Example #1
0
import boto
import time
import urllib

from boto.s3.connection import S3Connection
from functools import partial
from viewfinder.backend.base import constants, counters, util
from viewfinder.backend.base.secrets import GetSecret
from viewfinder.backend.storage.object_store import ObjectStore
from viewfinder.backend.storage.async_s3 import AsyncS3Connection
from xml.etree import ElementTree

_puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min',
                                     'Average S3 puts per minute.', 60)
_secs_per_put = counters.define_average(
    'viewfinder.s3.secs_per_put',
    'Average time in seconds to complete each S3 put')
_gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min',
                                     'Average S3 gets per minute.', 60)


class S3ObjectStore(ObjectStore):
    """Simple object storage interface supporting key/value pairs backed by S3.
  Methods that require network round-trips are asynchronous; they take a
  callback argument that will be invoked if the operation completes
  successfully. If the operation fails, then an exception is raised. To
  handle this exception, use a Barrier instance with the exception handler
  defined.
  """
    def __init__(self, bucket_name, temporary=False, read_only=False):
        assert not temporary, 'temporary can only be specified True for file object store'
__author__ = '[email protected] (Peter Mattis)'

import boto
import time
import urllib

from boto.s3.connection import S3Connection
from functools import partial
from viewfinder.backend.base import constants, counters, util
from viewfinder.backend.base.secrets import GetSecret
from viewfinder.backend.storage.object_store import ObjectStore
from viewfinder.backend.storage.async_s3 import AsyncS3Connection
from xml.etree import ElementTree

_puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60)
_secs_per_put = counters.define_average('viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put')
_gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60)

class S3ObjectStore(ObjectStore):
  """Simple object storage interface supporting key/value pairs backed by S3.
  Methods that require network round-trips are asynchronous; they take a
  callback argument that will be invoked if the operation completes
  successfully. If the operation fails, then an exception is raised. To
  handle this exception, use a Barrier instance with the exception handler
  defined.
  """
  def __init__(self, bucket_name, temporary=False, read_only=False):
    assert not temporary, 'temporary can only be specified True for file object store'
    self._bucket_name = bucket_name
    self._read_only = read_only
Example #3
0
from tornado import gen, stack_context
from tornado.ioloop import IOLoop
from viewfinder.backend.base import counters, message, util
from viewfinder.backend.base.exceptions import FailpointError, InvalidRequestError, LimitExceededError, PermissionError
from viewfinder.backend.base.exceptions import CannotWaitError, NotFoundError, LockFailedError, StopOperationError
from viewfinder.backend.db.lock import Lock
from viewfinder.backend.db.lock_resource_type import LockResourceType
from viewfinder.backend.db.operation import Operation
from viewfinder.backend.op.op_mgr_db_client import OpMgrDBClient
from viewfinder.backend.op.op_context import OpContext

# Performance counters for operation module.
# Average operation time is tracked for all operations - its historical value can be used to measure overall resource usage.
# A rate count of operations attempted and retries attempted - these numbers will indicate if a large number of operations are
# resulting in retry attempts.
_avg_op_time = counters.define_average('viewfinder.operation.avg_op_time', 'Average time in seconds per completed operation.')
_ops_per_min = counters.define_rate('viewfinder.operation.ops_per_min', 'Operations attempted per minute.', 60)
_retries_per_min = counters.define_rate('viewfinder.operation.retries_per_min', 'Operation retries attempted per minute.', 60)
_aborts_per_min = counters.define_rate('viewfinder.operation.aborts_per_min', 'Operations aborted per minute.', 60)

# Tuple of exceptions for which we will abort an operation (not retry).
# Any exception base class included here qualifies all of its subclasses.
_ABORTABLE_EXCEPTIONS = (
  PermissionError,
  InvalidRequestError,
  LimitExceededError,
  NotFoundError,
  )

# Tuple of exceptions which trigger a retry with a smaller initial backoff.
_SMALLER_RETRY_EXCEPTIONS = (