import boto import time import urllib from boto.s3.connection import S3Connection from functools import partial from viewfinder.backend.base import constants, counters, util from viewfinder.backend.base.secrets import GetSecret from viewfinder.backend.storage.object_store import ObjectStore from viewfinder.backend.storage.async_s3 import AsyncS3Connection from xml.etree import ElementTree _puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60) _secs_per_put = counters.define_average( 'viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put') _gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60) class S3ObjectStore(ObjectStore): """Simple object storage interface supporting key/value pairs backed by S3. Methods that require network round-trips are asynchronous; they take a callback argument that will be invoked if the operation completes successfully. If the operation fails, then an exception is raised. To handle this exception, use a Barrier instance with the exception handler defined. """ def __init__(self, bucket_name, temporary=False, read_only=False): assert not temporary, 'temporary can only be specified True for file object store'
__author__ = '[email protected] (Peter Mattis)' import boto import time import urllib from boto.s3.connection import S3Connection from functools import partial from viewfinder.backend.base import constants, counters, util from viewfinder.backend.base.secrets import GetSecret from viewfinder.backend.storage.object_store import ObjectStore from viewfinder.backend.storage.async_s3 import AsyncS3Connection from xml.etree import ElementTree _puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60) _secs_per_put = counters.define_average('viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put') _gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60) class S3ObjectStore(ObjectStore): """Simple object storage interface supporting key/value pairs backed by S3. Methods that require network round-trips are asynchronous; they take a callback argument that will be invoked if the operation completes successfully. If the operation fails, then an exception is raised. To handle this exception, use a Barrier instance with the exception handler defined. """ def __init__(self, bucket_name, temporary=False, read_only=False): assert not temporary, 'temporary can only be specified True for file object store' self._bucket_name = bucket_name self._read_only = read_only
from tornado import gen, stack_context from tornado.ioloop import IOLoop from viewfinder.backend.base import counters, message, util from viewfinder.backend.base.exceptions import FailpointError, InvalidRequestError, LimitExceededError, PermissionError from viewfinder.backend.base.exceptions import CannotWaitError, NotFoundError, LockFailedError, StopOperationError from viewfinder.backend.db.lock import Lock from viewfinder.backend.db.lock_resource_type import LockResourceType from viewfinder.backend.db.operation import Operation from viewfinder.backend.op.op_mgr_db_client import OpMgrDBClient from viewfinder.backend.op.op_context import OpContext # Performance counters for operation module. # Average operation time is tracked for all operations - its historical value can be used to measure overall resource usage. # A rate count of operations attempted and retries attempted - these numbers will indicate if a large number of operations are # resulting in retry attempts. _avg_op_time = counters.define_average('viewfinder.operation.avg_op_time', 'Average time in seconds per completed operation.') _ops_per_min = counters.define_rate('viewfinder.operation.ops_per_min', 'Operations attempted per minute.', 60) _retries_per_min = counters.define_rate('viewfinder.operation.retries_per_min', 'Operation retries attempted per minute.', 60) _aborts_per_min = counters.define_rate('viewfinder.operation.aborts_per_min', 'Operations aborted per minute.', 60) # Tuple of exceptions for which we will abort an operation (not retry). # Any exception base class included here qualifies all of its subclasses. _ABORTABLE_EXCEPTIONS = ( PermissionError, InvalidRequestError, LimitExceededError, NotFoundError, ) # Tuple of exceptions which trigger a retry with a smaller initial backoff. _SMALLER_RETRY_EXCEPTIONS = (