import boto
import time
import urllib
from boto.s3.connection import S3Connection
from functools import partial
from viewfinder.backend.base import constants, counters, util
from viewfinder.backend.base.secrets import GetSecret
from viewfinder.backend.storage.object_store import ObjectStore
from viewfinder.backend.storage.async_s3 import AsyncS3Connection
from xml.etree import ElementTree
_puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min',
'Average S3 puts per minute.', 60)
_secs_per_put = counters.define_average(
'viewfinder.s3.secs_per_put',
'Average time in seconds to complete each S3 put')
_gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min',
'Average S3 gets per minute.', 60)
class S3ObjectStore(ObjectStore):
"""Simple object storage interface supporting key/value pairs backed by S3.
Methods that require network round-trips are asynchronous; they take a
callback argument that will be invoked if the operation completes
successfully. If the operation fails, then an exception is raised. To
handle this exception, use a Barrier instance with the exception handler
defined.
"""
def __init__(self, bucket_name, temporary=False, read_only=False):
assert not temporary, 'temporary can only be specified True for file object store'
__author__ = '[email protected] (Peter Mattis)' import boto import time import urllib from boto.s3.connection import S3Connection from functools import partial from viewfinder.backend.base import constants, counters, util from viewfinder.backend.base.secrets import GetSecret from viewfinder.backend.storage.object_store import ObjectStore from viewfinder.backend.storage.async_s3 import AsyncS3Connection from xml.etree import ElementTree _puts_per_min = counters.define_rate('viewfinder.s3.puts_per_min', 'Average S3 puts per minute.', 60) _secs_per_put = counters.define_average('viewfinder.s3.secs_per_put', 'Average time in seconds to complete each S3 put') _gets_per_min = counters.define_rate('viewfinder.s3.gets_per_min', 'Average S3 gets per minute.', 60) class S3ObjectStore(ObjectStore): """Simple object storage interface supporting key/value pairs backed by S3. Methods that require network round-trips are asynchronous; they take a callback argument that will be invoked if the operation completes successfully. If the operation fails, then an exception is raised. To handle this exception, use a Barrier instance with the exception handler defined. """ def __init__(self, bucket_name, temporary=False, read_only=False): assert not temporary, 'temporary can only be specified True for file object store' self._bucket_name = bucket_name self._read_only = read_only
from tornado import gen, stack_context
from tornado.ioloop import IOLoop
from viewfinder.backend.base import counters, message, util
from viewfinder.backend.base.exceptions import FailpointError, InvalidRequestError, LimitExceededError, PermissionError
from viewfinder.backend.base.exceptions import CannotWaitError, NotFoundError, LockFailedError, StopOperationError
from viewfinder.backend.db.lock import Lock
from viewfinder.backend.db.lock_resource_type import LockResourceType
from viewfinder.backend.db.operation import Operation
from viewfinder.backend.op.op_mgr_db_client import OpMgrDBClient
from viewfinder.backend.op.op_context import OpContext
# Performance counters for operation module.
# Average operation time is tracked for all operations - its historical value can be used to measure overall resource usage.
# A rate count of operations attempted and retries attempted - these numbers will indicate if a large number of operations are
# resulting in retry attempts.
_avg_op_time = counters.define_average('viewfinder.operation.avg_op_time', 'Average time in seconds per completed operation.')
_ops_per_min = counters.define_rate('viewfinder.operation.ops_per_min', 'Operations attempted per minute.', 60)
_retries_per_min = counters.define_rate('viewfinder.operation.retries_per_min', 'Operation retries attempted per minute.', 60)
_aborts_per_min = counters.define_rate('viewfinder.operation.aborts_per_min', 'Operations aborted per minute.', 60)
# Tuple of exceptions for which we will abort an operation (not retry).
# Any exception base class included here qualifies all of its subclasses.
_ABORTABLE_EXCEPTIONS = (
PermissionError,
InvalidRequestError,
LimitExceededError,
NotFoundError,
)
# Tuple of exceptions which trigger a retry with a smaller initial backoff.
_SMALLER_RETRY_EXCEPTIONS = (
