def testCompletedBeforeException(self):
"""Make the barrier callback and then raise exception."""
val = [0]
def _Exception(type_, value_, traceback):
logging.info("Exception")
val[0] += 1
def _Completed():
logging.info("Completed")
val[0] += 1
def _RaiseException():
raise KeyError('key')
def _PropException(type_, value_, traceback):
self.io_loop.add_callback(self.stop)
with util.ExceptionBarrier(_PropException):
with util.Barrier(_Completed, _Exception):
self.io_loop.add_callback(_RaiseException)
self.wait()
self.assertEqual(val[0], 1,
'Both _Completed and _Exception were called.')
def main():
"""Parses command line options and, if directed, executes some operation
to transform or create secrets from the command line.
"""
io_loop = ioloop.IOLoop.current()
options.parse_command_line()
def _OnException(type, value, traceback):
logging.error('failed %s' % options.options.secrets_mode,
exc_info=(type, value, traceback))
io_loop.stop()
sys.exit(1)
with util.ExceptionBarrier(_OnException):
if options.options.secrets_mode == 'list_secrets':
_ListSecrets(io_loop)
elif options.options.secrets_mode == 'get_secret':
_GetSecret(io_loop, options.options.secret)
elif options.options.secrets_mode == 'put_secret':
_PutSecret(io_loop, options.options.secret)
elif options.options.secrets_mode == 'put_crypt_keyset':
_PutCryptKeyset(io_loop, options.options.secret)
elif options.options.secrets_mode == 'put_signing_keyset':
_PutSigningKeyset(io_loop, options.options.secret)
elif options.options.secrets_mode == 'encrypt_secrets':
_EncryptSecrets(io_loop)
else:
raise Exception('unknown secrets_mode: %s' %
options.options.secrets_mode)
io_loop.start()
def _FilterContext(self):
with util.ExceptionBarrier(util.LogExceptionCallback):
try:
filter = _NullFilter()
self._filterer.addFilter(filter)
yield
finally:
self._filterer.removeFilter(filter)
def testImmediateException(self):
"""Test exception raised before barrier context is exited."""
def _OnException(type, value, tb):
self.stop()
with util.ExceptionBarrier(_OnException):
raise Exception('an error')
self.wait()
def testCallback(self):
"""ERROR: Try to use Callback() method on barrier."""
def _OnException(type, value, tb):
self.assertEqual(type, AssertionError)
self.stop()
with util.ExceptionBarrier(_OnException) as b:
b.Callback()
self.wait()
def testDelayedException(self):
"""Test exception raised after initial barrier context has exited."""
def _OnException(type, value, tb):
self.stop()
def _RaiseException():
raise Exception('an error')
with util.ExceptionBarrier(_OnException):
self.io_loop.add_callback(_RaiseException)
self.wait()
def _wrapper(self, *args, **kwargs):
"""Disables automatic HTTP response completion on exit."""
self._auto_finish = False
if datastore:
self._client = DBClient.Instance()
if obj_store:
self._obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
if log_store:
self._log_store = ObjectStore.GetInstance(ObjectStore.USER_LOG)
with util.ExceptionBarrier(self._stack_context_handle_exception):
return method(self, *args, **kwargs)
def testMultipleExceptions(self):
"""ERROR: Raise multiple exceptions within scope of exception barrier."""
def _OnException(type, value, tb):
self.stop()
def _RaiseException():
raise Exception('an error')
with util.ExceptionBarrier(_OnException) as b:
self.io_loop.add_callback(_RaiseException)
self.io_loop.add_callback(_RaiseException)
self.wait()
def _OnRefresh(response):
try:
response_dict = www_util.ParseJSONResponse(response)
except web.HTTPError as e:
if e.status_code == 400:
logging.error(
'%s: failed to refresh access token; clearing refresh token'
% e)
with util.ExceptionBarrier(util.LogExceptionCallback):
self.refresh_token = None
self.Update(client, util.NoCallback)
raise
self.access_token = response_dict['access_token']
self.expires = time.time() + response_dict['expires_in']
callback()
def _ScanAbandonedLocks(self):
"""Periodically scans the Locks table looking for abandoned operation
locks. If any are found, the associated operations are executed.
TODO(Andy): Scanning for abandoned locks really should go into a
LockManager class. See header for lock.py.
"""
max_timeout_secs = OpManager._MAX_SCAN_ABANDONED_LOCKS_INTERVAL.total_seconds()
while True:
# If there are too many active users, do not scan.
if len(self._active_users) < self._MAX_USERS_OUTSTANDING:
try:
last_key = None
while True:
limit = min(self._MAX_USERS_OUTSTANDING - len(self._active_users), OpManager._SCAN_LIMIT)
locks, last_key = yield gen.Task(Lock.ScanAbandoned,
self._client,
limit=limit,
excl_start_key=last_key)
for lock in locks:
resource_type, resource_id = Lock.DeconstructLockId(lock.lock_id)
if resource_type == LockResourceType.Operation:
user_id = int(resource_id)
logging.info('scanned operation lock for user %d' % user_id)
# Create a clean context for this operation since we're not blocking the current
# coroutine on it.
with stack_context.NullContext():
with util.ExceptionBarrier(util.LogExceptionCallback):
self.MaybeExecuteOp(self._client, user_id, lock.resource_data)
# Keep iterating until all abandoned locks have been found, otherwise wait until the next scan time.
if last_key is None:
break
except Exception:
logging.exception('abandoned lock scan failed')
# Wait until next scan time.
timeout_secs = random.random() * max_timeout_secs
timeout_time = time.time() + timeout_secs
logging.debug('next scan in %.2fs' % timeout_secs)
yield gen.Task(IOLoop.current().add_timeout, timeout_time)
def _ScanFailedOps(self):
"""Periodically scans the Operation table for operations which have failed and are ready
to retry. If any are found, they are retried to see if the error that originally caused
them to fail has been fixed.
"""
from viewfinder.backend.db.operation import Operation
max_timeout_secs = OpManager._MAX_SCAN_FAILED_OPS_INTERVAL.total_seconds()
while True:
# If there are too many active users, do not scan.
if len(self._active_users) < self._MAX_USERS_OUTSTANDING:
try:
last_key = None
while True:
limit = min(self._MAX_USERS_OUTSTANDING - len(self._active_users), OpManager._SCAN_LIMIT)
ops, last_key = yield gen.Task(Operation.ScanFailed,
self._client,
limit=limit,
excl_start_key=last_key)
# Add each operation to the queue for the owning user.
for op in ops:
logging.info('scanned failed operation "%s" for user %d' % (op.operation_id, op.user_id))
if op.user_id not in self._active_users:
# Create a clean context for this operation since we're not blocking the current
# coroutine on it.
with stack_context.NullContext():
with util.ExceptionBarrier(util.LogExceptionCallback):
self.MaybeExecuteOp(self._client, op.user_id, op.operation_id)
# Keep iterating until all failed operations have been found, otherwise wait until the next scan time.
if last_key is None:
break
except Exception:
logging.exception('failed op scan failed')
# Wait until next scan time.
timeout_secs = random.random() * max_timeout_secs
timeout_time = time.time() + timeout_secs
logging.debug('next scan in %.2fs' % timeout_secs)
yield gen.Task(IOLoop.current().add_timeout, timeout_time)
def CreateAndExecute(
cls,
client,
user_id,
device_id,
method,
args,
callback,
message_version=message.MAX_SUPPORTED_MESSAGE_VERSION):
"""Creates a new operation with 'method' and 'args' describing the operation. After
successfully creating the operation, the operation is asynchronously executed. Returns
the op that was executed.
"""
# Get useful headers and strip all else.
headers = args.pop('headers', {})
synchronous = headers.pop('synchronous', False)
# Validate the op_id and op_timestamp fields.
op_id = headers.pop('op_id', None)
op_timestamp = headers.pop('op_timestamp', None)
assert (op_id is not None) == (op_timestamp
is not None), (op_id, op_timestamp)
# Validate that op_id is correctly formed and is allowed to be generated by the current device.
# No need to do this if the op_id was generated by the system as part of message upgrade.
if op_id is not None and headers.get(
'original_version',
0) >= message.Message.ADD_OP_HEADER_VERSION:
yield Operation.VerifyOperationId(client, user_id, device_id,
op_id)
# Use the op_id provided by the user, or generate a system op-id.
if op_id is None:
op_id = yield gen.Task(Operation.AllocateSystemOperationId, client)
# Possibly migrate backwards to a message version that is compatible with older versions of the
# server that may still be running.
op_message = message.Message(
args, default_version=message.MAX_MESSAGE_VERSION)
yield gen.Task(op_message.Migrate,
client,
migrate_version=message_version,
migrators=OpManager.Instance().op_map[method].migrators)
op = Operation(user_id, op_id)
op.device_id = device_id
op.method = method
op.json = json.dumps(args)
op.attempts = 0
# Set timestamp to header value if it was specified, or current timestamp if not.
if op_timestamp is not None:
op.timestamp = op_timestamp
else:
op.timestamp = util.GetCurrentTimestamp()
# Set expired backoff so that if this process fails before the op can be executed, in the worst
# case it will eventually get picked up by the OpManager's scan for failed ops. Note that in
# rare cases, this may mean that the op gets picked up immediately by another server (i.e. even
# though the current server has *not* failed), but that is fine -- it doesn't really matter what
# server executes the op, it just matters that the op gets executed in a timely manner.
op.backoff = 0
# Try to create the operation if it does not yet exist.
try:
yield gen.Task(op.Update, client, expected={'operation_id': False})
# Execute the op according to the 'synchronous' parameter. If 'synchronous' is True, the
# callback is invoked only after the operation has completed. Useful during unittests to
# ensure the mutations wrought by the operation are queryable.
logging.info('PERSIST: user: %d, device: %d, op: %s, method: %s' %
(user_id, device_id, op_id, method))
except Exception:
# Return existing op.
logging.warning('operation "%s" already exists', op_id)
existing_op = yield gen.Task(Operation.Query,
client,
user_id,
op_id,
None,
must_exist=False)
if existing_op is not None:
op = existing_op
# If not synchronous, we fire the callback, but continue to execute.
if not synchronous:
callback(op)
# Establish new "clean" context in which to execute the operation. The operation should not rely
# on any context, since it may end up run on a completely different machine. In addition, establish
# an exception barrier in order to handle any bugs or asserts, rather than letting the context
# established for the request handle it, since it will have already completed).
with stack_context.NullContext():
with util.ExceptionBarrier(util.LogExceptionCallback):
OpManager.Instance().MaybeExecuteOp(
client, user_id, op.operation_id)
else:
# Let exceptions flow up to request context so they'll be put into an error response.
OpManager.Instance().MaybeExecuteOp(client, user_id,
op.operation_id,
partial(callback, op))
def _InitSecrets():
with util.ExceptionBarrier(_OnInitException):
secrets.InitSecrets(_InitWatchdog, can_prompt=sys.stderr.isatty())
