def _ProcessPending():
while len(inflight) < _INFLIGHT_LIMIT and len(pending) > 0:
photo = pending.popleft()
#if options.options.copy_files and photo.new_assets == 'copied':
# continue
with util.Barrier(partial(_OnProcessed, photo,
_ProcessPending)) as b:
for ak, nak in [(fmt % photo.photo_id, nfmt % photo.photo_id) \
for fmt, nfmt in (('%s', '%s.o'),
('%s_f', '%s.f'),
('%s_m', '%s.m'),
('%s_t', '%s.t'))]:
assert ak not in inflight, ak
inflight.add(ak)
finish_cb = b.Callback()
with util.MonoBarrier(partial(_OnPhotoGet, ak, nak,
finish_cb),
on_exception=partial(
_OnErrorGet, ak,
finish_cb)) as get_b:
obj_store.Get(ak, get_b.Callback())
if len(pending) == 0 and len(inflight) == 0:
if last_key and count < max_count:
Photo.Scan(client,
col_names=None,
limit=_SCAN_LIMIT,
excl_start_key=last_key,
callback=partial(_OnScan, final_cb, client, count))
else:
logging.info('finished rename of %d photos' % count)
final_cb()
def _Get(self, key, include_body=True):
# On error, return 404 to client.
def _OnError(type, value, traceback):
self.send_error(404)
def _OnCompletedGet(content):
md5_hex = util.ComputeMD5Hex(content)
self.set_header(
"Expires",
int(time.time()) + FileObjectStoreHandler._CACHE_MAX_AGE)
self.set_header('Content-Type', self.content_type)
self.set_header('Content-Length', len(content))
self.set_header('Content-MD5', md5_hex)
self.set_header('Etag', '"%s"' % md5_hex)
cache_control = self.get_argument('response-cache-control', None)
if cache_control is not None:
self.set_header("Cache-Control", cache_control)
if include_body:
self.write(content)
self.finish()
with util.MonoBarrier(_OnCompletedGet, on_exception=_OnError) as b:
self.object_store.Get(key, b.Callback())
def post(self, method_name):
"""Parses the JSON request body, validates it, and invokes the
method as specified in the request URI. On completion, the
response is returned as a JSON-encoded HTTP response body.
"""
def _OnSuccess(method, start_time, response_dict):
validictory.validate(response_dict, method.response)
self.set_status(200)
self.set_header('Content-Type', 'application/json; charset=UTF-8')
self.write(response_dict) # tornado automatically serializes json
request_time = time.time() - start_time
logging.debug('serviced %s request in %.4fs: %s' %
(method_name, request_time, response_dict))
self.finish()
def _OnException(type, value, tb):
status, message = www_util.HTTPInfoFromException(value)
self.set_status(status)
if status == 500:
logging.error('failure processing %s:\n%s' %
(method_name, self.request.body),
exc_info=(type, value, tb))
error_dict = {
'error': {
'method': method_name,
'message': '%s %s' % (type, value)
}
}
validictory.validate(error_dict, json_schema.ERROR_RESPONSE)
self.set_header('Content-Type', 'application/json; charset=UTF-8')
self.write(error_dict)
self.finish()
start_time = time.time()
# Check service method; (501: Not Implemented).
if not AdminServiceHandler.SERVICE_MAP.has_key(method_name):
self.send_error(status_code=501)
return
# Verify application/json; (415: Unsupported Media Type).
# TODO(ben): Refactor BaseHandler so we can use _LoadJSONRequest here.
content_type = self.request.headers['Content-Type']
if not content_type.startswith('application/json'):
self.send_error(status_code=415)
return
method = AdminServiceHandler.SERVICE_MAP[method_name]
with util.MonoBarrier(partial(_OnSuccess, method, start_time),
on_exception=_OnException) as b:
request_dict = json.loads(self.request.body)
validictory.validate(request_dict, method.request)
method.handler(self._client, self._obj_store,
self.get_current_user(), request_dict, b.Callback())
def testMissing(self):
"""Verify query for a missing photo fails."""
def _OnQuery(p):
assert False, 'photo query should fail with missing key'
def _OnMissing(type, value, traceback):
self.stop()
return True
with util.MonoBarrier(_OnQuery, on_exception=_OnMissing) as b:
Photo.Query(self._client, str(1L << 63), None, b.Callback())
def testEmptyBarrier(self):
val = [False]
def _Callback(result):
self.assertEqual(result, None)
val[0] = True
self.stop()
with util.MonoBarrier(_Callback):
pass
self.wait()
self.assertTrue(val[0])
def testBarrier(self):
val = []
def _Callback(result):
val.append(result)
self.stop()
with util.MonoBarrier(_Callback) as b:
cb = b.Callback()
self.assertRaises(Exception, b.Callback)
cb(1)
self.wait()
self.assertEqual(1, val[0])
def testCallbackPositionalArguments(self):
val = [0]
def _Callback(arg1, arg2):
self.stop()
self.assertEqual(arg1, 'arg1')
self.assertEqual(arg2, 'arg2')
val[0] = 1
def _Exception(type_, instance_, traceback):
self.stop()
self.assertTrue(type_ is TypeError)
val[0] = 2
with util.MonoBarrier(_Callback) as b:
b.Callback()('arg1', 'arg2')
self.wait()
self.assertEqual(1, val[0])
with util.Barrier(_Callback, on_exception=_Exception) as b1:
with util.MonoBarrier(_Callback) as b2:
b2.Callback()(b1.Callback())
self.wait()
self.assertEqual(2, val[0])
def _OnPhotoGet(ak, nak, callback, photo_bytes):
"""Get the sink photo to determine whether we need to copy. If the
sink photo exists, the bytes are compared to the source as verification.
"""
logging.info('got photo %s: %d bytes' % (ak, len(photo_bytes)))
if options.options.verify_files or options.options.copy_files:
with util.MonoBarrier(partial(_OnSinkPhotoGet, ak, nak, callback,
photo_bytes),
on_exception=partial(_OnErrorSinkGet, ak,
nak, callback,
photo_bytes)) as b:
obj_store.Get(nak, b.Callback())
else:
logging.info(
'fetched %d bytes for asset %s; not copying or verifying' %
(len(photo_bytes), ak))
inflight.remove(ak)
callback()
def _ExecuteRequest(self, queue, dyn_req):
"""Helper function to execute a DynamoDB request within the context
in which is was scheduled. This way, if an unrecoverable exception is
thrown during execution, it can be re-raised to the appropriate caller.
"""
def _OnResponse(start_time, json_response):
if dyn_req.method in ('BatchGetItem',):
consumed_units = next(json_response.get('Responses').itervalues()).get('ConsumedCapacityUnits', 1)
else:
consumed_units = json_response.get('ConsumedCapacityUnits', 1)
logging.debug('%s response: %d bytes, %d units, %.3fs elapsed' %
(dyn_req.method, len(json_response), consumed_units,
time.time() - start_time))
queue.Report(True, consumed_units)
dyn_req.finish_cb(json_response)
def _OnException(type, value, tb):
if type in (DBProvisioningExceededError, DBLimitExceededError):
# Retry on DynamoDB throttling errors. Report the failure to the queue so that it will backoff the
# requests/sec rate.
queue.Report(False)
elif type == DynamoDBResponseError and value.status in [500, 599] and \
dyn_req.method in RequestScheduler._READ_ONLY_METHODS:
# DynamoDB returns 500 when the service is unavailable for some reason.
# Curl returns 599 when something goes wrong with the connection, such as a timeout or connection reset.
# Only retry if this is a read-only request, since otherwise an update may be applied twice.
pass
else:
# Re-raise the exception now that we're in the stack context of original caller.
logging.warning('error calling "%s" with this request: %s' % (dyn_req.method, dyn_req.request))
raise type, value, tb
if dyn_req.method in ('BatchGetItem',):
table_name = next(dyn_req.request['RequestItems'].iterkeys())
else:
table_name = dyn_req.request.get('TableName', None)
logging.warning('%s against %s table failed: %s' % (dyn_req.method, table_name, value))
queue.Push(dyn_req)
self._ProcessQueue(queue)
logging.debug('sending %s (%d bytes) dynamodb request' % (dyn_req.method, len(dyn_req.request)))
with util.MonoBarrier(partial(_OnResponse, time.time()), on_exception=partial(_OnException)) as b:
self._asyncdynamo.make_request(dyn_req.method, json.dumps(dyn_req.request), b.Callback())
def _AllocateIds(self, client):
"""Allocates the next batch of IDs. On success, processes all
pending waiters. If there are more waiters than ids, re-allocates.
Otherwise, resets _allocation_pending.
"""
assert self._cur_id == self._last_id, (self._cur_id, self._last_id)
def _OnAllocate(result):
self._last_id = result.return_values[self._next_id_key]
if self._last_id <= IdAllocator._START_ID:
self._cur_id = self._last_id
return self._AllocateIds(client)
self._cur_id = max(IdAllocator._START_ID,
self._last_id - self._allocation)
assert self._cur_id < self._last_id, 'cur id %d >= allocated last id %d' % \
(self._cur_id, self._last_id)
logging.debug(
"allocated %d %s IDs (%d-%d)" %
(self._allocation, self.id_type, self._cur_id, self._last_id))
if not self._ProcessWaiters():
self._AllocateIds(client)
else:
self._allocation_pending = False
def _OnError(type, value, traceback):
logging.error('failed to allocate new id; returning waiters...',
exc_info=(type, value, traceback))
while len(self._waiters):
self._waiters.popleft()(type, value, traceback)
self._allocation_pending = True
with util.MonoBarrier(_OnAllocate, on_exception=_OnError) as b:
client.UpdateItem(table=self._table.name,
key=self.GetKey(),
attributes={
self._next_id_key:
db_client.UpdateAttr(value=self._allocation,
action='ADD')
},
return_values='UPDATED_NEW',
callback=b.Callback())
def testWithBarrier(self):
"""Ensure Retry doesn't interfere with barriers."""
retry_policy = retry.RetryPolicy(max_tries=2, check_result=lambda res, err: err)
with util.MonoBarrier(self._OnCompleted) as b:
retry.CallWithRetryAsync(retry_policy, self._AsyncFuncFailOnce, callback=b.Callback())
self.wait()
