def _OnQueryAccounting(entry):
if entry is None:
# No previous entry. Set op_id and set replace to false.
# We can submit the accounting object directly since it has not been mutated.
accounting.op_ids = op_id
with util.Barrier(callback,
on_exception=partial(_OnException,
accounting)) as b:
accounting.Update(client, b.Callback(), replace=False)
else:
prev_op_ids = entry.op_ids
# Checks whether the op id has been applied and modifies op_ids accordingly.
found = entry.IsOpDuplicate(op_id)
if found:
# This operation has been applied: skip.
callback()
return
entry.IncrementStatsFrom(accounting)
# Entry exists: modify the object returned by Query and require that the op_ids
# field has not changed since. If the existing entry was created by dbchk, it will
# not have a op_ids field. Setting expected={'op_ids': None} is not equivalent to
# expected={'op_ids': False}.
with util.Barrier(callback,
on_exception=partial(_OnException,
accounting)) as b:
entry.Update(client,
b.Callback(),
expected={'op_ids': prev_op_ids or False})
def _OnQuery(lock):
"""Creates a new lock, takes control of an abandoned lock, or reports
failure to acquire the lock. The choice depends upon the current
state of the lock in the database. Other agents may be trying to
acquire the lock at the same time, so takes care to handle those race
conditions.
"""
if lock is None:
# Create new lock.
lock = Lock(lock_id, owner_id)
if resource_data is not None:
lock.resource_data = resource_data
if detect_abandonment:
lock.expiration = time.time() + Lock.ABANDONMENT_SECS
with util.Barrier(partial(_OnUpdate, lock, Lock.ACQUIRED_LOCK),
on_exception=_OnException) as b:
_DoUpdate(
partial(lock.Update,
client,
expected={'lock_id': False},
callback=b.Callback()))
elif lock._IsOwnedBy(owner_id):
assert not detect_abandonment, (resource_type, resource_data)
# Acquirer knew owner id, so sync _unique_id up with it.
lock._SyncUniqueIdToOwnerId()
_OnUpdate(lock, Lock.ACQUIRED_LOCK)
elif lock.IsAbandoned():
logging.warning(
'lock was abandoned; trying to take control of it: %s' %
lock)
# Try to take control of lock.
with util.Barrier(partial(_OnUpdate, lock,
Lock.ACQUIRED_ABANDONED_LOCK),
on_exception=_OnException) as b:
_DoUpdate(
partial(lock._TryTakeControl, client,
detect_abandonment, b.Callback()))
else:
logging.warning(
'acquire of lock failed; already held by another agent: %s'
% lock)
# Report the failure to acquire in order to track contention on the lock and to notify the current
# lock owner that another agent tried to acquire the lock.
with util.Barrier(partial(_OnUpdate, None,
Lock.FAILED_TO_ACQUIRE_LOCK),
on_exception=_OnException) as b:
_DoUpdate(
partial(lock._TryReportAcquireFailure, client,
b.Callback()))
def testExceptionPropagationInStackContext(self):
"""Verify exceptions propagated from the DynamoDB client are raised in
the stack context of the caller.
"""
entered = [False, False]
def _OnPut1():
assert False, 'Put1 should fail'
def _OnError1(type, value, tb):
#logging.info('in Put1 error handler')
if entered[0]:
print 'in error1 again!'
assert not entered[0], 'already entered error 1'
entered[0] = True
if all(entered):
self.stop()
def _OnPut2():
assert False, 'Put2 should fail'
def _OnError2(type, value, tb):
#logging.info('in Put2 error handler')
assert not entered[1], 'already entered error 2'
entered[1] = True
if all(entered):
self.stop()
# Pause all processing to allow two put operations to queue and for the
# latter's error handler to become the de-facto stack context.
self._client._scheduler._Pause()
with util.Barrier(_OnPut1, _OnError1) as b1:
# Put an item with a blank string, which is disallowed by DynamoDB.
self._client.PutItem(table=_table.name,
key=DBKey(u'1', 1),
attributes={'a2': ''},
callback=b1.Callback())
with util.Barrier(_OnPut2, _OnError2) as b2:
# Put a valid item; this should replace the previous stack context.
self._client.PutItem(table=_table.name,
key=DBKey(u'2', 1),
attributes={'a2': ''},
callback=b2.Callback())
# Resume the request scheduler queue processing.
self._client._scheduler._Resume()
def _OnCreate(locations, episodes):
with util.Barrier(self.stop) as b:
episode_ids = dict([(v.title, v.episode_id) for v in episodes])
# Exact search.
_QueryAndVerify(episode_ids, b.Callback(),
Location(40.727657, -73.994583, 30),
['kimball ph'])
_QueryAndVerify(episode_ids, b.Callback(),
Location(41.044048, -71.950622, 100),
['surf lodge'])
# A super-small search area, centered in middle of Great Jones Alley.
_QueryAndVerify(episode_ids, b.Callback(),
Location(40.727267, -73.994443, 10), [])
# Widen the search area to 50m, centered in middle of Great Jones Alley.
_QueryAndVerify(episode_ids, b.Callback(),
Location(40.727267, -73.994443, 50),
['kimball ph', 'bond st sushi'])
# Union square with a 2km radius.
_QueryAndVerify(episode_ids, b.Callback(),
Location(40.736462, -73.990517, 2000), [
'kimball ph', 'bond st sushi',
'viewfinder', 'soho house', 'google'
])
# The Dominican Republic.
_QueryAndVerify(episode_ids, b.Callback(),
Location(19.041349, -70.427856, 75000),
['casa kimball'])
# The Caribbean.
_QueryAndVerify(episode_ids, b.Callback(),
Location(22.593726, -76.662598, 800000),
['casa kimball', 'atlantis'])
# Long Island.
_QueryAndVerify(episode_ids, b.Callback(),
Location(40.989228, -72.144470, 40000),
['kimball east', 'surf lodge'])
def testNestedBarriers(self):
"""Verify that a handled exception in a nested barrier doesn't prevent
outer barrier from completing.
"""
exceptions = [False, False]
level1_reached = [False]
def _Level2Exception(type, value, traceback):
exceptions[1] = True
def _Level2(cb):
raise Exception('exception in level 2')
def _Level1Exception(type, value, traceback):
exceptions[0] = True
def _OnLevel1():
self.io_loop.add_callback(self.stop)
level1_reached[0] = True
def _Level1(cb):
with util.Barrier(None, on_exception=_Level2Exception) as b:
_Level2(b.Callback())
_OnLevel1()
with util.Barrier(_OnLevel1, on_exception=_Level1Exception) as b:
_Level1(b.Callback())
self.wait()
self.assertTrue(not exceptions[0])
self.assertTrue(exceptions[1])
self.assertTrue(level1_reached[0])
def _OnPut(result):
key = DBKey(u'1à朋', 1)
attrs = [u'a0', u'a1', u'a2']
with util.Barrier(self.stop) as b:
self._client.GetItem(table=_table.name,
key=key,
callback=partial(_VerifyGet,
b.Callback()),
attributes=attrs,
must_exist=False,
consistent_read=False)
self._client.GetItem(table=_table.name,
key=key,
callback=partial(_VerifyConsistentGet,
b.Callback()),
attributes=attrs,
must_exist=False,
consistent_read=True)
self._client.GetItem(table=_table.name,
key=DBKey('2', 2),
callback=partial(_VerifyEmptyGet,
b.Callback()),
attributes=attrs,
must_exist=False,
consistent_read=False)
def _OnListKeys(store, batches, cb, keys):
self.assertEqual(len(batches), len(keys))
with util.Barrier(cb) as b2:
for batch in batches:
self.assertIn(batch.Key(), keys)
store.Get(batch.Key(),
partial(_OnGetBatch, batch, b2.Callback()))
def testCompletedBeforeException(self):
"""Make the barrier callback and then raise exception."""
val = [0]
def _Exception(type_, value_, traceback):
logging.info("Exception")
val[0] += 1
def _Completed():
logging.info("Completed")
val[0] += 1
def _RaiseException():
raise KeyError('key')
def _PropException(type_, value_, traceback):
self.io_loop.add_callback(self.stop)
with util.ExceptionBarrier(_PropException):
with util.Barrier(_Completed, _Exception):
self.io_loop.add_callback(_RaiseException)
self.wait()
self.assertEqual(val[0], 1,
'Both _Completed and _Exception were called.')
def testQuery(self):
"""Adds a range of values and queries with start key and limit."""
num_items = 10
def _OnQuery(exp_count, result):
for i in xrange(exp_count):
self.assertEqual(len(result.items), exp_count)
self.assertEqual(
result.items[i], {
u'thk': u'test_query',
u'trk': i,
u'a1': i,
u'a2': ('test-%d' % i)
})
self.stop()
def _OnPutItems():
self._client.Query(table=_table.name,
hash_key='test_query',
range_operator=RangeOperator([0, 5], 'BETWEEN'),
callback=partial(_OnQuery, 6),
attributes=['thk', 'trk', 'a1', 'a2'])
with util.Barrier(_OnPutItems) as b:
for i in xrange(num_items):
self._client.PutItem(table=_table.name,
key=DBKey(u'test_query', i),
callback=b.Callback(),
attributes={
'a1': i,
'a2': ('test-%d' % i)
})
def testPerformanceCounters(self):
"""Verify that performance counters are working correctly for DynamoDB."""
meter = counters.Meter(counters.counters.viewfinder.dynamodb)
def _PutComplete():
self.stop()
self._client._scheduler._Pause()
with util.Barrier(_PutComplete) as b:
self._client.PutItem(table=_table.name,
key=DBKey(u'1', 1),
attributes={
'a1': 100,
'a2': 'test value'
},
callback=b.Callback())
self._client.PutItem(table=_table.name,
key=DBKey(u'2', 1),
attributes={
'a1': 200,
'a2': 'test value'
},
callback=b.Callback())
sample = meter.sample()
self.assertEqual(2, sample.viewfinder.dynamodb.requests_queued)
self._client._scheduler._Resume()
sample = meter.sample()
self.assertEqual(0, sample.viewfinder.dynamodb.requests_queued)
def _ProcessPending():
while len(inflight) < _INFLIGHT_LIMIT and len(pending) > 0:
photo = pending.popleft()
#if options.options.copy_files and photo.new_assets == 'copied':
# continue
with util.Barrier(partial(_OnProcessed, photo,
_ProcessPending)) as b:
for ak, nak in [(fmt % photo.photo_id, nfmt % photo.photo_id) \
for fmt, nfmt in (('%s', '%s.o'),
('%s_f', '%s.f'),
('%s_m', '%s.m'),
('%s_t', '%s.t'))]:
assert ak not in inflight, ak
inflight.add(ak)
finish_cb = b.Callback()
with util.MonoBarrier(partial(_OnPhotoGet, ak, nak,
finish_cb),
on_exception=partial(
_OnErrorGet, ak,
finish_cb)) as get_b:
obj_store.Get(ak, get_b.Callback())
if len(pending) == 0 and len(inflight) == 0:
if last_key and count < max_count:
Photo.Scan(client,
col_names=None,
limit=_SCAN_LIMIT,
excl_start_key=last_key,
callback=partial(_OnScan, final_cb, client, count))
else:
logging.info('finished rename of %d photos' % count)
final_cb()
def _OnAggregate(timestamp, reports, metrics):
new_report = HealthReport.CreateFromKeywords(group_key=group_key,
timestamp=timestamp)
if len(metrics.timestamps) == 0:
# No metrics collected for this period. Only the missing metrics criteria
# can be evaluated in this case.
new_report.warnings.add('MissingMetrics:' + CLUSTER_TOKEN)
for c in criteria:
c.InspectTrends(reports, new_report)
else:
# Pivot metrics data to time.
time_pivot = {
counter: {
'cluster_total': data.cluster_total[0][1],
'cluster_avg': data.cluster_avg[0][1],
'machine_data':
{k: v[0][1]
for k, v in data.machine_data.iteritems()},
}
for counter, data in metrics.counter_data.iteritems()
}
for c in criteria:
c.InspectMetrics(time_pivot, new_report)
c.InspectTrends(reports, new_report)
success_cb = partial(_OnCreateReport, reports, new_report)
with util.Barrier(success_cb, _OnFailCreateReport) as b:
new_report.Update(client, b.Callback(), replace=False)
def _OnQuery(activities_followers):
activities, (follower_ids, last_key) = activities_followers
with util.Barrier(partial(callback, viewpoint)) as b:
old_timestamp = viewpoint.last_updated
if len(activities) > 0:
new_timestamp = max(a.timestamp for a in activities)
else:
# Viewpoint has no activities.
new_timestamp = 0
viewpoint.last_updated = new_timestamp
self._LogUpdate(viewpoint)
for follower_id in follower_ids:
logging.info(
'Followed (user_id=%s, viewpoint_id=%s): %s => %s' %
(follower_id, viewpoint.viewpoint_id,
Followed._TruncateToDay(old_timestamp),
Followed._TruncateToDay(new_timestamp)))
if Version._mutate_items:
Followed.UpdateDateUpdated(client, follower_id,
viewpoint.viewpoint_id,
old_timestamp,
new_timestamp, b.Callback())
def _OnQuery(followers_activities):
(follower_ids, last_key), activities = followers_activities
activities = [
activity for activity in activities if activity.name == 'share'
]
if len(activities) > 0:
# Find the share activity with the lowest timestamp.
oldest_activity = None
for activity in activities:
if oldest_activity is None or activity.timestamp < oldest_activity.timestamp:
oldest_activity = activity
activity.name = 'share_existing'
# Override oldest activity as share_new and add followers.
oldest_activity.name = 'share_new'
act_dict = json.loads(activities[-1].json)
act_dict['follower_ids'] = [
f_id for f_id in follower_ids if f_id != viewpoint.user_id
]
oldest_activity.json = json.dumps(act_dict)
# Update all activities.
with util.Barrier(partial(callback, viewpoint)) as b:
for activity in activities:
self._LogUpdate(activity)
if Version._mutate_items:
activity.Update(client, b.Callback())
else:
b.Callback()()
else:
callback(viewpoint)
def _OnScan(result):
with util.Barrier(callback) as b:
for item in result.items:
self._client.DeleteItem(table=_table.name,
key=DBKey(item['thk'],
item['trk']),
callback=b.Callback())
def _OnQueryByPushToken(devices):
"""Disable alerts for all other devices."""
with util.Barrier(_DoUpdate) as b:
for device in devices:
if device.device_id != self.device_id:
device.push_token = None
device.alert_user_id = None
super(Device, device).Update(client, b.Callback())
def _OnQuery(items):
if len(items) < DBObject._VISIT_LIMIT:
barrier_callback = callback
else:
barrier_callback = partial(_DoQuery, excl_start_key=items[-1].GetKey())
with util.Barrier(barrier_callback) as b:
for item in items:
visitor(item, callback=b.Callback())
def _RunQueries(cls, query_expr, hash_key_25, hash_key_75, callback):
with util.Barrier(callback) as b:
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=None,
end_key=None,
limit=None)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=None,
end_key=None,
limit=50)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=None,
end_key=hash_key_75,
limit=50)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=None,
end_key=hash_key_25,
limit=50)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=hash_key_25,
end_key=None,
limit=50)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=hash_key_75,
end_key=None,
limit=50)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=hash_key_25,
end_key=hash_key_75,
limit=50)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=hash_key_25,
end_key=hash_key_75,
limit=1)
_QueryAndVerify(cls,
b.Callback(),
query_expr,
start_key=hash_key_25,
end_key=hash_key_75,
limit=100)
def _OnRenewalTimeout():
self._timeout = None
if not self._renewing:
return
logging.info('renewing lock: %s' % self)
with util.Barrier(_ScheduleNextRenew,
on_exception=_OnException) as b:
self.expiration = time.time() + Lock.ABANDONMENT_SECS
self.Update(client, b.Callback())
def RefreshDB(self, callback):
"""Refreshes the lookup database by iterating over all cities in
the geoprocessor and querying google maps for address components
for any that are missing.
"""
def _OnLookup(city, barrier_cb, response):
if response.code != 200:
logging.error('error in google maps API query: %s' % response)
else:
try:
json_response = json.loads(response.body)
components = []
if len(json_response['results']) > 0:
for comp in json_response['results'][0][
'address_components']:
for t in comp['types']:
if t in ('locality',
'administrative_area_level_1',
'country'):
components.append(comp)
break
self._lookup_db[city.id] = components
except:
logging.exception(
'unable to parse google maps API response: %s' %
response.body)
barrier_cb()
with util.Barrier(callback) as b:
def _ProcessCities(cities):
start_time = time.time()
lookup_count = 0
for index in xrange(len(cities)):
city = cities[index]
if city.id not in self._lookup_db:
lat, lon = float(city.lat), float(city.lon)
logging.info(
'looking up %s (%f, %f) via google maps API' %
(city.name, lat, lon))
if lookup_count / (1.0 + time.time() - start_time
) > options.options.query_rate:
logging.info('pausing to slow API query rate')
return self._io_loop.add_timeout(
time.time() + 1.0,
partial(_ProcessCities, cities[index:]))
lookup_count += 1
self._http_client.fetch(
'http://maps.googleapis.com/maps/api/geocode/json?latlng=%f,%f&sensor=false'
% (lat, lon),
callback=partial(_OnLookup, city, b.Callback()),
method='GET')
cities = self._geoproc.GetCities()
_ProcessCities(cities)
def _OnQuery(retry_cb, count, result):
count += len(result.items)
if not result.last_key:
result_cb = partial(callback, 'deleted %d items' % count)
else:
logging.info('deleting next %d items from %s' % (len(result.items), table.name))
result_cb = partial(retry_cb, result.last_key, count)
with util.Barrier(result_cb) as b:
for item in result.items:
key = db_client.DBKey(options.options.hash_key, item[table.range_key_col.key])
client.DeleteItem(table=table.name, key=key, callback=b.Callback())
def testEmptyBarrier(self):
val = [False]
def _Callback():
val[0] = True
self.stop()
with util.Barrier(_Callback):
pass
self.wait()
self.assertTrue(val[0])
def _OnQueryIndexTerms(get_result):
terms = [
t for term_set in get_result.attributes.values()
for t in term_set
]
with util.Barrier(_OnDeleteIndexTerms) as b:
index_key = self._GetIndexKey()
[
client.DeleteItem(table=vf_schema.INDEX,
key=db_client.DBKey(hash_key=term,
range_key=index_key),
callback=b.Callback()) for term in terms
]
def testNestedContexts(self):
"""Test the nesting of a single ContextLocal subclass."""
with util.Barrier(self._OnSuccess,
on_exception=self._OnException) as b:
with StackContext(ExampleContext(1)):
self.io_loop.add_callback(
partial(self._VerifyExampleContext, 1, b.Callback()))
with StackContext(ExampleContext(2)):
self._VerifyExampleContext(2, util.NoCallback)
self.io_loop.add_callback(
partial(self._VerifyExampleContext, 2, b.Callback()))
self._VerifyExampleContext(1, util.NoCallback)
self.wait()
def _OnUpdate(p):
with util.Barrier(self.stop) as b:
_QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {
'c': 'Class'
}), False)
_QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {
'c': 'reunion'
}), False)
_QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {
'c': '1992'
}), True)
_QueryAndVerify(p, b.Callback(), ('photo.caption={c}', {
'c': 'culumbia'
}), True)
def _PersistToObjStore(self, batch, restore=False):
"""Writes the given log batch to the object store. The 'restore'
parameter indicates that this is an attempt to restore the log, in
which case we do not rewrite it to backup on a subsequent failure.
If there are callbacks waiting on a pending flush and there are no
more inflight log buffers, returns all callbacks.
"""
batch_key = batch.Key()
def _ProcessWaitCallbacks():
# If this was the last currently uploading ('in-flight') log batch,
# invokes any callbacks waiting on the persistor to finish. This functionality
# is only intended for testing.
del self._in_flight[batch_key]
if not self._in_flight:
while self._wait_callbacks:
self._wait_callbacks.popleft()()
def _OnPut():
logging.info(
'Successfully persisted log batch %s to object store' %
batch_key)
# Delete the local backup file if this was a restore attempt.
if restore:
os.unlink(self._BackupFileName(batch))
_ProcessWaitCallbacks()
def _OnError(type, value, tb):
logging.error('Failed to put log batch %s to object store' %
batch_key,
exc_info=(type, value, tb))
# If this was the original attempt to upload a batch, save to local backup.
if not restore:
self._PersistToBackup(batch)
_ProcessWaitCallbacks()
# Add this batch to the 'in-flight' collection. This helps track any outstanding S3 requests, which
# is important if the persistor is closed with outstanding batches remaining.
if IOLoop.current() is not None:
# Because this can be called during a process-exit scenario with no IOLoop available, we need to
# check for it before using a barrier. Otherwise we persist to backup, which does not require an
# IOLoop.
self._in_flight[batch_key] = batch
with util.Barrier(_OnPut, on_exception=_OnError) as b:
ObjectStore.GetInstance(batch.store_name).Put(
batch_key, batch.buffer, callback=b.Callback())
else:
self._PersistToBackup(batch)
def _OnQuery(viewpoint):
with util.Barrier(partial(callback, follower)) as b:
follower.viewed_seq = 0
self._LogUpdate(follower)
if Version._mutate_items:
follower.Update(client, b.Callback())
if viewpoint.update_seq is None:
viewpoint._version = self.rank
viewpoint.update_seq = 0
self._LogUpdate(viewpoint)
if Version._mutate_items:
viewpoint.Update(client, b.Callback())
def testMultipleContextTypes(self):
"""Test the usage of multiple ContextLocal subclasses in tandem."""
with util.Barrier(self._OnSuccess,
on_exception=self._OnException) as b:
with StackContext(ExampleContext(1)):
with StackContext(ExampleContextTwoParams(2, 3)):
self._VerifyExampleContext(1, util.NoCallback)
self._VerifyExampleContextTwoParams(2, 3, util.NoCallback)
self.io_loop.add_callback(
partial(self._VerifyExampleContext, 1, b.Callback()))
self.io_loop.add_callback(
partial(self._VerifyExampleContextTwoParams, 2, 3,
b.Callback()))
self.wait()
def testBadRequest(self):
"""Verify exceptions are propagated on a bad request."""
def _OnPut(result):
assert False, 'Put should fail'
def _OnError(type, value, callback):
self.assertEqual(type, DynamoDBResponseError)
self.stop()
with util.Barrier(_OnPut, _OnError) as b:
# Put an item with a blank string, which is disallowed by DynamoDB.
self._client.PutItem(table=_table.name,
key=DBKey(u'1', 1),
attributes={'a2': ''},
callback=b.Callback())
def testConcurrentOperations(self):
"""Verify N concurrent synchronous uploads with an additional operation
manager also trying to scan and assume device operation queues.
"""
N = 50
with util.Barrier(self.stop) as b:
for i in xrange(N):
request_dict = {'activity': self._tester.CreateActivityDict(self._cookie),
'episode': self._CreateEpisodeDict(self._cookie),
'photos': [self._CreatePhotoDict(self._cookie)]}
self._tester.SendRequestAsync('upload_episode', self._cookie, request_dict, b.Callback())
self.wait()
self.assertEqual(len(self._RunAsync(Episode.Scan, self._client, None)[0]), N)
self.assertEqual(len(self._RunAsync(Photo.Scan, self._client, None)[0]), N)
