def _VerifyRange(r_op, limit, forward, start_key, exp_keys,
exp_last_key):
"""Returns the result"""
result = self._client.Query(
table='RangeTest',
hash_key=hash_key,
range_operator=r_op,
callback=None,
attributes=['test_hk', 'test_rk', 'attr1', 'attr2'],
limit=limit,
scan_forward=forward,
excl_start_key=(DBKey(hash_key, start_key)
if start_key else None))
self.assertEqual(len(exp_keys), result.count)
self.assertEqual(len(exp_keys), len(result.items))
self.assertEqual(exp_keys,
[item['test_rk'] for item in result.items])
for item in result.items:
self.assertEqual(items[item['test_rk']]['attr1'],
item['attr1'])
self.assertEqual(items[item['test_rk']]['attr2'],
item['attr2'])
if exp_last_key is not None:
self.assertEqual(DBKey(hash_key, exp_last_key),
result.last_key)
else:
self.assertTrue(result.last_key is None)
def testBatchQuery(self):
"""Test DBObject.BatchQuery."""
# Create some data to query.
keys = []
for i in xrange(3):
photo_id = Photo.ConstructPhotoId(time.time(),
self._mobile_dev.device_id, 1)
episode_id = Episode.ConstructEpisodeId(time.time(),
self._mobile_dev.device_id,
1)
ph_dict = {
'photo_id': photo_id,
'user_id': self._user.user_id,
'episode_id': episode_id
}
self._RunAsync(Photo.CreateNew, self._client, **ph_dict)
keys.append(DBKey(photo_id, None))
# Add a key that will not be found.
keys.append(DBKey('unk-photo', None))
photos = self._RunAsync(Photo.BatchQuery,
self._client,
keys,
None,
must_exist=False)
self.assertEqual(len(photos), 4)
for i in xrange(3):
self.assertEqual(photos[i].GetKey(), keys[i])
self.assertIsNone(photos[3])
def testPerformanceCounters(self):
"""Verify that performance counters are working correctly for DynamoDB."""
meter = counters.Meter(counters.counters.viewfinder.dynamodb)
def _PutComplete():
self.stop()
self._client._scheduler._Pause()
with util.Barrier(_PutComplete) as b:
self._client.PutItem(table=_table.name,
key=DBKey(u'1', 1),
attributes={
'a1': 100,
'a2': 'test value'
},
callback=b.Callback())
self._client.PutItem(table=_table.name,
key=DBKey(u'2', 1),
attributes={
'a1': 200,
'a2': 'test value'
},
callback=b.Callback())
sample = meter.sample()
self.assertEqual(2, sample.viewfinder.dynamodb.requests_queued)
self._client._scheduler._Resume()
sample = meter.sample()
self.assertEqual(0, sample.viewfinder.dynamodb.requests_queued)
def _OnPut(result):
key = DBKey(u'1à朋', 1)
attrs = [u'a0', u'a1', u'a2']
with util.Barrier(self.stop) as b:
self._client.GetItem(table=_table.name,
key=key,
callback=partial(_VerifyGet,
b.Callback()),
attributes=attrs,
must_exist=False,
consistent_read=False)
self._client.GetItem(table=_table.name,
key=key,
callback=partial(_VerifyConsistentGet,
b.Callback()),
attributes=attrs,
must_exist=False,
consistent_read=True)
self._client.GetItem(table=_table.name,
key=DBKey('2', 2),
callback=partial(_VerifyEmptyGet,
b.Callback()),
attributes=attrs,
must_exist=False,
consistent_read=False)
def testUnicode(self):
"""Test various interesting Unicode characters."""
base_name = escape.utf8(
u'ààà朋友你好abc123\U00010000\U00010000\x00\x01\b\n\t ')
timestamp = time.time()
contact_id_lookup = dict()
def _CreateContact(index):
name = base_name + str(index)
identity_key = 'Email:%s' % name
return Contact.CreateFromKeywords(100, [(identity_key, None)],
timestamp,
Contact.GMAIL,
name=name)
def _VerifyContacts(query_expr, start_key, end_key, exp_indexes):
actual_contacts = self._RunAsync(Contact.IndexQuery,
self._client,
query_expr,
None,
start_index_key=start_key,
end_index_key=end_key)
self.assertEqual(len(exp_indexes), len(actual_contacts))
for expected, actual in zip(
[_CreateContact(i) for i in exp_indexes], actual_contacts):
self.assertEqual(expected._asdict(), actual._asdict())
# Create 3 contacts under user 100 in the db.
for i in xrange(3):
contact = _CreateContact(i)
contact_id_lookup[i] = contact.contact_id
self._RunAsync(contact.Update, self._client)
# Get contact by identity.
identity_key = 'Email:%s' % base_name
_VerifyContacts(('contact.identities={i}', {
'i': identity_key + '0'
}), None, None, [0])
# Get multiple contacts.
_VerifyContacts(('contact.identities={i} | contact.identities={i2}', {
'i': identity_key + '0',
'i2': identity_key + '1'
}), None, None, [1, 0])
# Get contact with start key.
sort_key = Contact.CreateSortKey(contact_id_lookup[1], timestamp)
_VerifyContacts(('contact.identities={i} | contact.identities={i2}', {
'i': identity_key + '0',
'i2': identity_key + '1'
}), DBKey(100, sort_key), None, [0])
# Get contact with end key.
sort_key = Contact.CreateSortKey(contact_id_lookup[0], timestamp)
_VerifyContacts(('contact.identities={i} | contact.identities={i2}', {
'i': identity_key + '0',
'i2': identity_key + '1'
}), None, DBKey(100, sort_key), [1])
def testMethods(self):
# Read-only methods:
self._RunAsync(self._client.ListTables)
self._RunAsync(self._client.DescribeTable, 'LocalTest')
self._RunAsync(self._client.GetItem,
table='LocalTest',
key=DBKey(hash_key=1, range_key=1),
attributes=['num'])
batch_dict = {
'LocalTest':
BatchGetRequest(keys=[DBKey(1, 1)],
attributes=['num'],
consistent_read=True)
}
self._RunAsync(self._client.BatchGetItem, batch_dict)
self._RunAsync(self._client.Query,
table='LocalTest',
hash_key=1,
range_operator=None,
attributes=None)
self._RunAsync(self._client.Scan, table='LocalTest', attributes=None)
# Mutating methods:
self.assertRaisesRegexp(AssertionError,
'request on read-only database',
self._RunAsync, self._client.CreateTable,
'LocalTest', _hash_key_schema,
_range_key_schema, 5, 10)
self.assertRaisesRegexp(AssertionError,
'request on read-only database',
self._RunAsync,
self._client.DeleteTable,
table='LocalTest')
self.assertRaisesRegexp(AssertionError,
'request on read-only database',
self._RunAsync,
self._client.PutItem,
table='LocalTest',
key=DBKey(hash_key=1, range_key=2),
attributes={'num': 1})
self.assertRaisesRegexp(AssertionError,
'request on read-only database',
self._RunAsync,
self._client.DeleteItem,
table='LocalTest',
key=DBKey(hash_key=1, range_key=2))
self.assertRaisesRegexp(AssertionError,
'request on read-only database',
self._RunAsync,
self._client.UpdateItem,
table='LocalTest',
key=DBKey(hash_key=1, range_key=2),
attributes={'num': 1})
def _Validate(viewpoint_id):
self._validator.ValidateDeleteDBObject(Follower, DBKey(self._user.user_id, viewpoint_id))
self._validator.ValidateDeleteDBObject(Follower, DBKey(self._user2.user_id, viewpoint_id))
sort_key = Followed.CreateSortKey(viewpoint_id, 0)
self._validator.ValidateDeleteDBObject(Followed, DBKey(self._user.user_id, sort_key))
self._validator.ValidateDeleteDBObject(Followed, DBKey(self._user2.user_id, sort_key))
self._validator.ValidateDeleteDBObject(Viewpoint, viewpoint_id)
def _CreateExpectedPhotos(validator,
user_id,
device_id,
episode_id,
limit=None,
start_key=None):
"""Return a set of photo dicts that contain all the photo metadata for
photos in the episode with id "episode_id".
"""
photo_dicts = []
posts = validator.QueryModelObjects(Post,
episode_id,
limit=limit,
start_key=start_key)
for post in posts:
post_dict = post._asdict()
photo_dict = validator.GetModelObject(Photo, post.photo_id)._asdict()
photo_dict.pop('share_seq', None)
photo_dict.pop('client_data', None)
# Do not return access URLs for posts which have been removed.
if not post.IsRemoved():
obj_store = ObjectStore.GetInstance(ObjectStore.PHOTO)
_AddPhotoUrls(obj_store, photo_dict)
asset_keys = set()
user_photo = validator.GetModelObject(UserPhoto,
DBKey(user_id, post.photo_id),
must_exist=False)
if user_photo is not None and user_photo.asset_keys:
asset_keys.update(user_photo.asset_keys)
if asset_keys:
photo_dict['asset_keys'] = list(asset_keys)
photo_dicts.append(photo_dict)
post_id = Post.ConstructPostId(episode_id, post.photo_id)
user_post = validator.GetModelObject(UserPost,
DBKey(user_id, post_id),
must_exist=False)
labels = post.labels.combine()
if user_post is not None:
# Union together post labels and user_post labels.
labels = labels.union(user_post.labels.combine())
if len(labels) > 0:
photo_dict['labels'] = list(labels)
last_key = posts[-1].photo_id if len(posts) > 0 else None
return (photo_dicts, last_key)
def testExceptionPropagationInStackContext(self):
"""Verify exceptions propagated from the DynamoDB client are raised in
the stack context of the caller.
"""
entered = [False, False]
def _OnPut1():
assert False, 'Put1 should fail'
def _OnError1(type, value, tb):
#logging.info('in Put1 error handler')
if entered[0]:
print 'in error1 again!'
assert not entered[0], 'already entered error 1'
entered[0] = True
if all(entered):
self.stop()
def _OnPut2():
assert False, 'Put2 should fail'
def _OnError2(type, value, tb):
#logging.info('in Put2 error handler')
assert not entered[1], 'already entered error 2'
entered[1] = True
if all(entered):
self.stop()
# Pause all processing to allow two put operations to queue and for the
# latter's error handler to become the de-facto stack context.
self._client._scheduler._Pause()
with util.Barrier(_OnPut1, _OnError1) as b1:
# Put an item with a blank string, which is disallowed by DynamoDB.
self._client.PutItem(table=_table.name,
key=DBKey(u'1', 1),
attributes={'a2': ''},
callback=b1.Callback())
with util.Barrier(_OnPut2, _OnError2) as b2:
# Put a valid item; this should replace the previous stack context.
self._client.PutItem(table=_table.name,
key=DBKey(u'2', 1),
attributes={'a2': ''},
callback=b2.Callback())
# Resume the request scheduler queue processing.
self._client._scheduler._Resume()
def _Unshare(ep_dicts):
"""Recursively validates all photos that were unshared."""
for ep_dict in ep_dicts:
episode = validator.GetModelObject(Episode, ep_dict['episode_id'])
unshared_photo_ids = unshares_dict.get(episode.viewpoint_id,
{}).get(
episode.episode_id, [])
for ph_id in ep_dict['photo_ids']:
# Validate that each unshared post has UNSHARED label added to it.
key = DBKey(ep_dict['episode_id'], ph_id)
post = validator.GetModelObject(Post, key, must_exist=False)
if post is not None and Post.UNSHARED not in post.labels:
validator.ValidateUpdateDBObject(
Post,
episode_id=ep_dict['episode_id'],
photo_id=ph_id,
labels=post.labels.union([Post.UNSHARED,
Post.REMOVED]))
if post.photo_id not in unshared_photo_ids:
unshared_photo_ids.append(post.photo_id)
if len(unshared_photo_ids) > 0:
unshares_dict.setdefault(
episode.viewpoint_id,
{})[episode.episode_id] = unshared_photo_ids
predicate = lambda e: e.parent_ep_id == episode.episode_id
child_episodes = validator.QueryModelObjects(
Episode, predicate=predicate)
if len(child_episodes) > 0:
_Unshare([{
'episode_id': ep.episode_id,
'photo_ids': ep_dict['photo_ids']
} for ep in child_episodes])
def SharePhotos(self, client, sharer_id, viewpoint_id, photo_ids,
follower_ids):
"""Generate and update accounting entries for a ShareNew or ShareExisting event.
- photo_ids: list of *new* photos that were added (caller should exclude the ids of any
photos that already existed).
- follower_ids: list of ids of all followers of the viewpoint, *including* the sharer
if it is not removed from the viewpoint.
We need to query all photos for size information. Creates the following entries:
- (vs:<viewpoint>, sb:<sharer>): sum across all new photos for the sharer
- (vs:<viewpoint>, vt): sum across all new photos
- (us:<sharer>, sb): sum across all new photos for the sharer
"""
photo_keys = [DBKey(photo_id, None) for photo_id in photo_ids]
photos = yield gen.Task(Photo.BatchQuery, client, photo_keys, None)
acc = Accounting()
acc.IncrementFromPhotos(photos)
# Viewpoint visible_to for viewpoint.
self.GetViewpointVisibleTo(viewpoint_id).IncrementStatsFrom(acc)
if sharer_id in follower_ids:
# Viewpoint shared_by for sharer.
self.GetViewpointSharedBy(viewpoint_id,
sharer_id).IncrementStatsFrom(acc)
# User shared_by for sharer.
self.GetUserSharedBy(sharer_id).IncrementStatsFrom(acc)
# Viewpoint visible_to for followers.
for follower_id in follower_ids:
self.GetUserVisibleTo(follower_id).IncrementStatsFrom(acc)
def ReadTablePageRequest(self, table_name, op_type=None):
"""Formats the request data received from jQuery data table. The 'table_name'
parameter is required if a single handler can query more than one data table.
"""
requested_start = int(self.get_argument('iDisplayStart'))
requested_length = int(self.get_argument('iDisplayLength'))
s_echo = self.get_argument('sEcho')
# Load last key, which is stored in a cookie
cookie = self.get_secure_cookie(self._GetCookieName())
try:
last_table, op_type, last_index, last_key = json.loads(cookie)
if last_table != table_name or last_index != requested_start:
last_key = None
requested_start = 0
except:
logging.warn('Bad cookie value: %s = %s' %
(self._GetCookieName(), cookie))
self.clear_cookie(self._GetCookieName())
last_key = None
requested_start = 0
if last_key:
# Convert last key back into DBKey - this is lost in json serialization.
last_key = DBKey(last_key[0], last_key[1])
self._table_request = _TablePageRequest(table=table_name,
op_type=op_type,
start=requested_start,
length=requested_length,
last_key=last_key,
echo=s_echo)
return self._table_request
def _OnScan(result):
with util.Barrier(callback) as b:
for item in result.items:
self._client.DeleteItem(table=_table.name,
key=DBKey(item['thk'],
item['trk']),
callback=b.Callback())
def _ValidateUploadOneContact(contact):
contact = deepcopy(contact)
# Transform into proper form for Contact.CalculateContactId()
identities_properties = [
(identity_properties['identity'],
identity_properties.get('description', None))
for identity_properties in contact['identities']
]
contact.pop('identities')
contact_dict = Contact.CreateContactDict(user_id,
identities_properties,
op_dict['op_timestamp'],
**contact)
predicate = lambda c: c.contact_id == contact_dict['contact_id']
existing_contacts = validator.QueryModelObjects(Contact,
predicate=predicate)
# Create contact if it doesn't already exist or it's in a 'removed' state.
if len(existing_contacts) == 0 or existing_contacts[-1].IsRemoved():
if len(existing_contacts) != 0:
# Delete the 'removed' contact.
validator.ValidateDeleteDBObject(
Contact, DBKey(user_id, existing_contacts[-1].sort_key))
validator.ValidateCreateContact(**contact_dict)
return contact_dict['contact_id']
def testRangeScan(self):
items = {}
for h in xrange(2):
items[h] = {}
for r in xrange(2):
items[h][r] = {
'attr1': (h + r * 2),
'attr2': 'test-%d' % (h + r)
}
result = self._client.PutItem('RangeTest',
key=DBKey(hash_key=h,
range_key=r),
attributes=items[h][r],
callback=None)
self.assertEqual(result.write_units, 1)
self.assertFalse(result.return_values)
def _VerifyScan(limit, start_key, exp_keys, exp_last_key):
result = self._client.Scan(
table='RangeTest',
callback=None,
attributes=['test_hk', 'test_rk', 'attr1', 'attr2'],
limit=limit,
excl_start_key=start_key)
for item in result.items:
self.assertTrue((item['test_hk'], item['test_rk']) in exp_keys)
if exp_last_key is not None:
self.assertEqual(exp_last_key, result.last_key)
else:
self.assertTrue(result.last_key is None)
_VerifyScan(None, None, set([(0, 0), (0, 1), (1, 0), (1, 1)]), None)
self.stop()
def _MergeIdentities(self):
"""Re-binds all identities attached to the source user to the target user. Sends corresponding
notifications for any merged identities. Sets a checkpoint so that the exact same set of
identities will be merged if a restart occurs.
"""
if self._op.checkpoint is None:
# Get set of identities that need to re-bound to the target user.
query_expr = ('identity.user_id={id}', {'id': self._source_user_id})
identity_keys = yield gen.Task(Identity.IndexQueryKeys,
self._client,
query_expr,
limit=MergeAccountsOperation._MAX_IDENTITIES)
checkpoint = {'state': 'id',
'ids': [key.hash_key for key in identity_keys]}
yield self._op.SetCheckpoint(self._client, checkpoint)
else:
identity_keys = [DBKey(id, None) for id in self._op.checkpoint['ids']]
# Get all the identity objects and re-bind them to the target user.
identities = yield gen.Task(Identity.BatchQuery, self._client, identity_keys, None)
for identity in identities:
identity.expires = 0
identity.user_id = self._target_user_id
yield gen.Task(identity.Update, self._client)
# Send notifications for all identities that were re-bound.
yield NotificationManager.NotifyMergeIdentities(self._client,
self._target_user_id,
[identity.key for identity in identities],
self._op.timestamp)
def testQuery(self):
"""Adds a range of values and queries with start key and limit."""
num_items = 10
def _OnQuery(exp_count, result):
for i in xrange(exp_count):
self.assertEqual(len(result.items), exp_count)
self.assertEqual(
result.items[i], {
u'thk': u'test_query',
u'trk': i,
u'a1': i,
u'a2': ('test-%d' % i)
})
self.stop()
def _OnPutItems():
self._client.Query(table=_table.name,
hash_key='test_query',
range_operator=RangeOperator([0, 5], 'BETWEEN'),
callback=partial(_OnQuery, 6),
attributes=['thk', 'trk', 'a1', 'a2'])
with util.Barrier(_OnPutItems) as b:
for i in xrange(num_items):
self._client.PutItem(table=_table.name,
key=DBKey(u'test_query', i),
callback=b.Callback(),
attributes={
'a1': i,
'a2': ('test-%d' % i)
})
def testUpdateWithDelete(self):
"""Update an item by deleting its attributes."""
def _OnDeleteUpdate(result):
self.assertEquals(result.write_units, 1)
self.assertEquals(result.return_values, {
u'a0': 1,
u'thk': u'2',
u'trk': 2
})
self.stop()
def _OnUpdate(result):
self.assertEquals(result.write_units, 1)
self._client.UpdateItem(table=_table.name,
key=DBKey(u'2', 2),
callback=_OnDeleteUpdate,
attributes={
'a1': UpdateAttr(None, 'DELETE'),
'a2': UpdateAttr(None, 'DELETE'),
'a3': UpdateAttr(None, 'DELETE'),
'a4': UpdateAttr(None, 'DELETE')
},
return_values='ALL_NEW')
self._client.UpdateItem(table=_table.name,
key=DBKey(u'2', 2),
callback=_OnUpdate,
attributes={
'a0': UpdateAttr(1, 'PUT'),
'a1': UpdateAttr(5, 'PUT'),
'a2': UpdateAttr('update str', 'PUT'),
'a3': UpdateAttr(set([1, 2, 3]), 'PUT'),
'a4': UpdateAttr(set(['1', '2', '3']),
'PUT')
})
def ValidateCopyEpisodes(self, op_dict, viewpoint_id, ep_dicts):
"""Validates that a set of episodes and posts have been created within the specified
viewpoint via a sharing or save operation.
"""
ph_act_dict = {}
for ep_dict in ep_dicts:
existing_episode = self.GetModelObject(Episode, ep_dict['existing_episode_id'])
new_ep_dict = {'episode_id': ep_dict['new_episode_id'],
'parent_ep_id': ep_dict['existing_episode_id'],
'user_id': op_dict['user_id'],
'viewpoint_id': viewpoint_id,
'timestamp': existing_episode.timestamp,
'publish_timestamp': op_dict['op_timestamp'],
'location': existing_episode.location,
'placemark': existing_episode.placemark}
self.ValidateCreateDBObject(Episode, **new_ep_dict)
for photo_id in ep_dict['photo_ids']:
post = self.GetModelObject(Post, DBKey(ep_dict['new_episode_id'], photo_id), must_exist=False)
if post is None or post.IsRemoved():
if post is None:
self.ValidateCreateDBObject(Post, episode_id=ep_dict['new_episode_id'], photo_id=photo_id)
else:
self.ValidateUpdateDBObject(Post, episode_id=ep_dict['new_episode_id'], photo_id=photo_id, labels=[])
ph_act_dict.setdefault(viewpoint_id, {}).setdefault(ep_dict['new_episode_id'], []).append(photo_id)
def _OnUpdate(result):
self.assertEquals(result.write_units, 1)
self._client.GetItem(table=_table.name,
key=DBKey(u'2', 2),
attributes=[u'a1', u'a2'],
must_exist=False,
callback=_VerifyGet)
def _OnFirstUpdate(result):
self.assertEquals(result.write_units, 1)
self.assertEquals(
result.return_values, {
u'thk': u'2',
u'trk': 2,
u'a1': 5,
u'a2': u'update str',
u'a3': set([1, 2, 3]),
u'a4': set([u'1', u'3', u'2'])
})
self._client.UpdateItem(table=_table.name,
key=DBKey(u'2', 2),
callback=_OnSecondUpdate,
attributes={
'a1':
UpdateAttr(5, 'ADD'),
'a2':
UpdateAttr('update str 2', 'PUT'),
'a3':
UpdateAttr(set([4, 5, 6]), 'ADD'),
'a4':
UpdateAttr(set(['1', '2', '100']),
'DELETE')
},
return_values='UPDATED_NEW')
def testPutValues(self):
"""Verify operation with various attribute values."""
def _VerifyGet(result):
self.assertEqual(
{
u'a1': 0,
u'a2': u'str',
u'a3': set([0]),
u'a4': set([u'strà朋'])
}, result.attributes)
self.assertEqual(result.read_units, 0.5)
self.stop()
def _OnPut(result):
self._client.GetItem(table=_table.name,
key=DBKey(u'1', 1),
callback=_VerifyGet,
attributes=[u'a1', u'a2', u'a3', u'a4'])
self._client.PutItem(table=_table.name,
key=DBKey(u'1', 1),
callback=_OnPut,
attributes={
'a1': 0,
'a2': u'str',
'a3': set([0]),
'a4': set(['strà朋'])
})
def _ValidateRemoveOneContact(contact_id):
predicate = lambda c: c.contact_id == contact_id
existing_contacts = validator.QueryModelObjects(Contact,
predicate=predicate,
query_forward=True)
if len(existing_contacts) > 0:
last_contact = existing_contacts[-1]
if last_contact.IsRemoved():
# Keep the last matching contact because it's already been removed.
existing_contacts = existing_contacts[:-1]
elif not removed_contacts_reset:
removed_contact_dict = {
'user_id':
user_id,
'contact_id':
contact_id,
'contact_source':
Contact.GetContactSourceFromContactId(contact_id),
'timestamp':
util._TEST_TIME,
'sort_key':
Contact.CreateSortKey(contact_id, util._TEST_TIME),
'labels': [Contact.REMOVED]
}
validator.ValidateCreateContact(identities_properties=None,
**removed_contact_dict)
for contact in existing_contacts:
db_key = DBKey(user_id, contact.sort_key)
validator.ValidateDeleteDBObject(Contact, db_key)
def _TestRemoveFollowers(tester, user_cookie, request_dict):
"""Called by the ServiceTester in order to test remove_followers service API call."""
validator = tester.validator
user_id, device_id = tester.GetIdsFromCookie(user_cookie)
request_dict = deepcopy(request_dict)
viewpoint_id = request_dict['viewpoint_id']
remove_ids = request_dict['remove_ids']
# Send remove_followers request.
actual_dict = tester.SendRequest('remove_followers', user_cookie,
request_dict)
op_dict = tester._DeriveNotificationOpDict(user_id, device_id,
request_dict)
# Validate each of the removed followers.
for follower_id in remove_ids:
follower = validator.GetModelObject(Follower,
DBKey(follower_id, viewpoint_id),
must_exist=False)
if follower is not None:
new_labels = follower.labels.union(
[Follower.REMOVED, Follower.UNREVIVABLE])
follower = validator.ValidateFollower(
user_id=follower_id,
viewpoint_id=viewpoint_id,
labels=new_labels,
last_updated=op_dict['op_timestamp'])
# Validate activity and notifications for the add.
activity_dict = {
'name': 'remove_followers',
'activity_id': request_dict['activity']['activity_id'],
'timestamp': request_dict['activity']['timestamp'],
'follower_ids': remove_ids
}
def _GetInvalidate(follower_id):
if follower_id in remove_ids:
return {
'viewpoints': [{
'viewpoint_id': viewpoint_id,
'get_attributes': True
}]
}
else:
return {
'viewpoints': [{
'viewpoint_id': viewpoint_id,
'get_followers': True
}]
}
validator.ValidateFollowerNotifications(viewpoint_id, activity_dict,
op_dict, _GetInvalidate)
validator.ValidateViewpointAccounting(viewpoint_id)
tester._CompareResponseDicts('remove_followers', user_id, request_dict, {},
actual_dict)
return actual_dict
def testConditionalPutAndUpdate(self):
# Try to put an item with an attribute which must exist.
self.assertRaises(Exception,
self._client.PutItem,
table='Cond',
key=DBKey(hash_key=1, range_key=None),
expected={'attr1': 'val'},
attributes={'attr1': 'new_val'})
# Now add this item and try again.
self._client.PutItem(table='Cond',
key=DBKey(hash_key=1, range_key=None),
attributes={'attr1': 'new_val'},
callback=None)
# But with wrong value.
self.assertRaises(Exception,
self._client.PutItem,
table='Cond',
key=DBKey(hash_key=1, range_key=None),
expected={'attr1': 'val'},
attributes={'attr1': 'new_val'})
# Now with correct value.
self._client.PutItem(table='Cond',
key=DBKey(hash_key=1, range_key=None),
expected={'attr1': 'new_val'},
attributes={'attr1': 'even_newer_val'},
callback=None)
# Try to put an item which already exists, but which mustn't.
self.assertRaises(Exception,
self._client.PutItem,
table='Cond',
key=DBKey(hash_key=1, range_key=None),
expected={_hash_key_schema.name: False},
attributes={'attr1': 'new_val'},
callback=None)
# Try with a composite key object.
self.assertRaises(Exception,
self._client.PutItem,
table='Cond2',
key=DBKey(hash_key=1, range_key=1),
expected={'attr1': 'val'},
attributes={'attr1': 'new_val'})
self._client.PutItem(table='Cond2',
key=DBKey(hash_key=1, range_key=1),
attributes={'attr1': 'new_val'},
callback=None)
self.assertRaises(Exception,
self._client.PutItem,
table='Cond2',
key=DBKey(hash_key=1, range_key=1),
expected={_hash_key_schema.name: False},
attributes={'attr1': 'even_newer_val'},
callback=None)
self.stop()
def _Check(self):
"""Gathers pre-mutation information:
1. Episode and photos to upload.
2. Checkpoints list of episode and photo ids that need to be (re)created.
3. Checkpoints whether to attempt to set episode location and placemark from photos.
Validates the following:
1. Permissions to upload to the given episode.
2. Each photo can be uploaded into at most one episode.
"""
# Get existing episode, if it exists.
self._episode = yield gen.Task(Episode.Query,
self._client,
self._episode_id,
None,
must_exist=False)
if self._episode is not None and self._episode.parent_ep_id != None:
raise InvalidRequestError('Cannot upload photos into an episode that was saved.')
# Query for all photos in a batch.
photo_keys = [DBKey(ph_dict['photo_id'], None) for ph_dict in self._ph_dicts]
photos = yield gen.Task(Photo.BatchQuery,
self._client,
photo_keys,
None,
must_exist=False)
# Start populating the checkpoint if this the first time the operation has been run.
if self._op.checkpoint is None:
# Gather list of ids of new episode and photos that need to be created.
self._new_ids = set()
if self._episode is None:
self._new_ids.add(self._episode_id)
for photo, ph_dict in zip(photos, self._ph_dicts):
if photo is None:
self._new_ids.add(ph_dict['photo_id'])
elif photo.episode_id != self._episode_id:
raise InvalidRequestError('Cannot upload photo "%s" into multiple episodes.' % ph_dict['photo_id'])
# Determine whether episode location/placemark needs to be set.
self._set_location = self._episode is None or self._episode.location is None
self._set_placemark = self._episode is None or self._episode.placemark is None
# Set checkpoint.
# List of new episode/photo ids, and whether to set location/placemark need to be check-
# pointed because they may change in the UPDATE phase. If we fail after UPDATE, but
# before NOTIFY, we would not send correct notifications on retry.
checkpoint = {'new': list(self._new_ids),
'location': self._set_location,
'placemark': self._set_placemark}
yield self._op.SetCheckpoint(self._client, checkpoint)
else:
# Restore state from checkpoint.
self._new_ids = set(self._op.checkpoint['new'])
self._set_location = self._op.checkpoint['location']
self._set_placemark = self._op.checkpoint['placemark']
def _MakeMetadataDict(friend_user, forward_friend, reverse_friend):
user_dict = {'user_id': friend_user.user_id}
# Certain labels are visible even to non-friends.
labels = list(
friend_user.labels.intersection([User.REGISTERED,
User.TERMINATED]))
# User profile attributes should only be visible to those who consider caller a friend.
if reverse_friend is not None:
util.SetIfNotNone(user_dict, 'name', friend_user.name)
util.SetIfNotNone(user_dict, 'given_name', friend_user.given_name)
util.SetIfNotNone(user_dict, 'family_name',
friend_user.family_name)
util.SetIfNotNone(user_dict, 'email', friend_user.email)
util.SetIfNotNone(user_dict, 'picture', friend_user.picture)
util.SetIfNotNone(user_dict, 'merged_with',
friend_user.merged_with)
labels.append('friend')
user_dict['labels'] = labels
if friend_user.user_id == user_id:
# Subscriptions don't currently use the model so we can't access them here,
# but since most tests don't have subscriptions we just turn off validation
# in the ones that do.
user_dict['private'] = {'subscriptions': [], 'user_identities': []}
if friend_user.pwd_hash is None:
user_dict['private']['no_password'] = True
db_key = DBKey('us:%d' % user_id, AccountSettings.GROUP_NAME)
settings = validator.GetModelObject(AccountSettings,
db_key,
must_exist=False)
if settings is not None:
settings_dict = user_dict['private'].setdefault(
'account_settings', {})
util.SetIfNotNone(settings_dict, 'email_alerts',
settings.email_alerts)
util.SetIfNotNone(settings_dict, 'sms_alerts',
settings.sms_alerts)
util.SetIfNotEmpty(settings_dict, 'storage_options',
list(settings.storage_options))
predicate = lambda ident: ident.user_id == user_id
for expected_ident in validator.QueryModelObjects(
Identity, predicate=predicate):
ident_dict = {'identity': expected_ident.key}
if expected_ident.authority is not None:
ident_dict['authority'] = expected_ident.authority
user_dict['private']['user_identities'].append(ident_dict)
# Add attributes assigned to the friend by the user himself (such as nickname).
if forward_friend is not None:
util.SetIfNotNone(user_dict, 'nickname', forward_friend.nickname)
return user_dict
def _TestHidePhotos(tester, user_cookie, request_dict):
"""Called by the ServiceTester in order to test hide_photos service API call."""
validator = tester.validator
user_id, device_id = tester.GetIdsFromCookie(user_cookie)
request_dict = deepcopy(request_dict)
user = validator.GetModelObject(User, user_id)
# Send hide_photos request.
actual_dict = tester.SendRequest('hide_photos', user_cookie, request_dict)
op_dict = tester._DeriveNotificationOpDict(user_id, device_id,
request_dict)
# Validate UserPost objects.
ph_act_dict = {}
for request_ep in request_dict['episodes']:
episode_id = request_ep['episode_id']
episode = validator.GetModelObject(Episode, episode_id)
for photo_id in request_ep['photo_ids']:
post_id = Post.ConstructPostId(episode_id, photo_id)
user_post = validator.GetModelObject(UserPost,
DBKey(user_id, post_id),
must_exist=False)
# Gather set of photos that should have been hidden and will affect accounting.
if episode.viewpoint_id == user.private_vp_id:
if user_post is None or not user_post.IsHidden():
ph_act_dict.setdefault(episode.viewpoint_id,
{}).setdefault(episode_id,
[]).append(photo_id)
timestamp = op_dict[
'op_timestamp'] if user_post is None else user_post.timestamp
# Add HIDDEN label if not already there.
if user_post is None:
labels = [UserPost.HIDDEN]
else:
labels = user_post.labels
if not user_post.IsHidden():
labels = labels.union([UserPost.HIDDEN])
validator.ValidateUpdateDBObject(UserPost,
user_id=user_id,
post_id=post_id,
timestamp=timestamp,
labels=labels)
# Validate notification for the hide.
invalidate = {
'episodes': [{
'episode_id': request_ep['episode_id'],
'get_photos': True
} for request_ep in request_dict['episodes']]
}
validator.ValidateNotification('hide_photos', user_id, op_dict, invalidate)
return actual_dict
def _TestQueryEpisodes(tester, user_cookie, request_dict):
"""Called by the ServiceTester in order to test query_episodes
service API call.
"""
validator = tester.validator
cookie_dict = tester.DecodeUserCookie(user_cookie)
user_id = cookie_dict.get('user_id', None)
device_id = cookie_dict.get('device_id', None)
cookie_viewpoint_id = cookie_dict.get('viewpoint_id', None)
# Send query_episodes request.
actual_dict = tester.SendRequest('query_episodes', user_cookie,
request_dict)
limit = request_dict.get('photo_limit', None)
expected_dict = {'episodes': []}
for request_ep in request_dict['episodes']:
episode_id = request_ep['episode_id']
expected_ep = validator.GetModelObject(Episode,
episode_id,
must_exist=False)
if expected_ep is not None:
viewpoint_id = expected_ep.viewpoint_id
expected_vp = validator.GetModelObject(Viewpoint, viewpoint_id)
expected_foll = validator.GetModelObject(Follower,
DBKey(
user_id,
viewpoint_id),
must_exist=False)
# If follower can't view, or if there is a non-matching viewpoint in the cookie, then skip the episode.
is_cookie_viewpoint = cookie_viewpoint_id is None or cookie_viewpoint_id == viewpoint_id
can_view_content = is_cookie_viewpoint or expected_vp.IsSystem()
if can_view_content and expected_foll is not None and expected_foll.CanViewContent(
):
expected_ep_dict = {'episode_id': episode_id}
if request_ep.get('get_attributes', False):
expected_ep_dict.update(expected_ep._asdict())
if request_ep.get('get_photos', False):
photo_dicts, last_key = _CreateExpectedPhotos(
validator,
user_id,
device_id,
episode_id,
limit=limit,
start_key=request_ep.get('photo_start_key', None))
expected_ep_dict['photos'] = photo_dicts
if last_key is not None:
expected_ep_dict['last_key'] = last_key
expected_dict['episodes'].append(expected_ep_dict)
tester._CompareResponseDicts('query_episodes', user_id, request_dict,
expected_dict, actual_dict)
return actual_dict
def _QueryPosts(episode_id, photo_ids):
"""Queries the posts for the given episode id and photo ids."""
post_keys = [DBKey(episode_id, photo_id) for photo_id in photo_ids]
posts = yield gen.Task(Post.BatchQuery,
self._client,
post_keys,
None,
must_exist=False)
raise gen.Return(posts)
