def _ValidateRemoveOneContact(contact_id): predicate = lambda c: c.contact_id == contact_id existing_contacts = validator.QueryModelObjects(Contact, predicate=predicate, query_forward=True) if len(existing_contacts) > 0: last_contact = existing_contacts[-1] if last_contact.IsRemoved(): # Keep the last matching contact because it's already been removed. existing_contacts = existing_contacts[:-1] elif not removed_contacts_reset: removed_contact_dict = { 'user_id': user_id, 'contact_id': contact_id, 'contact_source': Contact.GetContactSourceFromContactId(contact_id), 'timestamp': util._TEST_TIME, 'sort_key': Contact.CreateSortKey(contact_id, util._TEST_TIME), 'labels': [Contact.REMOVED] } validator.ValidateCreateContact(identities_properties=None, **removed_contact_dict) for contact in existing_contacts: db_key = DBKey(user_id, contact.sort_key) validator.ValidateDeleteDBObject(Contact, db_key)
def testUnicode(self): """Test various interesting Unicode characters.""" base_name = escape.utf8( u'ààà朋友你好abc123\U00010000\U00010000\x00\x01\b\n\t ') timestamp = time.time() contact_id_lookup = dict() def _CreateContact(index): name = base_name + str(index) identity_key = 'Email:%s' % name return Contact.CreateFromKeywords(100, [(identity_key, None)], timestamp, Contact.GMAIL, name=name) def _VerifyContacts(query_expr, start_key, end_key, exp_indexes): actual_contacts = self._RunAsync(Contact.IndexQuery, self._client, query_expr, None, start_index_key=start_key, end_index_key=end_key) self.assertEqual(len(exp_indexes), len(actual_contacts)) for expected, actual in zip( [_CreateContact(i) for i in exp_indexes], actual_contacts): self.assertEqual(expected._asdict(), actual._asdict()) # Create 3 contacts under user 100 in the db. for i in xrange(3): contact = _CreateContact(i) contact_id_lookup[i] = contact.contact_id self._RunAsync(contact.Update, self._client) # Get contact by identity. identity_key = 'Email:%s' % base_name _VerifyContacts(('contact.identities={i}', { 'i': identity_key + '0' }), None, None, [0]) # Get multiple contacts. _VerifyContacts(('contact.identities={i} | contact.identities={i2}', { 'i': identity_key + '0', 'i2': identity_key + '1' }), None, None, [1, 0]) # Get contact with start key. sort_key = Contact.CreateSortKey(contact_id_lookup[1], timestamp) _VerifyContacts(('contact.identities={i} | contact.identities={i2}', { 'i': identity_key + '0', 'i2': identity_key + '1' }), DBKey(100, sort_key), None, [0]) # Get contact with end key. sort_key = Contact.CreateSortKey(contact_id_lookup[0], timestamp) _VerifyContacts(('contact.identities={i} | contact.identities={i2}', { 'i': identity_key + '0', 'i2': identity_key + '1' }), None, DBKey(100, sort_key), [1])
def _FormatAllAttributes(self, item): """Build list of (column, key, value, pretty_value). We need a list to keep the columns ordered. The interpretation of the 'key' column depends on the beginning of the 'term' column.""" attrs = [] term = item.get('t', None) key = item.get('k', None) data = item.get('d', None) split = term.split(':') table = split[0] key_pretty = key if table == 'co': db_key = Contact._ParseIndexKey(key) key_pretty = self._SortQueryLink('Contact', db_key.hash_key, db_key.range_key) elif table == 'ev': key_pretty = self._EpisodeLink(key) elif table == 'fo': db_key = Follower._ParseIndexKey(key) key_pretty = self._SortQueryLink('Follower', db_key.hash_key, db_key.range_key) elif table == 'id': key_pretty = self._HashQueryLink('Identity', key) elif table == 'vp': key_pretty = self._ViewpointLink(key) attrs.append(('term', 't', term, term)) attrs.append(('key', 'k', key, key_pretty)) attrs.append(('data', 't', data, data)) attrs.append(('_version', '_ve', data, data)) return attrs
def testMergeWithContacts(self): """Test merge in which another user has the source user as a contact.""" # Create user #1 contacts for user #2 and user #3. exp_contacts = [] for user, identity in zip(self._users[1:], self._identities[1:]): contact_dict = Contact.CreateContactDict(self._user.user_id, [(identity.key, None)], util._TEST_TIME, Contact.GMAIL) exp_contacts.append({ 'contact_id': contact_dict['contact_id'], 'contact_source': contact_dict['contact_source'], 'identities': [{ 'identity': identity.key, 'user_id': 3 }] }) self._UpdateOrAllocateDBObject(Contact, **contact_dict) # Merge user #2 into user #3. Contact's user id should be updated. self._tester.MergeAccounts(self._cookie3, source_user_cookie=self._confirmed_cookie2) self.assertEqual( sorted(self._tester.QueryContacts(self._cookie)['contacts']), sorted(exp_contacts)) # Remove cookie so that base class won't try to validate its user's assets (and fail). self._cookies.remove(self._cookie2)
def ValidateRewriteContacts(self, identity_key, op_dict): """Validates that all contacts that reference "identity_key" have been updated with the new timestamp. """ # Iterate over all contacts that reference the identity. for co in self.QueryModelObjects(Contact, predicate=lambda co: identity_key in co.identities): # Validate that contact has been updated. contact_dict = co._asdict() contact_dict['timestamp'] = op_dict['op_timestamp'] sort_key = Contact.CreateSortKey(Contact.CalculateContactId(contact_dict), contact_dict['timestamp']) contact_dict['sort_key'] = sort_key self.ValidateUpdateDBObject(Contact, **contact_dict) # Validate that any old contacts have been deleted. if sort_key != co.sort_key: self.ValidateDeleteDBObject(Contact, co.GetKey())
def _CreateContact(index): name = base_name + str(index) identity_key = 'Email:%s' % name return Contact.CreateFromKeywords(100, [(identity_key, None)], timestamp, Contact.GMAIL, name=name)
def testStringSetIndexing(self): """Tests indexing of items in string set columns.""" emails = [ '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**', '*****@*****.**' ] # Create a bunch of contacts with one or two identities. timestamp = util.GetCurrentTimestamp() for email in emails: for email2 in emails: contact = Contact.CreateFromKeywords( 1, [('Email:' + email, None), ('Email:' + email2, None)], timestamp, Contact.GMAIL) self._RunAsync(contact.Update, self._client) for email in emails: q_contacts = self._RunAsync(Contact.IndexQuery, self._client, ('contact.identities={i}', { 'i': 'Email:' + email }), col_names=None) logging.debug('querying for %s=%s yielded %d matches' % ('identities', 'Email:' + email, len(q_contacts))) for contact in q_contacts: self.assertTrue('Email:' + email in contact.identities) self.assertEqual(len(q_contacts), len(emails) * 2 - 1)
def _ValidateUploadOneContact(contact): contact = deepcopy(contact) # Transform into proper form for Contact.CalculateContactId() identities_properties = [ (identity_properties['identity'], identity_properties.get('description', None)) for identity_properties in contact['identities'] ] contact.pop('identities') contact_dict = Contact.CreateContactDict(user_id, identities_properties, op_dict['op_timestamp'], **contact) predicate = lambda c: c.contact_id == contact_dict['contact_id'] existing_contacts = validator.QueryModelObjects(Contact, predicate=predicate) # Create contact if it doesn't already exist or it's in a 'removed' state. if len(existing_contacts) == 0 or existing_contacts[-1].IsRemoved(): if len(existing_contacts) != 0: # Delete the 'removed' contact. validator.ValidateDeleteDBObject( Contact, DBKey(user_id, existing_contacts[-1].sort_key)) validator.ValidateCreateContact(**contact_dict) return contact_dict['contact_id']
def testRegisterProspectiveContact(self): """Register an identity that is the target of a contact (that is still a prospective user).""" for user_id in [self._user.user_id, self._user2.user_id]: # Create several contacts. identity_key = 'Email:%s' % self._prospective_user.email contact_dict = Contact.CreateContactDict(user_id, [(identity_key, None)], util._TEST_TIME, Contact.GMAIL, name='Mr. John') self._UpdateOrAllocateDBObject(Contact, **contact_dict) # Register the prospective user. user, device_id = self._tester.RegisterViewfinderUser(self._register_user_dict) # Expect friend & contact notifications. response_dict = self._tester.QueryNotifications(self._cookie, 2, scan_forward=False) self.assertEqual([notify_dict['name'] for notify_dict in response_dict['notifications']], ['register friend', 'first register contact']) # Expect only contact notification. response_dict = self._tester.QueryNotifications(self._cookie2, 1, scan_forward=False) self.assertEqual([notify_dict['name'] for notify_dict in response_dict['notifications']], ['first register contact']) # Expect only friend notification. cookie = self._GetSecureUserCookie(self._prospective_user, self._prospective_user.webapp_dev_id) response_dict = self._tester.QueryNotifications(cookie, 2, scan_forward=False) self.assertEqual([notify_dict['name'] for notify_dict in response_dict['notifications']], ['register friend', 'share_new'])
def _VerifyAccountStatus(cookie, results): u = results['user'] dev = results['device'] ident = results['identity'] self.assertEqual(ident.user_id, u.user_id) self.assertTrue(u.name) self.assertTrue(u.given_name) self.assertTrue(u.family_name) self.assertIsNotNone(u.webapp_dev_id) [ self.assertEqual(getattr(dev, k), v) for k, v in self._mobile_device_dict.items() ] # Keep querying until notifications are found. while True: response_dict = self._SendRequest('query_notifications', cookie, {}) if len(response_dict['notifications']) > 2: break time.sleep(0.100) self.assertEqual(response_dict['notifications'][1]['name'], 'register friend') notification = response_dict['notifications'][2] self.assertEqual(notification['name'], 'fetch_contacts') sort_key = Contact.CreateSortKey(None, notification['timestamp']) self.assertEqual( notification['invalidate']['contacts']['start_key'], sort_key) self.stop()
def ValidateContactNotifications(self, name, identity_key, op_dict): """Validates that a notification was created for each user who references a contact of the specified identity. """ invalidate = {'contacts': {'start_key': Contact.CreateSortKey(None, op_dict['op_timestamp'])}} for co in self.QueryModelObjects(Contact, predicate=lambda co: identity_key in co.identities): self.ValidateNotification(name, co.user_id, op_dict, invalidate)
def _ReplaceRemovedContact(contact_dict_to_insert, removed_contact_to_delete): contact_to_insert = Contact.CreateFromKeywords( **contact_dict_to_insert) yield gen.Task(contact_to_insert.Update, self._client) if removed_contact_to_delete is not None: yield Operation.TriggerFailpoint(self._client) yield gen.Task(removed_contact_to_delete.Delete, self._client)
def testUnicodeContactNames(self): """Test that contact_id generation works correctly when names include non-ascii characters.""" name = u'ààà朋友你好abc123\U00010000\U00010000\x00\x01\b\n\t ' # The following will assert if there are problems when calculating the hash for the contact_id: contact_a = Contact.CreateFromKeywords(1, [('Email:[email protected]', None)], util.GetCurrentTimestamp(), Contact.GMAIL, name=name) contact_b = Contact.CreateFromKeywords(1, [('Email:[email protected]', None)], util.GetCurrentTimestamp(), Contact.GMAIL, name=u'朋' + name[1:]) # Check that making a slight change to a unicode self.assertNotEqual(contact_a.contact_id, contact_b.contact_id)
def _RemoveContact(contact_to_remove): """Insert a 'removed' contact with the same contact_id as the one being removed and then delete the actual contact that's being removed.""" removed_contact = Contact.CreateRemovedContact( contact_to_remove.user_id, contact_to_remove.contact_id, self._notify_timestamp) yield gen.Task(removed_contact.Update, self._client) yield Operation.TriggerFailpoint(self._client) yield gen.Task(contact_to_remove.Delete, self._client)
def _CreateSimpleContacts(self): """Create multiple unbound contacts for user #1 and user #2.""" for user_id in [self._user.user_id, self._user2.user_id]: for identity_key in ['Local:identity1', 'Local:identity2']: contact_dict = Contact.CreateContactDict(user_id, [('Phone:+13191234567', 'mobile'), (identity_key, None)], util._TEST_TIME, Contact.GMAIL) self._UpdateOrAllocateDBObject(Contact, **contact_dict)
def ValidateCreateContact(self, user_id, identities_properties, timestamp, contact_source, **op_dict): """Validates creation of contact along with derived attributes. Returns created contact. """ contact_dict = op_dict contact_dict['user_id'] = user_id contact_dict['timestamp'] = timestamp if identities_properties is not None: contact_dict['identities_properties'] = identities_properties contact_dict['identities'] = set([Identity.Canonicalize(identity_properties[0]) for identity_properties in identities_properties]) else: contact_dict['identities_properties'] = None contact_dict['contact_source'] = contact_source if 'contact_id' not in contact_dict: contact_dict['contact_id'] = Contact.CalculateContactId(contact_dict) if 'sort_key' not in contact_dict: contact_dict['sort_key'] = Contact.CreateSortKey(contact_dict['contact_id'], timestamp) return self.ValidateCreateDBObject(Contact, **contact_dict)
def testUnlinkContacts(self): """Test unlinking identity that is referenced by contacts.""" for i in xrange(3): contact_dict = Contact.CreateContactDict( self._users[i].user_id, [('FacebookGraph:100', None)], util._TEST_TIME, Contact.FACEBOOK, rank=i) self._UpdateOrAllocateDBObject(Contact, **contact_dict) self._tester.LinkFacebookUser({'id': 100}, user_cookie=self._cookie) response_dict = self._tester.QueryContacts( self._cookie2, start_key=Contact.CreateSortKey(None, util._TEST_TIME)) self.assertEqual(len(response_dict['contacts']), 1) # Now unlink the identity and make sure contacts are updated. util._TEST_TIME += 1 self._tester.UnlinkIdentity(self._cookie, 'FacebookGraph:100') response_dict = self._tester.QueryContacts( self._cookie2, start_key=Contact.CreateSortKey(None, util._TEST_TIME)) self.assertEqual(len(response_dict['contacts']), 1) # Now repeat, but with older contacts. util._TEST_TIME += 1 timestamp = time.time() - 100 for i in xrange(3): contact_dict = Contact.CreateContactDict( self._users[i].user_id, [('FacebookGraph:101', None)], timestamp, Contact.FACEBOOK, rank=i) contact = self._UpdateOrAllocateDBObject(Contact, **contact_dict) self._tester.LinkFacebookUser({'id': 101}, user_cookie=self._cookie) self._tester.UnlinkIdentity(self._cookie, 'FacebookGraph:101') response_dict = self._tester.QueryContacts( self._cookie3, start_key=Contact.CreateSortKey(None, util._TEST_TIME)) self.assertEqual(len(response_dict['contacts']), 1)
def testQueryBoundContact(self): """Fetch a contact that is bound to a user.""" self._CreateContact([('Local:1', None, 100)]) response_dict = self._tester.QueryContacts(self._cookie) contact_dict = { 'identities_properties': [('Local:1', None)], 'contact_source': Contact.GMAIL } contact_dict['contact_id'] = Contact.CalculateContactId(contact_dict) contact_dict['identities'] = [{'identity': 'Local:1', 'user_id': 100}] contact_dict.pop('identities_properties') self.assertEqual(response_dict['contacts'][0], contact_dict)
def testQueryContactNoIdentity(self): """Fetch a contact with no corresponding identity object.""" self._CreateContact([('Local:1', None)], no_identity=True) response_dict = self._tester.QueryContacts(self._cookie) contact_dict = { 'identities_properties': [('Local:1', None)], 'contact_source': Contact.GMAIL } contact_dict['contact_id'] = Contact.CalculateContactId(contact_dict) contact_dict['identities'] = [{'identity': 'Local:1'}] contact_dict.pop('identities_properties') self.assertEqual(response_dict['contacts'][0], contact_dict)
def NotifyUploadContacts(cls, client, user_id, timestamp): """Adds a notification for the specified user that new contacts have been uploaded/updated and need to be pulled to the client. Invalidates all the newly uploaded/updated contacts, which all have sort_key greater than or equal to "timestamp". """ invalidate = { 'contacts': { 'start_key': Contact.CreateSortKey(None, timestamp) } } yield NotificationManager.CreateForUser(client, user_id, 'upload_contacts', invalidate=invalidate)
def testUnlinkIdentity(self): """Verify unlinking an identity causes every referencing contact to be updated.""" # Create a Peter contact for Spencer. timestamp = util.GetCurrentTimestamp() spencer = self._user contact_identity = 'Email:[email protected]' contact_name = 'Peter Mattis' contact_given_name = 'Peter' contact_family_name = 'Mattis' contact_rank = 42 contact = Contact.CreateFromKeywords(spencer.user_id, [(contact_identity, None)], timestamp, Contact.GMAIL, name='Peter Mattis', given_name='Peter', family_name='Mattis', rank=42) self._RunAsync(contact.Update, self._client) peter_ident = self._RunAsync(Identity.Query, self._client, contact_identity, None) # Unlink peter's identity, which should cause Spencer's contact to be updated. self._RunAsync(peter_ident.UnlinkIdentity, self._client, self._user2.user_id, contact_identity, timestamp + 1) contacts = self._RunAsync(Contact.RangeQuery, self._client, spencer.user_id, None, None, None) self.assertEqual(len(contacts), 1) self.assertEqual( contacts[0].sort_key, Contact.CreateSortKey(contact.contact_id, timestamp + 1)) self.assertEqual(contacts[0].name, contact_name) self.assertEqual(contacts[0].given_name, contact_given_name) self.assertEqual(contacts[0].family_name, contact_family_name) self.assertEqual(contacts[0].rank, contact_rank)
def _RewriteOneContact(co): """Update the given contact's timestamp.""" new_co = Contact.CreateFromKeywords(co.user_id, co.identities_properties, timestamp, co.contact_source, name=co.name, given_name=co.given_name, family_name=co.family_name, rank=co.rank) # Only rewrite if timestamp is different. if co.sort_key != new_co.sort_key: yield gen.Task(new_co.Update, client) yield gen.Task(co.Delete, client)
def NotifyFetchContacts(cls, client, user_id, timestamp, reload_all): """Adds a notification for the specified user that new contacts have been fetched and need to be pulled to the client. Invalidates all the newly fetched contacts, which all have sort_key greater than "timestamp". """ invalidate = { 'contacts': { 'start_key': Contact.CreateSortKey(None, 0 if reload_all else timestamp) } } if reload_all: invalidate['contacts']['all'] = True yield NotificationManager.CreateForUser(client, user_id, 'fetch_contacts', invalidate=invalidate)
def NotifyRemoveContacts(cls, client, user_id, timestamp, reload_all): """Adds a notification for the specified user that contacts have been removed and need to be removed from the client. Invalidates all the newly removed contacts, which all have sort_key greater than or equal to "timestamp". """ invalidate = { 'contacts': { 'start_key': Contact.CreateSortKey(None, 0 if reload_all else timestamp) } } if reload_all: invalidate['contacts']['all'] = True yield NotificationManager.CreateForUser(client, user_id, 'remove_contacts', invalidate=invalidate)
def _NotifyUsersWithContact(cls, client, name, identity_key, timestamp): """Adds a notification to all users having contacts with the specified identity. Invalidates all contacts added after the specified timestamp. """ @gen.coroutine def _VisitContactUserId(contact_user_id): yield NotificationManager.CreateForUser(client, contact_user_id, name, invalidate=invalidate) invalidate = { 'contacts': { 'start_key': Contact.CreateSortKey(None, timestamp) } } yield gen.Task(Contact.VisitContactUserIds, client, identity_key, _VisitContactUserId)
def testTerminateWithContact(self): """Terminate user account to which a contact was linked.""" # Create contact for user #1 identity_key = 'Email:[email protected]' contact_dict = Contact.CreateContactDict( self._user.user_id, [('Email:[email protected]', None)], util._TEST_TIME, Contact.GMAIL) self._UpdateOrAllocateDBObject(Contact, **contact_dict) self._tester.TerminateAccount(self._cookie3) response_dict = self._tester.QueryNotifications(self._cookie, 1, scan_forward=False) self.assertEqual(response_dict['notifications'][0]['name'], 'unlink_identity') # Remove cookie so that base class won't try to validate its user's assets (and fail). self._cookies.remove(self._cookie3)
def testRegisterContact(self): """Register an identity that is the target of a contact, which will be bound to a user_id as a result. """ # Create a contact. user_dict = {'name': 'Andrew Kimball', 'email': '*****@*****.**', 'verified_email': True} identity_key = 'Email:%s' % user_dict['email'] contact_dict = Contact.CreateContactDict(self._user.user_id, [(identity_key, None)], util._TEST_TIME, Contact.GMAIL, name=user_dict['name']) self._UpdateOrAllocateDBObject(Contact, **contact_dict) # Register the new user. user, device_id = self._tester.RegisterGoogleUser(user_dict) response_dict = self._tester.QueryNotifications(self._cookie, 1, scan_forward=False) self.assertEqual([notify_dict['name'] for notify_dict in response_dict['notifications']], ['first register contact'])
def _CreateContact(self, identities_properties, no_identity=False, **kwargs): if not no_identity: for identity_properties in identities_properties: contact_user_id = identity_properties[2] if len( identity_properties) > 2 else None self._UpdateOrAllocateDBObject(Identity, key=Identity.Canonicalize( identity_properties[0]), user_id=contact_user_id) contact_identities_properties = [] for identity_properties in identities_properties: contact_identities_properties.append(identity_properties[:2]) contact_dict = Contact.CreateContactDict( self._user.user_id, contact_identities_properties, util._TEST_TIME, Contact.GMAIL, **kwargs) self._UpdateOrAllocateDBObject(Contact, **contact_dict)
def testLinkWithContacts(self): """Test link of an identity which another user has as a contact.""" # Create contact for user #1 identity_key = 'Email:[email protected]' contact_dict = Contact.CreateContactDict(self._user.user_id, [(identity_key, None)], util._TEST_TIME, Contact.GMAIL) self._UpdateOrAllocateDBObject(Contact, **contact_dict) # Link the identity to user #2 and verify that user #1 is notified. source_identity_dict = self._TestGenerateMergeToken( identity_key, user_cookie=self._cookie2) self._tester.MergeAccounts(self._cookie2, source_identity_dict=source_identity_dict) response_dict = self._tester.QueryNotifications(self._cookie, 1, scan_forward=False) self.assertEqual(response_dict['notifications'][0]['name'], 'link identity')
def testDerivedAttributes(self): """Test that the identity and identities attributes are being properly derived from the identities_properties attribute. """ # Create a Peter contact for Spencer with multiple identical and nearly identical identities. spencer = self._user contact_identity_a = 'Email:[email protected]' contact_identity_b = 'Email:[email protected]' contact_identity_c = 'Email:[email protected]' timestamp = util.GetCurrentTimestamp() contact = Contact.CreateFromKeywords(spencer.user_id, [(contact_identity_a, None), (contact_identity_b, 'home'), (contact_identity_c, 'work')], timestamp, Contact.GMAIL, name='Peter Mattis', given_name='Peter', family_name='Mattis', rank=42) self.assertEqual(len(contact.identities_properties), 3) self.assertEqual(len(contact.identities), 2) self.assertFalse(contact_identity_a in contact.identities) self.assertFalse(contact_identity_b in contact.identities) self.assertFalse(contact_identity_c in contact.identities) self.assertTrue( Identity.Canonicalize(contact_identity_a) in contact.identities) self.assertTrue( Identity.Canonicalize(contact_identity_b) in contact.identities) self.assertTrue( Identity.Canonicalize(contact_identity_c) in contact.identities) self.assertTrue( [contact_identity_a, None] in contact.identities_properties) self.assertTrue( [contact_identity_b, 'home'] in contact.identities_properties) self.assertTrue( [contact_identity_c, 'work'] in contact.identities_properties)