def _CreateContact(index):
name = base_name + str(index)
identity_key = 'Email:%s' % name
return Contact.CreateFromKeywords(100, [(identity_key, None)],
timestamp,
Contact.GMAIL,
name=name)
def testStringSetIndexing(self):
"""Tests indexing of items in string set columns."""
emails = [
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**', '*****@*****.**',
'*****@*****.**'
]
# Create a bunch of contacts with one or two identities.
timestamp = util.GetCurrentTimestamp()
for email in emails:
for email2 in emails:
contact = Contact.CreateFromKeywords(
1, [('Email:' + email, None),
('Email:' + email2, None)], timestamp, Contact.GMAIL)
self._RunAsync(contact.Update, self._client)
for email in emails:
q_contacts = self._RunAsync(Contact.IndexQuery,
self._client,
('contact.identities={i}', {
'i': 'Email:' + email
}),
col_names=None)
logging.debug('querying for %s=%s yielded %d matches' %
('identities', 'Email:' + email, len(q_contacts)))
for contact in q_contacts:
self.assertTrue('Email:' + email in contact.identities)
self.assertEqual(len(q_contacts), len(emails) * 2 - 1)
def testUnicodeContactNames(self):
"""Test that contact_id generation works correctly when names include non-ascii characters."""
name = u'ààà朋友你好abc123\U00010000\U00010000\x00\x01\b\n\t '
# The following will assert if there are problems when calculating the hash for the contact_id:
contact_a = Contact.CreateFromKeywords(1, [('Email:[email protected]', None)],
util.GetCurrentTimestamp(),
Contact.GMAIL,
name=name)
contact_b = Contact.CreateFromKeywords(1, [('Email:[email protected]', None)],
util.GetCurrentTimestamp(),
Contact.GMAIL,
name=u'朋' + name[1:])
# Check that making a slight change to a unicode
self.assertNotEqual(contact_a.contact_id, contact_b.contact_id)
def _ReplaceRemovedContact(contact_dict_to_insert,
removed_contact_to_delete):
contact_to_insert = Contact.CreateFromKeywords(
**contact_dict_to_insert)
yield gen.Task(contact_to_insert.Update, self._client)
if removed_contact_to_delete is not None:
yield Operation.TriggerFailpoint(self._client)
yield gen.Task(removed_contact_to_delete.Delete, self._client)
def _RewriteOneContact(co):
"""Update the given contact's timestamp."""
new_co = Contact.CreateFromKeywords(co.user_id,
co.identities_properties,
timestamp,
co.contact_source,
name=co.name,
given_name=co.given_name,
family_name=co.family_name,
rank=co.rank)
# Only rewrite if timestamp is different.
if co.sort_key != new_co.sort_key:
yield gen.Task(new_co.Update, client)
yield gen.Task(co.Delete, client)
def testDerivedAttributes(self):
"""Test that the identity and identities attributes are being properly derived from the
identities_properties attribute.
"""
# Create a Peter contact for Spencer with multiple identical and nearly identical identities.
spencer = self._user
contact_identity_a = 'Email:[email protected]'
contact_identity_b = 'Email:[email protected]'
contact_identity_c = 'Email:[email protected]'
timestamp = util.GetCurrentTimestamp()
contact = Contact.CreateFromKeywords(spencer.user_id,
[(contact_identity_a, None),
(contact_identity_b, 'home'),
(contact_identity_c, 'work')],
timestamp,
Contact.GMAIL,
name='Peter Mattis',
given_name='Peter',
family_name='Mattis',
rank=42)
self.assertEqual(len(contact.identities_properties), 3)
self.assertEqual(len(contact.identities), 2)
self.assertFalse(contact_identity_a in contact.identities)
self.assertFalse(contact_identity_b in contact.identities)
self.assertFalse(contact_identity_c in contact.identities)
self.assertTrue(
Identity.Canonicalize(contact_identity_a) in contact.identities)
self.assertTrue(
Identity.Canonicalize(contact_identity_b) in contact.identities)
self.assertTrue(
Identity.Canonicalize(contact_identity_c) in contact.identities)
self.assertTrue(
[contact_identity_a, None] in contact.identities_properties)
self.assertTrue(
[contact_identity_b, 'home'] in contact.identities_properties)
self.assertTrue(
[contact_identity_c, 'work'] in contact.identities_properties)
def testUnlinkIdentity(self):
"""Verify unlinking an identity causes every referencing contact to be updated."""
# Create a Peter contact for Spencer.
timestamp = util.GetCurrentTimestamp()
spencer = self._user
contact_identity = 'Email:[email protected]'
contact_name = 'Peter Mattis'
contact_given_name = 'Peter'
contact_family_name = 'Mattis'
contact_rank = 42
contact = Contact.CreateFromKeywords(spencer.user_id,
[(contact_identity, None)],
timestamp,
Contact.GMAIL,
name='Peter Mattis',
given_name='Peter',
family_name='Mattis',
rank=42)
self._RunAsync(contact.Update, self._client)
peter_ident = self._RunAsync(Identity.Query, self._client,
contact_identity, None)
# Unlink peter's identity, which should cause Spencer's contact to be updated.
self._RunAsync(peter_ident.UnlinkIdentity, self._client,
self._user2.user_id, contact_identity, timestamp + 1)
contacts = self._RunAsync(Contact.RangeQuery, self._client,
spencer.user_id, None, None, None)
self.assertEqual(len(contacts), 1)
self.assertEqual(
contacts[0].sort_key,
Contact.CreateSortKey(contact.contact_id, timestamp + 1))
self.assertEqual(contacts[0].name, contact_name)
self.assertEqual(contacts[0].given_name, contact_given_name)
self.assertEqual(contacts[0].family_name, contact_family_name)
self.assertEqual(contacts[0].rank, contact_rank)
def _FetchFacebookContacts(self):
"""Do Facebook specific data gathering and checking.
Queries Facebook graph API for friend list using the identity's access token.
"""
@gen.coroutine
def _DetermineFacebookRankings():
"""Uses The tags from friends and the authors of the
photos are used to determine friend rank for facebook contacts. The
basic algorithm is:
sorted([sum(exp_decay(pc.time) * strength(pc)) for pc in photos])
A 'pc' in is a photo connection. There are three types, ordered by
the 'strength' they impart in the summation equation:
- from: the poster of a photo (strength=1.0)
- tag: another user tagged in the photo (strength=1.0)
- like: a facebook user who 'liked' the photo (strength=0.25)
Exponential decay uses _FACEBOOK_CONNECTION_HALF_LIFE for half life.
The rankings are passed to the provided callback as a dictionary of
identity ('FacebookGraph:<id>') => rank.
"""
logging.info(
'determining facebook contact rankings for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
friends = dict() # facebook id => connection strength
likes = dict()
now = util.GetCurrentTimestamp()
def _ComputeScore(create_iso8601, conn_type):
"""Computes the strength of a photo connection based on the time
that's passed and the connection type.
"""
decay = 0.001 # default is 1/1000th
if create_iso8601:
dt = iso8601.parse_date(create_iso8601)
create_time = calendar.timegm(dt.utctimetuple())
decay = math.exp(
-math.log(2) * (now - create_time) /
FetchContactsOperation._FACEBOOK_CONNECTION_HALF_LIFE)
return decay * FetchContactsOperation._PHOTO_CONNECTION_STRENGTHS[
conn_type]
# Construct the URL that will kick things off.
url = FetchContactsOperation._FACEBOOK_PHOTOS_URL + '?' + \
urllib.urlencode({'access_token': self._identity.access_token,
'format': 'json', 'limit': FetchContactsOperation._MAX_FETCH_COUNT})
while True:
logging.info(
'querying next %d Facebook photos for user %d' %
(FetchContactsOperation._MAX_FETCH_COUNT, self._user_id))
response = yield gen.Task(http_client.fetch, url, method='GET')
response_dict = www_util.ParseJSONResponse(response)
for p_dict in response_dict['data']:
created_time = p_dict.get('created_time', None)
if p_dict.get('from', None) and p_dict['from']['id']:
from_id = p_dict['from']['id']
friends[from_id] = friends.get(from_id, 0.0) + \
_ComputeScore(created_time, 'from')
if p_dict.get('tags', None):
for tag in p_dict['tags']['data']:
if tag.get('id', None) is not None:
friends[tag['id']] = friends.get(tag['id'], 0.0) + \
_ComputeScore(tag.get('created_time', None), 'tag')
if p_dict.get('likes', None):
for like in p_dict['likes']['data']:
if like.get('id', None) is not None:
likes[like['id']] = likes.get(like['id'], 0.0) + \
_ComputeScore(created_time, 'like')
if (len(response_dict['data'])
== FetchContactsOperation._MAX_FETCH_COUNT
and response_dict.has_key('paging')
and response_dict['paging'].has_key('next')):
url = response_dict['paging']['next']
else:
for fb_id in friends.keys():
friends[fb_id] += likes.get(fb_id, 0.0)
ranked_friends = sorted(friends.items(),
key=itemgetter(1),
reverse=True)
logging.info(
'successfully ranked %d Facebook contacts for user %d'
% (len(ranked_friends), self._user_id))
raise gen.Return(dict([('FacebookGraph:%s' % fb_id, rank) for rank, (fb_id, _) in \
izip(xrange(len(ranked_friends)), ranked_friends)]))
logging.info('fetching Facebook contacts for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
# Track fetched contacts regardless of rank in order to dedup contacts retrieved from Facebook.
rankless_ids = set()
# First get the rankings and then fetch the contacts.
rankings = yield _DetermineFacebookRankings()
url = FetchContactsOperation._FACEBOOK_FRIENDS_URL + '?' + \
urllib.urlencode({'fields': 'first_name,name,last_name',
'access_token': self._identity.access_token,
'format': 'json', 'limit': FetchContactsOperation._MAX_FETCH_COUNT})
retries = 0
while True:
if retries >= FetchContactsOperation._MAX_FETCH_RETRIES:
raise TooManyRetriesError(
'failed to fetch contacts %d times; aborting' % retries)
logging.info(
'fetching next %d Facebook contacts for user %d' %
(FetchContactsOperation._MAX_FETCH_COUNT, self._user_id))
response = yield gen.Task(http_client.fetch, url, method='GET')
try:
response_dict = www_util.ParseJSONResponse(response)
except Exception as exc:
logging.warning('failed to fetch Facebook contacts: %s' % exc)
retries += 1
continue
for c_dict in response_dict['data']:
if c_dict.has_key('id'):
ident = 'FacebookGraph:%s' % c_dict['id']
# Skip contact if name is not present, or is empty.
name = c_dict.get('name', None)
if name:
names = {
'name': name,
'given_name': c_dict.get('first_name', None),
'family_name': c_dict.get('last_name', None)
}
# Check to see if we've already processed an identical contact.
rankless_id = Contact.CalculateContactEncodedDigest(
identities_properties=[(ident, None)], **names)
if rankless_id in rankless_ids:
# Duplicate among fetched contacts. Skip it.
continue
else:
rankless_ids.add(rankless_id)
rank = rankings[ident] if ident in rankings else None
fetched_contact = Contact.CreateFromKeywords(
self._user_id, [(ident, None)],
self._notify_timestamp,
Contact.FACEBOOK,
rank=rank,
**names)
self._fetched_contacts[
fetched_contact.contact_id] = fetched_contact
# Prepare to fetch next batch.
if (len(response_dict['data'])
== FetchContactsOperation._MAX_FETCH_COUNT
and response_dict.has_key('paging')
and response_dict['paging'].has_key('next')):
retries = 0
url = response_dict['paging']['next']
else:
break
def _FetchGoogleContacts(self):
"""Do GMail specific data gathering and checking.
Queries Google data API for contacts in JSON format.
"""
# Track fetched contacts regardless of rank in order to dedup contacts retrieved from Google.
assert self._identity.refresh_token is not None, self._identity
if self._identity.expires and self._identity.expires < time.time():
yield gen.Task(self._identity.RefreshGoogleAccessToken,
self._client)
logging.info('fetching Google contacts for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
# Google data API uses 1-based start index.
start_index = 1
retries = 0
count = FetchContactsOperation._MAX_FETCH_COUNT
while True:
if retries >= FetchContactsOperation._MAX_FETCH_RETRIES:
raise TooManyRetriesError(
'failed to fetch contacts %d times; aborting' % retries)
logging.info('fetching next %d Google contacts for user %d' %
(count, self._user_id))
url = FetchContactsOperation._GOOGLE_CONTACTS_URL + '?' + \
urllib.urlencode({'max-results': count,
'start-index': start_index,
'alt': 'json'})
response = yield gen.Task(http_client.fetch,
url,
method='GET',
headers={
'Authorization':
'OAuth %s' %
self._identity.access_token,
'GData-Version':
3.0
})
try:
response_dict = www_util.ParseJSONResponse(response)['feed']
except Exception as exc:
logging.warning('failed to fetch Google contacts: %s' % exc)
retries += 1
continue
# Temporarily log additional information to figure out why some responses don't seem to have "entry" fields.
if 'entry' not in response_dict:
logging.warning('Missing entry: %s' %
json.dumps(response_dict, indent=True))
for c_dict in response_dict.get('entry', []):
# Build identities_properties list from all emails/phone numbers associated with this contact.
identities_properties = []
# Process emails first so that if there are any emails, one of them will be first in the
# identities_properties list. This will be *the* identity used for down-level client message
# migration.
for email_info in c_dict.get('gd$email', []):
email = email_info.get('address', None)
if email is not None:
email_type = FetchContactsOperation._GOOGLE_TYPE_LOOKUP.get(
email_info.get('rel', None), None)
identity_properties = (
'Email:' + Identity.CanonicalizeEmail(email),
email_info.get('label', email_type))
if email_info.get('primary', False):
# Insert the primary email address at the head of the list. Older clients will get this
# as the only email address for this contact when they query_contacts.
identities_properties.insert(
0, identity_properties)
else:
identities_properties.append(identity_properties)
for phone_info in c_dict.get('gd$phoneNumber', []):
# See RFC3966: "The tel URI for Telephone Numbers" for more information about this format.
# It should be 'tel:' + E.164 format phone number.
phone = phone_info.get('uri', None)
if phone is not None and phone.startswith(
'tel:+') and Identity.CanCanonicalizePhone(
phone[4:]):
phone_type = FetchContactsOperation._GOOGLE_TYPE_LOOKUP.get(
phone_info.get('rel', None), None)
identities_properties.append(
('Phone:' + Identity.CanonicalizePhone(phone[4:]),
phone_info.get('label', phone_type)))
if len(identities_properties) == 0:
continue
# Normalize name to None if empty.
gd_name = c_dict.get('gd$name', None)
if gd_name is not None:
names = {
'name':
gd_name.get('gd$fullName', {}).get('$t', None),
'given_name':
gd_name.get('gd$givenName', {}).get('$t', None),
'family_name':
gd_name.get('gd$familyName', {}).get('$t', None)
}
else:
names = {
'name': None,
'given_name': None,
'family_name': None
}
fetched_contact = Contact.CreateFromKeywords(
self._user_id,
identities_properties,
self._notify_timestamp,
Contact.GMAIL,
rank=None,
**names)
self._fetched_contacts[
fetched_contact.contact_id] = fetched_contact
# Prepare to fetch next batch.
# Indexes are 1-based, so add 1 to max_index.
if 'openSearch$totalResults' in response_dict:
max_index = int(
response_dict['openSearch$totalResults']['$t']) + 1
else:
max_index = FetchContactsOperation._MAX_GOOGLE_CONTACTS + 1
next_index = int(
response_dict['openSearch$startIndex']['$t']) + len(
response_dict.get('entry', []))
count = min(max_index - next_index,
FetchContactsOperation._MAX_FETCH_COUNT)
if len(
self._fetched_contacts
) < FetchContactsOperation._MAX_GOOGLE_CONTACTS and count > 0:
start_index = next_index
retries = 0
continue
else:
raise gen.Return()