def test_merge(config):
"""Test the basic logic of the merge() function."""
base = Contact(name='Original Name',
email_address='*****@*****.**')
remote = Contact(name='New Name',
email_address='*****@*****.**')
dest = Contact(name='Original Name',
email_address='*****@*****.**')
dest.merge_from(base, remote)
assert dest.name == 'New Name'
assert dest.email_address == '*****@*****.**'
def test_merge(config, contact_sync):
"""Test the basic logic of the merge() function."""
base = Contact(name='Original Name',
email_address='*****@*****.**')
remote = Contact(name='New Name',
email_address='*****@*****.**')
dest = Contact(name='Original Name',
email_address='*****@*****.**')
contact_sync.merge(base, remote, dest)
assert dest.name == 'New Name'
assert dest.email_address == '*****@*****.**'
def test_merge_conflict(config, contact_sync):
"""Test that merge() raises an error on conflict."""
base = Contact(name='Original Name',
email_address='*****@*****.**')
remote = Contact(name='New Name',
email_address='*****@*****.**')
dest = Contact(name='Some Other Name',
email_address='*****@*****.**')
with pytest.raises(MergeError):
contact_sync.merge(base, remote, dest)
# Check no update in case of conflict
assert dest.name == 'Some Other Name'
assert dest.email_address == '*****@*****.**'
def test_merge_conflict(config):
"""Test that merge() raises an error on conflict."""
base = Contact(name='Original Name',
email_address='*****@*****.**')
remote = Contact(name='New Name',
email_address='*****@*****.**')
dest = Contact(name='Some Other Name',
email_address='*****@*****.**')
with pytest.raises(MergeError):
dest.merge_from(base, remote)
# Check no update in case of conflict
assert dest.name == 'Some Other Name'
assert dest.email_address == '*****@*****.**'
def _vCard_raw_to_contact(self, cardstring):
card = vcard_from_string(cardstring)
def _x(key): # Ugly parsing helper for ugly formats
if key in card:
try:
return card[key][0][0]
except IndexError:
pass
# Skip contact groups for now
if _x('X-ADDRESSBOOKSERVER-KIND') == 'group':
return None
uid = _x('UID')
name = _x('FN')
email_address = _x('EMAIL')
# TODO add these later
# street_address = _x('ADR')
# phone_number = _x('TEL')
# organization = _x('ORG')
return Contact(namespace_id=self.namespace_id,
provider_name=self.PROVIDER_NAME,
uid=uid,
name=name,
email_address=email_address,
raw_data=cardstring)
def _get_contact_map(db_session, namespace_id, all_addresses):
"""
Retrieves or creates contacts for the given address pairs, returning a dict
with the canonicalized emails mapped to Contact objects.
"""
canonicalized_addresses = [canonicalize(addr) for _, addr in all_addresses]
if not canonicalized_addresses:
return {}
existing_contacts = (
db_session.query(Contact)
.filter(
Contact._canonicalized_address.in_(canonicalized_addresses),
Contact.namespace_id == namespace_id,
)
.all()
)
contact_map = {c._canonicalized_address: c for c in existing_contacts}
for name, email_address in all_addresses:
canonicalized_address = canonicalize(email_address)
if canonicalized_address not in contact_map:
new_contact = Contact(
name=name,
email_address=email_address,
namespace_id=namespace_id,
provider_name=INBOX_PROVIDER_NAME,
uid=uuid.uuid4().hex,
)
contact_map[canonicalized_address] = new_contact
return contact_map
def _parse_contact_result(self, google_contact):
"""Constructs a Contact object from a Google contact entry.
Parameters
----------
google_contact: gdata.contacts.entry.ContactEntry
The Google contact entry to parse.
Returns
-------
..models.tables.base.Contact
A corresponding Nylas Contact instance.
Raises
------
AttributeError
If the contact data could not be parsed correctly.
"""
email_addresses = [
email for email in google_contact.email if email.primary
]
if email_addresses and len(email_addresses) > 1:
self.log.error(
"Should not have more than one email per entry!",
num_email=len(email_addresses),
)
try:
# The id.text field of a ContactEntry object takes the form
# 'http://www.google.com/m8/feeds/contacts/<useremail>/base/<uid>'.
# We only want the <uid> part.
raw_google_id = google_contact.id.text
_, g_id = posixpath.split(raw_google_id)
name = (google_contact.name.full_name.text if
(google_contact.name
and google_contact.name.full_name) else None)
email_address = email_addresses[
0].address if email_addresses else None
# The entirety of the raw contact data in XML string
# representation.
raw_data = google_contact.to_string()
except AttributeError as e:
self.log.error("Something is wrong with contact",
contact=google_contact)
raise e
deleted = google_contact.deleted is not None
return Contact(
namespace_id=self.namespace_id,
uid=g_id,
name=name,
provider_name=self.PROVIDER_NAME,
email_address=email_address,
deleted=deleted,
raw_data=raw_data,
)
def create(namespace, db_session, name, email):
contact = Contact(namespace=namespace,
provider_name=INBOX_PROVIDER_NAME,
uid=uuid.uuid4().hex,
name=name,
email_address=email)
db_session.add(contact)
db_session.commit()
return contact
def contact(db, default_account):
from inbox.models import Contact
contact = Contact(namespace_id=default_account.namespace.id,
name='Ben Bitdiddle',
email_address='*****@*****.**',
uid='22')
db.session.add(contact)
db.session.commit()
return contact
def supply_contact(self, name, email_address, deleted=False):
from inbox.models import Contact
self._contacts.append(Contact(namespace_id=1,
uid=str(self._next_uid),
provider_name=self.PROVIDER_NAME,
name=name,
email_address=email_address,
deleted=deleted))
self._next_uid += 1
def update_contacts_from_message(db_session, message, namespace):
with db_session.no_autoflush:
# First create Contact objects for any email addresses that we haven't
# seen yet. We want to dedupe by canonicalized address, so this part is
# a bit finicky.
canonicalized_addresses = []
all_addresses = []
for field in (message.from_addr, message.to_addr, message.cc_addr,
message.bcc_addr, message.reply_to):
# We generally require these attributes to be non-null, but only
# set them to the default empty list at flush time. So it's better
# to be safe here.
if field is not None:
all_addresses.extend(field)
canonicalized_addresses = [
canonicalize(addr) for _, addr in all_addresses
]
existing_contacts = db_session.query(Contact).filter(
Contact._canonicalized_address.in_(canonicalized_addresses),
Contact.namespace_id == namespace.id).all()
contact_map = {c._canonicalized_address: c for c in existing_contacts}
for name, email_address in all_addresses:
canonicalized_address = canonicalize(email_address)
if canonicalized_address not in contact_map:
new_contact = Contact(name=name,
email_address=email_address,
namespace=namespace,
provider_name=INBOX_PROVIDER_NAME,
uid=uuid.uuid4().hex)
contact_map[canonicalized_address] = new_contact
# Now associate each contact to the message.
for field_name in ('from_addr', 'to_addr', 'cc_addr', 'bcc_addr',
'reply_to'):
field = getattr(message, field_name)
if field is None:
continue
for name, email_address in field:
if not valid_email(email_address):
continue
canonicalized_address = canonicalize(email_address)
contact = contact_map.get(canonicalized_address)
# Hackily address the condition that you get mail from e.g.
# "Ben Gotow (via Google Drive) <*****@*****.**"
# "Christine Spang (via Google Drive) <*****@*****.**"
# and so on: rather than creating many contacts with
# varying name, null out the name for the existing contact.
if contact.name != name and 'noreply' in canonicalized_address:
contact.name = None
message.contacts.append(
MessageContactAssociation(contact=contact,
field=field_name))
def add_fake_contact(db_session,
namespace_id,
name='Ben Bitdiddle',
email_address='*****@*****.**',
uid='22'):
from inbox.models import Contact
contact = Contact(namespace_id=namespace_id,
name=name,
email_address=email_address,
uid=uid)
db_session.add(contact)
db_session.commit()
return contact
def update_contacts_from_message(db_session, message, namespace):
with db_session.no_autoflush:
# First create Contact objects for any email addresses that we haven't
# seen yet. We want to dedupe by canonicalized address, so this part is
# a bit finicky.
canonicalized_addresses = []
all_addresses = []
for field in (message.from_addr, message.to_addr, message.cc_addr,
message.bcc_addr, message.reply_to):
# We generally require these attributes to be non-null, but only
# set them to the default empty list at flush time. So it's better
# to be safe here.
if field is not None:
all_addresses.extend(field)
canonicalized_addresses = [
canonicalize(addr) for _, addr in all_addresses
]
existing_contacts = db_session.query(Contact).filter(
Contact._canonicalized_address.in_(canonicalized_addresses),
Contact.namespace_id == namespace.id).all()
contact_map = {c._canonicalized_address: c for c in existing_contacts}
for name, email_address in all_addresses:
canonicalized_address = canonicalize(email_address)
if canonicalized_address not in contact_map:
new_contact = Contact(name=name,
email_address=email_address,
namespace=namespace,
provider_name=INBOX_PROVIDER_NAME,
uid=uuid.uuid4().hex)
contact_map[canonicalized_address] = new_contact
# Now associate each contact to the message.
for field_name in ('from_addr', 'to_addr', 'cc_addr', 'bcc_addr',
'reply_to'):
field = getattr(message, field_name)
if field is None:
continue
for name, email_address in field:
canonicalized_address = canonicalize(email_address)
contact = contact_map.get(canonicalized_address)
message.contacts.append(
MessageContactAssociation(contact=contact,
field=field_name))
def update_contacts_from_message(db_session, message, account_id):
with db_session.no_autoflush:
for field in ('to_addr', 'from_addr', 'cc_addr', 'bcc_addr'):
if getattr(message, field) is None:
continue
items = set(getattr(message, field))
for name, email_address in items:
contact = db_session.query(Contact).filter(
Contact.email_address == email_address).first()
if contact is None:
contact = Contact(name=name,
email_address=email_address,
account_id=account_id,
source='local',
provider_name='inbox',
uid=uuid.uuid4().hex)
message.contacts.append(
MessageContactAssociation(contact=contact, field=field))
class GoogleContactsProvider(BaseSyncProvider):
"""A utility class to fetch and parse Google contact data for the specified
account using the Google Contacts API.
Parameters
----------
db_session: sqlalchemy.orm.session.Session
Database session.
account: inbox.models.gmail.GmailAccount
The user account for which to fetch contact data.
Attributes
----------
google_client: gdata.contacts.client.ContactsClient
Google API client to do the actual data fetching.
log: logging.Logger
Logging handler.
"""
PROVIDER_NAME = 'google'
def __init__(self, account_id):
self.account_id = account_id
self.log = logger.new(account_id=account_id, component='contacts sync',
provider=self.PROVIDER_NAME)
def _get_google_client(self):
"""Return the Google API client."""
# TODO(emfree) figure out a better strategy for refreshing OAuth
# credentials as needed
with session_scope() as db_session:
try:
account = db_session.query(GmailAccount).get(self.account_id)
client_id = account.client_id or OAUTH_CLIENT_ID
client_secret = (account.client_secret or
OAUTH_CLIENT_SECRET)
two_legged_oauth_token = gdata.gauth.OAuth2Token(
client_id=client_id,
client_secret=client_secret,
scope=OAUTH_SCOPE,
user_agent=SOURCE_APP_NAME,
access_token=account.access_token,
refresh_token=account.refresh_token)
google_client = gdata.contacts.client.ContactsClient(
source=SOURCE_APP_NAME)
google_client.auth_token = two_legged_oauth_token
return google_client
except (gdata.client.BadAuthentication, OAuthError):
self.log.error('Invalid user credentials given')
account.sync_state = 'invalid'
db_session.add(account)
db_session.commit()
return None
except ConnectionError:
self.log.error('Connection error')
account.sync_state = 'connerror'
db_session.add(account)
db_session.commit()
return None
def _parse_contact_result(self, google_contact):
"""Constructs a Contact object from a Google contact entry.
Parameters
----------
google_contact: gdata.contacts.entry.ContactEntry
The Google contact entry to parse.
Returns
-------
..models.tables.base.Contact
A corresponding Inbox Contact instance.
Raises
------
AttributeError
If the contact data could not be parsed correctly.
"""
email_addresses = [email for email in google_contact.email if
email.primary]
if email_addresses and len(email_addresses) > 1:
self.log.error("Should not have more than one email per entry! {0}"
.format(email_addresses))
try:
# The id.text field of a ContactEntry object takes the form
# 'http://www.google.com/m8/feeds/contacts/<useremail>/base/<uid>'.
# We only want the <uid> part.
raw_google_id = google_contact.id.text
_, g_id = posixpath.split(raw_google_id)
name = (google_contact.name.full_name.text if (google_contact.name
and google_contact.name.full_name) else None)
email_address = (email_addresses[0].address if email_addresses else
None)
# The entirety of the raw contact data in XML string
# representation.
raw_data = google_contact.to_string()
except AttributeError, e:
self.log.error('Something is wrong with contact',
contact=google_contact)
raise e
deleted = google_contact.deleted is not None
return Contact(account_id=self.account_id, source='remote',
uid=g_id, name=name, provider_name=self.PROVIDER_NAME,
email_address=email_address, deleted=deleted,
raw_data=raw_data)
def poll(account_id, provider):
"""Query a remote contacts provider for updates and persist them to the
database.
Parameters
----------
account_id: int
ID for the account whose contacts should be queried.
db_session: sqlalchemy.orm.session.Session
Database session
provider: Interface to the remote contact data provider.
Must have a PROVIDER_NAME attribute and implement the get_contacts()
method.
"""
log = logger.new(account_id=account_id)
provider_name = provider.PROVIDER_NAME
with session_scope() as db_session:
account = db_session.query(Account).get(account_id)
change_counter = Counter()
last_sync = or_none(account.last_synced_contacts,
datetime.datetime.isoformat)
to_commit = []
for remote_contact in provider.get_contacts(last_sync):
remote_contact.account = account
assert remote_contact.uid is not None, \
'Got remote contact with null uid'
assert isinstance(remote_contact.uid, str)
matching_contacts = db_session.query(Contact).filter(
Contact.account == account,
Contact.provider_name == provider_name,
Contact.uid == remote_contact.uid)
# Snapshot of contact data from immediately after last sync:
cached_contact = matching_contacts. \
filter(Contact.source == 'remote').first()
# Contact data reflecting any local modifications since the last
# sync with the remote provider:
local_contact = matching_contacts. \
filter(Contact.source == 'local').first()
# If the remote contact was deleted, purge the corresponding
# database entries.
if remote_contact.deleted:
if cached_contact is not None:
db_session.delete(cached_contact)
change_counter['deleted'] += 1
if local_contact is not None:
db_session.delete(local_contact)
continue
# Otherwise, update the database.
if cached_contact is not None:
# The provider gave an update to a contact we already have.
if local_contact is not None:
try:
# Attempt to merge remote updates into local_contact
merge(cached_contact, remote_contact, local_contact)
# And update cached_contact to reflect both local and
# remote updates
cached_contact.copy_from(local_contact)
except MergeError:
log.error('Conflicting local and remote updates to '
'contact.',
local=local_contact, cached=cached_contact,
remote=remote_contact)
# TODO(emfree): Come up with a strategy for handling
# merge conflicts. For now, just don't update if there
# is a conflict.
continue
else:
log.warning('Contact already present as remote but not '
'local contact', cached_contact=cached_contact)
cached_contact.copy_from(remote_contact)
change_counter['updated'] += 1
else:
# This is a new contact, create both local and remote DB
# entries.
local_contact = Contact()
local_contact.copy_from(remote_contact)
local_contact.source = 'local'
to_commit.append(local_contact)
to_commit.append(remote_contact)
change_counter['added'] += 1
account.last_synced_contacts = datetime.datetime.now()
log.info('added contacts', count=change_counter['added'])
log.info('updated contacts', count=change_counter['updated'])
log.info('deleted contacts', count=change_counter['deleted'])
db_session.add_all(to_commit)
db_session.commit()
class GoogleContactsProvider(object):
"""
A utility class to fetch and parse Google contact data for the specified
account using the Google Contacts API.
Parameters
----------
db_session: sqlalchemy.orm.session.Session
Database session.
account: inbox.models.gmail.GmailAccount
The user account for which to fetch contact data.
Attributes
----------
google_client: gdata.contacts.client.ContactsClient
Google API client to do the actual data fetching.
log: logging.Logger
Logging handler.
"""
PROVIDER_NAME = 'google'
def __init__(self, account_id, namespace_id):
self.account_id = account_id
self.namespace_id = namespace_id
self.log = logger.new(account_id=account_id,
component='contacts sync',
provider=self.PROVIDER_NAME)
def _get_google_client(self, retry_conn_errors=True):
"""Return the Google API client."""
# TODO(emfree) figure out a better strategy for refreshing OAuth
# credentials as needed
with session_scope() as db_session:
try:
account = db_session.query(GmailAccount).get(self.account_id)
access_token, auth_creds_id = \
g_token_manager.get_token_and_auth_creds_id_for_contacts(
account)
auth_creds = db_session.query(GmailAuthCredentials) \
.get(auth_creds_id)
two_legged_oauth_token = gdata.gauth.OAuth2Token(
client_id=auth_creds.client_id,
client_secret=auth_creds.client_secret,
scope=auth_creds.scopes, # FIXME: string not list?
user_agent=SOURCE_APP_NAME,
access_token=access_token,
refresh_token=auth_creds.refresh_token)
google_client = gdata.contacts.client.ContactsClient(
source=SOURCE_APP_NAME)
google_client.auth_token = two_legged_oauth_token
return google_client
except (gdata.client.BadAuthentication, gdata.client.Unauthorized,
OAuthError):
if not retry_conn_errors: # end of the line
raise ValidationError
# If there are no valid refresh_tokens, will raise an
# OAuthError, stopping the sync
g_token_manager.get_token_for_contacts(account,
force_refresh=True)
return self._google_client(retry_conn_errors=False)
except ConnectionError:
self.log.error('Connection error')
raise
def _parse_contact_result(self, google_contact):
"""Constructs a Contact object from a Google contact entry.
Parameters
----------
google_contact: gdata.contacts.entry.ContactEntry
The Google contact entry to parse.
Returns
-------
..models.tables.base.Contact
A corresponding Inbox Contact instance.
Raises
------
AttributeError
If the contact data could not be parsed correctly.
"""
email_addresses = [
email for email in google_contact.email if email.primary
]
if email_addresses and len(email_addresses) > 1:
self.log.error(
"Should not have more than one email per entry! {0}".format(
email_addresses))
try:
# The id.text field of a ContactEntry object takes the form
# 'http://www.google.com/m8/feeds/contacts/<useremail>/base/<uid>'.
# We only want the <uid> part.
raw_google_id = google_contact.id.text
_, g_id = posixpath.split(raw_google_id)
name = (google_contact.name.full_name.text if
(google_contact.name
and google_contact.name.full_name) else None)
email_address = (email_addresses[0].address
if email_addresses else None)
# The entirety of the raw contact data in XML string
# representation.
raw_data = google_contact.to_string()
except AttributeError, e:
self.log.error('Something is wrong with contact',
contact=google_contact)
raise e
deleted = google_contact.deleted is not None
return Contact(namespace_id=self.namespace_id,
uid=g_id,
name=name,
provider_name=self.PROVIDER_NAME,
email_address=email_address,
deleted=deleted,
raw_data=raw_data)