def _get_google_client(self, retry_conn_errors=True):
"""Return the Google API client."""
# TODO(emfree) figure out a better strategy for refreshing OAuth
# credentials as needed
with session_scope() as db_session:
try:
account = db_session.query(GmailAccount).get(self.account_id)
access_token, auth_creds_id = g_token_manager.get_token_and_auth_creds_id_for_contacts(account)
auth_creds = db_session.query(GmailAuthCredentials).get(auth_creds_id)
two_legged_oauth_token = gdata.gauth.OAuth2Token(
client_id=auth_creds.client_id,
client_secret=auth_creds.client_secret,
scope=auth_creds.scopes, # FIXME: string not list?
user_agent=SOURCE_APP_NAME,
access_token=access_token,
refresh_token=auth_creds.refresh_token,
)
google_client = gdata.contacts.client.ContactsClient(source=SOURCE_APP_NAME)
google_client.auth_token = two_legged_oauth_token
return google_client
except (gdata.client.BadAuthentication, gdata.client.Unauthorized, OAuthError):
if not retry_conn_errors: # end of the line
raise ValidationError
# If there are no valid refresh_tokens, will raise an
# OAuthError, stopping the sync
g_token_manager.get_token_for_contacts(account, force_refresh=True)
return self._google_client(retry_conn_errors=False)
except ConnectionError:
self.log.error("Connection error")
raise
def get_items(self, sync_from_dt=None, max_results=100000):
"""
Fetches and parses fresh contact data.
Parameters
----------
sync_from_dt: datetime, optional
If given, fetch contacts that have been updated since this time.
Otherwise fetch all contacts
max_results: int, optional
The maximum number of contact entries to fetch.
Yields
------
..models.tables.base.Contact
The contacts that have been updated since the last account sync.
Raises
------
ValidationError
If no data could be fetched because of invalid credentials or
insufficient permissions, respectively.
"""
query = gdata.contacts.client.ContactsQuery()
# TODO(emfree): Implement batch fetching
# Note: The Google contacts API will only return 25 results if
# query.max_results is not explicitly set, so have to set it to a large
# number by default.
query.max_results = max_results
if sync_from_dt:
query.updated_min = datetime.isoformat(sync_from_dt) + 'Z'
query.showdeleted = True
while True:
try:
google_client = self._get_google_client()
results = google_client.GetContacts(q=query).entry
return [
self._parse_contact_result(result) for result in results
]
except gdata.client.RequestError as e:
if e.status == 503:
self.log.info('Ran into Google bot detection. Sleeping.',
message=e)
gevent.sleep(5 * 60 + random.randrange(0, 60))
else:
self.log.info('contact sync request failure; retrying',
message=e)
gevent.sleep(30 + random.randrange(0, 60))
except gdata.client.Unauthorized:
self.log.warning(
'Invalid access token; refreshing and retrying')
# Raises an OAuth error if no valid token exists
with session_scope(self.namespace_id) as db_session:
account = db_session.query(GmailAccount).get(
self.account_id)
g_token_manager.get_token_for_contacts(account,
force_refresh=True)
def get_items(self, sync_from_dt=None, max_results=100000):
"""
Fetches and parses fresh contact data.
Parameters
----------
sync_from_dt: datetime, optional
If given, fetch contacts that have been updated since this time.
Otherwise fetch all contacts
max_results: int, optional
The maximum number of contact entries to fetch.
Yields
------
..models.tables.base.Contact
The contacts that have been updated since the last account sync.
Raises
------
ValidationError
If no data could be fetched because of invalid credentials or
insufficient permissions, respectively.
"""
query = gdata.contacts.client.ContactsQuery()
# TODO(emfree): Implement batch fetching
# Note: The Google contacts API will only return 25 results if
# query.max_results is not explicitly set, so have to set it to a large
# number by default.
query.max_results = max_results
if sync_from_dt:
query.updated_min = datetime.isoformat(sync_from_dt) + 'Z'
query.showdeleted = True
while True:
try:
google_client = self._get_google_client()
results = google_client.GetContacts(q=query).entry
return [self._parse_contact_result(result) for result in
results]
except gdata.client.RequestError as e:
if e.status == 503:
self.log.info('Ran into Google bot detection. Sleeping.',
message=e)
gevent.sleep(5 * 60 + random.randrange(0, 60))
else:
self.log.info('contact sync request failure; retrying',
message=e)
gevent.sleep(30 + random.randrange(0, 60))
except gdata.client.Unauthorized:
self.log.warning(
'Invalid access token; refreshing and retrying')
# Raises an OAuth error if no valid token exists
with session_scope(self.namespace_id) as db_session:
account = db_session.query(GmailAccount).get(
self.account_id)
g_token_manager.get_token_for_contacts(
account, force_refresh=True)
def get_items(self, sync_from_dt=None, max_results=100000):
"""
Fetches and parses fresh contact data.
Parameters
----------
sync_from_dt: datetime, optional
If given, fetch contacts that have been updated since this time.
Otherwise fetch all contacts
max_results: int, optional
The maximum number of contact entries to fetch.
Yields
------
..models.tables.base.Contact
The contacts that have been updated since the last account sync.
Raises
------
ValidationError, PermissionsError
If no data could be fetched because of invalid credentials or
insufficient permissions, respectively.
"""
query = gdata.contacts.client.ContactsQuery()
# TODO(emfree): Implement batch fetching
# Note: The Google contacts API will only return 25 results if
# query.max_results is not explicitly set, so have to set it to a large
# number by default.
query.max_results = max_results
if sync_from_dt:
query.updated_min = datetime.isoformat(sync_from_dt) + 'Z'
query.showdeleted = True
google_client = self._get_google_client()
try:
results = google_client.GetContacts(q=query).entry
return [self._parse_contact_result(result) for result in results]
except gdata.client.RequestError as e:
# This is nearly always because we authed with Google OAuth
# credentials for which the contacts API is not enabled.
self.log.info('contact sync request failure', message=e)
raise PermissionsError('contact sync request failure')
except gdata.client.Unauthorized:
self.log.warning('Invalid access token; refreshing and retrying')
# Raises an OAuth error if no valid token exists
with session_scope() as db_session:
account = db_session.query(GmailAccount).get(self.account_id)
g_token_manager.get_token_for_contacts(account,
force_refresh=True)
# Retry - there must be some valid credentials left
return self.get_items(self,
sync_from_dt=sync_from_dt,
max_results=max_results)
def get_items(self, sync_from_dt=None, max_results=100000):
"""
Fetches and parses fresh contact data.
Parameters
----------
sync_from_dt: datetime, optional
If given, fetch contacts that have been updated since this time.
Otherwise fetch all contacts
max_results: int, optional
The maximum number of contact entries to fetch.
Yields
------
..models.tables.base.Contact
The contacts that have been updated since the last account sync.
Raises
------
ValidationError, PermissionsError
If no data could be fetched because of invalid credentials or
insufficient permissions, respectively.
"""
query = gdata.contacts.client.ContactsQuery()
# TODO(emfree): Implement batch fetching
# Note: The Google contacts API will only return 25 results if
# query.max_results is not explicitly set, so have to set it to a large
# number by default.
query.max_results = max_results
if sync_from_dt:
query.updated_min = datetime.isoformat(sync_from_dt) + 'Z'
query.showdeleted = True
google_client = self._get_google_client()
try:
results = google_client.GetContacts(q=query).entry
return [self._parse_contact_result(result) for result in results]
except gdata.client.RequestError as e:
# This is nearly always because we authed with Google OAuth
# credentials for which the contacts API is not enabled.
self.log.info('contact sync request failure', message=e)
raise PermissionsError('contact sync request failure')
except gdata.client.Unauthorized:
self.log.warning(
'Invalid access token; refreshing and retrying')
# Raises an OAuth error if no valid token exists
with session_scope() as db_session:
account = db_session.query(GmailAccount).get(self.account_id)
g_token_manager.get_token_for_contacts(
account, force_refresh=True)
# Retry - there must be some valid credentials left
return self.get_items(self, sync_from_dt=sync_from_dt,
max_results=max_results)
def _get_google_client(self, retry_conn_errors=True):
"""Return the Google API client."""
# TODO(emfree) figure out a better strategy for refreshing OAuth
# credentials as needed
with session_scope(self.namespace_id) as db_session:
try:
account = db_session.query(GmailAccount).get(self.account_id)
access_token, auth_creds_id = \
g_token_manager.get_token_and_auth_creds_id_for_contacts(
account)
auth_creds = db_session.query(GmailAuthCredentials) \
.get(auth_creds_id)
two_legged_oauth_token = gdata.gauth.OAuth2Token(
client_id=auth_creds.client_id,
client_secret=auth_creds.client_secret,
scope=auth_creds.scopes, # FIXME: string not list?
user_agent=SOURCE_APP_NAME,
access_token=access_token,
refresh_token=auth_creds.refresh_token)
google_client = gdata.contacts.client.ContactsClient(
source=SOURCE_APP_NAME)
google_client.auth_token = two_legged_oauth_token
return google_client
except (gdata.client.BadAuthentication, gdata.client.Unauthorized,
OAuthError):
if not retry_conn_errors: # end of the line
raise ValidationError
# If there are no valid refresh_tokens, will raise an
# OAuthError, stopping the sync
g_token_manager.get_token_for_contacts(account,
force_refresh=True)
return self._google_client(retry_conn_errors=False)
except ConnectionError:
self.log.error('Connection error')
raise
