def _get_google_client(self, retry_conn_errors=True): """Return the Google API client.""" # TODO(emfree) figure out a better strategy for refreshing OAuth # credentials as needed with session_scope() as db_session: try: account = db_session.query(GmailAccount).get(self.account_id) access_token, auth_creds_id = g_token_manager.get_token_and_auth_creds_id_for_contacts(account) auth_creds = db_session.query(GmailAuthCredentials).get(auth_creds_id) two_legged_oauth_token = gdata.gauth.OAuth2Token( client_id=auth_creds.client_id, client_secret=auth_creds.client_secret, scope=auth_creds.scopes, # FIXME: string not list? user_agent=SOURCE_APP_NAME, access_token=access_token, refresh_token=auth_creds.refresh_token, ) google_client = gdata.contacts.client.ContactsClient(source=SOURCE_APP_NAME) google_client.auth_token = two_legged_oauth_token return google_client except (gdata.client.BadAuthentication, gdata.client.Unauthorized, OAuthError): if not retry_conn_errors: # end of the line raise ValidationError # If there are no valid refresh_tokens, will raise an # OAuthError, stopping the sync g_token_manager.get_token_for_contacts(account, force_refresh=True) return self._google_client(retry_conn_errors=False) except ConnectionError: self.log.error("Connection error") raise
def get_items(self, sync_from_dt=None, max_results=100000): """ Fetches and parses fresh contact data. Parameters ---------- sync_from_dt: datetime, optional If given, fetch contacts that have been updated since this time. Otherwise fetch all contacts max_results: int, optional The maximum number of contact entries to fetch. Yields ------ ..models.tables.base.Contact The contacts that have been updated since the last account sync. Raises ------ ValidationError If no data could be fetched because of invalid credentials or insufficient permissions, respectively. """ query = gdata.contacts.client.ContactsQuery() # TODO(emfree): Implement batch fetching # Note: The Google contacts API will only return 25 results if # query.max_results is not explicitly set, so have to set it to a large # number by default. query.max_results = max_results if sync_from_dt: query.updated_min = datetime.isoformat(sync_from_dt) + 'Z' query.showdeleted = True while True: try: google_client = self._get_google_client() results = google_client.GetContacts(q=query).entry return [ self._parse_contact_result(result) for result in results ] except gdata.client.RequestError as e: if e.status == 503: self.log.info('Ran into Google bot detection. Sleeping.', message=e) gevent.sleep(5 * 60 + random.randrange(0, 60)) else: self.log.info('contact sync request failure; retrying', message=e) gevent.sleep(30 + random.randrange(0, 60)) except gdata.client.Unauthorized: self.log.warning( 'Invalid access token; refreshing and retrying') # Raises an OAuth error if no valid token exists with session_scope(self.namespace_id) as db_session: account = db_session.query(GmailAccount).get( self.account_id) g_token_manager.get_token_for_contacts(account, force_refresh=True)
def get_items(self, sync_from_dt=None, max_results=100000): """ Fetches and parses fresh contact data. Parameters ---------- sync_from_dt: datetime, optional If given, fetch contacts that have been updated since this time. Otherwise fetch all contacts max_results: int, optional The maximum number of contact entries to fetch. Yields ------ ..models.tables.base.Contact The contacts that have been updated since the last account sync. Raises ------ ValidationError If no data could be fetched because of invalid credentials or insufficient permissions, respectively. """ query = gdata.contacts.client.ContactsQuery() # TODO(emfree): Implement batch fetching # Note: The Google contacts API will only return 25 results if # query.max_results is not explicitly set, so have to set it to a large # number by default. query.max_results = max_results if sync_from_dt: query.updated_min = datetime.isoformat(sync_from_dt) + 'Z' query.showdeleted = True while True: try: google_client = self._get_google_client() results = google_client.GetContacts(q=query).entry return [self._parse_contact_result(result) for result in results] except gdata.client.RequestError as e: if e.status == 503: self.log.info('Ran into Google bot detection. Sleeping.', message=e) gevent.sleep(5 * 60 + random.randrange(0, 60)) else: self.log.info('contact sync request failure; retrying', message=e) gevent.sleep(30 + random.randrange(0, 60)) except gdata.client.Unauthorized: self.log.warning( 'Invalid access token; refreshing and retrying') # Raises an OAuth error if no valid token exists with session_scope(self.namespace_id) as db_session: account = db_session.query(GmailAccount).get( self.account_id) g_token_manager.get_token_for_contacts( account, force_refresh=True)
def get_items(self, sync_from_dt=None, max_results=100000): """ Fetches and parses fresh contact data. Parameters ---------- sync_from_dt: datetime, optional If given, fetch contacts that have been updated since this time. Otherwise fetch all contacts max_results: int, optional The maximum number of contact entries to fetch. Yields ------ ..models.tables.base.Contact The contacts that have been updated since the last account sync. Raises ------ ValidationError, PermissionsError If no data could be fetched because of invalid credentials or insufficient permissions, respectively. """ query = gdata.contacts.client.ContactsQuery() # TODO(emfree): Implement batch fetching # Note: The Google contacts API will only return 25 results if # query.max_results is not explicitly set, so have to set it to a large # number by default. query.max_results = max_results if sync_from_dt: query.updated_min = datetime.isoformat(sync_from_dt) + 'Z' query.showdeleted = True google_client = self._get_google_client() try: results = google_client.GetContacts(q=query).entry return [self._parse_contact_result(result) for result in results] except gdata.client.RequestError as e: # This is nearly always because we authed with Google OAuth # credentials for which the contacts API is not enabled. self.log.info('contact sync request failure', message=e) raise PermissionsError('contact sync request failure') except gdata.client.Unauthorized: self.log.warning('Invalid access token; refreshing and retrying') # Raises an OAuth error if no valid token exists with session_scope() as db_session: account = db_session.query(GmailAccount).get(self.account_id) g_token_manager.get_token_for_contacts(account, force_refresh=True) # Retry - there must be some valid credentials left return self.get_items(self, sync_from_dt=sync_from_dt, max_results=max_results)
def get_items(self, sync_from_dt=None, max_results=100000): """ Fetches and parses fresh contact data. Parameters ---------- sync_from_dt: datetime, optional If given, fetch contacts that have been updated since this time. Otherwise fetch all contacts max_results: int, optional The maximum number of contact entries to fetch. Yields ------ ..models.tables.base.Contact The contacts that have been updated since the last account sync. Raises ------ ValidationError, PermissionsError If no data could be fetched because of invalid credentials or insufficient permissions, respectively. """ query = gdata.contacts.client.ContactsQuery() # TODO(emfree): Implement batch fetching # Note: The Google contacts API will only return 25 results if # query.max_results is not explicitly set, so have to set it to a large # number by default. query.max_results = max_results if sync_from_dt: query.updated_min = datetime.isoformat(sync_from_dt) + 'Z' query.showdeleted = True google_client = self._get_google_client() try: results = google_client.GetContacts(q=query).entry return [self._parse_contact_result(result) for result in results] except gdata.client.RequestError as e: # This is nearly always because we authed with Google OAuth # credentials for which the contacts API is not enabled. self.log.info('contact sync request failure', message=e) raise PermissionsError('contact sync request failure') except gdata.client.Unauthorized: self.log.warning( 'Invalid access token; refreshing and retrying') # Raises an OAuth error if no valid token exists with session_scope() as db_session: account = db_session.query(GmailAccount).get(self.account_id) g_token_manager.get_token_for_contacts( account, force_refresh=True) # Retry - there must be some valid credentials left return self.get_items(self, sync_from_dt=sync_from_dt, max_results=max_results)
def _get_google_client(self, retry_conn_errors=True): """Return the Google API client.""" # TODO(emfree) figure out a better strategy for refreshing OAuth # credentials as needed with session_scope(self.namespace_id) as db_session: try: account = db_session.query(GmailAccount).get(self.account_id) access_token, auth_creds_id = \ g_token_manager.get_token_and_auth_creds_id_for_contacts( account) auth_creds = db_session.query(GmailAuthCredentials) \ .get(auth_creds_id) two_legged_oauth_token = gdata.gauth.OAuth2Token( client_id=auth_creds.client_id, client_secret=auth_creds.client_secret, scope=auth_creds.scopes, # FIXME: string not list? user_agent=SOURCE_APP_NAME, access_token=access_token, refresh_token=auth_creds.refresh_token) google_client = gdata.contacts.client.ContactsClient( source=SOURCE_APP_NAME) google_client.auth_token = two_legged_oauth_token return google_client except (gdata.client.BadAuthentication, gdata.client.Unauthorized, OAuthError): if not retry_conn_errors: # end of the line raise ValidationError # If there are no valid refresh_tokens, will raise an # OAuthError, stopping the sync g_token_manager.get_token_for_contacts(account, force_refresh=True) return self._google_client(retry_conn_errors=False) except ConnectionError: self.log.error('Connection error') raise