def test_auth_with_access_token(self):
"""
Tests the authentication routine when a cached auth token is provided.
"""
service = "gmail"
discovery = "gmail_discovery.json"
version = "v1"
email = "*****@*****.**"
access_token = "ya29.EAHi9TCEHi5Jaak_RV8A1KqCFg2G3soR5Wc0I" \
"j0dnRo56rLaPSmo4fgx1nkxed0OYBFFIuV_GlGH4A"
http = Mock()
credentials, http, service = authentication.authenticate(
service,
discovery,
version,
sub=email,
access_token=access_token,
http=http)
self.assertIsInstance(credentials, AccessTokenCredentials)
self.assertIsInstance(service, Resource)
self.assertEqual(credentials.access_token, access_token)
self.assertEqual(http, service._http)
def test_auth_with_access_token(self):
"""
Tests the authentication routine when a cached auth token is provided.
"""
service = "gmail"
discovery = "gmail_discovery.json"
version = "v1"
email = "*****@*****.**"
access_token = "ya29.EAHi9TCEHi5Jaak_RV8A1KqCFg2G3soR5Wc0I" \
"j0dnRo56rLaPSmo4fgx1nkxed0OYBFFIuV_GlGH4A"
http = Mock()
credentials, http, service = authentication.authenticate(
service,
discovery,
version,
sub=email,
access_token=access_token,
http=http)
self.assertIsInstance(credentials, AccessTokenCredentials)
self.assertIsInstance(service, Resource)
self.assertEqual(credentials.access_token, access_token)
self.assertEqual(http, service._http)
def update_user_list_v1(self, domain, admin_email, http=None):
"""
Get a list of users for a domain. An admin_email is required to act on the behalf of.
:param domain: The domain to retrieve users for.
:param admin_email: An admin on the Google Apps domain to act on behalf of.
:param http: Optional HTTP object to use (used in tests).
:return: A list of all users for the account.
"""
if not http: http=self.http
self.email = admin_email
self.credentials, self.http, self.service = authenticate(service="admin", discovery="admin_sdk_discovery.json",
version="directory_v1",
sub=admin_email,
access_token=self._get_auth_token(),
http=http)
all_users = []
page_token = None
params = {'domain': domain, 'fields': "users(id,name,primaryEmail,isAdmin)"}
# [{'isAdmin': True, 'name': {'familyName': 'Labs', 'givenName': 'Consus', 'fullName': 'Consus Labs'}, 'primaryEmail': '*****@*****.**', 'id': '104606403928391733102'}, {'isAdmin': False, 'name': {'familyName': 'Support', 'givenName': 'Consus Labs', 'fullName': 'Consus Labs Support'}, 'primaryEmail': '*****@*****.**', 'id': '117148913238924156783'}]
while True:
try:
if page_token:
params['pageToken'] = page_token
current_page = self.service.users().list(**params).execute(http=http)
all_users.extend(current_page['users'])
page_token = current_page.get('nextPageToken')
if not page_token:
break
except errors.HttpError as e:
if hasattr(e, 'content'):
content = e.content.decode('ascii')
try:
error = json.loads(content).get('error', None)
if error:
logging.exception("Error %d: %s" % (error.get('code'), error.get('message')),
extra={"email": self.email})
else:
logging.exception("An exception was raised while attempting to list domain users, "
"but no error information was returned by Google.",
extra={"email": self.email})
except ValueError:
logging.error("An exception was raised while attempting to list domain users and "
"information was returned by Google, but the JSON error "
"payload could not be loaded.",
extra={"email": self.email})
else:
logging.exception("Exception while attempting to list domain users via Directory API.",
extra={"email": self.email})
return
return all_users
def test_auth_with_no_access_token(self):
"""
Tests the authentication routine when no auth token is used.
"""
service = "gmail"
discovery = "gmail_discovery.json"
version = "v1"
email = "*****@*****.**"
access_token = None
http = Mock()
credentials, http, service = authentication.authenticate(
service,
discovery,
version,
sub=email,
access_token=access_token,
http=http)
self.assertIsInstance(credentials, SignedJwtAssertionCredentials)
self.assertIsInstance(service, Resource)
self.assertEqual(credentials.access_token, None)
self.assertEqual(http, service._http)
def test_auth_with_no_access_token(self):
"""
Tests the authentication routine when no auth token is used.
"""
service = "gmail"
discovery = "gmail_discovery.json"
version = "v1"
email = "*****@*****.**"
access_token = None
http = Mock()
credentials, http, service = authentication.authenticate(
service,
discovery,
version,
sub=email,
access_token=access_token,
http=http)
self.assertIsInstance(credentials, SignedJwtAssertionCredentials)
self.assertIsInstance(service, Resource)
self.assertEqual(credentials.access_token, None)
self.assertEqual(http, service._http)
def update_user_list_v1(self, domain, admin_email, http=None):
"""
Get a list of users for a domain. An admin_email is required to act on the behalf of.
:param domain: The domain to retrieve users for.
:param admin_email: An admin on the Google Apps domain to act on behalf of.
:param http: Optional HTTP object to use (used in tests).
:return: A list of all users for the account.
"""
if not http:
http = self.http
self.email = admin_email
self.credentials, self.http, self.service = authenticate(
service="admin",
discovery="admin_sdk_discovery.json",
version="directory_v1",
sub=admin_email,
access_token=self._get_auth_token(),
http=http,
)
all_users = []
page_token = None
params = {"domain": domain, "fields": "users(id,name,primaryEmail,isAdmin)"}
# [{'isAdmin': True, 'name': {'familyName': 'Labs', 'givenName': 'Consus', 'fullName': 'Consus Labs'}, 'primaryEmail': '*****@*****.**', 'id': '104606403928391733102'}, {'isAdmin': False, 'name': {'familyName': 'Support', 'givenName': 'Consus Labs', 'fullName': 'Consus Labs Support'}, 'primaryEmail': '*****@*****.**', 'id': '117148913238924156783'}]
while True:
try:
if page_token:
params["pageToken"] = page_token
current_page = self.service.users().list(**params).execute(http=http)
all_users.extend(current_page["users"])
page_token = current_page.get("nextPageToken")
if not page_token:
break
except errors.HttpError as e:
if hasattr(e, "content"):
content = e.content.decode("ascii")
try:
error = json.loads(content).get("error", None)
if error:
logging.exception(
"Error %d: %s" % (error.get("code"), error.get("message")), extra={"email": self.email}
)
else:
logging.exception(
"An exception was raised while attempting to list domain users, "
"but no error information was returned by Google.",
extra={"email": self.email},
)
except ValueError:
logging.error(
"An exception was raised while attempting to list domain users and "
"information was returned by Google, but the JSON error "
"payload could not be loaded.",
extra={"email": self.email},
)
else:
logging.exception(
"Exception while attempting to list domain users via Directory API.",
extra={"email": self.email},
)
return
return all_users
def check_account_v1(self, email, new_history_id, queue_time=None, http=None):
"""
Check a user's email account
:param email: The user's email account.
:param new_history_id: The new history ID for this account, as given by the Gmail Push message.
:param queue_time: The max amount of time the message can be on the queue before getting purged.
:param http: Optional HTTP object to use (used in tests).
"""
queue_timeout = self._check_for_queue_timeout(queue_time)
if queue_timeout:
return
self.email = email
self.domain = email.split("@")[1]
# Do a quick sanity check on the history ID value.
# if last_history_id < 1:
# logging.critical("History ID failed sanity check, cannot continue!",
# extra={'email': self.email,
# 'history_id': last_history_id})
# return
# Service object is returned from authentication request to Google.
self.credentials, self.http, self.service = authenticate(
service="gmail",
discovery="gmail_discovery.json",
version="v1",
sub=self.email,
access_token=self._get_auth_token(),
http=http,
)
# Get the Beaker EmailMeta object for this user.
email_meta = self.beaker_client.get_email_by_address(email)
if not email_meta:
logging.error(
"Could not associate the e-mail address to a " "Beaker user, cannot continue!",
extra={"email": self.email},
)
return
# Get some data from the EmailMeta for later use.
self.email_id = email_meta["id"]
last_history_id = email_meta["history_id"]
email_whitelisted = email_meta["whitelisted"]
email_blacklisted = email_meta["blacklisted"]
# Get the Beaker Domain object.
domain = self.beaker_client.get_domain_by_domain_name(self.domain)
if not domain:
logging.error(
"Could not associate the e-mail address to a " "Beaker domain, cannot continue!",
extra={"email": self.email, "domain": self.domain},
)
return
# If True, whitelist mode active; else blacklist mode
domain_whitelist_enabled = domain["whitelisted"]
# Time to determine whether or not we should actually check this account.
proceed = False
if domain_whitelist_enabled:
proceed = email_whitelisted
else:
proceed = email_blacklisted
if not proceed:
logging.info(
"Processing disabled for this address; will not continue!",
extra={
"email": self.email,
"domain_whitelist_enabled": domain_whitelist_enabled,
"email_whitelisted": email_whitelisted,
"email_blacklisted": email_blacklisted,
},
)
return
# Get the last value of the historyId, used to get changes.
last_history_id = email_meta["history_id"]
# If the last history ID is the same as was in the PubSub, ignore it.
if last_history_id >= new_history_id:
return
# Go get the history of this e-mail and return a list of changes.
try:
changes = self._get_history(last_history_id)
# This is our first attempt at using the access token, so make sure it works.
except AccessTokenCredentialsError:
logging.error("Access token not accepted. Clearing token from cache.", extra={"email": self.email})
self._clear_auth_token()
return
except AttributeError:
# Google returned a 404, meaning that the account's specified history ID did not exist.
# This is common, and happens if we ask for a history ID that is too old.
# Instead, we'll retrieve the most recent one and update Beaker with the info.
# logging.warning("History ID %d not found for user, getting current history." %
# last_history_id,
# extra={'email': self.email})
# new_history_id = self._get_history_id(last_history_id=last_history_id)
logging.warning(
"Specified history ID not found for user, setting to the value " "provided in the PubSub message.",
extra={"email": self.email, "last_history_id": last_history_id},
)
# Update Beaker
beaker_email = {"history_id": str(new_history_id)}
self.beaker_client.update_email(self.email_id, beaker_email)
return
# There will be a number of duplicate message IDs that we'll receive.
# We must find all message IDs, and then dedup them.
# TODO probably no longer needed with 'messagesAdded' feature. Strip this out.
updated_message_ids = []
for change in changes:
if not "messagesAdded" in change:
continue
for messageAdded in change["messagesAdded"]:
message = messageAdded["message"]
message_id = message["id"]
if not message_id in updated_message_ids:
updated_message_ids.append(message_id)
# Each of the message IDs need to be processed. We'll submit them in a bulk request.
batch = BatchHttpRequest()
items_in_batch = 0
for updated_message_id in updated_message_ids:
# Check Redis to see if this message ID is marked
# as being processed.
lock = self._get_message_lock(updated_message_id)
if lock:
logging.info(
"Message has already been locked for processing.",
extra={"email": self.email, "service_message_id": updated_message_id},
)
continue
else:
self._save_message_lock(updated_message_id)
# Check with Beaker to see if this is a message we've already processed.
# This is an expensive operation, so we cache heavily on the Beaker side (1hr).
# The above operation to check with Redis first should reduce/nearly eliminate
# the use of this check.
service_message = self.beaker_client.get_message_by_service_id(updated_message_id, cache_maxage=3600)
if service_message:
logging.info(
"Message has already been processed.",
extra={"email": self.email, "service_message_id": updated_message_id},
)
continue
logging.info(
"Preparing to process message.", extra={"email": self.email, "service_message_id": updated_message_id}
)
batch.add(
self.service.users().messages().get(userId=self.email, id=updated_message_id, format="raw"),
request_id=updated_message_id,
callback=self._process_message,
)
items_in_batch += 1
# new_history_id = last_history_id
new_history_id_from_profile = new_history_id
if items_in_batch > 0:
# The bulk query for the changed messages have been staged, now run them.
self._execute_batch(batch)
new_history_id_from_profile = self._get_history_id(last_history_id=last_history_id)
# After executing the batch, check and see if any of the runs called for an abort.
if self.abort_run:
return
# Once our run is finished, check and see if we
# had to initiate a new auth token and, if so,
# cache it. See function declaration for details.
self._save_auth_token()
logging.info(
"The last history ID was %s. The history ID in the PubSub was %s. "
"Next time, start at history ID %s (from profile)"
% (last_history_id, str(new_history_id), str(new_history_id_from_profile)),
extra={
"email": self.email,
"old_history_id": last_history_id,
"new_history_id_pubsub": new_history_id,
"new_history_id_profile": new_history_id_from_profile,
"processed_message_count": items_in_batch,
},
)
# If the history ID has changed, inform Beaker.
if last_history_id != new_history_id_from_profile:
beaker_email = {"history_id": str(new_history_id_from_profile)}
self.beaker_client.update_email(self.email_id, beaker_email)
def check_account_v1(self, email, new_history_id, queue_time=None, http=None):
"""
Check a user's email account
:param email: The user's email account.
:param new_history_id: The new history ID for this account, as given by the Gmail Push message.
:param queue_time: The max amount of time the message can be on the queue before getting purged.
:param http: Optional HTTP object to use (used in tests).
"""
queue_timeout = self._check_for_queue_timeout(queue_time)
if queue_timeout:
return
self.email = email
self.domain = email.split("@")[1]
# Do a quick sanity check on the history ID value.
#if last_history_id < 1:
# logging.critical("History ID failed sanity check, cannot continue!",
# extra={'email': self.email,
# 'history_id': last_history_id})
# return
# Service object is returned from authentication request to Google.
self.credentials, self.http, self.service = authenticate(service="gmail", discovery="gmail_discovery.json",
version="v1",
sub=self.email,
access_token=self._get_auth_token(),
http=http)
# Get the Beaker EmailMeta object for this user.
email_meta = self.beaker_client.get_email_by_address(email)
if not email_meta:
logging.error("Could not associate the e-mail address to a " \
"Beaker user, cannot continue!",
extra={'email': self.email})
return
# Get some data from the EmailMeta for later use.
self.email_id = email_meta['id']
last_history_id = email_meta['history_id']
email_whitelisted = email_meta['whitelisted']
email_blacklisted = email_meta['blacklisted']
# Get the Beaker Domain object.
domain = self.beaker_client.get_domain_by_domain_name(self.domain)
if not domain:
logging.error("Could not associate the e-mail address to a " \
"Beaker domain, cannot continue!",
extra={'email': self.email,
'domain': self.domain})
return
# If True, whitelist mode active; else blacklist mode
domain_whitelist_enabled = domain['whitelisted']
# Time to determine whether or not we should actually check this account.
proceed = False
if domain_whitelist_enabled:
proceed = email_whitelisted
else:
proceed = email_blacklisted
if not proceed:
logging.info("Processing disabled for this address; will not continue!",
extra={'email': self.email,
'domain_whitelist_enabled': domain_whitelist_enabled,
'email_whitelisted': email_whitelisted,
'email_blacklisted': email_blacklisted})
return
# Get the last value of the historyId, used to get changes.
last_history_id = email_meta['history_id']
# If the last history ID is the same as was in the PubSub, ignore it.
if last_history_id >= new_history_id:
return
# Go get the history of this e-mail and return a list of changes.
try:
changes = self._get_history(last_history_id)
# This is our first attempt at using the access token, so make sure it works.
except AccessTokenCredentialsError:
logging.error("Access token not accepted. Clearing token from cache.",
extra={'email': self.email})
self._clear_auth_token()
return
except AttributeError:
# Google returned a 404, meaning that the account's specified history ID did not exist.
# This is common, and happens if we ask for a history ID that is too old.
# Instead, we'll retrieve the most recent one and update Beaker with the info.
#logging.warning("History ID %d not found for user, getting current history." %
# last_history_id,
# extra={'email': self.email})
#new_history_id = self._get_history_id(last_history_id=last_history_id)
logging.warning("Specified history ID not found for user, setting to the value "
"provided in the PubSub message.",
extra={'email': self.email,
'last_history_id': last_history_id})
# Update Beaker
beaker_email = {
"history_id": str(new_history_id),
}
self.beaker_client.update_email(self.email_id, beaker_email)
return
# There will be a number of duplicate message IDs that we'll receive.
# We must find all message IDs, and then dedup them.
# TODO probably no longer needed with 'messagesAdded' feature. Strip this out.
updated_message_ids = []
for change in changes:
if not 'messagesAdded' in change:
continue
for messageAdded in change['messagesAdded']:
message = messageAdded['message']
message_id = message['id']
if not message_id in updated_message_ids:
updated_message_ids.append(message_id)
# Each of the message IDs need to be processed. We'll submit them in a bulk request.
batch = BatchHttpRequest()
items_in_batch = 0
for updated_message_id in updated_message_ids:
# Check Redis to see if this message ID is marked
# as being processed.
lock = self._get_message_lock(updated_message_id)
if lock:
logging.info("Message has already been locked for processing.",
extra={"email": self.email,
"service_message_id": updated_message_id})
continue
else:
self._save_message_lock(updated_message_id)
# Check with Beaker to see if this is a message we've already processed.
# This is an expensive operation, so we cache heavily on the Beaker side (1hr).
# The above operation to check with Redis first should reduce/nearly eliminate
# the use of this check.
service_message = self.beaker_client.get_message_by_service_id(updated_message_id, cache_maxage=3600)
if service_message:
logging.info("Message has already been processed.",
extra={"email": self.email,
"service_message_id": updated_message_id})
continue
logging.info("Preparing to process message.",
extra={"email": self.email,
"service_message_id": updated_message_id})
batch.add(self.service.users().messages().get(userId=self.email, id=updated_message_id, format='raw'),
request_id=updated_message_id, callback=self._process_message)
items_in_batch += 1
#new_history_id = last_history_id
new_history_id_from_profile = new_history_id
if items_in_batch > 0:
# The bulk query for the changed messages have been staged, now run them.
self._execute_batch(batch)
new_history_id_from_profile = self._get_history_id(last_history_id=last_history_id)
# After executing the batch, check and see if any of the runs called for an abort.
if self.abort_run:
return
# Once our run is finished, check and see if we
# had to initiate a new auth token and, if so,
# cache it. See function declaration for details.
self._save_auth_token()
logging.info("The last history ID was %s. The history ID in the PubSub was %s. "
"Next time, start at history ID %s (from profile)" % (last_history_id, str(new_history_id), str(new_history_id_from_profile)),
extra={'email': self.email,
"old_history_id": last_history_id,
"new_history_id_pubsub": new_history_id,
"new_history_id_profile": new_history_id_from_profile,
"processed_message_count": items_in_batch})
# If the history ID has changed, inform Beaker.
if last_history_id != new_history_id_from_profile:
beaker_email = {
"history_id": str(new_history_id_from_profile),
}
self.beaker_client.update_email(self.email_id, beaker_email)
