def test_auth_with_access_token(self):
        """
        Tests the authentication routine when a cached auth token is provided.
        """
        service = "gmail"
        discovery = "gmail_discovery.json"
        version = "v1"
        email = "*****@*****.**"
        access_token = "ya29.EAHi9TCEHi5Jaak_RV8A1KqCFg2G3soR5Wc0I" \
                       "j0dnRo56rLaPSmo4fgx1nkxed0OYBFFIuV_GlGH4A"
        http = Mock()

        credentials, http, service = authentication.authenticate(
                                    service,
                                    discovery,
                                    version,
                                    sub=email,
                                    access_token=access_token,
                                    http=http)

        self.assertIsInstance(credentials, AccessTokenCredentials)
        self.assertIsInstance(service, Resource)

        self.assertEqual(credentials.access_token, access_token)
        self.assertEqual(http, service._http)
Beispiel #2
0
    def test_auth_with_access_token(self):
        """
        Tests the authentication routine when a cached auth token is provided.
        """
        service = "gmail"
        discovery = "gmail_discovery.json"
        version = "v1"
        email = "*****@*****.**"
        access_token = "ya29.EAHi9TCEHi5Jaak_RV8A1KqCFg2G3soR5Wc0I" \
                       "j0dnRo56rLaPSmo4fgx1nkxed0OYBFFIuV_GlGH4A"
        http = Mock()

        credentials, http, service = authentication.authenticate(
            service,
            discovery,
            version,
            sub=email,
            access_token=access_token,
            http=http)

        self.assertIsInstance(credentials, AccessTokenCredentials)
        self.assertIsInstance(service, Resource)

        self.assertEqual(credentials.access_token, access_token)
        self.assertEqual(http, service._http)
Beispiel #3
0
    def update_user_list_v1(self, domain, admin_email, http=None):
        """
        Get a list of users for a domain. An admin_email is required to act on the behalf of.

        :param domain: The domain to retrieve users for.
        :param admin_email: An admin on the Google Apps domain to act on behalf of.
        :param http: Optional HTTP object to use (used in tests).
        :return: A list of all users for the account.
        """
        if not http: http=self.http
        self.email = admin_email

        self.credentials, self.http, self.service = authenticate(service="admin", discovery="admin_sdk_discovery.json",
                                                                 version="directory_v1",
                                                                 sub=admin_email,
                                                                 access_token=self._get_auth_token(),
                                                                 http=http)

        all_users = []
        page_token = None
        params = {'domain': domain, 'fields': "users(id,name,primaryEmail,isAdmin)"}
        # [{'isAdmin': True, 'name': {'familyName': 'Labs', 'givenName': 'Consus', 'fullName': 'Consus Labs'}, 'primaryEmail': '*****@*****.**', 'id': '104606403928391733102'}, {'isAdmin': False, 'name': {'familyName': 'Support', 'givenName': 'Consus Labs', 'fullName': 'Consus Labs Support'}, 'primaryEmail': '*****@*****.**', 'id': '117148913238924156783'}]
        while True:
            try:
                if page_token:
                    params['pageToken'] = page_token

                current_page = self.service.users().list(**params).execute(http=http)

                all_users.extend(current_page['users'])
                page_token = current_page.get('nextPageToken')
                if not page_token:
                   break
            except errors.HttpError as e:
                if hasattr(e, 'content'):
                    content = e.content.decode('ascii')
                    try:
                        error = json.loads(content).get('error', None)
                        if error:
                            logging.exception("Error %d: %s" % (error.get('code'), error.get('message')),
                                              extra={"email": self.email})
                        else:
                            logging.exception("An exception was raised while attempting to list domain users, "
                                              "but no error information was returned by Google.",
                                              extra={"email": self.email})
                    except ValueError:
                        logging.error("An exception was raised while attempting to list domain users and "
                                      "information was returned by Google, but the JSON error "
                                      "payload could not be loaded.",
                                      extra={"email": self.email})
                else:
                    logging.exception("Exception while attempting to list domain users via Directory API.",
                                      extra={"email": self.email})
                return

        return all_users
    def test_auth_with_no_access_token(self):
        """
        Tests the authentication routine when no auth token is used.
        """
        service = "gmail"
        discovery = "gmail_discovery.json"
        version = "v1"
        email = "*****@*****.**"
        access_token = None
        http = Mock()

        credentials, http, service = authentication.authenticate(
                                    service,
                                    discovery,
                                    version,
                                    sub=email,
                                    access_token=access_token,
                                    http=http)

        self.assertIsInstance(credentials, SignedJwtAssertionCredentials)
        self.assertIsInstance(service, Resource)

        self.assertEqual(credentials.access_token, None)
        self.assertEqual(http, service._http)
Beispiel #5
0
    def test_auth_with_no_access_token(self):
        """
        Tests the authentication routine when no auth token is used.
        """
        service = "gmail"
        discovery = "gmail_discovery.json"
        version = "v1"
        email = "*****@*****.**"
        access_token = None
        http = Mock()

        credentials, http, service = authentication.authenticate(
            service,
            discovery,
            version,
            sub=email,
            access_token=access_token,
            http=http)

        self.assertIsInstance(credentials, SignedJwtAssertionCredentials)
        self.assertIsInstance(service, Resource)

        self.assertEqual(credentials.access_token, None)
        self.assertEqual(http, service._http)
Beispiel #6
0
    def update_user_list_v1(self, domain, admin_email, http=None):
        """
        Get a list of users for a domain. An admin_email is required to act on the behalf of.

        :param domain: The domain to retrieve users for.
        :param admin_email: An admin on the Google Apps domain to act on behalf of.
        :param http: Optional HTTP object to use (used in tests).
        :return: A list of all users for the account.
        """
        if not http:
            http = self.http
        self.email = admin_email

        self.credentials, self.http, self.service = authenticate(
            service="admin",
            discovery="admin_sdk_discovery.json",
            version="directory_v1",
            sub=admin_email,
            access_token=self._get_auth_token(),
            http=http,
        )

        all_users = []
        page_token = None
        params = {"domain": domain, "fields": "users(id,name,primaryEmail,isAdmin)"}
        # [{'isAdmin': True, 'name': {'familyName': 'Labs', 'givenName': 'Consus', 'fullName': 'Consus Labs'}, 'primaryEmail': '*****@*****.**', 'id': '104606403928391733102'}, {'isAdmin': False, 'name': {'familyName': 'Support', 'givenName': 'Consus Labs', 'fullName': 'Consus Labs Support'}, 'primaryEmail': '*****@*****.**', 'id': '117148913238924156783'}]
        while True:
            try:
                if page_token:
                    params["pageToken"] = page_token

                current_page = self.service.users().list(**params).execute(http=http)

                all_users.extend(current_page["users"])
                page_token = current_page.get("nextPageToken")
                if not page_token:
                    break
            except errors.HttpError as e:
                if hasattr(e, "content"):
                    content = e.content.decode("ascii")
                    try:
                        error = json.loads(content).get("error", None)
                        if error:
                            logging.exception(
                                "Error %d: %s" % (error.get("code"), error.get("message")), extra={"email": self.email}
                            )
                        else:
                            logging.exception(
                                "An exception was raised while attempting to list domain users, "
                                "but no error information was returned by Google.",
                                extra={"email": self.email},
                            )
                    except ValueError:
                        logging.error(
                            "An exception was raised while attempting to list domain users and "
                            "information was returned by Google, but the JSON error "
                            "payload could not be loaded.",
                            extra={"email": self.email},
                        )
                else:
                    logging.exception(
                        "Exception while attempting to list domain users via Directory API.",
                        extra={"email": self.email},
                    )
                return

        return all_users
Beispiel #7
0
    def check_account_v1(self, email, new_history_id, queue_time=None, http=None):
        """
        Check a user's email account

        :param email: The user's email account.
        :param new_history_id: The new history ID for this account, as given by the Gmail Push message.
        :param queue_time: The max amount of time the message can be on the queue before getting purged.
        :param http: Optional HTTP object to use (used in tests).
        """
        queue_timeout = self._check_for_queue_timeout(queue_time)
        if queue_timeout:
            return

        self.email = email
        self.domain = email.split("@")[1]

        # Do a quick sanity check on the history ID value.
        # if last_history_id < 1:
        #    logging.critical("History ID failed sanity check, cannot continue!",
        #                     extra={'email': self.email,
        #                            'history_id': last_history_id})
        #    return

        # Service object is returned from authentication request to Google.
        self.credentials, self.http, self.service = authenticate(
            service="gmail",
            discovery="gmail_discovery.json",
            version="v1",
            sub=self.email,
            access_token=self._get_auth_token(),
            http=http,
        )
        # Get the Beaker EmailMeta object for this user.
        email_meta = self.beaker_client.get_email_by_address(email)

        if not email_meta:
            logging.error(
                "Could not associate the e-mail address to a " "Beaker user, cannot continue!",
                extra={"email": self.email},
            )
            return

        # Get some data from the EmailMeta for later use.
        self.email_id = email_meta["id"]
        last_history_id = email_meta["history_id"]
        email_whitelisted = email_meta["whitelisted"]
        email_blacklisted = email_meta["blacklisted"]

        # Get the Beaker Domain object.
        domain = self.beaker_client.get_domain_by_domain_name(self.domain)

        if not domain:
            logging.error(
                "Could not associate the e-mail address to a " "Beaker domain, cannot continue!",
                extra={"email": self.email, "domain": self.domain},
            )
            return

        # If True, whitelist mode active; else blacklist mode
        domain_whitelist_enabled = domain["whitelisted"]

        # Time to determine whether or not we should actually check this account.
        proceed = False
        if domain_whitelist_enabled:
            proceed = email_whitelisted
        else:
            proceed = email_blacklisted

        if not proceed:
            logging.info(
                "Processing disabled for this address; will not continue!",
                extra={
                    "email": self.email,
                    "domain_whitelist_enabled": domain_whitelist_enabled,
                    "email_whitelisted": email_whitelisted,
                    "email_blacklisted": email_blacklisted,
                },
            )
            return

        # Get the last value of the historyId, used to get changes.
        last_history_id = email_meta["history_id"]

        # If the last history ID is the same as was in the PubSub, ignore it.
        if last_history_id >= new_history_id:
            return

        # Go get the history of this e-mail and return a list of changes.
        try:
            changes = self._get_history(last_history_id)
        # This is our first attempt at using the access token, so make sure it works.
        except AccessTokenCredentialsError:
            logging.error("Access token not accepted. Clearing token from cache.", extra={"email": self.email})
            self._clear_auth_token()
            return
        except AttributeError:
            # Google returned a 404, meaning that the account's specified history ID did not exist.
            # This is common, and happens if we ask for a history ID that is too old.
            # Instead, we'll retrieve the most recent one and update Beaker with the info.
            # logging.warning("History ID %d not found for user, getting current history." %
            #                last_history_id,
            #                extra={'email': self.email})
            # new_history_id = self._get_history_id(last_history_id=last_history_id)
            logging.warning(
                "Specified history ID not found for user, setting to the value " "provided in the PubSub message.",
                extra={"email": self.email, "last_history_id": last_history_id},
            )

            # Update Beaker
            beaker_email = {"history_id": str(new_history_id)}
            self.beaker_client.update_email(self.email_id, beaker_email)
            return

        # There will be a number of duplicate message IDs that we'll receive.
        # We must find all message IDs, and then dedup them.
        # TODO probably no longer needed with 'messagesAdded' feature.  Strip this out.
        updated_message_ids = []
        for change in changes:
            if not "messagesAdded" in change:
                continue

            for messageAdded in change["messagesAdded"]:
                message = messageAdded["message"]
                message_id = message["id"]
                if not message_id in updated_message_ids:
                    updated_message_ids.append(message_id)

        # Each of the message IDs need to be processed. We'll submit them in a bulk request.
        batch = BatchHttpRequest()
        items_in_batch = 0
        for updated_message_id in updated_message_ids:
            # Check Redis to see if this message ID is marked
            # as being processed.
            lock = self._get_message_lock(updated_message_id)
            if lock:
                logging.info(
                    "Message has already been locked for processing.",
                    extra={"email": self.email, "service_message_id": updated_message_id},
                )
                continue
            else:
                self._save_message_lock(updated_message_id)

            # Check with Beaker to see if this is a message we've already processed.
            # This is an expensive operation, so we cache heavily on the Beaker side (1hr).
            # The above operation to check with Redis first should reduce/nearly eliminate
            # the use of this check.
            service_message = self.beaker_client.get_message_by_service_id(updated_message_id, cache_maxage=3600)
            if service_message:
                logging.info(
                    "Message has already been processed.",
                    extra={"email": self.email, "service_message_id": updated_message_id},
                )
                continue

            logging.info(
                "Preparing to process message.", extra={"email": self.email, "service_message_id": updated_message_id}
            )
            batch.add(
                self.service.users().messages().get(userId=self.email, id=updated_message_id, format="raw"),
                request_id=updated_message_id,
                callback=self._process_message,
            )
            items_in_batch += 1

        # new_history_id = last_history_id
        new_history_id_from_profile = new_history_id
        if items_in_batch > 0:
            # The bulk query for the changed messages have been staged, now run them.
            self._execute_batch(batch)
            new_history_id_from_profile = self._get_history_id(last_history_id=last_history_id)

            # After executing the batch, check and see if any of the runs called for an abort.
            if self.abort_run:
                return

        # Once our run is finished, check and see if we
        # had to initiate a new auth token and, if so,
        # cache it. See function declaration for details.
        self._save_auth_token()

        logging.info(
            "The last history ID was %s. The history ID in the PubSub was %s. "
            "Next time, start at history ID %s (from profile)"
            % (last_history_id, str(new_history_id), str(new_history_id_from_profile)),
            extra={
                "email": self.email,
                "old_history_id": last_history_id,
                "new_history_id_pubsub": new_history_id,
                "new_history_id_profile": new_history_id_from_profile,
                "processed_message_count": items_in_batch,
            },
        )

        # If the history ID has changed, inform Beaker.
        if last_history_id != new_history_id_from_profile:
            beaker_email = {"history_id": str(new_history_id_from_profile)}
            self.beaker_client.update_email(self.email_id, beaker_email)
Beispiel #8
0
    def check_account_v1(self, email, new_history_id, queue_time=None, http=None):
        """
        Check a user's email account

        :param email: The user's email account.
        :param new_history_id: The new history ID for this account, as given by the Gmail Push message.
        :param queue_time: The max amount of time the message can be on the queue before getting purged.
        :param http: Optional HTTP object to use (used in tests).
        """
        queue_timeout = self._check_for_queue_timeout(queue_time)
        if queue_timeout:
            return

        self.email = email
        self.domain = email.split("@")[1]

        # Do a quick sanity check on the history ID value.
        #if last_history_id < 1:
        #    logging.critical("History ID failed sanity check, cannot continue!",
        #                     extra={'email': self.email,
        #                            'history_id': last_history_id})
        #    return

        # Service object is returned from authentication request to Google.
        self.credentials, self.http, self.service = authenticate(service="gmail", discovery="gmail_discovery.json",
                                                                 version="v1",
                                                                 sub=self.email,
                                                                 access_token=self._get_auth_token(),
                                                                 http=http)
        # Get the Beaker EmailMeta object for this user.
        email_meta = self.beaker_client.get_email_by_address(email)

        if not email_meta:
            logging.error("Could not associate the e-mail address to a " \
                          "Beaker user, cannot continue!",
                          extra={'email': self.email})
            return

        # Get some data from the EmailMeta for later use.
        self.email_id = email_meta['id']
        last_history_id = email_meta['history_id']
        email_whitelisted = email_meta['whitelisted']
        email_blacklisted = email_meta['blacklisted']

        # Get the Beaker Domain object.
        domain = self.beaker_client.get_domain_by_domain_name(self.domain)

        if not domain:
            logging.error("Could not associate the e-mail address to a " \
                          "Beaker domain, cannot continue!",
                          extra={'email': self.email,
                                 'domain': self.domain})
            return

        # If True, whitelist mode active; else blacklist mode
        domain_whitelist_enabled = domain['whitelisted']

        # Time to determine whether or not we should actually check this account.
        proceed = False
        if domain_whitelist_enabled:
            proceed = email_whitelisted
        else:
            proceed = email_blacklisted

        if not proceed:
            logging.info("Processing disabled for this address; will not continue!",
                         extra={'email': self.email,
                                'domain_whitelist_enabled': domain_whitelist_enabled,
                                'email_whitelisted': email_whitelisted,
                                'email_blacklisted': email_blacklisted})
            return

        # Get the last value of the historyId, used to get changes.
        last_history_id = email_meta['history_id']

        # If the last history ID is the same as was in the PubSub, ignore it.
        if last_history_id >= new_history_id:
            return

        # Go get the history of this e-mail and return a list of changes.
        try:
            changes = self._get_history(last_history_id)
        # This is our first attempt at using the access token, so make sure it works.
        except AccessTokenCredentialsError:
            logging.error("Access token not accepted. Clearing token from cache.",
                          extra={'email': self.email})
            self._clear_auth_token()
            return
        except AttributeError:
            # Google returned a 404, meaning that the account's specified history ID did not exist.
            # This is common, and happens if we ask for a history ID that is too old.
            # Instead, we'll retrieve the most recent one and update Beaker with the info.
            #logging.warning("History ID %d not found for user, getting current history." %
            #                last_history_id,
            #                extra={'email': self.email})
            #new_history_id = self._get_history_id(last_history_id=last_history_id)
            logging.warning("Specified history ID not found for user, setting to the value "
                            "provided in the PubSub message.",
                            extra={'email': self.email,
                                   'last_history_id': last_history_id})

            # Update Beaker
            beaker_email = {
                "history_id": str(new_history_id),
            }
            self.beaker_client.update_email(self.email_id, beaker_email)
            return

        # There will be a number of duplicate message IDs that we'll receive.
        # We must find all message IDs, and then dedup them.
        # TODO probably no longer needed with 'messagesAdded' feature.  Strip this out.
        updated_message_ids = []
        for change in changes:
            if not 'messagesAdded' in change:
                continue

            for messageAdded in change['messagesAdded']:
                message = messageAdded['message']
                message_id = message['id']
                if not message_id in updated_message_ids:
                    updated_message_ids.append(message_id)

        # Each of the message IDs need to be processed. We'll submit them in a bulk request.
        batch = BatchHttpRequest()
        items_in_batch = 0
        for updated_message_id in updated_message_ids:
            # Check Redis to see if this message ID is marked
            # as being processed.
            lock = self._get_message_lock(updated_message_id)
            if lock:
                logging.info("Message has already been locked for processing.",
                             extra={"email": self.email,
                                    "service_message_id": updated_message_id})
                continue
            else:
                self._save_message_lock(updated_message_id)

            # Check with Beaker to see if this is a message we've already processed.
            # This is an expensive operation, so we cache heavily on the Beaker side (1hr).
            # The above operation to check with Redis first should reduce/nearly eliminate
            # the use of this check.
            service_message = self.beaker_client.get_message_by_service_id(updated_message_id, cache_maxage=3600)
            if service_message:
                logging.info("Message has already been processed.",
                             extra={"email": self.email,
                                    "service_message_id": updated_message_id})
                continue

            logging.info("Preparing to process message.",
                         extra={"email": self.email,
                                "service_message_id": updated_message_id})
            batch.add(self.service.users().messages().get(userId=self.email, id=updated_message_id, format='raw'),
                      request_id=updated_message_id, callback=self._process_message)
            items_in_batch += 1

        #new_history_id = last_history_id
        new_history_id_from_profile = new_history_id
        if items_in_batch > 0:
            # The bulk query for the changed messages have been staged, now run them.
            self._execute_batch(batch)
            new_history_id_from_profile = self._get_history_id(last_history_id=last_history_id)

            # After executing the batch, check and see if any of the runs called for an abort.
            if self.abort_run:
                return

        # Once our run is finished, check and see if we
        # had to initiate a new auth token and, if so,
        # cache it. See function declaration for details.
        self._save_auth_token()

        logging.info("The last history ID was %s. The history ID in the PubSub was %s. "
                     "Next time, start at history ID %s (from profile)" % (last_history_id, str(new_history_id), str(new_history_id_from_profile)),
                     extra={'email': self.email,
                            "old_history_id": last_history_id,
                            "new_history_id_pubsub": new_history_id,
                            "new_history_id_profile": new_history_id_from_profile,
                            "processed_message_count": items_in_batch})

        # If the history ID has changed, inform Beaker.
        if last_history_id != new_history_id_from_profile:
            beaker_email = {
                "history_id": str(new_history_id_from_profile),
            }
            self.beaker_client.update_email(self.email_id, beaker_email)