# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Functions to query billing info using the Google API."""
from apiclient.discovery import build
import credentials_utils
import log_utils
_LOG = log_utils.GetLogger('messagerecall.billing_info')
class BillingInfo(object):
"""Class to retrieve billing info.
Uses http_utils to add error handling and retry using backoff.
"""
def __init__(self, owner_email):
"""Initialize the billing client.
Args:
owner_email: String email address of the project admin.
"""
self._http = credentials_utils.GetAuthorizedHttp(owner_email)
from models import recall_task
from models import sharded_counter
import recall_errors
import user_retriever
import view_utils
import webapp2
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import runtime
from google.appengine.api.taskqueue import Error as TaskQueueError
from google.appengine.api.taskqueue import Queue
from google.appengine.api.taskqueue import Task
from google.appengine.ext import ndb
_LOG = log_utils.GetLogger('messagerecall.views')
_MONITOR_SLEEP_PERIOD_S = 10
# Write users to the db in batches to save time/rpcs.
_USER_PUT_MULTI_BATCH_SIZE = 100
def PartitionEmailPrefixes():
"""Divide the domain email namespace to allow concurrent tasks to search.
The rules for gmail usernames are:
a) Letters, numbers and . are allowed.
b) The first character must be a letter or number.
Returns:
List of Strings which are the valid prefixes the user search tasks will
import sys
_MESSAGE_RECALL_DIR = os.path.dirname(os.path.abspath(__file__))
def _SetMessageRecallLibPath(sys_path, message_recall_dir):
message_recall_lib_dir = os.path.join(message_recall_dir, 'lib')
if message_recall_lib_dir not in sys_path:
sys_path.append(message_recall_lib_dir)
_SetMessageRecallLibPath(sys.path, _MESSAGE_RECALL_DIR)
import log_utils # pylint: disable=g-import-not-at-top
_LOG = log_utils.GetLogger('messagerecall.setup_path')
try:
# pylint: disable=g-import-not-at-top, unused-import
import apiclient
from apiclient.discovery import build
import httplib2
import oauth2client
from oauth2client.tools import run
from oauth2client.client import SignedJwtAssertionCredentials
# pylint: enable=g-import-not-at-top, unused-import
except ImportError as e:
module_package_map = {
'apiclient': 'google-api-python-client',
'apiclient.discovery': 'google-api-python-client',
'oauth2client.tools': 'oauth2client.tools',
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Database models to track error reasons for reporting."""
import log_utils
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.ext import ndb
_LOG = log_utils.GetLogger('messagerecall.models.error_reason')
_REASON_ROWS_FETCH_PAGE = 20
class ErrorReasonModel(ndb.Model):
"""Model to track errors in the various recall tasks."""
recall_task_id = ndb.IntegerProperty(required=True)
user_email = ndb.StringProperty() # Empty if error before user activities.
error_datetime = ndb.DateTimeProperty(required=True, auto_now_add=True)
error_reason = ndb.StringProperty(required=True) # Max 500 character length.
@classmethod
def FetchOneUIPageOfErrorsForTask(cls, task_key_urlsafe, urlsafe_cursor):
"""Utility to query and fetch all error reasons for UI pages.
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Database models for candidate domain users to check for message presence."""
import log_utils
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.ext import ndb
_LOG = log_utils.GetLogger('messagerecall.models.domain_user')
_USER_ROWS_FETCH_PAGE = 10
USER_STARTED = 'Started'
USER_RECALLING = 'Recalling'
USER_CONNECT_FAILED = 'Imap Connect Failed'
USER_IMAP_DISABLED = 'Imap Disabled'
USER_DONE = 'Done'
USER_ABORTED = 'Aborted'
USER_SUSPENDED = 'Suspended'
USER_STATES = [USER_STARTED, USER_RECALLING, USER_CONNECT_FAILED,
USER_IMAP_DISABLED, USER_DONE, USER_ABORTED, USER_SUSPENDED]
ACTIVE_USER_STATES = [USER_STARTED, USER_RECALLING, USER_CONNECT_FAILED,
USER_IMAP_DISABLED, USER_ABORTED]
TERMINAL_USER_STATES = [USER_CONNECT_FAILED, USER_IMAP_DISABLED,
"""Database models for root Message Recall Task entity."""
import time
import log_utils
from models import domain_user
from models import error_reason
import recall_errors
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.ext import ndb
_GET_ENTITY_RETRIES = 5
_GET_ENTITY_SLEEP_S = 2
_LOG = log_utils.GetLogger('messagerecall.models.recall_task')
_TASK_ROWS_FETCH_PAGE = 10
TASK_STARTED = 'Started'
TASK_GETTING_USERS = 'Getting Users'
TASK_RECALLING = 'Recalling'
TASK_DONE = 'Done'
TASK_STATES = [TASK_STARTED, TASK_GETTING_USERS, TASK_RECALLING, TASK_DONE]
class RecallTaskModel(ndb.Model):
"""Model for each running/completed message recall task."""
owner_email = ndb.StringProperty(required=True)
message_criteria = ndb.StringProperty(required=True)
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Functions to search users using the Google Admin SDK API."""
import httplib
from apiclient.discovery import build
from apiclient.errors import HttpError
import credentials_utils
import log_utils
_LOG = log_utils.GetLogger('messagerecall.user_retriever')
_MAX_RESULT_PAGE_SIZE = 500 # Default is 100.
class DomainUserRetriever(object):
"""Class to organize large, multi-page user searches.
Uses http_utils to add error handling and retry using backoff.
"""
def __init__(self, owner_email, user_domain, search_query):
"""Initialize the search class.
Build the items needed to page through domain user lists which are expected
to be >100k users at times. Need a users collection object from the ADMIN
SDK to reference the search api and an authenticated http connection to
invoke it.
owned by multiple users in an apps domain.
"""
import os
import http_utils
import log_utils
from oauth2client import client
import recall_errors
import service_account
from google.appengine.api import memcache
_ACCESS_TOKEN_CACHE_S = 60 * 59 # Access tokens last about 1 hour.
_CACHE_NAMESPACE = 'messagerecall_accesstoken#ns'
_LOG = log_utils.GetLogger('messagerecall.credentials_utils')
# Load the key in PKCS 12 format that you downloaded from the Google API
# Console when you created your Service account.
_SERVICE_ACCOUNT_PEM_FILE_NAME = os.path.join(os.path.dirname(__file__),
'messagerecall_privatekey.pem')
with open(_SERVICE_ACCOUNT_PEM_FILE_NAME, 'rb') as f:
_SERVICE_ACCOUNT_KEY = f.read()
def _GetSignedJwtAssertionCredentials(user_email):
"""Retrieve an OAuth2 credentials object impersonating user_email.
This object is then used to authorize an http connection that will be
used to connect with Google services such as the Admin SDK.
# limitations under the License.
"""Sharded counter implementation: allows high volume counter access.
Reference: https://developers.google.com/appengine/articles/sharding_counters
"""
import random
import log_utils
import recall_errors
from google.appengine.api import memcache
from google.appengine.ext import ndb
_COUNTER_MEMCACHE_EXPIRATION_S = 60 * 60 * 24
_LOG = log_utils.GetLogger('messagerecall.models.sharded_counter')
_SHARD_KEY_TEMPLATE = 'shard-{}-{:d}'
_TRANSACTIONAL_RETRIES = 8
class CounterShardConfig(ndb.Model):
"""Allows customized shard count: highly used counters need more shards."""
num_shards = ndb.IntegerProperty(default=20)
@classmethod
def AllKeys(cls, name):
"""Returns all possible keys for the counter name given the config.
Args:
name: The name of the counter.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions and constants for http API requests."""
import httplib2
import log_utils
_EXTENDED_SOCKET_TIMEOUT_S = 10 # Default of 5s seems too short for Admin SDK.
_LOG = log_utils.GetLogger('messagerecall.http_utils')
def GetHttpObject():
"""Helper to abstract Http connection acquisition.
Not infrequently seeing 'HTTPException: Deadline exceeded...' error when
running in a scaled environment with > 900 users. Adding
_EXTENDED_SOCKET_TIMEOUT_S seems to mostly resolve this.
Returns:
Http connection object.
"""
return httplib2.Http(timeout=_EXTENDED_SOCKET_TIMEOUT_S)