def auth():
""" Check for account ID on all non-root URLS """
if request.path in ('/accounts', '/accounts/', '/') \
or request.path.startswith('/w/'):
return
if not request.authorization or not request.authorization.username:
return make_response((
"Could not verify access credential.", 401,
{'WWW-Authenticate': 'Basic realm="API '
'Access Token Required"'}))
namespace_public_id = request.authorization.username
with global_session_scope() as db_session:
try:
valid_public_id(namespace_public_id)
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
g.namespace_id = namespace.id
g.account_id = namespace.account.id
except NoResultFound:
return make_response((
"Could not verify access credential.", 401,
{'WWW-Authenticate': 'Basic realm="API '
'Access Token Required"'}))
def user_console(user_email_address):
with global_session_scope() as db_session:
account = db_session.query(Account).filter_by(
email_address=user_email_address).one()
if account.provider == 'eas':
banner = """
You can access the account instance with the 'account' variable.
"""
else:
with writable_connection_pool(account.id, pool_size=1).get()\
as crispin_client:
if account.provider == 'gmail' \
and 'all' in crispin_client.folder_names():
crispin_client.select_folder(
crispin_client.folder_names()['all'][0],
uidvalidity_cb)
banner = """
You can access the crispin instance with the 'crispin_client' variable,
and the account instance with the 'account' variable.
IMAPClient docs are at:
http://imapclient.readthedocs.org/en/latest/#imapclient-class-reference
"""
IPython.embed(banner1=banner)
def _process_log(self):
with global_session_scope() as db_session:
# Only actions on accounts associated with this sync-engine
query = db_session.query(ActionLog).join(Namespace).join(Account).\
filter(ActionLog.discriminator == 'actionlog',
ActionLog.status == 'pending',
Account.sync_host == platform.node(),
Account.sync_should_run).\
order_by(ActionLog.id).\
options(contains_eager(ActionLog.namespace, Namespace.account))
running_action_ids = [worker.action_log_id for worker in
self.workers]
if running_action_ids:
query = query.filter(~ActionLog.id.in_(running_action_ids))
for log_entry in query:
namespace = log_entry.namespace
self.log.info('delegating action',
action_id=log_entry.id,
msg=log_entry.action)
semaphore = self.account_semaphores[namespace.account_id]
worker = SyncbackWorker(action_name=log_entry.action,
semaphore=semaphore,
action_log_id=log_entry.id,
record_id=log_entry.record_id,
account_id=namespace.account_id,
provider=namespace.account.verbose_provider,
retry_interval=self.retry_interval,
extra_args=log_entry.extra_args)
self.workers.add(worker)
worker.start()
def _process_log(self):
with global_session_scope() as db_session:
# Only actions on accounts associated with this sync-engine
query = db_session.query(ActionLog).join(Namespace).join(Account).\
filter(ActionLog.discriminator == 'actionlog',
ActionLog.status == 'pending',
Account.sync_host == platform.node(),
Account.sync_should_run).\
order_by(ActionLog.id).\
options(contains_eager(ActionLog.namespace, Namespace.account))
running_action_ids = [worker.action_log_id for worker in
self.workers]
if running_action_ids:
query = query.filter(~ActionLog.id.in_(running_action_ids))
for log_entry in query:
namespace = log_entry.namespace
self.log.info('delegating action',
action_id=log_entry.id,
msg=log_entry.action)
semaphore = self.account_semaphores[namespace.account_id]
worker = SyncbackWorker(action_name=log_entry.action,
semaphore=semaphore,
action_log_id=log_entry.id,
record_id=log_entry.record_id,
account_id=namespace.account_id,
provider=namespace.account.provider,
retry_interval=self.retry_interval,
extra_args=log_entry.extra_args)
self.workers.add(worker)
worker.start()
def suspend_sync():
g.parser.add_argument('account_id',
required=True,
type=valid_public_id,
location='form')
args = strict_parse_args(g.parser, request.args)
namespace_public_id = args['account_id']
with global_session_scope() as db_session:
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
namespace_id = namespace.id
with session_scope(namespace_id) as db_session:
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
account = namespace.account
account.sync_should_run = False
account._sync_status[
'sync_disabled_reason'] = 'suspend_account API endpoint called'
account._sync_status['sync_disabled_on'] = datetime.utcnow()
account._sync_status['sync_disabled_by'] = 'api'
db_session.commit()
shared_queue = shared_sync_event_queue_for_zone(config.get('ZONE'))
shared_queue.send_event({'event': 'sync_suspended', 'id': account.id})
return make_response(('', 204, {}))
def modify_account(namespace_public_id):
"""
Modify an existing account
This stops syncing an account until it is explicitly resumed.
"""
data = request.get_json(force=True)
with global_session_scope() as db_session:
namespace = (db_session.query(Namespace).filter(
Namespace.public_id == namespace_public_id).one())
account = namespace.account
if isinstance(account, GenericAccount):
auth_handler = GenericAuthHandler()
account_data = _get_account_data_for_generic_account(data)
elif isinstance(account, GmailAccount):
auth_handler = GoogleAuthHandler()
account_data = _get_account_data_for_google_account(data)
elif isinstance(account, OutlookAccount):
auth_handler = MicrosoftAuthHandler()
account_data = _get_account_data_for_microsoft_account(data)
else:
raise ValueError("Account type not supported.")
account = auth_handler.update_account(account, account_data)
db_session.add(account)
db_session.commit()
encoder = APIEncoder()
return encoder.jsonify(account.namespace)
def account_ids_owned(self):
with global_session_scope() as db_session:
return {
r[0]
for r in db_session.query(Account.id).filter(
Account.sync_host == self.process_identifier).all()
}
def ns_all():
""" Return all namespaces """
# We do this outside the blueprint to support the case of an empty
# public_id. However, this means the before_request isn't run, so we need
# to make our own session
with global_session_scope() as db_session:
parser = reqparse.RequestParser(argument_class=ValidatableArgument)
parser.add_argument('limit', default=DEFAULT_LIMIT, type=limit,
location='args')
parser.add_argument('offset', default=0, type=int, location='args')
parser.add_argument('email_address', type=bounded_str, location='args')
args = strict_parse_args(parser, request.args)
query = db_session.query(Namespace)
if args['email_address']:
query = query.join(Account)
query = query.filter_by(email_address=args['email_address'])
query = query.limit(args['limit'])
if args['offset']:
query = query.offset(args['offset'])
namespaces = query.all()
encoder = APIEncoder(legacy_nsid=request.path.startswith('/n'))
return encoder.jsonify(namespaces)
def event_update(calendar_public_id):
try:
valid_public_id(calendar_public_id)
with global_session_scope() as db_session:
calendar = db_session.query(Calendar) \
.filter(Calendar.public_id == calendar_public_id) \
.one()
if calendar.gpush_last_ping is not None:
time_since_last_ping = (
datetime.utcnow() - calendar.gpush_last_ping
).total_seconds()
# Limit write volume, and de-herd, in case we're getting many
# concurrent updates for the same calendar.
if time_since_last_ping < 10 + random.randrange(0, 10):
return resp(200)
calendar.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
g.log.info('Getting push notifications for non-existing calendar',
calendar_public_id=calendar_public_id)
raise NotFoundError("Couldn't find calendar `{0}`"
.format(calendar_public_id))
def ns_all():
""" Return all namespaces """
# We do this outside the blueprint to support the case of an empty
# public_id. However, this means the before_request isn't run, so we need
# to make our own session
with global_session_scope() as db_session:
parser = reqparse.RequestParser(argument_class=ValidatableArgument)
parser.add_argument("limit",
default=DEFAULT_LIMIT,
type=limit,
location="args")
parser.add_argument("offset", default=0, type=int, location="args")
parser.add_argument("email_address", type=bounded_str, location="args")
args = strict_parse_args(parser, request.args)
query = db_session.query(Namespace)
if args["email_address"]:
query = query.join(Account)
query = query.filter_by(email_address=args["email_address"])
query = query.limit(args["limit"])
if args["offset"]:
query = query.offset(args["offset"])
namespaces = query.all()
encoder = APIEncoder()
return encoder.jsonify(namespaces)
def event_update(calendar_public_id):
g.log.info('Received request to update Google calendar',
calendar_public_id=calendar_public_id)
try:
valid_public_id(calendar_public_id)
with global_session_scope() as db_session:
calendar = db_session.query(Calendar) \
.filter(Calendar.public_id == calendar_public_id) \
.one()
if calendar.gpush_last_ping is not None:
time_since_last_ping = (
datetime.utcnow() -
calendar.gpush_last_ping).total_seconds()
# Limit write volume, and de-herd, in case we're getting many
# concurrent updates for the same calendar.
if time_since_last_ping < 10 + random.randrange(0, 10):
return resp(200)
calendar.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
g.log.info('Getting push notifications for non-existing calendar',
calendar_public_id=calendar_public_id)
raise NotFoundError(
"Couldn't find calendar `{0}`".format(calendar_public_id))
def find_account_shard(email_address):
with global_session_scope() as db_session:
account = db_session.query(Account).filter_by(
email_address=email_address).first()
if account is not None:
return account.id
return None
def main(hostname):
maybe_enable_rollbar()
with global_session_scope() as db_session:
account_ids = db_session.query(Account.id).filter(Account.sync_host == hostname)
print("Accounts being synced by {}:".format(hostname))
for account_id in account_ids:
print(account_id[0])
db_session.commit()
def account_ids_to_sync(self):
with global_session_scope() as db_session:
return {r[0] for r in db_session.query(Account.id).
filter(Account.sync_should_run,
or_(and_(Account.desired_sync_host == self.process_identifier,
Account.sync_host == None), # noqa
and_(Account.desired_sync_host == None, # noqa
Account.sync_host == self.process_identifier),
and_(Account.desired_sync_host == self.process_identifier,
Account.sync_host == self.process_identifier))).all()}
def main(min_id, max_id, shard_id):
maybe_enable_rollbar()
generic_accounts = []
failed = []
if min_id is not None or max_id is not None:
# Get the list of running Gmail accounts.
with global_session_scope() as db_session:
generic_accounts = db_session.query(GenericAccount).filter(
GenericAccount.sync_state == "running")
if min_id is not None:
generic_accounts = generic_accounts.filter(
GenericAccount.id > min_id)
if max_id is not None:
generic_accounts = generic_accounts.filter(
GenericAccount.id <= max_id)
generic_accounts = [acc.id for acc in generic_accounts]
db_session.expunge_all()
elif shard_id is not None:
with session_scope_by_shard_id(shard_id) as db_session:
generic_accounts = db_session.query(GenericAccount).filter(
GenericAccount.sync_state == "running")
generic_accounts = [acc.id for acc in generic_accounts]
db_session.expunge_all()
print("Total accounts", len(generic_accounts))
for account_id in generic_accounts:
try:
with session_scope(account_id) as db_session:
account = db_session.query(GenericAccount).get(account_id)
print("Updating", account.email_address)
with connection_pool(account.id).get() as crispin_client:
account.folder_prefix = crispin_client.folder_prefix
account.folder_separator = crispin_client.folder_separator
db_session.commit()
except Exception:
failed.append(account_id)
print("Processed accounts:")
print(generic_accounts)
print("Failed accounts:")
print(failed)
def main(dry_run, number, hostname, process):
"""
Unschedule all accounts assigned to a given sync host.
Intended primarily for use when decomissioning sync instances or for
manually unloading an overloaded sync instance.
"""
maybe_enable_rollbar()
if not number:
message = (
"You have not provided a --number option. This will "
"unschedule ALL syncs on the host. Proceed? [Y/n] "
)
if raw_input(message).strip().lower() == "n":
print("Will not proceed")
return
if not dry_run:
message = (
"It is unsafe to unassign hosts while mailsync processes are running. "
"Have you shut down the appropriate mailsync processes on {}? [Y/n]".format(
hostname
)
)
if raw_input(message).strip().lower() == "n":
print("Bailing out")
return
with global_session_scope() as db_session:
if process is not None:
hostname = ":".join([hostname, process])
to_unschedule = db_session.query(Account.id).filter(
Account.sync_host.like("{}%".format(hostname))
)
if number:
to_unschedule = to_unschedule.limit(number)
to_unschedule = [id_ for id_, in to_unschedule.all()]
if number:
to_unschedule = to_unschedule[:number]
for account_id in to_unschedule:
with session_scope(account_id) as db_session:
if dry_run:
print("Would unassign", account_id)
else:
account = db_session.query(Account).get(account_id)
print("Unassigning", account.id)
account.desired_sync_host = None
account.sync_host = None
db_session.commit()
def main(min_id, max_id, shard_id):
generic_accounts = []
failed = []
if min_id is not None or max_id is not None:
# Get the list of running Gmail accounts.
with global_session_scope() as db_session:
generic_accounts = db_session.query(GenericAccount).filter(
GenericAccount.sync_state == 'running')
if min_id is not None:
generic_accounts = generic_accounts.filter(
GenericAccount.id > min_id)
if max_id is not None:
generic_accounts = generic_accounts.filter(
GenericAccount.id <= max_id)
generic_accounts = [acc.id for acc in generic_accounts]
db_session.expunge_all()
elif shard_id is not None:
with session_scope_by_shard_id(shard_id) as db_session:
generic_accounts = db_session.query(GenericAccount).filter(
GenericAccount.sync_state == 'running')
generic_accounts = [acc.id for acc in generic_accounts]
db_session.expunge_all()
print "Total accounts: %d" % len(generic_accounts)
for account_id in generic_accounts:
try:
with session_scope(account_id) as db_session:
account = db_session.query(GenericAccount).get(account_id)
print "Updating %s" % account.email_address
with connection_pool(account.id).get() as crispin_client:
account.folder_prefix = crispin_client.folder_prefix
account.folder_separator = crispin_client.folder_separator
db_session.commit()
except Exception:
failed.append(account_id)
print "Processed accounts:"
print generic_accounts
print "Failed accounts:"
print failed
def auth():
""" Check for account ID on all non-root URLS """
if (
request.path == "/"
or request.path.startswith("/accounts")
or request.path.startswith("/w/")
or request.path.startswith("/metrics")
):
return
if not request.authorization or not request.authorization.username:
AUTH_ERROR_MSG = (
"Could not verify access credential.",
401,
{"WWW-Authenticate": 'Basic realm="API ' 'Access Token Required"'},
)
auth_header = request.headers.get("Authorization", None)
if not auth_header:
return make_response(AUTH_ERROR_MSG)
parts = auth_header.split()
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1]:
return make_response(AUTH_ERROR_MSG)
namespace_public_id = parts[1]
else:
namespace_public_id = request.authorization.username
with global_session_scope() as db_session:
try:
valid_public_id(namespace_public_id)
namespace = (
db_session.query(Namespace)
.filter(Namespace.public_id == namespace_public_id)
.one()
)
g.namespace_id = namespace.id
g.account_id = namespace.account.id
except NoResultFound:
return make_response(
(
"Could not verify access credential.",
401,
{"WWW-Authenticate": 'Basic realm="API ' 'Access Token Required"'},
)
)
def user_console(user_email_address):
with global_session_scope() as db_session:
result = (db_session.query(Account).filter_by(
email_address=user_email_address).all())
account = None
if len(result) == 1:
account = result[0]
elif len(result) > 1:
print("\n{} accounts found for that email.\n".format(len(result)))
for idx, acc in enumerate(result):
print("[{}] - {} {} {}".format(
idx,
acc.provider,
acc.namespace.email_address,
acc.namespace.public_id,
))
choice = int(input("\nWhich # do you want to select? "))
account = result[choice]
if account is None:
print(
"No account found with email '{}'".format(user_email_address))
return
if account.provider == "eas":
banner = """
You can access the account instance with the 'account' variable.
"""
IPython.embed(banner1=banner)
else:
with writable_connection_pool(account.id,
pool_size=1).get() as crispin_client:
if (account.provider == "gmail"
and "all" in crispin_client.folder_names()):
crispin_client.select_folder(
crispin_client.folder_names()["all"][0],
uidvalidity_cb)
banner = """
You can access the crispin instance with the 'crispin_client' variable,
and the account instance with the 'account' variable.
IMAPClient docs are at:
http://imapclient.readthedocs.org/en/latest/#imapclient-class-reference
"""
IPython.embed(banner1=banner)
def calendar_update(account_public_id):
request.environ["log_context"]["account_public_id"] = account_public_id
try:
valid_public_id(account_public_id)
with global_session_scope() as db_session:
account = (db_session.query(GmailAccount).filter(
GmailAccount.public_id == account_public_id).one())
account.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError("Invalid public ID")
except NoResultFound:
raise NotFoundError(
"Couldn't find account `{0}`".format(account_public_id))
def delete_account(namespace_public_id):
""" Mark an existing account for deletion. """
try:
with global_session_scope() as db_session:
namespace = (db_session.query(Namespace).filter(
Namespace.public_id == namespace_public_id).one())
account = namespace.account
account.mark_for_deletion()
db_session.commit()
except NoResultFound:
raise NotFoundError(
"Couldn't find account `{0}` ".format(namespace_public_id))
encoder = APIEncoder()
return encoder.jsonify({})
def calendar_update(account_public_id):
request.environ['log_context']['account_public_id'] = account_public_id
try:
valid_public_id(account_public_id)
with global_session_scope() as db_session:
account = db_session.query(GmailAccount) \
.filter(GmailAccount.public_id == account_public_id) \
.one()
account.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
raise NotFoundError("Couldn't find account `{0}`"
.format(account_public_id))
def enable_sync():
g.parser.add_argument('account_id',
required=True,
type=valid_public_id,
location='form')
args = strict_parse_args(g.parser, request.args)
account_id = None
namespace_public_id = args['account_id']
with global_session_scope() as db_session:
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
account_id = namespace.account.id
with session_scope(account_id) as db_session:
try:
account = db_session.query(Account).with_for_update() \
.filter(Account.id == account_id).one()
lease_period = timedelta(minutes=1)
time_ended = account.sync_status.get('sync_end_time')
time_now = datetime.utcnow()
if account.sync_host is None and account.sync_state != 'running' \
and (time_ended is None or time_now > time_ended + lease_period):
account.sync_should_run = True
if account.provider == 'gmail':
creds = account.auth_credentials
for c in creds:
c.is_valid = True
db_session.commit()
resp = json.dumps(account.sync_status, default=json_util.default)
return make_response((resp, 200, {
'Content-Type': 'application/json'
}))
except NotSupportedError as e:
resp = simplejson.dumps({
'message': str(e),
'type': 'custom_api_error'
})
return make_response((resp, 400, {
'Content-Type': 'application/json'
}))
def event_update(calendar_public_id):
try:
valid_public_id(calendar_public_id)
with global_session_scope() as db_session:
calendar = db_session.query(Calendar) \
.filter(Calendar.public_id == calendar_public_id) \
.one()
calendar.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
g.log.info('Getting push notifications for non-existing calendar',
calendar_public_id=calendar_public_id)
raise NotFoundError(
"Couldn't find calendar `{0}`".format(calendar_public_id))
def user_console(user_email_address):
with global_session_scope() as db_session:
result = db_session.query(Account).filter_by(
email_address=user_email_address).all()
account = None
if len(result) == 1:
account = result[0]
elif len(result) > 1:
print "\n{} accounts found for that email.\n".format(len(result))
for idx, acc in enumerate(result):
print "[{}] - {} {} {}".format(idx, acc.provider,
acc.namespace.email_address,
acc.namespace.public_id)
choice = int(raw_input("\nWhich # do you want to select? "))
account = result[choice]
if account is None:
print "No account found with email '{}'".format(user_email_address)
return
if account.provider == 'eas':
banner = """
You can access the account instance with the 'account' variable.
"""
IPython.embed(banner1=banner)
else:
with writable_connection_pool(account.id, pool_size=1).get()\
as crispin_client:
if account.provider == 'gmail' \
and 'all' in crispin_client.folder_names():
crispin_client.select_folder(
crispin_client.folder_names()['all'][0],
uidvalidity_cb)
banner = """
You can access the crispin instance with the 'crispin_client' variable,
and the account instance with the 'account' variable.
IMAPClient docs are at:
http://imapclient.readthedocs.org/en/latest/#imapclient-class-reference
"""
IPython.embed(banner1=banner)
def event_update(calendar_public_id):
try:
valid_public_id(calendar_public_id)
with global_session_scope() as db_session:
calendar = db_session.query(Calendar) \
.filter(Calendar.public_id == calendar_public_id) \
.one()
calendar.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
g.log.info('Getting push notifications for non-existing calendar',
calendar_public_id=calendar_public_id)
raise NotFoundError("Couldn't find calendar `{0}`"
.format(calendar_public_id))
def calendar_update(account_public_id):
try:
valid_public_id(account_public_id)
with global_session_scope() as db_session:
account = db_session.query(GmailAccount) \
.filter(GmailAccount.public_id == account_public_id) \
.one()
account.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
g.log.info('Getting push notifications for non-existing account',
account_public_id=account_public_id)
raise NotFoundError(
"Couldn't find account `{0}`".format(account_public_id))
def event_update(calendar_public_id):
request.environ["log_context"]["calendar_public_id"] = calendar_public_id
try:
valid_public_id(calendar_public_id)
allowed, tokens, sleep = limitlion.throttle(
"gcal:{}".format(calendar_public_id), rps=0.5)
if allowed:
with global_session_scope() as db_session:
calendar = (db_session.query(Calendar).filter(
Calendar.public_id == calendar_public_id).one())
calendar.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError("Invalid public ID")
except NoResultFound:
raise NotFoundError(
"Couldn't find calendar `{0}`".format(calendar_public_id))
def calendar_update(account_public_id):
try:
valid_public_id(account_public_id)
with global_session_scope() as db_session:
account = db_session.query(GmailAccount) \
.filter(GmailAccount.public_id == account_public_id) \
.one()
account.handle_gpush_notification()
db_session.commit()
return resp(200)
except ValueError:
raise InputError('Invalid public ID')
except NoResultFound:
g.log.info('Getting push notifications for non-existing account',
account_public_id=account_public_id)
raise NotFoundError("Couldn't find account `{0}`"
.format(account_public_id))
def auth():
""" Check for account ID on all non-root URLS """
if request.path in ('/accounts', '/accounts/', '/') \
or request.path.startswith('/w/') \
or request.path.startswith('/c/'):
return
if not request.authorization or not request.authorization.username:
AUTH_ERROR_MSG = ("Could not verify access credential.", 401, {
'WWW-Authenticate':
'Basic realm="API '
'Access Token Required"'
})
auth_header = request.headers.get('Authorization', None)
if not auth_header:
return make_response(AUTH_ERROR_MSG)
parts = auth_header.split()
if (len(parts) != 2 or parts[0].lower() != 'bearer' or not parts[1]):
return make_response(AUTH_ERROR_MSG)
namespace_public_id = parts[1]
else:
namespace_public_id = request.authorization.username
with global_session_scope() as db_session:
try:
valid_public_id(namespace_public_id)
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
g.namespace_id = namespace.id
g.account_id = namespace.account.id
except NoResultFound:
return make_response(("Could not verify access credential.", 401, {
'WWW-Authenticate':
'Basic realm="API '
'Access Token Required"'
}))
def main():
"""
Detects accounts with sync_state and sync_host inconsistent with
sync_should_run bit. (At one point, this could happen if, say, an account
was _started_ on a new host without being first stopped on its previous
host.)
"""
maybe_enable_rollbar()
with global_session_scope() as db_session:
for acc in (db_session.query(Account).options(
load_only("sync_state", "sync_should_run", "sync_host",
"desired_sync_host")).filter(
Account.sync_state == "stopped")):
if acc.desired_sync_host is not None:
print("account {} assigned to {} but has sync_state 'stopped'"
" ({}, {})".format(acc.id, acc.sync_host,
acc.sync_should_run, acc.sync_host))
def main(account_id, desired_host, dry_run, toggle_sync):
maybe_enable_rollbar()
with global_session_scope() as db_session:
account = db_session.query(Account).get(int(account_id))
print("Before sync host: {}".format(account.sync_host))
print("Before desired sync host: {}".format(account.desired_sync_host))
print("Before sync should run: {}".format(account.sync_should_run))
if dry_run:
return
account.desired_sync_host = desired_host
if toggle_sync:
account.sync_should_run = not account.sync_should_run
print("After sync host: {}".format(account.sync_host))
print("After desired sync host: {}".format(account.desired_sync_host))
print("After sync should run: {}".format(account.sync_should_run))
db_session.commit()
def auth_user(request):
"""
Authentication for user-specific routes, for example
getting messages for one user
"""
if not request.authorization or not request.authorization.username:
AUTH_ERROR_MSG = ("Could not verify access credential.", 401, {
'WWW-Authenticate':
'Basic realm="API '
'Access Token Required"'
})
auth_header = request.headers.get('Authorization', None)
if not auth_header:
return make_response(AUTH_ERROR_MSG)
parts = auth_header.split()
if len(parts) != 2 or parts[0].lower() != 'bearer' or not parts[1]:
return make_response(AUTH_ERROR_MSG)
namespace_public_id = parts[1]
else:
namespace_public_id = request.authorization.username
with global_session_scope() as db_session:
try:
valid_public_id(namespace_public_id)
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
g.namespace_id = namespace.id
g.account_id = namespace.account.id
except NoResultFound:
return make_response(("Could not verify access credential.", 401, {
'WWW-Authenticate':
'Basic realm="API '
'Access Token Required"'
}))
def auth():
""" Check for account ID on all non-root URLS """
if request.path in ('/accounts', '/accounts/', '/') \
or request.path.startswith('/w/'):
return
if not request.authorization or not request.authorization.username:
AUTH_ERROR_MSG = ("Could not verify access credential.", 401,
{'WWW-Authenticate': 'Basic realm="API '
'Access Token Required"'})
auth_header = request.headers.get('Authorization', None)
if not auth_header:
return make_response(AUTH_ERROR_MSG)
parts = auth_header.split()
if (len(parts) != 2 or parts[0].lower() != 'bearer' or not parts[1]):
return make_response(AUTH_ERROR_MSG)
namespace_public_id = parts[1]
else:
namespace_public_id = request.authorization.username
with global_session_scope() as db_session:
try:
valid_public_id(namespace_public_id)
namespace = db_session.query(Namespace) \
.filter(Namespace.public_id == namespace_public_id).one()
g.namespace_id = namespace.id
g.account_id = namespace.account.id
except NoResultFound:
return make_response((
"Could not verify access credential.", 401,
{'WWW-Authenticate': 'Basic realm="API '
'Access Token Required"'}))
def main(num_accounts):
maybe_enable_rollbar()
with global_session_scope() as db_session:
accounts = (db_session.query(Account).filter(
Account.sync_should_run == true()).order_by(
func.rand()).limit(num_accounts).all())
accounts = [acc.id for acc in accounts][:num_accounts]
db_session.expunge_all()
pool = Pool(size=100)
results = pool.map(process_account, accounts)
global_results = dict()
for ret in results:
for key in ret:
if key not in global_results:
global_results[key] = 0
global_results[key] += ret[key]
print(global_results)
def main(type, id, public_id):
maybe_enable_rollbar()
type = type.lower()
if type not in cls_for_type:
print("Error: unknown type '{}'".format(type))
sys.exit(-1)
cls = cls_for_type[type]
if public_id is None and id is None:
print("Error: you should specify an id or public id to query.")
sys.exit(-1)
with global_session_scope() as db_session:
if public_id:
obj = db_session.query(cls).filter(
cls.public_id == public_id).one()
print(obj.id)
elif id:
obj = db_session.query(cls).filter(cls.id == id).one()
print(obj.public_id)
def create_account():
""" Create a new account """
data = request.get_json(force=True)
if data["type"] == "generic":
auth_handler = GenericAuthHandler()
account_data = _get_account_data_for_generic_account(data)
elif data["type"] == "gmail":
auth_handler = GoogleAuthHandler()
account_data = _get_account_data_for_google_account(data)
elif data["type"] == "microsoft":
auth_handler = MicrosoftAuthHandler()
account_data = _get_account_data_for_microsoft_account(data)
else:
raise ValueError("Account type not supported.")
with global_session_scope() as db_session:
account = auth_handler.create_account(account_data)
db_session.add(account)
db_session.commit()
encoder = APIEncoder()
return encoder.jsonify(account.namespace)
def main(type, id, public_id, account_id, namespace_id, readwrite):
maybe_enable_rollbar()
type = type.lower()
if type not in cls_for_type:
print("Error: unknown type '{}'".format(type))
sys.exit(-1)
cls = cls_for_type[type]
if all([id, public_id, account_id, namespace_id]):
print("Error: you should specify an id or public id to query.")
sys.exit(-1)
with global_session_scope() as db_session, db_session.no_autoflush:
qu = db_session.query(cls)
if public_id:
qu = qu.filter(cls.public_id == public_id)
elif id:
qu = qu.filter(cls.id == id)
if account_id:
qu = qu.filter(cls.account_id == account_id)
elif namespace_id:
qu = qu.filter(cls.namespace_id == namespace_id)
qu.one() # noqa: F841
banner = """The object you queried is accessible as `obj`.
Note that the db session is read-only, unless if you start this script with --readwrite"""
IPython.embed(banner1=banner)
if readwrite is False:
print("Rolling-back db session.")
db_session.rollback()
def account_ids_owned(self):
with global_session_scope() as db_session:
return {r[0] for r in db_session.query(Account.id).
filter(Account.sync_host == self.process_identifier).all()}
def index():
with global_session_scope() as db_session:
if "namespace_id" in request.args:
try:
namespace = (
db_session.query(Namespace)
.filter(Namespace.public_id == request.args["namespace_id"])
.one()
)
except NoResultFound:
return APIEncoder().jsonify([])
else:
namespace = None
accounts = db_session.query(ImapAccount).with_polymorphic([GenericAccount])
if namespace:
accounts = accounts.filter(Account.namespace == namespace)
else:
# Get all account IDs that aren't deleted
account_ids = [
result[0]
for result in db_session.query(ImapAccount.id, ImapAccount._sync_status)
if result[1].get("sync_disabled_reason") != "account deleted"
]
# This is faster than fetching all accounts.
accounts = accounts.filter(ImapAccount.id.in_(account_ids))
accounts = list(accounts)
folder_data = _get_folder_data(db_session, accounts)
calendar_data = _get_calendar_data(db_session, namespace)
heartbeat = get_ping_status(account_ids=[acc.id for acc in accounts])
data = []
for account in accounts:
if account.id in heartbeat:
account_heartbeat = heartbeat[account.id]
account_folder_data = folder_data[account.id]
account_calendar_data = calendar_data[account.id]
events_alive = False
for folder_status in account_heartbeat.folders:
folder_status_id = int(folder_status.id)
if folder_status_id in account_folder_data:
account_folder_data[folder_status_id].update(
{
"alive": folder_status.alive,
"heartbeat_at": folder_status.timestamp,
}
)
elif folder_status_id == EVENT_SYNC_FOLDER_ID:
events_alive = folder_status.alive
email_alive = all(f["alive"] for f in account_folder_data.values())
alive = True
if account.sync_email and not email_alive:
alive = False
if account.sync_events and not events_alive:
alive = False
email_initial_sync = any(
f["state"] == "initial" for f in account_folder_data.values()
)
events_initial_sync = any(
c["state"] == "initial" for c in account_calendar_data
)
initial_sync = email_initial_sync or events_initial_sync
total_uids = sum(
f["remote_uid_count"] or 0 for f in account_folder_data.values()
)
remaining_uids = sum(
f["download_uid_count"] or 0 for f in account_folder_data.values()
)
if total_uids:
progress = 100.0 / total_uids * (total_uids - remaining_uids)
else:
progress = None
else:
alive = False
email_initial_sync = None
events_initial_sync = None
initial_sync = None
progress = None
sync_status = account.sync_status
is_running = sync_status["state"] == "running"
if (
is_running
and not sync_status.get("sync_start_time")
and not sync_status.get("sync_error")
):
sync_status_str = "starting"
elif is_running and alive:
if initial_sync:
sync_status_str = "initial"
else:
sync_status_str = "running"
elif is_running:
# Nylas is syncing, but not all heartbeats are reporting.
sync_status_str = "delayed"
else:
# Nylas is no longer syncing this account.
sync_status_str = "dead"
data.append(
{
"account_private_id": account.id,
"namespace_private_id": account.namespace.id,
"account_id": account.public_id,
"namespace_id": account.namespace.public_id,
"events_alive": events_alive,
"email_alive": email_alive,
"alive": alive,
"email_initial_sync": email_initial_sync,
"events_initial_sync": events_initial_sync,
"initial_sync": initial_sync,
"provider_name": account.provider,
"email_address": account.email_address,
"folders": sorted(
folder_data[account.id].values(), key=itemgetter("name")
),
"calendars": sorted(
calendar_data[account.id], key=itemgetter("name")
),
"sync_email": account.sync_email,
"sync_events": account.sync_events,
"sync_status": sync_status_str,
"sync_error": sync_status.get("sync_error"),
"sync_end_time": sync_status.get("sync_end_time"),
"sync_disabled_reason": sync_status.get("sync_disabled_reason"),
"sync_host": account.sync_host,
"progress": progress,
"throttled": account.throttled,
"created_at": account.created_at,
"updated_at": account.updated_at,
}
)
return APIEncoder().jsonify(data)
