def start(port, start_syncback, enable_tracer, enable_profiler):
# We need to import this down here, because this in turn imports
# ignition.engine, which has to happen *after* we read any config overrides
# for the database parameters. Boo for imports with side-effects.
from inbox.api.srv import app
if start_syncback:
# start actions service
from inbox.transactions.actions import SyncbackService
if enable_profiler:
inbox_config["DEBUG_PROFILING_ON"] = True
enable_profiler_api = inbox_config.get("DEBUG_PROFILING_ON")
syncback = SyncbackService(0, 0, 1)
profiling_frontend = SyncbackHTTPFrontend(
int(port) + 1, enable_tracer, enable_profiler_api)
profiling_frontend.start()
syncback.start()
nylas_logger = get_logger()
http_server = WSGIServer(("", int(port)),
app,
log=nylas_logger,
handler_class=NylasWSGIHandler)
nylas_logger.info("Starting API server", port=port)
http_server.serve_forever()
if start_syncback:
syncback.join()
def receive_checkout(dbapi_connection, connection_record, connection_proxy):
"""Log checkedout and overflow when a connection is checked out"""
hostname = gethostname().replace(".", "-")
process_name = str(config.get("PROCESS_NAME", "main_process"))
if config.get("ENABLE_DB_TXN_METRICS", False):
statsd_client.gauge(
".".join(
["dbconn", database_name, hostname, process_name, "checkedout"]
),
connection_proxy._pool.checkedout(),
)
statsd_client.gauge(
".".join(["dbconn", database_name, hostname, process_name, "overflow"]),
connection_proxy._pool.overflow(),
)
# Keep track of where and why this connection was checked out.
log = get_logger()
context = log._context._dict.copy()
f, name = find_first_app_frame_and_name(
ignores=["sqlalchemy", "inbox.ignition", "inbox.logging"]
)
source = "{}:{}".format(name, f.f_lineno)
pool_tracker[dbapi_connection] = {
"source": source,
"context": context,
"checkedout_at": time.time(),
}
def __init__(self, account):
self.account_id = account.id
self.log = get_logger()
self.log.bind(account_id=account.id)
if isinstance(account, GenericAccount):
self.smtp_username = account.smtp_username
else:
# Non-generic accounts have no smtp username
self.smtp_username = account.email_address
self.email_address = account.email_address
self.provider_name = account.provider
self.sender_name = account.name
self.smtp_endpoint = account.smtp_endpoint
self.auth_type = provider_info(self.provider_name)["auth"]
if self.auth_type == "oauth2":
try:
self.auth_token = token_manager.get_token(account)
except OAuthError:
raise SendMailException(
"Could not authenticate with the SMTP server.", 403
)
else:
assert self.auth_type == "password"
if isinstance(account, GenericAccount):
self.auth_token = account.smtp_password
else:
# non-generic accounts have no smtp password
self.auth_token = account.password
def default_json_error(ex):
""" Exception -> flask JSON responder """
logger = get_logger()
logger.error("Uncaught error thrown by Flask/Werkzeug", exc_info=ex)
response = jsonify(message=str(ex), type="api_error")
response.status_code = ex.code if isinstance(ex, HTTPException) else 500
return response
def __init__(
self,
process_identifier,
process_number,
poll_interval=SYNC_POLL_INTERVAL,
exit_after_min=None,
exit_after_max=None,
):
self.keep_running = True
self.host = platform.node()
self.process_number = process_number
self.process_identifier = process_identifier
self.monitor_cls_for = {
mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS)
for mod in module_registry.values()
if hasattr(mod, "SYNC_MONITOR_CLS")
}
for p_name, _ in iteritems(providers):
if p_name not in self.monitor_cls_for:
self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"]
self.log = get_logger()
self.log.bind(process_number=process_number)
self.log.info("starting mail sync process",
supported_providers=list(module_registry))
self.syncing_accounts = set()
self.email_sync_monitors = {}
self.contact_sync_monitors = {}
self.event_sync_monitors = {}
# Randomize the poll_interval so we maintain at least a little fairness
# when using a timeout while blocking on the redis queues.
min_poll_interval = 5
self.poll_interval = int((random.random() *
(poll_interval - min_poll_interval)) +
min_poll_interval)
self.semaphore = BoundedSemaphore(1)
self.zone = config.get("ZONE")
# Note that we don't partition by zone for the private queues.
# There's not really a reason to since there's one queue per machine
# anyways. Also, if you really want to send an Account to a mailsync
# machine in another zone you can do so.
self.private_queue = EventQueue(
SYNC_EVENT_QUEUE_NAME.format(self.process_identifier))
self.queue_group = EventQueueGroup([
shared_sync_event_queue_for_zone(self.zone),
self.private_queue,
])
self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True)
self._pending_avgs_provider = None
self.last_unloaded_account = time.time()
if exit_after_min and exit_after_max:
exit_after = random.randint(exit_after_min * 60,
exit_after_max * 60)
self.log.info("exit after", seconds=exit_after)
gevent.spawn_later(exit_after, self.stop)
def test_use_starttls():
conn = SMTPConnection(
account_id=1,
email_address="*****@*****.**",
smtp_username="******",
auth_type="password",
auth_token="secret_password",
smtp_endpoint=("smtp.gmail.com", 587),
log=get_logger(),
)
assert isinstance(conn.connection, smtplib.SMTP)
def publish(self, **kwargs):
try:
self.heartbeat_at = time.time()
self.store.publish(self.key, self.heartbeat_at)
except Exception:
log = get_logger()
log.error(
"Error while writing the heartbeat status",
account_id=self.key.account_id,
folder_id=self.key.folder_id,
device_id=self.device_id,
exc_info=True,
)
def __init__(self,
gather_stats=False,
max_blocking_time=MAX_BLOCKING_TIME):
self.gather_stats = gather_stats
self.max_blocking_time = max_blocking_time
self.time_spent_by_context = collections.defaultdict(float)
self.total_switches = 0
self._last_switch_time = None
self._switch_flag = False
self._active_greenlet = None
self._main_thread_id = gevent._threading.get_thread_ident()
self._hub = gevent.hub.get_hub()
self.log = get_logger()
def test_use_smtp_over_ssl():
# Auth won't actually work but we just want to test connection
# initialization here and below.
SMTPConnection.smtp_password = mock.Mock()
conn = SMTPConnection(
account_id=1,
email_address="*****@*****.**",
smtp_username="******",
auth_type="password",
auth_token="secret_password",
smtp_endpoint=("smtp.gmail.com", 465),
log=get_logger(),
)
assert isinstance(conn.connection, smtplib.SMTP_SSL)
def log_uncaught_errors(logger=None, **kwargs):
"""
Helper to log uncaught exceptions.
Parameters
----------
logger: structlog.BoundLogger, optional
The logging object to write to.
"""
logger = logger or get_logger()
kwargs.update(create_error_log_context(sys.exc_info()))
logger.error("Uncaught error", **kwargs)
rollbar.report_exc_info()
def timed_fn(self, *args, **kwargs):
start_time = time.time()
ret = fn(self, *args, **kwargs)
# TODO some modules like gmail.py don't have self.logger
try:
if self.log:
fn_logger = self.log
except AttributeError:
fn_logger = get_logger()
# out = None
fn_logger.info("[timer] {0} took {1:.3f} seconds.".format(
str(fn), float(time.time() - start_time)))
return ret
def test_use_plain():
# ssl = True
with pytest.raises(SendMailException):
conn = SMTPConnection(
account_id=1,
email_address="*****@*****.**",
smtp_username="******",
auth_type="password",
auth_token="testpwd",
smtp_endpoint=("tivertical.com", 587),
log=get_logger(),
)
# ssl = False
conn = SMTPConnection(
account_id=1,
email_address="*****@*****.**",
smtp_username="******",
auth_type="password",
auth_token="testpwd",
smtp_endpoint=("tivertical.com", 587),
log=get_logger(),
)
assert isinstance(conn.connection, smtplib.SMTP)
def callback(e):
is_transient = isinstance(e, TRANSIENT_NETWORK_ERRS)
mysql_error = None
log = logger or get_logger()
if isinstance(e, _mysql_exceptions.OperationalError):
mysql_error = e
elif isinstance(e, StatementError) and isinstance(
e.orig, _mysql_exceptions.OperationalError):
mysql_error = e.orig
if mysql_error and mysql_error.args and isinstance(
mysql_error.args[0], str):
for msg in TRANSIENT_MYSQL_MESSAGES:
if msg in mysql_error.args[0]:
is_transient = True
if is_transient:
occurrences[0] += 1
if occurrences[0] < 20:
return
else:
occurrences[0] = 1
if account_id:
try:
with session_scope(account_id) as db_session:
account = db_session.query(Account).get(account_id)
sync_error = account.sync_error
if not sync_error or isinstance(sync_error, basestring):
account.update_sync_error(e)
db_session.commit()
except Exception:
log.error("Error saving sync_error to account object",
account_id=account_id,
**create_error_log_context(sys.exc_info()))
log_uncaught_errors(logger,
account_id=account_id,
provider=provider,
occurrences=occurrences[0])
def test_handle_disconnect(monkeypatch, smtp_port):
def simulate_disconnect(self):
raise smtplib.SMTPServerDisconnected()
monkeypatch.setattr("smtplib.SMTP.rset", simulate_disconnect)
monkeypatch.setattr("smtplib.SMTP.mail", lambda *args: (550, "NOPE"))
monkeypatch.setattr(
"inbox.sendmail.smtp.postel.SMTPConnection.smtp_password",
lambda *args: None)
conn = SMTPConnection(
account_id=1,
email_address="*****@*****.**",
smtp_username="******",
auth_type="password",
auth_token="secret_password",
smtp_endpoint=("smtp.gmail.com", smtp_port),
log=get_logger(),
)
with pytest.raises(smtplib.SMTPSenderRefused):
conn.sendmail(["*****@*****.**"], "hello there")
from inbox.logging import get_logger
from inbox.models.base import MailSyncBase
from inbox.models.constants import MAX_INDEXABLE_LENGTH
from inbox.models.mixins import (
CaseInsensitiveComparator,
DeletedAtMixin,
HasPublicID,
HasRevisions,
UpdatedAtMixin,
)
from inbox.sqlalchemy_ext.util import StringWithTransform
from inbox.util.encoding import unicode_safe_truncate
from inbox.util.misc import fs_folder_path
log = get_logger()
EPOCH = datetime.utcfromtimestamp(0)
def sanitize_name(name):
return unicode_safe_truncate(name, MAX_INDEXABLE_LENGTH)
class CategoryNameString(StringWithTransform):
"""
CategoryNameString is a Column type that extends our
sqlalchemy_ext.util.StringWithTransform to initialize it with the correct
sanitization procedure and the correct string length and collation we use
for category names.
# I need to figure out what to do with it. This is temporary for the sake of running tests
# on Python 3.
import gdata.auth
import gdata.client
import gdata.contacts.client
import gevent
from inbox.auth.google import GoogleAuthHandler
from inbox.basicauth import ConnectionError, OAuthError, ValidationError
from inbox.logging import get_logger
from inbox.models import Contact
from inbox.models.backends.gmail import GmailAccount
from inbox.models.backends.oauth import token_manager
from inbox.models.session import session_scope
logger = get_logger()
SOURCE_APP_NAME = "Nylas Sync Engine"
class GoogleContactsProvider(object):
"""
A utility class to fetch and parse Google contact data for the specified
account using the Google Contacts API.
Parameters
----------
db_session: sqlalchemy.orm.session.Session
Database session.
account: inbox.models.gmail.GmailAccount
def __init__(self, account):
self.account = account
self.account_id = account.id
self.log = get_logger().new(account_id=account.id, component="search")
def main(prod, enable_tracer, enable_profiler, config, process_num,
exit_after):
""" Launch the Nylas sync service. """
level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL"))
configure_logging(log_level=level)
reconfigure_logging()
maybe_enable_rollbar()
if config is not None:
from inbox.util.startup import load_overrides
config_path = os.path.abspath(config)
load_overrides(config_path)
if not prod:
preflight()
total_processes = int(os.environ.get("MAILSYNC_PROCESSES", 1))
setproctitle.setproctitle("sync-engine-{}".format(process_num))
log = get_logger()
log.info(
"start",
components=["mail sync", "contact sync", "calendar sync"],
host=platform.node(),
process_num=process_num,
total_processes=total_processes,
recursion_limit=sys.getrecursionlimit(),
)
print(banner, file=sys.stderr)
print(file=sys.stderr)
print("Python", sys.version, file=sys.stderr)
if enable_profiler:
inbox_config["DEBUG_PROFILING_ON"] = True
port = 16384 + process_num
enable_profiler_api = inbox_config.get("DEBUG_PROFILING_ON")
process_identifier = "{}:{}".format(platform.node(), process_num)
if exit_after:
exit_after = exit_after.split(":")
exit_after_min, exit_after_max = int(exit_after[0]), int(exit_after[1])
else:
exit_after_min, exit_after_max = None, None
sync_service = SyncService(
process_identifier,
process_num,
exit_after_min=exit_after_min,
exit_after_max=exit_after_max,
)
signal.signal(signal.SIGTERM, sync_service.stop)
signal.signal(signal.SIGINT, sync_service.stop)
http_frontend = SyncHTTPFrontend(sync_service, port, enable_tracer,
enable_profiler_api)
sync_service.register_pending_avgs_provider(http_frontend)
http_frontend.start()
sync_service.run()
print("\033[94mNylas Sync Engine exiting...\033[0m", file=sys.stderr)
from future import standard_library
standard_library.install_aliases()
import re
import socket
from urllib.parse import urlencode
import dns
from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers, Resolver, Timeout
from future.utils import iteritems
from tldextract import extract as tld_extract
from inbox.logging import get_logger
log = get_logger("inbox.util.url")
from inbox.providers import providers
# http://www.regular-expressions.info/email.html
EMAIL_REGEX = re.compile(r"[A-Z0-9._%+-]+@(?:[A-Z0-9-]+\.)+[A-Z]{2,4}",
re.IGNORECASE)
# Use Google's Public DNS server (8.8.8.8)
GOOGLE_DNS_IP = "8.8.8.8"
dns_resolver = Resolver()
dns_resolver.nameservers = [GOOGLE_DNS_IP]
class InvalidEmailAddressError(Exception):
pass
def _monitoring_thread(self):
# Logger needs to be instantiated in new thread.
self.log = get_logger()
while not self._stopping:
retry_with_logging(self._run_impl, self.log)
from __future__ import print_function
import click
from inbox.crispin import connection_pool
from inbox.error_handling import maybe_enable_rollbar
from inbox.logging import configure_logging, get_logger
from inbox.models.backends.generic import GenericAccount
from inbox.models.session import (
global_session_scope,
session_scope,
session_scope_by_shard_id,
)
configure_logging()
log = get_logger(purpose="separator-backfix")
@click.command()
@click.option("--min-id", type=int, default=None)
@click.option("--max-id", type=int, default=None)
@click.option("--shard-id", type=int, default=None)
def main(min_id, max_id, shard_id):
maybe_enable_rollbar()
generic_accounts = []
failed = []
if min_id is not None or max_id is not None:
# Get the list of running Gmail accounts.
with global_session_scope() as db_session:
# messages, they are consolidated into a single category and the other
# categories are deleted
from itertools import chain
import click
from sqlalchemy import func
from sqlalchemy.sql import and_, exists
from inbox.error_handling import maybe_enable_rollbar
from inbox.ignition import engine_manager
from inbox.logging import configure_logging, get_logger
from inbox.models import Category, MessageCategory
from inbox.models.session import session_scope_by_shard_id
configure_logging()
log = get_logger(purpose="duplicate-category-backfill")
def backfix_shard(shard_id, dry_run):
categories_to_fix = []
with session_scope_by_shard_id(shard_id) as db_session:
# 'SELECT id FROM <table> GROUP BY <x>' does not select _all_ of the
# ids in the group. MySQL chooses one id and returns it. The id chosen
# is indeterminate. So we find the duplicate
# (namespace_id, display_name, name) pairs and use them to query
# for specific Category rows
category_query = db_session.query(Category.namespace_id,
Category.display_name, Category.name)
duplicate_attrs = (category_query.group_by(
Category.display_name, Category.namespace_id,
from __future__ import division
import click
from sqlalchemy import asc
from inbox.contacts.processing import update_contacts_from_event
from inbox.error_handling import maybe_enable_rollbar
from inbox.ignition import engine_manager
from inbox.logging import configure_logging, get_logger
from inbox.models import Event
from inbox.models.session import session_scope_by_shard_id
from inbox.models.util import limitlion
configure_logging()
log = get_logger(purpose="create-event-contact-associations")
def process_shard(shard_id, dry_run, id_start=0):
# At 500K events, we need to process 6 events per second to finish within a day.
batch_size = 100
rps = 6 / batch_size
window = 5
throttle = limitlion.throttle_wait("create-event-contact-associations",
rps=rps,
window=window)
with session_scope_by_shard_id(shard_id) as db_session:
# NOTE: The session is implicitly autoflushed, which ensures no
# duplicate contacts are created.
