def start(port, start_syncback, enable_tracer, enable_profiler): # We need to import this down here, because this in turn imports # ignition.engine, which has to happen *after* we read any config overrides # for the database parameters. Boo for imports with side-effects. from inbox.api.srv import app if start_syncback: # start actions service from inbox.transactions.actions import SyncbackService if enable_profiler: inbox_config["DEBUG_PROFILING_ON"] = True enable_profiler_api = inbox_config.get("DEBUG_PROFILING_ON") syncback = SyncbackService(0, 0, 1) profiling_frontend = SyncbackHTTPFrontend( int(port) + 1, enable_tracer, enable_profiler_api) profiling_frontend.start() syncback.start() nylas_logger = get_logger() http_server = WSGIServer(("", int(port)), app, log=nylas_logger, handler_class=NylasWSGIHandler) nylas_logger.info("Starting API server", port=port) http_server.serve_forever() if start_syncback: syncback.join()
def receive_checkout(dbapi_connection, connection_record, connection_proxy): """Log checkedout and overflow when a connection is checked out""" hostname = gethostname().replace(".", "-") process_name = str(config.get("PROCESS_NAME", "main_process")) if config.get("ENABLE_DB_TXN_METRICS", False): statsd_client.gauge( ".".join( ["dbconn", database_name, hostname, process_name, "checkedout"] ), connection_proxy._pool.checkedout(), ) statsd_client.gauge( ".".join(["dbconn", database_name, hostname, process_name, "overflow"]), connection_proxy._pool.overflow(), ) # Keep track of where and why this connection was checked out. log = get_logger() context = log._context._dict.copy() f, name = find_first_app_frame_and_name( ignores=["sqlalchemy", "inbox.ignition", "inbox.logging"] ) source = "{}:{}".format(name, f.f_lineno) pool_tracker[dbapi_connection] = { "source": source, "context": context, "checkedout_at": time.time(), }
def __init__(self, account): self.account_id = account.id self.log = get_logger() self.log.bind(account_id=account.id) if isinstance(account, GenericAccount): self.smtp_username = account.smtp_username else: # Non-generic accounts have no smtp username self.smtp_username = account.email_address self.email_address = account.email_address self.provider_name = account.provider self.sender_name = account.name self.smtp_endpoint = account.smtp_endpoint self.auth_type = provider_info(self.provider_name)["auth"] if self.auth_type == "oauth2": try: self.auth_token = token_manager.get_token(account) except OAuthError: raise SendMailException( "Could not authenticate with the SMTP server.", 403 ) else: assert self.auth_type == "password" if isinstance(account, GenericAccount): self.auth_token = account.smtp_password else: # non-generic accounts have no smtp password self.auth_token = account.password
def default_json_error(ex): """ Exception -> flask JSON responder """ logger = get_logger() logger.error("Uncaught error thrown by Flask/Werkzeug", exc_info=ex) response = jsonify(message=str(ex), type="api_error") response.status_code = ex.code if isinstance(ex, HTTPException) else 500 return response
def __init__( self, process_identifier, process_number, poll_interval=SYNC_POLL_INTERVAL, exit_after_min=None, exit_after_max=None, ): self.keep_running = True self.host = platform.node() self.process_number = process_number self.process_identifier = process_identifier self.monitor_cls_for = { mod.PROVIDER: getattr(mod, mod.SYNC_MONITOR_CLS) for mod in module_registry.values() if hasattr(mod, "SYNC_MONITOR_CLS") } for p_name, _ in iteritems(providers): if p_name not in self.monitor_cls_for: self.monitor_cls_for[p_name] = self.monitor_cls_for["generic"] self.log = get_logger() self.log.bind(process_number=process_number) self.log.info("starting mail sync process", supported_providers=list(module_registry)) self.syncing_accounts = set() self.email_sync_monitors = {} self.contact_sync_monitors = {} self.event_sync_monitors = {} # Randomize the poll_interval so we maintain at least a little fairness # when using a timeout while blocking on the redis queues. min_poll_interval = 5 self.poll_interval = int((random.random() * (poll_interval - min_poll_interval)) + min_poll_interval) self.semaphore = BoundedSemaphore(1) self.zone = config.get("ZONE") # Note that we don't partition by zone for the private queues. # There's not really a reason to since there's one queue per machine # anyways. Also, if you really want to send an Account to a mailsync # machine in another zone you can do so. self.private_queue = EventQueue( SYNC_EVENT_QUEUE_NAME.format(self.process_identifier)) self.queue_group = EventQueueGroup([ shared_sync_event_queue_for_zone(self.zone), self.private_queue, ]) self.stealing_enabled = config.get("SYNC_STEAL_ACCOUNTS", True) self._pending_avgs_provider = None self.last_unloaded_account = time.time() if exit_after_min and exit_after_max: exit_after = random.randint(exit_after_min * 60, exit_after_max * 60) self.log.info("exit after", seconds=exit_after) gevent.spawn_later(exit_after, self.stop)
def test_use_starttls(): conn = SMTPConnection( account_id=1, email_address="*****@*****.**", smtp_username="******", auth_type="password", auth_token="secret_password", smtp_endpoint=("smtp.gmail.com", 587), log=get_logger(), ) assert isinstance(conn.connection, smtplib.SMTP)
def publish(self, **kwargs): try: self.heartbeat_at = time.time() self.store.publish(self.key, self.heartbeat_at) except Exception: log = get_logger() log.error( "Error while writing the heartbeat status", account_id=self.key.account_id, folder_id=self.key.folder_id, device_id=self.device_id, exc_info=True, )
def __init__(self, gather_stats=False, max_blocking_time=MAX_BLOCKING_TIME): self.gather_stats = gather_stats self.max_blocking_time = max_blocking_time self.time_spent_by_context = collections.defaultdict(float) self.total_switches = 0 self._last_switch_time = None self._switch_flag = False self._active_greenlet = None self._main_thread_id = gevent._threading.get_thread_ident() self._hub = gevent.hub.get_hub() self.log = get_logger()
def test_use_smtp_over_ssl(): # Auth won't actually work but we just want to test connection # initialization here and below. SMTPConnection.smtp_password = mock.Mock() conn = SMTPConnection( account_id=1, email_address="*****@*****.**", smtp_username="******", auth_type="password", auth_token="secret_password", smtp_endpoint=("smtp.gmail.com", 465), log=get_logger(), ) assert isinstance(conn.connection, smtplib.SMTP_SSL)
def log_uncaught_errors(logger=None, **kwargs): """ Helper to log uncaught exceptions. Parameters ---------- logger: structlog.BoundLogger, optional The logging object to write to. """ logger = logger or get_logger() kwargs.update(create_error_log_context(sys.exc_info())) logger.error("Uncaught error", **kwargs) rollbar.report_exc_info()
def timed_fn(self, *args, **kwargs): start_time = time.time() ret = fn(self, *args, **kwargs) # TODO some modules like gmail.py don't have self.logger try: if self.log: fn_logger = self.log except AttributeError: fn_logger = get_logger() # out = None fn_logger.info("[timer] {0} took {1:.3f} seconds.".format( str(fn), float(time.time() - start_time))) return ret
def test_use_plain(): # ssl = True with pytest.raises(SendMailException): conn = SMTPConnection( account_id=1, email_address="*****@*****.**", smtp_username="******", auth_type="password", auth_token="testpwd", smtp_endpoint=("tivertical.com", 587), log=get_logger(), ) # ssl = False conn = SMTPConnection( account_id=1, email_address="*****@*****.**", smtp_username="******", auth_type="password", auth_token="testpwd", smtp_endpoint=("tivertical.com", 587), log=get_logger(), ) assert isinstance(conn.connection, smtplib.SMTP)
def callback(e): is_transient = isinstance(e, TRANSIENT_NETWORK_ERRS) mysql_error = None log = logger or get_logger() if isinstance(e, _mysql_exceptions.OperationalError): mysql_error = e elif isinstance(e, StatementError) and isinstance( e.orig, _mysql_exceptions.OperationalError): mysql_error = e.orig if mysql_error and mysql_error.args and isinstance( mysql_error.args[0], str): for msg in TRANSIENT_MYSQL_MESSAGES: if msg in mysql_error.args[0]: is_transient = True if is_transient: occurrences[0] += 1 if occurrences[0] < 20: return else: occurrences[0] = 1 if account_id: try: with session_scope(account_id) as db_session: account = db_session.query(Account).get(account_id) sync_error = account.sync_error if not sync_error or isinstance(sync_error, basestring): account.update_sync_error(e) db_session.commit() except Exception: log.error("Error saving sync_error to account object", account_id=account_id, **create_error_log_context(sys.exc_info())) log_uncaught_errors(logger, account_id=account_id, provider=provider, occurrences=occurrences[0])
def test_handle_disconnect(monkeypatch, smtp_port): def simulate_disconnect(self): raise smtplib.SMTPServerDisconnected() monkeypatch.setattr("smtplib.SMTP.rset", simulate_disconnect) monkeypatch.setattr("smtplib.SMTP.mail", lambda *args: (550, "NOPE")) monkeypatch.setattr( "inbox.sendmail.smtp.postel.SMTPConnection.smtp_password", lambda *args: None) conn = SMTPConnection( account_id=1, email_address="*****@*****.**", smtp_username="******", auth_type="password", auth_token="secret_password", smtp_endpoint=("smtp.gmail.com", smtp_port), log=get_logger(), ) with pytest.raises(smtplib.SMTPSenderRefused): conn.sendmail(["*****@*****.**"], "hello there")
from inbox.logging import get_logger from inbox.models.base import MailSyncBase from inbox.models.constants import MAX_INDEXABLE_LENGTH from inbox.models.mixins import ( CaseInsensitiveComparator, DeletedAtMixin, HasPublicID, HasRevisions, UpdatedAtMixin, ) from inbox.sqlalchemy_ext.util import StringWithTransform from inbox.util.encoding import unicode_safe_truncate from inbox.util.misc import fs_folder_path log = get_logger() EPOCH = datetime.utcfromtimestamp(0) def sanitize_name(name): return unicode_safe_truncate(name, MAX_INDEXABLE_LENGTH) class CategoryNameString(StringWithTransform): """ CategoryNameString is a Column type that extends our sqlalchemy_ext.util.StringWithTransform to initialize it with the correct sanitization procedure and the correct string length and collation we use for category names.
# I need to figure out what to do with it. This is temporary for the sake of running tests # on Python 3. import gdata.auth import gdata.client import gdata.contacts.client import gevent from inbox.auth.google import GoogleAuthHandler from inbox.basicauth import ConnectionError, OAuthError, ValidationError from inbox.logging import get_logger from inbox.models import Contact from inbox.models.backends.gmail import GmailAccount from inbox.models.backends.oauth import token_manager from inbox.models.session import session_scope logger = get_logger() SOURCE_APP_NAME = "Nylas Sync Engine" class GoogleContactsProvider(object): """ A utility class to fetch and parse Google contact data for the specified account using the Google Contacts API. Parameters ---------- db_session: sqlalchemy.orm.session.Session Database session. account: inbox.models.gmail.GmailAccount
def __init__(self, account): self.account = account self.account_id = account.id self.log = get_logger().new(account_id=account.id, component="search")
def main(prod, enable_tracer, enable_profiler, config, process_num, exit_after): """ Launch the Nylas sync service. """ level = os.environ.get("LOGLEVEL", inbox_config.get("LOGLEVEL")) configure_logging(log_level=level) reconfigure_logging() maybe_enable_rollbar() if config is not None: from inbox.util.startup import load_overrides config_path = os.path.abspath(config) load_overrides(config_path) if not prod: preflight() total_processes = int(os.environ.get("MAILSYNC_PROCESSES", 1)) setproctitle.setproctitle("sync-engine-{}".format(process_num)) log = get_logger() log.info( "start", components=["mail sync", "contact sync", "calendar sync"], host=platform.node(), process_num=process_num, total_processes=total_processes, recursion_limit=sys.getrecursionlimit(), ) print(banner, file=sys.stderr) print(file=sys.stderr) print("Python", sys.version, file=sys.stderr) if enable_profiler: inbox_config["DEBUG_PROFILING_ON"] = True port = 16384 + process_num enable_profiler_api = inbox_config.get("DEBUG_PROFILING_ON") process_identifier = "{}:{}".format(platform.node(), process_num) if exit_after: exit_after = exit_after.split(":") exit_after_min, exit_after_max = int(exit_after[0]), int(exit_after[1]) else: exit_after_min, exit_after_max = None, None sync_service = SyncService( process_identifier, process_num, exit_after_min=exit_after_min, exit_after_max=exit_after_max, ) signal.signal(signal.SIGTERM, sync_service.stop) signal.signal(signal.SIGINT, sync_service.stop) http_frontend = SyncHTTPFrontend(sync_service, port, enable_tracer, enable_profiler_api) sync_service.register_pending_avgs_provider(http_frontend) http_frontend.start() sync_service.run() print("\033[94mNylas Sync Engine exiting...\033[0m", file=sys.stderr)
from future import standard_library standard_library.install_aliases() import re import socket from urllib.parse import urlencode import dns from dns.resolver import NXDOMAIN, NoAnswer, NoNameservers, Resolver, Timeout from future.utils import iteritems from tldextract import extract as tld_extract from inbox.logging import get_logger log = get_logger("inbox.util.url") from inbox.providers import providers # http://www.regular-expressions.info/email.html EMAIL_REGEX = re.compile(r"[A-Z0-9._%+-]+@(?:[A-Z0-9-]+\.)+[A-Z]{2,4}", re.IGNORECASE) # Use Google's Public DNS server (8.8.8.8) GOOGLE_DNS_IP = "8.8.8.8" dns_resolver = Resolver() dns_resolver.nameservers = [GOOGLE_DNS_IP] class InvalidEmailAddressError(Exception): pass
def _monitoring_thread(self): # Logger needs to be instantiated in new thread. self.log = get_logger() while not self._stopping: retry_with_logging(self._run_impl, self.log)
from __future__ import print_function import click from inbox.crispin import connection_pool from inbox.error_handling import maybe_enable_rollbar from inbox.logging import configure_logging, get_logger from inbox.models.backends.generic import GenericAccount from inbox.models.session import ( global_session_scope, session_scope, session_scope_by_shard_id, ) configure_logging() log = get_logger(purpose="separator-backfix") @click.command() @click.option("--min-id", type=int, default=None) @click.option("--max-id", type=int, default=None) @click.option("--shard-id", type=int, default=None) def main(min_id, max_id, shard_id): maybe_enable_rollbar() generic_accounts = [] failed = [] if min_id is not None or max_id is not None: # Get the list of running Gmail accounts. with global_session_scope() as db_session:
# messages, they are consolidated into a single category and the other # categories are deleted from itertools import chain import click from sqlalchemy import func from sqlalchemy.sql import and_, exists from inbox.error_handling import maybe_enable_rollbar from inbox.ignition import engine_manager from inbox.logging import configure_logging, get_logger from inbox.models import Category, MessageCategory from inbox.models.session import session_scope_by_shard_id configure_logging() log = get_logger(purpose="duplicate-category-backfill") def backfix_shard(shard_id, dry_run): categories_to_fix = [] with session_scope_by_shard_id(shard_id) as db_session: # 'SELECT id FROM <table> GROUP BY <x>' does not select _all_ of the # ids in the group. MySQL chooses one id and returns it. The id chosen # is indeterminate. So we find the duplicate # (namespace_id, display_name, name) pairs and use them to query # for specific Category rows category_query = db_session.query(Category.namespace_id, Category.display_name, Category.name) duplicate_attrs = (category_query.group_by( Category.display_name, Category.namespace_id,
from __future__ import division import click from sqlalchemy import asc from inbox.contacts.processing import update_contacts_from_event from inbox.error_handling import maybe_enable_rollbar from inbox.ignition import engine_manager from inbox.logging import configure_logging, get_logger from inbox.models import Event from inbox.models.session import session_scope_by_shard_id from inbox.models.util import limitlion configure_logging() log = get_logger(purpose="create-event-contact-associations") def process_shard(shard_id, dry_run, id_start=0): # At 500K events, we need to process 6 events per second to finish within a day. batch_size = 100 rps = 6 / batch_size window = 5 throttle = limitlion.throttle_wait("create-event-contact-associations", rps=rps, window=window) with session_scope_by_shard_id(shard_id) as db_session: # NOTE: The session is implicitly autoflushed, which ensures no # duplicate contacts are created.