def poll(self, event): # Determine which accounts to sync start_accounts = self.account_ids_to_sync() statsd_client.gauge( "mailsync.account_counts.{}.mailsync-{}.count".format( self.host, self.process_number), len(start_accounts), ) # Perform the appropriate action on each account for account_id in start_accounts: if account_id not in self.syncing_accounts: try: self.start_sync(account_id) except OperationalError: self.log.error("Database error starting account sync", exc_info=True) log_uncaught_errors() stop_accounts = self.account_ids_owned() - set(start_accounts) for account_id in stop_accounts: self.log.info("sync service stopping sync", account_id=account_id) try: self.stop_sync(account_id) except OperationalError: self.log.error("Database error stopping account sync", exc_info=True) log_uncaught_errors()
def _run(self): """ Index into CloudSearch the contacts of all namespaces. """ try: self._set_transaction_pointers() self.log.info( "Starting contact-search-index service", transaction_pointers=self.transaction_pointers, ) while True: self._publish_heartbeat() self._index_transactions() except Exception: log_uncaught_errors(log)
def callback(e): is_transient = isinstance(e, TRANSIENT_NETWORK_ERRS) mysql_error = None log = logger or get_logger() if isinstance(e, _mysql_exceptions.OperationalError): mysql_error = e elif isinstance(e, StatementError) and isinstance( e.orig, _mysql_exceptions.OperationalError): mysql_error = e.orig if mysql_error and mysql_error.args and isinstance( mysql_error.args[0], str): for msg in TRANSIENT_MYSQL_MESSAGES: if msg in mysql_error.args[0]: is_transient = True if is_transient: occurrences[0] += 1 if occurrences[0] < 20: return else: occurrences[0] = 1 if account_id: try: with session_scope(account_id) as db_session: account = db_session.query(Account).get(account_id) sync_error = account.sync_error if not sync_error or isinstance(sync_error, basestring): account.update_sync_error(e) db_session.commit() except Exception: log.error("Error saving sync_error to account object", account_id=account_id, **create_error_log_context(sys.exc_info())) log_uncaught_errors(logger, account_id=account_id, provider=provider, occurrences=occurrences[0])
def batch_delete_namespaces(ids_to_delete, throttle=False, dry_run=False): start = time.time() for account_id, namespace_id in ids_to_delete: # try: try: delete_namespace(namespace_id, throttle=throttle, dry_run=dry_run) except AccountDeletionErrror as e: message = e.args[0] if e.args else "" log.critical("AccountDeletionErrror", error_message=message) except Exception: log_uncaught_errors(log, account_id=account_id) end = time.time() log.info( "All data deleted successfully for ids", ids_to_delete=ids_to_delete, time=end - start, count=len(ids_to_delete), )
def execute_with_lock(self): """ Process a task and return whether it executed successfully. """ self.log = logger.new( record_ids=list(set(self.record_ids)), action_log_ids=self.action_log_ids[:100], n_action_log_ids=len(self.action_log_ids), action=self.action_name, account_id=self.account_id, extra_args=self.extra_args, ) # Double-check that the action is still pending. # Although the task queue is populated based on pending actions, it's # possible that the processing of one action involved marking other # actions as failed. ( records_to_process, action_ids_to_process, ) = self._get_records_and_actions_to_process() if len(action_ids_to_process) == 0: return True try: before, after = self._execute_timed_action(records_to_process) self.log.debug("executing action", action_log_ids=action_ids_to_process) with session_scope(self.account_id) as db_session: action_log_entries = db_session.query(ActionLog).filter( ActionLog.id.in_(action_ids_to_process)) max_latency = max_func_latency = 0 for action_log_entry in action_log_entries: latency, func_latency = self._mark_action_as_successful( action_log_entry, before, after, db_session) if latency > max_latency: max_latency = latency if func_latency > max_func_latency: max_func_latency = func_latency self.log.info( "syncback action completed", latency=max_latency, process=self.parent_service().process_number, func_latency=max_func_latency, ) return True except Exception: log_uncaught_errors(self.log, account_id=self.account_id, provider=self.provider) with session_scope(self.account_id) as db_session: action_log_entries = db_session.query(ActionLog).filter( ActionLog.id.in_(action_ids_to_process)) marked_as_failed = False for action_log_entry in action_log_entries: action_log_entry.retries += 1 if action_log_entry.retries == ACTION_MAX_NR_OF_RETRIES: marked_as_failed = True if marked_as_failed: self.log.debug( "marking actions as failed", action_log_ids=action_ids_to_process, ) # If we merged actions, fail them all at the same time. for action_log_entry in action_log_entries: self._mark_action_as_failed(action_log_entry, db_session) db_session.commit() return False
def f(session): if obj_state["sent_event"]: return id = obj_state["id"] sync_should_run = obj_state["sync_should_run"] sync_host = obj_state["sync_host"] desired_sync_host = obj_state["desired_sync_host"] try: if sync_host is not None: # Somebody is actively syncing this Account, so notify them if # they should give up the Account. if not sync_should_run or (sync_host != desired_sync_host and desired_sync_host is not None): queue_name = SYNC_EVENT_QUEUE_NAME.format(sync_host) log.info( "Sending 'migrate_from' event for Account", account_id=id, queue_name=queue_name, ) EventQueue(queue_name).send_event({ "event": "migrate_from", "id": id }) return if not sync_should_run: # We don't need to notify anybody because the Account is not # actively being synced (sync_host is None) and sync_should_run is False, # so just return early. return if desired_sync_host is not None: # Nobody is actively syncing the Account, and we have somebody # who wants to sync this Account, so notify them. queue_name = SYNC_EVENT_QUEUE_NAME.format( desired_sync_host) log.info( "Sending 'migrate_to' event for Account", account_id=id, queue_name=queue_name, ) EventQueue(queue_name).send_event({ "event": "migrate_to", "id": id }) return # Nobody is actively syncing the Account, and nobody in particular # wants to sync the Account so notify the shared queue. shared_queue = shared_sync_event_queue_for_zone( config.get("ZONE")) log.info( "Sending 'migrate' event for Account", account_id=id, queue_name=shared_queue.queue_name, ) shared_queue.send_event({"event": "migrate", "id": id}) obj_state["sent_event"] = True except Exception: log_uncaught_errors( log, account_id=id, sync_host=sync_host, desired_sync_host=desired_sync_host, )