def _run(self): self.log.info('Starting syncback service', process_num=self.process_number, total_processes=self.total_processes, keys=self.keys) while self.keep_running: retry_with_logging(self._run_impl, self.log)
def test_no_logging_on_greenlet_exit(): logger = MockLogger() failing_function = FailingFunction(GreenletExit) with pytest.raises(GreenletExit): retry_with_logging(failing_function, logger=logger) assert logger.call_count == 0 assert failing_function.call_count == 1
def test_logging_on_critical_error(): critical = [ TypeError("Example TypeError"), StatementError(message="?", statement="SELECT *", params={}, orig=None), StatementError( message="?", statement="SELECT *", params={}, orig=_mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Incorrect string value " "'\\xE7\\x(a\\x84\\xE5'"), ), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Incorrect string value " "'\\xE7\\x(a\\x84\\xE5'"), _mysql_exceptions.IntegrityError( "(_mysql_exceptions.IntegrityError) Column not found"), ] for critical_exc in critical: logger = MockLogger() failing_function = FailingFunction(critical_exc, max_executions=2) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 1, "{} should be logged".format( critical_exc) assert failing_function.call_count == 2
def _run(self): # Bind greenlet-local logging context. self.log = log.new(account_id=self.account_id, folder=self.folder_name, provider=self.provider_name) # eagerly signal the sync status self.heartbeat_status.publish() try: self.update_folder_sync_status(lambda s: s.start_sync()) except IntegrityError: # The state insert failed because the folder ID ForeignKey # was no longer valid, ie. the folder for this engine was deleted # while we were starting up. # Exit the sync and let the monitor sort things out. log.info("Folder state loading failed due to IntegrityError", folder_id=self.folder_id, account_id=self.account_id) raise MailsyncDone() # NOTE: The parent ImapSyncMonitor handler could kill us at any # time if it receives a shutdown command. The shutdown command is # equivalent to ctrl-c. while True: retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=log)
def test_retry_with_logging(): logger = MockLogger() failing_function = FailingFunction(ValueError) with pytest.raises(ValueError): retry_with_logging(failing_function, logger=logger) assert logger.call_count == 3 assert failing_function.call_count == 3
def test_retry_with_logging(): logger = MockLogger() failing_function = FailingFunction(ValueError) with pytest.raises(ValueError): retry_with_logging(failing_function, logger=logger, max_count=3, backoff_delay=0) assert logger.call_count == 4 assert failing_function.call_count == 3
def test_selective_retry(): logger = MockLogger() failing_function = FailingFunction(ValueError) with pytest.raises(ValueError): retry_with_logging(failing_function, logger=logger, fail_classes=[ValueError]) assert logger.call_count == 1 assert failing_function.call_count == 1
def run(self): if config.get('DEBUG_PROFILING_ON'): # If config flag is set, get live top-level profiling output on # stdout by doing kill -SIGTRAP <sync_process>. # This slows things down so you probably don't want to do it # normally. attach_profiler() setproctitle('inbox-sync-{}'.format(self.cpu_id)) retry_with_logging(self._run_impl, self.log)
def test_selective_retry(): logger = MockLogger() failing_function = FailingFunction(ValueError) with pytest.raises(ValueError): retry_with_logging(failing_function, logger=logger, fail_classes=[ValueError]) assert logger.call_count == 0 assert failing_function.call_count == 1
def run(self): if config.get('DEBUG_CONSOLE_ON'): # Enable the debugging console if this flag is set. Connect to # localhost on the port shown in the logs to get access to a REPL port = None start_port = config.get('DEBUG_START_PORT') if start_port: port = start_port + self.cpu_id gevent.spawn(break_to_interpreter, port=port) setproctitle('inbox-sync-{}'.format(self.cpu_id)) retry_with_logging(self._run_impl, self.log)
def _run(self): # Bind greenlet-local logging context. self.log = log.new( account_id=self.account_id, folder=self.folder_name, provider=self.provider_name, ) # eagerly signal the sync status self.heartbeat_status.publish() def start_sync(saved_folder_status): # Ensure we don't cause an error if the folder was deleted. sync_end_time = (saved_folder_status.folder and saved_folder_status.metrics.get("sync_end_time")) if sync_end_time: sync_delay = datetime.utcnow() - sync_end_time if sync_delay > timedelta(days=1): saved_folder_status.state = "initial" log.info( "switching to initial sync due to delay", folder_id=self.folder_id, account_id=self.account_id, sync_delay=sync_delay.total_seconds(), ) saved_folder_status.start_sync() try: self.update_folder_sync_status(start_sync) except IntegrityError: # The state insert failed because the folder ID ForeignKey # was no longer valid, ie. the folder for this engine was deleted # while we were starting up. # Exit the sync and let the monitor sort things out. log.info( "Folder state loading failed due to IntegrityError", folder_id=self.folder_id, account_id=self.account_id, ) raise MailsyncDone() # NOTE: The parent ImapSyncMonitor handler could kill us at any # time if it receives a shutdown command. The shutdown command is # equivalent to ctrl-c. while self.state != "finish": retry_with_logging( self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=log, )
def test_no_logging_until_many_transient_error(): logger = MockLogger() failing_function = FailingFunction(socket.error, max_executions=2) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 0 assert failing_function.call_count == 2 failing_function = FailingFunction(socket.error, max_executions=21) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 1 assert failing_function.call_count == 21
def _run(self): # Bind greenlet-local logging context. self.log = self.log.new(account_id=self.account_id) try: while True: retry_with_logging(self._run_impl, account_id=self.account_id, fail_classes=[ValidationError], provider=self.provider_name, logger=self.log) except ValidationError: # Bad account credentials; exit. self.log.error('Credential validation error; exiting', exc_info=True, logstash_tag='mark_invalid') with session_scope(self.namespace_id) as db_session: account = db_session.query(Account).get(self.account_id) account.mark_invalid(scope=self.scope)
def _run(self): # Bind greenlet-local logging context. self.log = log.new(account_id=self.account_id, folder=self.folder_name, provider=self.provider_name, program='s3_sync') # eagerly signal the sync status return retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=log)
def _run(self): # Bind greenlet-local logging context. self.log = log.new(account_id=self.account_id, folder=self.folder_name) # eagerly signal the sync status self.heartbeat_status.publish() return retry_with_logging(self._run_impl, account_id=self.account_id, logger=log)
def _run(self): # Bind greenlet-local logging context. self.log = log.new(account_id=self.account_id, folder=self.folder_name, provider=self.provider_name) # eagerly signal the sync status self.heartbeat_status.publish() return retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=log)
def test_no_logging_until_many_transient_error(): transient = [ socket.timeout, socket.error, _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) (1213, 'Deadlock " "found when trying to get lock; try restarting transaction')"), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Lost connection to MySQL " "server during query"), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) MySQL server has gone away." ), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Can't connect to MySQL " "server on 127.0.0.1"), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Max connect timeout reached " "while reaching hostgroup 71"), StatementError( message="?", statement="SELECT *", params={}, orig=_mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) MySQL server has gone away." ), ), ] for transient_exc in transient: logger = MockLogger() failing_function = FailingFunction(transient_exc, max_executions=2) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 0, "{} should not be logged".format( transient_exc) assert failing_function.call_count == 2 failing_function = FailingFunction(socket.error, max_executions=21) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 1 assert failing_function.call_count == 21 failing_function = FailingFunction(socket.error, max_executions=2)
def _run(self): try: return retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=self.log) except GreenletExit: self._cleanup() raise
def run(self): if config.get('DEBUG_PROFILING_ON'): # If config flag is set, get live top-level profiling output on # stdout by doing kill -SIGTRAP <sync_process>. # This slows things down so you probably don't want to do it # normally. attach_profiler() if config.get('DEBUG_CONSOLE_ON'): # Enable the debugging console if this flag is set. Connect to # localhost on port $PID to get access to a REPL port = None start_port = config.get('DEBUG_START_PORT') if start_port: port = start_port + self.cpu_id gevent.spawn(break_to_interpreter, port=port) setproctitle('inbox-sync-{}'.format(self.cpu_id)) retry_with_logging(self._run_impl, self.log)
def run(self): if config.get('DEBUG_PROFILING_ON'): # If config flag is set, get live top-level profiling output on # stdout by doing kill -SIGTRAP <sync_process>. # This slows things down so you probably don't want to do it # normally. attach_profiler() if config.get('DEBUG_CONSOLE_ON'): # Enable the debugging console if this flag is set. Connect to # localhost on the port shown in the logs to get access to a REPL port = None start_port = config.get('DEBUG_START_PORT') if start_port: port = start_port + self.cpu_id gevent.spawn(break_to_interpreter, port=port) setproctitle('inbox-sync-{}'.format(self.cpu_id)) retry_with_logging(self._run_impl, self.log)
def test_no_logging_until_many_transient_error(): transient = [ socket.timeout, socket.error, _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) (1213, 'Deadlock " "found when trying to get lock; try restarting transaction')"), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Lost connection to MySQL " "server during query"), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) MySQL server has gone away."), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Can't connect to MySQL " "server on 127.0.0.1"), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Max connect timeout reached " "while reaching hostgroup 71"), StatementError( message="?", statement="SELECT *", params={}, orig=_mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) MySQL server has gone away.")), ] for transient_exc in transient: logger = MockLogger() failing_function = FailingFunction(transient_exc, max_executions=2) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 0, '{} should not be logged'.format(transient_exc) assert failing_function.call_count == 2 failing_function = FailingFunction(socket.error, max_executions=21) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 1 assert failing_function.call_count == 21 failing_function = FailingFunction(socket.error, max_executions=2)
def test_logging_on_critical_error(): critical = [ TypeError("Example TypeError"), StatementError( message="?", statement="SELECT *", params={}, orig=None), StatementError( message="?", statement="SELECT *", params={}, orig=_mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Incorrect string value " "'\\xE7\\x(a\\x84\\xE5'")), _mysql_exceptions.OperationalError( "(_mysql_exceptions.OperationalError) Incorrect string value " "'\\xE7\\x(a\\x84\\xE5'"), _mysql_exceptions.IntegrityError( "(_mysql_exceptions.IntegrityError) Column not found"), ] for critical_exc in critical: logger = MockLogger() failing_function = FailingFunction(critical_exc, max_executions=2) retry_with_logging(failing_function, logger=logger) assert logger.call_count == 1, '{} should be logged'.format(critical_exc) assert failing_function.call_count == 2
def _run(self): retry_with_logging(self._run_impl, self.log)
def _run(self): return retry_with_logging(self._run_impl, self.log)
def test_retry_with_logging(): logger = MockLogger() failing_function = FailingFunction(ValueError) retry_with_logging(failing_function, logger=logger, backoff_delay=0) assert logger.call_count == failing_function.max_executions - 1 assert failing_function.call_count == failing_function.max_executions
def _run(self): while True: retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name)
def run(self): log.info('Queueing accounts', zone=self.zone, shards=self.shards) while True: retry_with_logging(self._run_impl)
def run(self): return retry_with_logging(self._run_impl)
def _run(self): while True: retry_with_logging(self._run_impl)
def _run(self): # Bind greenlet-local logging context. self.log = self.log.new(account_id=self.account_id) return retry_with_logging(self._run_impl, account_id=self.account_id, logger=self.log)
def _run(self): gevent.spawn(retry_with_logging, self.retry_failed, self.log) retry_with_logging(self._run_impl, self.log)
def _monitoring_thread(self): # Logger needs to be instantiated in new thread. self.log = get_logger() while True: retry_with_logging(self._run_impl, self.log)
def _run(self): return retry_with_logging(self._run_impl, account_id=self.account_id, provider=self.provider_name, logger=self.log)
def run(self): while self.keep_running: retry_with_logging(self._run_impl, self.log)
def run(self): log.info("Queueing accounts", zone=self.zone, shards=self.shards) while True: retry_with_logging(self._run_impl)
def _run(self): return retry_with_logging(self._run_impl, account_id=self.account_id)
def run(self): while True: retry_with_logging(self._run_impl, self.log)