def _main_loop(self): # terminate gracefully on either SIGINT or SIGTERM signal.signal(signal.SIGINT, self._sigint_sigterm_handler) signal.signal(signal.SIGTERM, self._sigint_sigterm_handler) # in case at least one filter has been set up, we enter an infinite loop and let # the callbacks do the job. in case of no filters, we will not enter this loop # and the keeper will terminate soon after it started while any_filter_thread_present() or self._at_least_one_every: time.sleep(1) # if the keeper logic asked us to terminate, we do so if self.terminated_internally: self.logger.warning( "Keeper logic asked for termination, the keeper will terminate" ) break # if SIGINT/SIGTERM asked us to terminate, we do so if self.terminated_externally: self.logger.warning( "The keeper is terminating due do SIGINT/SIGTERM signal received" ) break # if any exception is raised in filter handling thread (could be an HTTP exception # while communicating with the node), web3.py does not retry and the filter becomes # dysfunctional i.e. no new callbacks will ever be fired. we detect it and terminate # the keeper so it can be restarted. if not all_filter_threads_alive(): self.logger.fatal( "One of filter threads is dead, the keeper will terminate") self.fatal_termination = True break # if we are watching for new blocks and no new block has been reported during # some time, we assume the watching filter died and terminate the keeper # so it can be restarted. # # this used to happen when the machine that has the node and the keeper running # was put to sleep and then woken up. # # TODO the same thing could possibly happen if we watch any event other than # TODO a new block. if that happens, we have no reliable way of detecting it now. if self._last_block_time and ( datetime.datetime.now(tz=pytz.UTC) - self._last_block_time).total_seconds() > 300: if not self.web3.eth.syncing: self.logger.fatal( "No new blocks received for 300 seconds, the keeper will terminate" ) self.fatal_termination = True break
def __exit__(self, exc_type, exc_val, exc_tb): # Initialization phase self.logger.info(f"Keeper connected to {self.web3.providers[0]}") self.logger.info(f"Keeper operating as {self.web3.eth.defaultAccount}") self._check_account_unlocked() self._wait_for_init() # Initial delay if self.delay > 0: self.logger.info(f"Waiting for {self.delay} seconds of initial delay...") time.sleep(self.delay) # Startup phase if self.startup_function: self.logger.info("Executing keeper startup logic") self.startup_function() # Bind `on_block`, bind `every` # Enter the main loop self._start_watching_blocks() self._start_every_timers() self._main_loop() # Enter shutdown process self.logger.info("Shutting down the keeper") # Disable all filters if any_filter_thread_present(): self.logger.info("Waiting for all threads to terminate...") stop_all_filter_threads() # If the `on_block` callback is still running, wait for it to terminate if self._on_block_callback is not None: self.logger.info("Waiting for outstanding callback to terminate...") self._on_block_callback.wait() # If any every (timer) callback is still running, wait for it to terminate if len(self.every_timers) > 0: self.logger.info("Waiting for outstanding timers to terminate...") for timer in self.every_timers: timer[1].wait() # Shutdown phase if self.shutdown_function: self.logger.info("Executing keeper shutdown logic...") self.shutdown_function() self.logger.info("Shutdown logic finished") self.logger.info("Keeper terminated") exit(10 if self.fatal_termination else 0)
def __exit__(self, exc_type, exc_val, exc_tb): # Initialization phase if self.web3: self.logger.info(f"Keeper connected to {self.web3.providers[0]}") if self.web3.eth.defaultAccount: self.logger.info( f"Keeper operating as {self.web3.eth.defaultAccount}") self._check_account_unlocked() else: self.logger.info( f"Keeper not operating as any particular account") # web3 calls do not work correctly if defaultAccount is empty self.web3.eth.defaultAccount = "0x0000000000000000000000000000000000000000" else: self.logger.info(f"Keeper initializing") # Wait for sync and peers if self.web3 and self.do_wait_for_sync: self._wait_for_init() # Initial delay if self.delay > 0: self.logger.info( f"Waiting for {self.delay} seconds of initial delay...") time.sleep(self.delay) # Initial checks if len(self.wait_for_functions) > 0: self.logger.info("Waiting for initial checks to pass...") for index, (wait_for_function, max_wait) in enumerate(self.wait_for_functions, start=1): start_time = time.time() while True: try: result = wait_for_function() except Exception as e: self.logger.exception( f"Initial check #{index} failed with an exception: '{e}'" ) result = False if result: break if time.time() - start_time >= max_wait: self.logger.warning( f"Initial check #{index} took more than {max_wait} seconds to pass, skipping" ) break time.sleep(0.1) # Startup phase if self.startup_function: self.logger.info("Executing keeper startup logic") self.startup_function() # Bind `on_block`, bind `every` # Enter the main loop self._start_watching_blocks() self._start_every_timers() self._main_loop() # Enter shutdown process self.logger.info("Shutting down the keeper") # Disable all filters if any_filter_thread_present(): self.logger.info("Waiting for all threads to terminate...") stop_all_filter_threads() # If the `on_block` callback is still running, wait for it to terminate if self._on_block_callback is not None: self.logger.info( "Waiting for outstanding callback to terminate...") self._on_block_callback.wait() # If any every (timer) callback is still running, wait for it to terminate if len(self.every_timers) > 0: self.logger.info("Waiting for outstanding timers to terminate...") for timer in self.every_timers: timer[1].wait() # If any condition callback is still running, wait for it to terminate if len(self.condition_timers) > 0: self.logger.info( "Waiting for outstanding conditions to terminate...") for timer in self.condition_timers: timer[2].wait() # Shutdown phase if self.shutdown_function: self.logger.info("Executing keeper shutdown logic...") self.shutdown_function() self.logger.info("Shutdown logic finished") self.logger.info("Keeper terminated") exit(10 if self.fatal_termination else 0)