def _main_loop(self):
# terminate gracefully on either SIGINT or SIGTERM
signal.signal(signal.SIGINT, self._sigint_sigterm_handler)
signal.signal(signal.SIGTERM, self._sigint_sigterm_handler)
# in case at least one filter has been set up, we enter an infinite loop and let
# the callbacks do the job. in case of no filters, we will not enter this loop
# and the keeper will terminate soon after it started
while any_filter_thread_present() or self._at_least_one_every:
time.sleep(1)
# if the keeper logic asked us to terminate, we do so
if self.terminated_internally:
self.logger.warning(
"Keeper logic asked for termination, the keeper will terminate"
)
break
# if SIGINT/SIGTERM asked us to terminate, we do so
if self.terminated_externally:
self.logger.warning(
"The keeper is terminating due do SIGINT/SIGTERM signal received"
)
break
# if any exception is raised in filter handling thread (could be an HTTP exception
# while communicating with the node), web3.py does not retry and the filter becomes
# dysfunctional i.e. no new callbacks will ever be fired. we detect it and terminate
# the keeper so it can be restarted.
if not all_filter_threads_alive():
self.logger.fatal(
"One of filter threads is dead, the keeper will terminate")
self.fatal_termination = True
break
# if we are watching for new blocks and no new block has been reported during
# some time, we assume the watching filter died and terminate the keeper
# so it can be restarted.
#
# this used to happen when the machine that has the node and the keeper running
# was put to sleep and then woken up.
#
# TODO the same thing could possibly happen if we watch any event other than
# TODO a new block. if that happens, we have no reliable way of detecting it now.
if self._last_block_time and (
datetime.datetime.now(tz=pytz.UTC) -
self._last_block_time).total_seconds() > 300:
if not self.web3.eth.syncing:
self.logger.fatal(
"No new blocks received for 300 seconds, the keeper will terminate"
)
self.fatal_termination = True
break
def __exit__(self, exc_type, exc_val, exc_tb):
# Initialization phase
self.logger.info(f"Keeper connected to {self.web3.providers[0]}")
self.logger.info(f"Keeper operating as {self.web3.eth.defaultAccount}")
self._check_account_unlocked()
self._wait_for_init()
# Initial delay
if self.delay > 0:
self.logger.info(f"Waiting for {self.delay} seconds of initial delay...")
time.sleep(self.delay)
# Startup phase
if self.startup_function:
self.logger.info("Executing keeper startup logic")
self.startup_function()
# Bind `on_block`, bind `every`
# Enter the main loop
self._start_watching_blocks()
self._start_every_timers()
self._main_loop()
# Enter shutdown process
self.logger.info("Shutting down the keeper")
# Disable all filters
if any_filter_thread_present():
self.logger.info("Waiting for all threads to terminate...")
stop_all_filter_threads()
# If the `on_block` callback is still running, wait for it to terminate
if self._on_block_callback is not None:
self.logger.info("Waiting for outstanding callback to terminate...")
self._on_block_callback.wait()
# If any every (timer) callback is still running, wait for it to terminate
if len(self.every_timers) > 0:
self.logger.info("Waiting for outstanding timers to terminate...")
for timer in self.every_timers:
timer[1].wait()
# Shutdown phase
if self.shutdown_function:
self.logger.info("Executing keeper shutdown logic...")
self.shutdown_function()
self.logger.info("Shutdown logic finished")
self.logger.info("Keeper terminated")
exit(10 if self.fatal_termination else 0)
def __exit__(self, exc_type, exc_val, exc_tb):
# Initialization phase
if self.web3:
self.logger.info(f"Keeper connected to {self.web3.providers[0]}")
if self.web3.eth.defaultAccount:
self.logger.info(
f"Keeper operating as {self.web3.eth.defaultAccount}")
self._check_account_unlocked()
else:
self.logger.info(
f"Keeper not operating as any particular account")
# web3 calls do not work correctly if defaultAccount is empty
self.web3.eth.defaultAccount = "0x0000000000000000000000000000000000000000"
else:
self.logger.info(f"Keeper initializing")
# Wait for sync and peers
if self.web3 and self.do_wait_for_sync:
self._wait_for_init()
# Initial delay
if self.delay > 0:
self.logger.info(
f"Waiting for {self.delay} seconds of initial delay...")
time.sleep(self.delay)
# Initial checks
if len(self.wait_for_functions) > 0:
self.logger.info("Waiting for initial checks to pass...")
for index, (wait_for_function,
max_wait) in enumerate(self.wait_for_functions,
start=1):
start_time = time.time()
while True:
try:
result = wait_for_function()
except Exception as e:
self.logger.exception(
f"Initial check #{index} failed with an exception: '{e}'"
)
result = False
if result:
break
if time.time() - start_time >= max_wait:
self.logger.warning(
f"Initial check #{index} took more than {max_wait} seconds to pass, skipping"
)
break
time.sleep(0.1)
# Startup phase
if self.startup_function:
self.logger.info("Executing keeper startup logic")
self.startup_function()
# Bind `on_block`, bind `every`
# Enter the main loop
self._start_watching_blocks()
self._start_every_timers()
self._main_loop()
# Enter shutdown process
self.logger.info("Shutting down the keeper")
# Disable all filters
if any_filter_thread_present():
self.logger.info("Waiting for all threads to terminate...")
stop_all_filter_threads()
# If the `on_block` callback is still running, wait for it to terminate
if self._on_block_callback is not None:
self.logger.info(
"Waiting for outstanding callback to terminate...")
self._on_block_callback.wait()
# If any every (timer) callback is still running, wait for it to terminate
if len(self.every_timers) > 0:
self.logger.info("Waiting for outstanding timers to terminate...")
for timer in self.every_timers:
timer[1].wait()
# If any condition callback is still running, wait for it to terminate
if len(self.condition_timers) > 0:
self.logger.info(
"Waiting for outstanding conditions to terminate...")
for timer in self.condition_timers:
timer[2].wait()
# Shutdown phase
if self.shutdown_function:
self.logger.info("Executing keeper shutdown logic...")
self.shutdown_function()
self.logger.info("Shutdown logic finished")
self.logger.info("Keeper terminated")
exit(10 if self.fatal_termination else 0)
