def test_terminated_due_to_exception_alert(self):
component = 'Some Monitor'
exception = Exception('Some Exception')
self.assertEqual(
'{} terminated due to exception: {}'.format(component, exception),
str(TerminatedDueToExceptionAlert(component, exception)))
def run_monitor_nodes(node: Node):
# Monitor name based on node
monitor_name = 'Node monitor ({})'.format(node.name)
# Logger initialisation
logger_monitor_node = create_logger(
InternalConf.node_monitor_general_log_file_template.format(node.name),
node.name,
InternalConf.logging_level,
rotating=True)
# Initialise monitor
node_monitor = NodeMonitor(monitor_name, channel_set, logger_monitor_node,
REDIS, node)
# Start
log_and_print('{} started.'.format(monitor_name))
sys.stdout.flush()
try:
start_node_monitor(node_monitor,
InternalConf.node_monitor_period_seconds,
logger_monitor_node)
except Exception as e:
channel_set.alert_error(TerminatedDueToExceptionAlert(monitor_name, e))
log_and_print('{} stopped.'.format(monitor_name))
def run_commands_telegram():
# Fixed monitor name
monitor_name = 'Telegram commands'
# Check if Telegram commands enabled
if not UserConf.telegram_cmds_enabled:
return
while True:
# Start
log_and_print('{} started.'.format(monitor_name))
sys.stdout.flush()
try:
TelegramCommands(
UserConf.telegram_cmds_bot_token,
UserConf.telegram_cmds_bot_chat_id,
logger_commands_telegram,
REDIS,
InternalConf.redis_twilio_snooze_key,
InternalConf.redis_periodic_alive_reminder_mute_key,
InternalConf.redis_node_monitor_alive_key_prefix,
InternalConf.redis_network_monitor_alive_key_prefix,
InternalConf.redis_network_monitor_last_height_key_prefix,
).start_listening()
except Exception as e:
full_channel_set.alert_error(
TerminatedDueToExceptionAlert(monitor_name, e))
log_and_print('{} stopped.'.format(monitor_name))
def run_monitor_network(network_nodes_tuple: Tuple[str, List[Node]]):
# Get network and nodes
network = network_nodes_tuple[0]
nodes = network_nodes_tuple[1]
# Monitor name based on network
monitor_name = 'Network monitor ({})'.format(network)
# Initialisation
try:
# Logger initialisation
logger_monitor_network = create_logger(
InternalConf.network_monitor_general_log_file_template.format(
network),
network,
InternalConf.logging_level,
rotating=True)
# Organize as validators and full nodes
validators = [n for n in nodes if n.is_validator]
full_nodes = [n for n in nodes if not n.is_validator]
# Do not start if not enough nodes
if 0 in [len(validators), len(full_nodes)]:
log_and_print('!!! Could not start {}. It must have at least 1 '
'validator and 1 full node!!!'.format(monitor_name))
return
# Initialise monitor
network_monitor = NetworkMonitor(
monitor_name, full_channel_set, logger_monitor_network,
InternalConf.network_monitor_max_catch_up_blocks, REDIS,
full_nodes, validators)
except Exception as e:
msg = '!!! Error when initialising {}: {} !!!'.format(monitor_name, e)
log_and_print(msg)
raise InitialisationException(msg)
while True:
# Start
log_and_print('{} started with {} validator(s) and {} full node(s).'
''.format(monitor_name, len(validators),
len(full_nodes)))
sys.stdout.flush()
try:
start_network_monitor(network_monitor,
InternalConf.network_monitor_period_seconds,
logger_monitor_network)
except Exception as e:
full_channel_set.alert_error(
TerminatedDueToExceptionAlert(monitor_name, e))
log_and_print('{} stopped.'.format(monitor_name))
def run_periodic_alive_reminder():
if not UserConf.periodic_alive_reminder_enabled:
return
name = "Periodic alive reminder"
# Initialisation
periodic_alive_reminder = PeriodicAliveReminder(
UserConf.interval_seconds, periodic_alive_reminder_channel_set,
InternalConf.redis_periodic_alive_reminder_mute_key, REDIS)
while True:
# Start
log_and_print('{} started.'.format(name))
sys.stdout.flush()
try:
periodic_alive_reminder.start()
except Exception as e:
periodic_alive_reminder_channel_set.alert_error(
TerminatedDueToExceptionAlert(name, e))
log_and_print('{} stopped.'.format(name))
def run_monitor_github(repo_config: RepoConfig):
# Monitor name based on repository
monitor_name = 'GitHub monitor ({})'.format(repo_config.repo_name)
# Initialisation
try:
# Logger initialisation
logger_monitor_github = create_logger(
InternalConf.github_monitor_general_log_file_template.format(
repo_config.repo_page.replace('/', '_')),
repo_config.repo_page,
InternalConf.logging_level,
rotating=True)
# Get releases page
releases_page = InternalConf.github_releases_template.format(
repo_config.repo_page)
# Initialise monitor
github_monitor = GitHubMonitor(
monitor_name, full_channel_set, logger_monitor_github, REDIS,
repo_config.repo_name, releases_page,
InternalConf.redis_github_releases_key_prefix)
except Exception as e:
msg = '!!! Error when initialising {}: {} !!!'.format(monitor_name, e)
log_and_print(msg)
raise InitialisationException(msg)
while True:
# Start
log_and_print('{} started.'.format(monitor_name))
sys.stdout.flush()
try:
start_github_monitor(github_monitor,
InternalConf.github_monitor_period_seconds,
logger_monitor_github)
except Exception as e:
full_channel_set.alert_error(
TerminatedDueToExceptionAlert(monitor_name, e))
log_and_print('{} stopped.'.format(monitor_name))