def set_as_down(self, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# If node was not down before, do not alert for now, just in case it's
# a connection hiccup but take note of the start of the downtime
if not self.is_down:
self._went_down_at = datetime.now()
self._experiencing_delays_alert_sent = False
self._initial_downtime_alert_sent = False
self._downtime_initial_alert_delayer.did_task()
# If node was down and we have not yet sent an alert about this, send
# an informational 'experiencing delays' alert as a warning
elif not self._experiencing_delays_alert_sent:
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._experiencing_delays_alert_sent = True
# If we have not yet sent an initial downtime alert, and enough
# time has passed for it, then send an initial alert
elif not self._initial_downtime_alert_sent:
if self._downtime_initial_alert_delayer.can_do_task():
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_major(
CannotAccessNodeAlert(self.name, self._went_down_at,
downtime))
else:
channels.alert_minor(
CannotAccessNodeAlert(self.name, self._went_down_at,
downtime))
self._downtime_reminder_limiter.did_task()
self._initial_downtime_alert_sent = True
# If we already sent an initial alert and enough time has passed
# for a reminder alert, then send a reminder alert
else:
if self._downtime_reminder_limiter.can_do_task():
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_major(
StillCannotAccessNodeAlert(self.name,
self._went_down_at,
downtime))
else:
channels.alert_minor(
StillCannotAccessNodeAlert(self.name,
self._went_down_at,
downtime))
self._downtime_reminder_limiter.did_task()
def set_as_down(self, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert (varies depending on whether was already down)
if self.is_down and not self._initial_downtime_alert_sent:
if self.is_validator:
channels.alert_critical(CannotAccessNodeAlert(self.name))
else:
channels.alert_warning(CannotAccessNodeAlert(self.name))
self._downtime_alert_limiter.did_task()
self._initial_downtime_alert_sent = True
elif self.is_down and self._downtime_alert_limiter.can_do_task():
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_critical(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
else:
channels.alert_warning(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
self._downtime_alert_limiter.did_task()
elif not self.is_down:
# Do not alert for now just in case this is a connection hiccup
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._went_down_at = datetime.now().timestamp()
self._initial_downtime_alert_sent = False
def set_as_down(self, channels: ChannelSet, error: Exception,
logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert (varies depending on whether was already down)
if self.is_down and not self._initial_downtime_alert_sent:
if self.is_validator:
channels.alert_major(CannotAccessNodeAlert(self.name))
else:
channels.alert_minor(CannotAccessNodeAlert(self.name))
logger.error('Error occurred when accessing {}: {}.'
''.format(self, error))
self._downtime_alert_limiter.did_task()
self._initial_downtime_alert_sent = True
elif self.is_down and self._downtime_alert_limiter.can_do_task():
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_major(StillCannotAccessNodeAlert(
self.name, self._went_down_at, downtime))
else:
channels.alert_minor(StillCannotAccessNodeAlert(
self.name, self._went_down_at, downtime))
self._downtime_alert_limiter.did_task()
elif not self.is_down:
# Do not alert for now just in case this is a connection hiccup
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._went_down_at = datetime.now()
self._initial_downtime_alert_sent = False
def update_finalized_block_height(self, new_finalized_height: int,
logger: logging.Logger,
channels: ChannelSet):
logger.debug(
'%s update_finalized_block_height: finalized_block_height'
' (currently)=%s', self, self._finalized_block_height)
current_timestamp = datetime.now().timestamp()
if self._finalized_block_height != new_finalized_height:
if self.is_no_change_in_height_warning_sent:
self._no_change_in_height_warning_sent = False
channels.alert_info(
NodeFinalizedBlockHeightHasNowBeenUpdatedAlert(self.name))
if self._finalized_block_height > new_finalized_height:
logger.info(
'The finalized height of node {} decreased to {}.'.format(
self, self._finalized_block_height))
self._finalized_block_height = new_finalized_height
self._time_of_last_height_change = current_timestamp
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter.set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
else:
timestamp_difference = current_timestamp - \
self._time_of_last_height_change
time_interval = strfdelta(
timedelta(seconds=int(timestamp_difference)),
"{hours}h, {minutes}m, {seconds}s")
if not self.is_no_change_in_height_warning_sent and \
timestamp_difference > \
self._no_change_in_height_first_warning_seconds:
self._no_change_in_height_warning_sent = True
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
elif self._finalized_height_alert_limiter.can_do_task() and \
self.is_no_change_in_height_warning_sent:
if self.is_validator:
channels.alert_critical(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
else:
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter. \
set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
def __init__(self, origin_name: str, difference: float, severity: str,
timestamp: float, parent_id: str, origin_id: str) -> None:
super().__init__(
SystemAlertCode.SystemStillDownAlert,
"{} System is still down, it has been down for {}.".format(
origin_name, strfdelta(timedelta(seconds=difference),
"{hours}h, {minutes}m, {seconds}s")),
severity, timestamp, parent_id, origin_id,
SystemMetricCode.SystemIsDown)
def test_strfdelta_includes_days_as_hours_if_days_not_in_fmt(self):
d = 1
h = 2
m = 3
s = 4
str_date = strfdelta(timedelta(days=d, hours=h, minutes=m, seconds=s),
'{hours}h {minutes}m {seconds}s')
expected_h = h + (d * 24)
expected_m = m
expected_s = s
self.assertEqual(
str_date, '{}h {}m {}s'.format(expected_h, expected_m, expected_s))
def test_strfdelta_returns_correct_string_for_integer_values(self):
d = 1
h = 2
m = 3
s = 4
str_date = strfdelta(timedelta(days=d, hours=h, minutes=m, seconds=s),
'{days}d {hours}h {minutes}m {seconds}s')
expected_d = d
expected_h = h
expected_m = m
expected_s = s
self.assertEqual(
str_date, '{}d {}h {}m {}s'.format(expected_d, expected_h,
expected_m, expected_s))
def test_strfdelta_returns_correct_string_for_float_values(self):
d = 1.5
h = 2.5
m = 3.5
s = 4.5
str_date = strfdelta(timedelta(days=d, hours=h, minutes=m, seconds=s),
'{days}d {hours}h {minutes}m {seconds}s')
expected_d = 1
expected_h = 2 + 12 # 0.5 days = 12 hours
expected_m = 3 + 30 # 0.5 hours = 30 minutes
expected_s = 4 + 30 # 0.5 minutes = 30 seconds
self.assertEqual(
str_date, '{}d {}h {}m {}s'.format(expected_d, expected_h,
expected_m, expected_s))
def set_as_up(self, channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug('%s set_as_up: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert if node was down
if self.is_down:
# Only send accessible alert if inaccessible alert was sent
if self._initial_downtime_alert_sent:
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
channels.alert_info(NowAccessibleAlert(
self.name, self._went_down_at, downtime))
# Reset downtime-related values
self._downtime_alert_limiter.reset()
self._went_down_at = None
def set_no_of_blocks_authored(self, channels: ChannelSet,
logger: logging.Logger,
new_no_of_blocks_authored: int,
era_index: int):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s set_no_of_blocks_authored: no_of_blocks_'
'authored(currently)=%s, channels=%s', self,
self._no_of_blocks_authored, channels)
if self.is_active:
if self._no_of_blocks_authored < new_no_of_blocks_authored:
self._no_of_blocks_authored = new_no_of_blocks_authored
self._time_of_last_block = datetime.now().timestamp()
self.blocks_authored_alert_limiter.did_task()
self._time_of_last_block_check_activity = \
datetime.now().timestamp()
if self._is_authoring is False:
self._is_authoring = True
channels.alert_info(
ANewBlockHasNowBeenAuthoredByValidatorAlert(self.name))
elif self._no_of_blocks_authored == \
new_no_of_blocks_authored and \
self.blocks_authored_alert_limiter.can_do_task():
if self._time_of_last_block != NONE:
time_interval = strfdelta(
datetime.now() -
datetime.fromtimestamp(self._time_of_last_block),
"{hours}h, {minutes}m, {seconds}s")
channels.alert_warning(
LastAuthoredBlockInEraAlert(self.name, time_interval,
era_index))
else:
channels.alert_warning(
NoBlocksHaveYetBeenAuthoredInEraAlert(
self.name, era_index))
self._is_authoring = False
self.blocks_authored_alert_limiter.did_task()
self._time_of_last_block_check_activity = \
datetime.now().timestamp()
def setup_periodic_alive_reminder(cp: ConfigParser) -> None:
print('---- Periodic alive reminder')
print('The periodic alive reminder is a way for the alerter to inform its '
'users that it is still running.')
if is_already_set_up(cp, 'periodic_alive_reminder') and \
not yn_prompt('The periodic alive reminder is already set up. '
'Do you wish to clear the current config? (Y/n)\n'):
return
reset_section('periodic_alive_reminder', cp)
cp['periodic_alive_reminder']['enabled'] = str(False)
cp['periodic_alive_reminder']['interval_seconds'] = ''
cp['periodic_alive_reminder']['email_enabled'] = ''
cp['periodic_alive_reminder']['telegram_enabled'] = ''
cp['periodic_alive_reminder']['mongo_enabled'] = ''
if not yn_prompt('Do you wish to set up the periodic alive reminder? '
'(Y/n)\n'):
return
interval = input("Please enter the amount of seconds you want to "
"pass for the periodic alive reminder. Make sure that "
"you insert a positive integer.\n")
while True:
try:
interval_number_rep = int(interval)
except ValueError:
interval = input("Input is not a valid integer. Please enter "
"another value:\n")
continue
if interval_number_rep > 0:
time = timedelta(seconds=int(interval_number_rep))
time = strfdelta(time, "{hours}h {minutes}m {seconds}s")
if yn_prompt(
'You will be reminded that the alerter is still running '
'every {}. Is this correct (Y/n) \n'.format(time)):
break
else:
interval = input(
"Please enter the amount of seconds you want to "
"pass for the periodic alive reminder. Make sure that "
"you insert a positive integer.\n")
else:
interval = input("Input is not a positive integer. Please enter "
"another value:\n")
if is_already_set_up(cp, 'email_alerts') and \
cp['email_alerts']['enabled'] and \
yn_prompt('Would you like the periodic alive reminder '
'to send alerts via e-mail? (Y/n)\n'):
email_enabled = str(True)
else:
email_enabled = str(False)
if is_already_set_up(cp, 'telegram_alerts') and \
cp['telegram_alerts']['enabled'] and \
yn_prompt('Would you like the periodic alive reminder '
'to send alerts via Telegram? (Y/n)\n'):
telegram_enabled = str(True)
else:
telegram_enabled = str(False)
if is_already_set_up(cp, 'mongo') and cp['mongo']['enabled'] and \
yn_prompt('Would you like the periodic alive reminder '
'to save alerts to Mongo? (Y/n)\n'):
mongo_enabled = str(True)
else:
mongo_enabled = str(False)
cp['periodic_alive_reminder']['enabled'] = str(True)
cp['periodic_alive_reminder']['interval_seconds'] = interval
cp['periodic_alive_reminder']['email_enabled'] = email_enabled
cp['periodic_alive_reminder']['telegram_enabled'] = telegram_enabled
cp['periodic_alive_reminder']['mongo_enabled'] = mongo_enabled
def time_interval_pretty(self) -> str:
return strfdelta(self.time_interval,
"{hours}h, {minutes}m, {seconds}s")
def test_time_interval_pretty_returns_strfdelta_result(self):
self.assertEqual(
self.ttl.time_interval_pretty,
strfdelta(self.interval_timedelta,
"{hours}h, {minutes}m, {seconds}s"))
