def set_as_down(self, channels: ChannelSet, logger: logging.Logger) -> None: logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s', self, self.is_down, channels) # If node was not down before, do not alert for now, just in case it's # a connection hiccup but take note of the start of the downtime if not self.is_down: self._went_down_at = datetime.now() self._experiencing_delays_alert_sent = False self._initial_downtime_alert_sent = False self._downtime_initial_alert_delayer.did_task() # If node was down and we have not yet sent an alert about this, send # an informational 'experiencing delays' alert as a warning elif not self._experiencing_delays_alert_sent: channels.alert_info(ExperiencingDelaysAlert(self.name)) self._experiencing_delays_alert_sent = True # If we have not yet sent an initial downtime alert, and enough # time has passed for it, then send an initial alert elif not self._initial_downtime_alert_sent: if self._downtime_initial_alert_delayer.can_do_task(): downtime = strfdelta(datetime.now() - self._went_down_at, "{hours}h, {minutes}m, {seconds}s") if self.is_validator: channels.alert_major( CannotAccessNodeAlert(self.name, self._went_down_at, downtime)) else: channels.alert_minor( CannotAccessNodeAlert(self.name, self._went_down_at, downtime)) self._downtime_reminder_limiter.did_task() self._initial_downtime_alert_sent = True # If we already sent an initial alert and enough time has passed # for a reminder alert, then send a reminder alert else: if self._downtime_reminder_limiter.can_do_task(): downtime = strfdelta(datetime.now() - self._went_down_at, "{hours}h, {minutes}m, {seconds}s") if self.is_validator: channels.alert_major( StillCannotAccessNodeAlert(self.name, self._went_down_at, downtime)) else: channels.alert_minor( StillCannotAccessNodeAlert(self.name, self._went_down_at, downtime)) self._downtime_reminder_limiter.did_task()
def set_as_down(self, channels: ChannelSet, logger: logging.Logger) -> None: logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s', self, self.is_down, channels) # Alert (varies depending on whether was already down) if self.is_down and not self._initial_downtime_alert_sent: if self.is_validator: channels.alert_critical(CannotAccessNodeAlert(self.name)) else: channels.alert_warning(CannotAccessNodeAlert(self.name)) self._downtime_alert_limiter.did_task() self._initial_downtime_alert_sent = True elif self.is_down and self._downtime_alert_limiter.can_do_task(): went_down_at = datetime.fromtimestamp(self._went_down_at) downtime = strfdelta(datetime.now() - went_down_at, "{hours}h, {minutes}m, {seconds}s") if self.is_validator: channels.alert_critical( StillCannotAccessNodeAlert(self.name, went_down_at, downtime)) else: channels.alert_warning( StillCannotAccessNodeAlert(self.name, went_down_at, downtime)) self._downtime_alert_limiter.did_task() elif not self.is_down: # Do not alert for now just in case this is a connection hiccup channels.alert_info(ExperiencingDelaysAlert(self.name)) self._went_down_at = datetime.now().timestamp() self._initial_downtime_alert_sent = False
def set_as_down(self, channels: ChannelSet, error: Exception, logger: logging.Logger) -> None: logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s', self, self.is_down, channels) # Alert (varies depending on whether was already down) if self.is_down and not self._initial_downtime_alert_sent: if self.is_validator: channels.alert_major(CannotAccessNodeAlert(self.name)) else: channels.alert_minor(CannotAccessNodeAlert(self.name)) logger.error('Error occurred when accessing {}: {}.' ''.format(self, error)) self._downtime_alert_limiter.did_task() self._initial_downtime_alert_sent = True elif self.is_down and self._downtime_alert_limiter.can_do_task(): downtime = strfdelta(datetime.now() - self._went_down_at, "{hours}h, {minutes}m, {seconds}s") if self.is_validator: channels.alert_major(StillCannotAccessNodeAlert( self.name, self._went_down_at, downtime)) else: channels.alert_minor(StillCannotAccessNodeAlert( self.name, self._went_down_at, downtime)) self._downtime_alert_limiter.did_task() elif not self.is_down: # Do not alert for now just in case this is a connection hiccup channels.alert_info(ExperiencingDelaysAlert(self.name)) self._went_down_at = datetime.now() self._initial_downtime_alert_sent = False
def update_finalized_block_height(self, new_finalized_height: int, logger: logging.Logger, channels: ChannelSet): logger.debug( '%s update_finalized_block_height: finalized_block_height' ' (currently)=%s', self, self._finalized_block_height) current_timestamp = datetime.now().timestamp() if self._finalized_block_height != new_finalized_height: if self.is_no_change_in_height_warning_sent: self._no_change_in_height_warning_sent = False channels.alert_info( NodeFinalizedBlockHeightHasNowBeenUpdatedAlert(self.name)) if self._finalized_block_height > new_finalized_height: logger.info( 'The finalized height of node {} decreased to {}.'.format( self, self._finalized_block_height)) self._finalized_block_height = new_finalized_height self._time_of_last_height_change = current_timestamp self._time_of_last_height_check_activity = current_timestamp self._finalized_height_alert_limiter.set_last_time_that_did_task( datetime.fromtimestamp(current_timestamp)) else: timestamp_difference = current_timestamp - \ self._time_of_last_height_change time_interval = strfdelta( timedelta(seconds=int(timestamp_difference)), "{hours}h, {minutes}m, {seconds}s") if not self.is_no_change_in_height_warning_sent and \ timestamp_difference > \ self._no_change_in_height_first_warning_seconds: self._no_change_in_height_warning_sent = True channels.alert_warning( NodeFinalizedBlockHeightDidNotChangeInAlert( self.name, time_interval)) elif self._finalized_height_alert_limiter.can_do_task() and \ self.is_no_change_in_height_warning_sent: if self.is_validator: channels.alert_critical( NodeFinalizedBlockHeightDidNotChangeInAlert( self.name, time_interval)) else: channels.alert_warning( NodeFinalizedBlockHeightDidNotChangeInAlert( self.name, time_interval)) self._time_of_last_height_check_activity = current_timestamp self._finalized_height_alert_limiter. \ set_last_time_that_did_task( datetime.fromtimestamp(current_timestamp))
def __init__(self, origin_name: str, difference: float, severity: str, timestamp: float, parent_id: str, origin_id: str) -> None: super().__init__( SystemAlertCode.SystemStillDownAlert, "{} System is still down, it has been down for {}.".format( origin_name, strfdelta(timedelta(seconds=difference), "{hours}h, {minutes}m, {seconds}s")), severity, timestamp, parent_id, origin_id, SystemMetricCode.SystemIsDown)
def test_strfdelta_includes_days_as_hours_if_days_not_in_fmt(self): d = 1 h = 2 m = 3 s = 4 str_date = strfdelta(timedelta(days=d, hours=h, minutes=m, seconds=s), '{hours}h {minutes}m {seconds}s') expected_h = h + (d * 24) expected_m = m expected_s = s self.assertEqual( str_date, '{}h {}m {}s'.format(expected_h, expected_m, expected_s))
def test_strfdelta_returns_correct_string_for_integer_values(self): d = 1 h = 2 m = 3 s = 4 str_date = strfdelta(timedelta(days=d, hours=h, minutes=m, seconds=s), '{days}d {hours}h {minutes}m {seconds}s') expected_d = d expected_h = h expected_m = m expected_s = s self.assertEqual( str_date, '{}d {}h {}m {}s'.format(expected_d, expected_h, expected_m, expected_s))
def test_strfdelta_returns_correct_string_for_float_values(self): d = 1.5 h = 2.5 m = 3.5 s = 4.5 str_date = strfdelta(timedelta(days=d, hours=h, minutes=m, seconds=s), '{days}d {hours}h {minutes}m {seconds}s') expected_d = 1 expected_h = 2 + 12 # 0.5 days = 12 hours expected_m = 3 + 30 # 0.5 hours = 30 minutes expected_s = 4 + 30 # 0.5 minutes = 30 seconds self.assertEqual( str_date, '{}d {}h {}m {}s'.format(expected_d, expected_h, expected_m, expected_s))
def set_as_up(self, channels: ChannelSet, logger: logging.Logger) -> None: logger.debug('%s set_as_up: is_down(currently)=%s, channels=%s', self, self.is_down, channels) # Alert if node was down if self.is_down: # Only send accessible alert if inaccessible alert was sent if self._initial_downtime_alert_sent: downtime = strfdelta(datetime.now() - self._went_down_at, "{hours}h, {minutes}m, {seconds}s") channels.alert_info(NowAccessibleAlert( self.name, self._went_down_at, downtime)) # Reset downtime-related values self._downtime_alert_limiter.reset() self._went_down_at = None
def set_no_of_blocks_authored(self, channels: ChannelSet, logger: logging.Logger, new_no_of_blocks_authored: int, era_index: int): # NOTE: This function assumes that the node is a validator. logger.debug( '%s set_no_of_blocks_authored: no_of_blocks_' 'authored(currently)=%s, channels=%s', self, self._no_of_blocks_authored, channels) if self.is_active: if self._no_of_blocks_authored < new_no_of_blocks_authored: self._no_of_blocks_authored = new_no_of_blocks_authored self._time_of_last_block = datetime.now().timestamp() self.blocks_authored_alert_limiter.did_task() self._time_of_last_block_check_activity = \ datetime.now().timestamp() if self._is_authoring is False: self._is_authoring = True channels.alert_info( ANewBlockHasNowBeenAuthoredByValidatorAlert(self.name)) elif self._no_of_blocks_authored == \ new_no_of_blocks_authored and \ self.blocks_authored_alert_limiter.can_do_task(): if self._time_of_last_block != NONE: time_interval = strfdelta( datetime.now() - datetime.fromtimestamp(self._time_of_last_block), "{hours}h, {minutes}m, {seconds}s") channels.alert_warning( LastAuthoredBlockInEraAlert(self.name, time_interval, era_index)) else: channels.alert_warning( NoBlocksHaveYetBeenAuthoredInEraAlert( self.name, era_index)) self._is_authoring = False self.blocks_authored_alert_limiter.did_task() self._time_of_last_block_check_activity = \ datetime.now().timestamp()
def setup_periodic_alive_reminder(cp: ConfigParser) -> None: print('---- Periodic alive reminder') print('The periodic alive reminder is a way for the alerter to inform its ' 'users that it is still running.') if is_already_set_up(cp, 'periodic_alive_reminder') and \ not yn_prompt('The periodic alive reminder is already set up. ' 'Do you wish to clear the current config? (Y/n)\n'): return reset_section('periodic_alive_reminder', cp) cp['periodic_alive_reminder']['enabled'] = str(False) cp['periodic_alive_reminder']['interval_seconds'] = '' cp['periodic_alive_reminder']['email_enabled'] = '' cp['periodic_alive_reminder']['telegram_enabled'] = '' cp['periodic_alive_reminder']['mongo_enabled'] = '' if not yn_prompt('Do you wish to set up the periodic alive reminder? ' '(Y/n)\n'): return interval = input("Please enter the amount of seconds you want to " "pass for the periodic alive reminder. Make sure that " "you insert a positive integer.\n") while True: try: interval_number_rep = int(interval) except ValueError: interval = input("Input is not a valid integer. Please enter " "another value:\n") continue if interval_number_rep > 0: time = timedelta(seconds=int(interval_number_rep)) time = strfdelta(time, "{hours}h {minutes}m {seconds}s") if yn_prompt( 'You will be reminded that the alerter is still running ' 'every {}. Is this correct (Y/n) \n'.format(time)): break else: interval = input( "Please enter the amount of seconds you want to " "pass for the periodic alive reminder. Make sure that " "you insert a positive integer.\n") else: interval = input("Input is not a positive integer. Please enter " "another value:\n") if is_already_set_up(cp, 'email_alerts') and \ cp['email_alerts']['enabled'] and \ yn_prompt('Would you like the periodic alive reminder ' 'to send alerts via e-mail? (Y/n)\n'): email_enabled = str(True) else: email_enabled = str(False) if is_already_set_up(cp, 'telegram_alerts') and \ cp['telegram_alerts']['enabled'] and \ yn_prompt('Would you like the periodic alive reminder ' 'to send alerts via Telegram? (Y/n)\n'): telegram_enabled = str(True) else: telegram_enabled = str(False) if is_already_set_up(cp, 'mongo') and cp['mongo']['enabled'] and \ yn_prompt('Would you like the periodic alive reminder ' 'to save alerts to Mongo? (Y/n)\n'): mongo_enabled = str(True) else: mongo_enabled = str(False) cp['periodic_alive_reminder']['enabled'] = str(True) cp['periodic_alive_reminder']['interval_seconds'] = interval cp['periodic_alive_reminder']['email_enabled'] = email_enabled cp['periodic_alive_reminder']['telegram_enabled'] = telegram_enabled cp['periodic_alive_reminder']['mongo_enabled'] = mongo_enabled
def time_interval_pretty(self) -> str: return strfdelta(self.time_interval, "{hours}h, {minutes}m, {seconds}s")
def test_time_interval_pretty_returns_strfdelta_result(self): self.assertEqual( self.ttl.time_interval_pretty, strfdelta(self.interval_timedelta, "{hours}h, {minutes}m, {seconds}s"))