def connect_with_api(self, channels: ChannelSet, logger: logging.Logger):
logger.debug('%s connect_with_api: channels=%s', self, channels)
if not self.is_connected_to_api_server:
channels.alert_info(NodeConnectedToApiServerAgainAlert(self.name))
self._connected_to_api_server = True
def set_as_down(self, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert (varies depending on whether was already down)
if self.is_down and not self._initial_downtime_alert_sent:
if self.is_validator:
channels.alert_critical(CannotAccessNodeAlert(self.name))
else:
channels.alert_warning(CannotAccessNodeAlert(self.name))
self._downtime_alert_limiter.did_task()
self._initial_downtime_alert_sent = True
elif self.is_down and self._downtime_alert_limiter.can_do_task():
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_critical(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
else:
channels.alert_warning(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
self._downtime_alert_limiter.did_task()
elif not self.is_down:
# Do not alert for now just in case this is a connection hiccup
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._went_down_at = datetime.now().timestamp()
self._initial_downtime_alert_sent = False
def slash(self, amount: float, channels: ChannelSet,
logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug('%s slash: channels=%s', self, channels)
if amount > 0:
channels.alert_critical(
ValidatorHasBeenSlashedAlert(self.name, amount))
def set_api_as_down(self, monitor: str, channels: ChannelSet) -> None:
self._logger.debug(
'%s set_api_as_down: api_down(currently)=%s, '
'channels=%s', self, self._api_down, channels)
if not self._api_down:
channels.alert_error(ApiIsDownAlert(monitor))
self._api_down = True
def set_api_as_up(self, monitor: str, channels: ChannelSet) -> None:
self._logger.debug(
'%s set_api_as_down: api_down(currently)=%s, '
'channels=%s', self, self._api_down, channels)
if self._api_down:
channels.alert_info(ApiIsUpAgainAlert(monitor))
self._api_down = False
def setUp(self) -> None:
self.alerter_name = 'testalerter'
self.logger = logging.getLogger('dummy')
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel],
TestInternalConfSomeAlertsDisabled)
self.dummy_alert = Alert(AlertCode.TestAlert, 'dummy')
self.severities_map_bkp = \
TestInternalConfSomeAlertsDisabled.severities_enabled_map.copy()
self.alerts_map_bkp = \
TestInternalConfSomeAlertsDisabled.alerts_enabled_map.copy()
def set_is_syncing(self, now_is_syncing: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_is_syncing: before=%s, new=%s, channels=%s', self,
self.is_syncing, now_is_syncing, channels)
# Alert if is syncing has changed
if not self.is_syncing and now_is_syncing:
channels.alert_warning(IsSyncingAlert(self.name))
elif self.is_syncing and not now_is_syncing:
channels.alert_info(IsNoLongerSyncingAlert(self.name))
# Update is-syncing
self._is_syncing = now_is_syncing
def disconnect_from_api(self, channels: ChannelSet,
logger: logging.Logger):
logger.debug('%s disconnect_from_api: channels=%s', self, channels)
if self.is_connected_to_api_server:
if self.is_validator:
channels.alert_critical(
NodeWasNotConnectedToApiServerAlert(self.name))
else:
channels.alert_warning(
NodeWasNotConnectedToApiServerAlert(self.name))
self._connected_to_api_server = False
def set_active(self, now_is_active: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator.
logger.debug('%s set_active: active(currently)=%s, channels=%s', self,
self.is_active, channels)
if self.is_active not in [now_is_active, None]:
if now_is_active:
channels.alert_info(ValidatorIsNowActiveAlert(self.name))
else:
channels.alert_critical(ValidatorIsNotActiveAlert(self.name))
self._active = now_is_active
def clear_missed_blocks(self, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
logger.debug('%s clear_missed_blocks: channels=%s', self, channels)
# Alert if validator was missing blocks (only if more than 1 block)
if self.is_missing_blocks and self._consecutive_blocks_missed > 1:
channels.alert_info(
NoLongerMissingBlocksAlert(self.name,
self._consecutive_blocks_missed))
# Reset missed blocks related values
self._consecutive_blocks_missed = 0
def set_voting_power(self, new_voting_power: int, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
logger.debug('%s set_voting_power: before=%s, new=%s, channels=%s',
self, self.voting_power, new_voting_power, channels)
# Alert if voting power has changed
if self.voting_power not in [None, new_voting_power]:
if self.is_validator and new_voting_power == 0: # N to 0
channels.alert_critical(
VotingPowerDecreasedAlert(self.name, self.voting_power,
new_voting_power))
elif self.is_validator and self.voting_power == 0: # 0 to N
channels.alert_info(
VotingPowerIncreasedAlert(self.name, self.voting_power,
new_voting_power))
else: # Any change
diff = new_voting_power - self.voting_power
if diff > 0:
channels.alert_info(
VotingPowerIncreasedByAlert(self.name,
self.voting_power,
new_voting_power))
else:
channels.alert_info(
VotingPowerDecreasedByAlert(self.name,
self.voting_power,
new_voting_power))
# Update voting power
self._voting_power = new_voting_power
def set_public_prop_count(self, new_public_prop_count: int,
channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug(
'%s set_public_prop_count: public_prop_count(currently)=%s, '
'channels=%s', self, self.public_prop_count, channels)
if self._public_prop_count is None:
self._public_prop_count = new_public_prop_count
return
while self._public_prop_count < new_public_prop_count:
channels.alert_info(NewPublicProposalAlert(self._public_prop_count))
self._public_prop_count += 1
def set_referendum_count(self, new_referendum_count: int,
channels: ChannelSet, logger: logging.Logger,
referendum_info: PolkadotWrapperType = None) \
-> None:
logger.debug(
'%s set_referendum_count: referendum_count(currently)=%s, '
'channels=%s', self, self.referendum_count, channels)
if self._referendum_count not in [None, new_referendum_count] \
and referendum_info is not None:
end_block = int(referendum_info['end'])
channels.alert_info(
NewReferendumAlert(self.referendum_count, end_block))
self._referendum_count = new_referendum_count
def update_finalized_block_height(self, new_finalized_height: int,
logger: logging.Logger,
channels: ChannelSet):
logger.debug(
'%s update_finalized_block_height: finalized_block_height'
' (currently)=%s', self, self._finalized_block_height)
current_timestamp = datetime.now().timestamp()
if self._finalized_block_height != new_finalized_height:
if self.is_no_change_in_height_warning_sent:
self._no_change_in_height_warning_sent = False
channels.alert_info(
NodeFinalizedBlockHeightHasNowBeenUpdatedAlert(self.name))
if self._finalized_block_height > new_finalized_height:
logger.info(
'The finalized height of node {} decreased to {}.'.format(
self, self._finalized_block_height))
self._finalized_block_height = new_finalized_height
self._time_of_last_height_change = current_timestamp
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter.set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
else:
timestamp_difference = current_timestamp - \
self._time_of_last_height_change
time_interval = strfdelta(
timedelta(seconds=int(timestamp_difference)),
"{hours}h, {minutes}m, {seconds}s")
if not self.is_no_change_in_height_warning_sent and \
timestamp_difference > \
self._no_change_in_height_first_warning_seconds:
self._no_change_in_height_warning_sent = True
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
elif self._finalized_height_alert_limiter.can_do_task() and \
self.is_no_change_in_height_warning_sent:
if self.is_validator:
channels.alert_critical(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
else:
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter. \
set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
def set_validator_set_size(self, new_validator_set_size: int,
channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug(
'%s set_validator_set_size: validator_set_size(currently)=%s, '
'channels=%s', self, self.validator_set_size, channels)
if self._validator_set_size not in [new_validator_set_size, None]:
if self._validator_set_size < new_validator_set_size:
channels.alert_info(ValidatorSetSizeIncreasedAlert(
new_validator_set_size))
elif self._validator_set_size > new_validator_set_size:
channels.alert_info(ValidatorSetSizeDecreasedAlert(
new_validator_set_size))
self._validator_set_size = new_validator_set_size
def set_elected(self, now_is_elected: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator.
logger.debug('%s set_elected: elected(currently)=%s, channels=%s',
self, self.is_elected, channels)
if self.is_elected not in [now_is_elected, None]:
if now_is_elected:
channels.alert_info(
ValidatorIsElectedForTheNextSessionAlert(self.name))
else:
channels.alert_warning(
ValidatorIsNotElectedForNextSessionAlert(self.name))
self._elected = now_is_elected
def setUp(self) -> None:
self.blockchain_name = 'testblockchain'
self.logger = logging.getLogger('dummy')
self.blockchain = Blockchain(name=self.blockchain_name, redis=None)
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel], TestInternalConf)
self.dummy_referendum_count, self.dummy_public_prop_count, \
self.dummy_council_prop_count = 10, 10, 10
self.dummy_validator_set_size = 120
self.dummy_referendum_info_ongoing = {
'Ongoing': {
'proposalHash': '0x345jtg8ergfg8df89h9we9t9sd9g9gsd9g9sdfg',
'end': 124143848,
'threshold': 'Supermajorityapproval',
'delay': 11549,
'tally': {
'ayes': '4544545 KSM',
'nayes': '3454 KSM',
'turnout': '4545454454 KSM'
}
}
}
self.dummy_referendum_info_finished = {
'Finished': {
'approved': False,
'end': 124143848
}
}
def set_council_member(self, now_is_council_member: bool,
channels: ChannelSet, logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s set_council_member: council_member(currently)=%s, '
'channels=%s', self, self.is_council_member, channels)
if self.is_council_member not in [now_is_council_member, None]:
if now_is_council_member:
channels.alert_info(
ValidatorIsNowPartOfTheCouncilAlert(self.name))
else:
channels.alert_info(
ValidatorIsNoLongerPartOfTheCouncilAlert(self.name))
self._council_member = now_is_council_member
def setUp(self) -> None:
self.logger = logging.getLogger('dummy')
self.monitor_name = 'testblockchainmonitor'
self.blockchain_name = 'testblockchain'
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel], TestInternalConf)
self.db = TestInternalConf.redis_test_database
self.host = TestUserConf.redis_host
self.port = TestUserConf.redis_port
self.password = TestUserConf.redis_password
self.redis = RedisApi(self.logger, self.db, self.host, self.port,
self.password)
self.redis.delete_all_unsafe()
try:
self.redis.ping_unsafe()
except RedisConnectionError:
self.fail('Redis is not online.')
self.data_sources = []
self.blockchain = Blockchain(self.blockchain_name, self.redis)
self.polkadot_api_endpoint = 'api_endpoint'
self.monitor = BlockchainMonitor(self.monitor_name, self.blockchain,
self.channel_set, self.logger,
self.redis, self.data_sources,
self.polkadot_api_endpoint,
TestInternalConf)
self.redis_alive_key_timeout = \
TestInternalConf.redis_blockchain_monitor_alive_key_timeout
self.alive_key_timeout = \
TestInternalConf.redis_blockchain_monitor_alive_key_timeout
def setUp(self) -> None:
self.logger = logging.getLogger('dummy')
self.monitor_name = 'testnodemonitor'
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel], TestInternalConf)
self.db = TestInternalConf.redis_test_database
self.host = TestUserConf.redis_host
self.port = TestUserConf.redis_port
self.password = TestUserConf.redis_password
self.redis = RedisApi(self.logger, self.db, self.host, self.port,
self.password)
self.redis.delete_all_unsafe()
try:
self.redis.ping_unsafe()
except RedisConnectionError:
self.fail('Redis is not online.')
self.node_monitor_max_catch_up_blocks = \
TestInternalConf.node_monitor_max_catch_up_blocks
self.node = None
self.archive_alerts_disabled = False
self.data_sources = []
self.monitor = NodeMonitor(self.monitor_name, self.channel_set,
self.logger,
self.node_monitor_max_catch_up_blocks,
self.redis, self.node,
self.archive_alerts_disabled,
self.data_sources, TestInternalConf)
self.dummy_last_height_checked = 1000
self.redis_alive_key_timeout = \
TestInternalConf.redis_node_monitor_alive_key_timeout
def setUp(self) -> None:
self.blockchain_name = 'testblockchain'
self.redis_prefix = self.blockchain_name
self.logger = logging.getLogger('dummy')
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel])
self.db = TestInternalConf.redis_test_database
self.host = TestUserConf.redis_host
self.port = TestUserConf.redis_port
self.password = TestUserConf.redis_password
self.redis = RedisApi(self.logger, self.db, self.host, self.port,
self.password)
self.redis.delete_all_unsafe()
try:
self.redis.ping_unsafe()
except RedisConnectionError:
self.fail('Redis is not online.')
self.blockchain = Blockchain(self.blockchain_name, self.redis)
self.dummy_referendum_count = 10
self.dummy_council_prop_count = 10
self.dummy_public_prop_count = 10
self.dummy_validator_set_size = 120
def setUp(self) -> None:
self.alerter_name = 'testalerter'
self.logger = logging.getLogger('dummy')
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel], TestInternalConf)
self.par = PeriodicAliveReminder(timedelta(), self.channel_set, None)
def setUp(self) -> None:
self.logger = logging.getLogger('dummy')
self.monitor_name = 'testmonitor'
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel])
self.repo_name = 'dummy/repository/'
self.releases_page = 'dummy.releases.page'
self.redis_prefix = TestInternalConf.redis_github_releases_key_prefix
self.db = TestInternalConf.redis_test_database
self.host = TestUserConf.redis_host
self.port = TestUserConf.redis_port
self.password = TestUserConf.redis_password
self.redis = RedisApi(self.logger, self.db, self.host,
self.port, self.password)
self.redis.delete_all_unsafe()
try:
self.redis.ping_unsafe()
except RedisConnectionError:
self.fail('Redis is not online.')
self.monitor = GitHubMonitor(self.monitor_name, self.channel_set,
self.logger, self.redis, self.repo_name,
self.releases_page, self.redis_prefix)
self.monitor._internal_conf = TestInternalConf
def setUp(self) -> None:
self.logger = logging.getLogger('dummy')
self.wrapper = PolkadotApiWrapper(self.logger, self.api_endpoint)
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel], TestInternalConf)
self.params = {'websocket': self.ws_url}
def set_virtual_memory_usage(self, new_virtual_memory_usage: int,
channels: ChannelSet, logger: logging.Logger) \
-> None:
logger.debug(
'%s set_virtual_memory_usage: '
'set_virtual_memory_usage(currently)=%s, channels=%s', self,
self._virtual_memory_usage, channels)
if self._virtual_memory_usage is None:
self._virtual_memory_usage = new_virtual_memory_usage
return
if self._virtual_memory_usage != new_virtual_memory_usage:
channels.alert_info(NewVirtualMemoryUsageAlert(self.name, \
new_virtual_memory_usage))
self._virtual_memory_usage = new_virtual_memory_usage
def set_disabled(self, now_is_disabled: bool, session: int,
channels: ChannelSet, logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s set_disabled: _disabled(currently)=%s, '
'channels=%s', self, self.is_disabled, channels)
if self.is_disabled not in [now_is_disabled, None]:
if now_is_disabled:
channels.alert_critical(
ValidatorHasBeenDisabledInSessionAlert(self.name, session))
else:
channels.alert_info(
ValidatorIsNoLongerDisabledInSessionAlert(
self.name, session))
self._disabled = now_is_disabled
def setUp(self) -> None:
self.logger = logging.getLogger('dummy')
self.monitor_name = 'testblockchainmonitor'
self.counter_channel = CounterChannel(self.logger)
self.channel_set = ChannelSet([self.counter_channel], TestInternalConf)
self.redis = None
self.data_sources = []
self.polkadot_api_endpoint = 'api_endpoint'
self.dummy_ws_url_1 = "11.22.33.11:9944"
self.dummy_ws_url_2 = "11.22.33.12:9944"
self.dummy_ws_url_3 = "11.22.33.13:9944"
self.dummy_ws_url_4 = "11.22.33.14:9944"
self.dummy_node_name_1 = "testnode1"
self.dummy_node_name_2 = "testnode2"
self.dummy_node_name_3 = "testnode3"
self.dummy_node_name_4 = "testnode4"
self.dummy_chain_name = "testchain"
self.validator_stash_account_address = "DFJGDF8G898fdghb98dg9wetg9we00w"
self.dummy_full_node_1 = Node(name=self.dummy_node_name_1,
ws_url=self.dummy_ws_url_1,
node_type=NodeType.NON_VALIDATOR_FULL_NODE,
stash_account_address='',
chain=self.dummy_chain_name, redis=None,
is_archive_node=True,
internal_conf=TestInternalConf)
self.dummy_full_node_2 = Node(name=self.dummy_node_name_2,
ws_url=self.dummy_ws_url_2,
node_type=NodeType.NON_VALIDATOR_FULL_NODE,
stash_account_address='',
chain=self.dummy_chain_name, redis=None,
is_archive_node=True,
internal_conf=TestInternalConf)
self.dummy_full_node_3 = Node(name=self.dummy_node_name_3,
ws_url=self.dummy_ws_url_3,
node_type=NodeType.NON_VALIDATOR_FULL_NODE,
stash_account_address='',
chain=self.dummy_chain_name, redis=None,
is_archive_node=True,
internal_conf=TestInternalConf)
self.dummy_validator_node_1 = Node(
name=self.dummy_node_name_4, ws_url=self.dummy_ws_url_4,
node_type=NodeType.VALIDATOR_FULL_NODE,
stash_account_address=self.validator_stash_account_address,
chain=self.dummy_chain_name, redis=None, is_archive_node=False,
internal_conf=TestInternalConf)
self.dummy_blockchain = Blockchain(self.dummy_chain_name, None)
self.monitor = BlockchainMonitor(self.monitor_name,
self.dummy_blockchain,
self.channel_set, self.logger,
self.redis, self.data_sources,
self.polkadot_api_endpoint,
TestInternalConf)
self.dummy_referendum_count = 10
self.dummy_public_prop_count = 10
self.dummy_council_prop_count = 10
self.dummy_validator_set_size = 120
def set_process_cpu_seconds_total(self, new_process_cpu_seconds: int,
channels: ChannelSet,
logger: logging.Logger) \
-> None:
logger.debug(
'%s set_process_cpu_seconds: '
'set_process_cpu_seconds(currently)=%s, channels=%s', self,
self._process_cpu_seconds_total, channels)
if self._process_cpu_seconds_total is None:
self._process_cpu_seconds_total = new_process_cpu_seconds
return
if self._process_cpu_seconds_total != new_process_cpu_seconds:
channels.alert_info(NewProcessCPUSecondsTotalAlert(self.name, \
new_process_cpu_seconds))
self._process_cpu_seconds_total = new_process_cpu_seconds
def set_referendum_count(self, new_referendum_count: int,
channels: ChannelSet, logger: logging.Logger,
referendum_info: PolkadotWrapperType = None) \
-> None:
logger.debug('%s set_referendum_count: referendum_count(currently)=%s, '
'channels=%s', self, self.referendum_count, channels)
# If referendum is new and still ongoing, alert the operator.
if self._referendum_count not in [None, new_referendum_count] \
and referendum_info is not None:
if 'Ongoing' in referendum_info:
end_block = parse_int_from_string(str(
referendum_info['Ongoing']['end']
))
channels.alert_info(NewReferendumAlert(
self.referendum_count, end_block)
)
self._referendum_count = new_referendum_count
def set_as_up(self, channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug('%s set_as_up: is_down(currently)=%s, channels=%s', self,
self.is_down, channels)
# Alert if node was down
if self.is_down:
# Only send accessible alert if inaccessible alert was sent
if self._initial_downtime_alert_sent:
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
channels.alert_info(
NowAccessibleAlert(self.name, went_down_at, downtime))
# Reset downtime-related values
self._downtime_alert_limiter.reset()
self._went_down_at = None
