def __init__(self, logger: logging.Logger, db: int,
host: str = 'localhost', port: int = 6379,
password: str = '', namespace: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60)) \
-> None:
self._logger = logger
# If no password was given establish passwordless connection
# otherwise establish authenticated connection
if password == '':
self._redis = redis.Redis(host=host, port=port, db=db)
else:
self._redis = redis.Redis(host=host,
port=port,
db=db,
password=password)
# Unique alerter identification
self._namespace = namespace
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
# Variable to show that Redis is live and being used.
self._is_live = True
self._set_as_live()
self._logger.info('Redis initialised.')
def __init__(self, name: str, rpc_url: Optional[str], node_type: NodeType,
pubkey: Optional[str], network: str, redis: Optional[RedisApi],
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self.rpc_url = rpc_url
self.node_type = node_type
self.pubkey = pubkey
self.network = network
self._redis = redis
self._redis_enabled = redis is not None
self._redis_prefix = self.name + "@" + self.network
self._went_down_at = None
self._consecutive_blocks_missed = 0
self._voting_power = None
self._catching_up = False
self._no_of_peers = None
self._initial_downtime_alert_sent = False
self._validator_peer_danger_boundary = \
internal_conf.validator_peer_danger_boundary
self._validator_peer_safe_boundary = \
internal_conf.validator_peer_safe_boundary
self._full_node_peer_danger_boundary = \
internal_conf.full_node_peer_danger_boundary
self._missed_blocks_danger_boundary = \
internal_conf.missed_blocks_danger_boundary
self._downtime_alert_limiter = TimedTaskLimiter(
internal_conf.downtime_alert_time_interval)
self._timed_block_miss_tracker = TimedOccurrenceTracker(
internal_conf.max_missed_blocks_in_time_interval,
internal_conf.max_missed_blocks_time_interval)
def __init__(self, logger: logging.Logger, db_name: str,
host: str = 'localhost', port: int = 27017,
username: str = '', password: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60),
timeout_ms: int = 10000) \
-> None:
self._logger = logger
self._db_name = db_name
if password == '':
self._client = MongoClient(
host=host, port=port, connectTimeoutMS=timeout_ms,
socketTimeoutMS=timeout_ms, serverSelectionTimeoutMS=timeout_ms)
else:
self._client = MongoClient(
host=host, port=port, connectTimeoutMS=timeout_ms,
socketTimeoutMS=timeout_ms, serverSelectionTimeoutMS=timeout_ms,
username=username, password=password)
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
self._is_live = True # This is necessary to initialise the variable
self._set_as_live()
self._logger.info('Mongo initialised.')
def __init__(self, logger: logging.Logger):
self._logger = logger
self._api_down = False
# If after 15 seconds connection between a validator monitor and API
# server is not re-established, user is informed via a critical alert
self._api_down_limiter = TimedTaskLimiter(timedelta(seconds=int(15)))
self._critical_alert_sent = False
def __init__(self, logger: logging.Logger, api_endpoint: str):
self._logger = logger
self._api_endpoint = api_endpoint
self._api_down = False
self._critical_alert_sent = False
# If 15 seconds pass since a validator monitor lost connection with the
# API server, the user is informed via critical alert once
self._api_down_limiter = TimedTaskLimiter(timedelta(seconds=int(15)))
def _create_state_for_system(self, system_id: str) -> None:
# initialise initial downtime alert sent
if system_id not in self._system_initial_downtime_alert_sent:
self._system_initial_downtime_alert_sent[system_id] = False
# initialise timed task limiters
if system_id not in self._system_critical_timed_task_limiters:
open_fd = self.alerts_configs.open_file_descriptors
cpu_use = self.alerts_configs.system_cpu_usage
storage = self.alerts_configs.system_storage_usage
ram_use = self.alerts_configs.system_ram_usage
is_down = self.alerts_configs.system_is_down
self._system_critical_timed_task_limiters[system_id] = {}
system_critical_limiters = \
self._system_critical_timed_task_limiters[system_id]
open_fd_critical_repeat = \
convert_to_float_if_not_none_and_not_empty_str(
open_fd['critical_repeat'],
timedelta.max.total_seconds() - 1)
cpu_use_critical_repeat = \
convert_to_float_if_not_none_and_not_empty_str(
cpu_use['critical_repeat'],
timedelta.max.total_seconds() - 1)
storage_critical_repeat = \
convert_to_float_if_not_none_and_not_empty_str(
storage['critical_repeat'],
timedelta.max.total_seconds() - 1)
ram_use_critical_repeat = \
convert_to_float_if_not_none_and_not_empty_str(
ram_use['critical_repeat'],
timedelta.max.total_seconds() - 1)
is_down_critical_repeat = \
convert_to_float_if_not_none_and_not_empty_str(
is_down['critical_repeat'],
timedelta.max.total_seconds() - 1)
system_critical_limiters[_OPEN_FD_LIMITER_NAME] = TimedTaskLimiter(
timedelta(seconds=float(open_fd_critical_repeat)))
system_critical_limiters[_CPU_USE_LIMITER_NAME] = TimedTaskLimiter(
timedelta(seconds=float(cpu_use_critical_repeat)))
system_critical_limiters[_STORAGE_USE_LIMITER_NAME] = \
TimedTaskLimiter(
timedelta(seconds=float(storage_critical_repeat))
)
system_critical_limiters[_RAM_USE_LIMITER_NAME] = TimedTaskLimiter(
timedelta(seconds=float(ram_use_critical_repeat)))
system_critical_limiters[_IS_DOWN_LIMITER_NAME] = TimedTaskLimiter(
timedelta(seconds=float(is_down_critical_repeat)))
def start_github_monitor(github_monitor: GitHubMonitor, monitor_period: int,
logger: logging.Logger):
# Set up alert limiter
github_error_alert_limiter = TimedTaskLimiter(
InternalConf.github_error_interval_seconds)
# Start
while True:
# Read GitHub releases page
try:
logger.debug('Reading %s.', github_monitor.releases_page)
github_monitor.monitor()
logger.debug('Done reading %s.', github_monitor.releases_page)
# Save all state
github_monitor.save_state()
# Reset alert limiter
github_error_alert_limiter.reset()
except (ConnectionError, ReadTimeout) as conn_err:
if github_error_alert_limiter.can_do_task():
github_monitor.channels.alert_error(
CannotAccessGitHubPageAlert(github_monitor.releases_page))
github_error_alert_limiter.did_task()
logger.error('Error occurred when accessing {}: {}.'
''.format(github_monitor.releases_page, conn_err))
except JSONDecodeError as json_error:
logger.error(json_error) # Ignore such errors
except Exception as e:
logger.error(e)
raise e
# Sleep
logger.debug('Sleeping for %s seconds.', monitor_period)
time.sleep(monitor_period)
def __init__(self, logger: logging.Logger, db: int,
host: str = 'localhost', port: int = 6379,
password: str = '', namespace: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60)) \
-> None:
self._logger = logger
if password == '':
self._redis = redis.Redis(host=host, port=port, db=db)
else:
self._redis = redis.Redis(host=host, port=port, db=db,
password=password)
self._namespace = namespace
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
self._is_live = True # This is necessary to initialise the variable
self._set_as_live()
self._logger.info('Redis initialised.')
def __init__(self, logger: logging.Logger, host: str = 'localhost',
port: int = 5672, username: str = '', password: str = '',
connection_check_time_interval: timedelta = timedelta(
seconds=30)) \
-> None:
self._logger = logger
self._host = host
self._connection = None
self._channel = None
self._port = port # Port used by the AMQP 0-9-1 and 1.0 clients
self._username = username
self._password = password
# The limiter will restrict usage of RabbitMQ whenever it is running
# into problems so that recovery is faster.
self._connection_check_limiter = TimedTaskLimiter(
connection_check_time_interval)
self._connection_check_time_interval_seconds = \
connection_check_time_interval.total_seconds()
# A boolean variable which keeps track of the connection status with
# RabbitMQ
self._is_connected = False
class OasisApiWrapper:
def __init__(self, logger: logging.Logger):
self._logger = logger
self._api_down = False
# If after 15 seconds connection between a validator monitor and API
# server is not re-established, user is informed via a critical alert
self._api_down_limiter = TimedTaskLimiter(timedelta(seconds=int(15)))
self._critical_alert_sent = False
@property
def is_api_down(self) -> bool:
return self._api_down
def set_api_as_down(self, monitor: str, is_validator_monitor,
channels: ChannelSet) -> None:
self._logger.debug('%s set_api_as_down: api_down(currently)=%s, '
'channels=%s', self, self._api_down, channels)
# If API is suddenly down, inform via a warning alert
if not self._api_down:
channels.alert_warning(ApiIsDownAlert(monitor))
self._api_down_limiter.did_task()
# If 15 seconds pass since a validator monitor lost connection with the
# API server, the user is informed via critical alert once
if is_validator_monitor and self._api_down_limiter.can_do_task() \
and not self._critical_alert_sent:
channels.alert_critical(ApiIsDownAlert(monitor))
self._critical_alert_sent = True
self._api_down = True
def set_api_as_up(self, monitor: str, channels: ChannelSet) -> None:
self._logger.debug('%s set_api_as_down: api_down(currently)=%s, '
'channels=%s', self, self._api_down, channels)
if self._api_down:
channels.alert_info(ApiIsUpAgainAlert(monitor))
self._critical_alert_sent = False
self._api_down = False
def get_web_sockets_connected_to_an_api(self, api_url: str) \
-> OasisWrapperType:
endpoint = api_url + '/api/getconnectionslist'
params = {}
return get_oasis_json(endpoint, params, self._logger)
def ping_api(self, api_url: str) -> OasisWrapperType:
endpoint = api_url + '/api/ping'
params = {}
return get_oasis_json(endpoint, params, self._logger)
def ping_node(self, api_url: str, node_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/pingnode'
params = {'name': node_name}
return get_oasis_json(endpoint, params, self._logger)
def get_node(self, api_url: str, node_name: str,
node_id: str) -> OasisWrapperType:
endpoint = api_url + '/api/registry/node'
params = {'name': node_name, 'nodeID': node_id}
return get_oasis_json(endpoint, params, self._logger)
def get_consensus_genesis(self, api_url: str,
node_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/consensus/genesis'
params = {'name': node_name}
return get_oasis_json(endpoint, params, self._logger)
def get_is_syncing(self, api_url: str,
node_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/nodecontroller/synced'
params = {'name': node_name}
return get_oasis_json(endpoint, params, self._logger)
def get_prometheus_gauge(self, api_url: str, node_name: str,
gauge_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/prometheus/gauge'
params = {'name': node_name, 'gauge': gauge_name}
return get_oasis_json(endpoint, params, self._logger)
def get_prometheus_counter(self, api_url: str, node_name: str,
counter_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/prometheus/counter'
params = {'name': node_name, 'counter': counter_name}
return get_oasis_json(endpoint, params, self._logger)
def get_node_exporter_gauge(self, api_url: str, node_name: str,
gauge_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/exporter/gauge'
params = {'name': node_name, 'gauge': gauge_name}
return get_oasis_json(endpoint, params, self._logger)
def get_node_export_counter(self, api_url: str, node_name: str,
counter_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/exporter/counter'
params = {'name': node_name, 'counter': counter_name}
return get_oasis_json(endpoint, params, self._logger)
def get_block_header(self, api_url: str,
node_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/consensus/blockheader'
params = {'name': node_name}
return get_oasis_json(endpoint, params, self._logger)
def get_session_validators(self, api_url: str,
node_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/scheduler/validators'
params = {'name': node_name}
return get_oasis_json(endpoint, params, self._logger)
def get_consensus_block(self, api_url: str,
node_name: str) -> OasisWrapperType:
endpoint = api_url + '/api/consensus/block'
params = {'name': node_name}
return get_oasis_json(endpoint, params, self._logger)
def get_block_header_height(self, api_url: str,
node_name: str,
height: str) -> OasisWrapperType:
endpoint = api_url + '/api/consensus/blockheader'
params = {'name': node_name, 'height': height}
return get_oasis_json(endpoint, params, self._logger)
def get_signed_blocks(self, api_url: str,
node_name: str, height: str) -> OasisWrapperType:
endpoint = api_url + '/api/consensus/blocklastcommit'
params = {'name': node_name, 'height': height}
return get_oasis_json(endpoint, params, self._logger)
def get_tendermint_address(self, api_url: str,
consensus_public_key: str) -> OasisWrapperType:
endpoint = api_url + '/api/consensus/pubkeyaddress'
params = {'consensus_public_key': consensus_public_key}
return get_oasis_json(endpoint, params, self._logger)
def get_registry_node(self, api_url: str, node_name: str,
node_id: str) -> OasisWrapperType:
endpoint = api_url + '/api/registry/node'
params = {'name': node_name, 'nodeID': node_id}
return get_oasis_json(endpoint, params, self._logger)
def get_staking_account(self, api_url: str, node_name: str,
address: str) -> OasisWrapperType:
endpoint = api_url + '/api/staking/account'
params = {'name': node_name, 'address': address}
return get_oasis_json(endpoint, params, self._logger)
def get_staking_address(self, api_url: str, public_key: \
str) -> OasisWrapperType:
endpoint = api_url + '/api/staking/publickeytoaddress'
params = {'pubKey': public_key}
return get_oasis_json(endpoint, params, self._logger)
def get_staking_delegations(self, api_url: str, node_name: str,
address: str) -> OasisWrapperType:
endpoint = api_url + '/api/staking/delegations'
params = {'name': node_name, 'address': address}
return get_oasis_json(endpoint, params, self._logger)
def get_events_by_height(self, api_url: str, node_name: str,
height: str) -> OasisWrapperType:
endpoint = api_url + '/api/staking/events'
params = {'name': node_name, 'height': height}
return get_oasis_json(endpoint, params, self._logger)
class PolkadotApiWrapper:
def __init__(self, logger: logging.Logger, api_endpoint: str):
self._logger = logger
self._api_endpoint = api_endpoint
self._api_down = False
self._critical_alert_sent = False
# If 15 seconds pass since a validator monitor lost connection with the
# API server, the user is informed via critical alert once
self._api_down_limiter = TimedTaskLimiter(timedelta(seconds=int(15)))
@property
def api_endpoint(self) -> str:
return self._api_endpoint
@property
def is_api_down(self) -> bool:
return self._api_down
def get_block_hash(self, ws_url: str, block_number: int) -> \
PolkadotWrapperType:
api_call = 'chain/getBlockHash'
endpoint = self._api_endpoint + '/api/rpc/' + api_call
params = {'websocket': ws_url, 'block_number': block_number}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_finalized_head(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'chain/getFinalizedHead'
endpoint = self._api_endpoint + '/api/rpc/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_header(self, ws_url: str, block_hash: str) -> PolkadotWrapperType:
api_call = 'chain/getHeader'
endpoint = self._api_endpoint + '/api/rpc/' + api_call
params = {'websocket': ws_url, 'hash': block_hash}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_system_chain(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'system/chain'
endpoint = self._api_endpoint + '/api/rpc/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_system_health(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'system/health'
endpoint = self._api_endpoint + '/api/rpc/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_council_members(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'council/members'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_council_proposal_count(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'council/proposalCount'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_public_proposal_count(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'democracy/publicPropCount'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_referendum_count(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'democracy/referendumCount'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_referendum_info_of(self, ws_url: str, referendum_index: int) \
-> PolkadotWrapperType:
api_call = 'democracy/referendumInfoOf'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url, 'referendum_index': referendum_index}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_authored_blocks(self, ws_url: str, session_index: int,
validator_id: str) -> PolkadotWrapperType:
api_call = 'imOnline/authoredBlocks'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {
'websocket': ws_url,
'validator_id': validator_id,
'session_index': session_index
}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_received_heartbeats(self, ws_url: str, session_index: int,
auth_index: int) -> PolkadotWrapperType:
api_call = 'imOnline/receivedHeartbeats'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {
'websocket': ws_url,
'auth_index': auth_index,
'session_index': session_index
}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_current_index(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'session/currentIndex'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_disabled_validators(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'session/disabledValidators'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_session_validators(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'session/validators'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_derive_staking_validators(self, ws_url: str) \
-> PolkadotWrapperType:
api_call = 'staking/validators'
endpoint = self._api_endpoint + '/api/derive/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_eras_stakers(self, ws_url: str, stash_account_address: str) \
-> PolkadotWrapperType:
api_call = 'staking/erasStakers'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url, 'account_id': stash_account_address}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_active_era(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'staking/activeEra'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_events(self, ws_url: str, block_hash: Optional[str]) \
-> PolkadotWrapperType:
api_call = 'system/events'
endpoint = self._api_endpoint + '/api/query/' + api_call
params = {'websocket': ws_url, 'block_hash': block_hash}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_slash_amount(self, ws_url: str, block_hash: Optional[str],
stash_account_address: str) -> PolkadotWrapperType:
api_call = 'custom/getSlashAmount'
endpoint = self._api_endpoint + '/api/' + api_call
params = {
'websocket': ws_url,
'block_hash': block_hash,
'account_address': stash_account_address
}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def get_web_sockets_connected_to_an_api(self) -> PolkadotWrapperType:
endpoint = self._api_endpoint + '/api/getConnectionsList'
params = {}
return get_polkadot_json(endpoint, params, self._logger)
def ping_api(self) -> PolkadotWrapperType:
endpoint = self._api_endpoint + '/api/pingApi'
params = {}
return get_polkadot_json(endpoint, params, self._logger)
def ping_node(self, ws_url: str) -> PolkadotWrapperType:
api_call = 'pingNode'
endpoint = self._api_endpoint + '/api/' + api_call
params = {'websocket': ws_url}
return get_polkadot_json(endpoint, params, self._logger, api_call)
def set_api_as_down(self, monitor: str, is_validator_monitor,
channels: ChannelSet) -> None:
self._logger.debug(
'%s set_api_as_down: api_down(currently)=%s, '
'channels=%s', self, self._api_down, channels)
# If API is suddenly down, inform via a warning alert
if not self._api_down:
channels.alert_warning(ApiIsDownAlert(monitor))
self._api_down_limiter.did_task()
# If 15 seconds pass since a validator monitor lost connection with the
# API server, the user is informed via critical alert once
if is_validator_monitor and self._api_down_limiter.can_do_task() \
and not self._critical_alert_sent:
channels.alert_critical(ApiIsDownAlert(monitor))
self._critical_alert_sent = True
self._api_down = True
def set_api_as_up(self, monitor: str, channels: ChannelSet) -> None:
self._logger.debug(
'%s set_api_as_down: api_down(currently)=%s, '
'channels=%s', self, self._api_down, channels)
if self._api_down:
channels.alert_info(ApiIsUpAgainAlert(monitor))
self._critical_alert_sent = False
self._api_down = False
class Node:
def __init__(self, name: str, rpc_url: Optional[str], node_type: NodeType,
pubkey: Optional[str], network: str, redis: Optional[RedisApi],
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self.rpc_url = rpc_url
self.node_type = node_type
self.pubkey = pubkey
self.network = network
self._redis = redis
self._redis_enabled = redis is not None
self._redis_prefix = self.name + "@" + self.network
self._went_down_at = None
self._consecutive_blocks_missed = 0
self._voting_power = None
self._catching_up = False
self._no_of_peers = None
self._experiencing_delays_alert_sent = False
self._initial_downtime_alert_sent = False
self._validator_peer_danger_boundary = \
internal_conf.validator_peer_danger_boundary
self._validator_peer_safe_boundary = \
internal_conf.validator_peer_safe_boundary
self._full_node_peer_danger_boundary = \
internal_conf.full_node_peer_danger_boundary
self._missed_blocks_danger_boundary = \
internal_conf.missed_blocks_danger_boundary
self._downtime_initial_alert_delayer = TimedTaskLimiter(
internal_conf.downtime_initial_alert_delay)
self._downtime_reminder_limiter = TimedTaskLimiter(
internal_conf.downtime_reminder_interval_seconds)
self._timed_block_miss_tracker = TimedOccurrenceTracker(
internal_conf.max_missed_blocks_in_time_interval,
internal_conf.max_missed_blocks_time_interval)
self._change_in_voting_power_threshold = \
internal_conf.change_in_voting_power_threshold
def __str__(self) -> str:
return self.name
@property
def is_validator(self) -> bool:
return self.node_type == NodeType.VALIDATOR_FULL_NODE
@property
def is_down(self) -> bool:
return self._went_down_at is not None
@property
def is_missing_blocks(self) -> bool:
return self.consecutive_blocks_missed_so_far > 0
@property
def consecutive_blocks_missed_so_far(self) -> int:
return self._consecutive_blocks_missed
@property
def voting_power(self) -> int:
return self._voting_power
@property
def catching_up(self) -> bool:
return self._catching_up
@property
def no_of_peers(self) -> int:
return self._no_of_peers
def status(self) -> str:
return "voting_power={}, catching_up={}, number_of_peers={}".format(
self.voting_power, self.catching_up, self.no_of_peers)
def load_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, load any previously stored state
if self._redis_enabled:
self._went_down_at = self._redis.get(
self._redis_prefix + '_went_down_at', None)
self._consecutive_blocks_missed = self._redis.get_int(
self._redis_prefix + '_consecutive_blocks_missed', 0)
self._voting_power = self._redis.get_int(
self._redis_prefix + '_voting_power', None)
self._catching_up = self._redis.get_bool(
self._redis_prefix + '_catching_up', False)
self._no_of_peers = self._redis.get_int(
self._redis_prefix + '_no_of_peers', None)
# String to actual values
if self._went_down_at is not None:
try:
self._went_down_at = \
dateutil.parser.parse(self._went_down_at)
except (TypeError, ValueError) as e:
logger.error('Error when parsing '
'_went_down_at: %s', e)
self._went_down_at = None
logger.debug(
'Restored %s state: _went_down_at=%s, '
'_consecutive_blocks_missed=%s, _voting_power=%s, '
'_catching_up=%s, _no_of_peers=%s',
self.name, self._went_down_at, self._consecutive_blocks_missed,
self._voting_power, self._catching_up, self._no_of_peers)
def save_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, store the current state
if self._redis_enabled:
logger.debug(
'Saving %s state: _went_down_at=%s, _consecutive_blocks_missed'
'=%s, _voting_power=%s, _catching_up=%s, _no_of_peers=%s',
self.name, self._went_down_at, self._consecutive_blocks_missed,
self._voting_power, self._catching_up, self._no_of_peers)
# Set values
self._redis.set_multiple({
self._redis_prefix + '_went_down_at': str(self._went_down_at),
self._redis_prefix + '_consecutive_blocks_missed':
self._consecutive_blocks_missed,
self._redis_prefix + '_voting_power': self._voting_power,
self._redis_prefix + '_catching_up': str(self._catching_up),
self._redis_prefix + '_no_of_peers': self._no_of_peers
})
def set_as_down(self, channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# If node was not down before, do not alert for now, just in case it's
# a connection hiccup but take note of the start of the downtime
if not self.is_down:
self._went_down_at = datetime.now()
self._experiencing_delays_alert_sent = False
self._initial_downtime_alert_sent = False
self._downtime_initial_alert_delayer.did_task()
# If node was down and we have not yet sent an alert about this, send
# an informational 'experiencing delays' alert as a warning
elif not self._experiencing_delays_alert_sent:
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._experiencing_delays_alert_sent = True
# If we have not yet sent an initial downtime alert, and enough
# time has passed for it, then send an initial alert
elif not self._initial_downtime_alert_sent:
if self._downtime_initial_alert_delayer.can_do_task():
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_major(CannotAccessNodeAlert(
self.name, self._went_down_at, downtime))
else:
channels.alert_minor(CannotAccessNodeAlert(
self.name, self._went_down_at, downtime))
self._downtime_reminder_limiter.did_task()
self._initial_downtime_alert_sent = True
# If we already sent an initial alert and enough time has passed
# for a reminder alert, then send a reminder alert
else:
if self._downtime_reminder_limiter.can_do_task():
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_major(StillCannotAccessNodeAlert(
self.name, self._went_down_at, downtime))
else:
channels.alert_minor(StillCannotAccessNodeAlert(
self.name, self._went_down_at, downtime))
self._downtime_reminder_limiter.did_task()
def set_as_up(self, channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug('%s set_as_up: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert if node was down
if self.is_down:
# Only send accessible alert if inaccessible alert was sent
if self._initial_downtime_alert_sent:
downtime = strfdelta(datetime.now() - self._went_down_at,
"{hours}h, {minutes}m, {seconds}s")
channels.alert_info(NowAccessibleAlert(
self.name, self._went_down_at, downtime))
# Reset downtime-related values
self._downtime_initial_alert_delayer.reset()
self._downtime_reminder_limiter.reset()
self._went_down_at = None
def add_missed_block(self, block_height: int, block_time: datetime,
missing_validators: int, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
# Calculate the actual blocks missed as of when this function was called
blocks_missed = self._consecutive_blocks_missed + 1
# Variable alias for improved readability
danger = self._missed_blocks_danger_boundary
logger.debug(
'%s add_missed_block: before=%s, new=%s, missing_validators = %s, '
'channels=%s', self, self.consecutive_blocks_missed_so_far,
blocks_missed, missing_validators, channels)
# Let timed tracker know that block missed
self._timed_block_miss_tracker.action_happened(at_time=block_time)
# Alert (varies depending on whether was already missing blocks)
if not self.is_missing_blocks:
pass # Do not alert on first missed block
elif 2 <= blocks_missed < danger:
channels.alert_info(MissedBlocksAlert(
self.name, blocks_missed, block_height, missing_validators)
) # 2+ blocks missed inside danger range
elif blocks_missed == 5:
channels.alert_minor(MissedBlocksAlert(
self.name, blocks_missed, block_height, missing_validators)
) # reached danger range
elif blocks_missed >= max(10, danger) and blocks_missed % 10 == 0:
channels.alert_major(MissedBlocksAlert(
self.name, blocks_missed, block_height, missing_validators)
) # Every (10N)th block missed for N >= 1 inside danger range
self._timed_block_miss_tracker.reset()
if self._timed_block_miss_tracker.too_many_occurrences(block_time):
blocks_in_interval = self._timed_block_miss_tracker.max_occurrences
time_interval = self._timed_block_miss_tracker.time_interval_pretty
channels.alert_major(TimedMissedBlocksAlert(
self.name, blocks_in_interval, time_interval,
block_height, missing_validators)
) # More blocks missed than is acceptable in the time interval
self._timed_block_miss_tracker.reset()
# Update consecutive blocks missed
self._consecutive_blocks_missed = blocks_missed
def clear_missed_blocks(self, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
logger.debug(
'%s clear_missed_blocks: channels=%s', self, channels)
# Alert if validator was missing blocks (only if more than 1 block)
if self.is_missing_blocks and self._consecutive_blocks_missed > 1:
channels.alert_info(NoLongerMissingBlocksAlert(
self.name, self._consecutive_blocks_missed))
# Reset missed blocks related values
self._consecutive_blocks_missed = 0
def set_voting_power(self, new_voting_power: int, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
logger.debug(
'%s set_voting_power: before=%s, new=%s, channels=%s',
self, self.voting_power, new_voting_power, channels)
# Alert if voting power has changed
if self.voting_power not in [None, new_voting_power]:
if self.is_validator and new_voting_power == 0: # N to 0
channels.alert_major(VotingPowerDecreasedAlert(
self.name, self.voting_power, new_voting_power))
elif self.is_validator and self.voting_power == 0: # 0 to N
channels.alert_info(VotingPowerIncreasedAlert(
self.name, self.voting_power, new_voting_power))
else: # Any change
diff = new_voting_power - self.voting_power
if abs(diff) > self._change_in_voting_power_threshold:
if diff > 0:
channels.alert_info(VotingPowerIncreasedByAlert(
self.name, self.voting_power, new_voting_power))
else:
channels.alert_info(VotingPowerDecreasedByAlert(
self.name, self.voting_power, new_voting_power))
# Update voting power
self._voting_power = new_voting_power
def set_catching_up(self, now_catching_up: bool,
channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug(
'%s set_catching_up: before=%s, new=%s, channels=%s',
self, self.catching_up, now_catching_up, channels)
# Alert if catching up has changed
if not self.catching_up and now_catching_up:
channels.alert_minor(IsCatchingUpAlert(self.name))
elif self.catching_up and not now_catching_up:
channels.alert_info(IsNoLongerCatchingUpAlert(self.name))
# Update catching-up
self._catching_up = now_catching_up
def set_no_of_peers(self, new_no_of_peers: int, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug(
'%s set_no_of_peers: before=%s, new=%s, channels=%s',
self, self.no_of_peers, new_no_of_peers, channels)
# Variable alias for improved readability
if self.is_validator:
danger = self._validator_peer_danger_boundary
safe = self._validator_peer_safe_boundary
else:
danger = self._full_node_peer_danger_boundary
safe = None
# Alert if number of peers has changed
if self.no_of_peers not in [None, new_no_of_peers]:
if self.is_validator:
if new_no_of_peers <= self._validator_peer_safe_boundary:
# beneath safe boundary
if new_no_of_peers > self.no_of_peers: # increase
channels.alert_info(PeersIncreasedAlert(
self.name, self.no_of_peers, new_no_of_peers))
elif new_no_of_peers > danger:
# decrease outside danger range
channels.alert_minor(PeersDecreasedAlert(
self.name, self.no_of_peers, new_no_of_peers))
else: # decrease inside danger range
channels.alert_major(PeersDecreasedAlert(
self.name, self.no_of_peers, new_no_of_peers))
elif self._no_of_peers <= self._validator_peer_safe_boundary \
< new_no_of_peers:
# increase outside safe range for the first time
channels.alert_info(
PeersIncreasedOutsideSafeRangeAlert(self.name, safe))
else:
if new_no_of_peers > self.no_of_peers: # increase
if new_no_of_peers <= danger:
# increase inside danger range
channels.alert_info(PeersIncreasedAlert(
self.name, self.no_of_peers, new_no_of_peers))
elif self.no_of_peers <= danger < new_no_of_peers:
# increase outside danger range
channels.alert_info(
PeersIncreasedOutsideDangerRangeAlert(
self.name, danger))
elif new_no_of_peers > danger: # decrease outside danger range
pass
else: # decrease inside danger range
channels.alert_minor(PeersDecreasedAlert(
self.name, self.no_of_peers, new_no_of_peers))
# Update number of peers
self._no_of_peers = new_no_of_peers
class TestTimedTaskLimiter(unittest.TestCase):
def setUp(self) -> None:
self.interval_seconds = 2
self.interval_seconds_with_error_margin = self.interval_seconds + 0.5
self.interval_timedelta = timedelta(seconds=self.interval_seconds)
self.ttl = TimedTaskLimiter(self.interval_timedelta)
def test_time_interval_is_supplied_time_interval(self):
self.assertEqual(self.ttl.time_interval, self.interval_timedelta)
def test_last_time_that_did_task_is_min_datetime(self):
self.assertEqual(self.ttl.last_time_that_did_task, datetime.min)
def test_can_do_task_if_not_done_before(self):
self.assertTrue(self.ttl.can_do_task())
def test_can_do_task_if_not_done_before_and_wait_time_interval(self):
sleep(self.interval_seconds_with_error_margin)
self.assertTrue(self.ttl.can_do_task())
def test_cannot_do_task_if_check_within_time_interval(self):
self.ttl.did_task()
self.assertFalse(self.ttl.can_do_task())
def test_cannot_do_task_if_check_after_time_interval(self):
self.ttl.did_task()
sleep(self.interval_seconds_with_error_margin)
self.assertTrue(self.ttl.can_do_task())
def test_do_task_updates_last_time_that_did_task_to_a_greater_time(self):
before = self.ttl.last_time_that_did_task
self.ttl.did_task()
after = self.ttl.last_time_that_did_task
self.assertGreater(after, before)
def test_do_task_actually_allowed_even_if_cannot_do_task(self):
self.ttl.did_task()
self.assertFalse(self.ttl.can_do_task())
self.ttl.did_task()
def test_reset_sets_last_time_to_min_datetime(self):
self.ttl.reset()
self.assertEqual(self.ttl.last_time_that_did_task, datetime.min)
def test_reset_sets_last_time_to_min_datetime_even_after_task_done(self):
self.ttl.did_task()
self.assertNotEqual(self.ttl.last_time_that_did_task, datetime.min)
self.ttl.reset()
self.assertEqual(self.ttl.last_time_that_did_task, datetime.min)
def setUp(self) -> None:
self.interval_seconds = 2
self.interval_seconds_with_error_margin = self.interval_seconds + 0.5
self.interval_timedelta = timedelta(seconds=self.interval_seconds)
self.ttl = TimedTaskLimiter(self.interval_timedelta)
class RedisApi:
# Init Redis object with logger and local database details
def __init__(self, logger: logging.Logger, db: int,
host: str = 'localhost', port: int = 6379,
password: str = '', namespace: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60)) \
-> None:
self._logger = logger
# If no password was given establish passwordless connection
# otherwise establish authenticated connection
if password == '':
self._redis = redis.Redis(host=host, port=port, db=db)
else:
self._redis = redis.Redis(host=host,
port=port,
db=db,
password=password)
# Unique alerter identification
self._namespace = namespace
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
# Variable to show that Redis is live and being used.
self._is_live = True
self._set_as_live()
self._logger.info('Redis initialised.')
@property
def is_live(self) -> bool:
return self._is_live
def _add_namespace(self, key: str) -> str:
if not key.startswith(self._namespace + ':'):
return self._namespace + ':' + key
else:
return key # prevent adding namespace twice
def _remove_namespace(self, key: str) -> str:
if not key.startswith(self._namespace + ':'):
return key # prevent removing namespace twice
else:
return key.replace(self._namespace + ':', '', 1)
def _set_as_live(self) -> None:
if not self._is_live:
self._logger.info('Redis is now accessible again.')
self._is_live = True
def _set_as_down(self) -> None:
# If Redis is live or if we can check whether it is live (because the
# live check time interval has passed), reset the live check limiter
# so that usage of Redis is skipped for as long as the time interval
if self._is_live or self._live_check_limiter.can_do_task():
self._live_check_limiter.did_task()
self._logger.warning('Redis is unusable for some reason. Stopping '
'usage temporarily to improve performance.')
self._is_live = False
def _do_not_use_if_recently_went_down(self) -> bool:
# If Redis is not live and cannot check if it is live (by using it)
# then stop the function called from happening by returning True
return not self._is_live and not self._live_check_limiter.can_do_task()
def _safe(self, function, args: List, default_return):
# Given an "unsafe" function from below and its arguments, safe calls
# the function with the provided arguments and performs exception
# handling as well as returns a specified default upon failure
try:
if self._do_not_use_if_recently_went_down():
return default_return
ret = function(*args)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in %s: %s', function.__name__, e)
self._set_as_down()
return default_return
def set_unsafe(self, key: str, value: RedisType):
key = self._add_namespace(key)
set_ret = self._redis.set(key, value)
return set_ret
def hset_unsafe(self, name: str, key: str, value: RedisType):
name = self._add_namespace(name)
set_ret = self._redis.hset(name, key, value)
return set_ret
def set_multiple_unsafe(self, key_values: Dict[str, RedisType]):
# Add namespace to keys
keys = list(key_values.keys())
unique_keys = [self._add_namespace(k) for k in keys]
for k, uk in zip(keys, unique_keys):
key_values[uk] = key_values.pop(k)
# Set multiple
pipe = self._redis.pipeline()
for key, value in key_values.items():
pipe.set(key, value if value is not None else 'None')
exec_ret = pipe.execute()
return exec_ret
def hset_multiple_unsafe(self, name: str, key_values: Dict[str,
RedisType]):
# Add namespace to hash name
name = self._add_namespace(name)
# Set multiple
pipe = self._redis.pipeline()
for key, value in key_values.items():
pipe.hset(name, key, value if value is not None else 'None')
exec_ret = pipe.execute()
return exec_ret
def set_for_unsafe(self, key: str, value: RedisType, time: timedelta):
key = self._add_namespace(key)
pipe = self._redis.pipeline()
pipe.set(key, value)
pipe.expire(key, time)
exec_ret = pipe.execute()
return exec_ret
def time_to_live_unsafe(self, key: str):
key = self._add_namespace(key)
time_to_live = self._redis.ttl(key)
# -1: Key exists but has no associated timeout
# -2: Key does not exist
if time_to_live == -1 or time_to_live == -2:
return None
else:
return time_to_live
def get_unsafe(self, key: str, default=None) -> Optional[bytes]:
key = self._add_namespace(key)
if self.exists_unsafe(key):
get_ret = self._redis.get(key)
if get_ret.decode('UTF-8') == 'None':
return None
else:
return get_ret
else:
return default
def hget_unsafe(self,
name: str,
key: str,
default=None) -> Optional[bytes]:
name = self._add_namespace(name)
if self.hexists_unsafe(name, key):
get_ret = self._redis.hget(name, key)
if get_ret.decode('UTF-8') == 'None':
return None
else:
return get_ret
else:
return default
def get_int_unsafe(self, key: str, default=None) -> Optional[int]:
key = self._add_namespace(key)
get_ret = self.get_unsafe(key, None)
try:
return int(get_ret) if get_ret is not None else default
except ValueError:
self._logger.error(
'Could not convert value %s of key %s to an integer. '
'Defaulting to value %s.', get_ret, key, default)
return default
def hget_int_unsafe(self, name: str, key: str, default=None) \
-> Optional[int]:
name = self._add_namespace(name)
get_ret = self.hget_unsafe(name, key, None)
try:
return int(get_ret) if get_ret is not None else default
except ValueError:
self._logger.error(
'Could not convert value %s of key %s to an integer. '
'Defaulting to value %s.', get_ret, key, default)
return default
def get_bool_unsafe(self, key: str, default=None) -> Optional[bool]:
key = self._add_namespace(key)
get_ret = self.get_unsafe(key, None)
return (get_ret.decode() == 'True') if get_ret is not None else default
def hget_bool_unsafe(self, name: str, key: str, default=None) \
-> Optional[bool]:
name = self._add_namespace(name)
get_ret = self.hget_unsafe(name, key, None)
return (get_ret.decode() == 'True') if get_ret is not None else default
def exists_unsafe(self, key: str) -> bool:
key = self._add_namespace(key)
return bool(self._redis.exists(key))
def hexists_unsafe(self, name: str, key: str) -> bool:
name = self._add_namespace(name)
return bool(self._redis.hexists(name, key))
def get_keys_unsafe(self, pattern: str = "*") -> List[str]:
pattern = self._add_namespace(pattern)
# Decode and remove namespace
keys_list = self._redis.keys(pattern)
keys_list = [k.decode('utf8') for k in keys_list]
keys_list = [self._remove_namespace(k) for k in keys_list]
return keys_list
def remove_unsafe(self, *keys):
keys = [self._add_namespace(k) for k in keys]
return self._redis.delete(*keys)
def delete_all_unsafe(self):
return self._redis.flushdb()
def set(self, key: str, value: RedisType):
return self._safe(self.set_unsafe, [key, value], None)
def hset(self, name: str, key: str, value: RedisType):
return self._safe(self.hset_unsafe, [name, key, value], None)
def set_multiple(self, key_values: Dict[str, RedisType]):
return self._safe(self.set_multiple_unsafe, [key_values], None)
def hset_multiple(self, name: str, key_values: Dict[str, RedisType]):
return self._safe(self.hset_multiple_unsafe, [name, key_values], None)
def set_for(self, key: str, value: RedisType, time: timedelta):
return self._safe(self.set_for_unsafe, [key, value, time], None)
def time_to_live(self, key: str):
return self._safe(self.time_to_live_unsafe, [key], None)
def get(self, key: str, default=None) -> Optional[bytes]:
return self._safe(self.get_unsafe, [key, default], default)
def hget(self, name: str, key: str, default=None) -> Optional[bytes]:
return self._safe(self.hget_unsafe, [name, key, default], default)
def get_int(self, key: str, default=None) -> Optional[int]:
return self._safe(self.get_int_unsafe, [key, default], default)
def hget_int(self, name: str, key: str, default=None) -> Optional[int]:
return self._safe(self.hget_int_unsafe, [name, key, default], default)
def get_bool(self, key: str, default=None) -> Optional[bool]:
return self._safe(self.get_bool_unsafe, [key, default], default)
def hget_bool(self, name: str, key: str, default=None) -> Optional[bool]:
return self._safe(self.hget_bool_unsafe, [name, key, default], default)
def exists(self, key: str) -> bool:
return self._safe(self.exists_unsafe, [key], False)
def hexists(self, name: str, key: str) -> bool:
return self._safe(self.hexists_unsafe, [name, key], False)
def get_keys(self, pattern: str = "*") -> List[str]:
return self._safe(self.get_keys_unsafe, [pattern], [])
def remove(self, *keys):
if type(keys) is tuple:
keys = ''.join(keys)
return self._safe(self.remove_unsafe, [keys], None)
def delete_all(self):
return self._safe(self.delete_all_unsafe, [], None)
def ping_unsafe(self) -> bool:
return self._redis.ping()
class MongoApi:
def __init__(self, logger: logging.Logger, db_name: str,
host: str = 'localhost', port: int = 27017,
username: str = '', password: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60),
timeout_ms: int = 10000) \
-> None:
self._logger = logger
self._db_name = db_name
if password == '':
self._client = MongoClient(
host=host, port=port, connectTimeoutMS=timeout_ms,
socketTimeoutMS=timeout_ms, serverSelectionTimeoutMS=timeout_ms)
else:
self._client = MongoClient(
host=host, port=port, connectTimeoutMS=timeout_ms,
socketTimeoutMS=timeout_ms, serverSelectionTimeoutMS=timeout_ms,
username=username, password=password)
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
self._is_live = True # This is necessary to initialise the variable
self._set_as_live()
self._logger.info('Mongo initialised.')
@property
def _db(self):
return self._client[self._db_name]
@property
def is_live(self) -> bool:
return self._is_live
def _set_as_live(self) -> None:
if not self._is_live:
self._logger.info('Mongo is now accessible again.')
self._is_live = True
def _set_as_down(self) -> None:
# If Mongo is live or if we can check whether it is live (because the
# live check time interval has passed), reset the live check limiter
# so that usage of Mongo is skipped for as long as the time interval
if self._is_live or self._live_check_limiter.can_do_task():
self._live_check_limiter.did_task()
self._logger.warning('Mongo is unusable for some reason. Stopping '
'usage temporarily to improve performance.')
self._is_live = False
def _do_not_use_if_recently_went_down(self) -> bool:
# If Mongo is not live and cannot check if it is live (by using it)
# then stop the function called from happening by returning True
return not self._is_live and not self._live_check_limiter.can_do_task()
def insert_one(self, collection: str, document: Dict) \
-> Optional[InsertOneResult]:
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self._db[collection].insert_one(document)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Mongo error in insert_one: %s', e)
self._set_as_down()
raise e
def insert_many(self, collection: str, documents: List[Dict]) \
-> Optional[InsertManyResult]:
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self._db[collection].insert_many(documents)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Mongo error in insert_many: %s', e)
self._set_as_down()
raise e
def get_all(self, collection: str) -> Optional[List[Dict]]:
try:
if self._do_not_use_if_recently_went_down():
return None
ret = list(self._db[collection].find({}))
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Mongo error in drop_collection: %s', e)
self._set_as_down()
raise e
def drop_collection(self, collection: str) -> Optional[Dict]:
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self._db.drop_collection(collection)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Mongo error in drop_collection: %s', e)
self._set_as_down()
raise e
def drop_db(self) -> None:
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self._client.drop_database(self._db.name)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Mongo error in drop_db: %s', e)
self._set_as_down()
raise e
def ping_unsafe(self):
return self._db.command('ping')
def ping_auth(self, username: str, password: str):
return self._db.authenticate(username, password, "admin")
def __init__(self,
name: str,
api_url: Optional[str],
prometheus_endpoint: Optional[str],
node_type: NodeType,
node_public_key: Optional[str],
chain: str,
redis: Optional[RedisApi],
is_archive_node: bool,
consensus_public_key: str,
tendermint_address_key: str,
staking_address: str,
entity_public_key: str,
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self._api_url = api_url
self._prometheus_endpoint = prometheus_endpoint
self._node_type = node_type
self._node_public_key = node_public_key
self._chain = chain
self._consensus_public_key = consensus_public_key
self._tendermint_address_key = tendermint_address_key
self._staking_address = staking_address
self._entity_public_key = entity_public_key
self._redis = redis
self._redis_enabled = redis is not None
self._redis_hash = Keys.get_hash_blockchain(self.chain)
self._connected_to_api_server = True
self._went_down_at = None
self._bonded_balance = None
self._debonding_balance = None
self._shares_balance = None
self._voting_power = None
self._is_syncing = False
self._no_of_peers = None
self._initial_downtime_alert_sent = False
self._no_change_in_height_warning_sent = False
self._active = None
self._is_missing_blocks = False
self._finalized_block_height = 0
self._time_of_last_height_check_activity = NONE
self._time_of_last_height_change = NONE
self._consecutive_blocks_missed = 0
self._is_archive_node = is_archive_node
self._validator_peer_danger_boundary = \
internal_conf.validator_peer_danger_boundary
self._validator_peer_safe_boundary = \
internal_conf.validator_peer_safe_boundary
self._full_node_peer_danger_boundary = \
internal_conf.full_node_peer_danger_boundary
self._no_change_in_height_first_warning_seconds = \
internal_conf.no_change_in_height_first_warning_seconds
self._no_change_in_height_interval_seconds = \
internal_conf.no_change_in_height_interval_seconds
self._downtime_alert_limiter = TimedTaskLimiter(
internal_conf.downtime_alert_interval_seconds)
self._finalized_height_alert_limiter = TimedTaskLimiter(
timedelta(seconds=int(self._no_change_in_height_interval_seconds)))
self._change_in_bonded_balance_threshold = \
internal_conf.change_in_bonded_balance_threshold
self._change_in_debonding_balance_threshold = \
internal_conf.change_in_debonding_balance_threshold
self._change_in_shares_balance_threshold = \
internal_conf.change_in_shares_balance_threshold
self._missed_blocks_danger_boundary = \
internal_conf.missed_blocks_danger_boundary
self._timed_block_miss_tracker = TimedOccurrenceTracker(
internal_conf.max_missed_blocks_in_time_interval,
internal_conf.max_missed_blocks_time_interval)
class Node:
def __init__(self,
name: str,
api_url: Optional[str],
prometheus_endpoint: Optional[str],
node_type: NodeType,
node_public_key: Optional[str],
chain: str,
redis: Optional[RedisApi],
is_archive_node: bool,
consensus_public_key: str,
tendermint_address_key: str,
staking_address: str,
entity_public_key: str,
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self._api_url = api_url
self._prometheus_endpoint = prometheus_endpoint
self._node_type = node_type
self._node_public_key = node_public_key
self._chain = chain
self._consensus_public_key = consensus_public_key
self._tendermint_address_key = tendermint_address_key
self._staking_address = staking_address
self._entity_public_key = entity_public_key
self._redis = redis
self._redis_enabled = redis is not None
self._redis_hash = Keys.get_hash_blockchain(self.chain)
self._connected_to_api_server = True
self._went_down_at = None
self._bonded_balance = None
self._debonding_balance = None
self._shares_balance = None
self._voting_power = None
self._is_syncing = False
self._no_of_peers = None
self._initial_downtime_alert_sent = False
self._no_change_in_height_warning_sent = False
self._active = None
self._is_missing_blocks = False
self._finalized_block_height = 0
self._time_of_last_height_check_activity = NONE
self._time_of_last_height_change = NONE
self._consecutive_blocks_missed = 0
self._is_archive_node = is_archive_node
self._validator_peer_danger_boundary = \
internal_conf.validator_peer_danger_boundary
self._validator_peer_safe_boundary = \
internal_conf.validator_peer_safe_boundary
self._full_node_peer_danger_boundary = \
internal_conf.full_node_peer_danger_boundary
self._no_change_in_height_first_warning_seconds = \
internal_conf.no_change_in_height_first_warning_seconds
self._no_change_in_height_interval_seconds = \
internal_conf.no_change_in_height_interval_seconds
self._downtime_alert_limiter = TimedTaskLimiter(
internal_conf.downtime_alert_interval_seconds)
self._finalized_height_alert_limiter = TimedTaskLimiter(
timedelta(seconds=int(self._no_change_in_height_interval_seconds)))
self._change_in_bonded_balance_threshold = \
internal_conf.change_in_bonded_balance_threshold
self._change_in_debonding_balance_threshold = \
internal_conf.change_in_debonding_balance_threshold
self._change_in_shares_balance_threshold = \
internal_conf.change_in_shares_balance_threshold
self._missed_blocks_danger_boundary = \
internal_conf.missed_blocks_danger_boundary
self._timed_block_miss_tracker = TimedOccurrenceTracker(
internal_conf.max_missed_blocks_in_time_interval,
internal_conf.max_missed_blocks_time_interval)
def __str__(self) -> str:
return self.name
@property
def voting_power(self) -> int:
return self._voting_power
@property
def is_validator(self) -> bool:
return self._node_type == NodeType.VALIDATOR_FULL_NODE
@property
def is_archive_node(self) -> bool:
return self._is_archive_node
@property
def is_down(self) -> bool:
return self._went_down_at is not None
@property
def is_active(self) -> bool:
return self._active
@property
def is_missing_blocks(self) -> bool:
return self.consecutive_blocks_missed_so_far > 0
@property
def consecutive_blocks_missed_so_far(self) -> int:
return self._consecutive_blocks_missed
@property
def is_syncing(self) -> bool:
return self._is_syncing
@property
def bonded_balance(self) -> int:
return self._bonded_balance
@property
def debonding_balance(self) -> int:
return self._debonding_balance
@property
def shares_balance(self) -> int:
return self._shares_balance
@property
def finalized_height_alert_limiter(self) -> TimedTaskLimiter:
return self._finalized_height_alert_limiter
@property
def node_public_key(self) -> str:
return self._node_public_key
@property
def staking_address(self) -> str:
return self._staking_address
@property
def api_url(self) -> str:
return self._api_url
@property
def prometheus_endpoint(self) -> str:
return self._prometheus_endpoint
@property
def chain(self) -> str:
return self._chain
@property
def entity_public_key(self) -> str:
return self._entity_public_key
@property
def is_connected_to_api_server(self) -> bool:
return self._connected_to_api_server
@property
def consensus_public_key(self) -> str:
return self._consensus_public_key
@property
def tendermint_address_key(self) -> str:
return self._tendermint_address_key
@property
def no_of_peers(self) -> int:
return self._no_of_peers
@property
def is_no_change_in_height_warning_sent(self) -> bool:
return self._no_change_in_height_warning_sent
@property
def finalized_block_height(self) -> int:
return self._finalized_block_height
def status(self) -> str:
return "bonded_balance={}, debonding_balance={}, shares_balance={}," \
" is_syncing={}, no_of_peers={}, active={}, " \
"finalized_block_height={}, is_missing_blocks={}" \
.format(self.bonded_balance, self.debonding_balance,
self.shares_balance, self.is_syncing, self.no_of_peers,
self.is_active, self.finalized_block_height,
self.is_missing_blocks)
def load_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, load any previously stored state
if self._redis_enabled:
self._went_down_at = self._redis.hget(
self._redis_hash, Keys.get_node_went_down_at(self.name), None)
self._bonded_balance = self._redis.hget_int(
self._redis_hash, Keys.get_node_bonded_balance(self.name),
None)
self._debonding_balance = self._redis.hget_int(
self._redis_hash, Keys.get_node_debonding_balance(self.name),
None)
self._shares_balance = self._redis.hget_int(
self._redis_hash, Keys.get_node_shares_balance(self.name),
None)
self._is_syncing = self._redis.hget_bool(
self._redis_hash, Keys.get_node_is_syncing(self.name), False)
self._voting_power = self._redis.hget_int(
self._redis_hash, Keys.get_voting_power(self.name), None)
self._consecutive_blocks_missed = self._redis.hget_int(
self._redis_hash,
Keys.get_consecutive_blocks_missed(self.name), 0)
self._no_of_peers = self._redis.hget_int(
self._redis_hash, Keys.get_node_no_of_peers(self.name), None)
self._active = self._redis.hget_bool(
self._redis_hash, Keys.get_node_active(self.name), None)
self._is_missing_blocks = self._redis.hget_bool(
self._redis_hash, Keys.get_node_is_missing_blocks(self.name),
False)
self._time_of_last_height_check_activity = float(
self._redis.hget(
self._redis_hash,
Keys.get_node_time_of_last_height_check_activity(
self.name), NONE))
self._time_of_last_height_change = float(
self._redis.hget(
self._redis_hash,
Keys.get_node_time_of_last_height_change(self.name), NONE))
self._finalized_block_height = self._redis.hget_int(
self._redis_hash,
Keys.get_node_finalized_block_height(self.name), 0)
self._no_change_in_height_warning_sent = self._redis.hget_bool(
self._redis_hash,
Keys.get_node_no_change_in_height_warning_sent(self.name),
False)
if self._time_of_last_height_check_activity != NONE:
self._finalized_height_alert_limiter.set_last_time_that_did_task(
datetime.fromtimestamp(
self._time_of_last_height_check_activity))
else:
self._finalized_height_alert_limiter.did_task()
self._time_of_last_height_change = datetime.now().timestamp()
# String to actual values
if self._went_down_at is not None:
try:
self._went_down_at = \
dateutil.parser.parse(self._went_down_at)
except (TypeError, ValueError) as e:
logger.error('Error when parsing ' '_went_down_at: %s', e)
self._went_down_at = None
logger.debug(
'Restored %s state: _went_down_at=%s, _bonded_balance=%s, '
'_debonding_balance=%s, _shares_balance=%s, _is_syncing=%s, '
'_no_of_peers=%s, _active=%s, _no_of_blocks_missed=%s, '
'_time_of_last_height_change=%s, '
'_time_of_last_height_check_activity, '
'_finalized_block_height=%s, '
'_no_change_in_height_warning_sent=%s, '
'_is_missing_blocks=%s ', self.name, self._went_down_at,
self._bonded_balance, self._debonding_balance,
self._shares_balance, self._is_syncing, self._no_of_peers,
self._active, self._consecutive_blocks_missed,
self._time_of_last_height_change,
self._time_of_last_height_check_activity,
self._finalized_block_height,
self._no_change_in_height_warning_sent, self.is_missing_blocks)
def save_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, store the current state
if self._redis_enabled:
logger.debug(
'Saved %s state: _went_down_at=%s, _bonded_balance=%s, '
'_debonding_balance=%s, _shares_balance=%s, _is_syncing=%s, '
'_no_of_peers=%s, _active=%s, _no_of_blocks_missed=%s, '
'_time_of_last_height_change=%s, '
'_time_of_last_height_check_activity, '
'_finalized_block_height=%s, '
'_no_change_in_height_warning_sent=%s, '
'_is_missing_blocks=%s', self.name, self._went_down_at,
self._bonded_balance, self._debonding_balance,
self._shares_balance, self._is_syncing, self._no_of_peers,
self._active, self._consecutive_blocks_missed,
self._time_of_last_height_change,
self._time_of_last_height_check_activity,
self._finalized_block_height,
self._no_change_in_height_warning_sent, self.is_missing_blocks)
# Set values
self._redis.hset_multiple(
self._redis_hash, {
Keys.get_node_went_down_at(self.name):
str(self._went_down_at),
Keys.get_node_bonded_balance(self.name):
self._bonded_balance,
Keys.get_node_debonding_balance(self.name):
self._debonding_balance,
Keys.get_node_shares_balance(self.name):
self._shares_balance,
Keys.get_node_is_syncing(self.name):
str(self._is_syncing),
Keys.get_node_no_of_peers(self.name):
self._no_of_peers,
Keys.get_voting_power(self.name):
self._voting_power,
Keys.get_consecutive_blocks_missed(self.name):
self._consecutive_blocks_missed,
Keys.get_node_is_missing_blocks(self.name):
str(self.is_missing_blocks),
Keys.get_node_active(self.name):
str(self._active),
Keys.get_node_time_of_last_height_check_activity(self.name):
self._time_of_last_height_check_activity,
Keys.get_node_time_of_last_height_change(self.name):
self._time_of_last_height_change,
Keys.get_node_finalized_block_height(self.name):
self._finalized_block_height,
Keys.get_node_no_change_in_height_warning_sent(self.name):
str(self._no_change_in_height_warning_sent),
})
def set_as_down(self, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert (varies depending on whether was already down)
if self.is_down and not self._initial_downtime_alert_sent:
if self.is_validator:
channels.alert_critical(CannotAccessNodeAlert(self.name))
else:
channels.alert_warning(CannotAccessNodeAlert(self.name))
self._downtime_alert_limiter.did_task()
self._initial_downtime_alert_sent = True
elif self.is_down and self._downtime_alert_limiter.can_do_task():
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_critical(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
else:
channels.alert_warning(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
self._downtime_alert_limiter.did_task()
elif not self.is_down:
# Do not alert for now just in case this is a connection hiccup
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._went_down_at = datetime.now().timestamp()
self._initial_downtime_alert_sent = False
def set_as_up(self, channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug('%s set_as_up: is_down(currently)=%s, channels=%s', self,
self.is_down, channels)
# Alert if node was down
if self.is_down:
# Only send accessible alert if inaccessible alert was sent
if self._initial_downtime_alert_sent:
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
channels.alert_info(
NowAccessibleAlert(self.name, went_down_at, downtime))
# Reset downtime-related values
self._downtime_alert_limiter.reset()
self._went_down_at = None
def set_bonded_balance(self, new_bonded_balance: int, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_bonded_balance: before=%s, new=%s, channels=%s',
self, self.bonded_balance, new_bonded_balance, channels)
# Alert if bonded_balance has changed
if self.bonded_balance not in [None, new_bonded_balance]:
# Extracted data is in giga, therefore, to give more meaningful
# alerts, the bonded balance will be scaled down.
threshold = scale_to_giga(self._change_in_bonded_balance_threshold)
scaled_new_bal = round(scale_to_nano(new_bonded_balance), 3)
scaled_bal = round(scale_to_nano(self.bonded_balance), 3)
if self.is_validator and new_bonded_balance == 0: # N to 0
channels.alert_critical(
BondedBalanceDecreasedAlert(self.name, scaled_bal,
scaled_new_bal))
elif self.is_validator and self.bonded_balance == 0: # 0 to N
channels.alert_info(
BondedBalanceIncreasedAlert(self.name, scaled_bal,
scaled_new_bal))
else: # Any change
diff = new_bonded_balance - self.bonded_balance
if abs(diff) > threshold:
if diff > 0:
channels.alert_info(
BondedBalanceIncreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
else:
channels.alert_info(
BondedBalanceDecreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
# Update bonded balance
self._bonded_balance = new_bonded_balance
def set_debonding_balance(self, new_debonding_balance: int, \
channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug(
'%s set_debonding_balance: before=%s, new=%s, channels=%s', self,
self.debonding_balance, new_debonding_balance, channels)
# Alert if debonding_balance has changed
if self.debonding_balance not in [None, new_debonding_balance]:
# Extracted data is in giga, therefore, to give more meaningful
# alerts, the debonding balance will be scaled down.
threshold = scale_to_giga(
self._change_in_debonding_balance_threshold)
scaled_new_bal = round(scale_to_nano(new_debonding_balance), 3)
scaled_bal = round(scale_to_nano(self.debonding_balance), 3)
if self.is_validator and new_debonding_balance == 0: # N to 0
channels.alert_info(
DebondingBalanceDecreasedAlert(self.name, scaled_bal,
scaled_new_bal))
elif self.is_validator and self.debonding_balance == 0: # 0 to N
channels.alert_info(
DebondingBalanceIncreasedAlert(self.name, scaled_bal,
scaled_new_bal))
else: # Any change
diff = new_debonding_balance - self.debonding_balance
if abs(diff) > threshold:
if diff > 0:
channels.alert_info(
DebondingBalanceIncreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
else:
channels.alert_info(
DebondingBalanceDecreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
# Update debonding balance
self._debonding_balance = new_debonding_balance
def set_shares_balance(self, new_shares_balance: int, \
channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_shares_balance: before=%s, new=%s, channels=%s',
self, self.shares_balance, new_shares_balance, channels)
# Alert if shares_balance has changed
if self.shares_balance not in [None, new_shares_balance]:
# Extracted data is in giga, therefore, to give more meaningful
# alerts, the debonding balance will be scaled down.
threshold = scale_to_giga(self._change_in_shares_balance_threshold)
scaled_new_bal = round(scale_to_nano(new_shares_balance), 3)
scaled_bal = round(scale_to_nano(self.shares_balance), 3)
if self.is_validator and new_shares_balance == 0: # N to 0
channels.alert_info(
SharesBalanceDecreasedAlert(self.name, scaled_bal,
scaled_new_bal))
elif self.is_validator and self.shares_balance == 0: # 0 to N
channels.alert_info(
SharesBalanceIncreasedAlert(self.name, scaled_bal,
scaled_new_bal))
else: # Any change
diff = new_shares_balance - self.shares_balance
if abs(diff) > threshold:
if diff > 0:
channels.alert_info(
SharesBalanceIncreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
else:
channels.alert_info(
SharesBalanceDecreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
# Update shares balance
self._shares_balance = new_shares_balance
def set_is_syncing(self, now_is_syncing: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_is_syncing: before=%s, new=%s, channels=%s', self,
self.is_syncing, now_is_syncing, channels)
# Alert if is syncing has changed
if not self.is_syncing and now_is_syncing:
channels.alert_warning(IsSyncingAlert(self.name))
elif self.is_syncing and not now_is_syncing:
channels.alert_info(IsNoLongerSyncingAlert(self.name))
# Update is-syncing
self._is_syncing = now_is_syncing
def set_voting_power(self, new_voting_power: int, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
logger.debug('%s set_voting_power: before=%s, new=%s, channels=%s',
self, self.voting_power, new_voting_power, channels)
# Alert if voting power has changed
if self.voting_power not in [None, new_voting_power]:
if self.is_validator and new_voting_power == 0: # N to 0
channels.alert_critical(
VotingPowerDecreasedAlert(self.name, self.voting_power,
new_voting_power))
elif self.is_validator and self.voting_power == 0: # 0 to N
channels.alert_info(
VotingPowerIncreasedAlert(self.name, self.voting_power,
new_voting_power))
else: # Any change
diff = new_voting_power - self.voting_power
if diff > 0:
channels.alert_info(
VotingPowerIncreasedByAlert(self.name,
self.voting_power,
new_voting_power))
else:
channels.alert_info(
VotingPowerDecreasedByAlert(self.name,
self.voting_power,
new_voting_power))
# Update voting power
self._voting_power = new_voting_power
def set_no_of_peers(self, new_no_of_peers: int, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_no_of_peers: before=%s, new=%s, channels=%s',
self, self.no_of_peers, new_no_of_peers, channels)
# Variable alias for improved readability
if self.is_validator:
danger = self._validator_peer_danger_boundary
safe = self._validator_peer_safe_boundary
else:
danger = self._full_node_peer_danger_boundary
safe = None
# Alert if number of peers has changed
if self.no_of_peers not in [None, new_no_of_peers]:
if self.is_validator:
if new_no_of_peers <= self._validator_peer_safe_boundary:
# beneath safe boundary
if new_no_of_peers > self.no_of_peers: # increase
channels.alert_info(
PeersIncreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
elif new_no_of_peers > danger:
# decrease outside danger range
channels.alert_warning(
PeersDecreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
else: # decrease inside danger range
channels.alert_critical(
PeersDecreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
elif self._no_of_peers <= self._validator_peer_safe_boundary \
< new_no_of_peers:
# increase outside safe range for the first time
channels.alert_info(
PeersIncreasedOutsideSafeRangeAlert(self.name, safe))
else:
if new_no_of_peers > self.no_of_peers: # increase
if new_no_of_peers <= danger:
# increase inside danger range
channels.alert_info(
PeersIncreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
elif self.no_of_peers <= danger < new_no_of_peers:
# increase outside danger range
channels.alert_info(
PeersIncreasedOutsideDangerRangeAlert(
self.name, danger))
elif new_no_of_peers > danger: # decrease outside danger range
pass
else: # decrease inside danger range
channels.alert_warning(
PeersDecreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
# Update number of peers
self._no_of_peers = new_no_of_peers
def set_active(self, now_is_active: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator.
logger.debug('%s set_active: active(currently)=%s, channels=%s', self,
self.is_active, channels)
if self.is_active not in [now_is_active, None]:
if now_is_active:
channels.alert_info(ValidatorIsNowActiveAlert(self.name))
else:
channels.alert_critical(ValidatorIsNotActiveAlert(self.name))
self._active = now_is_active
def update_finalized_block_height(self, new_finalized_height: int,
logger: logging.Logger,
channels: ChannelSet):
logger.debug(
'%s update_finalized_block_height: finalized_block_height'
' (currently)=%s', self, self._finalized_block_height)
current_timestamp = datetime.now().timestamp()
if self._finalized_block_height != new_finalized_height:
if self.is_no_change_in_height_warning_sent:
self._no_change_in_height_warning_sent = False
channels.alert_info(
NodeFinalizedBlockHeightHasNowBeenUpdatedAlert(self.name))
if self._finalized_block_height > new_finalized_height:
logger.info(
'The finalized height of node {} decreased to {}.'.format(
self, self._finalized_block_height))
self._finalized_block_height = new_finalized_height
self._time_of_last_height_change = current_timestamp
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter.set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
else:
timestamp_difference = current_timestamp - \
self._time_of_last_height_change
time_interval = strfdelta(
timedelta(seconds=int(timestamp_difference)),
"{hours}h, {minutes}m, {seconds}s")
if not self.is_no_change_in_height_warning_sent and \
timestamp_difference > \
self._no_change_in_height_first_warning_seconds:
self._no_change_in_height_warning_sent = True
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
elif self._finalized_height_alert_limiter.can_do_task() and \
self.is_no_change_in_height_warning_sent:
if self.is_validator:
channels.alert_critical(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
else:
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter. \
set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
def add_missed_block(self, block_height: int, block_time: datetime,
missing_validators: int, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
# Calculate the actual blocks missed as of when this function was called
blocks_missed = self._consecutive_blocks_missed + 1
# Variable alias for improved readability
danger = self._missed_blocks_danger_boundary
logger.debug(
'%s add_missed_block: before=%s, new=%s, missing_validators = %s, '
'channels=%s', self, self.consecutive_blocks_missed_so_far,
blocks_missed, missing_validators, channels)
# Let timed tracker know that block missed
self._timed_block_miss_tracker.action_happened(at_time=block_time)
# Alert (varies depending on whether was already missing blocks)
if not self.is_missing_blocks:
pass # Do not alert on first missed block
elif 2 <= blocks_missed < danger:
channels.alert_info(
MissedBlocksAlert(self.name, blocks_missed, block_height,
missing_validators)
) # 2+ blocks missed inside danger range
elif blocks_missed == 5:
channels.alert_warning(
MissedBlocksAlert(self.name, blocks_missed, block_height,
missing_validators)) # reached danger range
elif blocks_missed >= max(10, danger) and blocks_missed % 10 == 0:
channels.alert_critical(
MissedBlocksAlert(self.name, blocks_missed, block_height,
missing_validators)
) # Every (10N)th block missed for N >= 1 inside danger range
self._timed_block_miss_tracker.reset()
if self._timed_block_miss_tracker.too_many_occurrences(block_time):
blocks_in_interval = self._timed_block_miss_tracker.max_occurrences
time_interval = self._timed_block_miss_tracker.time_interval_pretty
channels.alert_critical(
TimedMissedBlocksAlert(self.name, blocks_in_interval,
time_interval, block_height,
missing_validators)
) # More blocks missed than is acceptable in the time interval
self._timed_block_miss_tracker.reset()
# Update consecutive blocks missed
self._consecutive_blocks_missed = blocks_missed
def clear_missed_blocks(self, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator
logger.debug('%s clear_missed_blocks: channels=%s', self, channels)
# Alert if validator was missing blocks (only if more than 1 block)
if self.is_missing_blocks and self._consecutive_blocks_missed > 1:
channels.alert_info(
NoLongerMissingBlocksAlert(self.name,
self._consecutive_blocks_missed))
# Reset missed blocks related values
self._consecutive_blocks_missed = 0
# Categorise the event and alert as needed
def process_event(self, event_height: str, event: dict,
channels: ChannelSet, logger: logging.Logger):
# An escrow event is when tokens are either taken/added or reclaimed
# from a delegation.
if self._check_dict_path(event, 'escrow'):
# Escrow events that take are usually done by the blockchain,
# such as when a validator is slashed
if self._check_dict_path(event, 'escrow', 'take'):
if event['escrow']['take']['owner'] == self.entity_public_key:
tokens = event['escrow']['take']['amount']
logger.debug('%s Node %s Slashed %s tokens at height %s',
self, self.name, tokens, event_height)
channels.alert_critical(
SlashedAlert(self.name, tokens, event_height))
# Escrow events that add occur when someone delegates tokens to a
# validator.
elif self._check_dict_path(event, 'escrow', 'add'):
if event['escrow']['add']['owner'] == self.entity_public_key:
tokens = event['escrow']['add']['amount']
escrow = event['escrow']['add']['escrow']
logger.debug('%s Node %s : Added %s tokens at height %s to '
'%s .', self, self.name, tokens, event_height, \
escrow)
channels.alert_info(
EscrowAddEventSelfOwner(self.name, tokens,
event_height, escrow))
elif event['escrow']['add'][
'escrow'] == self.entity_public_key:
tokens = event['escrow']['add']['amount']
owner = event['escrow']['add']['owner']
logger.debug('%s Node %s : Added %s tokens at height %s to '
'%s .', self, self.name, tokens, event_height, \
owner)
channels.alert_info(
EscrowAddEventSelfEscrow(self.name, tokens,
event_height, owner))
# Escrow events that reclaim occur when someone takes back their
# delegated tokens from a validator
elif self._check_dict_path(event, 'escrow', 'reclaim'):
if event['escrow']['reclaim']['owner'] == \
self.entity_public_key:
tokens = event['escrow']['reclaim']['amount']
escrow = event['escrow']['reclaim']['escrow']
logger.debug('%s Node %s : reclaimed %s tokens at height %s'
'to %s .', self, self.name, tokens, \
event_height, escrow)
channels.alert_info(
EscrowReclaimEventSelfOwner(self.name, tokens,
event_height, escrow))
elif event['escrow']['reclaim']['escrow'] == \
self.entity_public_key:
tokens = event['escrow']['reclaim']['amount']
owner = event['escrow']['reclaim']['owner']
logger.debug('%s Node %s : reclaimed %s tokens at height %s'
'to %s .', self, self.name, tokens, \
event_height, owner)
channels.alert_info(
EscrowReclaimEventSelfEscrow(self.name, tokens,
event_height, owner))
# Burn events occur when a user decides to destroy their own tokens.
elif self._check_dict_path(event, 'burn'):
if event['burn']['owner'] == self.entity_public_key:
tokens = event['burn']['amount']
logger.debug('%s Node %s Burned %s tokens at height %s', self,
self.name, tokens, event_height)
channels.alert_critical(
TokensBurnedAlert(self.name, tokens, event_height))
# Transfer events occur when a user decides to send tokens to another
# address.
elif self._check_dict_path(event, 'transfer'):
if event['transfer']['from'] == self.entity_public_key:
tokens = event['transfer']['amount']
destination = event['transfer']['to']
logger.debug(
'%s Node %s transfered %s tokens at height %s ' + 'to %s',
self, self.name, tokens, event_height,
event['transfer']['to'])
channels.alert_info(
TokensTransferedToAlert(self.name, tokens, event_height,
destination))
elif event['transfer']['to'] == self.entity_public_key:
tokens = event['transfer']['amount']
source = event['transfer']['from']
logger.debug(
'%s Node %s transfered %s tokens at height %s ' +
'from, %s', self, self.name, tokens, event_height,
event['transfer']['to'])
channels.alert_info(
TokensTransferedFromAlert(self.name, tokens, event_height,
source))
else:
logger.debug('%s Node received unknown event : %s', self,
self.name, event)
channels.alert_warning(
UnknownEventFound(self.name, event, event_height))
def disconnect_from_api(self, channels: ChannelSet,
logger: logging.Logger):
logger.debug('%s disconnect_from_api: channels=%s', self, channels)
if self.is_connected_to_api_server:
if self.is_validator:
channels.alert_critical(
NodeWasNotConnectedToApiServerAlert(self.name))
else:
channels.alert_warning(
NodeWasNotConnectedToApiServerAlert(self.name))
self._connected_to_api_server = False
def connect_with_api(self, channels: ChannelSet, logger: logging.Logger):
logger.debug('%s connect_with_api: channels=%s', self, channels)
if not self.is_connected_to_api_server:
channels.alert_info(NodeConnectedToApiServerAgainAlert(self.name))
self._connected_to_api_server = True
# Funciton to check if a path in a dictionray exists
def _check_dict_path(self, d: dict, *indices: str) -> bool:
sentinel = object()
for index in indices:
d = d.get(index, sentinel)
if d is sentinel:
return False
return True
def setUp(self) -> None:
self.interval_seconds = 2
self.interval_timedelta = timedelta(seconds=self.interval_seconds)
self.ttl = TimedTaskLimiter(self.interval_timedelta)
class Node:
def __init__(self,
name: str,
ws_url: Optional[str],
node_type: NodeType,
stash_account_address: Optional[str],
chain: str,
redis: Optional[RedisApi],
is_archive_node: bool,
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self._ws_url = ws_url
self._node_type = node_type
self._stash_account_address = stash_account_address
self._chain = chain
self._redis = redis
self._redis_enabled = redis is not None
self._redis_hash = Keys.get_hash_blockchain(self.chain)
self._connected_to_api_server = True
self._went_down_at = None
self._bonded_balance = None
self._is_syncing = False
self._no_of_peers = None
self._initial_downtime_alert_sent = False
self._no_change_in_height_warning_sent = False
self._active = None
self._disabled = None
self._elected = None
self._council_member = None
self._no_of_blocks_authored = 0
self._finalized_block_height = 0
self._time_of_last_block = NONE
self._time_of_last_block_check_activity = NONE
self._time_of_last_height_check_activity = NONE
self._time_of_last_height_change = NONE
self._auth_index = NONE
self._is_authoring = True
self._is_archive_node = is_archive_node
self._validator_peer_danger_boundary = \
internal_conf.validator_peer_danger_boundary
self._validator_peer_safe_boundary = \
internal_conf.validator_peer_safe_boundary
self._full_node_peer_danger_boundary = \
internal_conf.full_node_peer_danger_boundary
self._max_time_between_blocks_authored = \
internal_conf.max_time_alert_between_blocks_authored
self._no_change_in_height_first_warning_seconds = \
internal_conf.no_change_in_height_first_warning_seconds
self._no_change_in_height_interval_seconds = \
internal_conf.no_change_in_height_interval_seconds
self._downtime_alert_limiter = TimedTaskLimiter(
internal_conf.downtime_alert_time_interval)
self._blocks_authored_alert_limiter = TimedTaskLimiter(
self._max_time_between_blocks_authored)
self._finalized_height_alert_limiter = TimedTaskLimiter(
timedelta(seconds=int(self._no_change_in_height_interval_seconds)))
self._change_in_bonded_balance_threshold = \
internal_conf.change_in_bonded_balance_threshold
def __str__(self) -> str:
return self.name
@property
def is_validator(self) -> bool:
return self._node_type == NodeType.VALIDATOR_FULL_NODE
@property
def is_archive_node(self) -> bool:
return self._is_archive_node
@property
def is_down(self) -> bool:
return self._went_down_at is not None
@property
def is_active(self) -> bool:
return self._active
@property
def is_elected(self) -> bool:
return self._elected
@property
def is_disabled(self) -> bool:
return self._disabled
@property
def is_syncing(self) -> bool:
return self._is_syncing
@property
def is_council_member(self) -> bool:
return self._council_member
@property
def bonded_balance(self) -> int:
return self._bonded_balance
@property
def blocks_authored_alert_limiter(self) -> TimedTaskLimiter:
return self._blocks_authored_alert_limiter
@property
def finalized_height_alert_limiter(self) -> TimedTaskLimiter:
return self._finalized_height_alert_limiter
@property
def stash_account_address(self) -> str:
return self._stash_account_address
@property
def ws_url(self) -> str:
return self._ws_url
@property
def no_of_peers(self) -> int:
return self._no_of_peers
@property
def chain(self) -> str:
return self._chain
@property
def no_of_blocks_authored(self) -> int:
return self._no_of_blocks_authored
@property
def auth_index(self) -> int:
return self._auth_index
@property
def is_authoring(self) -> bool:
return self._is_authoring
@property
def is_no_change_in_height_warning_sent(self) -> bool:
return self._no_change_in_height_warning_sent
@property
def finalized_block_height(self) -> int:
return self._finalized_block_height
@property
def is_connected_to_api_server(self) -> bool:
return self._connected_to_api_server
def set_time_of_last_block(self, time_of_last_block: float,
channels: ChannelSet, logger: logging.Logger) \
-> None:
logger.debug(
'%s set_time_of_last_block: time_of_last_block(currently)'
'=%s, channels=%s', self, self._time_of_last_block, channels)
self._time_of_last_block = time_of_last_block
def set_is_authoring(self, is_authoring: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug(
'%s set_is_authoring: is_authoring(currently)=%s, '
'channels=%s', self, self._is_authoring, channels)
self._is_authoring = is_authoring
def set_time_of_last_block_check_activity(
self, time_of_last_block_check_activity: float,
channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug(
'%s set_time_of_last_block_check_activity: '
'time_of_last_block_check_activity(currently)=%s, '
'channels=%s', self, self._time_of_last_block_check_activity,
channels)
self._time_of_last_block_check_activity = \
time_of_last_block_check_activity
def status(self) -> str:
return "bonded_balance={}, is_syncing={}, no_of_peers={}, " \
"active={}, council_member={}, elected={}, disabled={}, " \
"no_of_blocks_authored={}, finalized_block_height={}". \
format(self.bonded_balance, self.is_syncing, self.no_of_peers,
self.is_active, self.is_council_member, self.is_elected,
self.is_disabled, self.no_of_blocks_authored,
self.finalized_block_height)
def load_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, load any previously stored state
if self._redis_enabled:
self._went_down_at = self._redis.hget(
self._redis_hash, Keys.get_node_went_down_at(self.name), None)
self._bonded_balance = self._redis.hget_int(
self._redis_hash, Keys.get_node_bonded_balance(self.name),
None)
self._is_syncing = self._redis.hget_bool(
self._redis_hash, Keys.get_node_is_syncing(self.name), False)
self._no_of_peers = self._redis.hget_int(
self._redis_hash, Keys.get_node_no_of_peers(self.name), None)
self._active = self._redis.hget_bool(
self._redis_hash, Keys.get_node_active(self.name), None)
self._council_member = self._redis.hget_bool(
self._redis_hash, Keys.get_node_council_member(self.name),
None)
self._elected = self._redis.hget_bool(
self._redis_hash, Keys.get_node_elected(self.name), None)
self._disabled = self._redis.hget_bool(
self._redis_hash, Keys.get_node_disabled(self.name), None)
self._no_of_blocks_authored = self._redis.hget_int(
self._redis_hash, Keys.get_node_blocks_authored(self.name), 0)
self._time_of_last_block = float(
self._redis.hget(self._redis_hash,
Keys.get_node_time_of_last_block(self.name),
NONE))
self._is_authoring = self._redis.hget_bool(
self._redis_hash, Keys.get_node_is_authoring(self.name), True)
self._time_of_last_block_check_activity = float(
self._redis.hget(
self._redis_hash,
Keys.get_node_time_of_last_block_check_activity(self.name),
NONE))
self._time_of_last_height_check_activity = float(
self._redis.hget(
self._redis_hash,
Keys.get_node_time_of_last_height_check_activity(
self.name), NONE))
self._time_of_last_height_change = float(
self._redis.hget(
self._redis_hash,
Keys.get_node_time_of_last_height_change(self.name), NONE))
self._finalized_block_height = self._redis.hget_int(
self._redis_hash,
Keys.get_node_finalized_block_height(self.name), 0)
self._no_change_in_height_warning_sent = self._redis.hget_bool(
self._redis_hash,
Keys.get_node_no_change_in_height_warning_sent(self.name),
False)
self._auth_index = self._redis.hget_int(
self._redis_hash, Keys.get_node_auth_index(self.name), NONE)
if self._time_of_last_block_check_activity != NONE:
self.blocks_authored_alert_limiter. \
set_last_time_that_did_task(datetime.fromtimestamp(
self._time_of_last_block_check_activity))
else:
self.blocks_authored_alert_limiter.did_task()
if self._time_of_last_height_check_activity != NONE:
self._finalized_height_alert_limiter.set_last_time_that_did_task(
datetime.fromtimestamp(
self._time_of_last_height_check_activity))
else:
self._finalized_height_alert_limiter.did_task()
self._time_of_last_height_change = datetime.now().timestamp()
# To avoid the return of byte hget values.
if self._went_down_at is not None:
self._went_down_at = float(self._went_down_at)
logger.debug(
'Restored %s state: _went_down_at=%s, _bonded_balance=%s, '
'_is_syncing=%s, _no_of_peers=%s, _active=%s, _council_member'
'=%s, _elected=%s, _disabled=%s, _no_of_blocks_authored=%s,'
' _time_of_last_block=%s, _is_authoring=%s, _'
'_time_of_last_block_check_activity=%s, '
'_time_of_last_height_change=%s, '
'_time_of_last_height_check_activity, '
'_finalized_block_height=%s,'
' _no_change_in_height_warning_sent=%s, _auth_index=%s',
self.name, self._went_down_at, self._bonded_balance,
self._is_syncing, self._no_of_peers, self._active,
self._council_member, self._elected, self._disabled,
self._no_of_blocks_authored, self._time_of_last_block,
self._is_authoring, self._time_of_last_block_check_activity,
self._time_of_last_height_change,
self._time_of_last_height_check_activity,
self._finalized_block_height,
self._no_change_in_height_warning_sent, self._auth_index)
def save_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, store the current state
if self._redis_enabled:
logger.debug(
'Saving %s state: _went_down_at=%s, _bonded_balance=%s, '
'_is_syncing=%s, _no_of_peers=%s, _active=%s, _council_member'
'=%s, _elected=%s, _disabled=%s, _no_of_blocks_authored=%s,'
' _time_of_last_block=%s, _is_authoring=%s, _'
'_time_of_last_block_check_activity=%s, '
'_time_of_last_height_change=%s, '
'_time_of_last_height_check_activity, '
'_finalized_block_height=%s,'
' _no_change_in_height_warning_sent=%s, _auth_index=%s',
self.name, self._went_down_at, self._bonded_balance,
self._is_syncing, self._no_of_peers, self._active,
self._council_member, self._elected, self._disabled,
self._no_of_blocks_authored, self._time_of_last_block,
self._is_authoring, self._time_of_last_block_check_activity,
self._time_of_last_height_change,
self._time_of_last_height_check_activity,
self._finalized_block_height,
self._no_change_in_height_warning_sent, self._auth_index)
# Set values
self._redis.hset_multiple(
self._redis_hash, {
Keys.get_node_went_down_at(self.name):
str(self._went_down_at),
Keys.get_node_bonded_balance(self.name):
self._bonded_balance,
Keys.get_node_is_syncing(self.name):
str(self._is_syncing),
Keys.get_node_no_of_peers(self.name):
self._no_of_peers,
Keys.get_node_active(self.name):
str(self._active),
Keys.get_node_council_member(self.name):
str(self._council_member),
Keys.get_node_elected(self.name):
str(self._elected),
Keys.get_node_disabled(self.name):
str(self._disabled),
Keys.get_node_blocks_authored(self.name):
self._no_of_blocks_authored,
Keys.get_node_time_of_last_block(self.name):
self._time_of_last_block,
Keys.get_node_is_authoring(self.name):
str(self._is_authoring),
Keys.get_node_time_of_last_block_check_activity(self.name):
self._time_of_last_block_check_activity,
Keys.get_node_time_of_last_height_check_activity(self.name):
self._time_of_last_height_check_activity,
Keys.get_node_time_of_last_height_change(self.name):
self._time_of_last_height_change,
Keys.get_node_finalized_block_height(self.name):
self._finalized_block_height,
Keys.get_node_no_change_in_height_warning_sent(self.name):
str(self._no_change_in_height_warning_sent),
Keys.get_node_auth_index(self.name):
self._auth_index
})
def set_as_down(self, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_as_down: is_down(currently)=%s, channels=%s',
self, self.is_down, channels)
# Alert (varies depending on whether was already down)
if self.is_down and not self._initial_downtime_alert_sent:
if self.is_validator:
channels.alert_critical(CannotAccessNodeAlert(self.name))
else:
channels.alert_warning(CannotAccessNodeAlert(self.name))
self._downtime_alert_limiter.did_task()
self._initial_downtime_alert_sent = True
elif self.is_down and self._downtime_alert_limiter.can_do_task():
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
if self.is_validator:
channels.alert_critical(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
else:
channels.alert_warning(
StillCannotAccessNodeAlert(self.name, went_down_at,
downtime))
self._downtime_alert_limiter.did_task()
elif not self.is_down:
# Do not alert for now just in case this is a connection hiccup
channels.alert_info(ExperiencingDelaysAlert(self.name))
self._went_down_at = datetime.now().timestamp()
self._initial_downtime_alert_sent = False
def set_as_up(self, channels: ChannelSet, logger: logging.Logger) -> None:
logger.debug('%s set_as_up: is_down(currently)=%s, channels=%s', self,
self.is_down, channels)
# Alert if node was down
if self.is_down:
# Only send accessible alert if inaccessible alert was sent
if self._initial_downtime_alert_sent:
went_down_at = datetime.fromtimestamp(self._went_down_at)
downtime = strfdelta(datetime.now() - went_down_at,
"{hours}h, {minutes}m, {seconds}s")
channels.alert_info(
NowAccessibleAlert(self.name, went_down_at, downtime))
# Reset downtime-related values
self._downtime_alert_limiter.reset()
self._went_down_at = None
def set_bonded_balance(self, new_bonded_balance: int, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_bonded_balance: before=%s, new=%s, channels=%s',
self, self.bonded_balance, new_bonded_balance, channels)
# Alert if bonded_balance has changed
if self.bonded_balance not in [None, new_bonded_balance]:
# Extracted data is in tera, therefore, to give more meaningful
# alerts, the bonded balance will be scaled down.
threshold = scale_to_tera(self._change_in_bonded_balance_threshold)
scaled_new_bal = round(scale_to_pico(new_bonded_balance), 3)
scaled_bal = round(scale_to_pico(self.bonded_balance), 3)
if self.is_validator and new_bonded_balance == 0: # N to 0
channels.alert_critical(
BondedBalanceDecreasedAlert(self.name, scaled_bal,
scaled_new_bal))
elif self.is_validator and self.bonded_balance == 0: # 0 to N
channels.alert_info(
BondedBalanceIncreasedAlert(self.name, scaled_bal,
scaled_new_bal))
else: # Any change
diff = new_bonded_balance - self.bonded_balance
if abs(diff) > threshold:
if diff > 0:
channels.alert_info(
BondedBalanceIncreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
else:
channels.alert_info(
BondedBalanceDecreasedByAlert(
self.name, scaled_bal, scaled_new_bal))
# Update bonded balance
self._bonded_balance = new_bonded_balance
def set_is_syncing(self, now_is_syncing: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_is_syncing: before=%s, new=%s, channels=%s', self,
self.is_syncing, now_is_syncing, channels)
# Alert if is syncing has changed
if not self.is_syncing and now_is_syncing:
channels.alert_warning(IsSyncingAlert(self.name))
elif self.is_syncing and not now_is_syncing:
channels.alert_info(IsNoLongerSyncingAlert(self.name))
# Update is-syncing
self._is_syncing = now_is_syncing
def set_no_of_peers(self, new_no_of_peers: int, channels: ChannelSet,
logger: logging.Logger) -> None:
logger.debug('%s set_no_of_peers: before=%s, new=%s, channels=%s',
self, self.no_of_peers, new_no_of_peers, channels)
# Variable alias for improved readability
if self.is_validator:
danger = self._validator_peer_danger_boundary
safe = self._validator_peer_safe_boundary
else:
danger = self._full_node_peer_danger_boundary
safe = None
# Alert if number of peers has changed
if self.no_of_peers not in [None, new_no_of_peers]:
if self.is_validator:
if new_no_of_peers <= self._validator_peer_safe_boundary:
# beneath safe boundary
if new_no_of_peers > self.no_of_peers: # increase
channels.alert_info(
PeersIncreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
elif new_no_of_peers > danger:
# decrease outside danger range
channels.alert_warning(
PeersDecreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
else: # decrease inside danger range
channels.alert_critical(
PeersDecreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
elif self._no_of_peers <= self._validator_peer_safe_boundary \
< new_no_of_peers:
# increase outside safe range for the first time
channels.alert_info(
PeersIncreasedOutsideSafeRangeAlert(self.name, safe))
else:
if new_no_of_peers > self.no_of_peers: # increase
if new_no_of_peers <= danger:
# increase inside danger range
channels.alert_info(
PeersIncreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
elif self.no_of_peers <= danger < new_no_of_peers:
# increase outside danger range
channels.alert_info(
PeersIncreasedOutsideDangerRangeAlert(
self.name, danger))
elif new_no_of_peers > danger: # decrease outside danger range
pass
else: # decrease inside danger range
channels.alert_warning(
PeersDecreasedAlert(self.name, self.no_of_peers,
new_no_of_peers))
# Update number of peers
self._no_of_peers = new_no_of_peers
def set_active(self, now_is_active: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator.
logger.debug('%s set_active: active(currently)=%s, channels=%s', self,
self.is_active, channels)
if self.is_active not in [now_is_active, None]:
if now_is_active:
channels.alert_info(ValidatorIsNowActiveAlert(self.name))
else:
channels.alert_critical(ValidatorIsNotActiveAlert(self.name))
self._active = now_is_active
def set_elected(self, now_is_elected: bool, channels: ChannelSet,
logger: logging.Logger) -> None:
# NOTE: This function assumes that the node is a validator.
logger.debug('%s set_elected: elected(currently)=%s, channels=%s',
self, self.is_elected, channels)
if self.is_elected not in [now_is_elected, None]:
if now_is_elected:
channels.alert_info(
ValidatorIsElectedForTheNextSessionAlert(self.name))
else:
channels.alert_warning(
ValidatorIsNotElectedForNextSessionAlert(self.name))
self._elected = now_is_elected
def set_council_member(self, now_is_council_member: bool,
channels: ChannelSet, logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s set_council_member: council_member(currently)=%s, '
'channels=%s', self, self.is_council_member, channels)
if self.is_council_member not in [now_is_council_member, None]:
if now_is_council_member:
channels.alert_info(
ValidatorIsNowPartOfTheCouncilAlert(self.name))
else:
channels.alert_info(
ValidatorIsNoLongerPartOfTheCouncilAlert(self.name))
self._council_member = now_is_council_member
def set_no_of_blocks_authored(self, channels: ChannelSet,
logger: logging.Logger,
new_no_of_blocks_authored: int,
era_index: int):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s set_no_of_blocks_authored: no_of_blocks_'
'authored(currently)=%s, channels=%s', self,
self._no_of_blocks_authored, channels)
if self.is_active:
if self._no_of_blocks_authored < new_no_of_blocks_authored:
self._no_of_blocks_authored = new_no_of_blocks_authored
self._time_of_last_block = datetime.now().timestamp()
self.blocks_authored_alert_limiter.did_task()
self._time_of_last_block_check_activity = \
datetime.now().timestamp()
if self._is_authoring is False:
self._is_authoring = True
channels.alert_info(
ANewBlockHasNowBeenAuthoredByValidatorAlert(self.name))
elif self._no_of_blocks_authored == \
new_no_of_blocks_authored and \
self.blocks_authored_alert_limiter.can_do_task():
if self._time_of_last_block != NONE:
time_interval = strfdelta(
datetime.now() -
datetime.fromtimestamp(self._time_of_last_block),
"{hours}h, {minutes}m, {seconds}s")
channels.alert_warning(
LastAuthoredBlockInEraAlert(self.name, time_interval,
era_index))
else:
channels.alert_warning(
NoBlocksHaveYetBeenAuthoredInEraAlert(
self.name, era_index))
self._is_authoring = False
self.blocks_authored_alert_limiter.did_task()
self._time_of_last_block_check_activity = \
datetime.now().timestamp()
def reset_no_of_blocks_authored(self, channels: ChannelSet,
logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s reset_no_of_blocks_authored: no_of_blocks_'
'authored(currently)=%s, channels=%s', self,
self._no_of_blocks_authored, channels)
self._no_of_blocks_authored = 0
def set_auth_index(self, new_auth_index: int, logger: logging.Logger):
logger.debug('%s set_auth_index: auth_index(currently)=%s', self,
self._auth_index)
if self.is_active:
self._auth_index = new_auth_index
def update_finalized_block_height(self, new_finalized_height: int,
logger: logging.Logger,
channels: ChannelSet):
logger.debug(
'%s update_finalized_block_height: finalized_block_height'
' (currently)=%s', self, self._finalized_block_height)
current_timestamp = datetime.now().timestamp()
if self._finalized_block_height != new_finalized_height:
if self.is_no_change_in_height_warning_sent:
self._no_change_in_height_warning_sent = False
channels.alert_info(
NodeFinalizedBlockHeightHasNowBeenUpdatedAlert(self.name))
if self._finalized_block_height > new_finalized_height:
logger.info(
'The finalized height of node {} decreased to {}.'.format(
self, self._finalized_block_height))
self._finalized_block_height = new_finalized_height
self._time_of_last_height_change = current_timestamp
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter.set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
else:
timestamp_difference = current_timestamp - \
self._time_of_last_height_change
time_interval = strfdelta(
timedelta(seconds=int(timestamp_difference)),
"{hours}h, {minutes}m, {seconds}s")
if not self.is_no_change_in_height_warning_sent and \
timestamp_difference > \
self._no_change_in_height_first_warning_seconds:
self._no_change_in_height_warning_sent = True
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
elif self._finalized_height_alert_limiter.can_do_task() and \
self.is_no_change_in_height_warning_sent:
if self.is_validator:
channels.alert_critical(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
else:
channels.alert_warning(
NodeFinalizedBlockHeightDidNotChangeInAlert(
self.name, time_interval))
self._time_of_last_height_check_activity = current_timestamp
self._finalized_height_alert_limiter. \
set_last_time_that_did_task(
datetime.fromtimestamp(current_timestamp))
def set_disabled(self, now_is_disabled: bool, session: int,
channels: ChannelSet, logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug(
'%s set_disabled: _disabled(currently)=%s, '
'channels=%s', self, self.is_disabled, channels)
if self.is_disabled not in [now_is_disabled, None]:
if now_is_disabled:
channels.alert_critical(
ValidatorHasBeenDisabledInSessionAlert(self.name, session))
else:
channels.alert_info(
ValidatorIsNoLongerDisabledInSessionAlert(
self.name, session))
self._disabled = now_is_disabled
def slash(self, amount: float, channels: ChannelSet,
logger: logging.Logger):
# NOTE: This function assumes that the node is a validator.
logger.debug('%s slash: channels=%s', self, channels)
if amount > 0:
channels.alert_critical(
ValidatorHasBeenSlashedAlert(self.name, amount))
def disconnect_from_api(self, channels: ChannelSet,
logger: logging.Logger):
logger.debug('%s disconnect_from_api: channels=%s', self, channels)
if self.is_connected_to_api_server:
if self.is_validator:
channels.alert_critical(
NodeWasNotConnectedToApiServerAlert(self.name))
else:
channels.alert_warning(
NodeWasNotConnectedToApiServerAlert(self.name))
self._connected_to_api_server = False
def connect_with_api(self, channels: ChannelSet, logger: logging.Logger):
logger.debug('%s connect_with_api: channels=%s', self, channels)
if not self.is_connected_to_api_server:
channels.alert_info(NodeConnectedToApiServerAgainAlert(self.name))
self._connected_to_api_server = True
def __init__(self,
name: str,
ws_url: Optional[str],
node_type: NodeType,
stash_account_address: Optional[str],
chain: str,
redis: Optional[RedisApi],
is_archive_node: bool,
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self._ws_url = ws_url
self._node_type = node_type
self._stash_account_address = stash_account_address
self._chain = chain
self._redis = redis
self._redis_enabled = redis is not None
self._redis_hash = Keys.get_hash_blockchain(self.chain)
self._connected_to_api_server = True
self._went_down_at = None
self._bonded_balance = None
self._is_syncing = False
self._no_of_peers = None
self._initial_downtime_alert_sent = False
self._no_change_in_height_warning_sent = False
self._active = None
self._disabled = None
self._elected = None
self._council_member = None
self._no_of_blocks_authored = 0
self._finalized_block_height = 0
self._time_of_last_block = NONE
self._time_of_last_block_check_activity = NONE
self._time_of_last_height_check_activity = NONE
self._time_of_last_height_change = NONE
self._auth_index = NONE
self._is_authoring = True
self._is_archive_node = is_archive_node
self._validator_peer_danger_boundary = \
internal_conf.validator_peer_danger_boundary
self._validator_peer_safe_boundary = \
internal_conf.validator_peer_safe_boundary
self._full_node_peer_danger_boundary = \
internal_conf.full_node_peer_danger_boundary
self._max_time_between_blocks_authored = \
internal_conf.max_time_alert_between_blocks_authored
self._no_change_in_height_first_warning_seconds = \
internal_conf.no_change_in_height_first_warning_seconds
self._no_change_in_height_interval_seconds = \
internal_conf.no_change_in_height_interval_seconds
self._downtime_alert_limiter = TimedTaskLimiter(
internal_conf.downtime_alert_time_interval)
self._blocks_authored_alert_limiter = TimedTaskLimiter(
self._max_time_between_blocks_authored)
self._finalized_height_alert_limiter = TimedTaskLimiter(
timedelta(seconds=int(self._no_change_in_height_interval_seconds)))
self._change_in_bonded_balance_threshold = \
internal_conf.change_in_bonded_balance_threshold
class RedisApi:
def __init__(self, logger: logging.Logger, db: int,
host: str = 'localhost', port: int = 6379,
password: str = '', namespace: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60)) \
-> None:
self._logger = logger
if password == '':
self._redis = redis.Redis(host=host, port=port, db=db)
else:
self._redis = redis.Redis(host=host,
port=port,
db=db,
password=password)
self._namespace = namespace
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
self._is_live = True # This is necessary to initialise the variable
self._set_as_live()
self._logger.info('Redis initialised.')
@property
def is_live(self) -> bool:
return self._is_live
def _add_namespace(self, key: str) -> str:
if not key.startswith(self._namespace + ':'):
return self._namespace + ':' + key
else:
return key # prevent adding namespace twice
def _remove_namespace(self, key: str) -> str:
if not key.startswith(self._namespace + ':'):
return key # prevent removing namespace twice
else:
return key.replace(self._namespace + ':', '', 1)
def _set_as_live(self) -> None:
if not self._is_live:
self._logger.info('Redis is now accessible again.')
self._is_live = True
def _set_as_down(self) -> None:
# If Redis is live or if we can check whether it is live (because the
# live check time interval has passed), reset the live check limiter
# so that usage of Redis is skipped for as long as the time interval
if self._is_live or self._live_check_limiter.can_do_task():
self._live_check_limiter.did_task()
self._logger.warning('Redis is unusable for some reason. Stopping '
'usage temporarily to improve performance.')
self._is_live = False
def _do_not_use_if_recently_went_down(self) -> bool:
# If Redis is not live and cannot check if it is live (by using it)
# then stop the function called from happening by returning True
return not self._is_live and not self._live_check_limiter.can_do_task()
def set_unsafe(self, key: str, value: RedisType):
key = self._add_namespace(key)
set_ret = self._redis.set(key, value)
return set_ret
def set_multiple_unsafe(self, key_values: Dict[str, RedisType]):
# Add namespace to keys
keys = list(key_values.keys())
unique_keys = [self._add_namespace(k) for k in keys]
for k, uk in zip(keys, unique_keys):
key_values[uk] = key_values.pop(k)
# Set multiple
pipe = self._redis.pipeline()
for key, value in key_values.items():
pipe.set(key, value if value is not None else 'None')
exec_ret = pipe.execute()
return exec_ret
def set_for_unsafe(self, key: str, value: RedisType, time: timedelta):
key = self._add_namespace(key)
pipe = self._redis.pipeline()
pipe.set(key, value)
pipe.expire(key, time)
exec_ret = pipe.execute()
return exec_ret
def get_unsafe(self, key: str, default=None) -> Optional[bytes]:
key = self._add_namespace(key)
if self.exists_unsafe(key):
get_ret = self._redis.get(key)
if get_ret.decode('UTF-8') == 'None':
return None
else:
return get_ret
else:
return default
def get_int_unsafe(self, key: str, default=None) -> Optional[int]:
key = self._add_namespace(key)
get_ret = self.get_unsafe(key, None)
try:
return int(get_ret) if get_ret is not None else default
except ValueError:
self._logger.error(
'Could not convert value %s of key %s to an integer. '
'Defaulting to value %s.', get_ret, key, default)
return default
def get_bool_unsafe(self, key: str, default=None) -> Optional[bool]:
key = self._add_namespace(key)
get_ret = self.get_unsafe(key, None)
return (get_ret.decode() == 'True') if get_ret is not None else default
def exists_unsafe(self, key: str) -> bool:
key = self._add_namespace(key)
exists_ret = self._redis.exists(key)
return bool(exists_ret)
def get_keys_unsafe(self, pattern: str = "*") -> List[str]:
pattern = self._add_namespace(pattern)
# Decode and remove namespace
keys_list = self._redis.keys(pattern)
keys_list = [k.decode('utf8') for k in keys_list]
keys_list = [self._remove_namespace(k) for k in keys_list]
return keys_list
def remove_unsafe(self, *keys):
keys = [self._add_namespace(k) for k in keys]
delete_ret = self._redis.delete(*keys)
return delete_ret
def delete_all_unsafe(self):
flushdb_ret = self._redis.flushdb()
return flushdb_ret
def set(self, key: str, value: RedisType):
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self.set_unsafe(key, value)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in set: %s', e)
self._set_as_down()
return None
def set_multiple(self, key_values: Dict[str, RedisType]):
# Add namespace to keys
keys = list(key_values.keys())
unique_keys = [self._add_namespace(k) for k in keys]
for k, uk in zip(keys, unique_keys):
key_values[uk] = key_values.pop(k)
# Set multiple
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self.set_multiple_unsafe(key_values)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in set_multiple: %s', e)
self._set_as_down()
return None
def set_for(self, key: str, value: RedisType, time: timedelta):
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self.set_for_unsafe(key, value, time)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in set_for: %s', e)
self._set_as_down()
return None
def time_to_live_unsafe(self, key: str):
key = self._add_namespace(key)
time_to_live = self._redis.ttl(key)
# -1: Key exists but has no associated timeout
# -2: Key does not exist
if time_to_live == -1 or time_to_live == -2:
return None
else:
return time_to_live
def time_to_live(self, key: str):
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self.time_to_live_unsafe(key)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in time_to_live: %s', e)
self._set_as_down()
return None
def get(self, key: str, default=None) -> Optional[bytes]:
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return default
ret = self.get_unsafe(key, default)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in get: %s', e)
self._set_as_down()
return default
def get_int(self, key: str, default=None) -> Optional[int]:
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return default
ret = self.get_int_unsafe(key, default)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in get_int: %s', e)
self._set_as_down()
return default
def get_bool(self, key: str, default=None) -> Optional[bool]:
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return default
ret = self.get_bool_unsafe(key, default)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in get_bool: %s', e)
self._set_as_down()
return default
def exists(self, key: str) -> bool:
key = self._add_namespace(key)
try:
if self._do_not_use_if_recently_went_down():
return False
ret = self.exists_unsafe(key)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in exists: %s', e)
self._set_as_down()
return False
def get_keys(self, pattern: str = "*") -> List[str]:
pattern = self._add_namespace(pattern)
try:
if self._do_not_use_if_recently_went_down():
return []
ret = self.get_keys_unsafe(pattern)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in get_keys: %s', e)
self._set_as_down()
return []
def remove(self, *keys):
keys = [self._add_namespace(k) for k in keys]
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self.remove_unsafe(*keys)
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in remove: %s', e)
self._set_as_down()
return None
def delete_all(self):
try:
if self._do_not_use_if_recently_went_down():
return None
ret = self.delete_all_unsafe()
self._set_as_live()
return ret
except Exception as e:
self._logger.error('Redis error in delete_all: %s', e)
self._set_as_down()
return None
def ping_unsafe(self) -> bool:
return self._redis.ping()
class MongoApi:
def __init__(self, logger: logging.Logger, db_name: str,
host: str = 'localhost', port: int = 27017,
username: str = '', password: str = '',
live_check_time_interval: timedelta = timedelta(seconds=60),
timeout_ms: int = 10000) \
-> None:
self._logger = logger
self._db_name = db_name
if password == '':
self._client = MongoClient(
host=host, port=port, connectTimeoutMS=timeout_ms,
socketTimeoutMS=timeout_ms, serverSelectionTimeoutMS=timeout_ms)
else:
self._client = MongoClient(
host=host, port=port, connectTimeoutMS=timeout_ms,
socketTimeoutMS=timeout_ms, serverSelectionTimeoutMS=timeout_ms,
username=username, password=password)
# The live check limiter means that we don't wait for connection
# errors to occur to be able to continue, thus speeding everything up
self._live_check_limiter = TimedTaskLimiter(live_check_time_interval)
self._is_live = True # This is necessary to initialise the variable
self._set_as_live()
self._logger.info("Mongo initialised.")
@property
def _db(self):
return self._client[self._db_name]
@property
def is_live(self) -> bool:
return self._is_live
def _set_as_live(self) -> None:
if not self._is_live:
self._logger.info("Mongo is now accessible again.")
self._is_live = True
def _set_as_down(self) -> None:
# If Mongo is live or if we can check whether it is live (because the
# live check time interval has passed), reset the live check limiter
# so that usage of Mongo is skipped for as long as the time interval
if self._is_live or self._live_check_limiter.can_do_task():
self._live_check_limiter.did_task()
self._logger.warning("Mongo is unusable for some reason. Stopping "
"usage temporarily to improve performance.")
self._is_live = False
def _do_not_use_if_recently_went_down(self) -> bool:
# If Mongo is not live and cannot check if it is live return true
return not self._is_live and not self._live_check_limiter.can_do_task()
def _safe(self, function, args: List[Any], default_return: Any):
# Calls the function with the provided arguments and performs exception
# logging as well as returns a specified default if mongo is running
# into difficulties.
try:
if self._do_not_use_if_recently_went_down():
return default_return
ret = function(*args)
self._set_as_live()
return ret
except Exception as e:
self._logger.error("Mongo error in %s: %s", function.__name__, e)
self._set_as_down()
return default_return
def insert_one(self, collection: str, document: Dict) \
-> Optional[InsertOneResult]:
return self._safe(
lambda col, doc: self._db[col].insert_one(doc),
[collection, document], None)
def insert_many(self, collection: str, documents: List[Dict]) \
-> Optional[InsertManyResult]:
return self._safe(
lambda col, doc: self._db[col].insert_many(doc),
[collection, documents], None)
def update_one(self, collection: str, query: Dict, document: Dict) \
-> Optional[UpdateResult]:
return self._safe(
lambda col, q, doc: self._db[col].update_one(q, doc, upsert=True),
[collection, query, document], None)
def get_all(self, collection: str) -> Optional[List[Dict]]:
return self._safe(
lambda col: list(self._db[col].find({})),
[collection], None)
def drop_collection(self, collection: str) -> Optional[Dict]:
return self._safe(
lambda col: self._db.drop_collection(col),
[collection], None)
def drop_db(self) -> None:
return self._safe(
lambda: self._client.drop_database(self._db.name),
[], None)
def ping_unsafe(self):
return self._db.command('ping')
def ping_auth(self, username: str, password: str):
return self._db.authenticate(username, password, 'admin')