def save_state(self) -> None:
# If Redis is enabled, save the current time, indicating that the node
# monitor was alive at this time, the current session index,
# and the last height checked.
if self.redis_enabled:
key_si = Keys.get_node_monitor_session_index(self.monitor_name)
key_lh = Keys.get_node_monitor_last_height_checked(
self.monitor_name)
key_alive = Keys.get_node_monitor_alive(self.monitor_name)
self.logger.debug('Saving node monitor state: %s=%s, %s=%s',
self._monitor_name, key_si, self._session_index,
key_lh, self._last_height_checked)
# Set session index key
self.redis.set(key_si, self._session_index)
# Set last height checked key
until = timedelta(seconds=self._redis_last_height_key_timeout)
self.redis.set_for(key_lh, self._last_height_checked, until)
# Set alive key (to be able to query latest update from Telegram)
until = timedelta(seconds=self._redis_alive_key_timeout)
self.redis.set_for(key_alive, str(datetime.now().timestamp()),
until)
def test_load_state_sets_values_to_saved_values(self):
# Set Redis values manually
hash_name = Keys.get_hash_blockchain(self.blockchain.name)
self.redis.hset_multiple_unsafe(
hash_name, {
Keys.get_blockchain_referendum_count(self.blockchain.name):
self.dummy_referendum_count,
Keys.get_blockchain_public_prop_count(self.blockchain.name):
self.dummy_public_prop_count,
Keys.get_blockchain_council_prop_count(self.blockchain.name):
self.dummy_council_prop_count,
Keys.get_blockchain_validator_set_size(self.blockchain.name):
self.dummy_validator_set_size,
})
# Load the Redis values
self.blockchain.load_state(self.logger)
# Assert
self.assertEqual(self.blockchain.referendum_count,
self.dummy_referendum_count)
self.assertEqual(self.blockchain.public_prop_count,
self.dummy_public_prop_count)
self.assertEqual(self.blockchain.council_prop_count,
self.dummy_council_prop_count)
self.assertEqual(self.blockchain.validator_set_size,
self.dummy_validator_set_size)
def load_state(self) -> None:
# If Redis is enabled, load the session index, and last height checked
# for slashing if any.
if self.redis_enabled:
key_si = Keys.get_node_monitor_session_index(self.monitor_name)
key_lh = Keys.get_node_monitor_last_height_checked(
self.monitor_name)
self._session_index = self.redis.get_int(key_si, NONE)
self._last_height_checked = self.redis.get_int(key_lh, NONE)
self.logger.debug('Restored %s state: %s=%s, %s=%s',
self._monitor_name, key_si, self._session_index,
key_lh, self._last_height_checked)
def load_state(self) -> None:
# If Redis is enabled, restore any previously stored number of releases
if self.redis_enabled:
key = Keys.get_github_releases(self.repo_name)
self._prev_no_of_releases = self.redis.get_int(key, None)
self.logger.debug('Restored github monitor state: %s=%s', key,
self._prev_no_of_releases)
def test_save_state_sets_nothing_if_no_previous_state(self):
# Save the values to Redis
self.monitor.save_state()
# Assert
key = Keys.get_github_releases(self.repo_name)
self.assertIsNone(self.redis.get_int(key))
def save_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, store the current state
if self._redis_enabled:
logger.debug(
'Saving %s state: _process_cpu_seconds_total=%s, '
'_process_memory_usage=%s, _virtual_memory_usage=%s, '
'_open_file_descriptors=%s, _system_cpu_usage=%s, '
'_system_ram_usage=%s, _system_storage_usage=%s', self.name,
self._process_cpu_seconds_total, self._process_memory_usage,
self._virtual_memory_usage, self._open_file_descriptors,
self._system_cpu_usage, self._system_ram_usage,
self._system_storage_usage)
# Set values
self._redis.hset_multiple(
self._redis_hash, {
Keys.get_system_get_process_cpu_seconds_total(self.name):
self._process_cpu_seconds_total,
Keys.get_system_get_process_memory_usage(self.name):
self._process_memory_usage,
Keys.get_system_get_virtual_memory_usage(self.name):
self._virtual_memory_usage,
Keys.get_system_get_open_file_descriptors(self.name):
self._open_file_descriptors,
Keys.get_system_get_system_cpu_usage(self.name):
self._system_cpu_usage,
Keys.get_system_get_system_ram_usage(self.name):
self._system_ram_usage,
Keys.get_system_get_system_storage_usage(self.name):
self._system_storage_usage
})
def load_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, load any previously stored state
if self._redis_enabled:
self._process_cpu_seconds_total = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_process_cpu_seconds_total(self.name), None)
self._process_memory_usage = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_process_memory_usage(self.name), None)
self._virtual_memory_usage = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_virtual_memory_usage(self.name), None)
self._open_file_descriptors = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_open_file_descriptors(self.name), None)
self._system_cpu_usage = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_system_cpu_usage(self.name), None)
self._system_ram_usage = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_system_ram_usage(self.name), None)
self._system_storage_usage = self._redis.hget_int(
self._redis_hash,
Keys.get_system_get_system_storage_usage(self.name), None)
logger.debug(
'Restored %s state: _process_cpu_seconds_total=%s, '
'_process_memory_usage=%s, _virtual_memory_usage=%s, '
'_open_file_descriptors=%s, _system_cpu_usage=%s, '
'_system_ram_usage=%s, _system_storage_usage=%s', self.name,
self._process_cpu_seconds_total, self._process_memory_usage,
self._virtual_memory_usage, self._open_file_descriptors,
self._system_cpu_usage, self._system_ram_usage,
self._system_storage_usage)
def test_save_state_saves_alive_key_temporarily(self):
self.monitor.save_state()
key = Keys.get_blockchain_monitor_alive(self.monitor_name)
last_update = self.redis.get(key)
timeout = self.redis.time_to_live(key)
self.assertIsNotNone(last_update)
self.assertEqual(timeout, self.alive_key_timeout)
def save_state(self) -> None:
# If Redis is enabled save the current time, indicating that the monitor
# was alive at this time.
if self.redis_enabled:
self.logger.debug('Saving %s state', self._monitor_name)
# Set alive key (to be able to query latest update from Telegram)
key = Keys.get_blockchain_monitor_alive(self.monitor_name)
until = timedelta(seconds=self._redis_alive_key_timeout)
self.redis.set_for(key, str(datetime.now().timestamp()), until)
def load_state(self) -> None:
# If Redis is enabled, load any previously stored state
if self.redis_enabled:
key_lh = Keys.get_node_monitor_last_height_checked(
self.monitor_name)
self._last_height_checked = self.redis.get_int(key_lh, NONE)
self.logger.debug('Restored %s state: %s=%s', self._monitor_name,
key_lh, self._last_height_checked)
def test_save_state_sets_values_to_current_values(self):
# Set monitor values manually
self.monitor._prev_no_of_releases = 10
# Save the values to Redis
self.monitor.save_state()
# Assert
key = Keys.get_github_releases(self.repo_name)
self.assertEqual(10, self.redis.get_int(key))
def test_load_state_sets_values_to_saved_values(self):
# Set Redis values manually
key = Keys.get_github_releases(self.repo_name)
self.redis.set_unsafe(key, 10)
# Load the values from Redis
self.monitor.load_state()
# Assert
self.assertEqual(10, self.monitor._prev_no_of_releases)
def test_load_state_sets_values_to_saved_values(self) -> None:
# Set Redis values manually
key_lh = Keys.get_node_monitor_last_height_checked(self.monitor_name)
self.redis.set_unsafe(key_lh, self.dummy_last_height_checked)
# Load the values from Redis
self.monitor.load_state()
# Assert
self.assertEqual(self.dummy_last_height_checked,
self.monitor.last_height_checked)
def save_state(self) -> None:
# If Redis is enabled, save the currently known number of releases
if self.redis_enabled:
if self._prev_no_of_releases is None:
self.logger.debug('Skipping github monitor state save '
'due to None previous number of releases.')
return
key = Keys.get_github_releases(self.repo_name)
self.logger.debug('Saving github monitor state: %s=%s', key,
self._prev_no_of_releases)
self.redis.set(key, self._prev_no_of_releases)
def _calls_snoozed(self, logger: logging.Logger) \
-> bool:
if self.redis_enabled:
key = Keys.get_twilio_snooze()
if self.redis.exists(key):
snooze_until = self.redis.get(key).decode("utf-8")
logger.info('Tried to call but calls are snoozed until {}.'
''.format(snooze_until))
return True
else:
logger.info('Twilio did not find a snooze in Redis.')
return False
def __init__(self, name: str, redis: Optional[RedisApi]) -> None:
super().__init__()
self.name = name
self._redis = redis
self._redis_enabled = redis is not None
self._redis_hash = Keys.get_hash_blockchain(name)
self._referendum_count = None
self._public_prop_count = None
self._council_prop_count = None
self._validator_set_size = None
def save_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, store the current state
if self._redis_enabled:
logger.debug(
'Saving %s state: _referendum_count=%s, '
'_public_prop_count=%s, _council_prop_count=%s, '
'_validator_set_size=%s',
self.name, self._referendum_count, self._public_prop_count,
self._council_prop_count, self._validator_set_size)
# Set values
self._redis.hset_multiple(self._redis_hash, {
Keys.get_blockchain_referendum_count(self.name):
self._referendum_count,
Keys.get_blockchain_public_prop_count(self.name):
self._public_prop_count,
Keys.get_blockchain_council_prop_count(self.name):
self._council_prop_count,
Keys.get_blockchain_validator_set_size(self.name):
self._validator_set_size
})
def test_periodic_alive_reminder_sends_no_alert_if_mute_key_present(self):
hours = timedelta(hours=float(1))
until = str(datetime.now() + hours)
key = Keys.get_alive_reminder_mute()
self.redis.set_for(key, until, hours)
self.counter_channel.reset() # ignore previous alerts
self.par.send_alive_alert()
self.redis.remove(key)
self.assertEqual(self.counter_channel.warning_count, 0)
self.assertEqual(self.counter_channel.critical_count, 0)
self.assertEqual(self.counter_channel.info_count, 0)
self.assertEqual(self.counter_channel.error_count, 0)
def load_state(self, logger: logging.Logger) -> None:
# If Redis is enabled, load any previously stored state
if self._redis_enabled:
self._referendum_count = self._redis.hget_int(
self._redis_hash,
Keys.get_blockchain_referendum_count(self.name), None)
self._public_prop_count = self._redis.hget_int(
self._redis_hash,
Keys.get_blockchain_public_prop_count(self.name), None)
self._council_prop_count = self._redis.hget_int(
self._redis_hash,
Keys.get_blockchain_council_prop_count(self.name), None)
self._validator_set_size = self._redis.hget_int(
self._redis_hash,
Keys.get_blockchain_validator_set_size(self.name), None)
logger.debug(
'Restored %s state: _referendum_count=%s, '
'_public_prop_count=%s, _council_prop_count=%s, '
'_validator_set_size=%s',
self.name, self._referendum_count, self._public_prop_count,
self._council_prop_count, self._validator_set_size)
def test_save_state_sets_values_to_current_values(self):
# Set blockchain values manually
self.blockchain.set_referendum_count(self.dummy_referendum_count,
self.channel_set, self.logger)
self.blockchain.set_council_prop_count(self.dummy_council_prop_count,
self.channel_set, self.logger)
self.blockchain.set_public_prop_count(self.dummy_public_prop_count,
self.channel_set, self.logger)
self.blockchain.set_validator_set_size(self.dummy_validator_set_size,
self.channel_set, self.logger)
# Save the values to Redis
self.blockchain.save_state(self.logger)
# Assert
hash_name = Keys.get_hash_blockchain(self.blockchain.name)
self.assertEqual(
self.redis.hget_int_unsafe(
hash_name,
Keys.get_blockchain_referendum_count(self.blockchain.name)),
self.dummy_referendum_count)
self.assertEqual(
self.redis.hget_int_unsafe(
hash_name,
Keys.get_blockchain_public_prop_count(self.blockchain.name)),
self.dummy_public_prop_count)
self.assertEqual(
self.redis.hget_int_unsafe(
hash_name,
Keys.get_blockchain_council_prop_count(self.blockchain.name)),
self.dummy_council_prop_count)
self.assertEqual(
self.redis.hget_int_unsafe(
hash_name,
Keys.get_blockchain_validator_set_size(self.blockchain.name)),
self.dummy_validator_set_size)
def test_save_state_sets_values_to_current_values_and_stores_alive_key_temp(
self) -> None:
# Set monitor values manually
self.monitor._last_height_checked = self.dummy_last_height_checked
# Save the values to Redis
self.monitor.save_state()
key_lh = Keys.get_node_monitor_last_height_checked(self.monitor_name)
# Get last update, and its timeout in Redis
last_update = self.redis.get(key_lh)
timeout = self.redis.time_to_live(key_lh)
# Assert
self.assertEqual(self.dummy_last_height_checked,
self.redis.get_int(key_lh))
self.assertIsNotNone(last_update)
self.assertEqual(timeout, self.redis_alive_key_timeout)
def send_alive_alert(self) -> None:
# If it is not the case that Redis is enabled and the reminder is muted,
# inform the node operator that the alerter is still running.
if not (self._redis_enabled
and self._redis.exists(Keys.get_alive_reminder_mute())):
self._channel_set.alert_info(AlerterAliveAlert())
def __init__(self,
name: str,
redis: Optional[RedisApi],
node: Node,
internal_conf: InternalConfig = InternalConf) -> None:
super().__init__()
self.name = name
self._node = node
self._redis = redis
self._redis_enabled = redis is not None
self._redis_hash = Keys.get_hash_blockchain(node.chain)
self._process_cpu_seconds_total = None
# Process memory usage
self._process_memory_usage = None
# Virtual memory usage
self._virtual_memory_usage = None
# Open file descriptors
self._open_file_descriptors = None
# System CPU usage
self._system_cpu_usage = None
# System RAM usage
self._system_ram_usage = None
# System storage used
self._system_storage_usage = None
self._validator_process_memory_usage_danger_boundary = \
internal_conf.validator_process_memory_usage_danger_boundary
self._validator_process_memory_usage_safe_boundary = \
internal_conf.validator_process_memory_usage_safe_boundary
self._validator_open_file_descriptors_danger_boundary = \
internal_conf.validator_open_file_descriptors_danger_boundary
self._validator_open_file_descriptors_safe_boundary = \
internal_conf.validator_open_file_descriptors_safe_boundary
self._validator_system_cpu_usage_danger_boundary = \
internal_conf.validator_system_cpu_usage_danger_boundary
self._validator_system_cpu_usage_safe_boundary = \
internal_conf.validator_system_cpu_usage_safe_boundary
self._validator_system_ram_usage_danger_boundary = \
internal_conf.validator_system_ram_usage_danger_boundary
self._validator_system_ram_usage_safe_boundary = \
internal_conf.validator_system_ram_usage_safe_boundary
self._validator_system_storage_usage_danger_boundary = \
internal_conf.validator_system_storage_usage_danger_boundary
self._validator_system_storage_usage_safe_boundary = \
internal_conf.validator_system_storage_usage_safe_boundary
self._node_process_memory_usage_danger_boundary = \
internal_conf.node_process_memory_usage_danger_boundary
self._node_process_memory_usage_safe_boundary = \
internal_conf.node_process_memory_usage_safe_boundary
self._node_open_file_descriptors_danger_boundary = \
internal_conf.node_open_file_descriptors_danger_boundary
self._node_open_file_descriptors_safe_boundary = \
internal_conf.node_open_file_descriptors_safe_boundary
self._node_system_cpu_usage_danger_boundary = \
internal_conf.node_system_cpu_usage_danger_boundary
self._node_system_cpu_usage_safe_boundary = \
internal_conf.node_system_cpu_usage_safe_boundary
self._node_system_ram_usage_danger_boundary = \
internal_conf.node_system_ram_usage_danger_boundary
self._node_system_ram_usage_safe_boundary = \
internal_conf.node_system_ram_usage_safe_boundary
self._node_system_storage_usage_danger_boundary = \
internal_conf.node_system_storage_usage_danger_boundary
self._node_system_storage_usage_safe_boundary = \
internal_conf.node_system_storage_usage_safe_boundary