def _process_redis_store(self, routing_key: str, data: Dict) -> None:
if data:
self.logger.debug("Saving for %s the data=%s.", routing_key, data)
self.redis.set(Keys.get_config(routing_key), json.dumps(data))
else:
self.logger.debug("Removing the saved config for key %s .",
routing_key)
if self.redis.exists(Keys.get_config(routing_key)):
self.redis.remove(Keys.get_config(routing_key))
def test_process_data_saves_in_redis(self, mock_config_data,
mock_routing_key, mock_send_hb,
mock_ack) -> None:
self.rabbitmq.connect()
mock_ack.return_value = None
try:
data = eval(mock_config_data)
routing_key = eval(mock_routing_key)
self.test_store._initialise_rabbitmq()
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=eval(mock_routing_key))
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
self.assertEqual(
data,
json.loads(
self.redis.get(
Keys.get_config(routing_key)).decode("utf-8")))
except Exception as e:
self.fail("Test failed: {}".format(e))
def _process_redis_error_store(self, data: Dict) -> None:
meta_data = data['meta_data']
error_code = data['code']
system_name = meta_data['system_name']
downtime_exception = SystemIsDownException(system_name)
if error_code == downtime_exception.code:
system_id = meta_data['system_id']
parent_id = meta_data['system_parent_id']
metrics = data['data']
self.logger.debug("Saving %s state: _went_down_at=%s", system_name,
metrics['went_down_at'])
self.redis.hset(Keys.get_hash_parent(parent_id),
Keys.get_system_went_down_at(system_id),
str(metrics['went_down_at']))
def _process_redis_store(self, alert: Dict) -> None:
metric_data = {
'severity': alert['severity'],
'message': alert['message']
}
key = alert['origin_id']
self.redis.hset(Keys.get_hash_parent(alert['parent_id']),
eval('Keys.get_alert_{}(key)'.format(alert['metric'])),
json.dumps(metric_data))
def _process_redis_result_store(self, data: Dict) -> None:
meta_data = data['meta_data']
repo_name = meta_data['repo_name']
repo_id = meta_data['repo_id']
parent_id = meta_data['repo_parent_id']
metrics = data['data']
self.logger.debug(
"Saving %s state: _no_of_releases=%s, _last_monitored=%s",
repo_name, metrics['no_of_releases'], meta_data['last_monitored'])
self.redis.hset_multiple(
Keys.get_hash_parent(parent_id), {
Keys.get_github_no_of_releases(repo_id):
str(metrics['no_of_releases']),
Keys.get_github_last_monitored(repo_id):
str(meta_data['last_monitored']),
})
def test_process_redis_store_redis_is_called_correctly(
self, mock_github_data, mock_hset_multiple) -> None:
data = eval(mock_github_data)
self.test_store._process_redis_store(data)
meta_data = data['result']['meta_data']
repo_id = meta_data['repo_id']
parent_id = meta_data['repo_parent_id']
metrics = data['result']['data']
call_1 = call(
Keys.get_hash_parent(parent_id), {
Keys.get_github_no_of_releases(repo_id):
str(metrics['no_of_releases']),
Keys.get_github_last_monitored(repo_id):
str(meta_data['last_monitored']),
})
mock_hset_multiple.assert_has_calls([call_1])
def test_process_redis_store_redis_stores_correctly(
self, mock_github_data) -> None:
data = eval(mock_github_data)
self.test_store._process_redis_store(data)
meta_data = data['result']['meta_data']
repo_id = meta_data['repo_id']
parent_id = meta_data['repo_parent_id']
metrics = data['result']['data']
self.assertEqual(
str(metrics['no_of_releases']),
self.redis.hget(
Keys.get_hash_parent(parent_id),
Keys.get_github_no_of_releases(repo_id)).decode("utf-8"))
self.assertEqual(
str(meta_data['last_monitored']),
self.redis.hget(
Keys.get_hash_parent(parent_id),
Keys.get_github_last_monitored(repo_id)).decode("utf-8"))
def _process_heartbeat(self, ch: BlockingChannel,
method: pika.spec.Basic.Deliver,
properties: pika.spec.BasicProperties, body: bytes) \
-> None:
heartbeat = json.loads(body)
self.logger.debug("Received %s. Now processing this data.", heartbeat)
try:
if method.routing_key == 'heartbeat.worker' or \
method.routing_key == 'heartbeat.manager':
component_name = heartbeat['component_name']
key_heartbeat = Keys.get_component_heartbeat(component_name)
transformed_heartbeat = json.dumps(heartbeat)
self._save_to_redis_and_add_to_state_if_fail(
key_heartbeat, transformed_heartbeat)
self._dump_unsavable_redis_data()
self.logger.debug("Successfully processed %s", heartbeat)
else:
raise ReceivedUnexpectedDataException(
"{}: _process_heartbeat".format(self))
except Exception as e:
self.logger.error("Error when processing %s", heartbeat)
self.logger.exception(e)
self.rabbitmq.basic_ack(method.delivery_tag, False)
self.logger.debug("Saving %s heartbeat to Redis", self)
key_heartbeat = Keys.get_component_heartbeat(self.name)
handler_heartbeat = {
'component_name': self.name,
'timestamp': datetime.now().timestamp()
}
transformed_handler_heartbeat = json.dumps(handler_heartbeat)
ret = self.redis.set(key_heartbeat, transformed_handler_heartbeat)
if ret is None:
self.logger.error("Could not save %s=%s to Redis.", key_heartbeat,
transformed_handler_heartbeat)
def load_state(self, repo: Union[System, GitHubRepo]) \
-> Union[System, GitHubRepo]:
# If Redis is down, the data passed as default will be stored as
# the repo state.
self.logger.debug("Loading the state of %s from Redis", repo)
redis_hash = Keys.get_hash_parent(repo.parent_id)
repo_id = repo.repo_id
# Below, we will try and get the data stored in redis and store it
# in the repo's state. If the data from Redis cannot be obtained, the
# state won't be updated.
# Load no_of_releases from Redis
state_no_of_releases = repo.no_of_releases
redis_no_of_releases = self.redis.hget(
redis_hash, Keys.get_github_no_of_releases(repo_id),
state_no_of_releases)
no_of_releases = \
convert_to_int_if_not_none(redis_no_of_releases, None)
repo.set_no_of_releases(no_of_releases)
# Load last_monitored from Redis
state_last_monitored = repo.last_monitored
redis_last_monitored = self.redis.hget(
redis_hash, Keys.get_github_last_monitored(repo_id),
state_last_monitored)
last_monitored = \
convert_to_float_if_not_none(redis_last_monitored, None)
repo.set_last_monitored(last_monitored)
self.logger.debug(
"Restored %s state: _no_of_releases=%s, _last_monitored=%s", repo,
no_of_releases, last_monitored)
return repo
def test_process_data_saves_in_redis(self, mock_github_data, mock_send_hb,
mock_ack) -> None:
self.rabbitmq.connect()
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
data = eval(mock_github_data)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=GITHUB_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
meta_data = data['result']['meta_data']
repo_id = meta_data['repo_id']
parent_id = meta_data['repo_parent_id']
metrics = data['result']['data']
self.assertEqual(
str(metrics['no_of_releases']),
self.redis.hget(
Keys.get_hash_parent(parent_id),
Keys.get_github_no_of_releases(repo_id)).decode("utf-8"))
self.assertEqual(
str(meta_data['last_monitored']),
self.redis.hget(
Keys.get_hash_parent(parent_id),
Keys.get_github_last_monitored(repo_id)).decode("utf-8"))
except Exception as e:
self.fail("Test failed: {}".format(e))
def test_process_redis_store_calls_redis_correctly(self, mock_system_data,
mock_hset) -> None:
data = eval(mock_system_data)
self.test_store._process_redis_store(data)
metric_data = {
'severity': data['severity'],
'message': data['message']
}
key = data['origin_id']
call_1 = call(Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])),
json.dumps(metric_data))
mock_hset.assert_has_calls([call_1])
def test_process_redis_store_redis_stores_correctly(
self, mock_system_data) -> None:
data = eval(mock_system_data)
self.test_store._process_redis_store(data)
key = data['origin_id']
stored_data = self.redis.hget(
Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])))
expected_data = {
'severity': data['severity'],
'message': data['message']
}
self.assertEqual(expected_data, json.loads(stored_data))
def test_process_data_results_stores_in_redis_correctly(
self, mock_system_data, mock_send_hb, mock_process_mongo_store,
mock_ack) -> None:
mock_ack.return_value = None
try:
self.test_store._initialise_rabbitmq()
data = eval(mock_system_data)
blocking_channel = self.test_store.rabbitmq.channel
method_chains = pika.spec.Basic.Deliver(
routing_key=ALERT_STORE_INPUT_ROUTING_KEY)
properties = pika.spec.BasicProperties()
self.test_store._process_data(blocking_channel, method_chains,
properties,
json.dumps(data).encode())
mock_process_mongo_store.assert_called_once()
mock_ack.assert_called_once()
mock_send_hb.assert_called_once()
key = data['origin_id']
stored_data = self.redis.hget(
Keys.get_hash_parent(data['parent_id']),
eval('Keys.get_alert_{}(key)'.format(data['metric'])))
expected_data = {
'severity': data['severity'],
'message': data['message']
}
self.assertEqual(expected_data, json.loads(stored_data))
except Exception as e:
self.fail("Test failed: {}".format(e))
def load_state(self, system: Union[System, GitHubRepo]) \
-> Union[System, GitHubRepo]:
# If Redis is down, the data passed as default will be stored as
# the system state.
self.logger.debug("Loading the state of %s from Redis", system)
redis_hash = Keys.get_hash_parent(system.parent_id)
system_id = system.system_id
# Below, we will try and get the data stored in redis and store it
# in the system's state. If the data from Redis cannot be obtained, the
# state won't be updated.
# Load process_cpu_seconds_total from Redis
state_process_cpu_seconds_total = system.process_cpu_seconds_total
redis_process_cpu_seconds_total = self.redis.hget(
redis_hash, Keys.get_system_process_cpu_seconds_total(system_id),
state_process_cpu_seconds_total)
process_cpu_seconds_total = \
convert_to_float_if_not_none(redis_process_cpu_seconds_total, None)
system.set_process_cpu_seconds_total(process_cpu_seconds_total)
# Load process_memory_usage from Redis
state_process_memory_usage = system.process_memory_usage
redis_process_memory_usage = self.redis.hget(
redis_hash, Keys.get_system_process_memory_usage(system_id),
state_process_memory_usage)
process_memory_usage = \
convert_to_float_if_not_none(redis_process_memory_usage, None)
system.set_process_memory_usage(process_memory_usage)
# Load virtual_memory_usage from Redis
state_virtual_memory_usage = system.virtual_memory_usage
redis_virtual_memory_usage = self.redis.hget(
redis_hash, Keys.get_system_virtual_memory_usage(system_id),
state_virtual_memory_usage)
virtual_memory_usage = \
convert_to_float_if_not_none(redis_virtual_memory_usage, None)
system.set_virtual_memory_usage(virtual_memory_usage)
# Load open_file_descriptors from Redis
state_open_file_descriptors = system.open_file_descriptors
redis_open_file_descriptors = self.redis.hget(
redis_hash, Keys.get_system_open_file_descriptors(system_id),
state_open_file_descriptors)
open_file_descriptors = \
convert_to_float_if_not_none(redis_open_file_descriptors, None)
system.set_open_file_descriptors(open_file_descriptors)
# Load system_cpu_usage from Redis
state_system_cpu_usage = system.system_cpu_usage
redis_system_cpu_usage = self.redis.hget(
redis_hash, Keys.get_system_system_cpu_usage(system_id),
state_system_cpu_usage)
system_cpu_usage = \
convert_to_float_if_not_none(redis_system_cpu_usage, None)
system.set_system_cpu_usage(system_cpu_usage)
# Load system_ram_usage from Redis
state_system_ram_usage = system.system_ram_usage
redis_system_ram_usage = self.redis.hget(
redis_hash, Keys.get_system_system_ram_usage(system_id),
state_system_ram_usage)
system_ram_usage = \
convert_to_float_if_not_none(redis_system_ram_usage, None)
system.set_system_ram_usage(system_ram_usage)
# Load system_storage_usage from Redis
state_system_storage_usage = system.system_storage_usage
redis_system_storage_usage = self.redis.hget(
redis_hash, Keys.get_system_system_storage_usage(system_id),
state_system_storage_usage)
system_storage_usage = \
convert_to_float_if_not_none(redis_system_storage_usage, None)
system.set_system_storage_usage(system_storage_usage)
# Load network_transmit_bytes_per_second from Redis
state_network_transmit_bytes_per_second = \
system.network_transmit_bytes_per_second
redis_network_transmit_bytes_per_second = self.redis.hget(
redis_hash,
Keys.get_system_network_transmit_bytes_per_second(system_id),
state_network_transmit_bytes_per_second)
network_transmit_bytes_per_second = \
convert_to_float_if_not_none(
redis_network_transmit_bytes_per_second, None)
system.set_network_transmit_bytes_per_second(
network_transmit_bytes_per_second)
# Load network_receive_bytes_per_second from Redis
state_network_receive_bytes_per_second = \
system.network_receive_bytes_per_second
redis_network_receive_bytes_per_second = self.redis.hget(
redis_hash,
Keys.get_system_network_receive_bytes_per_second(system_id),
state_network_receive_bytes_per_second)
network_receive_bytes_per_second = \
convert_to_float_if_not_none(
redis_network_receive_bytes_per_second, None)
system.set_network_receive_bytes_per_second(
network_receive_bytes_per_second)
# Load network_transmit_bytes_total from Redis
state_network_transmit_bytes_total = system.network_transmit_bytes_total
redis_network_transmit_bytes_total = self.redis.hget(
redis_hash, Keys.get_system_network_transmit_bytes_total(system_id),
state_network_transmit_bytes_total)
network_transmit_bytes_total = \
convert_to_float_if_not_none(redis_network_transmit_bytes_total,
None)
system.set_network_transmit_bytes_total(network_transmit_bytes_total)
# Load network_receive_bytes_total from Redis
state_network_receive_bytes_total = system.network_receive_bytes_total
redis_network_receive_bytes_total = self.redis.hget(
redis_hash, Keys.get_system_network_receive_bytes_total(system_id),
state_network_receive_bytes_total)
network_receive_bytes_total = \
convert_to_float_if_not_none(redis_network_receive_bytes_total,
None)
system.set_network_receive_bytes_total(network_receive_bytes_total)
# Load disk_io_time_seconds_in_interval from Redis
state_disk_io_time_seconds_in_interval = \
system.disk_io_time_seconds_in_interval
redis_disk_io_time_seconds_in_interval = self.redis.hget(
redis_hash,
Keys.get_system_disk_io_time_seconds_in_interval(system_id),
state_disk_io_time_seconds_in_interval)
disk_io_time_seconds_in_interval = \
convert_to_float_if_not_none(
redis_disk_io_time_seconds_in_interval, None)
system.set_disk_io_time_seconds_in_interval(
disk_io_time_seconds_in_interval)
# Load disk_io_time_seconds_total from Redis
state_disk_io_time_seconds_total = system.disk_io_time_seconds_total
redis_disk_io_time_seconds_total = self.redis.hget(
redis_hash, Keys.get_system_disk_io_time_seconds_total(system_id),
state_disk_io_time_seconds_total)
disk_io_time_seconds_total = \
convert_to_float_if_not_none(redis_disk_io_time_seconds_total, None)
system.set_disk_io_time_seconds_total(disk_io_time_seconds_total)
# Load last_monitored from Redis
state_last_monitored = system.last_monitored
redis_last_monitored = self.redis.hget(
redis_hash, Keys.get_system_last_monitored(system_id),
state_last_monitored)
last_monitored = convert_to_float_if_not_none(redis_last_monitored,
None)
system.set_last_monitored(last_monitored)
# Load went_down_at from Redis
state_went_down_at = system.went_down_at
redis_went_down_at = self.redis.hget(
redis_hash, Keys.get_system_went_down_at(system_id),
state_went_down_at)
went_down_at = convert_to_float_if_not_none(redis_went_down_at, None)
system.set_went_down_at(went_down_at)
self.logger.debug(
"Restored %s state: _process_cpu_seconds_total=%s, "
"_process_memory_usage=%s, _virtual_memory_usage=%s, "
"_open_file_descriptors=%s, _system_cpu_usage=%s, "
"_system_ram_usage=%s, _system_storage_usage=%s, "
"_network_transmit_bytes_per_second=%s, "
"_network_receive_bytes_per_second=%s, "
"_network_transmit_bytes_total=%s, "
"_network_receive_bytes_total=%s, "
"_disk_io_time_seconds_in_interval=%s, "
"_disk_io_time_seconds_total=%s, _last_monitored=%s, "
"_went_down_at=%s", system, process_cpu_seconds_total,
process_memory_usage, virtual_memory_usage, open_file_descriptors,
system_cpu_usage, system_ram_usage, system_storage_usage,
network_transmit_bytes_per_second, network_receive_bytes_per_second,
network_transmit_bytes_total, network_receive_bytes_total,
disk_io_time_seconds_in_interval, disk_io_time_seconds_total,
last_monitored, went_down_at)
return system
def _process_redis_result_store(self, data: Dict) -> None:
meta_data = data['meta_data']
system_name = meta_data['system_name']
system_id = meta_data['system_id']
parent_id = meta_data['system_parent_id']
metrics = data['data']
self.logger.debug(
'Saving %s state: _process_cpu_seconds_total=%s, '
'_process_memory_usage=%s, _virtual_memory_usage=%s, '
'_open_file_descriptors=%s, _system_cpu_usage=%s, '
'_system_ram_usage=%s, _system_storage_usage=%s, '
'_network_transmit_bytes_per_second=%s, '
'_network_receive_bytes_per_second=%s, '
'_network_receive_bytes_total=%s, '
'_network_transmit_bytes_total=%s, '
'_disk_io_time_seconds_total=%s, '
'_disk_io_time_seconds_in_interval=%s, _went_down_at=%s, '
'_last_monitored=%s', system_name,
metrics['process_cpu_seconds_total'],
metrics['process_memory_usage'], metrics['virtual_memory_usage'],
metrics['open_file_descriptors'], metrics['system_cpu_usage'],
metrics['system_ram_usage'], metrics['system_storage_usage'],
metrics['network_transmit_bytes_per_second'],
metrics['network_receive_bytes_per_second'],
metrics['network_receive_bytes_total'],
metrics['network_transmit_bytes_total'],
metrics['disk_io_time_seconds_total'],
metrics['disk_io_time_seconds_in_interval'],
metrics['went_down_at'], meta_data['last_monitored'])
self.redis.hset_multiple(
Keys.get_hash_parent(parent_id), {
Keys.get_system_process_cpu_seconds_total(system_id):
str(metrics['process_cpu_seconds_total']),
Keys.get_system_process_memory_usage(system_id):
str(metrics['process_memory_usage']),
Keys.get_system_virtual_memory_usage(system_id):
str(metrics['virtual_memory_usage']),
Keys.get_system_open_file_descriptors(system_id):
str(metrics['open_file_descriptors']),
Keys.get_system_system_cpu_usage(system_id):
str(metrics['system_cpu_usage']),
Keys.get_system_system_ram_usage(system_id):
str(metrics['system_ram_usage']),
Keys.get_system_system_storage_usage(system_id):
str(metrics['system_storage_usage']),
Keys.get_system_network_transmit_bytes_per_second(system_id):
str(metrics['network_transmit_bytes_per_second']),
Keys.get_system_network_receive_bytes_per_second(system_id):
str(metrics['network_receive_bytes_per_second']),
Keys.get_system_network_receive_bytes_total(system_id):
str(metrics['network_receive_bytes_total']),
Keys.get_system_network_transmit_bytes_total(system_id):
str(metrics['network_transmit_bytes_total']),
Keys.get_system_disk_io_time_seconds_total(system_id):
str(metrics['disk_io_time_seconds_total']),
Keys.get_system_disk_io_time_seconds_in_interval(system_id):
str(metrics['disk_io_time_seconds_in_interval']),
Keys.get_system_went_down_at(system_id):
str(metrics['went_down_at']),
Keys.get_system_last_monitored(system_id):
str(meta_data['last_monitored']),
})