def setUp(self):
self.config_commandline = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
"commandline": ".foo.*",
}
self.monitor = ProcessMonitor(self.config_commandline, scalyr_logging.getLogger("syslog_monitor[test]"))
def test_gather_sample_by_pid_failure_pid_doesnt_exist(self):
monitor_config = {
"module": "linux_process_metrics",
"id": "my-id",
"pid": 65555,
}
mock_logger = mock.Mock()
monitor = ProcessMonitor(monitor_config, mock_logger)
monitor.gather_sample()
self.assertEqual(mock_logger.error.call_count, 3)
self.assertEqual(mock_logger.emit_value.call_count, 0)
def test_initialize_monitor(self):
monitor = ProcessMonitor(
self.config_commandline,
scalyr_logging.getLogger("syslog_monitor[test]"))
self.assertEqual(monitor._ProcessMonitor__metrics_history,
defaultdict(dict))
self.assertEqual(monitor._ProcessMonitor__aggregated_metrics, {})
def setUp(self):
self.config_commandline = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
"commandline": ".foo.*",
}
self.monitor = ProcessMonitor(self.config_commandline, scalyr_logging.getLogger("syslog_monitor[test]"))
def test_gather_sample_by_commandline_success(self):
monitor_config = {
"module": "linux_process_metrics",
"id": "my-id",
"commandline": ".*%s.*" % (" ".join(sys.argv)),
}
mock_logger = mock.Mock()
monitor = ProcessMonitor(monitor_config, mock_logger)
monitor_module = "scalyr_agent.builtin_monitors.linux_process_metrics"
monitor_info = load_monitor_class(monitor_module, [])[1]
monitor.gather_sample()
self.assertEqual(mock_logger.error.call_count, 0)
self.assertEqual(mock_logger.emit_value.call_count, len(monitor_info.metrics))
def test_late_process_pid_setting(self):
self.config = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
# the process ID is not known yet.
"pid": "$$TBD",
}
self.monitor = ProcessMonitor(
self.config, scalyr_logging.getLogger("syslog_monitor[test]"))
# it can not select any process until the process id is set.
self.assertEqual(self.monitor._select_processes(), [])
self.assertEqual(self.monitor._select_processes(), [])
# set the process id.
self.monitor.set_pid(os.getpid())
# it has to select the current PID after it has been set.
self.assertEqual(self.monitor._select_processes(), [os.getpid()])
class TestProcessMonitorRunningTotal(ScalyrTestCase):
"""
Tests the calculations of the running totals of the metrics.
"""
def setUp(self):
super(TestProcessMonitorRunningTotal, self).setUp()
self.config_commandline = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
"commandline": ".foo.*",
}
self.monitor = ProcessMonitor(
self.config_commandline,
scalyr_logging.getLogger("syslog_monitor[test]"))
def test_no_history(self):
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_single_process_single_epoch(self):
metric = Metric("fakemetric", "faketype")
metrics = {metric: 21}
self.monitor.record_metrics(555, metrics)
self.monitor._ProcessMonitor__pids = [555]
self.monitor._calculate_aggregated_metrics()
expected_history = {555: {metric: [21]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics,
{metric: 21})
def test_single_process_multiple_epoch(self):
metric = Metric("fakemetric", "faketype")
# epoch 1
metrics = {metric: 21}
self.monitor.record_metrics(555, metrics)
self.monitor._ProcessMonitor__pids = [555]
self.monitor._calculate_aggregated_metrics()
expected_history = {555: {metric: [21]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics,
{metric: 21})
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics = {metric: 21.5}
self.monitor.record_metrics(555, metrics)
self.monitor._ProcessMonitor__pids = [555]
self.monitor._calculate_aggregated_metrics()
expected_history = {555: {metric: [21, 21.5]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics,
{metric: 21.5})
def test_multiple_process_multiple_epochs(self):
metric1 = Metric("fakemetric1", "faketype1")
metric2 = Metric("fakemetric2", "faketype2")
# epoch 1
metrics1 = {metric1: 21}
metrics2 = {metric2: 100.0}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {1: {metric1: [21]}, 2: {metric2: [100.0]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 21,
metric2: 100.0
},
)
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 21.11}
metrics2 = {metric2: 100.11}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {
metric1: [21, 21.11]
},
2: {
metric2: [100.0, 100.11]
}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 21.11,
metric2: 100.11
},
)
def test_multiple_process_multiple_epochs_cumulative_metrics(self):
metric1 = Metric("app.cpu", "system")
# epoch 1
metrics1 = {metric1: 20}
metrics2 = {metric1: 40}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {1: {metric1: [20]}, 2: {metric1: [40]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: 0})
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 22}
metrics2 = {metric1: 44}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {1: {metric1: [20, 22]}, 2: {metric1: [40, 44]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: (22 - 20) + (44 - 40)},
)
# epoch 3
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 25}
metrics2 = {metric1: 48}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
# we only keep the last 2 historical values
expected_history = {1: {metric1: [22, 25]}, 2: {metric1: [44, 48]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: (22 - 20) + (44 - 40) + (25 - 22) + (48 - 44)},
)
def test_multiple_process_multiple_epochs_cumulative_metrics_one_process_death(
self, ):
"""
Same as test_multiple_process_multiple_epochs_cumulative_metrics
but one process dies after epoch 2
"""
metric1 = Metric("app.cpu", "system")
# epoch 1
metrics1 = {metric1: 21}
metrics2 = {metric1: 100.0}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {1: {metric1: [21]}, 2: {metric1: [100.0]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: 0})
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 30.1}
metrics2 = {metric1: 100.2}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {
metric1: [21, 30.1]
},
2: {
metric1: [100.0, 100.2]
}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: (30.1 - 21) + (100.2 - 100.0)},
)
# epoch 3
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 26.0}
metrics2 = {metric1: 103}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
# Process 1 dies.. boom
self.monitor._ProcessMonitor__pids = [2]
self.monitor._calculate_aggregated_metrics()
# we only keep the last 2 historical values
expected_history = {
1: {
metric1: [30.1, 26.0]
},
2: {
metric1: [100.2, 103]
}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: (30.1 - 21) + (100.2 - 100.0) + (103 - 100.2)},
)
def test_multiple_process_multiple_epochs_cumulative_metrics_all_process_death(
self, ):
"""
Same as test_multiple_process_multiple_epochs_cumulative_metrics_one_process_death
but all processes die after epoch 2
"""
metric1 = Metric("app.cpu", "system")
# epoch 1
metrics1 = {metric1: 20}
metrics2 = {metric1: 40}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {1: {metric1: [20]}, 2: {metric1: [40]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: 0})
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 25}
metrics2 = {metric1: 46}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {1: {metric1: [20, 25]}, 2: {metric1: [40, 46]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: (25 - 20) + (46 - 40)},
)
# epoch 3
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 23}
metrics2 = {metric1: 43}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
# Process 1 and 2 die.. boom
# we should ensure the total running value for metric doesn't go down.
self.monitor._ProcessMonitor__pids = []
self.monitor._calculate_aggregated_metrics()
# we only keep the last 2 historical values
expected_history = {1: {metric1: [25, 23]}, 2: {metric1: [46, 43]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{metric1: (25 - 20) + (46 - 40)},
)
class TestProcessMonitorRecordMetrics(ScalyrTestCase):
"""
Tests the record_metrics method of ProcessMonitor class
"""
def setUp(self):
super(TestProcessMonitorRecordMetrics, self).setUp()
self.config_commandline = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
"commandline": ".foo.*",
}
self.monitor = ProcessMonitor(
self.config_commandline,
scalyr_logging.getLogger("syslog_monitor[test]"))
def test_empty_metrics(self):
self.monitor.record_metrics(666, {})
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
defaultdict(dict))
def test_single_process_single_epoch(self):
metric = Metric("fakemetric", "faketype")
metrics = {metric: 21}
self.monitor.record_metrics(555, metrics)
expected_history = {555: {metric: [21]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_single_process_multiple_epochs(self):
metric = Metric("fakemetric", "faketype")
# epoch 1
self.monitor.record_metrics(777, {metric: 1.2})
expected_history = {777: {metric: [1.2]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
# epoch 2
self.monitor.record_metrics(777, {metric: 1.9})
expected_history = {777: {metric: [1.2, 1.9]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_multi_process_single_epoch(self):
metric1 = Metric("fakemetric1", "faketype1")
metric2 = Metric("fakemetric2", "faketype2")
self.monitor.record_metrics(111, {metric1: 1.2})
self.monitor.record_metrics(222, {metric2: 2.87})
expected_history = {111: {metric1: [1.2]}, 222: {metric2: [2.87]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_multi_process_multi_epochs(self):
metric1 = Metric("fakemetric1", "faketype1")
metric2 = Metric("fakemetric2", "faketype2")
# epoch 1
self.monitor.record_metrics(111, {metric1: 1.2})
self.monitor.record_metrics(222, {metric2: 2.87})
expected_history = {111: {metric1: [1.2]}, 222: {metric2: [2.87]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
# epoch 2
self.monitor.record_metrics(111, {metric1: 1.6})
self.monitor.record_metrics(222, {metric2: 2.92})
expected_history = {
111: {
metric1: [1.2, 1.6]
},
222: {
metric2: [2.87, 2.92]
}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history,
expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
class TestProcessMonitorRecordMetrics(ScalyrTestCase):
"""
Tests the record_metrics method of ProcessMonitor class
"""
def setUp(self):
self.config_commandline = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
"commandline": ".foo.*",
}
self.monitor = ProcessMonitor(self.config_commandline, scalyr_logging.getLogger("syslog_monitor[test]"))
def test_empty_metrics(self):
self.monitor.record_metrics(666, {})
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, defaultdict(dict))
def test_single_process_single_epoch(self):
metric = Metric('fakemetric', 'faketype')
metrics = {
metric: 21
}
self.monitor.record_metrics(555, metrics)
expected_history = {
555: {metric: [21]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_single_process_multiple_epochs(self):
metric = Metric('fakemetric', 'faketype')
# epoch 1
self.monitor.record_metrics(777, {metric: 1.2})
expected_history = {
777: {metric: [1.2]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
# epoch 2
self.monitor.record_metrics(777, {metric: 1.9})
expected_history = {
777: {metric: [1.2, 1.9]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_multi_process_single_epoch(self):
metric1 = Metric('fakemetric1', 'faketype1')
metric2 = Metric('fakemetric2', 'faketype2')
self.monitor.record_metrics(111, {metric1: 1.2})
self.monitor.record_metrics(222, {metric2: 2.87})
expected_history = {
111: {metric1: [1.2]},
222: {metric2: [2.87]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_multi_process_multi_epochs(self):
metric1 = Metric('fakemetric1', 'faketype1')
metric2 = Metric('fakemetric2', 'faketype2')
# epoch 1
self.monitor.record_metrics(111, {metric1: 1.2})
self.monitor.record_metrics(222, {metric2: 2.87})
expected_history = {
111: {metric1: [1.2]},
222: {metric2: [2.87]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
# epoch 2
self.monitor.record_metrics(111, {metric1: 1.6})
self.monitor.record_metrics(222, {metric2: 2.92})
expected_history = {
111: {metric1: [1.2, 1.6]},
222: {metric2: [2.87, 2.92]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
class TestProcessMonitorRunningTotal(ScalyrTestCase):
"""
Tests the calculations of the running totals of the metrics.
"""
def setUp(self):
self.config_commandline = {
"module": "scalyr_agent.builtin_monitors.linux_process_metrics",
"id": "myapp",
"commandline": ".foo.*",
}
self.monitor = ProcessMonitor(self.config_commandline, scalyr_logging.getLogger("syslog_monitor[test]"))
def test_no_history(self):
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {})
def test_single_process_single_epoch(self):
metric = Metric('fakemetric', 'faketype')
metrics = {
metric: 21
}
self.monitor.record_metrics(555, metrics)
self.monitor._ProcessMonitor__pids = [555]
self.monitor._calculate_aggregated_metrics()
expected_history = {
555: {metric: [21]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {metric: 21})
def test_single_process_multiple_epoch(self):
metric = Metric('fakemetric', 'faketype')
# epoch 1
metrics = {metric: 21}
self.monitor.record_metrics(555, metrics)
self.monitor._ProcessMonitor__pids = [555]
self.monitor._calculate_aggregated_metrics()
expected_history = {555: {metric: [21]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {metric: 21})
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics = {metric: 21.5}
self.monitor.record_metrics(555, metrics)
self.monitor._ProcessMonitor__pids = [555]
self.monitor._calculate_aggregated_metrics()
expected_history = {555: {metric: [21, 21.5]}}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(self.monitor._ProcessMonitor__aggregated_metrics, {metric: 21.5})
def test_multiple_process_multiple_epochs(self):
metric1 = Metric('fakemetric1', 'faketype1')
metric2 = Metric('fakemetric2', 'faketype2')
# epoch 1
metrics1 = {metric1: 21}
metrics2 = {metric2: 100.0}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [21]},
2: {metric2: [100.0]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 21,
metric2: 100.0
}
)
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 21.11}
metrics2 = {metric2: 100.11}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [21, 21.11]},
2: {metric2: [100.0, 100.11]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 21.11,
metric2: 100.11
}
)
def test_multiple_process_multiple_epochs_cumulative_metrics(self):
metric1 = Metric('app.cpu', 'system')
# epoch 1
metrics1 = {metric1: 20}
metrics2 = {metric1: 40}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [20]},
2: {metric1: [40]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 0
}
)
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 22}
metrics2 = {metric1: 44}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [20, 22]},
2: {metric1: [40, 44]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: (22 - 20) + (44 - 40)
}
)
# epoch 3
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 25}
metrics2 = {metric1: 48}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
# we only keep the last 2 historical values
expected_history = {
1: {metric1: [22, 25]},
2: {metric1: [44, 48]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: (22 - 20) + (44 - 40) + (25 - 22) + (48 - 44)
}
)
def test_multiple_process_multiple_epochs_cumulative_metrics_one_process_death(self):
"""
Same as test_multiple_process_multiple_epochs_cumulative_metrics
but one process dies after epoch 2
"""
metric1 = Metric('app.cpu', 'system')
# epoch 1
metrics1 = {metric1: 21}
metrics2 = {metric1: 100.0}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [21]},
2: {metric1: [100.0]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 0
}
)
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 30.1}
metrics2 = {metric1: 100.2}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [21, 30.1]},
2: {metric1: [100.0, 100.2]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: (30.1 - 21) + (100.2 - 100.0)
}
)
# epoch 3
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 26.0}
metrics2 = {metric1: 103}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
# Process 1 dies.. boom
self.monitor._ProcessMonitor__pids = [2]
self.monitor._calculate_aggregated_metrics()
# we only keep the last 2 historical values
expected_history = {
1: {metric1: [30.1, 26.0]},
2: {metric1: [100.2, 103]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: (30.1 - 21) + (100.2 - 100.0) + (103 - 100.2)
}
)
def test_multiple_process_multiple_epochs_cumulative_metrics_all_process_death(self):
"""
Same as test_multiple_process_multiple_epochs_cumulative_metrics_one_process_death
but all processes die after epoch 2
"""
metric1 = Metric('app.cpu', 'system')
# epoch 1
metrics1 = {metric1: 20}
metrics2 = {metric1: 40}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1,2 ]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [20]},
2: {metric1: [40]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: 0
}
)
# epoch 2
# before epoch 2, the reset is called for absolute metrics
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 25}
metrics2 = {metric1: 46}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
self.monitor._ProcessMonitor__pids = [1, 2]
self.monitor._calculate_aggregated_metrics()
expected_history = {
1: {metric1: [20, 25]},
2: {metric1: [40, 46]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: (25 - 20) + (46 - 40)
}
)
# epoch 3
self.monitor._reset_absolute_metrics()
metrics1 = {metric1: 23}
metrics2 = {metric1: 43}
self.monitor.record_metrics(1, metrics1)
self.monitor.record_metrics(2, metrics2)
# Process 1 and 2 die.. boom
# we should ensure the total running value for metric doesn't go down.
self.monitor._ProcessMonitor__pids = []
self.monitor._calculate_aggregated_metrics()
# we only keep the last 2 historical values
expected_history = {
1: {metric1: [25, 23]},
2: {metric1: [46, 43]}
}
self.assertEqual(self.monitor._ProcessMonitor__metrics_history, expected_history)
self.assertEqual(
self.monitor._ProcessMonitor__aggregated_metrics,
{
metric1: (25 - 20) + (46 - 40)
}
)
