def test_metrics_send(self):
model_id = 'demo'
build_number = 10
metric = metrics.Metric.TEST_ACCURACY
value = 30.0
host, port, namespace = metrics.get_metric_endpoint()
with patch_environ({env.BUILD_NUMBER[0]: build_number}):
with patch('legion.metrics.send_tcp') as send_tcp_mock:
timestamp = int(time.time())
metrics.send_metric(model_id, metric, value)
self.assertTrue(len(send_tcp_mock.call_args_list) == 2, '2 calls founded')
for call in send_tcp_mock.call_args_list:
self.assertEqual(call[0][0], host)
self.assertEqual(call[0][1], port)
delimiter = ' '
call_with_metric = send_tcp_mock.call_args_list[0][0][2].strip().split(delimiter)
call_with_build_number = send_tcp_mock.call_args_list[1][0][2].strip().split(delimiter)
self.assertEqual(call_with_metric[0], '{}.{}.metrics.{}'.format(namespace, model_id, metric.value))
self.assertEqual(float(call_with_metric[1]), value)
self.assertEqual(call_with_metric[2], str(timestamp))
self.assertEqual(call_with_build_number[0], '{}.{}.metrics.build'.format(namespace, model_id))
self.assertEqual(int(float(call_with_build_number[1])), build_number)
self.assertEqual(call_with_build_number[2], str(timestamp))
def test_model_id_deduction_exception(self):
self.assertEqual(legion.model.model_id._model_id, None,
'Model ID not empty')
self.assertEqual(os.getenv(*env.MODEL_ID), None,
'Model ID ENV not empty')
with self.assertRaises(Exception) as context:
metrics.send_metric(metrics.Metric.TEST_ACCURACY, 30.0)
def test_metrics_send(self):
model_id = 'demo'
build_number = 10
metric = metrics.Metric.TEST_ACCURACY
value = 30.0
host, port, namespace = metrics.get_metric_endpoint()
os.environ[env.MODEL_ID[0]] = str(model_id)
with patch(
'legion.model.model_id.send_model_id') as send_model_id_mock:
with MetricContent(model_id, build_number, init_at_startup=False):
self.assertEqual(len(send_model_id_mock.call_args_list), 0)
with patch('legion.metrics.send_tcp') as send_tcp_mock:
timestamp = int(time.time())
metrics.send_metric(metric, value)
self.assertEqual(len(send_model_id_mock.call_args_list), 1)
del os.environ[env.MODEL_ID[0]]
self.assertTrue(
len(send_tcp_mock.call_args_list) == 2,
'2 calls founded')
for call in send_tcp_mock.call_args_list:
self.assertEqual(call[0][0], host)
self.assertEqual(call[0][1], port)
delimiter = ' '
call_with_metric = send_tcp_mock.call_args_list[0][0][
2].strip().split(delimiter)
call_with_build_number = send_tcp_mock.call_args_list[1][
0][2].strip().split(delimiter)
self.assertEqual(
call_with_metric[0], '%s.%s.metrics.%s' %
(namespace, model_id, metric.value))
self.assertEqual(float(call_with_metric[1]), value)
self.assertEqual(call_with_metric[2], str(timestamp))
self.assertEqual(
call_with_build_number[0],
'%s.%s.metrics.build' % (namespace, model_id))
self.assertEqual(int(float(call_with_build_number[1])),
build_number)
self.assertEqual(call_with_build_number[2], str(timestamp))
def __exit__(self, type, value, traceback):
duration = time.time() - self._start_time
print("Name: " + self._name +
" Elapsed time: {:.3f} sec".format(duration))
send_metric(self._name, duration)