def init_with_database(self):
"""Initializes the job execution metrics with the execution history from the database
"""
oldest_time = self._finished_metrics_over_time.time_blocks[0].start
# TODO: this should be in the manager, but the JobExecution model is going to be completely re-worked anyway
job_exe_query = JobExecution.objects.select_related('error')
job_exe_query = job_exe_query.filter(
status__in=['COMPLETED', 'FAILED'], ended__gte=oldest_time)
for job_exe_model in job_exe_query:
job_exe = RunningJobExecution(job_exe_model)
job_exe._set_finished_status(job_exe_model.status,
job_exe_model.ended,
job_exe_model.error)
self._finished_metrics.add_job_execution(job_exe)
self._finished_metrics_over_time.add_job_execution(job_exe)
def test_running_executions(self):
"""Tests the metrics with running executions that complete"""
node_model_1 = node_test_utils.create_node()
node_model_2 = node_test_utils.create_node()
job_type_1 = job_test_utils.create_job_type()
job_type_2 = job_test_utils.create_job_type()
job_exe_model_1 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_1)
job_exe_model_2 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_1)
job_exe_model_3 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_1)
job_exe_model_4 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_1)
job_exe_model_5 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_2)
job_exe_model_6 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_2)
job_exe_model_7 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2)
job_exe_model_8 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2)
job_exe_model_9 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2)
job_exe_model_10 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2)
job_exe_model_11 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2)
job_exe_1 = RunningJobExecution(job_exe_model_1)
job_exe_2 = RunningJobExecution(job_exe_model_2)
job_exe_3 = RunningJobExecution(job_exe_model_3)
job_exe_4 = RunningJobExecution(job_exe_model_4)
job_exe_5 = RunningJobExecution(job_exe_model_5)
job_exe_6 = RunningJobExecution(job_exe_model_6)
job_exe_7 = RunningJobExecution(job_exe_model_7)
job_exe_8 = RunningJobExecution(job_exe_model_8)
job_exe_9 = RunningJobExecution(job_exe_model_9)
job_exe_10 = RunningJobExecution(job_exe_model_10)
job_exe_11 = RunningJobExecution(job_exe_model_11)
# NOTE: This unit test is about to get CRAZY. I apologize for the complexity, but this is needed for a
# thorough testing
self.metrics.add_running_job_exes([job_exe_1, job_exe_2, job_exe_3, job_exe_4, job_exe_5, job_exe_6, job_exe_7,
job_exe_8, job_exe_9, job_exe_10, job_exe_11])
node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}]
self.metrics.generate_status_json(node_list_dict, now())
# Check expected totals
self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 4)
for job_type_dict in node_list_dict[0]['job_executions']['running']['by_job_type']:
if job_type_dict['job_type_id'] == job_type_1.id:
self.assertEqual(job_type_dict['count'], 3)
elif job_type_dict['job_type_id'] == job_type_2.id:
self.assertEqual(job_type_dict['count'], 1)
else:
self.fail('Unexpected job type ID')
self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 7)
for job_type_dict in node_list_dict[1]['job_executions']['running']['by_job_type']:
if job_type_dict['job_type_id'] == job_type_1.id:
self.assertEqual(job_type_dict['count'], 2)
elif job_type_dict['job_type_id'] == job_type_2.id:
self.assertEqual(job_type_dict['count'], 5)
else:
self.fail('Unexpected job type ID')
self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0)
# Finish some job executions
end_time_1 = now()
job_exe_1._set_finished_status('COMPLETED', end_time_1)
job_exe_2._set_finished_status('FAILED', end_time_1, error=self.data_error)
job_exe_4._set_finished_status('FAILED', end_time_1, error=self.alg_error)
self.metrics.job_exe_finished(job_exe_1)
self.metrics.job_exe_finished(job_exe_2)
self.metrics.job_exe_finished(job_exe_4)
node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}]
self.metrics.generate_status_json(node_list_dict, end_time_1 + datetime.timedelta(seconds=1))
# Check expected totals
self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['running']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['job_type_id'], job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['completed']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['job_type_id'],
job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 2)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['job_type_id'],
job_type_2.id)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['data']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['job_type_id'],
job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 7)
for job_type_dict in node_list_dict[1]['job_executions']['running']['by_job_type']:
if job_type_dict['job_type_id'] == job_type_1.id:
self.assertEqual(job_type_dict['count'], 2)
elif job_type_dict['job_type_id'] == job_type_2.id:
self.assertEqual(job_type_dict['count'], 5)
else:
self.fail('Unexpected job type ID')
self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0)
# Finish some job executions (all executions still on node 2)
end_time_2 = end_time_1 + FinishedJobExeMetricsOverTime.BLOCK_LENGTH
job_exe_5._set_finished_status('COMPLETED', end_time_2)
job_exe_6._set_finished_status('COMPLETED', end_time_2)
job_exe_7._set_finished_status('COMPLETED', end_time_2)
job_exe_8._set_finished_status('COMPLETED', end_time_2)
job_exe_9._set_finished_status('COMPLETED', end_time_2)
job_exe_10._set_finished_status('COMPLETED', end_time_2)
job_exe_11._set_finished_status('COMPLETED', end_time_2)
self.metrics.job_exe_finished(job_exe_5)
self.metrics.job_exe_finished(job_exe_6)
self.metrics.job_exe_finished(job_exe_7)
self.metrics.job_exe_finished(job_exe_8)
self.metrics.job_exe_finished(job_exe_9)
self.metrics.job_exe_finished(job_exe_10)
self.metrics.job_exe_finished(job_exe_11)
node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}]
self.metrics.generate_status_json(node_list_dict, end_time_2)
# Check expected totals
self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['running']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['job_type_id'], job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['completed']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['job_type_id'],
job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 2)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['job_type_id'],
job_type_2.id)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['data']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['job_type_id'],
job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 7)
for job_type_dict in node_list_dict[1]['job_executions']['completed']['by_job_type']:
if job_type_dict['job_type_id'] == job_type_1.id:
self.assertEqual(job_type_dict['count'], 2)
elif job_type_dict['job_type_id'] == job_type_2.id:
self.assertEqual(job_type_dict['count'], 5)
else:
self.fail('Unexpected job type ID')
self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0)
# Let all finished job executions roll off by time, only running remaining
end_time_3 = end_time_2 + FinishedJobExeMetricsOverTime.TOTAL_TIME_PERIOD
end_time_3 += FinishedJobExeMetricsOverTime.BLOCK_LENGTH + datetime.timedelta(seconds=1)
node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}]
self.metrics.generate_status_json(node_list_dict, end_time_3)
# Check expected totals
self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 1)
self.assertEqual(len(node_list_dict[0]['job_executions']['running']['by_job_type']), 1)
self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['count'], 1)
self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['job_type_id'], job_type_1.id)
self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 0)
self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0)
self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0)