def init_with_database(self): """Initializes the job execution metrics with the execution history from the database """ oldest_time = self._finished_metrics_over_time.time_blocks[0].start # TODO: this should be in the manager, but the JobExecution model is going to be completely re-worked anyway job_exe_query = JobExecution.objects.select_related('error') job_exe_query = job_exe_query.filter( status__in=['COMPLETED', 'FAILED'], ended__gte=oldest_time) for job_exe_model in job_exe_query: job_exe = RunningJobExecution(job_exe_model) job_exe._set_finished_status(job_exe_model.status, job_exe_model.ended, job_exe_model.error) self._finished_metrics.add_job_execution(job_exe) self._finished_metrics_over_time.add_job_execution(job_exe)
def test_running_executions(self): """Tests the metrics with running executions that complete""" node_model_1 = node_test_utils.create_node() node_model_2 = node_test_utils.create_node() job_type_1 = job_test_utils.create_job_type() job_type_2 = job_test_utils.create_job_type() job_exe_model_1 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_1) job_exe_model_2 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_1) job_exe_model_3 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_1) job_exe_model_4 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_1) job_exe_model_5 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_2) job_exe_model_6 = job_test_utils.create_job_exe(job_type=job_type_1, status='RUNNING', node=node_model_2) job_exe_model_7 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2) job_exe_model_8 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2) job_exe_model_9 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2) job_exe_model_10 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2) job_exe_model_11 = job_test_utils.create_job_exe(job_type=job_type_2, status='RUNNING', node=node_model_2) job_exe_1 = RunningJobExecution(job_exe_model_1) job_exe_2 = RunningJobExecution(job_exe_model_2) job_exe_3 = RunningJobExecution(job_exe_model_3) job_exe_4 = RunningJobExecution(job_exe_model_4) job_exe_5 = RunningJobExecution(job_exe_model_5) job_exe_6 = RunningJobExecution(job_exe_model_6) job_exe_7 = RunningJobExecution(job_exe_model_7) job_exe_8 = RunningJobExecution(job_exe_model_8) job_exe_9 = RunningJobExecution(job_exe_model_9) job_exe_10 = RunningJobExecution(job_exe_model_10) job_exe_11 = RunningJobExecution(job_exe_model_11) # NOTE: This unit test is about to get CRAZY. I apologize for the complexity, but this is needed for a # thorough testing self.metrics.add_running_job_exes([job_exe_1, job_exe_2, job_exe_3, job_exe_4, job_exe_5, job_exe_6, job_exe_7, job_exe_8, job_exe_9, job_exe_10, job_exe_11]) node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}] self.metrics.generate_status_json(node_list_dict, now()) # Check expected totals self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 4) for job_type_dict in node_list_dict[0]['job_executions']['running']['by_job_type']: if job_type_dict['job_type_id'] == job_type_1.id: self.assertEqual(job_type_dict['count'], 3) elif job_type_dict['job_type_id'] == job_type_2.id: self.assertEqual(job_type_dict['count'], 1) else: self.fail('Unexpected job type ID') self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 7) for job_type_dict in node_list_dict[1]['job_executions']['running']['by_job_type']: if job_type_dict['job_type_id'] == job_type_1.id: self.assertEqual(job_type_dict['count'], 2) elif job_type_dict['job_type_id'] == job_type_2.id: self.assertEqual(job_type_dict['count'], 5) else: self.fail('Unexpected job type ID') self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0) # Finish some job executions end_time_1 = now() job_exe_1._set_finished_status('COMPLETED', end_time_1) job_exe_2._set_finished_status('FAILED', end_time_1, error=self.data_error) job_exe_4._set_finished_status('FAILED', end_time_1, error=self.alg_error) self.metrics.job_exe_finished(job_exe_1) self.metrics.job_exe_finished(job_exe_2) self.metrics.job_exe_finished(job_exe_4) node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}] self.metrics.generate_status_json(node_list_dict, end_time_1 + datetime.timedelta(seconds=1)) # Check expected totals self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['running']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['completed']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 2) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['job_type_id'], job_type_2.id) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['data']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 7) for job_type_dict in node_list_dict[1]['job_executions']['running']['by_job_type']: if job_type_dict['job_type_id'] == job_type_1.id: self.assertEqual(job_type_dict['count'], 2) elif job_type_dict['job_type_id'] == job_type_2.id: self.assertEqual(job_type_dict['count'], 5) else: self.fail('Unexpected job type ID') self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0) # Finish some job executions (all executions still on node 2) end_time_2 = end_time_1 + FinishedJobExeMetricsOverTime.BLOCK_LENGTH job_exe_5._set_finished_status('COMPLETED', end_time_2) job_exe_6._set_finished_status('COMPLETED', end_time_2) job_exe_7._set_finished_status('COMPLETED', end_time_2) job_exe_8._set_finished_status('COMPLETED', end_time_2) job_exe_9._set_finished_status('COMPLETED', end_time_2) job_exe_10._set_finished_status('COMPLETED', end_time_2) job_exe_11._set_finished_status('COMPLETED', end_time_2) self.metrics.job_exe_finished(job_exe_5) self.metrics.job_exe_finished(job_exe_6) self.metrics.job_exe_finished(job_exe_7) self.metrics.job_exe_finished(job_exe_8) self.metrics.job_exe_finished(job_exe_9) self.metrics.job_exe_finished(job_exe_10) self.metrics.job_exe_finished(job_exe_11) node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}] self.metrics.generate_status_json(node_list_dict, end_time_2) # Check expected totals self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['running']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['completed']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['completed']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 2) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['by_job_type'][0]['job_type_id'], job_type_2.id) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['failed']['data']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 7) for job_type_dict in node_list_dict[1]['job_executions']['completed']['by_job_type']: if job_type_dict['job_type_id'] == job_type_1.id: self.assertEqual(job_type_dict['count'], 2) elif job_type_dict['job_type_id'] == job_type_2.id: self.assertEqual(job_type_dict['count'], 5) else: self.fail('Unexpected job type ID') self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0) # Let all finished job executions roll off by time, only running remaining end_time_3 = end_time_2 + FinishedJobExeMetricsOverTime.TOTAL_TIME_PERIOD end_time_3 += FinishedJobExeMetricsOverTime.BLOCK_LENGTH + datetime.timedelta(seconds=1) node_list_dict = [{'id': node_model_1.id}, {'id': node_model_2.id}] self.metrics.generate_status_json(node_list_dict, end_time_3) # Check expected totals self.assertEqual(node_list_dict[0]['job_executions']['running']['total'], 1) self.assertEqual(len(node_list_dict[0]['job_executions']['running']['by_job_type']), 1) self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['count'], 1) self.assertEqual(node_list_dict[0]['job_executions']['running']['by_job_type'][0]['job_type_id'], job_type_1.id) self.assertEqual(node_list_dict[0]['job_executions']['completed']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['algorithm']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['data']['total'], 0) self.assertEqual(node_list_dict[0]['job_executions']['failed']['system']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['running']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['completed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['algorithm']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['data']['total'], 0) self.assertEqual(node_list_dict[1]['job_executions']['failed']['system']['total'], 0)