def test_most_recent_end_time_job_failed(self): job_runs = { 'status': 'scheduled', 'next_run': None, 'runs': [ { 'id': 'MASTER.test.3', 'state': 'scheduled', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 600), ), 'end_time': None }, { 'id': 'MASTER.test.2', 'state': 'succeeded', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1800), ), 'end_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1700), ), }, { 'id': 'MASTER.test.1', 'state': 'failed', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), 'end_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 500), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.1') assert_equal(state, State.FAILED)
def test_job_has_no_runs_at_all(self): job_runs = { 'status': 'running', 'next_run': None, 'runs': [], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run, None) assert_equal(state, State.NO_RUN_YET)
def test_job_unknown(self): job_runs = { 'status': 'running', 'next_run': None, 'runs': [ { 'id': 'MASTER.test.3', 'state': 'scheduled', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), 'end_time': None, }, { 'id': 'MASTER.test.2', 'state': 'unknown', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1200), ), 'end_time': None, }, { 'id': 'MASTER.test.1', 'state': 'succeeded', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1800), ), 'end_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.2') assert_equal(state, State.UNKNOWN)
def test_job_running_job_exceeds_expected_runtime(self): job_runs = { 'status': 'running', 'next_run': None, 'expected_runtime': 480.0, 'runs': [ { 'id': 'MASTER.test.100', 'state': 'scheduled', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 600), ), 'end_time': None, 'start_time': None, 'duration': '', }, { 'id': 'MASTER.test.99', 'state': 'running', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), 'start_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), 'end_time': None, 'duration': '0:10:01.883601', }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.99') assert_equal(state, State.STUCK)
def test_no_job_scheduled_or_queuing(self): job_runs = { 'status': 'succeeded', 'next_run': None, 'runs': [ { 'id': 'MASTER.test.2', 'state': 'succeeded', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 300), ), 'end_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), }, { 'id': 'MASTER.test.1', 'state': 'succeeded', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 900), ), 'end_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1200), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.2') assert_equal(state, State.NOT_SCHEDULED)
def test_job_stuck_when_runtime_not_sorted(self): job_runs = { 'status': 'running', 'next_run': None, 'runs': [ { 'id': 'MASTER.test.2', 'state': 'running', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), 'end_time': None, }, { 'id': 'MASTER.test.1', 'state': 'scheduled', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time()), ), 'end_time': None, }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.2') assert_equal(state, State.STUCK)
def test_job_waiting_for_first_run(self): job_runs = { 'status': 'scheduled', 'next_run': None, 'runs': [{ 'id': 'MASTER.test.1', 'state': 'scheduled', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 1200), ), 'end_time': None, }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.1') assert_equal(state, State.WAITING_FOR_FIRST_RUN)
def test_job_running_action_exceeds_expected_runtime(self): job_runs = { 'status': 'running', 'next_run': None, 'actions_expected_runtime': { 'action1': 720.0, 'action2': 480.0 }, 'runs': [ dict( id='MASTER.test.3', state='scheduled', run_time=time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 600), ), end_time=None, duration='', ), dict( id='MASTER.test.2', state='running', run_time=time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), end_time=None, duration='0:10:01.883601', runs=[ dict( id='MASTER.test.2.action2', state='running', action_name='action2', start_time=time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), duration='0:10:01.883601', ), dict( id='MASTER.test.1.action1', state='running', action_name='action1', start_time=time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), duration='0:10:01.885401', ), ], ), dict( id='MASTER.test.1', state='succeeded', run_time=time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1800), ), end_time=time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1700), ), duration='0:15:00.453601', ), ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.2') assert_equal(state, State.STUCK)
def test_job_running_but_action_failed_already(self): job_runs = { 'status': 'running', 'next_run': None, 'runs': [ { 'id': 'MASTER.test.3', 'state': 'scheduled', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() + 600), ), 'end_time': None, }, { 'id': 'MASTER.test.2', 'state': 'running', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 600), ), 'end_time': None, 'runs': [ { 'id': 'MASTER.test.2.action2', 'state': 'running', }, { 'id': 'MASTER.test.1.action1', 'state': 'failed', }, ], # noqa: E122 }, { 'id': 'MASTER.test.1', 'state': 'succeeded', 'run_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1800), ), 'end_time': time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time() - 1700), ), }, ], } run, state = check_tron_jobs.get_relevant_run_and_state(job_runs) assert_equal(run['id'], 'MASTER.test.2') assert_equal(state, State.FAILED)