def test_job_waiting_for_first_run(self):
job_runs = {
'status':
'scheduled',
'next_run':
None,
'runs': [
{
'id':
'MASTER.test.1',
'state':
'scheduled',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + 1200),
),
'end_time':
None,
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.1')
assert_equal(state, State.WAITING_FOR_FIRST_RUN)
def test_job_has_no_runs_at_all(self):
job_runs = {
'status': 'running',
'next_run': None,
'runs': [],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run, None)
assert_equal(state, State.NO_RUN_YET)
def test_most_recent_end_time_job_failed(self):
job_runs = {
'status':
'scheduled',
'next_run':
None,
'runs': [
{
'id':
'MASTER.test.3',
'state':
'scheduled',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + 600),
),
'end_time':
None
},
{
'id':
'MASTER.test.2',
'state':
'succeeded',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1800),
),
'end_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1700),
),
},
{
'id':
'MASTER.test.1',
'state':
'failed',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
'end_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 500),
),
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.1')
assert_equal(state, State.FAILED)
def test_job_unknown(self):
job_runs = {
'status':
'running',
'next_run':
None,
'runs': [
{
'id':
'MASTER.test.3',
'state':
'scheduled',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
'end_time':
None,
},
{
'id':
'MASTER.test.2',
'state':
'unknown',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1200),
),
'end_time':
None,
},
{
'id':
'MASTER.test.1',
'state':
'succeeded',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1800),
),
'end_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1700),
),
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.2')
assert_equal(state, State.UNKNOWN)
def test_job_running_job_exceeds_expected_runtime(self):
job_runs = {
'status':
'running',
'next_run':
None,
'expected_runtime':
480.0,
'runs': [
{
'id':
'MASTER.test.100',
'state':
'scheduled',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + 600),
),
'end_time':
None,
'start_time':
None,
'duration':
'',
},
{
'id':
'MASTER.test.99',
'state':
'running',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
'start_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
'end_time':
None,
'duration':
'0:10:01.883601',
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.99')
assert_equal(state, State.STUCK)
def test_no_job_scheduled_or_queuing(self):
job_runs = {
'status':
'succeeded',
'next_run':
None,
'runs': [
{
'id':
'MASTER.test.2',
'state':
'succeeded',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 300),
),
'end_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
},
{
'id':
'MASTER.test.1',
'state':
'succeeded',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 900),
),
'end_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1200),
),
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.2')
assert_equal(state, State.NOT_SCHEDULED)
def test_job_stuck_when_runtime_not_sorted(self):
job_runs = {
'status':
'running',
'next_run':
None,
'runs': [
{
'id':
'MASTER.test.2',
'state':
'running',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
'end_time':
None,
},
{
'id':
'MASTER.test.1',
'state':
'scheduled',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time()),
),
'end_time':
None,
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.2')
assert_equal(state, State.STUCK)
def test_job_running_action_exceeds_expected_runtime(self):
job_runs = {
'status':
'running',
'next_run':
None,
'actions_expected_runtime': {
'action1': 720.0,
'action2': 480.0
},
'runs': [
dict(
id='MASTER.test.3',
state='scheduled',
run_time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + 600),
),
end_time=None,
duration='',
),
dict(
id='MASTER.test.2',
state='running',
run_time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
end_time=None,
duration='0:10:01.883601',
runs=[
dict(
id='MASTER.test.2.action2',
state='running',
action_name='action2',
start_time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
duration='0:10:01.883601',
),
dict(
id='MASTER.test.1.action1',
state='running',
action_name='action1',
start_time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
duration='0:10:01.885401',
),
],
),
dict(
id='MASTER.test.1',
state='succeeded',
run_time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1800),
),
end_time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1700),
),
duration='0:15:00.453601',
),
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.2')
assert_equal(state, State.STUCK)
def test_job_running_but_action_failed_already(self):
job_runs = {
'status':
'running',
'next_run':
None,
'runs': [
{
'id':
'MASTER.test.3',
'state':
'scheduled',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() + 600),
),
'end_time':
None,
},
{
'id':
'MASTER.test.2',
'state':
'running',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 600),
),
'end_time':
None,
'runs': [
{
'id': 'MASTER.test.2.action2',
'state': 'running',
},
{
'id': 'MASTER.test.1.action1',
'state': 'failed',
},
], # noqa: E122
},
{
'id':
'MASTER.test.1',
'state':
'succeeded',
'run_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1800),
),
'end_time':
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() - 1700),
),
},
],
}
run, state = check_tron_jobs.get_relevant_run_and_state(job_runs)
assert_equal(run['id'], 'MASTER.test.2')
assert_equal(state, State.FAILED)