def test_notify_broker_kill_single_runner(describe, session, assignment,
monkeypatch, stub_function_class):
with describe('setup'):
test = m.AutoTest(setup_script='',
run_setup_script='',
assignment=assignment)
run = m.AutoTestRun(_job_id=uuid.uuid4(),
auto_test=test,
batch_run_done=False)
session.flush()
session.add(run)
runner = m.AutoTestRunner(_ipaddr='localhost',
run=run,
_job_id=run.get_job_id())
session.commit()
ses = requests_stubs.Session()
describe.add_hook(ses.reset)
monkeypatch.setattr(psef.models.BrokerSetting, 'get_session',
lambda *_, **__: ses)
with describe('kill runner'):
psef.tasks._notify_broker_kill_single_runner_1(run.id, runner.id.hex)
call, = ses.calls
assert call['method'] == 'delete'
assert runner.job_id in call['args'][0]
with describe('invalid run'):
psef.tasks._notify_broker_kill_single_runner_1(-1, runner.id.hex)
assert not ses.calls
with describe('invalid runner'):
psef.tasks._notify_broker_kill_single_runner_1(run.id,
uuid.uuid4().hex)
assert not ses.calls
def test_get_runs_internal_api(test_client, describe, app, session, basic,
logged_in):
with describe('Get runs without password'):
test_client.req('get', '/api/v-internal/auto_tests/', 400)
with describe('Get runs when there are no runs'):
test_client.req('get',
'/api/v-internal/auto_tests/?get=tests_to_run',
204,
headers={
'CG-Internal-Api-Password':
app.config['AUTO_TEST_PASSWORD']
})
with describe('setup'):
run = m.AutoTestRun(started_date=None, auto_test=m.AutoTest())
session.add(run)
course, assig_id, teacher, student = basic
with logged_in(teacher):
test = helpers.create_auto_test(test_client, assig_id)
run = m.AutoTestRun(started_date=None,
auto_test_id=test['id'],
is_continuous_feedback_run=True)
session.commit()
with describe('With runs but with failing broker'):
test_client.req('get',
'/api/v-internal/auto_tests/?get=tests_to_run',
204,
headers={
'CG-Internal-Api-Password':
app.config['AUTO_TEST_PASSWORD']
})
def test_stop_auto_test_run(session, describe, monkeypatch,
stub_function_class, app, monkeypatch_celery):
with describe('setup'):
stub_stop = stub_function_class()
monkeypatch.setattr(t, 'stop_auto_test_run', stub_stop)
stub_notify_stop = stub_function_class()
monkeypatch.setattr(t, 'notify_broker_end_of_job', stub_notify_stop)
stub_notify_kill_single = stub_function_class()
monkeypatch.setattr(t, 'notify_broker_kill_single_runner',
stub_notify_kill_single)
test = m.AutoTest(setup_script='', run_setup_script='')
run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test)
run.state = m.AutoTestRunState.running
session.add(run)
with app.test_request_context('/non_existing', {}):
run.add_active_runner('localhost')
session.commit()
with describe('kill before deadline'):
t._stop_auto_test_run_1(run.id)
assert len(stub_stop.all_args) == 1
assert stub_stop.all_args[0][0] == (run.id, )
assert stub_stop.all_args[0]['eta'] == run.kill_date
assert not stub_notify_stop.called
assert not stub_notify_kill_single.called
with describe('kill after deadline'):
run.kill_date = datetime.utcnow()
old_job_id = run.get_job_id()
session.commit()
t._stop_auto_test_run_1(run.id)
assert not stub_stop.called
assert len(stub_notify_stop.all_args) == 1
assert len(stub_notify_kill_single.all_args) == 1
assert stub_notify_stop.all_args[0][0] == old_job_id
assert run.state == m.AutoTestRunState.timed_out
with describe('kill after already done'):
t._stop_auto_test_run_1(run.id)
assert not stub_stop.called
assert not stub_notify_stop.called
assert not stub_notify_kill_single.called
with describe('kill with unknown runner'):
t._stop_auto_test_run_1((run.id + 2)**4)
assert not stub_stop.called
assert not stub_notify_stop.called
assert not stub_notify_kill_single.called
def test_adjust_amomunt_runners(session, describe, monkeypatch,
stub_function_class, app,
assignment_real_works, monkeypatch_celery):
assignment, submission = assignment_real_works
with describe('setup'):
stub_notify_new = stub_function_class()
monkeypatch.setattr(t, '_notify_broker_of_new_job_1', stub_notify_new)
test = m.AutoTest(setup_script='',
run_setup_script='',
assignment=assignment)
run = m.AutoTestRun(_job_id=uuid.uuid4(),
auto_test=test,
batch_run_done=False)
run.results = [
m.AutoTestResult(work_id=submission['id'], final_result=False)
]
run.results[0].state = m.AutoTestStepResultState.passed
session.add(run)
with app.test_request_context('/non_existing', {}):
run.add_active_runner('localhost')
runner = run.runners[0]
assert runner.run
session.commit()
with describe("Should not request runners when we don't need any"):
t.adjust_amount_runners(run.id)
assert not stub_notify_new.called
with describe('Should be called when there are results'):
run.results.append(
m.AutoTestResult(work_id=submission['id'], final_result=False))
run.results[-1].state = m.AutoTestStepResultState.not_started
session.commit()
t.adjust_amount_runners(run.id)
assert stub_notify_new.called
with describe('Should be called when we have enough'):
run.runners_requested = 1
session.commit()
t.adjust_amount_runners(run.id)
assert not stub_notify_new.called
with describe('Should be called when we have too many'):
run.runners_requested = 2
session.commit()
t.adjust_amount_runners(run.id)
assert stub_notify_new.called
def test_check_heartbeat(session, describe, monkeypatch, stub_function_class,
app, assignment_real_works, monkeypatch_celery):
assignment, submission = assignment_real_works
sub2_id = m.Work.query.filter_by(assignment_id=assignment.id).filter(
m.Work.id != submission['id']).first().id
with describe('setup'):
stub_heart = stub_function_class()
monkeypatch.setattr(t, 'check_heartbeat_auto_test_run', stub_heart)
stub_notify_new = stub_function_class()
monkeypatch.setattr(t, '_notify_broker_of_new_job_1', stub_notify_new)
orig_kill_and_adjust = t._kill_runners_and_adjust_1
kill_and_adjust = stub_function_class(orig_kill_and_adjust)
monkeypatch.setattr(t, 'kill_runners_and_adjust', kill_and_adjust)
stub_notify_stop = stub_function_class()
monkeypatch.setattr(t, 'notify_broker_end_of_job', stub_notify_stop)
stub_notify_kill_single = stub_function_class()
monkeypatch.setattr(t, 'notify_broker_kill_single_runner',
stub_notify_kill_single)
test = m.AutoTest(setup_script='',
run_setup_script='',
assignment=assignment)
run = m.AutoTestRun(_job_id=uuid.uuid4(),
auto_test=test,
batch_run_done=False)
run.results = [
m.AutoTestResult(work_id=sub2_id, final_result=False),
m.AutoTestResult(work_id=submission['id'], final_result=False)
]
run.results[0].state = m.AutoTestStepResultState.running
run.results[1].state = m.AutoTestStepResultState.passed
session.add(run)
with app.test_request_context('/non_existing', {}):
run.add_active_runner('localhost')
runner = run.runners[0]
run.results[0].runner = runner
run.results[1].runner = runner
with app.test_request_context('/non_existing', {}):
run.add_active_runner('localhost2')
assert runner.run
session.commit()
with describe('not expired'):
t._check_heartbeat_stop_test_runner_1.delay(runner.id.hex)
now = DatetimeWithTimezone.utcnow()
# As the heartbeats have not expired yet a new check should be
# scheduled
assert len(stub_heart.all_args) == 1
assert stub_heart.all_args[0][0] == (runner.id.hex, )
assert (stub_heart.all_args[0]['eta'] - now).total_seconds() > 0
assert not stub_notify_new.called
assert not kill_and_adjust.called
assert not stub_notify_stop.all_args
assert not stub_notify_kill_single.all_args
with describe('expired'):
runner.last_heartbeat = DatetimeWithTimezone.utcfromtimestamp(0)
old_job_id = run.get_job_id()
session.commit()
run = runner.run
t._check_heartbeat_stop_test_runner_1.delay(runner.id.hex)
assert not stub_heart.called
assert len(stub_notify_new.all_args) == 0
assert len(kill_and_adjust.args) == 1
assert len(stub_notify_stop.all_args) == 0
assert len(stub_notify_kill_single.all_args) == 0
run.get_job_id() != old_job_id
assert runner.run is None
assert (run.results[0].state == m.AutoTestStepResultState.not_started
), 'The results should be cleared'
assert (run.results[1].state == m.AutoTestStepResultState.passed
), 'Passed results should not be cleared'
with describe('With non existing runner'):
t._check_heartbeat_stop_test_runner_1.delay(uuid.uuid4().hex)
assert not stub_heart.called
assert not stub_notify_new.called
assert not stub_notify_stop.called
assert not stub_notify_kill_single.called
def test_check_heartbeat(session, describe, monkeypatch, stub_function_class,
app, assignment_real_works, monkeypatch_celery):
assignment, submission = assignment_real_works
with describe('setup'):
stub_heart = stub_function_class()
monkeypatch.setattr(t, 'check_heartbeat_auto_test_run', stub_heart)
stub_notify_new = stub_function_class()
monkeypatch.setattr(t, '_notify_broker_of_new_job_1', stub_notify_new)
stub_notify_stop = stub_function_class()
monkeypatch.setattr(t, 'notify_broker_end_of_job', stub_notify_stop)
stub_notify_kill_single = stub_function_class()
monkeypatch.setattr(t, 'notify_broker_kill_single_runner',
stub_notify_kill_single)
test = m.AutoTest(setup_script='',
run_setup_script='',
assignment=assignment)
run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test)
run.results = [
m.AutoTestResult(work_id=submission['id']),
m.AutoTestResult(work_id=submission['id'])
]
run.results[0].state = m.AutoTestStepResultState.failed
run.results[1].state = m.AutoTestStepResultState.passed
session.add(run)
with app.test_request_context('/non_existing', {}):
run.add_active_runner('localhost')
runner = run.runners[0]
assert runner.run
session.commit()
with describe('not expired'):
t._check_heartbeat_stop_test_runner_1(runner.id.hex)
now = datetime.utcnow()
# As the heartbeats have not expired yet a new check should be
# scheduled
assert len(stub_heart.all_args) == 1
assert stub_heart.all_args[0][0] == (runner.id.hex, )
assert (stub_heart.all_args[0]['eta'] - now).total_seconds() > 0
assert not stub_notify_new.called
assert not stub_notify_stop.all_args
assert not stub_notify_kill_single.all_args
with describe('Already finished'):
with describe('Inner setup'):
# Use _state as the setter does logic we don't wan't right now
run._state = m.AutoTestRunState.done
session.commit()
t._check_heartbeat_stop_test_runner_1(runner.id.hex)
assert not stub_heart.called
assert not stub_notify_new.called
assert not stub_notify_stop.called
assert not stub_notify_kill_single.all_args
run._state = m.AutoTestRunState.running
session.commit()
with describe('expired'):
runner.last_heartbeat = datetime.fromtimestamp(0)
old_job_id = run.get_job_id()
session.commit()
run = runner.run
t._check_heartbeat_stop_test_runner_1(runner.id.hex)
assert not stub_heart.called
assert len(stub_notify_new.all_args) == 1
assert len(stub_notify_stop.all_args) == 1
assert len(stub_notify_kill_single.all_args) == 1
assert stub_notify_new.call_dates[0] > stub_notify_stop.call_dates[0]
run.get_job_id() != old_job_id
assert runner.run is None
assert run.state == m.AutoTestRunState.running
assert (run.results[0].state == m.AutoTestStepResultState.not_started
), 'The results should be cleared'
assert (run.results[1].state == m.AutoTestStepResultState.passed
), 'Passed results should not be cleared'
with describe('With non existing runner'):
t._check_heartbeat_stop_test_runner_1(uuid.uuid4().hex)
assert not stub_heart.called
assert not stub_notify_new.called
assert not stub_notify_stop.called
assert not stub_notify_kill_single.called