def test_get_runs_internal_api(test_client, describe, app, session, basic, logged_in): with describe('Get runs without password'): test_client.req('get', '/api/v-internal/auto_tests/', 400) with describe('Get runs when there are no runs'): test_client.req('get', '/api/v-internal/auto_tests/?get=tests_to_run', 204, headers={ 'CG-Internal-Api-Password': app.config['AUTO_TEST_PASSWORD'] }) with describe('setup'): run = m.AutoTestRun(started_date=None, auto_test=m.AutoTest()) session.add(run) course, assig_id, teacher, student = basic with logged_in(teacher): test = helpers.create_auto_test(test_client, assig_id) run = m.AutoTestRun(started_date=None, auto_test_id=test['id'], is_continuous_feedback_run=True) session.commit() with describe('With runs but with failing broker'): test_client.req('get', '/api/v-internal/auto_tests/?get=tests_to_run', 204, headers={ 'CG-Internal-Api-Password': app.config['AUTO_TEST_PASSWORD'] })
def test_notify_broker_kill_single_runner(describe, session, assignment, monkeypatch, stub_function_class): with describe('setup'): test = m.AutoTest(setup_script='', run_setup_script='', assignment=assignment) run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test, batch_run_done=False) session.flush() session.add(run) runner = m.AutoTestRunner(_ipaddr='localhost', run=run, _job_id=run.get_job_id()) session.commit() ses = requests_stubs.Session() describe.add_hook(ses.reset) monkeypatch.setattr(psef.models.BrokerSetting, 'get_session', lambda *_, **__: ses) with describe('kill runner'): psef.tasks._notify_broker_kill_single_runner_1(run.id, runner.id.hex) call, = ses.calls assert call['method'] == 'delete' assert runner.job_id in call['args'][0] with describe('invalid run'): psef.tasks._notify_broker_kill_single_runner_1(-1, runner.id.hex) assert not ses.calls with describe('invalid runner'): psef.tasks._notify_broker_kill_single_runner_1(run.id, uuid.uuid4().hex) assert not ses.calls
def test_stop_auto_test_run(session, describe, monkeypatch, stub_function_class, app, monkeypatch_celery): with describe('setup'): stub_stop = stub_function_class() monkeypatch.setattr(t, 'stop_auto_test_run', stub_stop) stub_notify_stop = stub_function_class() monkeypatch.setattr(t, 'notify_broker_end_of_job', stub_notify_stop) stub_notify_kill_single = stub_function_class() monkeypatch.setattr(t, 'notify_broker_kill_single_runner', stub_notify_kill_single) test = m.AutoTest(setup_script='', run_setup_script='') run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test) run.state = m.AutoTestRunState.running session.add(run) with app.test_request_context('/non_existing', {}): run.add_active_runner('localhost') session.commit() with describe('kill before deadline'): t._stop_auto_test_run_1(run.id) assert len(stub_stop.all_args) == 1 assert stub_stop.all_args[0][0] == (run.id, ) assert stub_stop.all_args[0]['eta'] == run.kill_date assert not stub_notify_stop.called assert not stub_notify_kill_single.called with describe('kill after deadline'): run.kill_date = datetime.utcnow() old_job_id = run.get_job_id() session.commit() t._stop_auto_test_run_1(run.id) assert not stub_stop.called assert len(stub_notify_stop.all_args) == 1 assert len(stub_notify_kill_single.all_args) == 1 assert stub_notify_stop.all_args[0][0] == old_job_id assert run.state == m.AutoTestRunState.timed_out with describe('kill after already done'): t._stop_auto_test_run_1(run.id) assert not stub_stop.called assert not stub_notify_stop.called assert not stub_notify_kill_single.called with describe('kill with unknown runner'): t._stop_auto_test_run_1((run.id + 2)**4) assert not stub_stop.called assert not stub_notify_stop.called assert not stub_notify_kill_single.called
def test_adjust_amomunt_runners(session, describe, monkeypatch, stub_function_class, app, assignment_real_works, monkeypatch_celery): assignment, submission = assignment_real_works with describe('setup'): stub_notify_new = stub_function_class() monkeypatch.setattr(t, '_notify_broker_of_new_job_1', stub_notify_new) test = m.AutoTest(setup_script='', run_setup_script='', assignment=assignment) run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test, batch_run_done=False) run.results = [ m.AutoTestResult(work_id=submission['id'], final_result=False) ] run.results[0].state = m.AutoTestStepResultState.passed session.add(run) with app.test_request_context('/non_existing', {}): run.add_active_runner('localhost') runner = run.runners[0] assert runner.run session.commit() with describe("Should not request runners when we don't need any"): t.adjust_amount_runners(run.id) assert not stub_notify_new.called with describe('Should be called when there are results'): run.results.append( m.AutoTestResult(work_id=submission['id'], final_result=False)) run.results[-1].state = m.AutoTestStepResultState.not_started session.commit() t.adjust_amount_runners(run.id) assert stub_notify_new.called with describe('Should be called when we have enough'): run.runners_requested = 1 session.commit() t.adjust_amount_runners(run.id) assert not stub_notify_new.called with describe('Should be called when we have too many'): run.runners_requested = 2 session.commit() t.adjust_amount_runners(run.id) assert stub_notify_new.called
def test_internal_api_auth(test_client, app, describe, session): with describe('setup'): ipaddr = object() other_ipaddr = object() run = m.AutoTestRun() runner = m.AutoTestRunner(_ipaddr=ipaddr, run=run, id=uuid.uuid4()) assert runner.id verify_runner = psef.v_internal.auto_tests._verify_and_get_runner with describe('Without correct global password'): test_client.req('get', '/api/v-internal/auto_tests/', 401, headers={'CG-Internal-Api-Password': '******'}) with describe('no runner to verify_runner'): with pytest.raises(psef.exceptions.PermissionException): verify_runner(None, ('', False)) with describe('wrong password to any verify_runner'): with pytest.raises(psef.exceptions.PermissionException): verify_runner(run, ('', False)) with pytest.raises(psef.exceptions.PermissionException): verify_runner(run, ('Not correct', False)) with app.test_request_context('/', environ_base={'REMOTE_ADDR': other_ipaddr}): with describe('Wrong ip address'): with pytest.raises(psef.exceptions.PermissionException): verify_runner(run, (str(runner.id), True)) # It should work when verification is turned off assert runner == verify_runner(run, (str(runner.id), False)) runner._ipaddr = other_ipaddr # It should work when ip is correct assert runner == verify_runner(run, (str(runner.id), True)) with describe('no run should not work'): runner.run = None with pytest.raises(psef.exceptions.PermissionException): verify_runner(run, (str(runner.id), True))
def make_assig(*args, **kwargs): hidden = kwargs.pop('has_hidden', False) with logged_in(admin_user): assig_id = create_assignment(*args, **kwargs)['id'] create_auto_test(test_client, assig_id, has_hidden_steps=hidden) assig = m.Assignment.query.get(assig_id) run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=assig.auto_test, batch_run_done=False) session.add(run) session.commit() with logged_in(admin_user): sub_id = create_submission(test_client, assig.id)['id'] assert run.id is not None session.add( m.AutoTestResult(final_result=False, work_id=sub_id, auto_test_run_id=run.id)) session.commit() return assig
def test_check_heartbeat(session, describe, monkeypatch, stub_function_class, app, assignment_real_works, monkeypatch_celery): assignment, submission = assignment_real_works sub2_id = m.Work.query.filter_by(assignment_id=assignment.id).filter( m.Work.id != submission['id']).first().id with describe('setup'): stub_heart = stub_function_class() monkeypatch.setattr(t, 'check_heartbeat_auto_test_run', stub_heart) stub_notify_new = stub_function_class() monkeypatch.setattr(t, '_notify_broker_of_new_job_1', stub_notify_new) orig_kill_and_adjust = t._kill_runners_and_adjust_1 kill_and_adjust = stub_function_class(orig_kill_and_adjust) monkeypatch.setattr(t, 'kill_runners_and_adjust', kill_and_adjust) stub_notify_stop = stub_function_class() monkeypatch.setattr(t, 'notify_broker_end_of_job', stub_notify_stop) stub_notify_kill_single = stub_function_class() monkeypatch.setattr(t, 'notify_broker_kill_single_runner', stub_notify_kill_single) test = m.AutoTest(setup_script='', run_setup_script='', assignment=assignment) run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test, batch_run_done=False) run.results = [ m.AutoTestResult(work_id=sub2_id, final_result=False), m.AutoTestResult(work_id=submission['id'], final_result=False) ] run.results[0].state = m.AutoTestStepResultState.running run.results[1].state = m.AutoTestStepResultState.passed session.add(run) with app.test_request_context('/non_existing', {}): run.add_active_runner('localhost') runner = run.runners[0] run.results[0].runner = runner run.results[1].runner = runner with app.test_request_context('/non_existing', {}): run.add_active_runner('localhost2') assert runner.run session.commit() with describe('not expired'): t._check_heartbeat_stop_test_runner_1.delay(runner.id.hex) now = DatetimeWithTimezone.utcnow() # As the heartbeats have not expired yet a new check should be # scheduled assert len(stub_heart.all_args) == 1 assert stub_heart.all_args[0][0] == (runner.id.hex, ) assert (stub_heart.all_args[0]['eta'] - now).total_seconds() > 0 assert not stub_notify_new.called assert not kill_and_adjust.called assert not stub_notify_stop.all_args assert not stub_notify_kill_single.all_args with describe('expired'): runner.last_heartbeat = DatetimeWithTimezone.utcfromtimestamp(0) old_job_id = run.get_job_id() session.commit() run = runner.run t._check_heartbeat_stop_test_runner_1.delay(runner.id.hex) assert not stub_heart.called assert len(stub_notify_new.all_args) == 0 assert len(kill_and_adjust.args) == 1 assert len(stub_notify_stop.all_args) == 0 assert len(stub_notify_kill_single.all_args) == 0 run.get_job_id() != old_job_id assert runner.run is None assert (run.results[0].state == m.AutoTestStepResultState.not_started ), 'The results should be cleared' assert (run.results[1].state == m.AutoTestStepResultState.passed ), 'Passed results should not be cleared' with describe('With non existing runner'): t._check_heartbeat_stop_test_runner_1.delay(uuid.uuid4().hex) assert not stub_heart.called assert not stub_notify_new.called assert not stub_notify_stop.called assert not stub_notify_kill_single.called
def test_check_heartbeat(session, describe, monkeypatch, stub_function_class, app, assignment_real_works, monkeypatch_celery): assignment, submission = assignment_real_works with describe('setup'): stub_heart = stub_function_class() monkeypatch.setattr(t, 'check_heartbeat_auto_test_run', stub_heart) stub_notify_new = stub_function_class() monkeypatch.setattr(t, '_notify_broker_of_new_job_1', stub_notify_new) stub_notify_stop = stub_function_class() monkeypatch.setattr(t, 'notify_broker_end_of_job', stub_notify_stop) stub_notify_kill_single = stub_function_class() monkeypatch.setattr(t, 'notify_broker_kill_single_runner', stub_notify_kill_single) test = m.AutoTest(setup_script='', run_setup_script='', assignment=assignment) run = m.AutoTestRun(_job_id=uuid.uuid4(), auto_test=test) run.results = [ m.AutoTestResult(work_id=submission['id']), m.AutoTestResult(work_id=submission['id']) ] run.results[0].state = m.AutoTestStepResultState.failed run.results[1].state = m.AutoTestStepResultState.passed session.add(run) with app.test_request_context('/non_existing', {}): run.add_active_runner('localhost') runner = run.runners[0] assert runner.run session.commit() with describe('not expired'): t._check_heartbeat_stop_test_runner_1(runner.id.hex) now = datetime.utcnow() # As the heartbeats have not expired yet a new check should be # scheduled assert len(stub_heart.all_args) == 1 assert stub_heart.all_args[0][0] == (runner.id.hex, ) assert (stub_heart.all_args[0]['eta'] - now).total_seconds() > 0 assert not stub_notify_new.called assert not stub_notify_stop.all_args assert not stub_notify_kill_single.all_args with describe('Already finished'): with describe('Inner setup'): # Use _state as the setter does logic we don't wan't right now run._state = m.AutoTestRunState.done session.commit() t._check_heartbeat_stop_test_runner_1(runner.id.hex) assert not stub_heart.called assert not stub_notify_new.called assert not stub_notify_stop.called assert not stub_notify_kill_single.all_args run._state = m.AutoTestRunState.running session.commit() with describe('expired'): runner.last_heartbeat = datetime.fromtimestamp(0) old_job_id = run.get_job_id() session.commit() run = runner.run t._check_heartbeat_stop_test_runner_1(runner.id.hex) assert not stub_heart.called assert len(stub_notify_new.all_args) == 1 assert len(stub_notify_stop.all_args) == 1 assert len(stub_notify_kill_single.all_args) == 1 assert stub_notify_new.call_dates[0] > stub_notify_stop.call_dates[0] run.get_job_id() != old_job_id assert runner.run is None assert run.state == m.AutoTestRunState.running assert (run.results[0].state == m.AutoTestStepResultState.not_started ), 'The results should be cleared' assert (run.results[1].state == m.AutoTestStepResultState.passed ), 'Passed results should not be cleared' with describe('With non existing runner'): t._check_heartbeat_stop_test_runner_1(uuid.uuid4().hex) assert not stub_heart.called assert not stub_notify_new.called assert not stub_notify_stop.called assert not stub_notify_kill_single.called