def test_delete_event_only_files(make_toy_db): # check the behavior when only file are present on disks runner = CliRunner() # create the event folder ramp_config = read_config(ramp_config_template()) ramp_config['ramp']['event_name'] = 'iris_test2' deployment_dir = os.path.commonpath([ramp_config['ramp']['kit_dir'], ramp_config['ramp']['data_dir']]) runner.invoke(main_utils, ['init-event', '--name', 'iris_test2', '--deployment-dir', deployment_dir]) event_config = os.path.join( deployment_dir, 'events', ramp_config['ramp']['event_name'], 'config.yml' ) with open(event_config, 'w+') as f: yaml.dump(ramp_config, f) # check that --from-disk will raise an error cmd = ['delete-event', '--config', database_config_template(), '--config-event', event_config, '--from-disk'] result = runner.invoke(main, cmd) assert result.exit_code == 1 assert 'add the option "--force"' in result.output cmd = ['delete-event', '--config', database_config_template(), '--config-event', event_config, '--from-disk', '--force'] result = runner.invoke(main, cmd) assert result.exit_code == 0, result.output assert not os.path.exists(os.path.dirname(event_config))
def test_deploy_ramp_event_options(session_scope_function): database_config = read_config(database_config_template()) ramp_config = generate_ramp_config(read_config(ramp_config_template())) deploy_ramp_event(database_config_template(), ramp_config_template()) # deploy again by forcing the deployment deploy_ramp_event(database_config_template(), ramp_config_template(), force=True) # do not deploy the kit to trigger the error in the problem with we don't # force the deployment msg_err = 'The RAMP problem already exists in the database.' with pytest.raises(ValueError, match=msg_err): with session_scope(database_config['sqlalchemy']) as session: problem = get_problem(session, 'iris') problem.path_ramp_kit = problem.path_ramp_kit + '_xxx' session.commit() deploy_ramp_event(database_config_template(), ramp_config_template(), setup_ramp_repo=False, force=False) problem = get_problem(session, 'iris') problem.path_ramp_kit = ramp_config['ramp_kit_dir'] problem.path_ramp_data = problem.path_ramp_data + '_xxx' session.commit() deploy_ramp_event(database_config_template(), ramp_config_template(), setup_ramp_repo=False, force=False)
def test_deploy_ramp_event(session_scope_function): database_config = read_config(database_config_template()) event_config_filename = ramp_config_template() event_config = read_config(event_config_filename) ramp_config = generate_ramp_config(event_config) deploy_ramp_event(database_config_template(), ramp_config_template()) # simulate that we add users and sign-up for the event and that they # submitted the starting kit with session_scope(database_config['sqlalchemy']) as session: add_users(session) sign_up_team(session, ramp_config['event_name'], 'test_user') submit_starting_kits(session, ramp_config['event_name'], 'test_user', ramp_config['ramp_kit_submissions_dir']) # run the dispatcher on the event which are in the dataset dispatcher = Dispatcher(config=database_config, event_config=event_config, worker=CondaEnvWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for a user with session_scope(database_config['sqlalchemy']) as session: submission = get_submissions(session, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 1
def test_deploy_ramp_event(): runner = CliRunner() result = runner.invoke(main, ['deploy-event', '--config', database_config_template(), '--event-config', ramp_config_template()]) assert result.exit_code == 0, result.output result = runner.invoke(main, ['deploy-event', '--config', database_config_template(), '--event-config', ramp_config_template(), '--force']) assert result.exit_code == 0, result.output
def test_delete_user(make_toy_db): runner = CliRunner() runner.invoke(main, [ 'add-user', '--config', database_config_template(), '--login', 'yyy', '--password', 'yyy', '--lastname', 'yyy', '--firstname', 'yyy', '--email', 'yyy', '--access-level', 'user' ], catch_exceptions=False) result = runner.invoke(main, [ 'delete-user', '--config', database_config_template(), '--login', 'yyy' ], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_make_user_admin(make_toy_db): runner = CliRunner() result = runner.invoke(main, ['make-user-admin', '--config', database_config_template(), '--login', 'glemaitre'], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_aws_dispatcher(session_toy): # noqa # copy of test_integration_dispatcher but with AWS if not os.path.isfile(os.path.join(HERE, 'config.yml')): pytest.skip("Only for local tests for now") config = read_config(database_config_template()) event_config = ramp_config_template() event_config = read_config(event_config) # patch the event_config to match local config.yml for AWS aws_event_config = read_config(os.path.join(HERE, 'config.yml')) event_config['worker'] = aws_event_config['worker'] dispatcher = Dispatcher(config=config, event_config=event_config, worker=AWSWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def test_update_all_user_leaderboards(make_toy_db): runner = CliRunner() result = runner.invoke(main, ['update-all-users-leaderboards', '--config', database_config_template(), '--event', 'iris_test'], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_get_submission_by_state(make_toy_db): runner = CliRunner() result = runner.invoke(main, ['get-submissions-by-state', '--config', database_config_template(), '--event', 'boston_housing_test', '--state', 'new'], catch_exceptions=False) assert result.exit_code == 0, result.output assert "ID" in result.output result = runner.invoke(main, ['get-submissions-by-state', '--config', database_config_template(), '--event', 'iris_test', '--state', 'scored'], catch_exceptions=False) assert result.exit_code == 0, result.output assert 'No submission for this event and this state' in result.output
def test_generate_ramp_config_short(): ramp_config = generate_ramp_config(_get_event_config('short'), database_config_template()) expected_config = { 'problem_name': 'iris', 'event_name': 'iris_test', 'event_title': 'Iris event', 'ramp_kit_dir': os.path.join('template', 'ramp-kits', 'iris'), 'ramp_data_dir': os.path.join('template', 'ramp-data', 'iris'), 'ramp_submissions_dir': os.path.join('template', 'events', 'iris_test', 'submissions'), 'sandbox_name': 'starting_kit', 'ramp_predictions_dir': os.path.join('template', 'events', 'iris_test', 'predictions'), 'ramp_logs_dir': os.path.join('template', 'events', 'iris_test', 'logs'), 'ramp_sandbox_dir': os.path.join('template', 'ramp-kits', 'iris', 'submissions', 'starting_kit'), 'ramp_kit_submissions_dir': os.path.join('template', 'ramp-kits', 'iris', 'submissions') } for key in expected_config: assert expected_config[key] in ramp_config[key]
def test_dispatcher_worker_retry(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=10, hunger_policy='exit') dispatcher.fetch_from_db(session_toy) dispatcher.launch_workers(session_toy) # Get one worker and set status to 'retry' worker, (submission_id, submission_name) = \ dispatcher._processing_worker_queue.get() setattr(worker, 'status', 'retry') assert worker.status == 'retry' # Add back to queue dispatcher._processing_worker_queue.put_nowait( (worker, (submission_id, submission_name))) while not dispatcher._processing_worker_queue.empty(): dispatcher.collect_result(session_toy) submissions = get_submissions(session_toy, 'iris_test', 'new') assert submission_name in [sub[1] for sub in submissions]
def test_error_handling_worker_setup_error(session_toy, caplog): # make sure the error on the worker.setup is dealt with correctly # set mock worker class Worker_mock(): def __init__(self, *args, **kwargs): self.state = None def setup(self): raise Exception('Test error') def teardown(self): pass config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) worker = Worker_mock() dispatcher = Dispatcher(config=config, event_config=event_config, worker=Worker_mock, n_workers=-1, hunger_policy='exit') dispatcher.launch() submissions = get_submissions(session_toy, event_config['ramp']['event_name'], 'checking_error') assert len(submissions) == 6 worker.status = 'error' assert 'Test error' in caplog.text
def teardown_function(function): database_config = read_config(database_config_template()) # FIXME: we are recreating the deployment directory but it should be # replaced by an temporary creation of folder. shutil.rmtree(function.deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_set_submission_state(): runner = CliRunner() result = runner.invoke(main, [ 'set-submission-state', '--config', database_config_template(), '--submission-id', 3, '--state', 'scored' ], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_sign_up_team(make_toy_db): runner = CliRunner() result = runner.invoke(main, ['sign-up-team', '--config', database_config_template(), '--event', 'iris_test', '--team', 'glemaitre'], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_add_event_admin(make_toy_db): runner = CliRunner() result = runner.invoke(main, ['add-event-admin', '--config', database_config_template(), '--event', 'iris_test', '--user', 'glemaitre'], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_set_user_access_level(make_toy_db): runner = CliRunner() result = runner.invoke(main, ['set-user-access-level', '--config', database_config_template(), '--login', 'glemaitre', '--access-level', 'admin'], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_approve_user(): runner = CliRunner() result = runner.invoke(main, [ 'approve-user', '--config', database_config_template(), '--login', 'glemaitre' ], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_setup_db(database): database_config = read_config(database_config_template(), filter_section='sqlalchemy') db, Session = setup_db(database_config) with db.connect() as conn: session = Session(bind=conn) file_type = session.query(SubmissionFileType).all() assert len(file_type) > 0
def test_dispatcher(): runner = CliRunner() result = runner.invoke(main, [ "dispatcher", "--config", database_config_template(), "--event-config", ramp_config_template() ]) assert result.exit_code == 0, result.output
def test_dispatcher(verbose_params, make_toy_db): runner = CliRunner() cmd = ["dispatcher", "--config", database_config_template(), "--event-config", ramp_config_template()] if verbose_params is not None: cmd += [verbose_params] result = runner.invoke(main, cmd) assert result.exit_code == 0, result.output
def test_export_submissions(make_toy_db): # when there are no submissions to save runner = CliRunner() result = runner.invoke(main, ['export-leaderboards', '--config', database_config_template(), '--event', 'iris_test', '--path', 'test.csv'], catch_exceptions=False) assert result.exit_code == 0 assert 'No score was found on the' in result.output
def session_scope_function(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: yield finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def make_toy_db(database_connection): database_config = read_config(database_config_template()) ramp_config = ramp_config_template() try: deployment_dir = create_toy_db(database_config, ramp_config) yield finally: shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_add_user(): runner = CliRunner() result = runner.invoke(main, [ 'add-user', '--config', database_config_template(), '--login', 'glemaitre', '--password', 'xxx', '--lastname', 'xxx', '--firstname', 'xxx', '--email', 'xxx', '--access-level', 'admin' ], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_delete_event_error(make_toy_db, from_disk, force): runner = CliRunner() with pytest.raises(FileNotFoundError): cmd = ('delete-event --config ' + database_config_template() + ' --config-event ' + "random") if from_disk: cmd += ' --from-disk' if force: cmd += ' --force' runner.invoke(main, cmd, catch_exceptions=False)
def test_add_problem(): runner = CliRunner() ramp_config = generate_ramp_config(read_config(ramp_config_template())) result = runner.invoke(main, [ 'add-problem', '--config', database_config_template(), '--problem', 'iris', '--kit-dir', ramp_config['ramp_kit_dir'], '--data-dir', ramp_config['ramp_data_dir'], '--force', True ], catch_exceptions=False) assert result.exit_code == 0, result.output
def session_scope_module(): database_config = read_config(database_config_template()) ramp_config = ramp_config_template() try: deployment_dir = create_toy_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: yield session finally: shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def base_db(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: create_test_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: yield session finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_dispatcher_error(): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) # check that passing a not a number will raise a TypeError err_msg = "The parameter 'n_threads' should be a positive integer" with pytest.raises(TypeError, match=err_msg): Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=100, n_threads='whatever', hunger_policy='exit')