def test_worker(): runner = CliRunner() result = runner.invoke(main, ["worker", "--config", ramp_config_template(), "--event-config", ramp_config_template(), "--submission", "starting_kit"]) assert result.exit_code == 0, result.output
def test_deploy_ramp_event(session_scope_function): database_config = read_config(database_config_template()) event_config_filename = ramp_config_template() event_config = read_config(event_config_filename) ramp_config = generate_ramp_config(event_config) deploy_ramp_event(database_config_template(), ramp_config_template()) # simulate that we add users and sign-up for the event and that they # submitted the starting kit with session_scope(database_config['sqlalchemy']) as session: add_users(session) sign_up_team(session, ramp_config['event_name'], 'test_user') submit_starting_kits(session, ramp_config['event_name'], 'test_user', ramp_config['ramp_kit_submissions_dir']) # run the dispatcher on the event which are in the dataset dispatcher = Dispatcher(config=database_config, event_config=event_config, worker=CondaEnvWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for a user with session_scope(database_config['sqlalchemy']) as session: submission = get_submissions(session, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 1
def test_deploy_ramp_event_options(session_scope_function): database_config = read_config(database_config_template()) ramp_config = generate_ramp_config(read_config(ramp_config_template())) deploy_ramp_event(database_config_template(), ramp_config_template()) # deploy again by forcing the deployment deploy_ramp_event(database_config_template(), ramp_config_template(), force=True) # do not deploy the kit to trigger the error in the problem with we don't # force the deployment msg_err = 'The RAMP problem already exists in the database.' with pytest.raises(ValueError, match=msg_err): with session_scope(database_config['sqlalchemy']) as session: problem = get_problem(session, 'iris') problem.path_ramp_kit = problem.path_ramp_kit + '_xxx' session.commit() deploy_ramp_event(database_config_template(), ramp_config_template(), setup_ramp_repo=False, force=False) problem = get_problem(session, 'iris') problem.path_ramp_kit = ramp_config['ramp_kit_dir'] problem.path_ramp_data = problem.path_ramp_data + '_xxx' session.commit() deploy_ramp_event(database_config_template(), ramp_config_template(), setup_ramp_repo=False, force=False)
def test_worker(verbose_params): runner = CliRunner() cmd = ["worker", "--config", ramp_config_template(), "--event-config", ramp_config_template(), "--submission", "starting_kit"] if verbose_params is not None: cmd += [verbose_params] result = runner.invoke(main, cmd) assert result.exit_code == 0, result.output
def test_deploy_ramp_event(): runner = CliRunner() result = runner.invoke(main, ['deploy-event', '--config', database_config_template(), '--event-config', ramp_config_template()]) assert result.exit_code == 0, result.output result = runner.invoke(main, ['deploy-event', '--config', database_config_template(), '--event-config', ramp_config_template(), '--force']) assert result.exit_code == 0, result.output
def test_dispatcher_worker_retry(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=10, hunger_policy='exit') dispatcher.fetch_from_db(session_toy) dispatcher.launch_workers(session_toy) # Get one worker and set status to 'retry' worker, (submission_id, submission_name) = \ dispatcher._processing_worker_queue.get() setattr(worker, 'status', 'retry') assert worker.status == 'retry' # Add back to queue dispatcher._processing_worker_queue.put_nowait( (worker, (submission_id, submission_name))) while not dispatcher._processing_worker_queue.empty(): dispatcher.collect_result(session_toy) submissions = get_submissions(session_toy, 'iris_test', 'new') assert submission_name in [sub[1] for sub in submissions]
def test_error_handling_worker_setup_error(session_toy, caplog): # make sure the error on the worker.setup is dealt with correctly # set mock worker class Worker_mock(): def __init__(self, *args, **kwargs): self.state = None def setup(self): raise Exception('Test error') def teardown(self): pass config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) worker = Worker_mock() dispatcher = Dispatcher(config=config, event_config=event_config, worker=Worker_mock, n_workers=-1, hunger_policy='exit') dispatcher.launch() submissions = get_submissions(session_toy, event_config['ramp']['event_name'], 'checking_error') assert len(submissions) == 6 worker.status = 'error' assert 'Test error' in caplog.text
def test_add_submission_wrong_submission_files(base_db): # check that we raise an error if the file required by the workflow is not # present in the submission or that it has the wrong extension session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'corrupted_submission' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) os.makedirs(path_submission) # case that there is not files in the submission err_msg = 'No file corresponding to the workflow element' with pytest.raises(MissingSubmissionFileError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # case that there is not file corresponding to the workflow component filename = os.path.join(path_submission, 'unknown_file.xxx') open(filename, "w+").close() err_msg = 'No file corresponding to the workflow element' with pytest.raises(MissingSubmissionFileError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # case that we have the correct filename but not the right extension filename = os.path.join(path_submission, 'classifier.xxx') open(filename, "w+").close() err_msg = 'All extensions "xxx" are unknown for the submission' with pytest.raises(MissingExtensionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission)
def test_aws_dispatcher(session_toy): # noqa # copy of test_integration_dispatcher but with AWS if not os.path.isfile(os.path.join(HERE, 'config.yml')): pytest.skip("Only for local tests for now") config = read_config(database_config_template()) event_config = ramp_config_template() event_config = read_config(event_config) # patch the event_config to match local config.yml for AWS aws_event_config = read_config(os.path.join(HERE, 'config.yml')) event_config['worker'] = aws_event_config['worker'] dispatcher = Dispatcher(config=config, event_config=event_config, worker=AWSWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def test_add_submission_too_early_submission(base_db): # check that we raise an error when the elapsed time was not large enough # between the new submission and the previous submission session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) # check that we have an awaiting time for the event event = (session.query(Event).filter( Event.name == event_name).one_or_none()) assert event.min_duration_between_submissions == 900 # make 2 submissions which are too close from each other for submission_idx, submission_name in enumerate( ['random_forest_10_10', 'too_early_submission']): path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) if submission_idx == 1: err_msg = 'You need to wait' with pytest.raises(TooEarlySubmissionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) else: add_submission(session, event_name, username, submission_name, path_submission)
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # check that the submissions have been copied for sub_id, _, _ in all_submissions: sub = get_submission_by_id(session, sub_id) assert os.path.exists(sub.path) assert os.path.exists(os.path.join(sub.path, 'classifier.py')) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 # check that the number of submissions for an event was updated event = session.query(Event).filter(Event.name == event_name).one_or_none() assert event.n_submissions == 1 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_delete_event_only_files(make_toy_db): # check the behavior when only file are present on disks runner = CliRunner() # create the event folder ramp_config = read_config(ramp_config_template()) ramp_config['ramp']['event_name'] = 'iris_test2' deployment_dir = os.path.commonpath([ramp_config['ramp']['kit_dir'], ramp_config['ramp']['data_dir']]) runner.invoke(main_utils, ['init-event', '--name', 'iris_test2', '--deployment-dir', deployment_dir]) event_config = os.path.join( deployment_dir, 'events', ramp_config['ramp']['event_name'], 'config.yml' ) with open(event_config, 'w+') as f: yaml.dump(ramp_config, f) # check that --from-disk will raise an error cmd = ['delete-event', '--config', database_config_template(), '--config-event', event_config, '--from-disk'] result = runner.invoke(main, cmd) assert result.exit_code == 1 assert 'add the option "--force"' in result.output cmd = ['delete-event', '--config', database_config_template(), '--config-event', event_config, '--from-disk', '--force'] result = runner.invoke(main, cmd) assert result.exit_code == 0, result.output assert not os.path.exists(os.path.dirname(event_config))
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = read_config(ramp_config_template()) event_name, username = _setup_sign_up(session, config) ramp_config = generate_ramp_config(config) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_check_event(session_scope_function): config = read_config(ramp_config_template()) # addition of event require some problem problem_names = ['iris', 'boston_housing'] for problem_name in problem_names: setup_ramp_kits_ramp_data(config, problem_name) ramp_config = generate_ramp_config(config) add_problem(session_scope_function, problem_name, ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir']) for problem_name in problem_names: event_name = '{}_test'.format(problem_name) event_title = 'event title' add_event(session_scope_function, problem_name, event_name, event_title, ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], is_public=True, force=False) event = get_event(session_scope_function, None) assert len(event) == 2 assert isinstance(event, list) event = get_event(session_scope_function, 'iris_test') scores_iris = ('acc', 'error', 'nll', 'f1_70') _check_event(session_scope_function, event, 'iris_test', 'event title', True, scores_iris) # add event for second time without forcing should raise an error err_msg = 'Attempting to overwrite existing event.' with pytest.raises(ValueError, match=err_msg): add_event(session_scope_function, 'iris', 'iris_test', event_title, ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], is_public=True, force=False) # add event by force add_event(session_scope_function, 'iris', 'iris_test', event_title, ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], is_public=True, force=True) event = get_event(session_scope_function, 'iris_test') _check_event(session_scope_function, event, 'iris_test', 'event title', True, scores_iris) delete_event(session_scope_function, 'iris_test') event = get_event(session_scope_function, None) assert len(event) == 1
def test_generate_worker_config_missing_params(): ramp_config = read_config(ramp_config_template()) # rename on of the key to make the generation failed ramp_config['worker']['env'] = ramp_config['worker']['conda_env'] del ramp_config['worker']['conda_env'] err_msg = "The conda worker is missing the parameter" with pytest.raises(ValueError, match=err_msg): generate_worker_config(ramp_config)
def test_dispatcher(): runner = CliRunner() result = runner.invoke(main, [ "dispatcher", "--config", database_config_template(), "--event-config", ramp_config_template() ]) assert result.exit_code == 0, result.output
def test_dispatcher(verbose_params, make_toy_db): runner = CliRunner() cmd = ["dispatcher", "--config", database_config_template(), "--event-config", ramp_config_template()] if verbose_params is not None: cmd += [verbose_params] result = runner.invoke(main, cmd) assert result.exit_code == 0, result.output
def session_scope_function(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: yield finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def make_toy_db(database_connection): database_config = read_config(database_config_template()) ramp_config = ramp_config_template() try: deployment_dir = create_toy_db(database_config, ramp_config) yield finally: shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_add_problem(): runner = CliRunner() ramp_config = generate_ramp_config(read_config(ramp_config_template())) result = runner.invoke(main, [ 'add-problem', '--config', database_config_template(), '--problem', 'iris', '--kit-dir', ramp_config['ramp_kit_dir'], '--data-dir', ramp_config['ramp_data_dir'], '--force', True ], catch_exceptions=False) assert result.exit_code == 0, result.output
def session_scope_module(): database_config = read_config(database_config_template()) ramp_config = ramp_config_template() try: deployment_dir = create_toy_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: yield session finally: shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def base_db(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: create_test_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: yield session finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_sandbox_upload_file(client_session, makedrop_event, submission_dir, filename): client, session = client_session sign_up_team(session, "iris_test_4event", "test_user") config = ramp_config_template() ramp_config = generate_ramp_config(read_config(config)) # upload file in sandbox.html path_submissions = os.path.join(ramp_config["ramp_kit_dir"], submission_dir) with login_scope(client, "test_user", "test") as client: rv = client.get("http://localhost/events/iris_test_4event/sandbox") assert rv.status_code == 200 # choose file and check if it was uploaded correctly path_submission = os.path.join(path_submissions, filename) assert os.path.isfile(path_submission) rv = client.post( "http://localhost/events/iris_test_4event/sandbox", headers={ "Referer": "http://localhost/events/iris_test_4event/sandbox" }, data={"file": (open(path_submission, "rb"), filename)}, follow_redirects=False, ) assert rv.status_code == 302 assert ( rv.location == "http://localhost/events/iris_test_4event/sandbox") # code of the saved file with open(path_submission, "r") as file: submitted_data = file.read() # code from the db event = get_event(session, "iris_test_4event") sandbox_submission = get_submission_by_name(session, "iris_test_4event", "test_user", event.ramp_sandbox_name) submission_code = sandbox_submission.files[-1].get_code() # get user interactions from db and check if 'upload' was added user_interactions = get_user_interactions_by_name(session, "test_user") # check if the code of the submitted file in the 'submission_code' assert submitted_data is not None assert submitted_data in submission_code # check if the user_interaction was added to the db assert "upload" in user_interactions["interaction"].values
def test_add_event(): runner = CliRunner() ramp_config = generate_ramp_config(read_config(ramp_config_template())) result = runner.invoke(main, [ 'add-event', '--config', database_config_template(), '--problem', 'iris', '--event', 'iris_test', '--title', 'Iris classification', '--sandbox', ramp_config['sandbox_name'], '--submissions-dir', ramp_config['ramp_submissions_dir'], '--is-public', False, '--force', True ], catch_exceptions=False) assert result.exit_code == 0, result.output
def test_dispatcher_error(): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) # check that passing a not a number will raise a TypeError err_msg = "The parameter 'n_threads' should be a positive integer" with pytest.raises(TypeError, match=err_msg): Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=100, n_threads='whatever', hunger_policy='exit')
def session_scope_function(database_connection): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: yield finally: # FIXME: we are recreating the deployment directory but it should be # replaced by an temporary creation of folder. deployment_dir = os.path.commonpath( [ramp_config['ramp']['kit_dir'], ramp_config['ramp']['data_dir']]) shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_generate_worker_config(): worker_config = generate_worker_config(ramp_config_template(), database_config_template()) expected_config = { 'worker_type': 'conda', 'conda_env': 'ramp-iris', 'kit_dir': os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris'), 'data_dir': os.path.join('/tmp/databoard_test', 'ramp-data', 'iris'), 'submissions_dir': os.path.join('/tmp/databoard_test', 'submissions'), 'predictions_dir': os.path.join('/tmp/databoard_test', 'preds'), 'logs_dir': os.path.join('/tmp/databoard_test', 'log') } assert worker_config == expected_config
def session_scope_function(): database_config = read_config(database_config_template()) ramp_config = ramp_config_template() try: deployment_dir = create_test_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: add_users(session) add_problems(session) add_events(session) yield session finally: shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_integration_dispatcher(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_worker=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def test_make_submission_resubmission(base_db): # check that resubmitting the a submission with the same name will raise # an error session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) # submitting the starting_kit which is used as the default submission for # the sandbox should raise an error err_msg = ('Submission "starting_kit" of team "test_user" at event ' '"iris_test" exists already') with pytest.raises(DuplicateSubmissionError, match=err_msg): add_submission(session, event_name, username, os.path.basename(ramp_config['ramp_sandbox_dir']), ramp_config['ramp_sandbox_dir']) # submitting twice a normal submission should raise an error as well submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) # first submission add_submission( session, event_name, username, submission_name, path_submission, ) # mock that we scored the submission set_submission_state(session, 5, 'scored') # second submission err_msg = ('Submission "random_forest_10_10" of team "test_user" at event ' '"iris_test" exists already') with pytest.raises(DuplicateSubmissionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # a resubmission can take place if it is tagged as "new" or failed # mock that the submission failed during the training set_submission_state(session, 5, 'training_error') add_submission(session, event_name, username, submission_name, path_submission) # mock that the submissions are new submissions set_submission_state(session, 5, 'new') add_submission(session, event_name, username, submission_name, path_submission)