def test_worker():
runner = CliRunner()
result = runner.invoke(main, ["worker",
"--config", ramp_config_template(),
"--event-config", ramp_config_template(),
"--submission", "starting_kit"])
assert result.exit_code == 0, result.output
def test_deploy_ramp_event(session_scope_function):
database_config = read_config(database_config_template())
event_config_filename = ramp_config_template()
event_config = read_config(event_config_filename)
ramp_config = generate_ramp_config(event_config)
deploy_ramp_event(database_config_template(), ramp_config_template())
# simulate that we add users and sign-up for the event and that they
# submitted the starting kit
with session_scope(database_config['sqlalchemy']) as session:
add_users(session)
sign_up_team(session, ramp_config['event_name'], 'test_user')
submit_starting_kits(session, ramp_config['event_name'], 'test_user',
ramp_config['ramp_kit_submissions_dir'])
# run the dispatcher on the event which are in the dataset
dispatcher = Dispatcher(config=database_config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for a user
with session_scope(database_config['sqlalchemy']) as session:
submission = get_submissions(session,
event_config['ramp']['event_name'],
'training_error')
assert len(submission) == 1
def test_deploy_ramp_event_options(session_scope_function):
database_config = read_config(database_config_template())
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
deploy_ramp_event(database_config_template(), ramp_config_template())
# deploy again by forcing the deployment
deploy_ramp_event(database_config_template(),
ramp_config_template(),
force=True)
# do not deploy the kit to trigger the error in the problem with we don't
# force the deployment
msg_err = 'The RAMP problem already exists in the database.'
with pytest.raises(ValueError, match=msg_err):
with session_scope(database_config['sqlalchemy']) as session:
problem = get_problem(session, 'iris')
problem.path_ramp_kit = problem.path_ramp_kit + '_xxx'
session.commit()
deploy_ramp_event(database_config_template(),
ramp_config_template(),
setup_ramp_repo=False,
force=False)
problem = get_problem(session, 'iris')
problem.path_ramp_kit = ramp_config['ramp_kit_dir']
problem.path_ramp_data = problem.path_ramp_data + '_xxx'
session.commit()
deploy_ramp_event(database_config_template(),
ramp_config_template(),
setup_ramp_repo=False,
force=False)
def test_worker(verbose_params):
runner = CliRunner()
cmd = ["worker",
"--config", ramp_config_template(),
"--event-config", ramp_config_template(),
"--submission", "starting_kit"]
if verbose_params is not None:
cmd += [verbose_params]
result = runner.invoke(main, cmd)
assert result.exit_code == 0, result.output
def test_deploy_ramp_event():
runner = CliRunner()
result = runner.invoke(main, ['deploy-event',
'--config', database_config_template(),
'--event-config', ramp_config_template()])
assert result.exit_code == 0, result.output
result = runner.invoke(main, ['deploy-event',
'--config', database_config_template(),
'--event-config', ramp_config_template(),
'--force'])
assert result.exit_code == 0, result.output
def test_dispatcher_worker_retry(session_toy):
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=10,
hunger_policy='exit')
dispatcher.fetch_from_db(session_toy)
dispatcher.launch_workers(session_toy)
# Get one worker and set status to 'retry'
worker, (submission_id, submission_name) = \
dispatcher._processing_worker_queue.get()
setattr(worker, 'status', 'retry')
assert worker.status == 'retry'
# Add back to queue
dispatcher._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
while not dispatcher._processing_worker_queue.empty():
dispatcher.collect_result(session_toy)
submissions = get_submissions(session_toy, 'iris_test', 'new')
assert submission_name in [sub[1] for sub in submissions]
def test_error_handling_worker_setup_error(session_toy, caplog):
# make sure the error on the worker.setup is dealt with correctly
# set mock worker
class Worker_mock():
def __init__(self, *args, **kwargs):
self.state = None
def setup(self):
raise Exception('Test error')
def teardown(self):
pass
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
worker = Worker_mock()
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=Worker_mock,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
submissions = get_submissions(session_toy,
event_config['ramp']['event_name'],
'checking_error')
assert len(submissions) == 6
worker.status = 'error'
assert 'Test error' in caplog.text
def test_add_submission_wrong_submission_files(base_db):
# check that we raise an error if the file required by the workflow is not
# present in the submission or that it has the wrong extension
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submission_name = 'corrupted_submission'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
os.makedirs(path_submission)
# case that there is not files in the submission
err_msg = 'No file corresponding to the workflow element'
with pytest.raises(MissingSubmissionFileError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# case that there is not file corresponding to the workflow component
filename = os.path.join(path_submission, 'unknown_file.xxx')
open(filename, "w+").close()
err_msg = 'No file corresponding to the workflow element'
with pytest.raises(MissingSubmissionFileError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# case that we have the correct filename but not the right extension
filename = os.path.join(path_submission, 'classifier.xxx')
open(filename, "w+").close()
err_msg = 'All extensions "xxx" are unknown for the submission'
with pytest.raises(MissingExtensionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
def test_aws_dispatcher(session_toy): # noqa
# copy of test_integration_dispatcher but with AWS
if not os.path.isfile(os.path.join(HERE, 'config.yml')):
pytest.skip("Only for local tests for now")
config = read_config(database_config_template())
event_config = ramp_config_template()
event_config = read_config(event_config)
# patch the event_config to match local config.yml for AWS
aws_event_config = read_config(os.path.join(HERE, 'config.yml'))
event_config['worker'] = aws_event_config['worker']
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=AWSWorker,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for each user
submission = get_submissions(session_toy,
event_config['ramp']['event_name'],
'training_error')
assert len(submission) == 2
def test_add_submission_too_early_submission(base_db):
# check that we raise an error when the elapsed time was not large enough
# between the new submission and the previous submission
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
# check that we have an awaiting time for the event
event = (session.query(Event).filter(
Event.name == event_name).one_or_none())
assert event.min_duration_between_submissions == 900
# make 2 submissions which are too close from each other
for submission_idx, submission_name in enumerate(
['random_forest_10_10', 'too_early_submission']):
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
if submission_idx == 1:
err_msg = 'You need to wait'
with pytest.raises(TooEarlySubmissionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
else:
add_submission(session, event_name, username, submission_name,
path_submission)
def test_add_submission_create_new_submission(base_db):
# check that we can make a new submission to the database
# it will require to have already a team and an event
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
add_submission(session, event_name, username, submission_name,
path_submission)
all_submissions = get_submissions(session, event_name, None)
# check that the submissions have been copied
for sub_id, _, _ in all_submissions:
sub = get_submission_by_id(session, sub_id)
assert os.path.exists(sub.path)
assert os.path.exists(os.path.join(sub.path, 'classifier.py'))
# `sign_up_team` make a submission (sandbox) by user. This submission will
# be the third submission.
assert len(all_submissions) == 3
# check that the number of submissions for an event was updated
event = session.query(Event).filter(Event.name == event_name).one_or_none()
assert event.n_submissions == 1
submission = get_submission_by_name(session, event_name, username,
submission_name)
assert submission.name == submission_name
submission_file = submission.files[0]
assert submission_file.name == 'classifier'
assert submission_file.extension == 'py'
assert (os.path.join('submission_000000005', 'classifier.py')
in submission_file.path)
def test_delete_event_only_files(make_toy_db):
# check the behavior when only file are present on disks
runner = CliRunner()
# create the event folder
ramp_config = read_config(ramp_config_template())
ramp_config['ramp']['event_name'] = 'iris_test2'
deployment_dir = os.path.commonpath([ramp_config['ramp']['kit_dir'],
ramp_config['ramp']['data_dir']])
runner.invoke(main_utils, ['init-event',
'--name', 'iris_test2',
'--deployment-dir', deployment_dir])
event_config = os.path.join(
deployment_dir, 'events', ramp_config['ramp']['event_name'],
'config.yml'
)
with open(event_config, 'w+') as f:
yaml.dump(ramp_config, f)
# check that --from-disk will raise an error
cmd = ['delete-event',
'--config', database_config_template(),
'--config-event', event_config,
'--from-disk']
result = runner.invoke(main, cmd)
assert result.exit_code == 1
assert 'add the option "--force"' in result.output
cmd = ['delete-event',
'--config', database_config_template(),
'--config-event', event_config,
'--from-disk', '--force']
result = runner.invoke(main, cmd)
assert result.exit_code == 0, result.output
assert not os.path.exists(os.path.dirname(event_config))
def test_add_submission_create_new_submission(base_db):
# check that we can make a new submission to the database
# it will require to have already a team and an event
session = base_db
config = read_config(ramp_config_template())
event_name, username = _setup_sign_up(session, config)
ramp_config = generate_ramp_config(config)
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
add_submission(session, event_name, username, submission_name,
path_submission)
all_submissions = get_submissions(session, event_name, None)
# `sign_up_team` make a submission (sandbox) by user. This submission will
# be the third submission.
assert len(all_submissions) == 3
submission = get_submission_by_name(session, event_name, username,
submission_name)
assert submission.name == submission_name
submission_file = submission.files[0]
assert submission_file.name == 'classifier'
assert submission_file.extension == 'py'
assert (os.path.join('submission_000000005', 'classifier.py')
in submission_file.path)
def test_check_event(session_scope_function):
config = read_config(ramp_config_template())
# addition of event require some problem
problem_names = ['iris', 'boston_housing']
for problem_name in problem_names:
setup_ramp_kits_ramp_data(config, problem_name)
ramp_config = generate_ramp_config(config)
add_problem(session_scope_function, problem_name,
ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir'])
for problem_name in problem_names:
event_name = '{}_test'.format(problem_name)
event_title = 'event title'
add_event(session_scope_function,
problem_name,
event_name,
event_title,
ramp_config['sandbox_name'],
ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
event = get_event(session_scope_function, None)
assert len(event) == 2
assert isinstance(event, list)
event = get_event(session_scope_function, 'iris_test')
scores_iris = ('acc', 'error', 'nll', 'f1_70')
_check_event(session_scope_function, event, 'iris_test', 'event title',
True, scores_iris)
# add event for second time without forcing should raise an error
err_msg = 'Attempting to overwrite existing event.'
with pytest.raises(ValueError, match=err_msg):
add_event(session_scope_function,
'iris',
'iris_test',
event_title,
ramp_config['sandbox_name'],
ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
# add event by force
add_event(session_scope_function,
'iris',
'iris_test',
event_title,
ramp_config['sandbox_name'],
ramp_config['ramp_submissions_dir'],
is_public=True,
force=True)
event = get_event(session_scope_function, 'iris_test')
_check_event(session_scope_function, event, 'iris_test', 'event title',
True, scores_iris)
delete_event(session_scope_function, 'iris_test')
event = get_event(session_scope_function, None)
assert len(event) == 1
def test_generate_worker_config_missing_params():
ramp_config = read_config(ramp_config_template())
# rename on of the key to make the generation failed
ramp_config['worker']['env'] = ramp_config['worker']['conda_env']
del ramp_config['worker']['conda_env']
err_msg = "The conda worker is missing the parameter"
with pytest.raises(ValueError, match=err_msg):
generate_worker_config(ramp_config)
def test_dispatcher():
runner = CliRunner()
result = runner.invoke(main, [
"dispatcher", "--config",
database_config_template(), "--event-config",
ramp_config_template()
])
assert result.exit_code == 0, result.output
def test_dispatcher(verbose_params, make_toy_db):
runner = CliRunner()
cmd = ["dispatcher",
"--config", database_config_template(),
"--event-config", ramp_config_template()]
if verbose_params is not None:
cmd += [verbose_params]
result = runner.invoke(main, cmd)
assert result.exit_code == 0, result.output
def session_scope_function():
database_config = read_config(database_config_template())
ramp_config = read_config(ramp_config_template())
try:
yield
finally:
shutil.rmtree(ramp_config['ramp']['deployment_dir'],
ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def make_toy_db(database_connection):
database_config = read_config(database_config_template())
ramp_config = ramp_config_template()
try:
deployment_dir = create_toy_db(database_config, ramp_config)
yield
finally:
shutil.rmtree(deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_add_problem():
runner = CliRunner()
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
result = runner.invoke(main, [
'add-problem', '--config',
database_config_template(), '--problem', 'iris', '--kit-dir',
ramp_config['ramp_kit_dir'], '--data-dir',
ramp_config['ramp_data_dir'], '--force', True
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def session_scope_module():
database_config = read_config(database_config_template())
ramp_config = ramp_config_template()
try:
deployment_dir = create_toy_db(database_config, ramp_config)
with session_scope(database_config['sqlalchemy']) as session:
yield session
finally:
shutil.rmtree(deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def base_db():
database_config = read_config(database_config_template())
ramp_config = read_config(ramp_config_template())
try:
create_test_db(database_config, ramp_config)
with session_scope(database_config['sqlalchemy']) as session:
yield session
finally:
shutil.rmtree(ramp_config['ramp']['deployment_dir'],
ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_sandbox_upload_file(client_session, makedrop_event, submission_dir,
filename):
client, session = client_session
sign_up_team(session, "iris_test_4event", "test_user")
config = ramp_config_template()
ramp_config = generate_ramp_config(read_config(config))
# upload file in sandbox.html
path_submissions = os.path.join(ramp_config["ramp_kit_dir"],
submission_dir)
with login_scope(client, "test_user", "test") as client:
rv = client.get("http://localhost/events/iris_test_4event/sandbox")
assert rv.status_code == 200
# choose file and check if it was uploaded correctly
path_submission = os.path.join(path_submissions, filename)
assert os.path.isfile(path_submission)
rv = client.post(
"http://localhost/events/iris_test_4event/sandbox",
headers={
"Referer": "http://localhost/events/iris_test_4event/sandbox"
},
data={"file": (open(path_submission, "rb"), filename)},
follow_redirects=False,
)
assert rv.status_code == 302
assert (
rv.location == "http://localhost/events/iris_test_4event/sandbox")
# code of the saved file
with open(path_submission, "r") as file:
submitted_data = file.read()
# code from the db
event = get_event(session, "iris_test_4event")
sandbox_submission = get_submission_by_name(session,
"iris_test_4event",
"test_user",
event.ramp_sandbox_name)
submission_code = sandbox_submission.files[-1].get_code()
# get user interactions from db and check if 'upload' was added
user_interactions = get_user_interactions_by_name(session, "test_user")
# check if the code of the submitted file in the 'submission_code'
assert submitted_data is not None
assert submitted_data in submission_code
# check if the user_interaction was added to the db
assert "upload" in user_interactions["interaction"].values
def test_add_event():
runner = CliRunner()
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
result = runner.invoke(main, [
'add-event', '--config',
database_config_template(), '--problem', 'iris', '--event',
'iris_test', '--title', 'Iris classification', '--sandbox',
ramp_config['sandbox_name'], '--submissions-dir',
ramp_config['ramp_submissions_dir'], '--is-public', False, '--force',
True
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_dispatcher_error():
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
# check that passing a not a number will raise a TypeError
err_msg = "The parameter 'n_threads' should be a positive integer"
with pytest.raises(TypeError, match=err_msg):
Dispatcher(config=config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=100,
n_threads='whatever',
hunger_policy='exit')
def session_scope_function(database_connection):
database_config = read_config(database_config_template())
ramp_config = read_config(ramp_config_template())
try:
yield
finally:
# FIXME: we are recreating the deployment directory but it should be
# replaced by an temporary creation of folder.
deployment_dir = os.path.commonpath(
[ramp_config['ramp']['kit_dir'], ramp_config['ramp']['data_dir']])
shutil.rmtree(deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_generate_worker_config():
worker_config = generate_worker_config(ramp_config_template(),
database_config_template())
expected_config = {
'worker_type': 'conda',
'conda_env': 'ramp-iris',
'kit_dir': os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris'),
'data_dir': os.path.join('/tmp/databoard_test', 'ramp-data', 'iris'),
'submissions_dir': os.path.join('/tmp/databoard_test', 'submissions'),
'predictions_dir': os.path.join('/tmp/databoard_test', 'preds'),
'logs_dir': os.path.join('/tmp/databoard_test', 'log')
}
assert worker_config == expected_config
def session_scope_function():
database_config = read_config(database_config_template())
ramp_config = ramp_config_template()
try:
deployment_dir = create_test_db(database_config, ramp_config)
with session_scope(database_config['sqlalchemy']) as session:
add_users(session)
add_problems(session)
add_events(session)
yield session
finally:
shutil.rmtree(deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_integration_dispatcher(session_toy):
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=CondaEnvWorker,
n_worker=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for each user
submission = get_submissions(session_toy,
event_config['ramp']['event_name'],
'training_error')
assert len(submission) == 2
def test_make_submission_resubmission(base_db):
# check that resubmitting the a submission with the same name will raise
# an error
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
# submitting the starting_kit which is used as the default submission for
# the sandbox should raise an error
err_msg = ('Submission "starting_kit" of team "test_user" at event '
'"iris_test" exists already')
with pytest.raises(DuplicateSubmissionError, match=err_msg):
add_submission(session, event_name, username,
os.path.basename(ramp_config['ramp_sandbox_dir']),
ramp_config['ramp_sandbox_dir'])
# submitting twice a normal submission should raise an error as well
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
# first submission
add_submission(
session,
event_name,
username,
submission_name,
path_submission,
)
# mock that we scored the submission
set_submission_state(session, 5, 'scored')
# second submission
err_msg = ('Submission "random_forest_10_10" of team "test_user" at event '
'"iris_test" exists already')
with pytest.raises(DuplicateSubmissionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# a resubmission can take place if it is tagged as "new" or failed
# mock that the submission failed during the training
set_submission_state(session, 5, 'training_error')
add_submission(session, event_name, username, submission_name,
path_submission)
# mock that the submissions are new submissions
set_submission_state(session, 5, 'new')
add_submission(session, event_name, username, submission_name,
path_submission)
