def test_delete_event_only_files(make_toy_db):
# check the behavior when only file are present on disks
runner = CliRunner()
# create the event folder
ramp_config = read_config(ramp_config_template())
ramp_config['ramp']['event_name'] = 'iris_test2'
deployment_dir = os.path.commonpath([ramp_config['ramp']['kit_dir'],
ramp_config['ramp']['data_dir']])
runner.invoke(main_utils, ['init-event',
'--name', 'iris_test2',
'--deployment-dir', deployment_dir])
event_config = os.path.join(
deployment_dir, 'events', ramp_config['ramp']['event_name'],
'config.yml'
)
with open(event_config, 'w+') as f:
yaml.dump(ramp_config, f)
# check that --from-disk will raise an error
cmd = ['delete-event',
'--config', database_config_template(),
'--config-event', event_config,
'--from-disk']
result = runner.invoke(main, cmd)
assert result.exit_code == 1
assert 'add the option "--force"' in result.output
cmd = ['delete-event',
'--config', database_config_template(),
'--config-event', event_config,
'--from-disk', '--force']
result = runner.invoke(main, cmd)
assert result.exit_code == 0, result.output
assert not os.path.exists(os.path.dirname(event_config))
def test_deploy_ramp_event_options(session_scope_function):
database_config = read_config(database_config_template())
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
deploy_ramp_event(database_config_template(), ramp_config_template())
# deploy again by forcing the deployment
deploy_ramp_event(database_config_template(),
ramp_config_template(),
force=True)
# do not deploy the kit to trigger the error in the problem with we don't
# force the deployment
msg_err = 'The RAMP problem already exists in the database.'
with pytest.raises(ValueError, match=msg_err):
with session_scope(database_config['sqlalchemy']) as session:
problem = get_problem(session, 'iris')
problem.path_ramp_kit = problem.path_ramp_kit + '_xxx'
session.commit()
deploy_ramp_event(database_config_template(),
ramp_config_template(),
setup_ramp_repo=False,
force=False)
problem = get_problem(session, 'iris')
problem.path_ramp_kit = ramp_config['ramp_kit_dir']
problem.path_ramp_data = problem.path_ramp_data + '_xxx'
session.commit()
deploy_ramp_event(database_config_template(),
ramp_config_template(),
setup_ramp_repo=False,
force=False)
def test_deploy_ramp_event(session_scope_function):
database_config = read_config(database_config_template())
event_config_filename = ramp_config_template()
event_config = read_config(event_config_filename)
ramp_config = generate_ramp_config(event_config)
deploy_ramp_event(database_config_template(), ramp_config_template())
# simulate that we add users and sign-up for the event and that they
# submitted the starting kit
with session_scope(database_config['sqlalchemy']) as session:
add_users(session)
sign_up_team(session, ramp_config['event_name'], 'test_user')
submit_starting_kits(session, ramp_config['event_name'], 'test_user',
ramp_config['ramp_kit_submissions_dir'])
# run the dispatcher on the event which are in the dataset
dispatcher = Dispatcher(config=database_config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for a user
with session_scope(database_config['sqlalchemy']) as session:
submission = get_submissions(session,
event_config['ramp']['event_name'],
'training_error')
assert len(submission) == 1
def test_deploy_ramp_event():
runner = CliRunner()
result = runner.invoke(main, ['deploy-event',
'--config', database_config_template(),
'--event-config', ramp_config_template()])
assert result.exit_code == 0, result.output
result = runner.invoke(main, ['deploy-event',
'--config', database_config_template(),
'--event-config', ramp_config_template(),
'--force'])
assert result.exit_code == 0, result.output
def test_delete_user(make_toy_db):
runner = CliRunner()
runner.invoke(main, [
'add-user', '--config',
database_config_template(), '--login', 'yyy', '--password', 'yyy',
'--lastname', 'yyy', '--firstname', 'yyy', '--email', 'yyy',
'--access-level', 'user'
],
catch_exceptions=False)
result = runner.invoke(main, [
'delete-user', '--config',
database_config_template(), '--login', 'yyy'
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_make_user_admin(make_toy_db):
runner = CliRunner()
result = runner.invoke(main, ['make-user-admin',
'--config', database_config_template(),
'--login', 'glemaitre'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_aws_dispatcher(session_toy): # noqa
# copy of test_integration_dispatcher but with AWS
if not os.path.isfile(os.path.join(HERE, 'config.yml')):
pytest.skip("Only for local tests for now")
config = read_config(database_config_template())
event_config = ramp_config_template()
event_config = read_config(event_config)
# patch the event_config to match local config.yml for AWS
aws_event_config = read_config(os.path.join(HERE, 'config.yml'))
event_config['worker'] = aws_event_config['worker']
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=AWSWorker,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for each user
submission = get_submissions(session_toy,
event_config['ramp']['event_name'],
'training_error')
assert len(submission) == 2
def test_update_all_user_leaderboards(make_toy_db):
runner = CliRunner()
result = runner.invoke(main, ['update-all-users-leaderboards',
'--config', database_config_template(),
'--event', 'iris_test'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_get_submission_by_state(make_toy_db):
runner = CliRunner()
result = runner.invoke(main, ['get-submissions-by-state',
'--config', database_config_template(),
'--event', 'boston_housing_test',
'--state', 'new'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
assert "ID" in result.output
result = runner.invoke(main, ['get-submissions-by-state',
'--config', database_config_template(),
'--event', 'iris_test',
'--state', 'scored'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
assert 'No submission for this event and this state' in result.output
def test_generate_ramp_config_short():
ramp_config = generate_ramp_config(_get_event_config('short'),
database_config_template())
expected_config = {
'problem_name':
'iris',
'event_name':
'iris_test',
'event_title':
'Iris event',
'ramp_kit_dir':
os.path.join('template', 'ramp-kits', 'iris'),
'ramp_data_dir':
os.path.join('template', 'ramp-data', 'iris'),
'ramp_submissions_dir':
os.path.join('template', 'events', 'iris_test', 'submissions'),
'sandbox_name':
'starting_kit',
'ramp_predictions_dir':
os.path.join('template', 'events', 'iris_test', 'predictions'),
'ramp_logs_dir':
os.path.join('template', 'events', 'iris_test', 'logs'),
'ramp_sandbox_dir':
os.path.join('template', 'ramp-kits', 'iris', 'submissions',
'starting_kit'),
'ramp_kit_submissions_dir':
os.path.join('template', 'ramp-kits', 'iris', 'submissions')
}
for key in expected_config:
assert expected_config[key] in ramp_config[key]
def test_dispatcher_worker_retry(session_toy):
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=10,
hunger_policy='exit')
dispatcher.fetch_from_db(session_toy)
dispatcher.launch_workers(session_toy)
# Get one worker and set status to 'retry'
worker, (submission_id, submission_name) = \
dispatcher._processing_worker_queue.get()
setattr(worker, 'status', 'retry')
assert worker.status == 'retry'
# Add back to queue
dispatcher._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
while not dispatcher._processing_worker_queue.empty():
dispatcher.collect_result(session_toy)
submissions = get_submissions(session_toy, 'iris_test', 'new')
assert submission_name in [sub[1] for sub in submissions]
def test_error_handling_worker_setup_error(session_toy, caplog):
# make sure the error on the worker.setup is dealt with correctly
# set mock worker
class Worker_mock():
def __init__(self, *args, **kwargs):
self.state = None
def setup(self):
raise Exception('Test error')
def teardown(self):
pass
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
worker = Worker_mock()
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=Worker_mock,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
submissions = get_submissions(session_toy,
event_config['ramp']['event_name'],
'checking_error')
assert len(submissions) == 6
worker.status = 'error'
assert 'Test error' in caplog.text
def teardown_function(function):
database_config = read_config(database_config_template())
# FIXME: we are recreating the deployment directory but it should be
# replaced by an temporary creation of folder.
shutil.rmtree(function.deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_set_submission_state():
runner = CliRunner()
result = runner.invoke(main, [
'set-submission-state', '--config',
database_config_template(), '--submission-id', 3, '--state', 'scored'
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_sign_up_team(make_toy_db):
runner = CliRunner()
result = runner.invoke(main, ['sign-up-team',
'--config', database_config_template(),
'--event', 'iris_test',
'--team', 'glemaitre'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_add_event_admin(make_toy_db):
runner = CliRunner()
result = runner.invoke(main, ['add-event-admin',
'--config', database_config_template(),
'--event', 'iris_test',
'--user', 'glemaitre'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_set_user_access_level(make_toy_db):
runner = CliRunner()
result = runner.invoke(main, ['set-user-access-level',
'--config', database_config_template(),
'--login', 'glemaitre',
'--access-level', 'admin'],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_approve_user():
runner = CliRunner()
result = runner.invoke(main, [
'approve-user', '--config',
database_config_template(), '--login', 'glemaitre'
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_setup_db(database):
database_config = read_config(database_config_template(),
filter_section='sqlalchemy')
db, Session = setup_db(database_config)
with db.connect() as conn:
session = Session(bind=conn)
file_type = session.query(SubmissionFileType).all()
assert len(file_type) > 0
def test_dispatcher():
runner = CliRunner()
result = runner.invoke(main, [
"dispatcher", "--config",
database_config_template(), "--event-config",
ramp_config_template()
])
assert result.exit_code == 0, result.output
def test_dispatcher(verbose_params, make_toy_db):
runner = CliRunner()
cmd = ["dispatcher",
"--config", database_config_template(),
"--event-config", ramp_config_template()]
if verbose_params is not None:
cmd += [verbose_params]
result = runner.invoke(main, cmd)
assert result.exit_code == 0, result.output
def test_export_submissions(make_toy_db):
# when there are no submissions to save
runner = CliRunner()
result = runner.invoke(main, ['export-leaderboards',
'--config', database_config_template(),
'--event', 'iris_test',
'--path', 'test.csv'],
catch_exceptions=False)
assert result.exit_code == 0
assert 'No score was found on the' in result.output
def session_scope_function():
database_config = read_config(database_config_template())
ramp_config = read_config(ramp_config_template())
try:
yield
finally:
shutil.rmtree(ramp_config['ramp']['deployment_dir'],
ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def make_toy_db(database_connection):
database_config = read_config(database_config_template())
ramp_config = ramp_config_template()
try:
deployment_dir = create_toy_db(database_config, ramp_config)
yield
finally:
shutil.rmtree(deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_add_user():
runner = CliRunner()
result = runner.invoke(main, [
'add-user', '--config',
database_config_template(), '--login', 'glemaitre', '--password',
'xxx', '--lastname', 'xxx', '--firstname', 'xxx', '--email', 'xxx',
'--access-level', 'admin'
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_delete_event_error(make_toy_db, from_disk, force):
runner = CliRunner()
with pytest.raises(FileNotFoundError):
cmd = ('delete-event --config ' + database_config_template() +
' --config-event ' + "random")
if from_disk:
cmd += ' --from-disk'
if force:
cmd += ' --force'
runner.invoke(main, cmd, catch_exceptions=False)
def test_add_problem():
runner = CliRunner()
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
result = runner.invoke(main, [
'add-problem', '--config',
database_config_template(), '--problem', 'iris', '--kit-dir',
ramp_config['ramp_kit_dir'], '--data-dir',
ramp_config['ramp_data_dir'], '--force', True
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def session_scope_module():
database_config = read_config(database_config_template())
ramp_config = ramp_config_template()
try:
deployment_dir = create_toy_db(database_config, ramp_config)
with session_scope(database_config['sqlalchemy']) as session:
yield session
finally:
shutil.rmtree(deployment_dir, ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def base_db():
database_config = read_config(database_config_template())
ramp_config = read_config(ramp_config_template())
try:
create_test_db(database_config, ramp_config)
with session_scope(database_config['sqlalchemy']) as session:
yield session
finally:
shutil.rmtree(ramp_config['ramp']['deployment_dir'],
ignore_errors=True)
db, _ = setup_db(database_config['sqlalchemy'])
Model.metadata.drop_all(db)
def test_dispatcher_error():
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
# check that passing a not a number will raise a TypeError
err_msg = "The parameter 'n_threads' should be a positive integer"
with pytest.raises(TypeError, match=err_msg):
Dispatcher(config=config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=100,
n_threads='whatever',
hunger_policy='exit')
