def test_check_problem(session_scope_function):
ramp_configs = {
'iris': read_config(ramp_config_iris()),
'boston_housing': read_config(ramp_config_boston_housing())
}
for problem_name, ramp_config in ramp_configs.items():
internal_ramp_config = generate_ramp_config(ramp_config)
setup_ramp_kit_ramp_data(internal_ramp_config, problem_name)
add_problem(session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'])
problem_name = 'iris'
problem = get_problem(session_scope_function, problem_name)
assert problem.name == problem_name
assert isinstance(problem, Problem)
problem = get_problem(session_scope_function, None)
assert len(problem) == 2
assert isinstance(problem, list)
# Without forcing, we cannot write the same problem twice
internal_ramp_config = generate_ramp_config(ramp_configs[problem_name])
err_msg = 'Attempting to overwrite a problem and delete all linked events'
with pytest.raises(ValueError, match=err_msg):
add_problem(
session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'],
force=False
)
# Force add the problem
add_problem(
session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'],
force=True
)
problem = get_problem(session_scope_function, problem_name)
assert problem.name == problem_name
assert isinstance(problem, Problem)
delete_problem(session_scope_function, problem_name)
problem = get_problem(session_scope_function, problem_name)
assert problem is None
problem = get_problem(session_scope_function, None)
assert len(problem) == 1
assert isinstance(problem, list)
def __init__(self,
config,
event_config,
worker=None,
n_workers=1,
n_threads=None,
hunger_policy=None):
self.worker = CondaEnvWorker if worker is None else worker
self.n_workers = (max(multiprocessing.cpu_count() + 1 +
n_workers, 1) if n_workers < 0 else n_workers)
self.hunger_policy = hunger_policy
# init the poison pill to kill the dispatcher
self._poison_pill = False
# create the different dispatcher queues
self._awaiting_worker_queue = Queue()
self._processing_worker_queue = LifoQueue(maxsize=self.n_workers)
self._processed_submission_queue = Queue()
# split the different configuration required
if (isinstance(config, str) and isinstance(event_config, str)):
self._database_config = read_config(config,
filter_section='sqlalchemy')
self._ramp_config = generate_ramp_config(event_config, config)
else:
self._database_config = config['sqlalchemy']
self._ramp_config = event_config['ramp']
self._worker_config = generate_worker_config(event_config, config)
# set the number of threads for openmp, openblas, and mkl
self.n_threads = n_threads
if self.n_threads is not None:
if not isinstance(self.n_threads, numbers.Integral):
raise TypeError(
"The parameter 'n_threads' should be a positive integer. "
"Got {} instead.".format(repr(self.n_threads)))
for lib in ('OMP', 'MKL', 'OPENBLAS'):
os.environ[lib + '_NUM_THREADS'] = str(self.n_threads)
def delete_event(config, config_event, dry_run, from_disk, force):
"""Delete event."""
internal_config = read_config(config)
ramp_config = generate_ramp_config(config_event, config)
event_name = ramp_config["event_name"]
with session_scope(internal_config['sqlalchemy']) as session:
db_event = event_module.get_event(session, event_name)
if db_event:
if not dry_run:
event_module.delete_event(session, event_name)
click.echo('{} was removed from the database'.format(event_name))
if from_disk:
if not db_event and not force:
err_msg = ('{} event not found in the database. If you want '
'to force removing event files from the disk, add '
'the option "--force".'.format(event_name))
raise click.ClickException(err_msg)
for key in ("ramp_submissions_dir", "ramp_predictions_dir",
"ramp_logs_dir"):
dir_to_remove = ramp_config[key]
if os.path.exists(dir_to_remove):
if not dry_run:
shutil.rmtree(dir_to_remove)
click.echo("Removed directory:\n{}".format(dir_to_remove))
else:
click.echo("Directory not found. Skip removal for the "
"directory:\n{}".format(dir_to_remove))
event_dir = os.path.dirname(config_event)
if not dry_run:
shutil.rmtree(event_dir)
click.echo("Removed directory:\n{}".format(event_dir))
def test_generate_ramp_config_short():
ramp_config = generate_ramp_config(_get_event_config('short'),
database_config_template())
expected_config = {
'problem_name':
'iris',
'event_name':
'iris_test',
'event_title':
'Iris event',
'ramp_kit_dir':
os.path.join('template', 'ramp-kits', 'iris'),
'ramp_data_dir':
os.path.join('template', 'ramp-data', 'iris'),
'ramp_submissions_dir':
os.path.join('template', 'events', 'iris_test', 'submissions'),
'sandbox_name':
'starting_kit',
'ramp_predictions_dir':
os.path.join('template', 'events', 'iris_test', 'predictions'),
'ramp_logs_dir':
os.path.join('template', 'events', 'iris_test', 'logs'),
'ramp_sandbox_dir':
os.path.join('template', 'ramp-kits', 'iris', 'submissions',
'starting_kit'),
'ramp_kit_submissions_dir':
os.path.join('template', 'ramp-kits', 'iris', 'submissions')
}
for key in expected_config:
assert expected_config[key] in ramp_config[key]
def submit_all_starting_kits(session):
"""Submit all starting kits.
Parameters
----------
session : :class:`sqlalchemy.orm.Session`
The session to directly perform the operation on the database.
"""
ramp_configs = {
'iris': read_config(ramp_config_iris()),
'iris_aws': read_config(ramp_config_aws_iris()),
'boston_housing': read_config(ramp_config_boston_housing())
}
for problem_name, ramp_config in ramp_configs.items():
ramp_config_problem = generate_ramp_config(ramp_config)
path_submissions = os.path.join(
ramp_config_problem['ramp_kit_dir'], 'submissions'
)
submit_starting_kits(
session, ramp_config_problem['event_name'], 'test_user',
path_submissions
)
submit_starting_kits(
session, ramp_config_problem['event_name'], 'test_user_2',
path_submissions
)
def test_ramp_kit_ramp_data(session_scope_function, ramp_config):
internal_ramp_config = generate_ramp_config(read_config(ramp_config))
setup_ramp_kit_ramp_data(internal_ramp_config,
'iris',
depth=1,
mock_html_conversion=True)
msg_err = 'The RAMP kit repository was previously cloned.'
with pytest.raises(ValueError, match=msg_err):
setup_ramp_kit_ramp_data(internal_ramp_config,
'iris',
depth=1,
mock_html_conversion=True)
# retrieve the path to the ramp kit to remove it
shutil.rmtree(internal_ramp_config['ramp_kit_dir'])
msg_err = 'The RAMP data repository was previously cloned.'
with pytest.raises(ValueError, match=msg_err):
setup_ramp_kit_ramp_data(internal_ramp_config,
'iris',
depth=1,
mock_html_conversion=True)
setup_ramp_kit_ramp_data(internal_ramp_config,
'iris',
force=True,
mock_html_conversion=True)
def test_generate_ramp_config(config):
ramp_config = generate_ramp_config(config)
expected_config = {
'event':
'iris',
'event_name':
'iris_test',
'event_title':
'Iris event',
'event_is_public':
True,
'sandbox_name':
'starting_kit',
'deployment_dir':
'/tmp/databoard_test',
'ramp_kits_dir':
os.path.join('/tmp/databoard_test', 'ramp-kits'),
'ramp_data_dir':
os.path.join('/tmp/databoard_test', 'ramp-data'),
'ramp_kit_submissions_dir':
os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris',
'submissions'),
'ramp_submissions_dir':
os.path.join('/tmp/databoard_test', 'submissions'),
'ramp_sandbox_dir':
os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris', 'submissions',
'starting_kit')
}
assert ramp_config == expected_config
def add_events(session):
"""Add events in the database.
Parameters
----------
session : :class:`sqlalchemy.orm.Session`
The session to directly perform the operation on the database.
Notes
-----
Be aware that :func:`add_problems` needs to be called before.
"""
ramp_configs = {
'iris': read_config(ramp_config_iris()),
'boston_housing': read_config(ramp_config_boston_housing())
}
for problem_name, ramp_config in ramp_configs.items():
ramp_config_problem = generate_ramp_config(ramp_config)
add_event(
session,
problem_name=problem_name,
event_name=ramp_config_problem['event_name'],
event_title=ramp_config_problem['event_title'],
ramp_sandbox_name=ramp_config_problem['sandbox_name'],
ramp_submissions_path=ramp_config_problem['ramp_submissions_dir'],
is_public=True,
force=False)
def test_deploy_ramp_event_options(session_scope_function):
database_config = read_config(database_config_template())
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
deploy_ramp_event(database_config_template(), ramp_config_template())
# deploy again by forcing the deployment
deploy_ramp_event(database_config_template(),
ramp_config_template(),
force=True)
# do not deploy the kit to trigger the error in the problem with we don't
# force the deployment
msg_err = 'The RAMP problem already exists in the database.'
with pytest.raises(ValueError, match=msg_err):
with session_scope(database_config['sqlalchemy']) as session:
problem = get_problem(session, 'iris')
problem.path_ramp_kit = problem.path_ramp_kit + '_xxx'
session.commit()
deploy_ramp_event(database_config_template(),
ramp_config_template(),
setup_ramp_repo=False,
force=False)
problem = get_problem(session, 'iris')
problem.path_ramp_kit = ramp_config['ramp_kit_dir']
problem.path_ramp_data = problem.path_ramp_data + '_xxx'
session.commit()
deploy_ramp_event(database_config_template(),
ramp_config_template(),
setup_ramp_repo=False,
force=False)
def add_problems(session, ramp_config):
"""Add dummy problems into the database. In addition, we add couple of
keyword.
Parameters
----------
session : :class:`sqlalchemy.orm.Session`
The session to directly perform the operation on the database.
ramp_config : dict
The configuration file containing the information about a RAMP event.
"""
problems = ['iris', 'boston_housing']
for problem_name in problems:
setup_ramp_kits_ramp_data(ramp_config, problem_name)
internal_ramp_config = generate_ramp_config(ramp_config)
add_problem(session, problem_name,
internal_ramp_config['ramp_kits_dir'],
internal_ramp_config['ramp_data_dir'])
add_keyword(session,
problem_name,
'data_domain',
category='scientific data')
add_problem_keyword(session,
problem_name=problem_name,
keyword_name=problem_name)
add_keyword(session,
problem_name + '_theme',
'data_science_theme',
category='classification')
add_problem_keyword(session,
problem_name=problem_name,
keyword_name=problem_name + '_theme')
def add_problems(session):
"""Add dummy problems into the database. In addition, we add couple of
keyword.
Parameters
----------
session : :class:`sqlalchemy.orm.Session`
The session to directly perform the operation on the database.
"""
ramp_configs = {
'iris': read_config(ramp_config_iris()),
'boston_housing': read_config(ramp_config_boston_housing())
}
for problem_name, ramp_config in ramp_configs.items():
internal_ramp_config = generate_ramp_config(ramp_config)
setup_ramp_kit_ramp_data(internal_ramp_config, problem_name)
add_problem(session, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'])
add_keyword(session,
problem_name,
'data_domain',
category='scientific data')
add_problem_keyword(session,
problem_name=problem_name,
keyword_name=problem_name)
add_keyword(session,
problem_name + '_theme',
'data_science_theme',
category='classification')
add_problem_keyword(session,
problem_name=problem_name,
keyword_name=problem_name + '_theme')
def test_generate_ramp_config(event_config, database_config):
ramp_config = generate_ramp_config(event_config, database_config)
expected_config = {
'problem_name':
'iris',
'event_name':
'iris_test',
'event_title':
'Iris event',
'event_is_public':
True,
'sandbox_name':
'starting_kit',
'ramp_kit_dir':
os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris'),
'ramp_data_dir':
os.path.join('/tmp/databoard_test', 'ramp-data', 'iris'),
'ramp_kit_submissions_dir':
os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris',
'submissions'),
'ramp_submissions_dir':
os.path.join('/tmp/databoard_test', 'submissions'),
'ramp_sandbox_dir':
os.path.join('/tmp/databoard_test', 'ramp-kits', 'iris', 'submissions',
'starting_kit'),
'ramp_logs_dir':
os.path.join('/tmp/databoard_test', 'log'),
'ramp_predictions_dir':
os.path.join('/tmp/databoard_test', 'preds')
}
assert ramp_config == expected_config
def test_add_submission_create_new_submission(base_db):
# check that we can make a new submission to the database
# it will require to have already a team and an event
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
add_submission(session, event_name, username, submission_name,
path_submission)
all_submissions = get_submissions(session, event_name, None)
# check that the submissions have been copied
for sub_id, _, _ in all_submissions:
sub = get_submission_by_id(session, sub_id)
assert os.path.exists(sub.path)
assert os.path.exists(os.path.join(sub.path, 'classifier.py'))
# `sign_up_team` make a submission (sandbox) by user. This submission will
# be the third submission.
assert len(all_submissions) == 3
# check that the number of submissions for an event was updated
event = session.query(Event).filter(Event.name == event_name).one_or_none()
assert event.n_submissions == 1
submission = get_submission_by_name(session, event_name, username,
submission_name)
assert submission.name == submission_name
submission_file = submission.files[0]
assert submission_file.name == 'classifier'
assert submission_file.extension == 'py'
assert (os.path.join('submission_000000005', 'classifier.py')
in submission_file.path)
def test_add_submission_wrong_submission_files(base_db):
# check that we raise an error if the file required by the workflow is not
# present in the submission or that it has the wrong extension
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submission_name = 'corrupted_submission'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
os.makedirs(path_submission)
# case that there is not files in the submission
err_msg = 'No file corresponding to the workflow element'
with pytest.raises(MissingSubmissionFileError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# case that there is not file corresponding to the workflow component
filename = os.path.join(path_submission, 'unknown_file.xxx')
open(filename, "w+").close()
err_msg = 'No file corresponding to the workflow element'
with pytest.raises(MissingSubmissionFileError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# case that we have the correct filename but not the right extension
filename = os.path.join(path_submission, 'classifier.xxx')
open(filename, "w+").close()
err_msg = 'All extensions "xxx" are unknown for the submission'
with pytest.raises(MissingExtensionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
def test_add_submission_too_early_submission(base_db):
# check that we raise an error when the elapsed time was not large enough
# between the new submission and the previous submission
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
# check that we have an awaiting time for the event
event = (session.query(Event).filter(
Event.name == event_name).one_or_none())
assert event.min_duration_between_submissions == 900
# make 2 submissions which are too close from each other
for submission_idx, submission_name in enumerate(
['random_forest_10_10', 'too_early_submission']):
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
if submission_idx == 1:
err_msg = 'You need to wait'
with pytest.raises(TooEarlySubmissionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
else:
add_submission(session, event_name, username, submission_name,
path_submission)
def create_test_db(database_config, ramp_config):
"""Create an empty test database and the setup the files for RAMP.
Note: this will forcedly remove any existing content in the deployment
directory.
Parameters
----------
database_config : dict
The configuration file containing the database information.
ramp_config : dict
The configuration file containing the information about a RAMP event.
"""
database_config = database_config['sqlalchemy']
# we can automatically setup the database from the config file used for the
# tests.
ramp_config = generate_ramp_config(ramp_config)
shutil.rmtree(ramp_config['deployment_dir'], ignore_errors=True)
os.makedirs(ramp_config['ramp_kits_dir'])
os.makedirs(ramp_config['ramp_data_dir'])
os.makedirs(ramp_config['ramp_submissions_dir'])
db, _ = setup_db(database_config)
Model.metadata.drop_all(db)
Model.metadata.create_all(db)
with session_scope(database_config) as session:
setup_files_extension_type(session)
def __init__(self,
config,
event_config,
worker=None,
n_worker=1,
hunger_policy=None):
self.worker = CondaEnvWorker if worker is None else worker
self.n_worker = (max(multiprocessing.cpu_count() + 1 +
n_worker, 1) if n_worker < 0 else n_worker)
self.hunger_policy = hunger_policy
# init the poison pill to kill the dispatcher
self._poison_pill = False
# create the different dispatcher queues
self._awaiting_worker_queue = Queue()
self._processing_worker_queue = LifoQueue(maxsize=self.n_worker)
self._processed_submission_queue = Queue()
# split the different configuration required
if (isinstance(config, six.string_types)
and isinstance(event_config, six.string_types)):
self._database_config = read_config(config,
filter_section='sqlalchemy')
self._ramp_config = generate_ramp_config(event_config, config)
else:
self._database_config = config['sqlalchemy']
self._ramp_config = event_config['ramp']
self._worker_config = generate_worker_config(event_config, config)
def add_events(session, ramp_config):
"""Add events in the database.
Parameters
----------
session : :class:`sqlalchemy.orm.Session`
The session to directly perform the operation on the database.
ramp_config : dict
The configuration file containing the information about a RAMP event.
Notes
-----
Be aware that :func:`add_problems` needs to be called before.
"""
ramp_config = generate_ramp_config(ramp_config)
problems = ['iris', 'boston_housing']
for problem_name in problems:
event_name = '{}_test'.format(problem_name)
event_title = 'test event'
add_event(session,
problem_name=problem_name,
event_name=event_name,
event_title=event_title,
ramp_sandbox_name=ramp_config['sandbox_name'],
ramp_submissions_path=ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
def test_deploy_ramp_event(session_scope_function):
database_config = read_config(database_config_template())
event_config_filename = ramp_config_template()
event_config = read_config(event_config_filename)
ramp_config = generate_ramp_config(event_config)
deploy_ramp_event(database_config_template(), ramp_config_template())
# simulate that we add users and sign-up for the event and that they
# submitted the starting kit
with session_scope(database_config['sqlalchemy']) as session:
add_users(session)
sign_up_team(session, ramp_config['event_name'], 'test_user')
submit_starting_kits(session, ramp_config['event_name'], 'test_user',
ramp_config['ramp_kit_submissions_dir'])
# run the dispatcher on the event which are in the dataset
dispatcher = Dispatcher(config=database_config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for a user
with session_scope(database_config['sqlalchemy']) as session:
submission = get_submissions(session,
event_config['ramp']['event_name'],
'training_error')
assert len(submission) == 1
def test_add_submission_create_new_submission(base_db):
# check that we can make a new submission to the database
# it will require to have already a team and an event
session = base_db
config = read_config(ramp_config_template())
event_name, username = _setup_sign_up(session, config)
ramp_config = generate_ramp_config(config)
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
add_submission(session, event_name, username, submission_name,
path_submission)
all_submissions = get_submissions(session, event_name, None)
# `sign_up_team` make a submission (sandbox) by user. This submission will
# be the third submission.
assert len(all_submissions) == 3
submission = get_submission_by_name(session, event_name, username,
submission_name)
assert submission.name == submission_name
submission_file = submission.files[0]
assert submission_file.name == 'classifier'
assert submission_file.extension == 'py'
assert (os.path.join('submission_000000005', 'classifier.py')
in submission_file.path)
def test_check_event(session_scope_function):
config = read_config(ramp_config_template())
# addition of event require some problem
problem_names = ['iris', 'boston_housing']
for problem_name in problem_names:
setup_ramp_kits_ramp_data(config, problem_name)
ramp_config = generate_ramp_config(config)
add_problem(session_scope_function, problem_name,
ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir'])
for problem_name in problem_names:
event_name = '{}_test'.format(problem_name)
event_title = 'event title'
add_event(session_scope_function,
problem_name,
event_name,
event_title,
ramp_config['sandbox_name'],
ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
event = get_event(session_scope_function, None)
assert len(event) == 2
assert isinstance(event, list)
event = get_event(session_scope_function, 'iris_test')
scores_iris = ('acc', 'error', 'nll', 'f1_70')
_check_event(session_scope_function, event, 'iris_test', 'event title',
True, scores_iris)
# add event for second time without forcing should raise an error
err_msg = 'Attempting to overwrite existing event.'
with pytest.raises(ValueError, match=err_msg):
add_event(session_scope_function,
'iris',
'iris_test',
event_title,
ramp_config['sandbox_name'],
ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
# add event by force
add_event(session_scope_function,
'iris',
'iris_test',
event_title,
ramp_config['sandbox_name'],
ramp_config['ramp_submissions_dir'],
is_public=True,
force=True)
event = get_event(session_scope_function, 'iris_test')
_check_event(session_scope_function, event, 'iris_test', 'event title',
True, scores_iris)
delete_event(session_scope_function, 'iris_test')
event = get_event(session_scope_function, None)
assert len(event) == 1
def test_add_problem():
runner = CliRunner()
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
result = runner.invoke(main, [
'add-problem', '--config',
database_config_template(), '--problem', 'iris', '--kit-dir',
ramp_config['ramp_kit_dir'], '--data-dir',
ramp_config['ramp_data_dir'], '--force', True
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_sandbox_upload_file(client_session, makedrop_event, submission_dir,
filename):
client, session = client_session
sign_up_team(session, "iris_test_4event", "test_user")
config = ramp_config_template()
ramp_config = generate_ramp_config(read_config(config))
# upload file in sandbox.html
path_submissions = os.path.join(ramp_config["ramp_kit_dir"],
submission_dir)
with login_scope(client, "test_user", "test") as client:
rv = client.get("http://localhost/events/iris_test_4event/sandbox")
assert rv.status_code == 200
# choose file and check if it was uploaded correctly
path_submission = os.path.join(path_submissions, filename)
assert os.path.isfile(path_submission)
rv = client.post(
"http://localhost/events/iris_test_4event/sandbox",
headers={
"Referer": "http://localhost/events/iris_test_4event/sandbox"
},
data={"file": (open(path_submission, "rb"), filename)},
follow_redirects=False,
)
assert rv.status_code == 302
assert (
rv.location == "http://localhost/events/iris_test_4event/sandbox")
# code of the saved file
with open(path_submission, "r") as file:
submitted_data = file.read()
# code from the db
event = get_event(session, "iris_test_4event")
sandbox_submission = get_submission_by_name(session,
"iris_test_4event",
"test_user",
event.ramp_sandbox_name)
submission_code = sandbox_submission.files[-1].get_code()
# get user interactions from db and check if 'upload' was added
user_interactions = get_user_interactions_by_name(session, "test_user")
# check if the code of the submitted file in the 'submission_code'
assert submitted_data is not None
assert submitted_data in submission_code
# check if the user_interaction was added to the db
assert "upload" in user_interactions["interaction"].values
def setup_ramp_kits_ramp_data(ramp_config, problem_name, force=False):
"""Clone ramp-kits and ramp-data repository and setup it up.
Parameters
----------
ramp_config : dict
The configuration file containing the information about a RAMP event.
problem_name : str
The name of the problem.
force : bool, default is False
Whether or not to overwrite the RAMP kit and data repositories if they
already exists.
"""
ramp_config = generate_ramp_config(ramp_config)
problem_kits_path = os.path.join(ramp_config['ramp_kits_dir'],
problem_name)
if not os.path.exists(ramp_config['ramp_kits_dir']):
os.makedirs(ramp_config['ramp_kits_dir'])
if os.path.exists(problem_kits_path):
if not force:
raise ValueError(
'The RAMP kit repository was previously cloned. To replace '
'it, you need to set "force=True".')
shutil.rmtree(problem_kits_path, ignore_errors=True)
ramp_kits_url = 'https://github.com/ramp-kits/{}.git'.format(problem_name)
Repo.clone_from(ramp_kits_url, problem_kits_path)
problem_data_path = os.path.join(ramp_config['ramp_data_dir'],
problem_name)
if not os.path.exists(ramp_config['ramp_data_dir']):
os.makedirs(ramp_config['ramp_data_dir'])
if os.path.exists(problem_data_path):
if not force:
raise ValueError(
'The RAMP data repository was previously cloned. To replace '
'it, you need to set "force=True".')
shutil.rmtree(problem_data_path, ignore_errors=True)
ramp_data_url = 'https://github.com/ramp-data/{}.git'.format(problem_name)
Repo.clone_from(ramp_data_url, problem_data_path)
current_directory = os.getcwd()
os.chdir(problem_data_path)
subprocess.check_output(["python", "prepare_data.py"])
os.chdir(problem_kits_path)
subprocess.check_output([
"jupyter", "nbconvert", "--to", "html",
"{}_starting_kit.ipynb".format(problem_name)
])
# delete this line since it trigger in the front-end
# (try to open execute "custom.css".)
_delete_line_from_file("{}_starting_kit.html".format(problem_name),
'<link rel="stylesheet" href="custom.css">\n')
os.chdir(current_directory)
def test_add_event():
runner = CliRunner()
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
result = runner.invoke(main, [
'add-event', '--config',
database_config_template(), '--problem', 'iris', '--event',
'iris_test', '--title', 'Iris classification', '--sandbox',
ramp_config['sandbox_name'], '--submissions-dir',
ramp_config['ramp_submissions_dir'], '--is-public', False, '--force',
True
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def create_conda_env(config, event_config):
"""Create the conda environment for a specific event"""
conda_env_name = read_config(event_config)["worker"]["conda_env"]
ramp_config = generate_ramp_config(event_config, database_config=config)
path_environment_file = os.path.join(
ramp_config["ramp_kit_dir"], "environment.yml"
)
subprocess.run(
["conda", "create", "--name", conda_env_name, "--yes"]
)
subprocess.run(
["conda", "env", "update",
"--name", conda_env_name,
"--file", path_environment_file]
)
def test_make_submission_resubmission(base_db):
# check that resubmitting the a submission with the same name will raise
# an error
session = base_db
config = ramp_config_template()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
# submitting the starting_kit which is used as the default submission for
# the sandbox should raise an error
err_msg = ('Submission "starting_kit" of team "test_user" at event '
'"iris_test" exists already')
with pytest.raises(DuplicateSubmissionError, match=err_msg):
add_submission(session, event_name, username,
os.path.basename(ramp_config['ramp_sandbox_dir']),
ramp_config['ramp_sandbox_dir'])
# submitting twice a normal submission should raise an error as well
submission_name = 'random_forest_10_10'
path_submission = os.path.join(
os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name)
# first submission
add_submission(
session,
event_name,
username,
submission_name,
path_submission,
)
# mock that we scored the submission
set_submission_state(session, 5, 'scored')
# second submission
err_msg = ('Submission "random_forest_10_10" of team "test_user" at event '
'"iris_test" exists already')
with pytest.raises(DuplicateSubmissionError, match=err_msg):
add_submission(session, event_name, username, submission_name,
path_submission)
# a resubmission can take place if it is tagged as "new" or failed
# mock that the submission failed during the training
set_submission_state(session, 5, 'training_error')
add_submission(session, event_name, username, submission_name,
path_submission)
# mock that the submissions are new submissions
set_submission_state(session, 5, 'new')
add_submission(session, event_name, username, submission_name,
path_submission)
def test_add_submission():
ramp_config = generate_ramp_config(read_config(ramp_config_template()))
submission_name = 'new_submission'
submission_path = os.path.join(ramp_config['ramp_kit_submissions_dir'],
submission_name)
shutil.copytree(
os.path.join(ramp_config['ramp_kit_submissions_dir'],
'random_forest_10_10'), submission_path)
runner = CliRunner()
result = runner.invoke(main, [
'add-submission', '--config',
database_config_template(), '--event', 'iris_test', '--team',
'glemaitre', '--submission', submission_name, '--path', submission_path
],
catch_exceptions=False)
assert result.exit_code == 0, result.output
def test_submit_starting_kits(base_db):
session = base_db
config = ramp_config_iris()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submit_starting_kits(session, event_name, username,
ramp_config['ramp_kit_submissions_dir'])
submissions = get_submissions(session, event_name, None)
submissions_id = [sub[0] for sub in submissions]
assert len(submissions) == 5
expected_submission_name = {'starting_kit', 'starting_kit_test',
'random_forest_10_10', 'error'}
submission_name = {get_submission_by_id(session, sub_id).name
for sub_id in submissions_id}
assert submission_name == expected_submission_name
def submit_all_starting_kits(session, ramp_config):
"""Submit all starting kits.
Parameters
----------
session : :class:`sqlalchemy.orm.Session`
The session to directly perform the operation on the database.
ramp_config : dict
The configuration file containing the information about a RAMP event.
"""
ramp_config = generate_ramp_config(ramp_config)
for event, event_name in zip(['iris', 'boston_housing'],
['iris_test', 'boston_housing_test']):
path_submissions = os.path.join(ramp_config['ramp_kits_dir'], event,
'submissions')
submit_starting_kits(session, event_name, 'test_user',
path_submissions)
submit_starting_kits(session, event_name, 'test_user_2',
path_submissions)