def test_check_problem(session_scope_function):
ramp_configs = {
'iris': read_config(ramp_config_iris()),
'boston_housing': read_config(ramp_config_boston_housing())
}
for problem_name, ramp_config in ramp_configs.items():
internal_ramp_config = generate_ramp_config(ramp_config)
setup_ramp_kit_ramp_data(internal_ramp_config, problem_name)
add_problem(session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'])
problem_name = 'iris'
problem = get_problem(session_scope_function, problem_name)
assert problem.name == problem_name
assert isinstance(problem, Problem)
problem = get_problem(session_scope_function, None)
assert len(problem) == 2
assert isinstance(problem, list)
# Without forcing, we cannot write the same problem twice
internal_ramp_config = generate_ramp_config(ramp_configs[problem_name])
err_msg = 'Attempting to overwrite a problem and delete all linked events'
with pytest.raises(ValueError, match=err_msg):
add_problem(
session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'],
force=False
)
# Force add the problem
add_problem(
session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'],
force=True
)
problem = get_problem(session_scope_function, problem_name)
assert problem.name == problem_name
assert isinstance(problem, Problem)
delete_problem(session_scope_function, problem_name)
problem = get_problem(session_scope_function, problem_name)
assert problem is None
problem = get_problem(session_scope_function, None)
assert len(problem) == 1
assert isinstance(problem, list)
def test_submit_starting_kits(base_db):
session = base_db
config = ramp_config_iris()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submit_starting_kits(session, event_name, username,
ramp_config['ramp_kit_submissions_dir'])
submissions = get_submissions(session, event_name, None)
submissions_id = [sub[0] for sub in submissions]
assert len(submissions) == 5
expected_submission_name = {'starting_kit', 'starting_kit_test',
'random_forest_10_10', 'error'}
submission_name = {get_submission_by_id(session, sub_id).name
for sub_id in submissions_id}
assert submission_name == expected_submission_name
def test_check_event(session_scope_function):
ramp_configs = {
'iris': read_config(ramp_config_iris()),
'boston_housing': read_config(ramp_config_boston_housing())
}
for problem_name, ramp_config in ramp_configs.items():
internal_ramp_config = generate_ramp_config(ramp_config)
setup_ramp_kit_ramp_data(internal_ramp_config, problem_name)
add_problem(session_scope_function, problem_name,
internal_ramp_config['ramp_kit_dir'],
internal_ramp_config['ramp_data_dir'])
for problem_name, ramp_config in ramp_configs.items():
internal_ramp_config = generate_ramp_config(ramp_config)
add_event(session_scope_function,
problem_name,
internal_ramp_config['event_name'],
internal_ramp_config['event_title'],
internal_ramp_config['sandbox_name'],
internal_ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
event = get_event(session_scope_function, None)
assert len(event) == 2
assert isinstance(event, list)
problem_name = 'iris'
internal_ramp_config = generate_ramp_config(ramp_configs[problem_name])
event = get_event(session_scope_function,
internal_ramp_config['event_name'])
scores_iris = ('acc', 'error', 'nll', 'f1_70')
_check_event(session_scope_function, event,
internal_ramp_config['event_name'],
internal_ramp_config['event_title'], True, scores_iris)
# add event for second time without forcing should raise an error
err_msg = 'Attempting to overwrite existing event.'
with pytest.raises(ValueError, match=err_msg):
add_event(session_scope_function,
problem_name,
internal_ramp_config['event_name'],
internal_ramp_config['event_title'],
internal_ramp_config['sandbox_name'],
internal_ramp_config['ramp_submissions_dir'],
is_public=True,
force=False)
# add event by force
add_event(session_scope_function,
problem_name,
internal_ramp_config['event_name'],
internal_ramp_config['event_title'],
internal_ramp_config['sandbox_name'],
internal_ramp_config['ramp_submissions_dir'],
is_public=True,
force=True)
event = get_event(session_scope_function,
internal_ramp_config['event_name'])
_check_event(session_scope_function, event,
internal_ramp_config['event_name'],
internal_ramp_config['event_title'], True, scores_iris)
delete_event(session_scope_function, internal_ramp_config['event_name'])
event = get_event(session_scope_function, None)
assert len(event) == 1
def test_ramp_config_iris():
filename = ramp_config_iris()
assert os.path.join('tests', 'data', 'ramp_config_iris.yml') in filename