def test_check_event(session_scope_function): config = read_config(ramp_config_template()) # addition of event require some problem problem_names = ['iris', 'boston_housing'] for problem_name in problem_names: setup_ramp_kits_ramp_data(config, problem_name) ramp_config = generate_ramp_config(config) add_problem(session_scope_function, problem_name, ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir']) for problem_name in problem_names: event_name = '{}_test'.format(problem_name) event_title = 'event title' add_event(session_scope_function, problem_name, event_name, event_title, ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], is_public=True, force=False) event = get_event(session_scope_function, None) assert len(event) == 2 assert isinstance(event, list) event = get_event(session_scope_function, 'iris_test') scores_iris = ('acc', 'error', 'nll', 'f1_70') _check_event(session_scope_function, event, 'iris_test', 'event title', True, scores_iris) # add event for second time without forcing should raise an error err_msg = 'Attempting to overwrite existing event.' with pytest.raises(ValueError, match=err_msg): add_event(session_scope_function, 'iris', 'iris_test', event_title, ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], is_public=True, force=False) # add event by force add_event(session_scope_function, 'iris', 'iris_test', event_title, ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], is_public=True, force=True) event = get_event(session_scope_function, 'iris_test') _check_event(session_scope_function, event, 'iris_test', 'event title', True, scores_iris) delete_event(session_scope_function, 'iris_test') event = get_event(session_scope_function, None) assert len(event) == 1
def test_check_problem(session_scope_function): ramp_configs = { 'iris': read_config(ramp_config_iris()), 'boston_housing': read_config(ramp_config_boston_housing()) } for problem_name, ramp_config in ramp_configs.items(): internal_ramp_config = generate_ramp_config(ramp_config) setup_ramp_kit_ramp_data(internal_ramp_config, problem_name) add_problem(session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir']) problem_name = 'iris' problem = get_problem(session_scope_function, problem_name) assert problem.name == problem_name assert isinstance(problem, Problem) problem = get_problem(session_scope_function, None) assert len(problem) == 2 assert isinstance(problem, list) # Without forcing, we cannot write the same problem twice internal_ramp_config = generate_ramp_config(ramp_configs[problem_name]) err_msg = 'Attempting to overwrite a problem and delete all linked events' with pytest.raises(ValueError, match=err_msg): add_problem( session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir'], force=False ) # Force add the problem add_problem( session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir'], force=True ) problem = get_problem(session_scope_function, problem_name) assert problem.name == problem_name assert isinstance(problem, Problem) delete_problem(session_scope_function, problem_name) problem = get_problem(session_scope_function, problem_name) assert problem is None problem = get_problem(session_scope_function, None) assert len(problem) == 1 assert isinstance(problem, list)
def test_check_problem(session_scope_function): config = read_config(ramp_config_template()) problem_names = ['iris', 'boston_housing'] for problem_name in problem_names: setup_ramp_kits_ramp_data(config, problem_name) ramp_config = generate_ramp_config(config) add_problem(session_scope_function, problem_name, ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir']) problem = get_problem(session_scope_function, problem_names[0]) assert problem.name == problem_names[0] assert isinstance(problem, Problem) problem = get_problem(session_scope_function, None) assert len(problem) == 2 assert isinstance(problem, list) # Without forcing, we cannot write the same problem twice err_msg = 'Attempting to overwrite a problem and delete all linked events' with pytest.raises(ValueError, match=err_msg): add_problem(session_scope_function, problem_names[0], ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir'], force=False) # Force add the problem add_problem(session_scope_function, problem_names[0], ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir'], force=True) problem = get_problem(session_scope_function, problem_names[0]) assert problem.name == problem_names[0] assert isinstance(problem, Problem) delete_problem(session_scope_function, problem_names[0]) problem = get_problem(session_scope_function, problem_names[0]) assert problem is None problem = get_problem(session_scope_function, None) assert len(problem) == 1 assert isinstance(problem, list)
def test_check_event(session_scope_function): ramp_configs = { 'iris': read_config(ramp_config_iris()), 'boston_housing': read_config(ramp_config_boston_housing()) } for problem_name, ramp_config in ramp_configs.items(): internal_ramp_config = generate_ramp_config(ramp_config) setup_ramp_kit_ramp_data(internal_ramp_config, problem_name) add_problem(session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir']) for problem_name, ramp_config in ramp_configs.items(): internal_ramp_config = generate_ramp_config(ramp_config) add_event(session_scope_function, problem_name, internal_ramp_config['event_name'], internal_ramp_config['event_title'], internal_ramp_config['sandbox_name'], internal_ramp_config['ramp_submissions_dir'], is_public=True, force=False) event = get_event(session_scope_function, None) assert len(event) == 2 assert isinstance(event, list) problem_name = 'iris' internal_ramp_config = generate_ramp_config(ramp_configs[problem_name]) event = get_event(session_scope_function, internal_ramp_config['event_name']) scores_iris = ('acc', 'error', 'nll', 'f1_70') _check_event(session_scope_function, event, internal_ramp_config['event_name'], internal_ramp_config['event_title'], True, scores_iris) # add event for second time without forcing should raise an error err_msg = 'Attempting to overwrite existing event.' with pytest.raises(ValueError, match=err_msg): add_event(session_scope_function, problem_name, internal_ramp_config['event_name'], internal_ramp_config['event_title'], internal_ramp_config['sandbox_name'], internal_ramp_config['ramp_submissions_dir'], is_public=True, force=False) # add event by force add_event(session_scope_function, problem_name, internal_ramp_config['event_name'], internal_ramp_config['event_title'], internal_ramp_config['sandbox_name'], internal_ramp_config['ramp_submissions_dir'], is_public=True, force=True) event = get_event(session_scope_function, internal_ramp_config['event_name']) _check_event(session_scope_function, event, internal_ramp_config['event_name'], internal_ramp_config['event_title'], True, scores_iris) delete_event(session_scope_function, internal_ramp_config['event_name']) event = get_event(session_scope_function, None) assert len(event) == 1
def deploy_ramp_event(config, event_config, setup_ramp_repo=True, force=False): """Deploy a RAMP event using a configuration file. This utility is in charge of creating the kit and data repository for a given RAMP event. It will also setup the database. Parameters ---------- config : str The path to the YAML file containing the database information. event_config : str The path to the YAML file containing the RAMP infomation. setup_ramp_repo : bool, default is True Whether or not to setup the RAMP kit and data repositories. force : bool, default is False Whether or not to potentially overwrite the repositories, problem and event in the database. """ database_config = read_config(config, filter_section='sqlalchemy') ramp_config = generate_ramp_config(event_config, config) with session_scope(database_config) as session: setup_files_extension_type(session) if setup_ramp_repo: setup_ramp_kit_ramp_data(ramp_config, ramp_config['problem_name'], force) else: # we do not clone the repository but we need to convert the # notebook to html current_directory = os.getcwd() problem_kit_path = ramp_config['ramp_kit_dir'] os.chdir(problem_kit_path) subprocess.check_output([ "jupyter", "nbconvert", "--to", "html", "{}_starting_kit.ipynb".format(ramp_config['problem_name']) ]) # delete this line since it trigger in the front-end # (try to open execute "custom.css".) _delete_line_from_file( "{}_starting_kit.html".format(ramp_config['problem_name']), '<link rel="stylesheet" href="custom.css">\n') os.chdir(current_directory) # check if the repository exists problem = get_problem(session, ramp_config['problem_name']) if problem is None: add_problem(session, ramp_config['problem_name'], ramp_config['ramp_kit_dir'], ramp_config['ramp_data_dir']) else: if ((ramp_config['ramp_kit_dir'] != problem.path_ramp_kit or ramp_config['ramp_data_dir'] != problem.path_ramp_data) and not force): raise ValueError( 'The RAMP problem already exists in the database. The path' ' to the kit or to the data is different. You need to set' ' "force=True" if you want to overwrite these parameters.') if setup_ramp_repo: setup_ramp_kit_ramp_data(ramp_config, ramp_config['problem_name'], force) add_problem(session, ramp_config['problem_name'], ramp_config['ramp_kit_dir'], ramp_config['ramp_data_dir'], force) if not os.path.exists(ramp_config['ramp_submissions_dir']): os.makedirs(ramp_config['ramp_submissions_dir']) add_event(session, ramp_config['problem_name'], ramp_config['event_name'], ramp_config['event_title'], ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], ramp_config['event_is_public'], force)
def deploy_ramp_event(config, event_config, setup_ramp_repo=True, force=False): """Deploy a RAMP event using a configuration file. This utility is in charge of creating the kit and data repository for a given RAMP event. It will also setup the database. Parameters ---------- config : str The path to the YAML file containing the database information. event_config : str The path to the YAML file containing the RAMP infomation. setup_ramp_repo : bool, default is True Whether or not to setup the RAMP kit and data repositories. force : bool, default is False Whether or not to potentially overwrite the repositories, problem and event in the database. """ database_config = read_config(config, filter_section='sqlalchemy') event_config = read_config(event_config) ramp_config = generate_ramp_config(event_config) with session_scope(database_config) as session: setup_files_extension_type(session) if setup_ramp_repo: setup_ramp_kits_ramp_data( event_config, ramp_config['event'], force ) # check if the repository exists problem = get_problem(session, ramp_config['event']) if problem is None: add_problem(session, ramp_config['event'], ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir']) else: if ((ramp_config['ramp_kits_dir'] != problem.path_ramp_kits or ramp_config['ramp_data_dir'] != problem.path_ramp_data) and not force): raise ValueError( 'The RAMP problem already exists in the database. The path' 'to the kit or to the data is different. You need to set' '"force=True" if you want to overwrite these parameters.' ) if setup_ramp_repo: setup_ramp_kits_ramp_data( event_config, ramp_config['event'], force ) add_problem(session, ramp_config['event'], ramp_config['ramp_kits_dir'], ramp_config['ramp_data_dir'], force) if not os.path.exists(ramp_config['ramp_submissions_dir']): os.makedirs(ramp_config['ramp_submissions_dir']) add_event(session, ramp_config['event'], ramp_config['event_name'], ramp_config['event_title'], ramp_config['sandbox_name'], ramp_config['ramp_submissions_dir'], ramp_config['event_is_public'], force)