def test_deploy_ramp_event(session_scope_function): database_config = read_config(database_config_template()) event_config_filename = ramp_config_template() event_config = read_config(event_config_filename) ramp_config = generate_ramp_config(event_config) deploy_ramp_event(database_config_template(), ramp_config_template()) # simulate that we add users and sign-up for the event and that they # submitted the starting kit with session_scope(database_config['sqlalchemy']) as session: add_users(session) sign_up_team(session, ramp_config['event_name'], 'test_user') submit_starting_kits(session, ramp_config['event_name'], 'test_user', ramp_config['ramp_kit_submissions_dir']) # run the dispatcher on the event which are in the dataset dispatcher = Dispatcher(config=database_config, event_config=event_config, worker=CondaEnvWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for a user with session_scope(database_config['sqlalchemy']) as session: submission = get_submissions(session, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 1
def test_deploy_ramp_event_options(session_scope_function): database_config = read_config(database_config_template()) ramp_config = generate_ramp_config(read_config(ramp_config_template())) deploy_ramp_event(database_config_template(), ramp_config_template()) # deploy again by forcing the deployment deploy_ramp_event(database_config_template(), ramp_config_template(), force=True) # do not deploy the kit to trigger the error in the problem with we don't # force the deployment msg_err = 'The RAMP problem already exists in the database.' with pytest.raises(ValueError, match=msg_err): with session_scope(database_config['sqlalchemy']) as session: problem = get_problem(session, 'iris') problem.path_ramp_kit = problem.path_ramp_kit + '_xxx' session.commit() deploy_ramp_event(database_config_template(), ramp_config_template(), setup_ramp_repo=False, force=False) problem = get_problem(session, 'iris') problem.path_ramp_kit = ramp_config['ramp_kit_dir'] problem.path_ramp_data = problem.path_ramp_data + '_xxx' session.commit() deploy_ramp_event(database_config_template(), ramp_config_template(), setup_ramp_repo=False, force=False)
def add_events(session): """Add events in the database. Parameters ---------- session : :class:`sqlalchemy.orm.Session` The session to directly perform the operation on the database. Notes ----- Be aware that :func:`add_problems` needs to be called before. """ ramp_configs = { 'iris': read_config(ramp_config_iris()), 'boston_housing': read_config(ramp_config_boston_housing()) } for problem_name, ramp_config in ramp_configs.items(): ramp_config_problem = generate_ramp_config(ramp_config) add_event( session, problem_name=problem_name, event_name=ramp_config_problem['event_name'], event_title=ramp_config_problem['event_title'], ramp_sandbox_name=ramp_config_problem['sandbox_name'], ramp_submissions_path=ramp_config_problem['ramp_submissions_dir'], is_public=True, force=False)
def add_problems(session): """Add dummy problems into the database. In addition, we add couple of keyword. Parameters ---------- session : :class:`sqlalchemy.orm.Session` The session to directly perform the operation on the database. """ ramp_configs = { 'iris': read_config(ramp_config_iris()), 'boston_housing': read_config(ramp_config_boston_housing()) } for problem_name, ramp_config in ramp_configs.items(): internal_ramp_config = generate_ramp_config(ramp_config) setup_ramp_kit_ramp_data(internal_ramp_config, problem_name) add_problem(session, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir']) add_keyword(session, problem_name, 'data_domain', category='scientific data') add_problem_keyword(session, problem_name=problem_name, keyword_name=problem_name) add_keyword(session, problem_name + '_theme', 'data_science_theme', category='classification') add_problem_keyword(session, problem_name=problem_name, keyword_name=problem_name + '_theme')
def test_dispatcher_worker_retry(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=10, hunger_policy='exit') dispatcher.fetch_from_db(session_toy) dispatcher.launch_workers(session_toy) # Get one worker and set status to 'retry' worker, (submission_id, submission_name) = \ dispatcher._processing_worker_queue.get() setattr(worker, 'status', 'retry') assert worker.status == 'retry' # Add back to queue dispatcher._processing_worker_queue.put_nowait( (worker, (submission_id, submission_name))) while not dispatcher._processing_worker_queue.empty(): dispatcher.collect_result(session_toy) submissions = get_submissions(session_toy, 'iris_test', 'new') assert submission_name in [sub[1] for sub in submissions]
def test_error_handling_worker_setup_error(session_toy, caplog): # make sure the error on the worker.setup is dealt with correctly # set mock worker class Worker_mock(): def __init__(self, *args, **kwargs): self.state = None def setup(self): raise Exception('Test error') def teardown(self): pass config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) worker = Worker_mock() dispatcher = Dispatcher(config=config, event_config=event_config, worker=Worker_mock, n_workers=-1, hunger_policy='exit') dispatcher.launch() submissions = get_submissions(session_toy, event_config['ramp']['event_name'], 'checking_error') assert len(submissions) == 6 worker.status = 'error' assert 'Test error' in caplog.text
def submit_all_starting_kits(session): """Submit all starting kits. Parameters ---------- session : :class:`sqlalchemy.orm.Session` The session to directly perform the operation on the database. """ ramp_configs = { 'iris': read_config(ramp_config_iris()), 'iris_aws': read_config(ramp_config_aws_iris()), 'boston_housing': read_config(ramp_config_boston_housing()) } for problem_name, ramp_config in ramp_configs.items(): ramp_config_problem = generate_ramp_config(ramp_config) path_submissions = os.path.join( ramp_config_problem['ramp_kit_dir'], 'submissions' ) submit_starting_kits( session, ramp_config_problem['event_name'], 'test_user', path_submissions ) submit_starting_kits( session, ramp_config_problem['event_name'], 'test_user_2', path_submissions )
def test_aws_dispatcher(session_toy): # noqa # copy of test_integration_dispatcher but with AWS if not os.path.isfile(os.path.join(HERE, 'config.yml')): pytest.skip("Only for local tests for now") config = read_config(database_config_template()) event_config = ramp_config_template() event_config = read_config(event_config) # patch the event_config to match local config.yml for AWS aws_event_config = read_config(os.path.join(HERE, 'config.yml')) event_config['worker'] = aws_event_config['worker'] dispatcher = Dispatcher(config=config, event_config=event_config, worker=AWSWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def session_scope_function(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: yield finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def session_scope_module(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: create_toy_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: yield session finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def session_scope_function(database_connection): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: yield finally: # FIXME: we are recreating the deployment directory but it should be # replaced by an temporary creation of folder. deployment_dir = os.path.commonpath( [ramp_config['ramp']['kit_dir'], ramp_config['ramp']['data_dir']]) shutil.rmtree(deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_dispatcher_error(): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) # check that passing a not a number will raise a TypeError err_msg = "The parameter 'n_threads' should be a positive integer" with pytest.raises(TypeError, match=err_msg): Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=100, n_threads='whatever', hunger_policy='exit')
def test_integration_dispatcher(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_worker=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def session_scope_function(): database_config = read_config(database_config_template()) ramp_config = read_config(ramp_config_template()) try: create_test_db(database_config, ramp_config) with session_scope(database_config['sqlalchemy']) as session: add_users(session) add_problems(session, ramp_config['ramp']) add_events(session, ramp_config['ramp']) yield session finally: shutil.rmtree(ramp_config['ramp']['deployment_dir'], ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def test_check_problem(session_scope_function): ramp_configs = { 'iris': read_config(ramp_config_iris()), 'boston_housing': read_config(ramp_config_boston_housing()) } for problem_name, ramp_config in ramp_configs.items(): internal_ramp_config = generate_ramp_config(ramp_config) setup_ramp_kit_ramp_data(internal_ramp_config, problem_name) add_problem(session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir']) problem_name = 'iris' problem = get_problem(session_scope_function, problem_name) assert problem.name == problem_name assert isinstance(problem, Problem) problem = get_problem(session_scope_function, None) assert len(problem) == 2 assert isinstance(problem, list) # Without forcing, we cannot write the same problem twice internal_ramp_config = generate_ramp_config(ramp_configs[problem_name]) err_msg = 'Attempting to overwrite a problem and delete all linked events' with pytest.raises(ValueError, match=err_msg): add_problem( session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir'], force=False ) # Force add the problem add_problem( session_scope_function, problem_name, internal_ramp_config['ramp_kit_dir'], internal_ramp_config['ramp_data_dir'], force=True ) problem = get_problem(session_scope_function, problem_name) assert problem.name == problem_name assert isinstance(problem, Problem) delete_problem(session_scope_function, problem_name) problem = get_problem(session_scope_function, problem_name) assert problem is None problem = get_problem(session_scope_function, None) assert len(problem) == 1 assert isinstance(problem, list)
def test_restart_on_sudden_instance_termination(training_finished, launch_train, spot_terminated, caplog): class DummyInstance: id = 1 launch_train.return_value = 0 # setup the AWS worker event_config = read_config(ramp_aws_config_template())['worker'] worker = AWSWorker(event_config, submission='starting_kit_local') worker.config = event_config worker.submission = 'dummy submissions' worker.instance = DummyInstance # set the submission did not yet finish training training_finished.return_value = False spot_terminated.return_value = False worker.launch_submission() assert worker.status == 'running' assert caplog.text == '' # call CalledProcessError on checking if submission was finished training_finished.side_effect = subprocess.CalledProcessError(255, 'test') # make sure that the worker status is set to 'retry' assert worker.status == 'retry' assert 'Unable to connect to the instance' in caplog.text assert 'Adding the submission back to the queue' in caplog.text
def __init__(self, config, event_config, worker=None, n_worker=1, hunger_policy=None): self.worker = CondaEnvWorker if worker is None else worker self.n_worker = (max(multiprocessing.cpu_count() + 1 + n_worker, 1) if n_worker < 0 else n_worker) self.hunger_policy = hunger_policy # init the poison pill to kill the dispatcher self._poison_pill = False # create the different dispatcher queues self._awaiting_worker_queue = Queue() self._processing_worker_queue = LifoQueue(maxsize=self.n_worker) self._processed_submission_queue = Queue() # split the different configuration required if (isinstance(config, six.string_types) and isinstance(event_config, six.string_types)): self._database_config = read_config(config, filter_section='sqlalchemy') self._ramp_config = generate_ramp_config(event_config, config) else: self._database_config = config['sqlalchemy'] self._ramp_config = event_config['ramp'] self._worker_config = generate_worker_config(event_config, config)
def dispatcher(config, event_config, verbose): """Launch the RAMP dispatcher. The RAMP dispatcher is in charge of starting RAMP workers, collecting results from them, and update the database. """ if verbose: if verbose == 1: level = logging.INFO else: level = logging.DEBUG logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', level=level, datefmt='%Y:%m:%d %H:%M:%S' ) internal_event_config = read_config(event_config) worker_type = available_workers[ internal_event_config['worker']['worker_type'] ] dispatcher_config = (internal_event_config['dispatcher'] if 'dispatcher' in internal_event_config else {}) n_workers = dispatcher_config.get('n_workers', -1) n_threads = dispatcher_config.get('n_threads', None) hunger_policy = dispatcher_config.get('hunger_policy', 'sleep') time_between_collection = dispatcher_config.get( 'time_between_collection', 1) disp = Dispatcher( config=config, event_config=event_config, worker=worker_type, n_workers=n_workers, n_threads=n_threads, hunger_policy=hunger_policy, time_between_collection=time_between_collection ) disp.launch()
def dispatcher(config, event_config, n_workers, n_threads, hunger_policy, verbose): """Launch the RAMP dispatcher. The RAMP dispatcher is in charge of starting RAMP workers, collecting results from them, and update the database. """ if verbose: if verbose == 1: level = logging.INFO else: level = logging.DEBUG logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', level=level, datefmt='%Y:%m:%d %H:%M:%S') internal_event_config = read_config(event_config) worker_type = available_workers[internal_event_config['worker'] ['worker_type']] disp = Dispatcher(config=config, event_config=event_config, worker=worker_type, n_workers=n_workers, n_threads=n_threads, hunger_policy=hunger_policy) disp.launch()
def teardown_function(function): database_config = read_config(database_config_template()) # FIXME: we are recreating the deployment directory but it should be # replaced by an temporary creation of folder. shutil.rmtree(function.deployment_dir, ignore_errors=True) db, _ = setup_db(database_config['sqlalchemy']) Model.metadata.drop_all(db)
def __init__(self, config, event_config, worker=None, n_workers=1, n_threads=None, hunger_policy=None): self.worker = CondaEnvWorker if worker is None else worker self.n_workers = (max(multiprocessing.cpu_count() + 1 + n_workers, 1) if n_workers < 0 else n_workers) self.hunger_policy = hunger_policy # init the poison pill to kill the dispatcher self._poison_pill = False # create the different dispatcher queues self._awaiting_worker_queue = Queue() self._processing_worker_queue = LifoQueue(maxsize=self.n_workers) self._processed_submission_queue = Queue() # split the different configuration required if (isinstance(config, str) and isinstance(event_config, str)): self._database_config = read_config(config, filter_section='sqlalchemy') self._ramp_config = generate_ramp_config(event_config, config) else: self._database_config = config['sqlalchemy'] self._ramp_config = event_config['ramp'] self._worker_config = generate_worker_config(event_config, config) # set the number of threads for openmp, openblas, and mkl self.n_threads = n_threads if self.n_threads is not None: if not isinstance(self.n_threads, numbers.Integral): raise TypeError( "The parameter 'n_threads' should be a positive integer. " "Got {} instead.".format(repr(self.n_threads))) for lib in ('OMP', 'MKL', 'OPENBLAS'): os.environ[lib + '_NUM_THREADS'] = str(self.n_threads)
def test_add_submission_wrong_submission_files(base_db): # check that we raise an error if the file required by the workflow is not # present in the submission or that it has the wrong extension session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'corrupted_submission' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) os.makedirs(path_submission) # case that there is not files in the submission err_msg = 'No file corresponding to the workflow element' with pytest.raises(MissingSubmissionFileError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # case that there is not file corresponding to the workflow component filename = os.path.join(path_submission, 'unknown_file.xxx') open(filename, "w+").close() err_msg = 'No file corresponding to the workflow element' with pytest.raises(MissingSubmissionFileError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) # case that we have the correct filename but not the right extension filename = os.path.join(path_submission, 'classifier.xxx') open(filename, "w+").close() err_msg = 'All extensions "xxx" are unknown for the submission' with pytest.raises(MissingExtensionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission)
def add_user(config, login, password, lastname, firstname, email, access_level, hidden_notes): """Add a new user in the database.""" config = read_config(config) with session_scope(config['sqlalchemy']) as session: user_module.add_user(session, login, password, lastname, firstname, email, access_level, hidden_notes)
def test_aws_worker(): if not os.path.isfile(os.path.join(HERE, 'config.yml')): pytest.skip("Only for local tests for now") ramp_kit_dir = os.path.join(HERE, 'kits', 'iris') # make sure predictio and log dirs exist, if not, add them add_empty_dir(os.path.join(ramp_kit_dir, 'predictions')) add_empty_dir(os.path.join(ramp_kit_dir, 'logs')) # if the prediction / log files are still there, remove them for subdir in os.listdir(os.path.join(ramp_kit_dir, 'predictions')): if os.path.isdir(subdir): shutil.rmtree(subdir) for subdir in os.listdir(os.path.join(ramp_kit_dir, 'logs')): if os.path.isdir(subdir): shutil.rmtree(subdir) config = read_config(os.path.join(HERE, 'config.yml')) worker_config = generate_worker_config(config) worker = AWSWorker(worker_config, submission='starting_kit_local') worker.setup() assert worker.status == 'setup' worker.launch_submission() assert worker.status in ('running', 'finished') worker.collect_results() assert worker.status == 'collected' assert os.path.isdir( os.path.join(ramp_kit_dir, 'predictions', 'starting_kit_local', 'fold_0')) assert os.path.isfile( os.path.join(ramp_kit_dir, 'logs', 'starting_kit_local', 'log')) worker.teardown() assert worker.status == 'killed'
def test_add_submission_too_early_submission(base_db): # check that we raise an error when the elapsed time was not large enough # between the new submission and the previous submission session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) # check that we have an awaiting time for the event event = (session.query(Event).filter( Event.name == event_name).one_or_none()) assert event.min_duration_between_submissions == 900 # make 2 submissions which are too close from each other for submission_idx, submission_name in enumerate( ['random_forest_10_10', 'too_early_submission']): path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) if submission_idx == 1: err_msg = 'You need to wait' with pytest.raises(TooEarlySubmissionError, match=err_msg): add_submission(session, event_name, username, submission_name, path_submission) else: add_submission(session, event_name, username, submission_name, path_submission)
def delete_event(config, config_event, dry_run, from_disk, force): """Delete event.""" internal_config = read_config(config) ramp_config = generate_ramp_config(config_event, config) event_name = ramp_config["event_name"] with session_scope(internal_config['sqlalchemy']) as session: db_event = event_module.get_event(session, event_name) if db_event: if not dry_run: event_module.delete_event(session, event_name) click.echo('{} was removed from the database'.format(event_name)) if from_disk: if not db_event and not force: err_msg = ('{} event not found in the database. If you want ' 'to force removing event files from the disk, add ' 'the option "--force".'.format(event_name)) raise click.ClickException(err_msg) for key in ("ramp_submissions_dir", "ramp_predictions_dir", "ramp_logs_dir"): dir_to_remove = ramp_config[key] if os.path.exists(dir_to_remove): if not dry_run: shutil.rmtree(dir_to_remove) click.echo("Removed directory:\n{}".format(dir_to_remove)) else: click.echo("Directory not found. Skip removal for the " "directory:\n{}".format(dir_to_remove)) event_dir = os.path.dirname(config_event) if not dry_run: shutil.rmtree(event_dir) click.echo("Removed directory:\n{}".format(event_dir))
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # check that the submissions have been copied for sub_id, _, _ in all_submissions: sub = get_submission_by_id(session, sub_id) assert os.path.exists(sub.path) assert os.path.exists(os.path.join(sub.path, 'classifier.py')) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 # check that the number of submissions for an event was updated event = session.query(Event).filter(Event.name == event_name).one_or_none() assert event.n_submissions == 1 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_delete_event_only_files(make_toy_db): # check the behavior when only file are present on disks runner = CliRunner() # create the event folder ramp_config = read_config(ramp_config_template()) ramp_config['ramp']['event_name'] = 'iris_test2' deployment_dir = os.path.commonpath([ramp_config['ramp']['kit_dir'], ramp_config['ramp']['data_dir']]) runner.invoke(main_utils, ['init-event', '--name', 'iris_test2', '--deployment-dir', deployment_dir]) event_config = os.path.join( deployment_dir, 'events', ramp_config['ramp']['event_name'], 'config.yml' ) with open(event_config, 'w+') as f: yaml.dump(ramp_config, f) # check that --from-disk will raise an error cmd = ['delete-event', '--config', database_config_template(), '--config-event', event_config, '--from-disk'] result = runner.invoke(main, cmd) assert result.exit_code == 1 assert 'add the option "--force"' in result.output cmd = ['delete-event', '--config', database_config_template(), '--config-event', event_config, '--from-disk', '--force'] result = runner.invoke(main, cmd) assert result.exit_code == 0, result.output assert not os.path.exists(os.path.dirname(event_config))
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = read_config(ramp_config_template()) event_name, username = _setup_sign_up(session, config) ramp_config = generate_ramp_config(config) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def add_event(config, problem, event, title, sandbox, submissions_dir, is_public, force): """Add an event in the database.""" config = read_config(config) with session_scope(config['sqlalchemy']) as session: event_module.add_event(session, problem, event, title, sandbox, submissions_dir, is_public, force)