def test_deploy_ramp_event(session_scope_function): database_config = read_config(database_config_template()) event_config_filename = ramp_config_template() event_config = read_config(event_config_filename) ramp_config = generate_ramp_config(event_config) deploy_ramp_event(database_config_template(), ramp_config_template()) # simulate that we add users and sign-up for the event and that they # submitted the starting kit with session_scope(database_config['sqlalchemy']) as session: add_users(session) sign_up_team(session, ramp_config['event_name'], 'test_user') submit_starting_kits(session, ramp_config['event_name'], 'test_user', ramp_config['ramp_kit_submissions_dir']) # run the dispatcher on the event which are in the dataset dispatcher = Dispatcher(config=database_config, event_config=event_config, worker=CondaEnvWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for a user with session_scope(database_config['sqlalchemy']) as session: submission = get_submissions(session, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 1
def test_aws_dispatcher(session_toy): # noqa # copy of test_integration_dispatcher but with AWS if not os.path.isfile(os.path.join(HERE, 'config.yml')): pytest.skip("Only for local tests for now") config = read_config(database_config_template()) event_config = ramp_config_template() event_config = read_config(event_config) # patch the event_config to match local config.yml for AWS aws_event_config = read_config(os.path.join(HERE, 'config.yml')) event_config['worker'] = aws_event_config['worker'] dispatcher = Dispatcher(config=config, event_config=event_config, worker=AWSWorker, n_workers=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def fetch_from_db(self, session): """Fetch the submission from the database and create the workers.""" submissions = get_submissions(session, self._ramp_config['event_name'], state='new') if not submissions: return for submission_id, submission_name, _ in submissions: # do not train the sandbox submission submission = get_submission_by_id(session, submission_id) if not submission.is_not_sandbox: continue # create the worker worker = self.worker(self._worker_config, submission_name) set_submission_state(session, submission_id, 'sent_to_training') update_user_leaderboards( session, self._ramp_config['event_name'], submission.team.name, new_only=True, ) self._awaiting_worker_queue.put_nowait( (worker, (submission_id, submission_name))) logger.info('Submission {} added to the queue of submission to be ' 'processed'.format(submission_name))
def test_dispatcher_worker_retry(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=10, hunger_policy='exit') dispatcher.fetch_from_db(session_toy) dispatcher.launch_workers(session_toy) # Get one worker and set status to 'retry' worker, (submission_id, submission_name) = \ dispatcher._processing_worker_queue.get() setattr(worker, 'status', 'retry') assert worker.status == 'retry' # Add back to queue dispatcher._processing_worker_queue.put_nowait( (worker, (submission_id, submission_name))) while not dispatcher._processing_worker_queue.empty(): dispatcher.collect_result(session_toy) submissions = get_submissions(session_toy, 'iris_test', 'new') assert submission_name in [sub[1] for sub in submissions]
def test_error_handling_worker_setup_error(session_toy, caplog): # make sure the error on the worker.setup is dealt with correctly # set mock worker class Worker_mock(): def __init__(self, *args, **kwargs): self.state = None def setup(self): raise Exception('Test error') def teardown(self): pass config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) worker = Worker_mock() dispatcher = Dispatcher(config=config, event_config=event_config, worker=Worker_mock, n_workers=-1, hunger_policy='exit') dispatcher.launch() submissions = get_submissions(session_toy, event_config['ramp']['event_name'], 'checking_error') assert len(submissions) == 6 worker.status = 'error' assert 'Test error' in caplog.text
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = ramp_config_template() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # check that the submissions have been copied for sub_id, _, _ in all_submissions: sub = get_submission_by_id(session, sub_id) assert os.path.exists(sub.path) assert os.path.exists(os.path.join(sub.path, 'classifier.py')) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 # check that the number of submissions for an event was updated event = session.query(Event).filter(Event.name == event_name).one_or_none() assert event.n_submissions == 1 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_add_submission_create_new_submission(base_db): # check that we can make a new submission to the database # it will require to have already a team and an event session = base_db config = read_config(ramp_config_template()) event_name, username = _setup_sign_up(session, config) ramp_config = generate_ramp_config(config) submission_name = 'random_forest_10_10' path_submission = os.path.join( os.path.dirname(ramp_config['ramp_sandbox_dir']), submission_name) add_submission(session, event_name, username, submission_name, path_submission) all_submissions = get_submissions(session, event_name, None) # `sign_up_team` make a submission (sandbox) by user. This submission will # be the third submission. assert len(all_submissions) == 3 submission = get_submission_by_name(session, event_name, username, submission_name) assert submission.name == submission_name submission_file = submission.files[0] assert submission_file.name == 'classifier' assert submission_file.extension == 'py' assert (os.path.join('submission_000000005', 'classifier.py') in submission_file.path)
def test_unit_test_dispatcher(session_toy): # make sure that the size of the list is bigger than the number of # submissions config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=100, hunger_policy='exit') # check that all the queue are empty assert dispatcher._awaiting_worker_queue.empty() assert dispatcher._processing_worker_queue.empty() assert dispatcher._processed_submission_queue.empty() # check that all submissions are queued submissions = get_submissions(session_toy, 'iris_test', 'new') dispatcher.fetch_from_db(session_toy) # we should remove the starting kit from the length of the submissions for # each user assert dispatcher._awaiting_worker_queue.qsize() == len(submissions) - 2 submissions = get_submissions(session_toy, 'iris_test', 'sent_to_training') assert len(submissions) == 6 # start the training dispatcher.launch_workers(session_toy) # be sure that the training is finished while not dispatcher._processing_worker_queue.empty(): dispatcher.collect_result(session_toy) assert len(get_submissions(session_toy, 'iris_test', 'new')) == 2 assert (len(get_submissions(session_toy, 'iris_test', 'training_error')) == 2) assert len(get_submissions(session_toy, 'iris_test', 'tested')) == 4 dispatcher.update_database_results(session_toy) assert dispatcher._processed_submission_queue.empty() event = get_event(session_toy, 'iris_test') assert event.private_leaderboard_html assert event.public_leaderboard_html_with_links assert event.public_leaderboard_html_no_links assert event.failed_leaderboard_html assert event.new_leaderboard_html is None assert event.public_competition_leaderboard_html assert event.private_competition_leaderboard_html
def test_get_submissions(session_scope_module, state, expected_id): submissions = get_submissions(session_scope_module, 'iris_test', state=state) assert len(submissions) == len(expected_id) for submission_id, sub_name, sub_path in submissions: assert submission_id in expected_id assert 'submission_{:09d}'.format(submission_id) == sub_name path_file = os.path.join('submission_{:09d}'.format(submission_id), 'classifier.py') assert path_file in sub_path[0]
def test_integration_dispatcher(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_worker=-1, hunger_policy='exit') dispatcher.launch() # the iris kit contain a submission which should fail for each user submission = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submission) == 2
def test_dispatcher_aws_not_launching(session_toy_aws, caplog): # given the test config file the instance should not be able to launch # due to authentication error # after unsuccessful try the worker should teardown config = read_config(database_config_template()) event_config = read_config(ramp_aws_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=AWSWorker, n_workers=10, hunger_policy='exit') dispatcher.fetch_from_db(session_toy_aws) submissions = get_submissions(session_toy_aws, 'iris_aws_test', 'new') dispatcher.launch_workers(session_toy_aws) assert 'AuthFailure' in caplog.text # training should not have started assert 'training' not in caplog.text num_running_workers = dispatcher._processing_worker_queue.qsize() assert num_running_workers == 0 submissions2 = get_submissions(session_toy_aws, 'iris_aws_test', 'new') # assert that all the submissions are still in the 'new' state assert len(submissions) == len(submissions2)
def test_submit_starting_kits(base_db): session = base_db config = ramp_config_iris() event_name, username = _setup_sign_up(session) ramp_config = generate_ramp_config(read_config(config)) submit_starting_kits(session, event_name, username, ramp_config['ramp_kit_submissions_dir']) submissions = get_submissions(session, event_name, None) submissions_id = [sub[0] for sub in submissions] assert len(submissions) == 5 expected_submission_name = {'starting_kit', 'starting_kit_test', 'random_forest_10_10', 'error'} submission_name = {get_submission_by_id(session, sub_id).name for sub_id in submissions_id} assert submission_name == expected_submission_name
def test_dispatcher_timeout(session_toy): config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=CondaEnvWorker, n_workers=-1, hunger_policy='exit') # override the timeout of the worker dispatcher._worker_config["timeout"] = 1 dispatcher.launch() # we should have at least 3 submissions which will fail: # 2 for errors and 1 for timeout submissions = get_submissions(session_toy, event_config['ramp']['event_name'], 'training_error') assert len(submissions) >= 2
def test_submit_starting_kits(base_db): session = base_db config = read_config(ramp_config_template()) event_name, username = _setup_sign_up(session, config) ramp_config = generate_ramp_config(config) submit_starting_kits( session, event_name, username, os.path.join(ramp_config['ramp_kits_dir'], ramp_config['event'], config['ramp']['submissions_dir'])) submissions = get_submissions(session, event_name, None) submissions_id = [sub[0] for sub in submissions] assert len(submissions) == 5 expected_submission_name = { 'starting_kit', 'starting_kit_test', 'random_forest_10_10', 'error' } submission_name = set( get_submission_by_id(session, sub_id).name for sub_id in submissions_id) assert submission_name == expected_submission_name
def launch(self): """Launch the dispatcher.""" logger.info('Starting the RAMP dispatcher') with session_scope(self._database_config) as session: logger.info('Open a session to the database') try: while not self._poison_pill: self.fetch_from_db(session) self.launch_workers(session) self.collect_result(session) self.update_database_results(session) finally: # reset the submissions to 'new' in case of error or unfinished # training submissions = get_submissions(session, self._ramp_config['event_name'], state=None) for submission_id, _, _ in submissions: submission_state = get_submission_state( session, submission_id) if submission_state in ('training', 'send_to_training'): set_submission_state(session, submission_id, 'new') logger.info('Dispatcher killed by the poison pill')
def test_get_submission_unknown_state(session_scope_module): with pytest.raises(UnknownStateError, match='Unrecognized state'): get_submissions(session_scope_module, 'iris_test', state='whatever')
def test_info_on_training_error(test_launch_ec2_instances, upload_submission, launch_train, is_spot_terminated, training_finished, training_successful, get_log_content, check_instance_status, download_log, session_toy_aws, caplog): # make sure that the Python error from the solution is passed to the # dispatcher # everything shoud be mocked as correct output from AWS instances # on setting up the instance and loading the submission # mock dummy AWS instance class DummyInstance: id = 1 test_launch_ec2_instances.return_value = (DummyInstance(), ), 0 upload_submission.return_value = 0 launch_train.return_value = 0 is_spot_terminated.return_value = 0 training_finished.return_value = False download_log.return_value = 0 config = read_config(database_config_template()) event_config = read_config(ramp_aws_config_template()) dispatcher = Dispatcher(config=config, event_config=event_config, worker=AWSWorker, n_workers=10, hunger_policy='exit') dispatcher.fetch_from_db(session_toy_aws) dispatcher.launch_workers(session_toy_aws) num_running_workers = dispatcher._processing_worker_queue.qsize() # worker, (submission_id, submission_name) = \ # dispatcher._processing_worker_queue.get() # assert worker.status == 'running' submissions = get_submissions(session_toy_aws, 'iris_aws_test', 'training') ids = [submissions[idx][0] for idx in range(len(submissions))] assert len(submissions) > 1 assert num_running_workers == len(ids) dispatcher.time_between_collection = 0 training_successful.return_value = False # now we will end the submission with training error training_finished.return_value = True training_error_msg = 'Python error here' get_log_content.return_value = training_error_msg check_instance_status.return_value = 'finished' dispatcher.collect_result(session_toy_aws) # the worker which we were using should have been teared down num_running_workers = dispatcher._processing_worker_queue.qsize() assert num_running_workers == 0 submissions = get_submissions(session_toy_aws, 'iris_aws_test', 'training_error') assert len(submissions) == len(ids) submission = get_submission_by_id(session_toy_aws, submissions[0][0]) assert training_error_msg in submission.error_msg
def score_plot(session, event): from bokeh.plotting import figure from bokeh.models.sources import ColumnDataSource from bokeh.models.formatters import DatetimeTickFormatter submissions = get_submissions(session, event.name, None) submissions = [ get_submission_by_id(session, sub_id) for sub_id, _, _ in submissions if get_submission_by_id(session, sub_id).is_public_leaderboard and get_submission_by_id(session, sub_id).is_valid ] score_names = [score_type.name for score_type in event.score_types] scoress = np.array([[ score.valid_score_cv_bag for score in submission.ordered_scores(score_names) ] for submission in submissions]).T score_plot_df = pd.DataFrame() score_plot_df['submitted at (UTC)'] = [ submission.submission_timestamp for submission in submissions ] score_plot_df['contributivity'] = [ submission.contributivity for submission in submissions ] score_plot_df['historical contributivity'] = [ submission.historical_contributivity for submission in submissions ] for score_name in score_names: # to make sure the column is created score_plot_df[score_name] = 0 for score_name, scores in zip(score_names, scoress): score_plot_df[score_name] = scores score_name = event.official_score_name score_plot_df = score_plot_df[ score_plot_df['submitted at (UTC)'] > event.opening_timestamp] score_plot_df = score_plot_df.sort_values('submitted at (UTC)') score_plot_df = add_pareto(score_plot_df, score_name, event.official_score_type.worst, event.official_score_type.is_lower_the_better) is_open = (score_plot_df['submitted at (UTC)'] > event.public_opening_timestamp).values max_contributivity = max(0.0000001, max(score_plot_df['contributivity'].values)) max_historical_contributivity = max( 0.0000001, max(score_plot_df['historical contributivity'].values)) fill_color_1 = (176, 23, 31) fill_color_2 = (16, 78, 139) fill_colors_1 = color_gradient( fill_color_1, score_plot_df['contributivity'].values / max_contributivity) fill_colors_2 = color_gradient( fill_color_2, score_plot_df['historical contributivity'].values / max_historical_contributivity) fill_colors = np.minimum(fill_colors_1, fill_colors_2).astype(int) fill_colors = ["#%02x%02x%02x" % (c[0], c[1], c[2]) for c in fill_colors] score_plot_df['x'] = score_plot_df['submitted at (UTC)'] score_plot_df['y'] = score_plot_df[score_name] score_plot_df['line_color'] = 'royalblue' score_plot_df['circle_size'] = 8 score_plot_df['line_color'] = 'royalblue' score_plot_df.loc[is_open, 'line_color'] = 'coral' score_plot_df['fill_color'] = fill_colors score_plot_df['fill_alpha'] = 0.5 score_plot_df['line_width'] = 0 score_plot_df['label'] = 'closed phase' score_plot_df.loc[is_open, 'label'] = 'open phase' source = ColumnDataSource(score_plot_df) pareto_df = score_plot_df[score_plot_df[score_name + ' pareto'] == 1].copy() pareto_df = pareto_df.append(pareto_df.iloc[-1]) pareto_df.iloc[-1, pareto_df.columns.get_loc('x')] = (max(score_plot_df['x'])) pareto_df = make_step_df(pareto_df, event.official_score_type.is_lower_the_better) source_pareto = ColumnDataSource(pareto_df) tools = ['pan,wheel_zoom,box_zoom,reset,save,tap'] p = figure(plot_width=900, plot_height=600, tools=tools, title='Scores') p.circle('x', 'y', size='circle_size', line_color='line_color', fill_color='fill_color', fill_alpha='fill_alpha', line_width=1, source=source, legend='label') p.line('x', 'y', line_width=3, line_color='goldenrod', source=source_pareto, legend='best score', alpha=0.9) p.xaxis.formatter = DatetimeTickFormatter( hours=['%d %B %Y'], days=['%d %B %Y'], months=['%d %B %Y'], years=['%d %B %Y'], ) p.xaxis.major_label_orientation = np.pi / 4 if event.official_score_type.is_lower_the_better: p.yaxis.axis_label = score_name + ' (the lower the better)' p.legend.location = 'top_right' else: p.yaxis.axis_label = score_name + ' (the greater the better)' p.legend.location = 'bottom_right' p.xaxis.axis_label = 'submission timestamp (UTC)' p.xaxis.axis_label_text_font_size = '14pt' p.yaxis.axis_label_text_font_size = '14pt' p.legend.label_text_font_size = '14pt' p.title.text_font_size = '16pt' p.xaxis.major_label_text_font_size = '10pt' p.yaxis.major_label_text_font_size = '10pt' return p
def _reset_submission_after_failure(session, even_name): submissions = get_submissions(session, even_name, state=None) for submission_id, _, _ in submissions: submission_state = get_submission_state(session, submission_id) if submission_state in ('training', 'send_to_training'): set_submission_state(session, submission_id, 'new')