def test_submission_model_set_error(session_scope_module):
submission = get_submission_by_id(session_scope_module, 5)
error = 'training_error'
error_msg = 'simulate an error'
submission.set_error(error, error_msg)
assert submission.state == error
assert submission.error_msg == error_msg
for cv_fold in submission.on_cv_folds:
assert cv_fold.state == error
assert cv_fold.error_msg == error_msg
def test_submission_model_set_contributivity(session_scope_module, state,
expected_contributivity):
submission = get_submission_by_id(session_scope_module, 5)
# set the state of the submission such that the contributivity
submission.set_state(state)
# set the fold contributivity to non-default
for cv_fold in submission.on_cv_folds:
cv_fold.contributivity = 0.3
submission.set_contributivity()
assert submission.contributivity == pytest.approx(expected_contributivity)
def test_submission_model_set_state(session_scope_module):
submission = get_submission_by_id(session_scope_module, 5)
submission.set_state('sent_to_training')
assert submission.state == 'sent_to_training'
assert (submission.sent_to_training_timestamp -
datetime.utcnow()).total_seconds() < 10
submission.set_state('training')
assert submission.state == 'training'
assert (submission.training_timestamp -
datetime.utcnow()).total_seconds() < 10
submission.set_state('scored')
assert submission.state == 'scored'
for cv_fold in submission.on_cv_folds:
assert cv_fold.state == 'scored'
def test_integration_dispatcher(session_toy):
config = read_config(database_config_template())
event_config = read_config(ramp_config_template())
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=CondaEnvWorker,
n_workers=-1,
hunger_policy='exit')
dispatcher.launch()
# the iris kit contain a submission which should fail for each user
submissions = get_submissions(session_toy,
event_config['ramp']['event_name'],
'training_error')
assert len(submissions) == 2
submission = get_submission_by_id(session_toy, submissions[0][0])
assert 'ValueError' in submission.error_msg
def test_submit_starting_kits(base_db):
session = base_db
config = ramp_config_iris()
event_name, username = _setup_sign_up(session)
ramp_config = generate_ramp_config(read_config(config))
submit_starting_kits(session, event_name, username,
ramp_config['ramp_kit_submissions_dir'])
submissions = get_submissions(session, event_name, None)
submissions_id = [sub[0] for sub in submissions]
assert len(submissions) == 5
expected_submission_name = {'starting_kit', 'starting_kit_test',
'random_forest_10_10', 'error'}
submission_name = {get_submission_by_id(session, sub_id).name
for sub_id in submissions_id}
assert submission_name == expected_submission_name
def test_submission_model_property(session_scope_module):
# check that the property of Submission
submission = get_submission_by_id(session_scope_module, 5)
assert re.match(r'Submission\(iris_test/test_user/.*\)', str(submission))
assert re.match(r'Submission\(event_name.*\)', repr(submission))
assert isinstance(submission.team, Team)
assert isinstance(submission.event, Event)
assert submission.official_score_name == 'acc'
assert isinstance(submission.official_score, SubmissionScore)
assert all([
isinstance(score, EventScoreType) for score in submission.score_types
])
assert issubclass(submission.Predictions, BasePrediction)
assert submission.is_not_sandbox is True
assert submission.is_error is False
assert submission.is_public_leaderboard is False
assert submission.is_private_leaderboard is False
assert (os.path.join('submissions', 'submission_000000005')
in submission.path)
assert submission.basename == 'submission_000000005'
assert "submissions.submission_000000005" in submission.module
assert len(submission.f_names) == 1
assert submission.f_names[0] == 'estimator.py'
assert submission.link == '/' + os.path.join(submission.hash_,
'estimator.py')
assert re.match(
'<a href={}>{}/{}/{}</a>'.format(submission.link,
submission.event.name,
submission.team.name,
submission.name),
submission.full_name_with_link)
assert re.match(
'<a href={}>{}</a>'.format(submission.link, submission.name),
submission.name_with_link)
assert re.match(
'<a href=.*{}.*error.txt>{}</a>'.format(submission.hash_,
submission.state),
submission.state_with_link)
for score in submission.ordered_scores(score_names=['acc', 'error']):
assert isinstance(score, SubmissionScore)
def launch_workers(self, session):
"""Launch the awaiting workers if possible."""
while (not self._processing_worker_queue.full()
and not self._awaiting_worker_queue.empty()):
worker, (submission_id, submission_name) = \
self._awaiting_worker_queue.get()
self._logger.info(f'Starting worker: {worker}')
try:
worker.setup()
if worker.status != "error":
worker.launch_submission()
except Exception as e:
self._logger.error(
f'Worker finished with unhandled exception:\n {e}')
worker.status = 'error'
if worker.status == 'error':
set_submission_state(session, submission_id, 'checking_error')
worker.teardown() # kill the worker
self._logger.info(f'Worker {worker} killed due to an error '
f'while connecting to AWS worker')
stderr = ("There was a problem with sending your submission"
" for training. This problem is on RAMP side"
" and most likely it is not related to your"
" code. If this happened for the first time"
" to this submission you might"
" consider submitting the same code once again."
" Else, please contact the event organizers.")
set_submission_error_msg(session, submission_id, stderr)
continue
set_submission_state(session, submission_id, 'training')
submission = get_submission_by_id(session, submission_id)
update_user_leaderboards(
session,
self._ramp_config['event_name'],
submission.team.name,
new_only=True,
)
self._processing_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
self._logger.info(
f'Store the worker {worker} into the processing queue')
def fetch_from_db(self, session):
"""Fetch the submission from the database and create the workers."""
submissions = get_submissions(session,
self._ramp_config['event_name'],
state='new')
if not submissions:
logger.info('No new submissions fetch from the database')
return
for submission_id, submission_name, _ in submissions:
# do not train the sandbox submission
submission = get_submission_by_id(session, submission_id)
if not submission.is_not_sandbox:
continue
# create the worker
worker = self.worker(self._worker_config, submission_name)
set_submission_state(session, submission_id, 'sent_to_training')
self._awaiting_worker_queue.put_nowait(
(worker, (submission_id, submission_name)))
logger.info('Submission {} added to the queue of submission to be '
'processed'.format(submission_name))
def test_submit_starting_kits(base_db):
session = base_db
config = read_config(ramp_config_template())
event_name, username = _setup_sign_up(session, config)
ramp_config = generate_ramp_config(config)
submit_starting_kits(
session, event_name, username,
os.path.join(ramp_config['ramp_kits_dir'], ramp_config['event'],
config['ramp']['submissions_dir']))
submissions = get_submissions(session, event_name, None)
submissions_id = [sub[0] for sub in submissions]
assert len(submissions) == 5
expected_submission_name = {
'starting_kit', 'starting_kit_test', 'random_forest_10_10', 'error'
}
submission_name = set(
get_submission_by_id(session, sub_id).name
for sub_id in submissions_id)
assert submission_name == expected_submission_name
def test_submission_model_reset(session_scope_module):
submission = get_submission_by_id(session_scope_module, 5)
for score in submission.ordered_scores(score_names=['acc', 'error']):
assert isinstance(score, SubmissionScore)
# set the score to later test the reset function
score.valid_score_cv_bag = 1.0
score.test_score_cv_bag = 1.0
score.valid_score_cv_bags = np.ones(2)
score.test_score_cv_bags = np.ones(2)
# set to non-default the variable that should change with reset
submission.error_msg = 'simulate an error'
submission.contributivity = 100.
submission.reset()
assert submission.contributivity == pytest.approx(0)
assert submission.state == 'new'
assert submission.error_msg == ''
for score, worse_score in zip(submission.ordered_scores(['acc', 'error']),
[0, 1]):
assert score.valid_score_cv_bag == pytest.approx(worse_score)
assert score.test_score_cv_bag == pytest.approx(worse_score)
assert score.valid_score_cv_bags is None
assert score.test_score_cv_bags is None
def test_info_on_training_error(test_launch_ec2_instances, upload_submission,
launch_train, is_spot_terminated,
training_finished, training_successful,
get_log_content, check_instance_status,
download_log, session_toy_aws, caplog):
# make sure that the Python error from the solution is passed to the
# dispatcher
# everything shoud be mocked as correct output from AWS instances
# on setting up the instance and loading the submission
# mock dummy AWS instance
class DummyInstance:
id = 1
test_launch_ec2_instances.return_value = (DummyInstance(), ), 0
upload_submission.return_value = 0
launch_train.return_value = 0
is_spot_terminated.return_value = 0
training_finished.return_value = False
download_log.return_value = 0
config = read_config(database_config_template())
event_config = read_config(ramp_aws_config_template())
dispatcher = Dispatcher(config=config,
event_config=event_config,
worker=AWSWorker,
n_workers=10,
hunger_policy='exit')
dispatcher.fetch_from_db(session_toy_aws)
dispatcher.launch_workers(session_toy_aws)
num_running_workers = dispatcher._processing_worker_queue.qsize()
# worker, (submission_id, submission_name) = \
# dispatcher._processing_worker_queue.get()
# assert worker.status == 'running'
submissions = get_submissions(session_toy_aws, 'iris_aws_test', 'training')
ids = [submissions[idx][0] for idx in range(len(submissions))]
assert len(submissions) > 1
assert num_running_workers == len(ids)
dispatcher.time_between_collection = 0
training_successful.return_value = False
# now we will end the submission with training error
training_finished.return_value = True
training_error_msg = 'Python error here'
get_log_content.return_value = training_error_msg
check_instance_status.return_value = 'finished'
dispatcher.collect_result(session_toy_aws)
# the worker which we were using should have been teared down
num_running_workers = dispatcher._processing_worker_queue.qsize()
assert num_running_workers == 0
submissions = get_submissions(session_toy_aws, 'iris_aws_test',
'training_error')
assert len(submissions) == len(ids)
submission = get_submission_by_id(session_toy_aws, submissions[0][0])
assert training_error_msg in submission.error_msg
def test_submission_model_set_state(session_scope_module):
submission = get_submission_by_id(session_scope_module, 5)
submission.set_state('scored')
assert submission.state == 'scored'
for cv_fold in submission.on_cv_folds:
assert cv_fold.state == 'scored'
def score_plot(session, event):
from bokeh.plotting import figure
from bokeh.models.sources import ColumnDataSource
from bokeh.models.formatters import DatetimeTickFormatter
submissions = get_submissions(session, event.name, None)
submissions = [
get_submission_by_id(session, sub_id) for sub_id, _, _ in submissions
if get_submission_by_id(session, sub_id).is_public_leaderboard
and get_submission_by_id(session, sub_id).is_valid
]
score_names = [score_type.name for score_type in event.score_types]
scoress = np.array([[
score.valid_score_cv_bag
for score in submission.ordered_scores(score_names)
] for submission in submissions]).T
score_plot_df = pd.DataFrame()
score_plot_df['submitted at (UTC)'] = [
submission.submission_timestamp for submission in submissions
]
score_plot_df['contributivity'] = [
submission.contributivity for submission in submissions
]
score_plot_df['historical contributivity'] = [
submission.historical_contributivity for submission in submissions
]
for score_name in score_names: # to make sure the column is created
score_plot_df[score_name] = 0
for score_name, scores in zip(score_names, scoress):
score_plot_df[score_name] = scores
score_name = event.official_score_name
score_plot_df = score_plot_df[
score_plot_df['submitted at (UTC)'] > event.opening_timestamp]
score_plot_df = score_plot_df.sort_values('submitted at (UTC)')
score_plot_df = add_pareto(score_plot_df, score_name,
event.official_score_type.worst,
event.official_score_type.is_lower_the_better)
is_open = (score_plot_df['submitted at (UTC)'] >
event.public_opening_timestamp).values
max_contributivity = max(0.0000001,
max(score_plot_df['contributivity'].values))
max_historical_contributivity = max(
0.0000001, max(score_plot_df['historical contributivity'].values))
fill_color_1 = (176, 23, 31)
fill_color_2 = (16, 78, 139)
fill_colors_1 = color_gradient(
fill_color_1,
score_plot_df['contributivity'].values / max_contributivity)
fill_colors_2 = color_gradient(
fill_color_2, score_plot_df['historical contributivity'].values /
max_historical_contributivity)
fill_colors = np.minimum(fill_colors_1, fill_colors_2).astype(int)
fill_colors = ["#%02x%02x%02x" % (c[0], c[1], c[2]) for c in fill_colors]
score_plot_df['x'] = score_plot_df['submitted at (UTC)']
score_plot_df['y'] = score_plot_df[score_name]
score_plot_df['line_color'] = 'royalblue'
score_plot_df['circle_size'] = 8
score_plot_df['line_color'] = 'royalblue'
score_plot_df.loc[is_open, 'line_color'] = 'coral'
score_plot_df['fill_color'] = fill_colors
score_plot_df['fill_alpha'] = 0.5
score_plot_df['line_width'] = 0
score_plot_df['label'] = 'closed phase'
score_plot_df.loc[is_open, 'label'] = 'open phase'
source = ColumnDataSource(score_plot_df)
pareto_df = score_plot_df[score_plot_df[score_name +
' pareto'] == 1].copy()
pareto_df = pareto_df.append(pareto_df.iloc[-1])
pareto_df.iloc[-1,
pareto_df.columns.get_loc('x')] = (max(score_plot_df['x']))
pareto_df = make_step_df(pareto_df,
event.official_score_type.is_lower_the_better)
source_pareto = ColumnDataSource(pareto_df)
tools = ['pan,wheel_zoom,box_zoom,reset,save,tap']
p = figure(plot_width=900, plot_height=600, tools=tools, title='Scores')
p.circle('x',
'y',
size='circle_size',
line_color='line_color',
fill_color='fill_color',
fill_alpha='fill_alpha',
line_width=1,
source=source,
legend='label')
p.line('x',
'y',
line_width=3,
line_color='goldenrod',
source=source_pareto,
legend='best score',
alpha=0.9)
p.xaxis.formatter = DatetimeTickFormatter(
hours=['%d %B %Y'],
days=['%d %B %Y'],
months=['%d %B %Y'],
years=['%d %B %Y'],
)
p.xaxis.major_label_orientation = np.pi / 4
if event.official_score_type.is_lower_the_better:
p.yaxis.axis_label = score_name + ' (the lower the better)'
p.legend.location = 'top_right'
else:
p.yaxis.axis_label = score_name + ' (the greater the better)'
p.legend.location = 'bottom_right'
p.xaxis.axis_label = 'submission timestamp (UTC)'
p.xaxis.axis_label_text_font_size = '14pt'
p.yaxis.axis_label_text_font_size = '14pt'
p.legend.label_text_font_size = '14pt'
p.title.text_font_size = '16pt'
p.xaxis.major_label_text_font_size = '10pt'
p.yaxis.major_label_text_font_size = '10pt'
return p
def test_get_submission_by_id(session_scope_module):
submission = get_submission_by_id(session_scope_module, 1)
assert isinstance(submission, Submission)
assert submission.basename == 'submission_000000001'
assert os.path.exists(os.path.join(submission.path, 'classifier.py'))
assert submission.state == 'trained'
