def update_database_results(self, session):
"""Update the database with the results of ramp_test_submission."""
while not self._processed_submission_queue.empty():
submission_id, submission_name = \
self._processed_submission_queue.get_nowait()
if 'error' in get_submission_state(session, submission_id):
update_leaderboards(session, self._ramp_config['event_name'])
update_all_user_leaderboards(session,
self._ramp_config['event_name'])
logger.info('Skip update for {} due to failure during the '
'processing'.format(submission_name))
continue
logger.info('Update the results obtained on each fold for '
'{}'.format(submission_name))
path_predictions = os.path.join(
self._worker_config['predictions_dir'], submission_name
)
set_predictions(session, submission_id, path_predictions)
set_time(session, submission_id, path_predictions)
set_scores(session, submission_id, path_predictions)
set_bagged_scores(session, submission_id, path_predictions)
set_submission_state(session, submission_id, 'scored')
update_leaderboards(session, self._ramp_config['event_name'])
update_all_user_leaderboards(session,
self._ramp_config['event_name'])
def test_check_predictions(session_scope_module):
# check both set_predictions and get_predictions
submission_id = 1
path_results = os.path.join(HERE, 'data', 'iris_predictions')
set_predictions(session_scope_module, submission_id, path_results)
predictions = get_predictions(session_scope_module, submission_id)
for fold_idx in range(2):
path_fold = os.path.join(path_results, 'fold_{}'.format(fold_idx))
expected_y_pred_train = np.load(
os.path.join(path_fold, 'y_pred_train.npz'))['y_pred']
expected_y_pred_test = np.load(
os.path.join(path_fold, 'y_pred_test.npz'))['y_pred']
assert_allclose(predictions.loc[fold_idx, 'y_pred_train'],
expected_y_pred_train)
assert_allclose(predictions.loc[fold_idx, 'y_pred_test'],
expected_y_pred_test)
def test_score_submission(session_scope_module):
submission_id = 9
multi_index = pd.MultiIndex.from_product(
[[0, 1], ['train', 'valid', 'test']], names=['fold', 'step']
)
expected_df = pd.DataFrame(
{'acc': [0.604167, 0.583333, 0.733333, 0.604167, 0.583333, 0.733333],
'error': [0.395833, 0.416667, 0.266667, 0.395833, 0.416667, 0.266667],
'nll': [0.732763, 2.194549, 0.693464, 0.746132, 2.030762, 0.693992],
'f1_70': [0.333333, 0.33333, 0.666667, 0.33333, 0.33333, 0.666667]},
index=multi_index
)
path_results = os.path.join(HERE, 'data', 'iris_predictions')
with pytest.raises(ValueError, match='Submission state must be "tested"'):
score_submission(session_scope_module, submission_id)
set_submission_state(session_scope_module, submission_id, 'tested')
set_predictions(session_scope_module, submission_id, path_results)
score_submission(session_scope_module, submission_id)
scores = get_scores(session_scope_module, submission_id)
assert_frame_equal(scores, expected_df, check_less_precise=True)
def update_database_results(self, session):
"""Update the database with the results of ramp_test_submission."""
make_update_leaderboard = False
while not self._processed_submission_queue.empty():
make_update_leaderboard = True
submission_id, submission_name = \
self._processed_submission_queue.get_nowait()
if 'error' in get_submission_state(session, submission_id):
continue
logger.info('Write info in data base for submission {}'.format(
submission_name))
path_predictions = os.path.join(
self._worker_config['predictions_dir'], submission_name)
set_predictions(session, submission_id, path_predictions)
set_time(session, submission_id, path_predictions)
set_scores(session, submission_id, path_predictions)
set_bagged_scores(session, submission_id, path_predictions)
set_submission_state(session, submission_id, 'scored')
if make_update_leaderboard:
logger.info('Update all leaderboards')
update_leaderboards(session, self._ramp_config['event_name'])
update_all_user_leaderboards(session,
self._ramp_config['event_name'])