def update_database_results(self, session): """Update the database with the results of ramp_test_submission.""" make_update_leaderboard = False while not self._processed_submission_queue.empty(): make_update_leaderboard = True submission_id, submission_name = \ self._processed_submission_queue.get_nowait() if 'error' in get_submission_state(session, submission_id): continue logger.info('Write info in database for submission {}'.format( submission_name)) path_predictions = os.path.join( self._worker_config['predictions_dir'], submission_name) # NOTE: In the past we were adding the predictions into the # database. Since they require too much space, we stop to store # them in the database and instead, keep it onto the disk. # set_predictions(session, submission_id, path_predictions) set_time(session, submission_id, path_predictions) set_scores(session, submission_id, path_predictions) set_bagged_scores(session, submission_id, path_predictions) set_submission_state(session, submission_id, 'scored') if make_update_leaderboard: logger.info('Update all leaderboards') update_leaderboards(session, self._ramp_config['event_name']) update_all_user_leaderboards(session, self._ramp_config['event_name'])
def update_database_results(self, session): """Update the database with the results of ramp_test_submission.""" while not self._processed_submission_queue.empty(): submission_id, submission_name = \ self._processed_submission_queue.get_nowait() if 'error' in get_submission_state(session, submission_id): update_leaderboards(session, self._ramp_config['event_name']) update_all_user_leaderboards(session, self._ramp_config['event_name']) logger.info('Skip update for {} due to failure during the ' 'processing'.format(submission_name)) continue logger.info('Update the results obtained on each fold for ' '{}'.format(submission_name)) path_predictions = os.path.join( self._worker_config['predictions_dir'], submission_name ) set_predictions(session, submission_id, path_predictions) set_time(session, submission_id, path_predictions) set_scores(session, submission_id, path_predictions) set_bagged_scores(session, submission_id, path_predictions) set_submission_state(session, submission_id, 'scored') update_leaderboards(session, self._ramp_config['event_name']) update_all_user_leaderboards(session, self._ramp_config['event_name'])
def test_check_scores(session_scope_module): # check both set_scores and get_scores submission_id = 1 path_results = os.path.join(HERE, 'data', 'iris_predictions') set_scores(session_scope_module, submission_id, path_results) scores = get_scores(session_scope_module, submission_id) multi_index = pd.MultiIndex.from_product( [[0, 1], ['train', 'valid', 'test']], names=['fold', 'step'] ) expected_df = pd.DataFrame( {'acc': [0.604167, 0.583333, 0.733333, 0.604167, 0.583333, 0.733333], 'error': [0.395833, 0.416667, 0.266667, 0.395833, 0.416667, 0.266667], 'nll': [0.732763, 2.194549, 0.693464, 0.746132, 2.030762, 0.693992], 'f1_70': [0.333333, 0.33333, 0.666667, 0.33333, 0.33333, 0.666667]}, index=multi_index ) assert_frame_equal(scores, expected_df, check_less_precise=True)
def update_database_results(self, session): """Update the database with the results of ramp_test_submission.""" make_update_leaderboard = False while not self._processed_submission_queue.empty(): make_update_leaderboard = True submission_id, submission_name = \ self._processed_submission_queue.get_nowait() if 'error' in get_submission_state(session, submission_id): continue logger.info('Write info in data base for submission {}'.format( submission_name)) path_predictions = os.path.join( self._worker_config['predictions_dir'], submission_name) set_predictions(session, submission_id, path_predictions) set_time(session, submission_id, path_predictions) set_scores(session, submission_id, path_predictions) set_bagged_scores(session, submission_id, path_predictions) set_submission_state(session, submission_id, 'scored') if make_update_leaderboard: logger.info('Update all leaderboards') update_leaderboards(session, self._ramp_config['event_name']) update_all_user_leaderboards(session, self._ramp_config['event_name'])