def fetch_from_db(self, session): """Fetch the submission from the database and create the workers.""" submissions = get_submissions(session, self._ramp_config['event_name'], state='new') if not submissions: return for submission_id, submission_name, _ in submissions: # do not train the sandbox submission submission = get_submission_by_id(session, submission_id) if not submission.is_not_sandbox: continue # create the worker worker = self.worker(self._worker_config, submission_name) set_submission_state(session, submission_id, 'sent_to_training') update_user_leaderboards( session, self._ramp_config['event_name'], submission.team.name, new_only=True, ) self._awaiting_worker_queue.put_nowait( (worker, (submission_id, submission_name))) logger.info('Submission {} added to the queue of submission to be ' 'processed'.format(submission_name))
def launch_workers(self, session): """Launch the awaiting workers if possible.""" while (not self._processing_worker_queue.full() and not self._awaiting_worker_queue.empty()): worker, (submission_id, submission_name) = \ self._awaiting_worker_queue.get() logger.info('Starting worker: {}'.format(worker)) worker.setup() if worker.status == 'error': set_submission_state(session, submission_id, 'checking_error') continue worker.launch_submission() if worker.status == 'error': set_submission_state(session, submission_id, 'checking_error') continue set_submission_state(session, submission_id, 'training') submission = get_submission_by_id(session, submission_id) update_user_leaderboards( session, self._ramp_config['event_name'], submission.team.name, new_only=True, ) self._processing_worker_queue.put_nowait( (worker, (submission_id, submission_name))) logger.info( 'Store the worker {} into the processing queue'.format(worker))
def launch_workers(self, session): """Launch the awaiting workers if possible.""" while (not self._processing_worker_queue.full() and not self._awaiting_worker_queue.empty()): worker, (submission_id, submission_name) = \ self._awaiting_worker_queue.get() self._logger.info(f'Starting worker: {worker}') try: worker.setup() if worker.status != "error": worker.launch_submission() except Exception as e: self._logger.error( f'Worker finished with unhandled exception:\n {e}') worker.status = 'error' if worker.status == 'error': set_submission_state(session, submission_id, 'checking_error') worker.teardown() # kill the worker self._logger.info(f'Worker {worker} killed due to an error ' f'while connecting to AWS worker') stderr = ("There was a problem with sending your submission" " for training. This problem is on RAMP side" " and most likely it is not related to your" " code. If this happened for the first time" " to this submission you might" " consider submitting the same code once again." " Else, please contact the event organizers.") set_submission_error_msg(session, submission_id, stderr) continue set_submission_state(session, submission_id, 'training') submission = get_submission_by_id(session, submission_id) update_user_leaderboards( session, self._ramp_config['event_name'], submission.team.name, new_only=True, ) self._processing_worker_queue.put_nowait( (worker, (submission_id, submission_name))) self._logger.info( f'Store the worker {worker} into the processing queue')
def test_update_leaderboard_functions(session_toy_function): event_name = 'iris_test' user_name = 'test_user' for leaderboard_type in ['public', 'private', 'failed', 'public competition', 'private competition']: leaderboard = get_leaderboard(session_toy_function, leaderboard_type, event_name) assert leaderboard is None leaderboard = get_leaderboard(session_toy_function, 'new', event_name) assert leaderboard event = get_event(session_toy_function, event_name) assert event.private_leaderboard_html is None assert event.public_leaderboard_html_with_links is None assert event.public_leaderboard_html_no_links is None assert event.failed_leaderboard_html is None assert event.public_competition_leaderboard_html is None assert event.private_competition_leaderboard_html is None assert event.new_leaderboard_html event_team = get_event_team_by_name(session_toy_function, event_name, user_name) assert event_team.leaderboard_html is None assert event_team.failed_leaderboard_html is None assert event_team.new_leaderboard_html event_teams = (session_toy_function.query(EventTeam) .filter_by(event=event) .all()) for et in event_teams: assert et.leaderboard_html is None assert et.failed_leaderboard_html is None assert et.new_leaderboard_html # run the dispatcher to process the different submissions config = read_config(database_config_template()) event_config = read_config(ramp_config_template()) dispatcher = Dispatcher( config, event_config, n_workers=-1, hunger_policy='exit' ) dispatcher.launch() session_toy_function.commit() update_leaderboards(session_toy_function, event_name) event = get_event(session_toy_function, event_name) assert event.private_leaderboard_html assert event.public_leaderboard_html_with_links assert event.public_leaderboard_html_no_links assert event.failed_leaderboard_html assert event.public_competition_leaderboard_html assert event.private_competition_leaderboard_html assert event.new_leaderboard_html is None update_user_leaderboards(session_toy_function, event_name, user_name) event_team = get_event_team_by_name(session_toy_function, event_name, user_name) assert event_team.leaderboard_html assert event_team.failed_leaderboard_html assert event_team.new_leaderboard_html is None update_all_user_leaderboards(session_toy_function, event_name) event_teams = (session_toy_function.query(EventTeam) .filter_by(event=event) .all()) for et in event_teams: assert et.leaderboard_html assert et.failed_leaderboard_html assert et.new_leaderboard_html is None