def update_submission(submission, status, job_id): """ Updates the status of a submission. submission: The CompetitionSubmission object to update. status: The new status string: 'running', 'finished' or 'failed'. job_id: The job ID used to track the progress of the evaluation. """ if status == 'running': _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING) return Job.RUNNING if status == 'finished': result = Job.FAILED state = {} if len(submission.execution_key) > 0: logger.debug("update_submission_task loading state: %s", submission.execution_key) state = json.loads(submission.execution_key) if 'score' in state: logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk) submission.output_file.name = pathname2url(submission_output_filename(submission)) submission.save() logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id) ozip = ZipFile(io.BytesIO(submission.output_file.read())) scores = open(ozip.extract('scores.txt'), 'r').read() logger.debug("Processing scores... (submission_id=%s)", submission.id) for line in scores.split("\n"): if len(line) > 0: label, value = line.split(":") try: scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition, key=label.strip()) SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value)) except SubmissionScoreDef.DoesNotExist: logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id) logger.debug("Done processing scores... (submission_id=%s)", submission.id) _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED) # Automatically submit to the leaderboard? if submission.phase.is_blind: logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id) add_submission_to_leaderboard(submission) logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id) result = Job.FINISHED else: logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk) url_name = pathname2url(submission_prediction_output_filename(submission)) submission.prediction_output_file.name = url_name submission.save() try: score(submission, job_id) result = Job.RUNNING logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk) except Exception: logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk) return result if status != 'failed': logger.error("Invalid status: %s (submission_id=%s)", status, submission.id) _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def setUp(self): self.organizer = User.objects.create_user(username="******", password="******") self.participant_user = User.objects.create_user(username="******", password="******") self.other_user = User.objects.create_user(username="******", password="******") self.competition = Competition.objects.create(creator=self.organizer, modified_by=self.organizer, published=True) self.participant_1 = CompetitionParticipant.objects.create( user=self.participant_user, competition=self.competition, status=ParticipantStatus.objects.get_or_create(name='approved', codename=ParticipantStatus.APPROVED)[0] ) self.organizer_participant = CompetitionParticipant.objects.create( user=self.organizer, competition=self.competition, status=ParticipantStatus.objects.get_or_create(name='approved', codename=ParticipantStatus.APPROVED)[0] ) self.phase_1 = CompetitionPhase.objects.create( competition=self.competition, phasenumber=1, start_date=datetime.datetime.now() - datetime.timedelta(days=30), ) submission_finished = CompetitionSubmissionStatus.objects.create(name="finished", codename="finished") submission_failed = CompetitionSubmissionStatus.objects.create(name="failed", codename="failed") self.submission_1 = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29) ) self.submission_2 = CompetitionSubmission.objects.create( participant=self.organizer_participant, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29) ) self.submission_3 = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_failed, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29) ) self.submission_1.status = submission_finished self.submission_1.save() self.submission_2.status = submission_finished self.submission_2.save() self.submission_3.status = submission_failed self.submission_3.save() add_submission_to_leaderboard(self.submission_2) self.client = Client() self.url = reverse("competitions:submission_toggle_leaderboard", kwargs={"submission_pk": self.submission_1.pk})
def test_toggle_leaderboard_returns_200_and_removes_submission_from_leaderboard(self): add_submission_to_leaderboard(self.submission_1) self.client.login(username="******", password="******") resp = self.client.post(self.url) self.assertEquals(resp.status_code, 200) submission_exists_on_leaderboard = PhaseLeaderBoardEntry.objects.filter( result=self.submission_1 ).exists() self.assertFalse(submission_exists_on_leaderboard) # Make sure the other submission remains untouched self.assertTrue(PhaseLeaderBoardEntry.objects.get(result=self.submission_2))
def addToLeaderboard(self, request, pk=None, competition_id=None): try: participant = webmodels.CompetitionParticipant.objects.filter( competition=self.kwargs['competition_id'], user=self.request.user).get() except ObjectDoesNotExist: raise PermissionDenied() if not participant.is_approved: raise PermissionDenied() submission = webmodels.CompetitionSubmission.objects.get(id=pk) if not submission.phase.is_active: raise PermissionDenied(detail='Competition phase is closed.') if submission.phase.is_blind: raise PermissionDenied( detail= 'Competition phase does not allow participants to modify the leaderboard.' ) if submission.participant.user != self.request.user: raise ParseError(detail='Invalid submission') response = dict() _, cr = webmodels.add_submission_to_leaderboard(submission) response['status'] = (201 if cr else 200) return Response(response, status=response['status'], content_type="application/json")
def push_submission_to_leaderboard_if_best(submission): from apps.web.models import PhaseLeaderBoard, PhaseLeaderBoardEntry, add_submission_to_leaderboard # In this phase get the submission score from the column with the lowest ordering score_def = submission.get_default_score_def() lb = PhaseLeaderBoard.objects.get(phase=submission.phase) # Get our leaderboard entries: Related Submissions should be in our participant's submissions, # and the leaderboard should be the one attached to our phase entries = PhaseLeaderBoardEntry.objects.filter(result__in=submission.participant.submissions.all(), board=lb) submissions = [(entry.result, entry.result.get_default_score()) for entry in entries] sorted_list = sorted(submissions, key=lambda x: x[1]) if sorted_list: top_sub, top_score = sorted_list[0] score_value = submission.get_default_score() if score_def.sorting == 'asc': # The last value in ascending is the top score, 1 beats 3 if score_value <= top_score: add_submission_to_leaderboard(submission) logger.info("Force best submission added submission to leaderboard in ascending order " "(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score, score_value) elif score_def.sorting == 'desc': # The first value in descending is the top score, 3 beats 1 if score_value >= top_score: add_submission_to_leaderboard(submission) logger.info( "Force best submission added submission to leaderboard in descending order " "(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score, score_value) else: add_submission_to_leaderboard(submission) logger.info( "Force best submission added submission: {0} with score: {1} to leaderboard: {2}" " because no submission was present".format( submission, submission.get_default_score(), lb) )
def addToLeaderboard(self, request, pk=None, competition_id=None): try: participant = webmodels.CompetitionParticipant.objects.filter(competition=self.kwargs['competition_id'], user=self.request.user).get() except ObjectDoesNotExist: raise PermissionDenied() if not participant.is_approved: raise PermissionDenied() submission = webmodels.CompetitionSubmission.objects.get(id=pk) if not submission.phase.is_active: raise PermissionDenied(detail='Competition phase is closed.') if submission.phase.is_blind: raise PermissionDenied(detail='Competition phase does not allow participants to modify the leaderboard.') if submission.participant.user != self.request.user: raise ParseError(detail='Invalid submission') response = dict() _, cr = webmodels.add_submission_to_leaderboard(submission) response['status'] = (201 if cr else 200) return Response(response, status=response['status'], content_type="application/json")
def push_submission_to_leaderboard_if_best(submission): from apps.web.models import PhaseLeaderBoard, PhaseLeaderBoardEntry, add_submission_to_leaderboard # In this phase get the submission score from the column with the lowest ordering score_def = submission.get_default_score_def() lb = PhaseLeaderBoard.objects.get(phase=submission.phase) # Get our leaderboard entries: Related Submissions should be in our participant's submissions, # and the leaderboard should be the one attached to our phase entries = PhaseLeaderBoardEntry.objects.filter( result__in=submission.participant.submissions.all(), board=lb) submissions = [(entry.result, entry.result.get_default_score()) for entry in entries] sorted_list = sorted(submissions, key=lambda x: x[1]) if sorted_list: top_sub, top_score = sorted_list[0] score_value = submission.get_default_score() if score_def.sorting == 'asc': # The last value in ascending is the top score, 1 beats 3 if score_value <= top_score: add_submission_to_leaderboard(submission) logger.info( "Force best submission added submission to leaderboard in ascending order " "(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score, score_value) elif score_def.sorting == 'desc': # The first value in descending is the top score, 3 beats 1 if score_value >= top_score: add_submission_to_leaderboard(submission) logger.info( "Force best submission added submission to leaderboard in descending order " "(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score, score_value) else: add_submission_to_leaderboard(submission) logger.info( "Force best submission added submission: {0} with score: {1} to leaderboard: {2}" " because no submission was present".format( submission, submission.get_default_score(), lb))
def setUp(self): self.organizer = User.objects.create_user(username="******", password="******") self.participant_user = User.objects.create_user( username="******", password="******") self.other_user = User.objects.create_user(username="******", password="******") self.competition = Competition.objects.create( creator=self.organizer, modified_by=self.organizer, published=True) self.participant_1 = CompetitionParticipant.objects.create( user=self.participant_user, competition=self.competition, status=ParticipantStatus.objects.get_or_create( name='approved', codename=ParticipantStatus.APPROVED)[0]) self.organizer_participant = CompetitionParticipant.objects.create( user=self.organizer, competition=self.competition, status=ParticipantStatus.objects.get_or_create( name='approved', codename=ParticipantStatus.APPROVED)[0]) self.phase_1 = CompetitionPhase.objects.create( competition=self.competition, phasenumber=1, start_date=datetime.datetime.now() - datetime.timedelta(days=30), ) self.phase_1.reference_data.save( 'test-file.txt', ContentFile("This is some fake reference data")) self.phase_1.save() assert self.phase_1.reference_data.name submission_finished = CompetitionSubmissionStatus.objects.create( name="finished", codename="finished") submission_failed = CompetitionSubmissionStatus.objects.create( name="failed", codename="failed") self.submission_1 = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)) self.submission_2 = CompetitionSubmission.objects.create( participant=self.organizer_participant, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)) self.submission_3 = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_failed, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)) self.submission_4 = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)) # Scores setup self.leader_board = PhaseLeaderBoard.objects.create(phase=self.phase_1) self.leader_board_entry_1 = PhaseLeaderBoardEntry.objects.create( board=self.leader_board, result=self.submission_1) self.result_group = SubmissionResultGroup.objects.create( competition=self.competition, key="Key", label=u"Test \u2020", ordering=1) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.result_group) self.score_def = SubmissionScoreDef.objects.create( competition=self.competition, key="Key", label=u"Test \u2020", sorting='desc', ) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def, group=self.result_group) SubmissionScore.objects.create(result=self.submission_1, scoredef=self.score_def, value=123) SubmissionScore.objects.create(result=self.submission_2, scoredef=self.score_def, value=120) SubmissionScoreSet.objects.create(competition=self.competition, key="Key", label=u"Test \u2020", scoredef=self.score_def) # End scores setup self.submission_1.status = submission_finished self.submission_1.save() self.submission_2.status = submission_finished self.submission_2.save() self.submission_3.status = submission_failed self.submission_3.save() self.submission_4.status = submission_finished self.submission_4.save() add_submission_to_leaderboard(self.submission_2) self.client = Client() self.toggle_url = reverse( "competitions:submission_toggle_leaderboard", kwargs={"submission_pk": self.submission_1.pk})
def test_submission_scoring_forced_best_to_leaderboard_exact_conditions( self): self.competition.force_submission_to_leaderboard = True self.competition.disallow_leaderboard_modifying = True self.competition.save() self.phase_1.force_best_submission_to_leaderboard = True self.phase_1.save() # Score def 2 self.result_group2 = SubmissionResultGroup.objects.create( competition=self.competition, key="Key2", label=u"Test2 \u2020", ordering=2) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.result_group2) self.score_def2 = SubmissionScoreDef.objects.create( competition=self.competition, key="Key2", label=u"Test2 \u2020", sorting='desc', ordering=2) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def2, group=self.result_group2) # Score def 3 self.resultgroup3 = SubmissionResultGroup.objects.create( competition=self.competition, key="Key3", label=u"Test3 \u2020", ordering=3) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.resultgroup3) self.score_def3 = SubmissionScoreDef.objects.create( competition=self.competition, key="Key3", label=u"Test3 \u2020", sorting='asc', ordering=3) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def3, group=self.resultgroup3) # Score def 4 self.resultgroup4 = SubmissionResultGroup.objects.create( competition=self.competition, key="Key4", label=u"Test4 \u2020", ordering=4) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.resultgroup4) self.score_def4 = SubmissionScoreDef.objects.create( competition=self.competition, key="Key4", label=u"Test4 \u2020", sorting='asc', ordering=4) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def4, group=self.resultgroup4) # Score def 5 self.resultgroup5 = SubmissionResultGroup.objects.create( competition=self.competition, key="Key5", label=u"Test5 \u2020", ordering=5) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.resultgroup5) self.score_def5 = SubmissionScoreDef.objects.create( competition=self.competition, key="Key5", label=u"Test5 \u2020", sorting='desc', ordering=5) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def5, group=self.resultgroup5) # Score def 6 self.resultgroup6 = SubmissionResultGroup.objects.create( competition=self.competition, key="Key6", label=u"Test6 \u2020", ordering=6) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.resultgroup6) self.score_def6 = SubmissionScoreDef.objects.create( competition=self.competition, key="Key6", label=u"Test6 \u2020", sorting='desc', ordering=6) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def6, group=self.resultgroup6) # score def 7 self.resultgroup7 = SubmissionResultGroup.objects.create( competition=self.competition, key="Key7", label=u"Test7 \u2020", ordering=7) self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create( phase=self.phase_1, group=self.resultgroup7) self.score_def7 = SubmissionScoreDef.objects.create( competition=self.competition, key="Key7", label=u"Test7 \u2020", sorting='desc', ordering=7) SubmissionScoreDefGroup.objects.create(scoredef=self.score_def7, group=self.resultgroup7) submission_finished = CompetitionSubmissionStatus.objects.get( name="finished", codename="finished") self.sub_test = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def, value=float(1.0510)) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def2, value=0.5) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def3, value=1.0) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def4, value=1.0) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def5, value=1.0) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def6, value=1.0) SubmissionScore.objects.create(result=self.sub_test, scoredef=self.score_def7, value=1.0) add_submission_to_leaderboard(self.sub_test) print( PhaseLeaderBoardEntry.objects.filter( board=self.leader_board, result__participant=self.participant_1).first().result) self.sub_test_two = CompetitionSubmission.objects.create( participant=self.participant_1, phase=self.phase_1, status=submission_finished, submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def, value=float(1.0533)) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def2, value=0.4) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def3, value=0.65) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def4, value=0.65) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def5, value=0.65) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def6, value=0.65) SubmissionScore.objects.create(result=self.sub_test_two, scoredef=self.score_def7, value=0.65) push_submission_to_leaderboard_if_best(self.sub_test_two) # result__participant = self.participant_1).first().result) assert PhaseLeaderBoardEntry.objects.filter(board=self.leader_board, result=self.sub_test_two)
def update_submission(submission, status, job_id, traceback=None, metadata=None): """ Updates the status of a submission. submission: The CompetitionSubmission object to update. status: The new status string: 'running', 'finished' or 'failed'. job_id: The job ID used to track the progress of the evaluation. """ state = {} if len(submission.execution_key) > 0: logger.debug("update_submission_task loading state: %s", submission.execution_key) state = json.loads(submission.execution_key) logger.debug("update_submission_task state = %s" % submission.execution_key) if metadata: is_predict = 'score' not in state sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create( is_predict=is_predict, is_scoring=not is_predict, submission=submission, ) sub_metadata.__dict__.update(metadata) sub_metadata.save() logger.debug( "saving extra metadata, was a new object created? %s" % created) if status == 'running': _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING) return Job.RUNNING if status == 'finished': result = Job.FAILED if 'score' in state: logger.debug( "update_submission_task loading final scores (pk=%s)", submission.pk) submission.output_file.name = pathname2url( submission_output_filename(submission)) submission.private_output_file.name = pathname2url( submission_private_output_filename(submission)) submission.detailed_results_file.name = pathname2url( submission_detailed_results_filename(submission)) submission.save() logger.debug( "Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id) logger.debug("Output.zip location=%s" % submission.output_file.file.name) ozip = ZipFile(io.BytesIO(submission.output_file.read())) scores = None try: scores = open(ozip.extract('scores.txt'), 'r').read() except Exception: logger.error( "Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id) _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED) return Job.FAILED logger.debug("Processing scores... (submission_id=%s)", submission.id) for line in scores.split("\n"): if len(line) > 0: label, value = line.split(":") logger.debug("Attempting to submit score %s:%s" % (label, value)) try: scoredef = SubmissionScoreDef.objects.get( competition=submission.phase.competition, key=label.strip()) SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value)) except SubmissionScoreDef.DoesNotExist: logger.warning( "Score %s does not exist (submission_id=%s)", label, submission.id) logger.debug("Done processing scores... (submission_id=%s)", submission.id) _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED) # Automatically submit to the leaderboard? if submission.phase.is_blind: logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id) add_submission_to_leaderboard(submission) logger.debug( "Leaderboard updated with latest submission (submission_id=%s)", submission.id) if submission.phase.competition.force_submission_to_leaderboard: add_submission_to_leaderboard(submission) logger.debug( "Force submission added submission to leaderboard (submission_id=%s)", submission.id) result = Job.FINISHED if submission.participant.user.email_on_submission_finished_successfully: email = submission.participant.user.email site_url = "https://%s%s" % ( Site.objects.get_current().domain, submission.phase.competition.get_absolute_url()) send_mail( 'Submission has finished successfully!', 'Your submission to the competition "%s" has finished successfully! View it here: %s' % (submission.phase.competition.title, site_url), settings.DEFAULT_FROM_EMAIL, [email], fail_silently=False) else: logger.debug( "update_submission_task entering scoring phase (pk=%s)", submission.pk) url_name = pathname2url( submission_prediction_output_filename(submission)) submission.prediction_output_file.name = url_name submission.prediction_stderr_file.name = pathname2url( predict_submission_stdout_filename(submission)) submission.prediction_stdout_file.name = pathname2url( predict_submission_stderr_filename(submission)) submission.save() try: score(submission, job_id) result = Job.RUNNING logger.debug( "update_submission_task scoring phase entered (pk=%s)", submission.pk) except Exception: logger.exception( "update_submission_task failed to enter scoring phase (pk=%s)", submission.pk) return result if status != 'failed': logger.error("Invalid status: %s (submission_id=%s)", status, submission.id) if traceback: submission.exception_details = traceback submission.save() _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def update_submission(submission, status, job_id, traceback=None, metadata=None): """ Updates the status of a submission. submission: The CompetitionSubmission object to update. status: The new status string: 'running', 'finished' or 'failed'. job_id: The job ID used to track the progress of the evaluation. """ state = {} if len(submission.execution_key) > 0: logger.debug("update_submission_task loading state: %s", submission.execution_key) state = json.loads(submission.execution_key) logger.debug("update_submission_task state = %s" % submission.execution_key) if metadata: is_predict = 'score' not in state sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create( is_predict=is_predict, is_scoring=not is_predict, submission=submission, ) sub_metadata.__dict__.update(metadata) sub_metadata.save() logger.debug("saving extra metadata, was a new object created? %s" % created) if status == 'running': _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING) return Job.RUNNING if status == 'finished': result = Job.FAILED if 'score' in state: logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk) submission.output_file.name = pathname2url(submission_output_filename(submission)) submission.private_output_file.name = pathname2url(submission_private_output_filename(submission)) submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission)) submission.save() logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id) logger.debug("Output.zip location=%s" % submission.output_file.file.name) ozip = ZipFile(io.BytesIO(submission.output_file.read())) scores = None try: scores = open(ozip.extract('scores.txt'), 'r').read() except Exception: logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id) _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED) return Job.FAILED logger.debug("Processing scores... (submission_id=%s)", submission.id) for line in scores.split("\n"): if len(line) > 0: label, value = line.split(":") logger.debug("Attempting to submit score %s:%s" % (label, value)) try: scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition, key=label.strip()) SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value)) except SubmissionScoreDef.DoesNotExist: logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id) logger.debug("Done processing scores... (submission_id=%s)", submission.id) _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED) # Automatically submit to the leaderboard? if submission.phase.is_blind: logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id) add_submission_to_leaderboard(submission) logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id) if submission.phase.competition.force_submission_to_leaderboard: add_submission_to_leaderboard(submission) logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id) result = Job.FINISHED if submission.participant.user.email_on_submission_finished_successfully: email = submission.participant.user.email site_url = "https://%s%s" % (Site.objects.get_current().domain, submission.phase.competition.get_absolute_url()) send_mail( 'Submission has finished successfully!', 'Your submission to the competition "%s" has finished successfully! View it here: %s' % (submission.phase.competition.title, site_url), settings.DEFAULT_FROM_EMAIL, [email], fail_silently=False ) else: logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk) url_name = pathname2url(submission_prediction_output_filename(submission)) submission.prediction_output_file.name = url_name submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission)) submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission)) submission.save() try: score(submission, job_id) result = Job.RUNNING logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk) except Exception: logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk) return result if status != 'failed': logger.error("Invalid status: %s (submission_id=%s)", status, submission.id) if traceback: submission.exception_details = traceback submission.save() _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)