def update_submission(submission, status, job_id):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = open(ozip.extract('scores.txt'), 'r').read()
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
result = Job.FINISHED
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def setUp(self):
self.organizer = User.objects.create_user(username="******", password="******")
self.participant_user = User.objects.create_user(username="******", password="******")
self.other_user = User.objects.create_user(username="******", password="******")
self.competition = Competition.objects.create(creator=self.organizer, modified_by=self.organizer, published=True)
self.participant_1 = CompetitionParticipant.objects.create(
user=self.participant_user,
competition=self.competition,
status=ParticipantStatus.objects.get_or_create(name='approved', codename=ParticipantStatus.APPROVED)[0]
)
self.organizer_participant = CompetitionParticipant.objects.create(
user=self.organizer,
competition=self.competition,
status=ParticipantStatus.objects.get_or_create(name='approved', codename=ParticipantStatus.APPROVED)[0]
)
self.phase_1 = CompetitionPhase.objects.create(
competition=self.competition,
phasenumber=1,
start_date=datetime.datetime.now() - datetime.timedelta(days=30),
)
submission_finished = CompetitionSubmissionStatus.objects.create(name="finished", codename="finished")
submission_failed = CompetitionSubmissionStatus.objects.create(name="failed", codename="failed")
self.submission_1 = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)
)
self.submission_2 = CompetitionSubmission.objects.create(
participant=self.organizer_participant,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)
)
self.submission_3 = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_failed,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29)
)
self.submission_1.status = submission_finished
self.submission_1.save()
self.submission_2.status = submission_finished
self.submission_2.save()
self.submission_3.status = submission_failed
self.submission_3.save()
add_submission_to_leaderboard(self.submission_2)
self.client = Client()
self.url = reverse("competitions:submission_toggle_leaderboard", kwargs={"submission_pk": self.submission_1.pk})
def test_toggle_leaderboard_returns_200_and_removes_submission_from_leaderboard(self):
add_submission_to_leaderboard(self.submission_1)
self.client.login(username="******", password="******")
resp = self.client.post(self.url)
self.assertEquals(resp.status_code, 200)
submission_exists_on_leaderboard = PhaseLeaderBoardEntry.objects.filter(
result=self.submission_1
).exists()
self.assertFalse(submission_exists_on_leaderboard)
# Make sure the other submission remains untouched
self.assertTrue(PhaseLeaderBoardEntry.objects.get(result=self.submission_2))
def addToLeaderboard(self, request, pk=None, competition_id=None):
try:
participant = webmodels.CompetitionParticipant.objects.filter(
competition=self.kwargs['competition_id'],
user=self.request.user).get()
except ObjectDoesNotExist:
raise PermissionDenied()
if not participant.is_approved:
raise PermissionDenied()
submission = webmodels.CompetitionSubmission.objects.get(id=pk)
if not submission.phase.is_active:
raise PermissionDenied(detail='Competition phase is closed.')
if submission.phase.is_blind:
raise PermissionDenied(
detail=
'Competition phase does not allow participants to modify the leaderboard.'
)
if submission.participant.user != self.request.user:
raise ParseError(detail='Invalid submission')
response = dict()
_, cr = webmodels.add_submission_to_leaderboard(submission)
response['status'] = (201 if cr else 200)
return Response(response,
status=response['status'],
content_type="application/json")
def push_submission_to_leaderboard_if_best(submission):
from apps.web.models import PhaseLeaderBoard, PhaseLeaderBoardEntry, add_submission_to_leaderboard
# In this phase get the submission score from the column with the lowest ordering
score_def = submission.get_default_score_def()
lb = PhaseLeaderBoard.objects.get(phase=submission.phase)
# Get our leaderboard entries: Related Submissions should be in our participant's submissions,
# and the leaderboard should be the one attached to our phase
entries = PhaseLeaderBoardEntry.objects.filter(result__in=submission.participant.submissions.all(), board=lb)
submissions = [(entry.result, entry.result.get_default_score()) for entry in entries]
sorted_list = sorted(submissions, key=lambda x: x[1])
if sorted_list:
top_sub, top_score = sorted_list[0]
score_value = submission.get_default_score()
if score_def.sorting == 'asc':
# The last value in ascending is the top score, 1 beats 3
if score_value <= top_score:
add_submission_to_leaderboard(submission)
logger.info("Force best submission added submission to leaderboard in ascending order "
"(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score,
score_value)
elif score_def.sorting == 'desc':
# The first value in descending is the top score, 3 beats 1
if score_value >= top_score:
add_submission_to_leaderboard(submission)
logger.info(
"Force best submission added submission to leaderboard in descending order "
"(submission_id=%s, top_score=%s, score=%s)", submission.id, top_score, score_value)
else:
add_submission_to_leaderboard(submission)
logger.info(
"Force best submission added submission: {0} with score: {1} to leaderboard: {2}"
" because no submission was present".format(
submission, submission.get_default_score(), lb)
)
def addToLeaderboard(self, request, pk=None, competition_id=None):
try:
participant = webmodels.CompetitionParticipant.objects.filter(competition=self.kwargs['competition_id'],
user=self.request.user).get()
except ObjectDoesNotExist:
raise PermissionDenied()
if not participant.is_approved:
raise PermissionDenied()
submission = webmodels.CompetitionSubmission.objects.get(id=pk)
if not submission.phase.is_active:
raise PermissionDenied(detail='Competition phase is closed.')
if submission.phase.is_blind:
raise PermissionDenied(detail='Competition phase does not allow participants to modify the leaderboard.')
if submission.participant.user != self.request.user:
raise ParseError(detail='Invalid submission')
response = dict()
_, cr = webmodels.add_submission_to_leaderboard(submission)
response['status'] = (201 if cr else 200)
return Response(response, status=response['status'], content_type="application/json")
def push_submission_to_leaderboard_if_best(submission):
from apps.web.models import PhaseLeaderBoard, PhaseLeaderBoardEntry, add_submission_to_leaderboard
# In this phase get the submission score from the column with the lowest ordering
score_def = submission.get_default_score_def()
lb = PhaseLeaderBoard.objects.get(phase=submission.phase)
# Get our leaderboard entries: Related Submissions should be in our participant's submissions,
# and the leaderboard should be the one attached to our phase
entries = PhaseLeaderBoardEntry.objects.filter(
result__in=submission.participant.submissions.all(), board=lb)
submissions = [(entry.result, entry.result.get_default_score())
for entry in entries]
sorted_list = sorted(submissions, key=lambda x: x[1])
if sorted_list:
top_sub, top_score = sorted_list[0]
score_value = submission.get_default_score()
if score_def.sorting == 'asc':
# The last value in ascending is the top score, 1 beats 3
if score_value <= top_score:
add_submission_to_leaderboard(submission)
logger.info(
"Force best submission added submission to leaderboard in ascending order "
"(submission_id=%s, top_score=%s, score=%s)",
submission.id, top_score, score_value)
elif score_def.sorting == 'desc':
# The first value in descending is the top score, 3 beats 1
if score_value >= top_score:
add_submission_to_leaderboard(submission)
logger.info(
"Force best submission added submission to leaderboard in descending order "
"(submission_id=%s, top_score=%s, score=%s)",
submission.id, top_score, score_value)
else:
add_submission_to_leaderboard(submission)
logger.info(
"Force best submission added submission: {0} with score: {1} to leaderboard: {2}"
" because no submission was present".format(
submission, submission.get_default_score(), lb))
def setUp(self):
self.organizer = User.objects.create_user(username="******",
password="******")
self.participant_user = User.objects.create_user(
username="******", password="******")
self.other_user = User.objects.create_user(username="******",
password="******")
self.competition = Competition.objects.create(
creator=self.organizer, modified_by=self.organizer, published=True)
self.participant_1 = CompetitionParticipant.objects.create(
user=self.participant_user,
competition=self.competition,
status=ParticipantStatus.objects.get_or_create(
name='approved', codename=ParticipantStatus.APPROVED)[0])
self.organizer_participant = CompetitionParticipant.objects.create(
user=self.organizer,
competition=self.competition,
status=ParticipantStatus.objects.get_or_create(
name='approved', codename=ParticipantStatus.APPROVED)[0])
self.phase_1 = CompetitionPhase.objects.create(
competition=self.competition,
phasenumber=1,
start_date=datetime.datetime.now() - datetime.timedelta(days=30),
)
self.phase_1.reference_data.save(
'test-file.txt', ContentFile("This is some fake reference data"))
self.phase_1.save()
assert self.phase_1.reference_data.name
submission_finished = CompetitionSubmissionStatus.objects.create(
name="finished", codename="finished")
submission_failed = CompetitionSubmissionStatus.objects.create(
name="failed", codename="failed")
self.submission_1 = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29))
self.submission_2 = CompetitionSubmission.objects.create(
participant=self.organizer_participant,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29))
self.submission_3 = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_failed,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29))
self.submission_4 = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29))
# Scores setup
self.leader_board = PhaseLeaderBoard.objects.create(phase=self.phase_1)
self.leader_board_entry_1 = PhaseLeaderBoardEntry.objects.create(
board=self.leader_board, result=self.submission_1)
self.result_group = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key",
label=u"Test \u2020",
ordering=1)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.result_group)
self.score_def = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key",
label=u"Test \u2020",
sorting='desc',
)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def,
group=self.result_group)
SubmissionScore.objects.create(result=self.submission_1,
scoredef=self.score_def,
value=123)
SubmissionScore.objects.create(result=self.submission_2,
scoredef=self.score_def,
value=120)
SubmissionScoreSet.objects.create(competition=self.competition,
key="Key",
label=u"Test \u2020",
scoredef=self.score_def)
# End scores setup
self.submission_1.status = submission_finished
self.submission_1.save()
self.submission_2.status = submission_finished
self.submission_2.save()
self.submission_3.status = submission_failed
self.submission_3.save()
self.submission_4.status = submission_finished
self.submission_4.save()
add_submission_to_leaderboard(self.submission_2)
self.client = Client()
self.toggle_url = reverse(
"competitions:submission_toggle_leaderboard",
kwargs={"submission_pk": self.submission_1.pk})
def test_submission_scoring_forced_best_to_leaderboard_exact_conditions(
self):
self.competition.force_submission_to_leaderboard = True
self.competition.disallow_leaderboard_modifying = True
self.competition.save()
self.phase_1.force_best_submission_to_leaderboard = True
self.phase_1.save()
# Score def 2
self.result_group2 = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key2",
label=u"Test2 \u2020",
ordering=2)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.result_group2)
self.score_def2 = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key2",
label=u"Test2 \u2020",
sorting='desc',
ordering=2)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def2,
group=self.result_group2)
# Score def 3
self.resultgroup3 = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key3",
label=u"Test3 \u2020",
ordering=3)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.resultgroup3)
self.score_def3 = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key3",
label=u"Test3 \u2020",
sorting='asc',
ordering=3)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def3,
group=self.resultgroup3)
# Score def 4
self.resultgroup4 = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key4",
label=u"Test4 \u2020",
ordering=4)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.resultgroup4)
self.score_def4 = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key4",
label=u"Test4 \u2020",
sorting='asc',
ordering=4)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def4,
group=self.resultgroup4)
# Score def 5
self.resultgroup5 = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key5",
label=u"Test5 \u2020",
ordering=5)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.resultgroup5)
self.score_def5 = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key5",
label=u"Test5 \u2020",
sorting='desc',
ordering=5)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def5,
group=self.resultgroup5)
# Score def 6
self.resultgroup6 = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key6",
label=u"Test6 \u2020",
ordering=6)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.resultgroup6)
self.score_def6 = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key6",
label=u"Test6 \u2020",
sorting='desc',
ordering=6)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def6,
group=self.resultgroup6)
# score def 7
self.resultgroup7 = SubmissionResultGroup.objects.create(
competition=self.competition,
key="Key7",
label=u"Test7 \u2020",
ordering=7)
self.submission_result_group_phase = SubmissionResultGroupPhase.objects.create(
phase=self.phase_1, group=self.resultgroup7)
self.score_def7 = SubmissionScoreDef.objects.create(
competition=self.competition,
key="Key7",
label=u"Test7 \u2020",
sorting='desc',
ordering=7)
SubmissionScoreDefGroup.objects.create(scoredef=self.score_def7,
group=self.resultgroup7)
submission_finished = CompetitionSubmissionStatus.objects.get(
name="finished", codename="finished")
self.sub_test = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29))
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def,
value=float(1.0510))
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def2,
value=0.5)
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def3,
value=1.0)
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def4,
value=1.0)
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def5,
value=1.0)
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def6,
value=1.0)
SubmissionScore.objects.create(result=self.sub_test,
scoredef=self.score_def7,
value=1.0)
add_submission_to_leaderboard(self.sub_test)
print(
PhaseLeaderBoardEntry.objects.filter(
board=self.leader_board,
result__participant=self.participant_1).first().result)
self.sub_test_two = CompetitionSubmission.objects.create(
participant=self.participant_1,
phase=self.phase_1,
status=submission_finished,
submitted_at=datetime.datetime.now() - datetime.timedelta(days=29))
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def,
value=float(1.0533))
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def2,
value=0.4)
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def3,
value=0.65)
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def4,
value=0.65)
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def5,
value=0.65)
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def6,
value=0.65)
SubmissionScore.objects.create(result=self.sub_test_two,
scoredef=self.score_def7,
value=0.65)
push_submission_to_leaderboard_if_best(self.sub_test_two)
# result__participant = self.participant_1).first().result)
assert PhaseLeaderBoardEntry.objects.filter(board=self.leader_board,
result=self.sub_test_two)
def update_submission(submission,
status,
job_id,
traceback=None,
metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" %
submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug(
"saving extra metadata, was a new object created? %s" %
created)
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(
submission_detailed_results_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
logger.debug("Output.zip location=%s" %
submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error(
"Scores.txt not found, unable to process submission: %s (submission_id=%s)",
status, submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" %
(label, value))
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (
Site.objects.get_current().domain,
submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s'
% (submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL, [email],
fail_silently=False)
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(
predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(
predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
def update_submission(submission, status, job_id, traceback=None, metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" % submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug("saving extra metadata, was a new object created? %s" % created)
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
logger.debug("Output.zip location=%s" % submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" % (label, value))
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (Site.objects.get_current().domain, submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s' %
(submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL,
[email],
fail_silently=False
)
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
