def update_submission(submission, status, job_id, traceback=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
logger.debug("Output.zip location=%s" % submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" % (label, value))
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)
result = Job.FINISHED
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def update_submission(submission, status, job_id, traceback=None, metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" % submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug("saving extra metadata, was a new object created? %s" % created)
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
logger.debug("Output.zip location=%s" % submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" % (label, value))
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (Site.objects.get_current().domain, submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s' %
(submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL,
[email],
fail_silently=False
)
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def update_submission(submission,
status,
job_id,
traceback=None,
metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" %
submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug(
"saving extra metadata, was a new object created? %s" %
created)
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(
submission_detailed_results_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
logger.debug("Output.zip location=%s" %
submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error(
"Scores.txt not found, unable to process submission: %s (submission_id=%s)",
status, submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" %
(label, value))
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (
Site.objects.get_current().domain,
submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s'
% (submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL, [email],
fail_silently=False)
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(
predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(
predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
def update_submission(submission, status, job_id, traceback=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(
submission_detailed_results_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
logger.debug("Output.zip location=%s" %
submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error(
"Scores.txt not found, unable to process submission: %s (submission_id=%s)",
status, submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" %
(label, value))
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(
predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(
predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)