def update_submission(submission, status, job_id, traceback=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
logger.debug("Output.zip location=%s" % submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" % (label, value))
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)
result = Job.FINISHED
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def score(submission, job_id):
"""
Dispatches the scoring task for the given submission to an appropriate compute worker.
submission: The CompetitionSubmission object.
job_id: The job ID used to track the progress of the evaluation.
"""
# Loads the computation state.
state = {}
if len(submission.execution_key) > 0:
state = json.loads(submission.execution_key)
has_generated_predictions = 'predict' in state
#generate metadata-only bundle describing the history of submissions and phases
last_submissions = CompetitionSubmission.objects.filter(
participant=submission.participant,
status__codename=CompetitionSubmissionStatus.FINISHED
).order_by('-submitted_at')
lines = []
lines.append("description: history of all previous successful runs output files")
if last_submissions:
for past_submission in last_submissions:
if past_submission.pk != submission.pk:
#pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
past_submission_number = '%03d' % past_submission.submission_number
lines.append('%s/%s/output/: %s' % (
past_submission_phasenumber,
past_submission_number,
submission_private_output_filename(past_submission),
)
)
else:
pass
submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))
submission.scores_file.save('scores.txt', ContentFile(submission.phase.competition.get_results_csv(submission.phase.pk)))
# Generate metadata-only bundle describing the inputs. Reference data is an optional
# dataset provided by the competition organizer. Results are provided by the participant
# either indirectly (has_generated_predictions is True i.e. participant provides a program
# which is run to generate results) ordirectly (participant uploads results directly).
lines = []
ref_value = submission.phase.reference_data.name
if len(ref_value) > 0:
lines.append("ref: %s" % ref_value)
res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
if len(res_value) > 0:
lines.append("res: %s" % res_value)
else:
raise ValueError("Results are missing.")
lines.append("history: %s" % submission_history_file_name(submission))
lines.append("scores: %s" % submission_scores_file_name(submission))
lines.append("submitted-by: %s" % submission.participant.user.username)
lines.append("submitted-at: %s" % submission.submitted_at.replace(microsecond=0).isoformat())
lines.append("competition-submission: %s" % submission.submission_number)
lines.append("competition-phase: %s" % submission.phase.phasenumber)
is_automatic_submission = False
if submission.phase.auto_migration:
# If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
submissions_this_phase = CompetitionSubmission.objects.filter(
phase=submission.phase,
participant=submission.participant
).count()
is_automatic_submission = submissions_this_phase == 1
lines.append("automatic-submission: %s" % is_automatic_submission)
submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))
# Generate metadata-only bundle describing the computation.
lines = []
program_value = submission.phase.scoring_program.name
if len(program_value) > 0:
lines.append("program: %s" % program_value)
else:
raise ValueError("Program is missing.")
lines.append("input: %s" % submission.inputfile.name)
lines.append("stdout: %s" % submission_stdout_filename(submission))
lines.append("stderr: %s" % submission_stderr_filename(submission))
submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))
# Create stdout.txt & stderr.txt
if has_generated_predictions == False:
username = submission.participant.user.username
lines = ["Standard output for submission #{0} by {1}.".format(submission.submission_number, username), ""]
submission.stdout_file.save('stdout.txt', ContentFile('\n'.join(lines)))
lines = ["Standard error for submission #{0} by {1}.".format(submission.submission_number, username), ""]
submission.stderr_file.save('stderr.txt', ContentFile('\n'.join(lines)))
# Update workflow state
state['score'] = job_id
submission.execution_key = json.dumps(state)
submission.save()
# Submit the request to the computation service
body = json.dumps({
"id" : job_id,
"task_type": "run",
"task_args": {
"bundle_id" : submission.runfile.name,
"container_name" : settings.BUNDLE_AZURE_CONTAINER,
"reply_to" : settings.SBS_RESPONSE_QUEUE,
"execution_time_limit": submission.phase.execution_time_limit,
"predict": False,
}
})
getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
if has_generated_predictions == False:
_set_submission_status(submission.id, CompetitionSubmissionStatus.SUBMITTED)
def score(submission, job_id):
"""
Dispatches the scoring task for the given submission to an appropriate compute worker.
submission: The CompetitionSubmission object.
job_id: The job ID used to track the progress of the evaluation.
"""
# Loads the computation state.
state = {}
if len(submission.execution_key) > 0:
state = json.loads(submission.execution_key)
has_generated_predictions = 'predict' in state
#generate metadata-only bundle describing the history of submissions and phases
last_submissions = CompetitionSubmission.objects.filter(
participant=submission.participant,
status__codename=CompetitionSubmissionStatus.FINISHED
).order_by('-submitted_at')
lines = []
lines.append("description: history of all previous successful runs output files")
if last_submissions:
for past_submission in last_submissions:
if past_submission.pk != submission.pk:
#pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
past_submission_number = '%03d' % past_submission.submission_number
lines.append('%s/%s/output/: %s' % (
past_submission_phasenumber,
past_submission_number,
submission_private_output_filename(past_submission),
)
)
else:
pass
submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))
score_csv = submission.phase.competition.get_results_csv(submission.phase.pk)
submission.scores_file.save('scores.txt', ContentFile(score_csv))
# Extra submission info
coopetition_zip_buffer = StringIO.StringIO()
coopetition_zip_file = zipfile.ZipFile(coopetition_zip_buffer, "w")
for phase in submission.phase.competition.phases.all():
coopetition_field_names = (
"participant__user__username",
"pk",
"when_made_public",
"when_unmade_public",
"started_at",
"completed_at",
"download_count",
"submission_number",
)
annotated_submissions = phase.submissions.filter(status__codename=CompetitionSubmissionStatus.FINISHED).values(
*coopetition_field_names
).annotate(like_count=Count("likes"), dislike_count=Count("dislikes"))
# Add this after fetching annotated count from db
coopetition_field_names += ("like_count", "dislike_count")
coopetition_csv = StringIO.StringIO()
writer = csv.DictWriter(coopetition_csv, coopetition_field_names)
writer.writeheader()
for row in annotated_submissions:
writer.writerow(row)
coopetition_zip_file.writestr('coopetition_phase_%s.txt' % phase.phasenumber, coopetition_csv.getvalue())
# Scores metadata
for phase in submission.phase.competition.phases.all():
coopetition_zip_file.writestr(
'coopetition_scores_phase_%s.txt' % phase.phasenumber,
phase.competition.get_results_csv(phase.pk, include_scores_not_on_leaderboard=True)
)
# Download metadata
coopetition_downloads_csv = StringIO.StringIO()
writer = csv.writer(coopetition_downloads_csv)
writer.writerow((
"submission_pk",
"submission_owner",
"downloaded_by",
"time_of_download",
))
for download in DownloadRecord.objects.filter(submission__phase__competition=submission.phase.competition):
writer.writerow((
download.submission.pk,
download.submission.participant.user.username,
download.user.username,
str(download.timestamp),
))
coopetition_zip_file.writestr('coopetition_downloads.txt', coopetition_downloads_csv.getvalue())
coopetition_zip_file.close()
submission.coopetition_file.save('coopetition.zip', ContentFile(coopetition_zip_buffer.getvalue()))
# Generate metadata-only bundle describing the inputs. Reference data is an optional
# dataset provided by the competition organizer. Results are provided by the participant
# either indirectly (has_generated_predictions is True i.e. participant provides a program
# which is run to generate results) ordirectly (participant uploads results directly).
lines = []
ref_value = submission.phase.reference_data.name
if len(ref_value) > 0:
lines.append("ref: %s" % ref_value)
res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
if len(res_value) > 0:
lines.append("res: %s" % res_value)
else:
raise ValueError("Results are missing.")
lines.append("history: %s" % submission_history_file_name(submission))
lines.append("scores: %s" % submission_scores_file_name(submission))
lines.append("coopetition: %s" % submission_coopetition_file_name(submission))
lines.append("submitted-by: %s" % submission.participant.user.username)
lines.append("submitted-at: %s" % submission.submitted_at.replace(microsecond=0).isoformat())
lines.append("competition-submission: %s" % submission.submission_number)
lines.append("competition-phase: %s" % submission.phase.phasenumber)
is_automatic_submission = False
if submission.phase.auto_migration:
# If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
submissions_this_phase = CompetitionSubmission.objects.filter(
phase=submission.phase,
participant=submission.participant
).count()
is_automatic_submission = submissions_this_phase == 1
lines.append("automatic-submission: %s" % is_automatic_submission)
submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))
# Generate metadata-only bundle describing the computation.
lines = []
program_value = submission.phase.scoring_program.name
if len(program_value) > 0:
lines.append("program: %s" % program_value)
else:
raise ValueError("Program is missing.")
lines.append("input: %s" % submission.inputfile.name)
lines.append("stdout: %s" % submission_stdout_filename(submission))
lines.append("stderr: %s" % submission_stderr_filename(submission))
submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))
# Create stdout.txt & stderr.txt
if has_generated_predictions == False:
username = submission.participant.user.username
lines = ["Standard output for submission #{0} by {1}.".format(submission.submission_number, username), ""]
submission.stdout_file.save('stdout.txt', ContentFile('\n'.join(lines)))
lines = ["Standard error for submission #{0} by {1}.".format(submission.submission_number, username), ""]
submission.stderr_file.save('stderr.txt', ContentFile('\n'.join(lines)))
# Update workflow state
state['score'] = job_id
submission.execution_key = json.dumps(state)
submission.save()
# Submit the request to the computation service
body = json.dumps({
"id" : job_id,
"task_type": "run",
"task_args": {
"bundle_id" : submission.runfile.name,
"container_name" : settings.BUNDLE_AZURE_CONTAINER,
"reply_to" : settings.SBS_RESPONSE_QUEUE,
"execution_time_limit": submission.phase.execution_time_limit,
"predict": False,
}
})
getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
if has_generated_predictions == False:
_set_submission_status(submission.id, CompetitionSubmissionStatus.SUBMITTED)
def update_submission(submission,
status,
job_id,
traceback=None,
metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" %
submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug(
"saving extra metadata, was a new object created? %s" %
created)
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(
submission_detailed_results_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
logger.debug("Output.zip location=%s" %
submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error(
"Scores.txt not found, unable to process submission: %s (submission_id=%s)",
status, submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" %
(label, value))
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (
Site.objects.get_current().domain,
submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s'
% (submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL, [email],
fail_silently=False)
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(
predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(
predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
def score(submission, job_id):
"""
Dispatches the scoring task for the given submission to an appropriate compute worker.
submission: The CompetitionSubmission object.
job_id: The job ID used to track the progress of the evaluation.
"""
# Loads the computation state.
state = {}
if len(submission.execution_key) > 0:
state = json.loads(submission.execution_key)
has_generated_predictions = 'predict' in state
#generate metadata-only bundle describing the history of submissions and phases
last_submissions = CompetitionSubmission.objects.filter(
participant=submission.participant,
status__codename=CompetitionSubmissionStatus.FINISHED).order_by(
'-submitted_at')
lines = []
lines.append(
"description: history of all previous successful runs output files")
if last_submissions:
for past_submission in last_submissions:
if past_submission.pk != submission.pk:
#pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
past_submission_number = '%03d' % past_submission.submission_number
lines.append('%s/%s/output/: %s' % (
past_submission_phasenumber,
past_submission_number,
submission_private_output_filename(past_submission),
))
else:
pass
submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))
score_csv = submission.phase.competition.get_results_csv(
submission.phase.pk)
submission.scores_file.save('scores.txt', ContentFile(score_csv))
# Extra submission info
coopetition_zip_buffer = StringIO.StringIO()
coopetition_zip_file = zipfile.ZipFile(coopetition_zip_buffer, "w")
for phase in submission.phase.competition.phases.all():
coopetition_field_names = (
"participant__user__username",
"pk",
"when_made_public",
"when_unmade_public",
"started_at",
"completed_at",
"download_count",
"submission_number",
)
annotated_submissions = phase.submissions.filter(
status__codename=CompetitionSubmissionStatus.FINISHED).values(
*coopetition_field_names).annotate(
like_count=Count("likes"), dislike_count=Count("dislikes"))
# Add this after fetching annotated count from db
coopetition_field_names += ("like_count", "dislike_count")
coopetition_csv = StringIO.StringIO()
writer = csv.DictWriter(coopetition_csv, coopetition_field_names)
writer.writeheader()
for row in annotated_submissions:
writer.writerow(row)
coopetition_zip_file.writestr(
'coopetition_phase_%s.txt' % phase.phasenumber,
coopetition_csv.getvalue().encode('utf-8'))
# Scores metadata
for phase in submission.phase.competition.phases.all():
coopetition_zip_file.writestr(
'coopetition_scores_phase_%s.txt' % phase.phasenumber,
phase.competition.get_results_csv(
phase.pk,
include_scores_not_on_leaderboard=True).encode('utf-8'))
# Download metadata
coopetition_downloads_csv = StringIO.StringIO()
writer = csv.writer(coopetition_downloads_csv)
writer.writerow((
"submission_pk",
"submission_owner",
"downloaded_by",
"time_of_download",
))
for download in DownloadRecord.objects.filter(
submission__phase__competition=submission.phase.competition):
writer.writerow((
download.submission.pk,
download.submission.participant.user.username,
download.user.username,
str(download.timestamp),
))
coopetition_zip_file.writestr(
'coopetition_downloads.txt',
coopetition_downloads_csv.getvalue().encode('utf-8'))
# Current user
coopetition_zip_file.writestr(
'current_user.txt',
submission.participant.user.username.encode('utf-8'))
coopetition_zip_file.close()
# Save them all
submission.coopetition_file.save(
'coopetition.zip', ContentFile(coopetition_zip_buffer.getvalue()))
# Generate metadata-only bundle describing the inputs. Reference data is an optional
# dataset provided by the competition organizer. Results are provided by the participant
# either indirectly (has_generated_predictions is True i.e. participant provides a program
# which is run to generate results) ordirectly (participant uploads results directly).
lines = []
ref_value = submission.phase.reference_data.name
if len(ref_value) > 0:
lines.append("ref: %s" % ref_value)
res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
if len(res_value) > 0:
lines.append("res: %s" % res_value)
else:
raise ValueError("Results are missing.")
lines.append("history: %s" % submission_history_file_name(submission))
lines.append("scores: %s" % submission_scores_file_name(submission))
lines.append("coopetition: %s" %
submission_coopetition_file_name(submission))
lines.append("submitted-by: %s" % submission.participant.user.username)
lines.append("submitted-at: %s" %
submission.submitted_at.replace(microsecond=0).isoformat())
lines.append("competition-submission: %s" % submission.submission_number)
lines.append("competition-phase: %s" % submission.phase.phasenumber)
is_automatic_submission = False
if submission.phase.auto_migration:
# If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
submissions_this_phase = CompetitionSubmission.objects.filter(
phase=submission.phase,
participant=submission.participant).count()
is_automatic_submission = submissions_this_phase == 1
lines.append("automatic-submission: %s" % is_automatic_submission)
submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))
# Generate metadata-only bundle describing the computation.
lines = []
program_value = submission.phase.scoring_program.name
if len(program_value) > 0:
lines.append("program: %s" % program_value)
else:
raise ValueError("Program is missing.")
lines.append("input: %s" % submission.inputfile.name)
lines.append("stdout: %s" % submission_stdout_filename(submission))
lines.append("stderr: %s" % submission_stderr_filename(submission))
submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))
# Create stdout.txt & stderr.txt
if has_generated_predictions == False:
username = submission.participant.user.username
lines = [
"Standard output for submission #{0} by {1}.".format(
submission.submission_number, username), ""
]
submission.stdout_file.save('stdout.txt',
ContentFile('\n'.join(lines)))
lines = [
"Standard error for submission #{0} by {1}.".format(
submission.submission_number, username), ""
]
submission.stderr_file.save('stderr.txt',
ContentFile('\n'.join(lines)))
# Update workflow state
state['score'] = job_id
submission.execution_key = json.dumps(state)
submission.save()
# Submit the request to the computation service
body = json.dumps({
"id": job_id,
"task_type": "run",
"task_args": {
"bundle_id": submission.runfile.name,
"container_name": settings.BUNDLE_AZURE_CONTAINER,
"reply_to": settings.SBS_RESPONSE_QUEUE,
"execution_time_limit": submission.phase.execution_time_limit,
"predict": False,
}
})
getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
if has_generated_predictions == False:
_set_submission_status(submission.id,
CompetitionSubmissionStatus.SUBMITTED)
def update_submission(submission, status, job_id, traceback=None, metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" % submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug("saving extra metadata, was a new object created? %s" % created)
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
logger.debug("Output.zip location=%s" % submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" % (label, value))
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (Site.objects.get_current().domain, submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s' %
(submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL,
[email],
fail_silently=False
)
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def update_submission(submission, status, job_id, traceback=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(
submission_detailed_results_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
logger.debug("Output.zip location=%s" %
submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error(
"Scores.txt not found, unable to process submission: %s (submission_id=%s)",
status, submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" %
(label, value))
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(
predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(
predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
def score(submission, job_id):
"""
Dispatches the scoring task for the given submission to an appropriate compute worker.
submission: The CompetitionSubmission object.
job_id: The job ID used to track the progress of the evaluation.
"""
# Loads the computation state.
state = {}
if len(submission.execution_key) > 0:
state = json.loads(submission.execution_key)
has_generated_predictions = 'predict' in state
#generate metadata-only bundle describing the history of submissions and phases
last_submissions = CompetitionSubmission.objects.filter(
participant=submission.participant,
status__codename=CompetitionSubmissionStatus.FINISHED).order_by(
'-submitted_at')
lines = []
lines.append(
"description: history of all previous successful runs output files")
if last_submissions:
for past_submission in last_submissions:
if past_submission.pk != submission.pk:
#pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
past_submission_number = '%03d' % past_submission.submission_number
lines.append('%s/%s/output/: %s' % (
past_submission_phasenumber,
past_submission_number,
submission_private_output_filename(past_submission),
))
else:
pass
submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))
# Generate metadata-only bundle describing the inputs. Reference data is an optional
# dataset provided by the competition organizer. Results are provided by the participant
# either indirectly (has_generated_predictions is True i.e. participant provides a program
# which is run to generate results) ordirectly (participant uploads results directly).
lines = []
ref_value = submission.phase.reference_data.name
if len(ref_value) > 0:
lines.append("ref: %s" % ref_value)
res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
if len(res_value) > 0:
lines.append("res: %s" % res_value)
else:
raise ValueError("Results are missing.")
lines.append("history: %s" % submission_history_file_name(submission))
lines.append("submitted-by: %s" % submission.participant.user.username)
lines.append("submitted-at: %s" %
submission.submitted_at.replace(microsecond=0).isoformat())
lines.append("competition-submission: %s" % submission.submission_number)
lines.append("competition-phase: %s" % submission.phase.phasenumber)
is_automatic_submission = False
if submission.phase.auto_migration:
# If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
submissions_this_phase = CompetitionSubmission.objects.filter(
phase=submission.phase,
participant=submission.participant).count()
is_automatic_submission = submissions_this_phase == 1
lines.append("automatic-submission: %s" % is_automatic_submission)
submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))
# Generate metadata-only bundle describing the computation.
lines = []
program_value = submission.phase.scoring_program.name
if len(program_value) > 0:
lines.append("program: %s" % program_value)
else:
raise ValueError("Program is missing.")
lines.append("input: %s" % submission.inputfile.name)
lines.append("stdout: %s" % submission_stdout_filename(submission))
lines.append("stderr: %s" % submission_stderr_filename(submission))
submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))
# Create stdout.txt & stderr.txt
if has_generated_predictions == False:
username = submission.participant.user.username
lines = [
"Standard output for submission #{0} by {1}.".format(
submission.submission_number, username), ""
]
submission.stdout_file.save('stdout.txt',
ContentFile('\n'.join(lines)))
lines = [
"Standard error for submission #{0} by {1}.".format(
submission.submission_number, username), ""
]
submission.stderr_file.save('stderr.txt',
ContentFile('\n'.join(lines)))
# Update workflow state
state['score'] = job_id
submission.execution_key = json.dumps(state)
submission.save()
# Submit the request to the computation service
body = json.dumps({
"id": job_id,
"task_type": "run",
"task_args": {
"bundle_id": submission.runfile.name,
"container_name": settings.BUNDLE_AZURE_CONTAINER,
"reply_to": settings.SBS_RESPONSE_QUEUE,
"execution_time_limit": submission.phase.execution_time_limit
}
})
getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
if has_generated_predictions == False:
_set_submission_status(submission.id,
CompetitionSubmissionStatus.SUBMITTED)