def create_competition_task(job_id, args):
"""
A task to create a competition from a bundle with the competition's definition.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['comp_def_id']: The ID of the bundle holding the competition definition.
Once the task succeeds, a new competition will be ready to use in CodaLab.
"""
def create_it(job):
"""Handles the actual creation of the competition"""
comp_def_id = args["comp_def_id"]
logger.info("Creating competition for competition bundle (bundle_id=%s, job_id=%s)", comp_def_id, job.id)
competition_def = CompetitionDefBundle.objects.get(pk=comp_def_id)
competition = competition_def.unpack()
logger.info(
"Created competition for competition bundle (bundle_id=%s, job_id=%s, comp_id=%s)",
comp_def_id,
job.id,
competition.pk,
)
return JobTaskResult(status=Job.FINISHED, info={"competition_id": competition.pk})
run_job_task(job_id, create_it)
def test_run_job_task2(self):
"""Exercise basics of run_job_task function: exception no handler."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
run_job_task(job.id, JobsTests._comp_with_ex)
self.assertEqual(Job.objects.get(pk=job.id).status, Job.FAILED)
job.delete()
def test_run_job_task2(self):
"""Exercise basics of run_job_task function: exception no handler."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
run_job_task(job.id, JobsTests._comp_with_ex)
self.assertEqual(Job.objects.get(pk=job.id).status, Job.FAILED)
job.delete()
def test_run_job_task1(self):
"""Exercise basics of run_job_task function: no exceptions."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
self.assertDictEqual(job.get_task_info(), {})
run_job_task(job.id, lambda ajob: JobTaskResult(status=Job.RUNNING))
self.assertEqual(Job.objects.get(pk=job.id).status, Job.RUNNING)
run_job_task(job.id, lambda ajob: JobTaskResult(status=Job.FINISHED))
self.assertEqual(Job.objects.get(pk=job.id).status, Job.FINISHED)
job.delete()
def test_run_job_task1(self):
"""Exercise basics of run_job_task function: no exceptions."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
self.assertDictEqual(job.get_task_info(), {})
run_job_task(job.id, lambda ajob : JobTaskResult(status=Job.RUNNING))
self.assertEqual(Job.objects.get(pk=job.id).status, Job.RUNNING)
run_job_task(job.id, lambda ajob : JobTaskResult(status=Job.FINISHED))
self.assertEqual(Job.objects.get(pk=job.id).status, Job.FINISHED)
job.delete()
def echo_task(job_id, args):
"""
A simple task to echo a message provided as args['message']. The associated job will
be marked as Finished if the message is echoes successfully. Otherwise the job will be
marked as Failed.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['message']: string to send as info to the module's logger.
"""
def echo_it(job):
"""Echoes the message specified."""
logger.info("Echoing (job id=%s): %s", job.id, args['message'])
return JobTaskResult(status=Job.FINISHED)
run_job_task(job_id, echo_it)
def echo_task(job_id, args):
"""
A simple task to echo a message provided as args['message']. The associated job will
be marked as Finished if the message is echoes successfully. Otherwise the job will be
marked as Failed.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['message']: string to send as info to the module's logger.
"""
def echo_it(job):
"""Echoes the message specified."""
logger.info("Echoing (job id=%s): %s", job.id, args['message'])
return JobTaskResult(status=Job.FINISHED)
run_job_task(job_id, echo_it)
def test_run_job_task4(self):
"""Exercise basics of run_job_task function: with info provided."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
info1 = { 'key1': 'value1', 'key2': 2 }
info2 = { 'key1': 'value1', 'key2': 2, 'key3': 3.3 }
run_job_task(job.id, lambda ajob : JobTaskResult(status=Job.RUNNING, info=info1))
j = Job.objects.get(pk=job.id)
self.assertEqual(j.status, Job.RUNNING)
self.assertDictEqual(j.get_task_info(), info1)
run_job_task(job.id, lambda ajob : JobTaskResult(status=Job.FINISHED, info=info2))
j = Job.objects.get(pk=job.id)
self.assertEqual(j.status, Job.FINISHED)
self.assertDictEqual(j.get_task_info(), info2)
job.delete()
def test_run_job_task4(self):
"""Exercise basics of run_job_task function: with info provided."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
info1 = { 'key1': 'value1', 'key2': 2 }
info2 = { 'key1': 'value1', 'key2': 2, 'key3': 3.3 }
run_job_task(job.id, lambda ajob : JobTaskResult(status=Job.RUNNING, info=info1))
j = Job.objects.get(pk=job.id)
self.assertEqual(j.status, Job.RUNNING)
self.assertDictEqual(j.get_task_info(), info1)
run_job_task(job.id, lambda ajob : JobTaskResult(status=Job.FINISHED, info=info2))
j = Job.objects.get(pk=job.id)
self.assertEqual(j.status, Job.FINISHED)
self.assertDictEqual(j.get_task_info(), info2)
job.delete()
def create_competition_task(job_id, args):
"""
A task to create a competition from a bundle with the competition's definition.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['comp_def_id']: The ID of the bundle holding the competition definition.
Once the task succeeds, a new competition will be ready to use in CodaLab.
"""
def create_it(job):
"""Handles the actual creation of the competition"""
comp_def_id = args['comp_def_id']
logger.info("Creating competition for competition bundle (bundle_id=%s, job_id=%s)",
comp_def_id, job.id)
competition_def = CompetitionDefBundle.objects.get(pk=comp_def_id)
competition = competition_def.unpack()
logger.info("Created competition for competition bundle (bundle_id=%s, job_id=%s, comp_id=%s)",
comp_def_id, job.id, competition.pk)
return JobTaskResult(status=Job.FINISHED, info={'competition_id': competition.pk})
run_job_task(job_id, create_it)
def test_run_job_task3(self):
"""Exercise basics of run_job_task function: exception with handler."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
run_job_task(job.id, JobsTests._comp_with_ex, self._ex_handler)
self.assertEqual(Job.objects.get(pk=job.id).status, Job.RUNNING)
run_job_task(job.id, JobsTests._comp_with_ex, lambda j, e: JobTaskResult())
self.assertEqual(Job.objects.get(pk=job.id).status, Job.RUNNING)
run_job_task(job.id, JobsTests._comp_with_ex, lambda j, e: JobTaskResult(status=Job.FAILED))
self.assertEqual(Job.objects.get(pk=job.id).status, Job.FAILED)
job.delete()
def test_run_job_task3(self):
"""Exercise basics of run_job_task function: exception with handler."""
job = Job.objects.create()
self.assertIsNotNone(job)
self.assertEqual(job.status, Job.PENDING)
run_job_task(job.id, JobsTests._comp_with_ex, self._ex_handler)
self.assertEqual(Job.objects.get(pk=job.id).status, Job.RUNNING)
run_job_task(job.id, JobsTests._comp_with_ex, lambda j, e: JobTaskResult())
self.assertEqual(Job.objects.get(pk=job.id).status, Job.RUNNING)
run_job_task(job.id, JobsTests._comp_with_ex, lambda j, e: JobTaskResult(status=Job.FAILED))
self.assertEqual(Job.objects.get(pk=job.id).status, Job.FAILED)
job.delete()
def update_submission_task(job_id, args):
"""
A task to update the status of a submission in a competition.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['status']: The evaluation status, which is one of 'running', 'finished' or 'failed'.
"""
def update_submission(submission, status, job_id):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == "running":
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == "finished":
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
if "score" in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = open(ozip.extract("scores.txt"), "r").read()
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition, key=label.strip()
)
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
result = Job.FINISHED
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != "failed":
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def handle_update_exception(job, ex):
"""
Handles exception that occur while attempting to update the status of a submission.
job: The running Job instance.
ex: The exception. The handler tries to acquire the CompetitionSubmission instance
from a submission attribute on the exception.
"""
try:
submission = ex.submission
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
except Exception:
logger.exception("Unable to set the submission status to Failed (job_id=%s)", job.id)
return JobTaskResult(status=Job.FAILED)
def update_it(job):
"""Updates the database to reflect the state of the evaluation of the given competition submission."""
logger.debug("Entering update_submission_task::update_it (job_id=%s)", job.id)
if job.task_type != "evaluate_submission":
raise ValueError("Job has incorrect task_type (job.task_type=%s)", job.task_type)
task_args = job.get_task_args()
submission_id = task_args["submission_id"]
logger.debug("Looking for submission (job_id=%s, submission_id=%s)", job.id, submission_id)
submission = CompetitionSubmission.objects.get(pk=submission_id)
status = args["status"]
logger.debug(
"Ready to update submission status (job_id=%s, submission_id=%s, status=%s)", job.id, submission_id, status
)
result = None
try:
result = update_submission(submission, status, job.id)
except Exception as e:
logger.exception(
"Failed to update submission (job_id=%s, submission_id=%s, status=%s)", job.id, submission_id, status
)
raise SubmissionUpdateException(submission, e)
return JobTaskResult(status=result)
run_job_task(job_id, update_it, handle_update_exception)
def update_submission_task(job_id, args):
"""
A task to update the status of a submission in a competition.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['status']: The evaluation status, which is one of 'running', 'finished' or 'failed'.
"""
def update_submission(submission, status, job_id, traceback=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s", submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
submission.output_file.name = pathname2url(submission_output_filename(submission))
submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
submission.save()
logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
logger.debug("Output.zip location=%s" % submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)", submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" % (label, value))
try:
scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)", submission.id)
_set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
add_submission_to_leaderboard(submission)
logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)
result = Job.FINISHED
else:
logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
url_name = pathname2url(submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
except Exception:
logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
def handle_update_exception(job, ex):
"""
Handles exception that occur while attempting to update the status of a submission.
job: The running Job instance.
ex: The exception. The handler tries to acquire the CompetitionSubmission instance
from a submission attribute on the exception.
"""
try:
submission = ex.submission
_set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
except Exception:
logger.exception("Unable to set the submission status to Failed (job_id=%s)", job.id)
return JobTaskResult(status=Job.FAILED)
def update_it(job):
"""Updates the database to reflect the state of the evaluation of the given competition submission."""
logger.debug("Entering update_submission_task::update_it (job_id=%s)", job.id)
if job.task_type != 'evaluate_submission':
raise ValueError("Job has incorrect task_type (job.task_type=%s)", job.task_type)
task_args = job.get_task_args()
submission_id = task_args['submission_id']
logger.debug("Looking for submission (job_id=%s, submission_id=%s)", job.id, submission_id)
submission = CompetitionSubmission.objects.get(pk=submission_id)
status = args['status']
logger.debug("Ready to update submission status (job_id=%s, submission_id=%s, status=%s)",
job.id, submission_id, status)
result = None
try:
traceback = None
if 'extra' in args and 'traceback' in args['extra']:
traceback = args['extra']['traceback']
result = update_submission(submission, status, job.id, traceback)
except Exception as e:
logger.exception("Failed to update submission (job_id=%s, submission_id=%s, status=%s)",
job.id, submission_id, status)
raise SubmissionUpdateException(submission, e)
return JobTaskResult(status=result)
run_job_task(job_id, update_it, handle_update_exception)
def update_submission_task(job_id, args):
"""
A task to update the status of a submission in a competition.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['status']: The evaluation status, which is one of 'running', 'finished' or 'failed'.
"""
def update_submission(submission,
status,
job_id,
traceback=None,
metadata=None):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
logger.debug("update_submission_task state = %s" %
submission.execution_key)
if metadata:
is_predict = 'score' not in state
sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
is_predict=is_predict,
is_scoring=not is_predict,
submission=submission,
)
sub_metadata.__dict__.update(metadata)
sub_metadata.save()
logger.debug(
"saving extra metadata, was a new object created? %s" %
created)
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.detailed_results_file.name = pathname2url(
submission_detailed_results_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
logger.debug("Output.zip location=%s" %
submission.output_file.file.name)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = None
try:
scores = open(ozip.extract('scores.txt'), 'r').read()
except Exception:
logger.error(
"Scores.txt not found, unable to process submission: %s (submission_id=%s)",
status, submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
return Job.FAILED
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
logger.debug("Attempting to submit score %s:%s" %
(label, value))
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
if submission.participant.user.email_on_submission_finished_successfully:
email = submission.participant.user.email
site_url = "https://%s%s" % (
Site.objects.get_current().domain,
submission.phase.competition.get_absolute_url())
send_mail(
'Submission has finished successfully!',
'Your submission to the competition "%s" has finished successfully! View it here: %s'
% (submission.phase.competition.title, site_url),
settings.DEFAULT_FROM_EMAIL, [email],
fail_silently=False)
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.prediction_stderr_file.name = pathname2url(
predict_submission_stdout_filename(submission))
submission.prediction_stdout_file.name = pathname2url(
predict_submission_stderr_filename(submission))
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
if traceback:
submission.exception_details = traceback
submission.save()
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
def handle_update_exception(job, ex):
"""
Handles exception that occur while attempting to update the status of a submission.
job: The running Job instance.
ex: The exception. The handler tries to acquire the CompetitionSubmission instance
from a submission attribute on the exception.
"""
try:
submission = ex.submission
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
except Exception:
logger.exception(
"Unable to set the submission status to Failed (job_id=%s)",
job.id)
return JobTaskResult(status=Job.FAILED)
def update_it(job):
"""Updates the database to reflect the state of the evaluation of the given competition submission."""
logger.debug("Entering update_submission_task::update_it (job_id=%s)",
job.id)
if job.task_type != 'evaluate_submission':
raise ValueError("Job has incorrect task_type (job.task_type=%s)",
job.task_type)
task_args = job.get_task_args()
submission_id = task_args['submission_id']
logger.debug("Looking for submission (job_id=%s, submission_id=%s)",
job.id, submission_id)
submission = CompetitionSubmission.objects.get(pk=submission_id)
status = args['status']
logger.debug(
"Ready to update submission status (job_id=%s, submission_id=%s, status=%s)",
job.id, submission_id, status)
result = None
try:
traceback = None
metadata = None
if 'extra' in args:
if 'traceback' in args['extra']:
traceback = args['extra']['traceback']
if 'metadata' in args['extra']:
metadata = args['extra']['metadata']
result = update_submission(submission, status, job.id, traceback,
metadata)
except Exception as e:
logger.exception(
"Failed to update submission (job_id=%s, submission_id=%s, status=%s)",
job.id, submission_id, status)
raise SubmissionUpdateException(submission, e)
return JobTaskResult(status=result)
run_job_task(job_id, update_it, handle_update_exception)
def update_submission_task(job_id, args):
"""
A task to update the status of a submission in a competition.
job_id: The ID of the job.
args: A dictionary with the arguments for the task. Expected items are:
args['status']: The evaluation status, which is one of 'running', 'finished' or 'failed'.
"""
def update_submission(submission, status, job_id):
"""
Updates the status of a submission.
submission: The CompetitionSubmission object to update.
status: The new status string: 'running', 'finished' or 'failed'.
job_id: The job ID used to track the progress of the evaluation.
"""
if status == 'running':
_set_submission_status(submission.id,
CompetitionSubmissionStatus.RUNNING)
return Job.RUNNING
if status == 'finished':
result = Job.FAILED
state = {}
if len(submission.execution_key) > 0:
logger.debug("update_submission_task loading state: %s",
submission.execution_key)
state = json.loads(submission.execution_key)
if 'score' in state:
logger.debug(
"update_submission_task loading final scores (pk=%s)",
submission.pk)
submission.output_file.name = pathname2url(
submission_output_filename(submission))
submission.private_output_file.name = pathname2url(
submission_private_output_filename(submission))
submission.save()
logger.debug(
"Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
submission.id)
ozip = ZipFile(io.BytesIO(submission.output_file.read()))
scores = open(ozip.extract('scores.txt'), 'r').read()
logger.debug("Processing scores... (submission_id=%s)",
submission.id)
for line in scores.split("\n"):
if len(line) > 0:
label, value = line.split(":")
try:
scoredef = SubmissionScoreDef.objects.get(
competition=submission.phase.competition,
key=label.strip())
SubmissionScore.objects.create(result=submission,
scoredef=scoredef,
value=float(value))
except SubmissionScoreDef.DoesNotExist:
logger.warning(
"Score %s does not exist (submission_id=%s)",
label, submission.id)
logger.debug("Done processing scores... (submission_id=%s)",
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FINISHED)
# Automatically submit to the leaderboard?
if submission.phase.is_blind:
logger.debug("Adding to leaderboard... (submission_id=%s)",
submission.id)
add_submission_to_leaderboard(submission)
logger.debug(
"Leaderboard updated with latest submission (submission_id=%s)",
submission.id)
if submission.phase.competition.force_submission_to_leaderboard:
add_submission_to_leaderboard(submission)
logger.debug(
"Force submission added submission to leaderboard (submission_id=%s)",
submission.id)
result = Job.FINISHED
else:
logger.debug(
"update_submission_task entering scoring phase (pk=%s)",
submission.pk)
url_name = pathname2url(
submission_prediction_output_filename(submission))
submission.prediction_output_file.name = url_name
submission.save()
try:
score(submission, job_id)
result = Job.RUNNING
logger.debug(
"update_submission_task scoring phase entered (pk=%s)",
submission.pk)
except Exception:
logger.exception(
"update_submission_task failed to enter scoring phase (pk=%s)",
submission.pk)
return result
if status != 'failed':
logger.error("Invalid status: %s (submission_id=%s)", status,
submission.id)
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
def handle_update_exception(job, ex):
"""
Handles exception that occur while attempting to update the status of a submission.
job: The running Job instance.
ex: The exception. The handler tries to acquire the CompetitionSubmission instance
from a submission attribute on the exception.
"""
try:
submission = ex.submission
_set_submission_status(submission.id,
CompetitionSubmissionStatus.FAILED)
except Exception:
logger.exception(
"Unable to set the submission status to Failed (job_id=%s)",
job.id)
return JobTaskResult(status=Job.FAILED)
def update_it(job):
"""Updates the database to reflect the state of the evaluation of the given competition submission."""
logger.debug("Entering update_submission_task::update_it (job_id=%s)",
job.id)
if job.task_type != 'evaluate_submission':
raise ValueError("Job has incorrect task_type (job.task_type=%s)",
job.task_type)
task_args = job.get_task_args()
submission_id = task_args['submission_id']
logger.debug("Looking for submission (job_id=%s, submission_id=%s)",
job.id, submission_id)
submission = CompetitionSubmission.objects.get(pk=submission_id)
status = args['status']
logger.debug(
"Ready to update submission status (job_id=%s, submission_id=%s, status=%s)",
job.id, submission_id, status)
result = None
try:
result = update_submission(submission, status, job.id)
except Exception as e:
logger.exception(
"Failed to update submission (job_id=%s, submission_id=%s, status=%s)",
job.id, submission_id, status)
raise SubmissionUpdateException(submission, e)
return JobTaskResult(status=result)
run_job_task(job_id, update_it, handle_update_exception)