Exemplo n.º 1
0
    def update_submission(submission, status, job_id, traceback=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        if status == 'running':
            _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            state = {}
            if len(submission.execution_key) > 0:
                logger.debug("update_submission_task loading state: %s", submission.execution_key)
                state = json.loads(submission.execution_key)
            if 'score' in state:
                logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
                submission.output_file.name = pathname2url(submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
                submission.save()
                logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
                logger.debug("Output.zip location=%s" % submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
                    _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)", submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" % (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
                                                                      key=label.strip())
                            SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)", submission.id)
                _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)

                result = Job.FINISHED
            else:
                logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
                url_name = pathname2url(submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
                except Exception:
                    logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()
        _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
Exemplo n.º 2
0
def score(submission, job_id):
    """
    Dispatches the scoring task for the given submission to an appropriate compute worker.

    submission: The CompetitionSubmission object.
    job_id: The job ID used to track the progress of the evaluation.
    """
    # Loads the computation state.
    state = {}
    if len(submission.execution_key) > 0:
        state = json.loads(submission.execution_key)
    has_generated_predictions = 'predict' in state

    #generate metadata-only bundle describing the history of submissions and phases
    last_submissions = CompetitionSubmission.objects.filter(
        participant=submission.participant,
        status__codename=CompetitionSubmissionStatus.FINISHED
    ).order_by('-submitted_at')


    lines = []
    lines.append("description: history of all previous successful runs output files")

    if last_submissions:
        for past_submission in last_submissions:
            if past_submission.pk != submission.pk:
                #pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
                past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
                past_submission_number = '%03d' % past_submission.submission_number
                lines.append('%s/%s/output/: %s' % (
                        past_submission_phasenumber,
                        past_submission_number,
                        submission_private_output_filename(past_submission),
                    )
                )
    else:
        pass

    submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))
    submission.scores_file.save('scores.txt', ContentFile(submission.phase.competition.get_results_csv(submission.phase.pk)))

    # Generate metadata-only bundle describing the inputs. Reference data is an optional
    # dataset provided by the competition organizer. Results are provided by the participant
    # either indirectly (has_generated_predictions is True i.e. participant provides a program
    # which is run to generate results) ordirectly (participant uploads results directly).
    lines = []
    ref_value = submission.phase.reference_data.name
    if len(ref_value) > 0:
        lines.append("ref: %s" % ref_value)
    res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
    if len(res_value) > 0:
        lines.append("res: %s" % res_value)
    else:
        raise ValueError("Results are missing.")

    lines.append("history: %s" % submission_history_file_name(submission))
    lines.append("scores: %s" % submission_scores_file_name(submission))
    lines.append("submitted-by: %s" % submission.participant.user.username)
    lines.append("submitted-at: %s" % submission.submitted_at.replace(microsecond=0).isoformat())
    lines.append("competition-submission: %s" % submission.submission_number)
    lines.append("competition-phase: %s" % submission.phase.phasenumber)
    is_automatic_submission = False
    if submission.phase.auto_migration:
        # If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
        submissions_this_phase = CompetitionSubmission.objects.filter(
            phase=submission.phase,
            participant=submission.participant
        ).count()
        is_automatic_submission = submissions_this_phase == 1

    lines.append("automatic-submission: %s" % is_automatic_submission)
    submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))


    # Generate metadata-only bundle describing the computation.
    lines = []
    program_value = submission.phase.scoring_program.name
    if len(program_value) > 0:
        lines.append("program: %s" % program_value)
    else:
        raise ValueError("Program is missing.")
    lines.append("input: %s" % submission.inputfile.name)
    lines.append("stdout: %s" % submission_stdout_filename(submission))
    lines.append("stderr: %s" % submission_stderr_filename(submission))
    submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))

    # Create stdout.txt & stderr.txt
    if has_generated_predictions == False:
        username = submission.participant.user.username
        lines = ["Standard output for submission #{0} by {1}.".format(submission.submission_number, username), ""]
        submission.stdout_file.save('stdout.txt', ContentFile('\n'.join(lines)))
        lines = ["Standard error for submission #{0} by {1}.".format(submission.submission_number, username), ""]
        submission.stderr_file.save('stderr.txt', ContentFile('\n'.join(lines)))
    # Update workflow state
    state['score'] = job_id
    submission.execution_key = json.dumps(state)
    submission.save()
    # Submit the request to the computation service
    body = json.dumps({
        "id" : job_id,
        "task_type": "run",
        "task_args": {
            "bundle_id" : submission.runfile.name,
            "container_name" : settings.BUNDLE_AZURE_CONTAINER,
            "reply_to" : settings.SBS_RESPONSE_QUEUE,
            "execution_time_limit": submission.phase.execution_time_limit,
            "predict": False,
        }
    })
    getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
    if has_generated_predictions == False:
        _set_submission_status(submission.id, CompetitionSubmissionStatus.SUBMITTED)
Exemplo n.º 3
0
def score(submission, job_id):
    """
    Dispatches the scoring task for the given submission to an appropriate compute worker.

    submission: The CompetitionSubmission object.
    job_id: The job ID used to track the progress of the evaluation.
    """
    # Loads the computation state.
    state = {}
    if len(submission.execution_key) > 0:
        state = json.loads(submission.execution_key)
    has_generated_predictions = 'predict' in state

    #generate metadata-only bundle describing the history of submissions and phases
    last_submissions = CompetitionSubmission.objects.filter(
        participant=submission.participant,
        status__codename=CompetitionSubmissionStatus.FINISHED
    ).order_by('-submitted_at')


    lines = []
    lines.append("description: history of all previous successful runs output files")

    if last_submissions:
        for past_submission in last_submissions:
            if past_submission.pk != submission.pk:
                #pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
                past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
                past_submission_number = '%03d' % past_submission.submission_number
                lines.append('%s/%s/output/: %s' % (
                        past_submission_phasenumber,
                        past_submission_number,
                        submission_private_output_filename(past_submission),
                    )
                )
    else:
        pass

    submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))

    score_csv = submission.phase.competition.get_results_csv(submission.phase.pk)
    submission.scores_file.save('scores.txt', ContentFile(score_csv))

    # Extra submission info
    coopetition_zip_buffer = StringIO.StringIO()
    coopetition_zip_file = zipfile.ZipFile(coopetition_zip_buffer, "w")

    for phase in submission.phase.competition.phases.all():
        coopetition_field_names = (
            "participant__user__username",
            "pk",
            "when_made_public",
            "when_unmade_public",
            "started_at",
            "completed_at",
            "download_count",
            "submission_number",
        )
        annotated_submissions = phase.submissions.filter(status__codename=CompetitionSubmissionStatus.FINISHED).values(
            *coopetition_field_names
        ).annotate(like_count=Count("likes"), dislike_count=Count("dislikes"))

        # Add this after fetching annotated count from db
        coopetition_field_names += ("like_count", "dislike_count")

        coopetition_csv = StringIO.StringIO()
        writer = csv.DictWriter(coopetition_csv, coopetition_field_names)
        writer.writeheader()
        for row in annotated_submissions:
            writer.writerow(row)

        coopetition_zip_file.writestr('coopetition_phase_%s.txt' % phase.phasenumber, coopetition_csv.getvalue())

    # Scores metadata
    for phase in submission.phase.competition.phases.all():
        coopetition_zip_file.writestr(
            'coopetition_scores_phase_%s.txt' % phase.phasenumber,
            phase.competition.get_results_csv(phase.pk, include_scores_not_on_leaderboard=True)
        )

    # Download metadata
    coopetition_downloads_csv = StringIO.StringIO()
    writer = csv.writer(coopetition_downloads_csv)
    writer.writerow((
        "submission_pk",
        "submission_owner",
        "downloaded_by",
        "time_of_download",
    ))
    for download in DownloadRecord.objects.filter(submission__phase__competition=submission.phase.competition):
        writer.writerow((
            download.submission.pk,
            download.submission.participant.user.username,
            download.user.username,
            str(download.timestamp),
        ))

    coopetition_zip_file.writestr('coopetition_downloads.txt', coopetition_downloads_csv.getvalue())

    coopetition_zip_file.close()
    submission.coopetition_file.save('coopetition.zip', ContentFile(coopetition_zip_buffer.getvalue()))

    # Generate metadata-only bundle describing the inputs. Reference data is an optional
    # dataset provided by the competition organizer. Results are provided by the participant
    # either indirectly (has_generated_predictions is True i.e. participant provides a program
    # which is run to generate results) ordirectly (participant uploads results directly).
    lines = []
    ref_value = submission.phase.reference_data.name
    if len(ref_value) > 0:
        lines.append("ref: %s" % ref_value)
    res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
    if len(res_value) > 0:
        lines.append("res: %s" % res_value)
    else:
        raise ValueError("Results are missing.")

    lines.append("history: %s" % submission_history_file_name(submission))
    lines.append("scores: %s" % submission_scores_file_name(submission))
    lines.append("coopetition: %s" % submission_coopetition_file_name(submission))
    lines.append("submitted-by: %s" % submission.participant.user.username)
    lines.append("submitted-at: %s" % submission.submitted_at.replace(microsecond=0).isoformat())
    lines.append("competition-submission: %s" % submission.submission_number)
    lines.append("competition-phase: %s" % submission.phase.phasenumber)
    is_automatic_submission = False
    if submission.phase.auto_migration:
        # If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
        submissions_this_phase = CompetitionSubmission.objects.filter(
            phase=submission.phase,
            participant=submission.participant
        ).count()
        is_automatic_submission = submissions_this_phase == 1

    lines.append("automatic-submission: %s" % is_automatic_submission)
    submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))


    # Generate metadata-only bundle describing the computation.
    lines = []
    program_value = submission.phase.scoring_program.name
    if len(program_value) > 0:
        lines.append("program: %s" % program_value)
    else:
        raise ValueError("Program is missing.")
    lines.append("input: %s" % submission.inputfile.name)
    lines.append("stdout: %s" % submission_stdout_filename(submission))
    lines.append("stderr: %s" % submission_stderr_filename(submission))
    submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))

    # Create stdout.txt & stderr.txt
    if has_generated_predictions == False:
        username = submission.participant.user.username
        lines = ["Standard output for submission #{0} by {1}.".format(submission.submission_number, username), ""]
        submission.stdout_file.save('stdout.txt', ContentFile('\n'.join(lines)))
        lines = ["Standard error for submission #{0} by {1}.".format(submission.submission_number, username), ""]
        submission.stderr_file.save('stderr.txt', ContentFile('\n'.join(lines)))
    # Update workflow state
    state['score'] = job_id
    submission.execution_key = json.dumps(state)
    submission.save()
    # Submit the request to the computation service
    body = json.dumps({
        "id" : job_id,
        "task_type": "run",
        "task_args": {
            "bundle_id" : submission.runfile.name,
            "container_name" : settings.BUNDLE_AZURE_CONTAINER,
            "reply_to" : settings.SBS_RESPONSE_QUEUE,
            "execution_time_limit": submission.phase.execution_time_limit,
            "predict": False,
        }
    })
    getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
    if has_generated_predictions == False:
        _set_submission_status(submission.id, CompetitionSubmissionStatus.SUBMITTED)
Exemplo n.º 4
0
    def update_submission(submission,
                          status,
                          job_id,
                          traceback=None,
                          metadata=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        state = {}
        if len(submission.execution_key) > 0:
            logger.debug("update_submission_task loading state: %s",
                         submission.execution_key)
            state = json.loads(submission.execution_key)
            logger.debug("update_submission_task state = %s" %
                         submission.execution_key)

        if metadata:
            is_predict = 'score' not in state
            sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
                is_predict=is_predict,
                is_scoring=not is_predict,
                submission=submission,
            )
            sub_metadata.__dict__.update(metadata)
            sub_metadata.save()
            logger.debug(
                "saving extra metadata, was a new object created? %s" %
                created)

        if status == 'running':
            _set_submission_status(submission.id,
                                   CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            if 'score' in state:
                logger.debug(
                    "update_submission_task loading final scores (pk=%s)",
                    submission.pk)
                submission.output_file.name = pathname2url(
                    submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(
                    submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(
                    submission_detailed_results_filename(submission))
                submission.save()
                logger.debug(
                    "Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
                    submission.id)
                logger.debug("Output.zip location=%s" %
                             submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error(
                        "Scores.txt not found, unable to process submission: %s (submission_id=%s)",
                        status, submission.id)
                    _set_submission_status(submission.id,
                                           CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)",
                             submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" %
                                     (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(
                                competition=submission.phase.competition,
                                key=label.strip())
                            SubmissionScore.objects.create(result=submission,
                                                           scoredef=scoredef,
                                                           value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning(
                                "Score %s does not exist (submission_id=%s)",
                                label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)",
                             submission.id)
                _set_submission_status(submission.id,
                                       CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)",
                                 submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Leaderboard updated with latest submission (submission_id=%s)",
                        submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Force submission added submission to leaderboard (submission_id=%s)",
                        submission.id)

                result = Job.FINISHED

                if submission.participant.user.email_on_submission_finished_successfully:
                    email = submission.participant.user.email
                    site_url = "https://%s%s" % (
                        Site.objects.get_current().domain,
                        submission.phase.competition.get_absolute_url())
                    send_mail(
                        'Submission has finished successfully!',
                        'Your submission to the competition "%s" has finished successfully! View it here: %s'
                        % (submission.phase.competition.title, site_url),
                        settings.DEFAULT_FROM_EMAIL, [email],
                        fail_silently=False)
            else:
                logger.debug(
                    "update_submission_task entering scoring phase (pk=%s)",
                    submission.pk)
                url_name = pathname2url(
                    submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(
                    predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(
                    predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug(
                        "update_submission_task scoring phase entered (pk=%s)",
                        submission.pk)
                except Exception:
                    logger.exception(
                        "update_submission_task failed to enter scoring phase (pk=%s)",
                        submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status,
                         submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()

        _set_submission_status(submission.id,
                               CompetitionSubmissionStatus.FAILED)
Exemplo n.º 5
0
def score(submission, job_id):
    """
    Dispatches the scoring task for the given submission to an appropriate compute worker.

    submission: The CompetitionSubmission object.
    job_id: The job ID used to track the progress of the evaluation.
    """
    # Loads the computation state.
    state = {}
    if len(submission.execution_key) > 0:
        state = json.loads(submission.execution_key)
    has_generated_predictions = 'predict' in state

    #generate metadata-only bundle describing the history of submissions and phases
    last_submissions = CompetitionSubmission.objects.filter(
        participant=submission.participant,
        status__codename=CompetitionSubmissionStatus.FINISHED).order_by(
            '-submitted_at')

    lines = []
    lines.append(
        "description: history of all previous successful runs output files")

    if last_submissions:
        for past_submission in last_submissions:
            if past_submission.pk != submission.pk:
                #pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
                past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
                past_submission_number = '%03d' % past_submission.submission_number
                lines.append('%s/%s/output/: %s' % (
                    past_submission_phasenumber,
                    past_submission_number,
                    submission_private_output_filename(past_submission),
                ))
    else:
        pass

    submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))

    score_csv = submission.phase.competition.get_results_csv(
        submission.phase.pk)
    submission.scores_file.save('scores.txt', ContentFile(score_csv))

    # Extra submission info
    coopetition_zip_buffer = StringIO.StringIO()
    coopetition_zip_file = zipfile.ZipFile(coopetition_zip_buffer, "w")

    for phase in submission.phase.competition.phases.all():
        coopetition_field_names = (
            "participant__user__username",
            "pk",
            "when_made_public",
            "when_unmade_public",
            "started_at",
            "completed_at",
            "download_count",
            "submission_number",
        )
        annotated_submissions = phase.submissions.filter(
            status__codename=CompetitionSubmissionStatus.FINISHED).values(
                *coopetition_field_names).annotate(
                    like_count=Count("likes"), dislike_count=Count("dislikes"))

        # Add this after fetching annotated count from db
        coopetition_field_names += ("like_count", "dislike_count")

        coopetition_csv = StringIO.StringIO()
        writer = csv.DictWriter(coopetition_csv, coopetition_field_names)
        writer.writeheader()
        for row in annotated_submissions:
            writer.writerow(row)

        coopetition_zip_file.writestr(
            'coopetition_phase_%s.txt' % phase.phasenumber,
            coopetition_csv.getvalue().encode('utf-8'))

    # Scores metadata
    for phase in submission.phase.competition.phases.all():
        coopetition_zip_file.writestr(
            'coopetition_scores_phase_%s.txt' % phase.phasenumber,
            phase.competition.get_results_csv(
                phase.pk,
                include_scores_not_on_leaderboard=True).encode('utf-8'))

    # Download metadata
    coopetition_downloads_csv = StringIO.StringIO()
    writer = csv.writer(coopetition_downloads_csv)
    writer.writerow((
        "submission_pk",
        "submission_owner",
        "downloaded_by",
        "time_of_download",
    ))
    for download in DownloadRecord.objects.filter(
            submission__phase__competition=submission.phase.competition):
        writer.writerow((
            download.submission.pk,
            download.submission.participant.user.username,
            download.user.username,
            str(download.timestamp),
        ))

    coopetition_zip_file.writestr(
        'coopetition_downloads.txt',
        coopetition_downloads_csv.getvalue().encode('utf-8'))

    # Current user
    coopetition_zip_file.writestr(
        'current_user.txt',
        submission.participant.user.username.encode('utf-8'))
    coopetition_zip_file.close()

    # Save them all
    submission.coopetition_file.save(
        'coopetition.zip', ContentFile(coopetition_zip_buffer.getvalue()))

    # Generate metadata-only bundle describing the inputs. Reference data is an optional
    # dataset provided by the competition organizer. Results are provided by the participant
    # either indirectly (has_generated_predictions is True i.e. participant provides a program
    # which is run to generate results) ordirectly (participant uploads results directly).
    lines = []
    ref_value = submission.phase.reference_data.name
    if len(ref_value) > 0:
        lines.append("ref: %s" % ref_value)
    res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
    if len(res_value) > 0:
        lines.append("res: %s" % res_value)
    else:
        raise ValueError("Results are missing.")

    lines.append("history: %s" % submission_history_file_name(submission))
    lines.append("scores: %s" % submission_scores_file_name(submission))
    lines.append("coopetition: %s" %
                 submission_coopetition_file_name(submission))
    lines.append("submitted-by: %s" % submission.participant.user.username)
    lines.append("submitted-at: %s" %
                 submission.submitted_at.replace(microsecond=0).isoformat())
    lines.append("competition-submission: %s" % submission.submission_number)
    lines.append("competition-phase: %s" % submission.phase.phasenumber)
    is_automatic_submission = False
    if submission.phase.auto_migration:
        # If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
        submissions_this_phase = CompetitionSubmission.objects.filter(
            phase=submission.phase,
            participant=submission.participant).count()
        is_automatic_submission = submissions_this_phase == 1

    lines.append("automatic-submission: %s" % is_automatic_submission)
    submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))

    # Generate metadata-only bundle describing the computation.
    lines = []
    program_value = submission.phase.scoring_program.name
    if len(program_value) > 0:
        lines.append("program: %s" % program_value)
    else:
        raise ValueError("Program is missing.")
    lines.append("input: %s" % submission.inputfile.name)
    lines.append("stdout: %s" % submission_stdout_filename(submission))
    lines.append("stderr: %s" % submission_stderr_filename(submission))
    submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))

    # Create stdout.txt & stderr.txt
    if has_generated_predictions == False:
        username = submission.participant.user.username
        lines = [
            "Standard output for submission #{0} by {1}.".format(
                submission.submission_number, username), ""
        ]
        submission.stdout_file.save('stdout.txt',
                                    ContentFile('\n'.join(lines)))
        lines = [
            "Standard error for submission #{0} by {1}.".format(
                submission.submission_number, username), ""
        ]
        submission.stderr_file.save('stderr.txt',
                                    ContentFile('\n'.join(lines)))
    # Update workflow state
    state['score'] = job_id
    submission.execution_key = json.dumps(state)
    submission.save()
    # Submit the request to the computation service
    body = json.dumps({
        "id": job_id,
        "task_type": "run",
        "task_args": {
            "bundle_id": submission.runfile.name,
            "container_name": settings.BUNDLE_AZURE_CONTAINER,
            "reply_to": settings.SBS_RESPONSE_QUEUE,
            "execution_time_limit": submission.phase.execution_time_limit,
            "predict": False,
        }
    })
    getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
    if has_generated_predictions == False:
        _set_submission_status(submission.id,
                               CompetitionSubmissionStatus.SUBMITTED)
Exemplo n.º 6
0
    def update_submission(submission, status, job_id, traceback=None, metadata=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        state = {}
        if len(submission.execution_key) > 0:
            logger.debug("update_submission_task loading state: %s", submission.execution_key)
            state = json.loads(submission.execution_key)
            logger.debug("update_submission_task state = %s" % submission.execution_key)

        if metadata:
            is_predict = 'score' not in state
            sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
                is_predict=is_predict,
                is_scoring=not is_predict,
                submission=submission,
            )
            sub_metadata.__dict__.update(metadata)
            sub_metadata.save()
            logger.debug("saving extra metadata, was a new object created? %s" % created)

        if status == 'running':
            _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            if 'score' in state:
                logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
                submission.output_file.name = pathname2url(submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
                submission.save()
                logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
                logger.debug("Output.zip location=%s" % submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
                    _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)", submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" % (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
                                                                      key=label.strip())
                            SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)", submission.id)
                _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)

                result = Job.FINISHED

                if submission.participant.user.email_on_submission_finished_successfully:
                    email = submission.participant.user.email
                    site_url = "https://%s%s" % (Site.objects.get_current().domain, submission.phase.competition.get_absolute_url())
                    send_mail(
                        'Submission has finished successfully!',
                        'Your submission to the competition "%s" has finished successfully! View it here: %s' %
                        (submission.phase.competition.title, site_url),
                        settings.DEFAULT_FROM_EMAIL,
                        [email],
                        fail_silently=False
                    )
            else:
                logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
                url_name = pathname2url(submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
                except Exception:
                    logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()

        _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
Exemplo n.º 7
0
    def update_submission(submission, status, job_id, traceback=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        if status == 'running':
            _set_submission_status(submission.id,
                                   CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            state = {}
            if len(submission.execution_key) > 0:
                logger.debug("update_submission_task loading state: %s",
                             submission.execution_key)
                state = json.loads(submission.execution_key)
            if 'score' in state:
                logger.debug(
                    "update_submission_task loading final scores (pk=%s)",
                    submission.pk)
                submission.output_file.name = pathname2url(
                    submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(
                    submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(
                    submission_detailed_results_filename(submission))
                submission.save()
                logger.debug(
                    "Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
                    submission.id)
                logger.debug("Output.zip location=%s" %
                             submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error(
                        "Scores.txt not found, unable to process submission: %s (submission_id=%s)",
                        status, submission.id)
                    _set_submission_status(submission.id,
                                           CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)",
                             submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" %
                                     (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(
                                competition=submission.phase.competition,
                                key=label.strip())
                            SubmissionScore.objects.create(result=submission,
                                                           scoredef=scoredef,
                                                           value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning(
                                "Score %s does not exist (submission_id=%s)",
                                label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)",
                             submission.id)
                _set_submission_status(submission.id,
                                       CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)",
                                 submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Leaderboard updated with latest submission (submission_id=%s)",
                        submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Force submission added submission to leaderboard (submission_id=%s)",
                        submission.id)

                result = Job.FINISHED
            else:
                logger.debug(
                    "update_submission_task entering scoring phase (pk=%s)",
                    submission.pk)
                url_name = pathname2url(
                    submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(
                    predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(
                    predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug(
                        "update_submission_task scoring phase entered (pk=%s)",
                        submission.pk)
                except Exception:
                    logger.exception(
                        "update_submission_task failed to enter scoring phase (pk=%s)",
                        submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status,
                         submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()
        _set_submission_status(submission.id,
                               CompetitionSubmissionStatus.FAILED)
Exemplo n.º 8
0
def score(submission, job_id):
    """
    Dispatches the scoring task for the given submission to an appropriate compute worker.

    submission: The CompetitionSubmission object.
    job_id: The job ID used to track the progress of the evaluation.
    """
    # Loads the computation state.
    state = {}
    if len(submission.execution_key) > 0:
        state = json.loads(submission.execution_key)
    has_generated_predictions = 'predict' in state

    #generate metadata-only bundle describing the history of submissions and phases
    last_submissions = CompetitionSubmission.objects.filter(
        participant=submission.participant,
        status__codename=CompetitionSubmissionStatus.FINISHED).order_by(
            '-submitted_at')

    lines = []
    lines.append(
        "description: history of all previous successful runs output files")

    if last_submissions:
        for past_submission in last_submissions:
            if past_submission.pk != submission.pk:
                #pad folder numbers for sorting os side, 001, 002, 003,... 010, etc...
                past_submission_phasenumber = '%03d' % past_submission.phase.phasenumber
                past_submission_number = '%03d' % past_submission.submission_number
                lines.append('%s/%s/output/: %s' % (
                    past_submission_phasenumber,
                    past_submission_number,
                    submission_private_output_filename(past_submission),
                ))
    else:
        pass

    submission.history_file.save('history.txt', ContentFile('\n'.join(lines)))

    # Generate metadata-only bundle describing the inputs. Reference data is an optional
    # dataset provided by the competition organizer. Results are provided by the participant
    # either indirectly (has_generated_predictions is True i.e. participant provides a program
    # which is run to generate results) ordirectly (participant uploads results directly).
    lines = []
    ref_value = submission.phase.reference_data.name
    if len(ref_value) > 0:
        lines.append("ref: %s" % ref_value)
    res_value = submission.prediction_output_file.name if has_generated_predictions else submission.file.name
    if len(res_value) > 0:
        lines.append("res: %s" % res_value)
    else:
        raise ValueError("Results are missing.")

    lines.append("history: %s" % submission_history_file_name(submission))
    lines.append("submitted-by: %s" % submission.participant.user.username)
    lines.append("submitted-at: %s" %
                 submission.submitted_at.replace(microsecond=0).isoformat())
    lines.append("competition-submission: %s" % submission.submission_number)
    lines.append("competition-phase: %s" % submission.phase.phasenumber)
    is_automatic_submission = False
    if submission.phase.auto_migration:
        # If this phase has auto_migration and this submission is the first in the phase, it is an automatic submission!
        submissions_this_phase = CompetitionSubmission.objects.filter(
            phase=submission.phase,
            participant=submission.participant).count()
        is_automatic_submission = submissions_this_phase == 1

    lines.append("automatic-submission: %s" % is_automatic_submission)
    submission.inputfile.save('input.txt', ContentFile('\n'.join(lines)))

    # Generate metadata-only bundle describing the computation.
    lines = []
    program_value = submission.phase.scoring_program.name
    if len(program_value) > 0:
        lines.append("program: %s" % program_value)
    else:
        raise ValueError("Program is missing.")
    lines.append("input: %s" % submission.inputfile.name)
    lines.append("stdout: %s" % submission_stdout_filename(submission))
    lines.append("stderr: %s" % submission_stderr_filename(submission))
    submission.runfile.save('run.txt', ContentFile('\n'.join(lines)))

    # Create stdout.txt & stderr.txt
    if has_generated_predictions == False:
        username = submission.participant.user.username
        lines = [
            "Standard output for submission #{0} by {1}.".format(
                submission.submission_number, username), ""
        ]
        submission.stdout_file.save('stdout.txt',
                                    ContentFile('\n'.join(lines)))
        lines = [
            "Standard error for submission #{0} by {1}.".format(
                submission.submission_number, username), ""
        ]
        submission.stderr_file.save('stderr.txt',
                                    ContentFile('\n'.join(lines)))
    # Update workflow state
    state['score'] = job_id
    submission.execution_key = json.dumps(state)
    submission.save()
    # Submit the request to the computation service
    body = json.dumps({
        "id": job_id,
        "task_type": "run",
        "task_args": {
            "bundle_id": submission.runfile.name,
            "container_name": settings.BUNDLE_AZURE_CONTAINER,
            "reply_to": settings.SBS_RESPONSE_QUEUE,
            "execution_time_limit": submission.phase.execution_time_limit
        }
    })
    getQueue(settings.SBS_COMPUTE_QUEUE).send_message(body)
    if has_generated_predictions == False:
        _set_submission_status(submission.id,
                               CompetitionSubmissionStatus.SUBMITTED)