示例#1
0
文件: tasks.py 项目: wguo123/codalab
    def update_submission(submission, status, job_id, traceback=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        if status == 'running':
            _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            state = {}
            if len(submission.execution_key) > 0:
                logger.debug("update_submission_task loading state: %s", submission.execution_key)
                state = json.loads(submission.execution_key)
            if 'score' in state:
                logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
                submission.output_file.name = pathname2url(submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
                submission.save()
                logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
                logger.debug("Output.zip location=%s" % submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
                    _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)", submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" % (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
                                                                      key=label.strip())
                            SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)", submission.id)
                _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)

                result = Job.FINISHED
            else:
                logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
                url_name = pathname2url(submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
                except Exception:
                    logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()
        _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
示例#2
0
    def update_submission(submission, status, job_id, traceback=None, metadata=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        state = {}
        if len(submission.execution_key) > 0:
            logger.debug("update_submission_task loading state: %s", submission.execution_key)
            state = json.loads(submission.execution_key)
            logger.debug("update_submission_task state = %s" % submission.execution_key)

        if metadata:
            is_predict = 'score' not in state
            sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
                is_predict=is_predict,
                is_scoring=not is_predict,
                submission=submission,
            )
            sub_metadata.__dict__.update(metadata)
            sub_metadata.save()
            logger.debug("saving extra metadata, was a new object created? %s" % created)

        if status == 'running':
            _set_submission_status(submission.id, CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            if 'score' in state:
                logger.debug("update_submission_task loading final scores (pk=%s)", submission.pk)
                submission.output_file.name = pathname2url(submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(submission_detailed_results_filename(submission))
                submission.save()
                logger.debug("Retrieving output.zip and 'scores.txt' file (submission_id=%s)", submission.id)
                logger.debug("Output.zip location=%s" % submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error("Scores.txt not found, unable to process submission: %s (submission_id=%s)", status, submission.id)
                    _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)", submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" % (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(competition=submission.phase.competition,
                                                                      key=label.strip())
                            SubmissionScore.objects.create(result=submission, scoredef=scoredef, value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning("Score %s does not exist (submission_id=%s)", label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)", submission.id)
                _set_submission_status(submission.id, CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)", submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug("Leaderboard updated with latest submission (submission_id=%s)", submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug("Force submission added submission to leaderboard (submission_id=%s)", submission.id)

                result = Job.FINISHED

                if submission.participant.user.email_on_submission_finished_successfully:
                    email = submission.participant.user.email
                    site_url = "https://%s%s" % (Site.objects.get_current().domain, submission.phase.competition.get_absolute_url())
                    send_mail(
                        'Submission has finished successfully!',
                        'Your submission to the competition "%s" has finished successfully! View it here: %s' %
                        (submission.phase.competition.title, site_url),
                        settings.DEFAULT_FROM_EMAIL,
                        [email],
                        fail_silently=False
                    )
            else:
                logger.debug("update_submission_task entering scoring phase (pk=%s)", submission.pk)
                url_name = pathname2url(submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug("update_submission_task scoring phase entered (pk=%s)", submission.pk)
                except Exception:
                    logger.exception("update_submission_task failed to enter scoring phase (pk=%s)", submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status, submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()

        _set_submission_status(submission.id, CompetitionSubmissionStatus.FAILED)
示例#3
0
    def update_submission(submission,
                          status,
                          job_id,
                          traceback=None,
                          metadata=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        state = {}
        if len(submission.execution_key) > 0:
            logger.debug("update_submission_task loading state: %s",
                         submission.execution_key)
            state = json.loads(submission.execution_key)
            logger.debug("update_submission_task state = %s" %
                         submission.execution_key)

        if metadata:
            is_predict = 'score' not in state
            sub_metadata, created = CompetitionSubmissionMetadata.objects.get_or_create(
                is_predict=is_predict,
                is_scoring=not is_predict,
                submission=submission,
            )
            sub_metadata.__dict__.update(metadata)
            sub_metadata.save()
            logger.debug(
                "saving extra metadata, was a new object created? %s" %
                created)

        if status == 'running':
            _set_submission_status(submission.id,
                                   CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            if 'score' in state:
                logger.debug(
                    "update_submission_task loading final scores (pk=%s)",
                    submission.pk)
                submission.output_file.name = pathname2url(
                    submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(
                    submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(
                    submission_detailed_results_filename(submission))
                submission.save()
                logger.debug(
                    "Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
                    submission.id)
                logger.debug("Output.zip location=%s" %
                             submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error(
                        "Scores.txt not found, unable to process submission: %s (submission_id=%s)",
                        status, submission.id)
                    _set_submission_status(submission.id,
                                           CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)",
                             submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" %
                                     (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(
                                competition=submission.phase.competition,
                                key=label.strip())
                            SubmissionScore.objects.create(result=submission,
                                                           scoredef=scoredef,
                                                           value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning(
                                "Score %s does not exist (submission_id=%s)",
                                label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)",
                             submission.id)
                _set_submission_status(submission.id,
                                       CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)",
                                 submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Leaderboard updated with latest submission (submission_id=%s)",
                        submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Force submission added submission to leaderboard (submission_id=%s)",
                        submission.id)

                result = Job.FINISHED

                if submission.participant.user.email_on_submission_finished_successfully:
                    email = submission.participant.user.email
                    site_url = "https://%s%s" % (
                        Site.objects.get_current().domain,
                        submission.phase.competition.get_absolute_url())
                    send_mail(
                        'Submission has finished successfully!',
                        'Your submission to the competition "%s" has finished successfully! View it here: %s'
                        % (submission.phase.competition.title, site_url),
                        settings.DEFAULT_FROM_EMAIL, [email],
                        fail_silently=False)
            else:
                logger.debug(
                    "update_submission_task entering scoring phase (pk=%s)",
                    submission.pk)
                url_name = pathname2url(
                    submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(
                    predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(
                    predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug(
                        "update_submission_task scoring phase entered (pk=%s)",
                        submission.pk)
                except Exception:
                    logger.exception(
                        "update_submission_task failed to enter scoring phase (pk=%s)",
                        submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status,
                         submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()

        _set_submission_status(submission.id,
                               CompetitionSubmissionStatus.FAILED)
示例#4
0
    def update_submission(submission, status, job_id, traceback=None):
        """
        Updates the status of a submission.

        submission: The CompetitionSubmission object to update.
        status: The new status string: 'running', 'finished' or 'failed'.
        job_id: The job ID used to track the progress of the evaluation.
        """
        if status == 'running':
            _set_submission_status(submission.id,
                                   CompetitionSubmissionStatus.RUNNING)
            return Job.RUNNING

        if status == 'finished':
            result = Job.FAILED
            state = {}
            if len(submission.execution_key) > 0:
                logger.debug("update_submission_task loading state: %s",
                             submission.execution_key)
                state = json.loads(submission.execution_key)
            if 'score' in state:
                logger.debug(
                    "update_submission_task loading final scores (pk=%s)",
                    submission.pk)
                submission.output_file.name = pathname2url(
                    submission_output_filename(submission))
                submission.private_output_file.name = pathname2url(
                    submission_private_output_filename(submission))
                submission.detailed_results_file.name = pathname2url(
                    submission_detailed_results_filename(submission))
                submission.save()
                logger.debug(
                    "Retrieving output.zip and 'scores.txt' file (submission_id=%s)",
                    submission.id)
                logger.debug("Output.zip location=%s" %
                             submission.output_file.file.name)
                ozip = ZipFile(io.BytesIO(submission.output_file.read()))
                scores = None
                try:
                    scores = open(ozip.extract('scores.txt'), 'r').read()
                except Exception:
                    logger.error(
                        "Scores.txt not found, unable to process submission: %s (submission_id=%s)",
                        status, submission.id)
                    _set_submission_status(submission.id,
                                           CompetitionSubmissionStatus.FAILED)
                    return Job.FAILED

                logger.debug("Processing scores... (submission_id=%s)",
                             submission.id)
                for line in scores.split("\n"):
                    if len(line) > 0:
                        label, value = line.split(":")
                        logger.debug("Attempting to submit score %s:%s" %
                                     (label, value))
                        try:
                            scoredef = SubmissionScoreDef.objects.get(
                                competition=submission.phase.competition,
                                key=label.strip())
                            SubmissionScore.objects.create(result=submission,
                                                           scoredef=scoredef,
                                                           value=float(value))
                        except SubmissionScoreDef.DoesNotExist:
                            logger.warning(
                                "Score %s does not exist (submission_id=%s)",
                                label, submission.id)
                logger.debug("Done processing scores... (submission_id=%s)",
                             submission.id)
                _set_submission_status(submission.id,
                                       CompetitionSubmissionStatus.FINISHED)
                # Automatically submit to the leaderboard?
                if submission.phase.is_blind:
                    logger.debug("Adding to leaderboard... (submission_id=%s)",
                                 submission.id)
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Leaderboard updated with latest submission (submission_id=%s)",
                        submission.id)

                if submission.phase.competition.force_submission_to_leaderboard:
                    add_submission_to_leaderboard(submission)
                    logger.debug(
                        "Force submission added submission to leaderboard (submission_id=%s)",
                        submission.id)

                result = Job.FINISHED
            else:
                logger.debug(
                    "update_submission_task entering scoring phase (pk=%s)",
                    submission.pk)
                url_name = pathname2url(
                    submission_prediction_output_filename(submission))
                submission.prediction_output_file.name = url_name
                submission.prediction_stderr_file.name = pathname2url(
                    predict_submission_stdout_filename(submission))
                submission.prediction_stdout_file.name = pathname2url(
                    predict_submission_stderr_filename(submission))
                submission.save()
                try:
                    score(submission, job_id)
                    result = Job.RUNNING
                    logger.debug(
                        "update_submission_task scoring phase entered (pk=%s)",
                        submission.pk)
                except Exception:
                    logger.exception(
                        "update_submission_task failed to enter scoring phase (pk=%s)",
                        submission.pk)
            return result

        if status != 'failed':
            logger.error("Invalid status: %s (submission_id=%s)", status,
                         submission.id)

        if traceback:
            submission.exception_details = traceback
            submission.save()
        _set_submission_status(submission.id,
                               CompetitionSubmissionStatus.FAILED)