Esempio n. 1
0
    def dataset_updated(self, task_id):
        """This function updates RWS with new data about a task. It should be
        called after the live dataset of a task is changed.

        task_id (int): id of the task whose dataset has changed.

        """
        with SessionGen(commit=False) as session:
            task = Task.get_from_id(task_id, session)
            dataset_id = task.active_dataset_id

        logger.info("Dataset update for task %d (dataset now is %d)." % (
            task_id, dataset_id))

        submission_ids = get_submissions(self.contest_id, task_id=task_id)

        subchanges = []
        with SessionGen(commit=False) as session:
            for submission_id in submission_ids:
                submission = Submission.get_from_id(submission_id, session)
                submission_result = SubmissionResult.get_from_id(
                    (submission_id, dataset_id), session)

                if submission_result is None:
                    # Not yet compiled, evaluated or scored.
                    score = None
                    ranking_score_details = None
                else:
                    score = submission_result.score
                    try:
                        ranking_score_details = json.loads(
                                submission_result.ranking_score_details)
                    except (json.decoder.JSONDecodeError, TypeError):
                        # It may be blank.
                        ranking_score_details = None

                # Data to send to remote rankings.
                subchange_id = "%s%ss" % \
                    (int(make_timestamp(submission.timestamp)),
                     submission_id)
                subchange_put_data = {
                    "submission": encode_id(submission_id),
                    "time": int(make_timestamp(submission.timestamp))}
                    # We're sending the unrounded score to RWS
                if score is not None:
                    subchange_put_data["score"] = score
                if ranking_score_details is not None:
                    subchange_put_data["extra"] = ranking_score_details
                subchanges.append((subchange_id, subchange_put_data))

        # Adding operations to the queue.
        with self.operation_queue_lock:
            for ranking in self.rankings:
                for subchange_id, data in subchanges:
                    self.subchange_queue.setdefault(
                        ranking,
                        dict())[encode_id(subchange_id)] = data
Esempio n. 2
0
    def search_jobs_not_done(self):
        """Look in the database for submissions that have not been
        scored for no good reasons. Put the missing job in the queue.

        """
        # Do this only if we are not still loading old submission
        # (from the start of the service).
        if self.scoring_old_submission:
            return True

        with SessionGen(commit=False) as session:
            new_submission_ids_to_score = set([])
            new_submission_ids_to_token = set([])
            contest = session.query(Contest).\
                      filter_by(id=self.contest_id).first()
            for submission in contest.get_submissions():
                for dataset in get_autojudge_datasets(submission.task):
                    # If a submission result does not yet exist, then we don't
                    # need to score it.
                    r = SubmissionResult.get_from_id(
                        (submission.id, dataset.id),
                        session)
                    if r is None:
                        continue

                    x = (r.submission_id, r.dataset_id)
                    if r is not None and (r.evaluated()
                        or r.compilation_outcome == "fail") \
                            and x not in self.submission_ids_scored:
                        new_submission_ids_to_score.add(x)
                    if r.submission.tokened() and r.submission_id not in \
                            self.submission_ids_tokened:
                        new_submission_ids_to_token.add(
                            (r.submission_id,
                             make_timestamp(r.submission.token.timestamp)))

        new_s = len(new_submission_ids_to_score)
        old_s = len(self.submission_ids_to_score)
        new_t = len(new_submission_ids_to_token)
        old_t = len(self.submission_ids_to_token)
        logger.info("Submissions found to score/token: %d, %d." %
                    (new_s, new_t))
        if new_s + new_t > 0:
            self.submission_ids_to_score |= new_submission_ids_to_score
            self.submission_ids_to_token |= new_submission_ids_to_token
            if old_s + old_t == 0:
                self.add_timeout(self.score_old_submissions, None,
                                 0.5, immediately=False)

        # Run forever.
        return True
Esempio n. 3
0
    def invalidate_submission(self,
                              submission_id=None,
                              user_id=None,
                              dataset_id=None,
                              task_id=None):
        """Request for invalidating the scores of some submissions.

        The scores to be cleared are the one regarding
            1) a submission (only submission_id and dataset_id given)
         or 2) all submissions of a user for the active dataset (only user_id
               given)
         or 3) all submissions for a dataset (only dataset_id given)

        submission_id (int): id of the submission to invalidate, or
                             None.
        user_id (int): id of the user we want to invalidate, or None.
        dataset_id (int): id of the dataset we want to invalidate, or None.

        """
        logger.info("Invalidation request received.")

        # If we are invalidating a dataset, get the task so we can find all the
        # relevant submissions.
        if submission_id is None and dataset_id is not None:
            with SessionGen(commit=False) as session:
                dataset = Dataset.get_from_id(dataset_id, session)
                task_id = dataset.task_id
        else:
            task_id = None

        submission_ids = get_submissions(
            self.contest_id,
            submission_id, user_id, task_id)

        logger.info("Submissions to invalidate: %s" % len(submission_ids))
        if len(submission_ids) == 0:
            return

        new_submission_ids = []
        with SessionGen(commit=True) as session:
            for submission_id in submission_ids:
                submission = Submission.get_from_id(submission_id, session)
                if dataset_id is None:
                    this_dataset_id = submission.task.active_dataset_id
                else:
                    this_dataset_id = dataset_id
                sr = SubmissionResult.get_from_id(
                    (submission_id, this_dataset_id), session)
                # If the submission is not evaluated, it does not have
                # a score to invalidate, and, when evaluated,
                # ScoringService will be prompted to score it. So in
                # that case we do not have to do anything.
                if sr is not None and sr.evaluated():
                    sr.invalidate_score()
                    new_submission_ids.append((submission_id, this_dataset_id))

        old_s = len(self.submission_ids_to_score)
        old_t = len(self.submission_ids_to_token)
        self.submission_ids_to_score |= set(new_submission_ids)
        if old_s + old_t == 0:
            self.add_timeout(self.score_old_submissions, None,
                             0.5, immediately=False)
Esempio n. 4
0
    def new_evaluation(self, submission_id, dataset_id):
        """This RPC inform ScoringService that ES finished the work on
        a submission (either because it has been evaluated, or because
        the compilation failed).

        submission_id (int): the id of the submission that changed.
        dataset_verion (int): the dataset version used.

        """
        with SessionGen(commit=True) as session:
            submission_result = SubmissionResult.get_from_id(
                (submission_id, dataset_id), session)

            if submission_result is None:
                logger.error("[new_evaluation] Couldn't find "
                             " submission %d in the database." %
                             submission_id)
                raise KeyError
            submission = submission_result.submission

            if not submission_result.compiled():
                logger.warning("[new_evaluation] Submission %d(%d) "
                               "is not compiled." % (
                               submission_id, dataset_id))
                return
            elif submission_result.compilation_outcome == "ok" \
                    and not submission_result.evaluated():
                logger.warning("[new_evaluation] Submission %d(%d) compiled "
                               "correctly but is not evaluated."
                               % (submission_id, dataset_id))
                return
            elif submission.user.hidden:
                logger.info("[new_evaluation] Submission %d not scored "
                            "because user is hidden." % submission_id)
                return

            # Assign score to the submission.
            scorer = self.scorers.get(dataset_id)
            if scorer is None:
                # We may get here because the scorer threw an exception whilst
                # initalizing, or we may be scoring for the wrong contest.
                logger.error(
                    "Not scoring submission %d because scorer is broken." %
                        submission_id)
                return

            try:
                scorer.add_submission(submission_id, dataset_id,
                      submission.timestamp,
                      submission.user.username,
                      submission_result.evaluated(),
                      dict((ev.num,
                            {"outcome": ev.outcome,
                             "text": ev.text,
                             "time": ev.execution_time,
                             "memory": ev.memory_used})
                           for ev in submission_result.evaluations),
                      submission.tokened())
            except:
                logger.error("Failed to score submission %d. "
                    "Scorer threw an exception: %s" % (
                        submission_id, traceback.format_exc()))
                return

            # Mark submission as scored.
            self.submission_ids_scored.add((submission_id, dataset_id))

            # Filling submission's score info in the db.
            submission_result.score = scorer.pool[submission_id]["score"]
            submission_result.public_score = \
                scorer.pool[submission_id]["public_score"]

            # And details.
            submission_result.score_details = \
                scorer.pool[submission_id]["details"]
            submission_result.public_score_details = \
                scorer.pool[submission_id]["public_details"]
            submission_result.ranking_score_details = \
                scorer.pool[submission_id]["ranking_details"]

            try:
                ranking_score_details = json.loads(
                        submission_result.ranking_score_details)
            except (json.decoder.JSONDecodeError, TypeError):
                # It may be blank.
                ranking_score_details = None

            # If we are not a live dataset then we can bail out here, and avoid
            # updating RWS.
            if dataset_id != submission.task.active_dataset_id:
                return

            # Data to send to remote rankings.
            submission_put_data = {
                "user": encode_id(submission.user.username),
                "task": encode_id(submission.task.name),
                "time": int(make_timestamp(submission.timestamp))}
            subchange_id = "%s%ss" % \
                (int(make_timestamp(submission.timestamp)),
                 submission_id)
            subchange_put_data = {
                "submission": encode_id(submission_id),
                "time": int(make_timestamp(submission.timestamp)),
                # We're sending the unrounded score to RWS
                "score": submission_result.score}
            if ranking_score_details is not None:
                subchange_put_data["extra"] = ranking_score_details

        # TODO: ScoreRelative here does not work with remote
        # rankings (it does in the ranking view) because we
        # update only the user owning the submission.

        # Adding operations to the queue.
        with self.operation_queue_lock:
            for ranking in self.rankings:
                self.submission_queue.setdefault(
                    ranking,
                    dict())[encode_id(submission_id)] = \
                    submission_put_data
                self.subchange_queue.setdefault(
                    ranking,
                    dict())[encode_id(subchange_id)] = \
                    subchange_put_data