Beispiel #1
0
    def invalidate_submission(self,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              contest_id=None):
        """Invalidate (and re-score) some submission results.

        Invalidate the scores of the submission results that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of contest_id or, if None, to any submission in
          the database.
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of contest_id or, if None, to any
          dataset in the database.

        submission_id (int|None): id of the submission whose results
            should be invalidated, or None.
        dataset_id (int|None): id of the dataset whose results should
            be invalidated, or None.
        participation_id (int|None): id of the participation whose results
            should be invalidated, or None.
        task_id (int|None): id of the task whose results should be
            invalidated, or None.
        contest_id (int|None): id of the contest whose results should
            be invalidated, or None.

        """
        logger.info("Invalidation request received.")

        # We can put results in the scorer queue only after they have
        # been invalidated (and committed to the database). Therefore
        # we temporarily save them somewhere else.
        temp_queue = list()

        with SessionGen() as session:
            submission_results = \
                get_submission_results(contest_id,
                                       participation_id, task_id,
                                       submission_id, dataset_id,
                                       session=session)

            for sr in submission_results:
                if sr.scored():
                    sr.invalidate_score()
                    # We also save the timestamp of the submission, to
                    # rescore them in order (for fairness, not for a
                    # specific need).
                    temp_queue.append(
                        (ScoringOperation(sr.submission_id, sr.dataset_id),
                         sr.submission.timestamp))

            session.commit()

        for item, timestamp in temp_queue:
            self.enqueue(item, timestamp=timestamp)

        logger.info("Invalidated %d submission results.", len(temp_queue))
Beispiel #2
0
    def invalidate_submission(self,
                              submission_id=None,
                              dataset_id=None,
                              user_id=None,
                              task_id=None):
        """Request for invalidating some scores.

        Invalidate the scores of the SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          user_id and/or task_id or, if both None, to any submission
          of the contest this service is running for.
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        submission_id (int): id of the submission to invalidate, or
                             None.
        dataset_id (int): id of the dataset to invalidate, or None.
        user_id (int): id of the user to invalidate, or None.
        task_id (int): id of the task to invalidate, or None.

        """
        logger.info("Invalidation request received.")

        # Validate arguments
        # TODO Check that all these objects belong to this contest.

        with SessionGen(commit=True) as session:
            submission_results = get_submission_results(
                # Give contest_id only if all others are None.
                self.contest_id
                    if {user_id, task_id, submission_id, dataset_id} == {None}
                    else None,
                user_id, task_id, submission_id, dataset_id, session)

            logger.info("Submission results to invalidate scores for: %d." %
                        len(submission_results))
            if len(submission_results) == 0:
                return

            new_submission_results_to_score = set()

            for submission_result in submission_results:
                # If the submission is not evaluated, it does not have
                # a score to invalidate, and, when evaluated,
                # ScoringService will be prompted to score it. So in
                # that case we do not have to do anything.
                if submission_result.evaluated():
                    submission_result.invalidate_score()
                    new_submission_results_to_score.add(
                        (submission_result.submission_id,
                         submission_result.dataset_id))

        old_s = len(self.submission_results_to_score)
        old_t = len(self.submissions_to_token)
        self.submission_results_to_score |= new_submission_results_to_score
        if old_s + old_t == 0:
            self.add_timeout(self.score_old_submissions, None,
                             0.5, immediately=False)
Beispiel #3
0
    def invalidate_submission(self, submission_id=None, dataset_id=None,
                              participation_id=None, task_id=None,
                              contest_id=None):
        """Invalidate (and re-score) some submission results.

        Invalidate the scores of the submission results that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of contest_id or, if None, to any submission in
          the database.
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of contest_id or, if None, to any
          dataset in the database.

        submission_id (int|None): id of the submission whose results
            should be invalidated, or None.
        dataset_id (int|None): id of the dataset whose results should
            be invalidated, or None.
        participation_id (int|None): id of the participation whose results
            should be invalidated, or None.
        task_id (int|None): id of the task whose results should be
            invalidated, or None.
        contest_id (int|None): id of the contest whose results should
            be invalidated, or None.

        """
        logger.info("Invalidation request received.")

        # We can put results in the scorer queue only after they have
        # been invalidated (and committed to the database). Therefore
        # we temporarily save them somewhere else.
        temp_queue = list()

        with SessionGen() as session:
            submission_results = \
                get_submission_results(contest_id,
                                       participation_id, task_id,
                                       submission_id, dataset_id,
                                       session=session)

            for sr in submission_results:
                if sr.scored():
                    sr.invalidate_score()
                    # We also save the timestamp of the submission, to
                    # rescore them in order (for fairness, not for a
                    # specific need).
                    temp_queue.append((
                        ScoringOperation(sr.submission_id, sr.dataset_id),
                        sr.submission.timestamp))

            session.commit()

        for item, timestamp in temp_queue:
            self.enqueue(item, timestamp=timestamp)

        logger.info("Invalidated %d submission results.", len(temp_queue))
Beispiel #4
0
    def invalidate_submission(self,
                              submission_id=None,
                              dataset_id=None,
                              user_id=None,
                              task_id=None,
                              contest_id=None):
        """Invalidate (and re-score) some submission results.

        Invalidate the scores of the submission results that:
        - belong to submission_id or, if None, to any submission of
          user_id and/or task_id or, if both None, to any submission
          of contest_id or, if None, to any submission in the database.
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of contest_id or, if None, to any
          dataset in the database.

        submission_id (int): id of the submission whose results should
            be invalidated, or None.
        dataset_id (int): id of the dataset whose results should be
            invalidated, or None.
        user_id (int): id of the user whose results should be
            invalidated, or None.
        task_id (int): id of the task whose results should be
            invalidated, or None.
        contest_id (int): id of the contest whose results should be
            invalidated, or None.

        """
        logger.info("Invalidation request received.")

        # We can put results in the scorer queue only after they have
        # been invalidated (and committed to the database). Therefore
        # we temporarily save them somewhere else.
        temp_queue = list()

        with SessionGen() as session:
            submission_results = \
                get_submission_results(contest_id, user_id, task_id,
                                       submission_id, dataset_id,
                                       session=session)

            for sr in submission_results:
                if sr.scored():
                    sr.invalidate_score()
                    temp_queue.append((sr.submission_id, sr.dataset_id))

            session.commit()

        for item in temp_queue:
            self._scorer_queue.put(item)

        logger.info("Invalidated %d submissions.", len(temp_queue))
Beispiel #5
0
    def _missing_operations(self):
        """Return a generator of unscored submission results.

        Obtain a list of all the submission results in the database,
        check each of them to see if it's still unscored and if so
        enqueue them.

        """
        counter = 0
        with SessionGen() as session:
            for sr in get_submission_results(session=session):
                if sr is not None and sr.needs_scoring():
                    self.enqueue(ScoringOperation(sr.submission_id, sr.dataset_id), timestamp=sr.submission.timestamp)
                    counter += 1
        return counter
Beispiel #6
0
    def invalidate_submission(self, submission_id=None, dataset_id=None,
                              user_id=None, task_id=None, contest_id=None):
        """Invalidate (and re-score) some submission results.

        Invalidate the scores of the submission results that:
        - belong to submission_id or, if None, to any submission of
          user_id and/or task_id or, if both None, to any submission
          of contest_id or, if None, to any submission in the database.
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of contest_id or, if None, to any
          dataset in the database.

        submission_id (int|None): id of the submission whose results
            should be invalidated, or None.
        dataset_id (int|None): id of the dataset whose results should
            be invalidated, or None.
        user_id (int|None): id of the user whose results should be
            invalidated, or None.
        task_id (int|None): id of the task whose results should be
            invalidated, or None.
        contest_id (int|None): id of the contest whose results should
            be invalidated, or None.

        """
        logger.info("Invalidation request received.")

        # We can put results in the scorer queue only after they have
        # been invalidated (and committed to the database). Therefore
        # we temporarily save them somewhere else.
        temp_queue = list()

        with SessionGen() as session:
            submission_results = \
                get_submission_results(contest_id, user_id, task_id,
                                       submission_id, dataset_id,
                                       session=session)

            for sr in submission_results:
                if sr.scored():
                    sr.invalidate_score()
                    temp_queue.append((sr.submission_id, sr.dataset_id))

            session.commit()

        for item in temp_queue:
            self._scorer_queue.put(item)

        logger.info("Invalidated %d submissions.", len(temp_queue))
Beispiel #7
0
    def _missing_operations(self):
        """Return a generator of unscored submission results.

        Obtain a list of all the submission results in the database,
        check each of them to see if it's still unscored and if so
        enqueue them.

        """
        counter = 0
        with SessionGen() as session:
            for sr in get_submission_results(session=session):
                if sr is not None and sr.needs_scoring():
                    self.enqueue(ScoringOperation(sr.submission_id,
                                                  sr.dataset_id),
                                 timestamp=sr.submission.timestamp)
                    counter += 1
        return counter
Beispiel #8
0
    def _sweep(self):
        """Check the database for unscored submission results.

        Obtain a list of all the submission results in the database,
        check each of them to see if it's still unscored and, in case,
        put it in the queue.

        """
        counter = 0

        with SessionGen() as session:
            for sr in get_submission_results(session=session):
                if sr is not None and sr.needs_scoring():
                    self._scorer_queue.put((sr.submission_id, sr.dataset_id))
                    counter += 1

        if counter > 0:
            logger.info("Found %d unscored submissions.", counter)
Beispiel #9
0
    def invalidate_submission(self,
                              contest_id=None,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              level="compilation"):
        """Request to invalidate some computed data.

        Invalidate the compilation and/or evaluation data of the
        SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of the contest asked for, or, if all three are
          None, the contest this service is running for (or all contests).
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        The data is cleared, the operations involving the submissions
        currently enqueued are deleted, and the ones already assigned to
        the workers are ignored. New appropriate operations are
        enqueued.

        submission_id (int|None): id of the submission to invalidate,
            or None.
        dataset_id (int|None): id of the dataset to invalidate, or
            None.
        participation_id (int|None): id of the participation to
            invalidate, or None.
        task_id (int|None): id of the task to invalidate, or None.
        level (string): 'compilation' or 'evaluation'

        """
        logger.info("Invalidation request received.")

        # Validate arguments
        # TODO Check that all these objects belong to this contest.
        if level not in ("compilation", "evaluation"):
            raise ValueError("Unexpected invalidation level `%s'." % level)

        if contest_id is None:
            contest_id = self.contest_id

        with SessionGen() as session:
            # When invalidating a dataset we need to know the task_id, otherwise
            # get_submissions will return all the submissions of the contest.
            if dataset_id is not None and task_id is None \
                    and submission_id is None:
                task_id = Dataset.get_from_id(dataset_id, session).task_id
            # First we load all involved submissions.
            submissions = get_submissions(
                # Give contest_id only if all others are None.
                contest_id if {participation_id, task_id, submission_id}
                == {None} else None,
                participation_id,
                task_id,
                submission_id,
                session)

            # Then we get all relevant operations, and we remove them
            # both from the queue and from the pool (i.e., we ignore
            # the workers involved in those operations).
            operations = get_relevant_operations(level, submissions,
                                                 dataset_id)
            for operation in operations:
                try:
                    self.dequeue(operation)
                except KeyError:
                    pass  # Ok, the operation wasn't in the queue.
                try:
                    self.get_executor().pool.ignore_operation(operation)
                except LookupError:
                    pass  # Ok, the operation wasn't in the pool.

            # Then we find all existing results in the database, and
            # we remove them.
            submission_results = get_submission_results(
                # Give contest_id only if all others are None.
                contest_id if {
                    participation_id, task_id, submission_id, dataset_id
                } == {None} else None,
                participation_id,
                # Provide the task_id only if the entire task has to be
                # reevaluated and not only a specific dataset.
                task_id if dataset_id is None else None,
                submission_id,
                dataset_id,
                session)
            logger.info("Submission results to invalidate %s for: %d.", level,
                        len(submission_results))
            for submission_result in submission_results:
                # We invalidate the appropriate data and queue the
                # operations to recompute those data.
                if level == "compilation":
                    submission_result.invalidate_compilation()
                elif level == "evaluation":
                    submission_result.invalidate_evaluation()

            # Finally, we re-enqueue the operations for the
            # submissions.
            for submission in submissions:
                self.submission_enqueue_operations(submission)

            session.commit()
        logger.info("Invalidate successfully completed.")
Beispiel #10
0
    def invalidate_submission(self,
                              contest_id=None,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              testcases=None,
                              overwrite=None,
                              force_priority=None,
                              level="compilation"):
        """Request to invalidate some computed data.

        Invalidate the compilation and/or evaluation data of the
        SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of the contest asked for, or, if all three are
          None, the contest this service is running for (or all contests).
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        The data is cleared, the operations involving the submissions
        currently enqueued are deleted, and the ones already assigned to
        the workers are ignored. New appropriate operations are
        enqueued.

        submission_id (int|None): id of the submission to invalidate,
            or None.
        dataset_id (int|None): id of the dataset to invalidate, or
            None.
        participation_id (int|None): id of the participation to
            invalidate, or None.
        task_id (int|None): id of the task to invalidate, or None.
        level (string): 'compilation' or 'evaluation'

        """
        logger.info("Invalidation request received.")

        # Avoid running the sweeper for the next 10-ish minutes, to
        # avoid race conditions between invalidate's requeuing of
        # submissions and the sweeper's.
        self.avoid_next_sweepers = 1

        # Validate arguments
        # TODO Check that all these objects belong to this contest.
        if level not in ("compilation", "evaluation"):
            raise ValueError("Unexpected invalidation level `%s'." % level)

        if contest_id is None:
            contest_id = self.contest_id

        with SessionGen() as session:
            # First we load all involved submissions.
            if (dataset_id is not None) and (submission_id is None):
                dataset = Dataset.get_from_id(dataset_id, session)
                task_id_for_submissions = dataset.task_id
            else:
                task_id_for_submissions = task_id
            submissions = get_submissions(
                # Give contest_id only if all others are None.
                contest_id if {
                    participation_id, task_id_for_submissions, submission_id
                } == {None} else None,
                participation_id,
                task_id_for_submissions,
                submission_id,
                session)

            # Then we get all relevant operations, and we remove them
            # both from the queue and from the pool (i.e., we ignore
            # the workers involved in those operations).
            operations = get_relevant_operations(level, submissions,
                                                 dataset_id, testcases)
            for operation in operations:
                try:
                    self.dequeue(operation)
                except KeyError:
                    pass  # Ok, the operation wasn't in the queue.
                try:
                    self.get_executor().pool.ignore_operation(operation)
                except LookupError:
                    pass  # Ok, the operation wasn't in the pool.

            # Then we find all existing results in the database, and
            # we remove them.
            submission_results = get_submission_results(
                # Give contest_id only if all others are None.
                contest_id if {
                    participation_id, task_id, submission_id, dataset_id
                } == {None} else None,
                participation_id,
                task_id,
                submission_id,
                dataset_id,
                session)
            logger.info("Submission results to invalidate %s for: %d.", level,
                        len(submission_results))
            for submission_result in submission_results:
                # We invalidate the appropriate data and queue the
                # operations to recompute those data.
                if level == "compilation":
                    submission_result.invalidate_compilation(
                        testcases if overwrite is not False else [])
                elif level == "evaluation":
                    submission_result.invalidate_evaluation(
                        testcases if overwrite is not False else [])

            # There is a small chance that an invalidate won't
            # succeed: if a result is in the pending structure, it
            # might be written after the commit here.
            session.commit()

            # Collecting the ids so that we can close the session
            # before the rpcs.
            submission_ids = [submission.id for submission in submissions]

        # Finally, we re-enqueue the operations for the submissions.
        for idx in range(0, len(submission_ids), 500):
            random_service(self.evaluation_services).new_submissions(
                submission_ids=submission_ids[idx:min(idx +
                                                      500, len(submission_ids
                                                               ))],
                dataset_id=dataset_id,
                force_priority=force_priority)

        logger.info("Invalidate successfully completed.")
Beispiel #11
0
    def invalidate_submission(self,
                              contest_id=None,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              level="compilation"):
        """Request to invalidate some computed data.

        Invalidate the compilation and/or evaluation data of the
        SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of the contest asked for, or, if all three are
          None, the contest this service is running for (or all contests).
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        The data is cleared, the operations involving the submissions
        currently enqueued are deleted, and the ones already assigned to
        the workers are ignored. New appropriate operations are
        enqueued.

        submission_id (int|None): id of the submission to invalidate,
            or None.
        dataset_id (int|None): id of the dataset to invalidate, or
            None.
        participation_id (int|None): id of the participation to
            invalidate, or None.
        task_id (int|None): id of the task to invalidate, or None.
        level (string): 'compilation' or 'evaluation'

        """
        logger.info("Invalidation request received.")

        # Validate arguments
        # TODO Check that all these objects belong to this contest.
        if level not in ("compilation", "evaluation"):
            raise ValueError(
                "Unexpected invalidation level `%s'." % level)

        if contest_id is None:
            contest_id = self.contest_id

        with SessionGen() as session:
            # First we load all involved submissions.
            submissions = get_submissions(
                # Give contest_id only if all others are None.
                contest_id
                if {participation_id, task_id, submission_id} == {None}
                else None,
                participation_id, task_id, submission_id, session)

            # Then we get all relevant operations, and we remove them
            # both from the queue and from the pool (i.e., we ignore
            # the workers involved in those operations).
            operations = get_relevant_operations(
                level, submissions, dataset_id)
            for operation in operations:
                try:
                    self.dequeue(operation)
                except KeyError:
                    pass  # Ok, the operation wasn't in the queue.
                try:
                    self.get_executor().pool.ignore_operation(operation)
                except LookupError:
                    pass  # Ok, the operation wasn't in the pool.

            # Then we find all existing results in the database, and
            # we remove them.
            submission_results = get_submission_results(
                # Give contest_id only if all others are None.
                contest_id
                if {participation_id,
                    task_id,
                    submission_id,
                    dataset_id} == {None}
                else None,
                participation_id, task_id, submission_id, dataset_id, session)
            logger.info("Submission results to invalidate %s for: %d.",
                        level, len(submission_results))
            for submission_result in submission_results:
                # We invalidate the appropriate data and queue the
                # operations to recompute those data.
                if level == "compilation":
                    submission_result.invalidate_compilation()
                elif level == "evaluation":
                    submission_result.invalidate_evaluation()

            # Finally, we re-enqueue the operations for the
            # submissions.
            for submission in submissions:
                self.submission_enqueue_operations(submission)

            session.commit()
        logger.info("Invalidate successfully completed.")