Ejemplo n.º 1
0
    def post(self, dataset_id_to_copy):
        fallback_page = "/dataset/%s/clone" % dataset_id_to_copy

        dataset = self.safe_get_item(Dataset, dataset_id_to_copy)
        task = self.safe_get_item(Task, dataset.task_id)
        task_id = task.id

        try:
            original_dataset = self.safe_get_item(Dataset, dataset_id_to_copy)
        except ValueError:
            raise tornado.web.HTTPError(404)

        try:
            attrs = dict()

            self.get_string(attrs, "description")

            # Ensure description is unique.
            if any(attrs["description"] == d.description for d in task.datasets):
                self.application.service.add_notification(
                    make_datetime(),
                    "Dataset name %r is already taken." % attrs["description"],
                    "Please choose a unique name for this dataset.",
                )
                self.redirect(fallback_page)
                return

            self.get_time_limit(attrs, "time_limit")
            self.get_memory_limit(attrs, "memory_limit")
            self.get_task_type(attrs, "task_type", "TaskTypeOptions_")
            self.get_score_type(attrs, "score_type", "score_type_parameters")

            # Create the dataset.
            attrs["autojudge"] = False
            attrs["task"] = task
            dataset = Dataset(**attrs)
            self.sql_session.add(dataset)

        except Exception as error:
            logger.warning("Invalid field.", exc_info=True)
            self.application.service.add_notification(make_datetime(), "Invalid field(s)", repr(error))
            self.redirect(fallback_page)
            return

        if original_dataset is not None:
            # If we were cloning the dataset, copy all managers and
            # testcases across too. If the user insists, clone all
            # evaluation information too.
            clone_results = bool(self.get_argument("clone_results", False))
            dataset.clone_from(original_dataset, True, True, clone_results)

        # If the task does not yet have an active dataset, make this
        # one active.
        if task.active_dataset is None:
            task.active_dataset = dataset

        if self.try_commit():
            self.redirect("/task/%s" % task_id)
        else:
            self.redirect(fallback_page)
Ejemplo n.º 2
0
    def build_job(self, session):
        """Produce the Job for this operation.

        Return the Job object that has to be sent to Workers to have
        them perform the operation this object describes.

        session (Session): the database session to use to fetch objects
            if necessary.

        return (Job): the job encoding of the operation, as understood
            by Workers and TaskTypes.

        """
        result = None
        dataset = Dataset.get_from_id(self.dataset_id, session)
        if self.type_ == ESOperation.COMPILATION:
            submission = Submission.get_from_id(self.object_id, session)
            result = CompilationJob.from_submission(submission, dataset)
        elif self.type_ == ESOperation.EVALUATION:
            submission = Submission.get_from_id(self.object_id, session)
            result = EvaluationJob.from_submission(
                submission, dataset, self.testcase_codename)
        elif self.type_ == ESOperation.USER_TEST_COMPILATION:
            user_test = UserTest.get_from_id(self.object_id, session)
            result = CompilationJob.from_user_test(user_test, dataset)
        elif self.type_ == ESOperation.USER_TEST_EVALUATION:
            user_test = UserTest.get_from_id(self.object_id, session)
            result = EvaluationJob.from_user_test(user_test, dataset)
        return result
Ejemplo n.º 3
0
    def enqueue(self, operation, priority, timestamp, job=None):
        """Push an operation in the queue.

        Push an operation in the operation queue if the submission is
        not already in the queue or assigned to a worker.

        operation (ESOperation): the operation to put in the queue.
        priority (int): the priority of the operation.
        timestamp (datetime): the time of the submission.
        job (dict|None): the job associated; if None will be computed

        return (bool): True if pushed, False if not.

        """
        if job is None:
            with SessionGen() as session:
                dataset = Dataset.get_from_id(operation.dataset_id, session)
                if operation.for_submission():
                    object_ = Submission.get_from_id(operation.object_id,
                                                     session)
                else:
                    object_ = UserTest.get_from_id(operation.object_id,
                                                   session)
                job = Job.from_operation(operation, object_,
                                         dataset).export_to_dict()
        return self.queue_service.enqueue(
            operation=operation.to_list(),
            priority=priority,
            timestamp=(timestamp - EvaluationService.EPOCH).total_seconds(),
            job=job)
Ejemplo n.º 4
0
    def new_submission(self,
                       submission_id,
                       dataset_id=None,
                       force_priority=None):
        """This RPC prompts ES of the existence of a new
        submission. ES takes the right countermeasures, i.e., it
        schedules it for compilation.

        submission_id (int): the id of the new submission.

        """
        with SessionGen() as session:
            submission = Submission.get_from_id(submission_id, session)
            if dataset_id is not None:
                dataset = Dataset.get_from_id(dataset_id, session)
            else:
                dataset = None
            if submission is None:
                logger.error(
                    "[new_submission] Couldn't find submission "
                    "%d in the database.", submission_id)
                return

            self.enqueue_all(self.get_submission_operations(
                submission, dataset),
                             force_priority=force_priority)

            session.commit()
Ejemplo n.º 5
0
    def build_job(self, session):
        """Produce the Job for this operation.

        Return the Job object that has to be sent to Workers to have
        them perform the operation this object describes.

        session (Session): the database session to use to fetch objects
            if necessary.

        return (Job): the job encoding of the operation, as understood
            by Workers and TaskTypes.

        """
        result = None
        dataset = Dataset.get_from_id(self.dataset_id, session)
        if self.type_ == ESOperation.COMPILATION:
            submission = Submission.get_from_id(self.object_id, session)
            result = CompilationJob.from_submission(submission, dataset)
        elif self.type_ == ESOperation.EVALUATION:
            submission = Submission.get_from_id(self.object_id, session)
            result = EvaluationJob.from_submission(submission, dataset,
                                                   self.testcase_codename)
        elif self.type_ == ESOperation.USER_TEST_COMPILATION:
            user_test = UserTest.get_from_id(self.object_id, session)
            result = CompilationJob.from_user_test(user_test, dataset)
        elif self.type_ == ESOperation.USER_TEST_EVALUATION:
            user_test = UserTest.get_from_id(self.object_id, session)
            result = EvaluationJob.from_user_test(user_test, dataset)
        return result
Ejemplo n.º 6
0
    def acquire_worker(self, operations):
        """Tries to assign an operation to an available worker. If no workers
        are available then this returns None, otherwise this returns
        the chosen worker.

        operations ([ESOperation]): the operations to assign to a worker.

        return (int|None): None if no workers are available, the worker
            assigned to the operation otherwise.

        """
        # We look for an available worker.
        try:
            shard = self.find_worker(WorkerPool.WORKER_INACTIVE,
                                     require_connection=True,
                                     random_worker=True)
        except LookupError:
            self._workers_available_event.clear()
            return None

        # Then we fill the info for future memory.
        self._add_operations(shard, operations)

        logger.debug("Worker %s acquired.", shard)
        self._start_time[shard] = make_datetime()

        with SessionGen() as session:
            jobs = []
            datasets = {}
            submissions = {}
            user_tests = {}
            for operation in operations:
                if operation.dataset_id not in datasets:
                    datasets[operation.dataset_id] = Dataset.get_from_id(
                        operation.dataset_id, session)
                object_ = None
                if operation.for_submission():
                    if operation.object_id not in submissions:
                        submissions[operation.object_id] = \
                            Submission.get_from_id(
                                operation.object_id, session)
                    object_ = submissions[operation.object_id]
                else:
                    if operation.object_id not in user_tests:
                        user_tests[operation.object_id] = \
                            UserTest.get_from_id(operation.object_id, session)
                    object_ = user_tests[operation.object_id]
                logger.info("Asking worker %s to `%s'.", shard, operation)

                jobs.append(
                    Job.from_operation(operation, object_,
                                       datasets[operation.dataset_id]))
            job_group_dict = JobGroup(jobs).export_to_dict()

        self._worker[shard].execute_job_group(
            job_group_dict=job_group_dict,
            callback=self._service.action_finished,
            plus=shard)
        return shard
Ejemplo n.º 7
0
    def execute(self, entry):
        """Assign a score to a submission result.

        This is the core of ScoringService: here we retrieve the result
        from the database, check if it is in the correct status,
        instantiate its ScoreType, compute its score, store it back in
        the database and tell ProxyService to update RWS if needed.

        entry (QueueEntry): entry containing the operation to perform.

        """
        operation = entry.item
        with SessionGen() as session:
            # Obtain submission.
            submission = Submission.get_from_id(operation.submission_id, session)
            if submission is None:
                raise ValueError("Submission %d not found in the database." % operation.submission_id)

            # Obtain dataset.
            dataset = Dataset.get_from_id(operation.dataset_id, session)
            if dataset is None:
                raise ValueError("Dataset %d not found in the database." % operation.dataset_id)

            # Obtain submission result.
            submission_result = submission.get_result(dataset)

            # It means it was not even compiled (for some reason).
            if submission_result is None:
                raise ValueError(
                    "Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id)
                )

            # Check if it's ready to be scored.
            if not submission_result.needs_scoring():
                if submission_result.scored():
                    logger.info(
                        "Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id
                    )
                    return
                else:
                    raise ValueError(
                        "The state of the submission result "
                        "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id)
                    )

            # Instantiate the score type.
            score_type = get_score_type(dataset=dataset)

            # Compute score and fill it in the database.
            submission_result.score, submission_result.score_details, submission_result.public_score, submission_result.public_score_details, submission_result.ranking_score_details = score_type.compute_score(
                submission_result
            )

            # Store it.
            session.commit()

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                self.proxy_service.submission_scored(submission_id=submission.id)
Ejemplo n.º 8
0
    def acquire_worker(self, operations):
        """Tries to assign an operation to an available worker. If no workers
        are available then this returns None, otherwise this returns
        the chosen worker.

        operations ([ESOperation]): the operations to assign to a worker.

        return (int|None): None if no workers are available, the worker
            assigned to the operation otherwise.

        """
        # We look for an available worker.
        try:
            shard = self.find_worker(WorkerPool.WORKER_INACTIVE,
                                     require_connection=True,
                                     random_worker=True)
        except LookupError:
            self._workers_available_event.clear()
            return None

        # Then we fill the info for future memory.
        self._add_operations(shard, operations)

        logger.debug("Worker %s acquired.", shard)
        self._start_time[shard] = make_datetime()

        with SessionGen() as session:
            jobs = []
            datasets = {}
            submissions = {}
            user_tests = {}
            for operation in operations:
                if operation.dataset_id not in datasets:
                    datasets[operation.dataset_id] = Dataset.get_from_id(
                        operation.dataset_id, session)
                object_ = None
                if operation.for_submission():
                    if operation.object_id not in submissions:
                        submissions[operation.object_id] = \
                            Submission.get_from_id(
                                operation.object_id, session)
                    object_ = submissions[operation.object_id]
                else:
                    if operation.object_id not in user_tests:
                        user_tests[operation.object_id] = \
                            UserTest.get_from_id(operation.object_id, session)
                    object_ = user_tests[operation.object_id]
                logger.info("Asking worker %s to `%s'.", shard, operation)

                jobs.append(Job.from_operation(
                    operation, object_, datasets[operation.dataset_id]))
            job_group_dict = JobGroup(jobs).export_to_dict()

        self._worker[shard].execute_job_group(
            job_group_dict=job_group_dict,
            callback=self._service.action_finished,
            plus=shard)
        return shard
Ejemplo n.º 9
0
    def post(self):
        fallback_page = "/tasks/add"

        try:
            attrs = dict()

            self.get_string(attrs, "name", empty=None)
            self.get_string(attrs, "category")

            assert attrs.get("name") is not None, "No task name specified."
            attrs["title"] = attrs["name"]

            # Set default submission format as ["taskname.%l"]
            attrs["submission_format"] = \
                [SubmissionFormatElement("%s.%%l" % attrs["name"])]

            # Create the task.
            task = Task(**attrs)
            self.sql_session.add(task)

        except Exception as error:
            self.application.service.add_notification(make_datetime(),
                                                      "Invalid field(s)",
                                                      repr(error))
            self.redirect(fallback_page)
            return

        try:
            attrs = dict()

            # Create its first dataset.
            attrs["description"] = "Default"
            attrs["autojudge"] = True
            attrs["task_type"] = "Batch"
            attrs["task_type_parameters"] = '["alone", ["", ""], "diff"]'
            attrs["score_type"] = "Sum"
            attrs["score_type_parameters"] = '100'
            attrs["task"] = task
            dataset = Dataset(**attrs)
            self.sql_session.add(dataset)

            # Make the dataset active. Life works better that way.
            task.active_dataset = dataset

        except Exception as error:
            self.application.service.add_notification(make_datetime(),
                                                      "Invalid field(s)",
                                                      repr(error))
            self.redirect(fallback_page)
            return

        if self.try_commit():
            # Create the task on RWS.
            self.application.service.proxy_service.reinitialize()
            self.redirect("/task/%s" % task.id)
        else:
            self.redirect(fallback_page)
Ejemplo n.º 10
0
    def from_operations(operations, session):
        jobs = []
        for operation in operations:
            # The get_from_id method loads from the instance map (if the
            # object exists there), which thus acts as a cache.
            if operation.for_submission():
                object_ = Submission.get_from_id(operation.object_id, session)
            else:
                object_ = UserTest.get_from_id(operation.object_id, session)
            dataset = Dataset.get_from_id(operation.dataset_id, session)

            jobs.append(Job.from_operation(operation, object_, dataset))
        return JobGroup(jobs)
Ejemplo n.º 11
0
    def create_dataset(self, task):
        """
        Create the main dataset for this task.
        """

        args = {}
        self.put_dataset_basic_info(args, task)
        self.put_dataset_limits(args)
        self.put_dataset_score_type(args)
        self.put_dataset_type_parameters(args)
        self.put_dataset_managers(args)
        self.put_dataset_testcases(args)

        return Dataset(**args)
Ejemplo n.º 12
0
    def post(self, task_id):
        fallback_page = self.url("task", task_id, "add_dataset")

        task = self.safe_get_item(Task, task_id)

        try:
            attrs = dict()

            self.get_string(attrs, "description")

            # Ensure description is unique.
            if any(attrs["description"] == d.description
                   for d in task.datasets):
                self.service.add_notification(
                    make_datetime(),
                    "Dataset name %r is already taken." % attrs["description"],
                    "Please choose a unique name for this dataset.")
                self.redirect(fallback_page)
                return

            self.get_time_limit(attrs, "time_limit")
            self.get_time_limit(attrs, "time_limit_python",
                                "time_limit_python")
            self.get_memory_limit(attrs, "memory_limit")
            self.get_task_type(attrs, "task_type", "TaskTypeOptions_")
            self.get_score_type(attrs, "score_type", "score_type_parameters")

            # Create the dataset.
            attrs["autojudge"] = False
            attrs["task"] = task
            dataset = Dataset(**attrs)
            self.sql_session.add(dataset)

        except Exception as error:
            logger.warning("Invalid field: %s" % (traceback.format_exc()))
            self.service.add_notification(make_datetime(), "Invalid field(s)",
                                          repr(error))
            self.redirect(fallback_page)
            return

        # If the task does not yet have an active dataset, make this
        # one active.
        if task.active_dataset is None:
            task.active_dataset = dataset

        if self.try_commit():
            # self.service.scoring_service.reinitialize()
            self.redirect(self.url("task", task_id))
        else:
            self.redirect(fallback_page)
Ejemplo n.º 13
0
 def get_dataset(cls, task=None, **kwargs):
     """Create a dataset"""
     task = task if task is not None else cls.get_task()
     args = {
         "task": task,
         "description": unique_unicode_id(),
         "task_type": "Batch",
         "task_type_parameters": ["alone", ["", ""], "diff"],
         "score_type": "Sum",
         "score_type_parameters": 100,
     }
     args.update(kwargs)
     dataset = Dataset(**args)
     return dataset
Ejemplo n.º 14
0
 def add_dataset(self, task=None, **kwargs):
     """Add a dataset."""
     task = task if task is not None else self.add_task()
     args = {
         "task": task,
         "description": unique_unicode_id(),
         "task_type": "",
         "task_type_parameters": "",
         "score_type": "",
         "score_type_parameters": "",
     }
     args.update(kwargs)
     dataset = Dataset(**args)
     self.session.add(dataset)
     return dataset
Ejemplo n.º 15
0
 def get_dataset(cls, task=None, **kwargs):
     """Create a dataset"""
     task = task if task is not None else cls.get_task()
     args = {
         "task": task,
         "description": unique_unicode_id(),
         "task_type": "",
         # "None" won't work here as the column is defined as non
         # nullable. As soon as we'll depend on SQLAlchemy 1.1 we
         # will be able to put JSON.NULL here instead.
         "task_type_parameters": {},
         "score_type": "",
         # Same here.
         "score_type_parameters": {},
     }
     args.update(kwargs)
     dataset = Dataset(**args)
     return dataset
Ejemplo n.º 16
0
    def get_submission_ops(self, submission_id, dataset_id=None):
        """This RPC returns the operations (including job) for the given
        submission and dataset.

        """
        with SessionGen() as session:
            submission = Submission.get_from_id(submission_id, session)
            if dataset_id is not None:
                dataset = Dataset.get_from_id(dataset_id, session)
            else:
                dataset = None
            if submission is None:
                logger.error(
                    "[get_submission_ops] Couldn't find submission "
                    "%d in the database.", submission_id)
                return []

            return [(operation.to_list(), priority,
                     (timestamp - EvaluationService.EPOCH).total_seconds(),
                     job) for operation, priority, timestamp, job in
                    self.get_submission_operations(submission, dataset)]
Ejemplo n.º 17
0
    def _score(self, submission_id, dataset_id):
        """Assign a score to a submission result.

        This is the core of ScoringService: here we retrieve the result
        from the database, check if it is in the correct status,
        instantiate its ScoreType, compute its score, store it back in
        the database and tell ProxyService to update RWS if needed.

        submission_id (int): the id of the submission that has to be
            scored.
        dataset_id (int): the id of the dataset to use.

        """
        with SessionGen() as session:
            # Obtain submission.
            submission = Submission.get_from_id(submission_id, session)
            if submission is None:
                raise ValueError("Submission %d not found in the database." %
                                 submission_id)

            # Obtain dataset.
            dataset = Dataset.get_from_id(dataset_id, session)
            if dataset is None:
                raise ValueError("Dataset %d not found in the database." %
                                 dataset_id)

            # Obtain submission result.
            submission_result = submission.get_result(dataset)

            # It means it was not even compiled (for some reason).
            if submission_result is None:
                raise ValueError("Submission result %d(%d) was not found." %
                                 (submission_id, dataset_id))

            # Check if it's ready to be scored.
            if not submission_result.needs_scoring():
                if submission_result.scored():
                    logger.info("Submission result %d(%d) is already scored.",
                                submission_id, dataset_id)
                    return
                else:
                    raise ValueError("The state of the submission result "
                                     "%d(%d) doesn't allow scoring." %
                                     (submission_id, dataset_id))

            # Instantiate the score type.
            score_type = get_score_type(dataset=dataset)

            # Compute score and fill it in the database.
            submission_result.score, \
                submission_result.score_details, \
                submission_result.public_score, \
                submission_result.public_score_details, \
                submission_result.ranking_score_details = \
                score_type.compute_score(submission_result)

            # Store it.
            session.commit()

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                self.proxy_service.submission_scored(
                    submission_id=submission.id)
Ejemplo n.º 18
0
Archivo: cps.py Proyecto: ioi-2017/cms
    def get_task(self, get_statement=True):
        """See docstring in class Loader.

        """

        json_src = os.path.join(self.path, 'problem.json')
        if not os.path.exists(json_src):
            logger.error('No task found.')
        with open(json_src) as json_file:
            data = json.load(json_file)

        name = data['code']
        logger.info("Loading parameters for task %s.", name)

        args = {}

        # Here we update the time of the last import.
        touch(os.path.join(self.path, ".itime"))
        # If this file is not deleted, then the import failed.
        touch(os.path.join(self.path, ".import_error"))

        args["name"] = name
        args["title"] = data['name']

        # Statements
        if get_statement:
            statements_dir = os.path.join(self.path, 'statements')
            if os.path.exists(statements_dir):
                statements = [
                    filename for filename in os.listdir(statements_dir)
                    if filename[-4:] == ".pdf"
                ]
                if len(statements) > 0:
                    args['statements'] = dict()
                    logger.info('Statements found')
                for statement in statements:
                    language = statement[:-4]
                    if language == "en_US":
                        args["primary_statements"] = '["en_US"]'
                    digest = self.file_cacher.put_file_from_path(
                        os.path.join(statements_dir, statement),
                        "Statement for task %s (lang: %s)" % (name, language))
                    args['statements'][language] = Statement(language, digest)

        # Attachments
        args["attachments"] = dict()
        attachments_dir = os.path.join(self.path, 'attachments')
        if os.path.exists(attachments_dir):
            logger.info("Attachments found")
            for filename in os.listdir(attachments_dir):
                digest = self.file_cacher.put_file_from_path(
                    os.path.join(attachments_dir, filename),
                    "Attachment %s for task %s" % (filename, name))
                args["attachments"][filename] = Attachment(filename, digest)

        data["task_type"] = data["task_type"][0].upper(
        ) + data["task_type"][1:]

        # Setting the submission format
        # Obtaining testcases' codename
        testcases_dir = os.path.join(self.path, 'tests')
        if not os.path.exists(testcases_dir):
            logger.warning('Testcase folder was not found')
            testcase_codenames = []
        else:
            testcase_codenames = sorted([
                filename[:-3] for filename in os.listdir(testcases_dir)
                if filename[-3:] == '.in'
            ])
        if data["task_type"] == 'OutputOnly':
            args["submission_format"] = list()
            for codename in testcase_codenames:
                args["submission_format"].append(
                    SubmissionFormatElement("%s.out" % codename))
        elif data["task_type"] == 'Notice':
            args["submission_format"] = list()
        else:
            args["submission_format"] = [
                SubmissionFormatElement("%s.%%l" % name)
            ]

        # These options cannot be configured in the CPS format.
        # Uncomment the following to set specific values for them.

        # args['max_submission_number'] = 100
        # args['max_user_test_number'] = 100
        # args['min_submission_interval'] = make_timedelta(60)
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['max_user_test_number'] = 10
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['token_mode'] = 'infinite'
        # args['token_max_number'] = 100
        # args['token_min_interval'] = make_timedelta(60)
        # args['token_gen_initial'] = 1
        # args['token_gen_number'] = 1
        # args['token_gen_interval'] = make_timedelta(1800)
        # args['token_gen_max'] = 2
        if "score_precision" in data:
            args['score_precision'] = int(data["score_precision"])
        else:
            args['score_precision'] = 2
        args['max_submission_number'] = 50
        args['max_user_test_number'] = 50
        if data["task_type"] == 'OutputOnly':
            args['max_submission_number'] = 100
            args['max_user_test_number'] = 100

        args['min_submission_interval'] = make_timedelta(60)
        args['min_user_test_interval'] = make_timedelta(60)

        task = Task(**args)

        args = dict()

        args["task"] = task
        args["description"] = "Default"
        args["autojudge"] = True

        if data['task_type'] != 'OutputOnly' and data['task_type'] != 'Notice':
            args["time_limit"] = float(data['time_limit'])
            args["memory_limit"] = int(data['memory_limit'])

        args["managers"] = {}

        # Checker
        checker_dir = os.path.join(self.path, "checker")
        checker_src = os.path.join(checker_dir, "checker.cpp")

        if os.path.exists(checker_src):
            logger.info("Checker found, compiling")
            checker_exe = os.path.join(checker_dir, "checker")
            os.system("g++ -x c++ -std=gnu++14 -O2 -static -o %s %s" %
                      (checker_exe, checker_src))
            digest = self.file_cacher.put_file_from_path(
                checker_exe, "Manager for task %s" % name)
            args["managers"]['checker'] = Manager("checker", digest)
            evaluation_param = "comparator"
        else:
            logger.info("Checker not found, using diff if neccessary")
            evaluation_param = "diff"

        args["task_type"] = data['task_type']
        if data['task_type'] != 'Notice':
            args["task_type"] += '2017'
        args["task_type_parameters"] = \
            self._get_task_type_parameters(data, data['task_type'], evaluation_param)

        # Graders
        graders_dir = os.path.join(self.path, 'graders')

        if data['task_type'] == 'TwoSteps':
            pas_manager = name + 'lib.pas'
            pas_manager_path = os.path.join(graders_dir, pas_manager)
            if not os.path.exists(pas_manager_path):
                digest = self.file_cacher.put_file_content(
                    ''.encode('utf-8'), 'Pascal manager for task %s' % name)
                args["managers"][pas_manager] = Manager(pas_manager, digest)

        if not os.path.exists(graders_dir):
            logger.warning('Grader folder was not found')
            graders_list = []
        else:
            graders_list = \
                [filename for filename in os.listdir(graders_dir) if filename != 'manager.cpp']
        for grader_name in graders_list:
            grader_src = os.path.join(graders_dir, grader_name)
            digest = self.file_cacher.put_file_from_path(
                grader_src, "Manager for task %s" % name)
            args["managers"][grader_name] = Manager(grader_name, digest)

        # Manager
        manager_src = os.path.join(graders_dir, 'manager.cpp')

        if os.path.exists(manager_src):
            logger.info("Manager found, compiling")
            manager_exe = os.path.join(graders_dir, "manager")
            os.system("cat %s | \
                            g++ -x c++ -O2 -static -o %s -" %
                      (manager_src, manager_exe))
            digest = self.file_cacher.put_file_from_path(
                manager_exe, "Manager for task %s" % name)
            args["managers"]["manager"] = Manager("manager", digest)

        # Testcases
        args["testcases"] = {}

        for codename in testcase_codenames:
            infile = os.path.join(testcases_dir, "%s.in" % codename)
            outfile = os.path.join(testcases_dir, "%s.out" % codename)
            if not os.path.exists(outfile):
                logger.critical(
                    'Could not file the output file for testcase %s' %
                    codename)
                logger.critical('Aborting...')
                return

            input_digest = self.file_cacher.put_file_from_path(
                infile, "Input %s for task %s" % (codename, name))
            output_digest = self.file_cacher.put_file_from_path(
                outfile, "Output %s for task %s" % (codename, name))
            testcase = Testcase(codename, True, input_digest, output_digest)
            args["testcases"][codename] = testcase

        # Score Type
        subtasks_dir = os.path.join(self.path, 'subtasks')
        if not os.path.exists(subtasks_dir):
            logger.warning('Subtask folder was not found')
            subtasks = []
        else:
            subtasks = sorted(os.listdir(subtasks_dir))

        if len(subtasks) == 0:
            number_tests = max(len(testcase_codenames), 1)
            args["score_type"] = "Sum"
            args["score_type_parameters"] = str(100 / number_tests)
        else:
            args["score_type"] = "GroupMinWithMaxScore"
            parsed_data = [
                100,
            ]
            subtask_no = -1
            add_optional_name = False
            for subtask in subtasks:
                subtask_no += 1
                with open(os.path.join(subtasks_dir, subtask)) as subtask_json:
                    subtask_data = json.load(subtask_json)
                    score = int(subtask_data["score"])
                    testcases = "|".join(
                        re.escape(testcase)
                        for testcase in subtask_data["testcases"])
                    optional_name = "Subtask %d" % subtask_no
                    if subtask_no == 0 and score == 0:
                        add_optional_name = True
                        optional_name = "Samples"
                    if add_optional_name:
                        parsed_data.append([score, testcases, optional_name])
                    else:
                        parsed_data.append([score, testcases])
            args["score_type_parameters"] = json.dumps(parsed_data)
        args["description"] = datetime.utcnow()\
            .strftime("%Y-%m-%d %H:%M:%S %Z%z")

        dataset = Dataset(**args)
        task.active_dataset = dataset

        os.remove(os.path.join(self.path, ".import_error"))

        logger.info("Task parameters loaded.")

        return task
Ejemplo n.º 19
0
    def get_task(self, name):
        """See docstring in class Loader.

        """
        try:
            num = self.tasks_order[name]

        # Here we expose an undocumented behavior, so that cmsMake can
        # import a task even without the whole contest; this is not to
        # be relied upon in general.
        except AttributeError:
            num = 1

        task_path = os.path.join(self.path, "problems", name)

        logger.info("Loading parameters for task %s.", name)

        args = {}

        # Here we update the time of the last import.
        touch(os.path.join(task_path, ".itime"))
        # If this file is not deleted, then the import failed.
        touch(os.path.join(task_path, ".import_error"))

        args["num"] = num

        # Get alphabetical task index for use in title.

        index = None
        contest_tree = ET.parse(os.path.join(self.path, "contest.xml"))
        contest_root = contest_tree.getroot()
        for problem in contest_root.find('problems'):
            if os.path.basename(problem.attrib['url']) == name:
                index = problem.attrib['index']

        tree = ET.parse(os.path.join(task_path, "problem.xml"))
        root = tree.getroot()

        args["name"] = name
        if index is not None:
            args["title"] = index.upper() + '. '
        else:
            args["title"] = ''
        args["title"] += root.find('names') \
            .find("name[@language='%s']" % self.primary_language) \
            .attrib['value']

        args["statements"] = []
        args["primary_statements"] = []
        for language in self.languages:
            path = os.path.join(task_path, 'statements', '.pdf', language,
                                'problem.pdf')
            if os.path.exists(path):
                lang = LANGUAGE_MAP[language]
                digest = self.file_cacher.put_file_from_path(
                    path,
                    "Statement for task %s (lang: %s)" % (name, language))
                args["statements"].append(Statement(lang, digest))
                args["primary_statements"].append(lang)
        args["primary_statements"] = '["%s"]' % \
            '","'.join(args["primary_statements"])
        args["submission_format"] = [SubmissionFormatElement("%s.%%l" % name)]

        # These options cannot be configured in the Polygon format.
        # Uncomment the following to set specific values for them.

        # args['max_submission_number'] = 100
        # args['max_user_test_number'] = 100
        # args['min_submission_interval'] = make_timedelta(60)
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['max_user_test_number'] = 10
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['token_mode'] = 'infinite'
        # args['token_max_number'] = 100
        # args['token_min_interval'] = make_timedelta(60)
        # args['token_gen_initial'] = 1
        # args['token_gen_number'] = 1
        # args['token_gen_interval'] = make_timedelta(1800)
        # args['token_gen_max'] = 2

        task_cms_conf_path = os.path.join(task_path, 'files')
        task_cms_conf = None
        if os.path.exists(os.path.join(task_cms_conf_path, 'cms_conf.py')):
            sys.path.append(task_cms_conf_path)
            logger.info("Found additional CMS options for task %s.", name)
            task_cms_conf = __import__('cms_conf')
            # TODO: probably should find more clever way to get rid of caching
            task_cms_conf = reload(task_cms_conf)
            sys.path.pop()
        if task_cms_conf is not None and hasattr(task_cms_conf, "general"):
            args.update(task_cms_conf.general)

        task = Task(**args)

        judging = root.find('judging')
        testset = None
        for testset in judging:
            testset_name = testset.attrib["name"]

            args = {}
            args["task"] = task
            args["description"] = testset_name
            args["autojudge"] = False

            tl = float(testset.find('time-limit').text)
            ml = float(testset.find('memory-limit').text)
            args["time_limit"] = tl * 0.001
            args["memory_limit"] = int(ml / (1024 * 1024))

            args["managers"] = []
            infile_param = judging.attrib['input-file']
            outfile_param = judging.attrib['output-file']

            checker_src = os.path.join(task_path, "files", "check.cpp")
            if os.path.exists(checker_src):
                logger.info("Checker found, compiling")
                checker_exe = os.path.join(task_path, "files", "checker")
                testlib_path = "/usr/local/include/cms/testlib.h"
                if not config.installed:
                    testlib_path = os.path.join(os.path.dirname(__file__),
                                                "polygon", "testlib.h")
                os.system("cat %s | \
                    sed 's$testlib.h$%s$' | \
                    g++ -x c++ -O2 -static -o %s -" %
                          (checker_src, testlib_path, checker_exe))
                digest = self.file_cacher.put_file_from_path(
                    checker_exe, "Manager for task %s" % name)
                args["managers"] += [Manager("checker", digest)]
                evaluation_param = "comparator"
            else:
                logger.info("Checker not found, using diff")
                evaluation_param = "diff"

            args["task_type"] = "Batch"
            args["task_type_parameters"] = \
                '["%s", ["%s", "%s"], "%s"]' % \
                ("alone", infile_param, outfile_param, evaluation_param)

            args["score_type"] = "Sum"
            total_value = 100.0
            input_value = 0.0

            testcases = int(testset.find('test-count').text)

            n_input = testcases
            if n_input != 0:
                input_value = total_value / n_input
            args["score_type_parameters"] = str(input_value)

            args["testcases"] = []

            for i in xrange(testcases):
                infile = os.path.join(task_path, testset_name,
                                      "%02d" % (i + 1))
                outfile = os.path.join(task_path, testset_name,
                                       "%02d.a" % (i + 1))
                if self.dos2unix_found:
                    os.system('dos2unix -q %s' % (infile, ))
                    os.system('dos2unix -q %s' % (outfile, ))
                input_digest = self.file_cacher.put_file_from_path(
                    infile, "Input %d for task %s" % (i, name))
                output_digest = self.file_cacher.put_file_from_path(
                    outfile, "Output %d for task %s" % (i, name))
                testcase = Testcase("%03d" % (i, ), False, input_digest,
                                    output_digest)
                testcase.public = True
                args["testcases"] += [testcase]

            if task_cms_conf is not None and \
               hasattr(task_cms_conf, "datasets") and \
               testset_name in task_cms_conf.datasets:
                args.update(task_cms_conf.datasets[testset_name])

            dataset = Dataset(**args)
            if testset_name == "tests":
                task.active_dataset = dataset

        os.remove(os.path.join(task_path, ".import_error"))

        logger.info("Task parameters loaded.")
        return task
Ejemplo n.º 20
0
    def get_task(self, get_statement=True):
        """See docstring in class TaskLoader."""
        name = os.path.split(self.path)[1]

        if (not os.path.exists(os.path.join(self.path, "task.yaml"))) and \
           (not os.path.exists(os.path.join(self.path, "..", name + ".yaml"))):
            logger.critical("File missing: \"task.yaml\"")
            return None

        # We first look for the yaml file inside the task folder,
        # and eventually fallback to a yaml file in its parent folder.
        try:
            conf = load_yaml_from_path(os.path.join(self.path, "task.yaml"))
        except OSError as err:
            try:
                deprecated_path = os.path.join(self.path, "..", name + ".yaml")
                conf = load_yaml_from_path(deprecated_path)

                logger.warning("You're using a deprecated location for the "
                               "task.yaml file. You're advised to move %s to "
                               "%s.", deprecated_path,
                               os.path.join(self.path, "task.yaml"))
            except OSError:
                # Since both task.yaml and the (deprecated) "../taskname.yaml"
                # are missing, we will only warn the user that task.yaml is
                # missing (to avoid encouraging the use of the deprecated one)
                raise err

        # Here we update the time of the last import
        touch(os.path.join(self.path, ".itime"))
        # If this file is not deleted, then the import failed
        touch(os.path.join(self.path, ".import_error"))

        args = {}

        load(conf, args, ["name", "nome_breve"])
        load(conf, args, ["title", "nome"])

        if name != args["name"]:
            logger.info("The task name (%s) and the directory name (%s) are "
                        "different. The former will be used.", args["name"],
                        name)

        if args["name"] == args["title"]:
            logger.warning("Short name equals long name (title). "
                           "Please check.")

        name = args["name"]

        logger.info("Loading parameters for task %s.", name)

        if get_statement:
            primary_language = load(conf, None, "primary_language")
            if primary_language is None:
                primary_language = 'it'
            paths = [os.path.join(self.path, "statement", "statement.pdf"),
                     os.path.join(self.path, "testo", "testo.pdf")]
            for path in paths:
                if os.path.exists(path):
                    digest = self.file_cacher.put_file_from_path(
                        path,
                        "Statement for task %s (lang: %s)" %
                        (name, primary_language))
                    break
            else:
                logger.critical("Couldn't find any task statement, aborting.")
                sys.exit(1)
            args["statements"] = {
                primary_language: Statement(primary_language, digest)
            }

            args["primary_statements"] = [primary_language]

        args["submission_format"] = ["%s.%%l" % name]

        # Import the feedback level when explicitly set to full
        # (default behaviour is restricted)
        if conf.get("feedback_level", None) == FEEDBACK_LEVEL_FULL:
            args["feedback_level"] = FEEDBACK_LEVEL_FULL
        elif conf.get("feedback_level", None) == FEEDBACK_LEVEL_RESTRICTED:
            args["feedback_level"] = FEEDBACK_LEVEL_RESTRICTED

        if conf.get("score_mode", None) == SCORE_MODE_MAX:
            args["score_mode"] = SCORE_MODE_MAX
        elif conf.get("score_mode", None) == SCORE_MODE_MAX_SUBTASK:
            args["score_mode"] = SCORE_MODE_MAX_SUBTASK
        elif conf.get("score_mode", None) == SCORE_MODE_MAX_TOKENED_LAST:
            args["score_mode"] = SCORE_MODE_MAX_TOKENED_LAST

        # Use the new token settings format if detected.
        if "token_mode" in conf:
            load(conf, args, "token_mode")
            load(conf, args, "token_max_number")
            load(conf, args, "token_min_interval", conv=make_timedelta)
            load(conf, args, "token_gen_initial")
            load(conf, args, "token_gen_number")
            load(conf, args, "token_gen_interval", conv=make_timedelta)
            load(conf, args, "token_gen_max")
        # Otherwise fall back on the old one.
        else:
            logger.warning(
                "task.yaml uses a deprecated format for token settings which "
                "will soon stop being supported, you're advised to update it.")
            # Determine the mode.
            if conf.get("token_initial", None) is None:
                args["token_mode"] = TOKEN_MODE_DISABLED
            elif conf.get("token_gen_number", 0) > 0 and \
                    conf.get("token_gen_time", 0) == 0:
                args["token_mode"] = TOKEN_MODE_INFINITE
            else:
                args["token_mode"] = TOKEN_MODE_FINITE
            # Set the old default values.
            args["token_gen_initial"] = 0
            args["token_gen_number"] = 0
            args["token_gen_interval"] = timedelta()
            # Copy the parameters to their new names.
            load(conf, args, "token_total", "token_max_number")
            load(conf, args, "token_min_interval", conv=make_timedelta)
            load(conf, args, "token_initial", "token_gen_initial")
            load(conf, args, "token_gen_number")
            load(conf, args, "token_gen_time", "token_gen_interval",
                 conv=make_timedelta)
            load(conf, args, "token_max", "token_gen_max")
            # Remove some corner cases.
            if args["token_gen_initial"] is None:
                args["token_gen_initial"] = 0
            if args["token_gen_interval"].total_seconds() == 0:
                args["token_gen_interval"] = timedelta(minutes=1)

        load(conf, args, "max_submission_number")
        load(conf, args, "max_user_test_number")
        load(conf, args, "min_submission_interval", conv=make_timedelta)
        load(conf, args, "min_user_test_interval", conv=make_timedelta)

        # Attachments
        args["attachments"] = dict()
        if os.path.exists(os.path.join(self.path, "att")):
            for filename in os.listdir(os.path.join(self.path, "att")):
                digest = self.file_cacher.put_file_from_path(
                    os.path.join(self.path, "att", filename),
                    "Attachment %s for task %s" % (filename, name))
                args["attachments"][filename] = Attachment(filename, digest)

        task = Task(**args)

        args = {}
        args["task"] = task
        args["description"] = conf.get("version", "Default")
        args["autojudge"] = False

        load(conf, args, ["time_limit", "timeout"], conv=float)
        # The Italian YAML format specifies memory limits in MiB.
        load(conf, args, ["memory_limit", "memlimit"],
             conv=lambda mb: mb * 1024 * 1024)

        # Builds the parameters that depend on the task type
        args["managers"] = []
        infile_param = conf.get("infile", "input.txt")
        outfile_param = conf.get("outfile", "output.txt")

        # If there is sol/grader.%l for some language %l, then,
        # presuming that the task type is Batch, we retrieve graders
        # in the form sol/grader.%l
        graders = False
        for lang in LANGUAGES:
            if os.path.exists(os.path.join(
                    self.path, "sol", "grader%s" % lang.source_extension)):
                graders = True
                break
        if graders:
            # Read grader for each language
            for lang in LANGUAGES:
                extension = lang.source_extension
                grader_filename = os.path.join(
                    self.path, "sol", "grader%s" % extension)
                if os.path.exists(grader_filename):
                    digest = self.file_cacher.put_file_from_path(
                        grader_filename,
                        "Grader for task %s and language %s" %
                        (task.name, lang))
                    args["managers"] += [
                        Manager("grader%s" % extension, digest)]
                else:
                    logger.warning("Grader for language %s not found ", lang)
            # Read managers with other known file extensions
            for other_filename in os.listdir(os.path.join(self.path, "sol")):
                if any(other_filename.endswith(header)
                       for header in HEADER_EXTS):
                    digest = self.file_cacher.put_file_from_path(
                        os.path.join(self.path, "sol", other_filename),
                        "Manager %s for task %s" % (other_filename, task.name))
                    args["managers"] += [
                        Manager(other_filename, digest)]
            compilation_param = "grader"
        else:
            compilation_param = "alone"

        # If there is check/checker (or equivalent), then, presuming
        # that the task type is Batch or OutputOnly, we retrieve the
        # comparator
        paths = [os.path.join(self.path, "check", "checker"),
                 os.path.join(self.path, "cor", "correttore")]
        for path in paths:
            if os.path.exists(path):
                digest = self.file_cacher.put_file_from_path(
                    path,
                    "Manager for task %s" % task.name)
                args["managers"] += [
                    Manager("checker", digest)]
                evaluation_param = "comparator"
                break
        else:
            evaluation_param = "diff"

        # Detect subtasks by checking GEN
        gen_filename = os.path.join(self.path, 'gen', 'GEN')
        try:
            with open(gen_filename, "rt", encoding="utf-8") as gen_file:
                subtasks = []
                testcases = 0
                points = None
                for line in gen_file:
                    line = line.strip()
                    splitted = line.split('#', 1)

                    if len(splitted) == 1:
                        # This line represents a testcase, otherwise
                        # it's just a blank
                        if splitted[0] != '':
                            testcases += 1

                    else:
                        testcase, comment = splitted
                        testcase = testcase.strip()
                        comment = comment.strip()
                        testcase_detected = len(testcase) > 0
                        copy_testcase_detected = comment.startswith("COPY:")
                        subtask_detected = comment.startswith('ST:')

                        flags = [testcase_detected,
                                 copy_testcase_detected,
                                 subtask_detected]
                        if len([x for x in flags if x]) > 1:
                            raise Exception("No testcase and command in"
                                            " the same line allowed")

                        # This line represents a testcase and contains a
                        # comment, but the comment doesn't start a new
                        # subtask
                        if testcase_detected or copy_testcase_detected:
                            testcases += 1

                        # This line starts a new subtask
                        if subtask_detected:
                            # Close the previous subtask
                            if points is None:
                                assert(testcases == 0)
                            else:
                                subtasks.append([points, testcases])
                            # Open the new one
                            testcases = 0
                            points = int(comment[3:].strip())

                # Close last subtask (if no subtasks were defined, just
                # fallback to Sum)
                if points is None:
                    args["score_type"] = "Sum"
                    total_value = float(conf.get("total_value", 100.0))
                    input_value = 0.0
                    n_input = testcases
                    if n_input != 0:
                        input_value = total_value / n_input
                    args["score_type_parameters"] = input_value
                else:
                    subtasks.append([points, testcases])
                    assert(100 == sum([int(st[0]) for st in subtasks]))
                    n_input = sum([int(st[1]) for st in subtasks])
                    args["score_type"] = "GroupMin"
                    args["score_type_parameters"] = subtasks

                if "n_input" in conf:
                    assert int(conf['n_input']) == n_input

        # If gen/GEN doesn't exist, just fallback to Sum
        except OSError:
            args["score_type"] = "Sum"
            total_value = float(conf.get("total_value", 100.0))
            input_value = 0.0
            n_input = int(conf['n_input'])
            if n_input != 0:
                input_value = total_value / n_input
            args["score_type_parameters"] = input_value

        # Override score_type if explicitly specified
        if "score_type" in conf and "score_type_parameters" in conf:
            logger.info("Overriding 'score_type' and 'score_type_parameters' "
                        "as per task.yaml")
            load(conf, args, "score_type")
            load(conf, args, "score_type_parameters")
        elif "score_type" in conf or "score_type_parameters" in conf:
            logger.warning("To override score type data, task.yaml must "
                           "specify both 'score_type' and "
                           "'score_type_parameters'.")

        # If output_only is set, then the task type is OutputOnly
        if conf.get('output_only', False):
            args["task_type"] = "OutputOnly"
            args["time_limit"] = None
            args["memory_limit"] = None
            args["task_type_parameters"] = [evaluation_param]
            task.submission_format = \
                ["output_%03d.txt" % i for i in range(n_input)]

        # If there is check/manager (or equivalent), then the task
        # type is Communication
        else:
            paths = [os.path.join(self.path, "check", "manager"),
                     os.path.join(self.path, "cor", "manager")]
            for path in paths:
                if os.path.exists(path):
                    num_processes = load(conf, None, "num_processes")
                    if num_processes is None:
                        num_processes = 1
                    logger.info("Task type Communication")
                    args["task_type"] = "Communication"
                    args["task_type_parameters"] = \
                        [num_processes, "stub", "fifo_io"]
                    digest = self.file_cacher.put_file_from_path(
                        path,
                        "Manager for task %s" % task.name)
                    args["managers"] += [
                        Manager("manager", digest)]
                    for lang in LANGUAGES:
                        stub_name = os.path.join(
                            self.path, "sol", "stub%s" % lang.source_extension)
                        if os.path.exists(stub_name):
                            digest = self.file_cacher.put_file_from_path(
                                stub_name,
                                "Stub for task %s and language %s" % (
                                    task.name, lang.name))
                            args["managers"] += [
                                Manager(
                                    "stub%s" % lang.source_extension, digest)]
                        else:
                            logger.warning("Stub for language %s not "
                                           "found.", lang.name)
                    for other_filename in os.listdir(os.path.join(self.path,
                                                                  "sol")):
                        if any(other_filename.endswith(header)
                               for header in HEADER_EXTS):
                            digest = self.file_cacher.put_file_from_path(
                                os.path.join(self.path, "sol", other_filename),
                                "Stub %s for task %s" % (other_filename,
                                                         task.name))
                            args["managers"] += [
                                Manager(other_filename, digest)]
                    break

            # Otherwise, the task type is Batch
            else:
                args["task_type"] = "Batch"
                args["task_type_parameters"] = \
                    [compilation_param, [infile_param, outfile_param],
                     evaluation_param]

        args["testcases"] = []
        for i in range(n_input):
            input_digest = self.file_cacher.put_file_from_path(
                os.path.join(self.path, "input", "input%d.txt" % i),
                "Input %d for task %s" % (i, task.name))
            output_digest = self.file_cacher.put_file_from_path(
                os.path.join(self.path, "output", "output%d.txt" % i),
                "Output %d for task %s" % (i, task.name))
            args["testcases"] += [
                Testcase("%03d" % i, False, input_digest, output_digest)]
            if args["task_type"] == "OutputOnly":
                task.attachments.set(
                    Attachment("input_%03d.txt" % i, input_digest))
        public_testcases = load(conf, None, ["public_testcases", "risultati"],
                                conv=lambda x: "" if x is None else x)
        if public_testcases == "all":
            for t in args["testcases"]:
                t.public = True
        elif len(public_testcases) > 0:
            for x in public_testcases.split(","):
                args["testcases"][int(x.strip())].public = True
        args["testcases"] = dict((tc.codename, tc) for tc in args["testcases"])
        args["managers"] = dict((mg.filename, mg) for mg in args["managers"])

        dataset = Dataset(**args)
        task.active_dataset = dataset

        # Import was successful
        os.remove(os.path.join(self.path, ".import_error"))

        logger.info("Task parameters loaded.")

        return task
Ejemplo n.º 21
0
    def get_task(self, get_statement=True):
        """See docstring in class TaskLoader."""
        name = os.path.split(self.path)[1]

        if (not os.path.exists(os.path.join(self.path, "task.yaml"))) and \
           (not os.path.exists(os.path.join(self.path, "problema.yaml"))) and \
           (not os.path.exists(os.path.join(self.path, "..", name + ".yaml"))):
            logger.critical("File missing: \"task.yaml\"")
            return None

        # We first look for the yaml file inside the task folder,
        # and eventually fallback to a yaml file in its parent folder.
        try:
            conf = yaml.safe_load(
                io.open(os.path.join(self.path, "task.yaml"),
                        "rt",
                        encoding="utf-8"))
        except IOError as err:
            try:
                conf = yaml.safe_load(
                    io.open(os.path.join(self.path, "problema.yaml"),
                            "rt",
                            encoding="utf-8"))
            except:
                try:
                    deprecated_path = os.path.join(self.path, "..",
                                                   name + ".yaml")
                    conf = yaml.safe_load(
                        io.open(deprecated_path, "rt", encoding="utf-8"))

                    logger.warning(
                        "You're using a deprecated location for the "
                        "task.yaml file. You're advised to move %s to "
                        "%s.", deprecated_path,
                        os.path.join(self.path, "task.yaml"))
                except IOError:
                    # Since both task.yaml and the (deprecated) "../taskname.yaml"
                    # are missing, we will only warn the user that task.yaml is
                    # missing (to avoid encouraging the use of the deprecated one)
                    raise err

        # Here we update the time of the last import
        touch(os.path.join(self.path, ".itime"))
        # If this file is not deleted, then the import failed
        touch(os.path.join(self.path, ".import_error"))

        args = {}

        load(conf, args, ["name", "nome_breve"])
        load(conf, args, ["title", "nome"])
        load(conf, args, "hide_task_prefix")
        load(conf, args, "category")
        load(conf, args, "level")
        if "level" in args:
            args["level"] = unicode(args["level"])

        if name != args["name"]:
            logger.info(
                "The task name (%s) and the directory name (%s) are "
                "different. The former will be used.", args["name"], name)

        if args["name"] == args["title"]:
            logger.warning("Short name equals long name (title). "
                           "Please check.")

        name = args["name"]

        logger.info("Loading parameters for task %s.", name)

        if get_statement:
            primary_language = load(conf, None, "primary_language")
            if primary_language is None:
                primary_language = 'it'
            paths = [
                os.path.join(self.path, "statement", "statement.pdf"),
                os.path.join(self.path, "statement.pdf"),
                os.path.join(self.path, "enunciado.pdf"),
                os.path.join(self.path, args["name"] + ".pdf"),
                os.path.join(self.path, "testo", "testo.pdf")
            ]
            for path in paths:
                if os.path.exists(path):
                    digest = self.file_cacher.put_file_from_path(
                        path, "Statement for task %s (lang: %s)" %
                        (name, primary_language))
                    break
            else:
                logger.critical("Couldn't find any task statement, aborting.")
                sys.exit(1)
            args["statements"] = [Statement(primary_language, digest)]

            args["primary_statements"] = '["%s"]' % (primary_language)

        args["attachments"] = []  # FIXME Use auxiliary

        args["submission_format"] = [SubmissionFormatElement("%s.%%l" % name)]

        if conf.get("score_mode", None) == SCORE_MODE_MAX:
            args["score_mode"] = SCORE_MODE_MAX
        elif conf.get("score_mode", None) == SCORE_MODE_MAX_TOKENED_LAST:
            args["score_mode"] = SCORE_MODE_MAX_TOKENED_LAST

        # Use the new token settings format if detected.
        if "token_mode" in conf:
            load(conf, args, "token_mode")
            load(conf, args, "token_max_number")
            load(conf, args, "token_min_interval", conv=make_timedelta)
            load(conf, args, "token_gen_initial")
            load(conf, args, "token_gen_number")
            load(conf, args, "token_gen_interval", conv=make_timedelta)
            load(conf, args, "token_gen_max")
        # Otherwise fall back on the old one.
        else:
            logger.warning(
                "task.yaml uses a deprecated format for token settings which "
                "will soon stop being supported, you're advised to update it.")
            # Determine the mode.
            if conf.get("token_initial", None) is None:
                args["token_mode"] = "disabled"
            elif conf.get("token_gen_number", 0) > 0 and \
                    conf.get("token_gen_time", 0) == 0:
                args["token_mode"] = "infinite"
            else:
                args["token_mode"] = "finite"
            # Set the old default values.
            args["token_gen_initial"] = 0
            args["token_gen_number"] = 0
            args["token_gen_interval"] = timedelta()
            # Copy the parameters to their new names.
            load(conf, args, "token_total", "token_max_number")
            load(conf, args, "token_min_interval", conv=make_timedelta)
            load(conf, args, "token_initial", "token_gen_initial")
            load(conf, args, "token_gen_number")
            load(conf,
                 args,
                 "token_gen_time",
                 "token_gen_interval",
                 conv=make_timedelta)
            load(conf, args, "token_max", "token_gen_max")
            # Remove some corner cases.
            if args["token_gen_initial"] is None:
                args["token_gen_initial"] = 0
            if args["token_gen_interval"].total_seconds() == 0:
                args["token_gen_interval"] = timedelta(minutes=1)

        load(conf, args, "max_submission_number")
        load(conf, args, "max_user_test_number")
        load(conf, args, "min_submission_interval", conv=make_timedelta)
        load(conf, args, "min_user_test_interval", conv=make_timedelta)

        # Attachments
        args["attachments"] = []
        if os.path.exists(os.path.join(self.path, "att")):
            for filename in os.listdir(os.path.join(self.path, "att")):
                digest = self.file_cacher.put_file_from_path(
                    os.path.join(self.path, "att", filename),
                    "Attachment %s for task %s" % (filename, name))
                args["attachments"] += [Attachment(filename, digest)]

        task = Task(**args)

        args = {}
        args["task"] = task
        args["description"] = conf.get("version", "Default")
        args["autojudge"] = False

        load(conf, args, ["time_limit", "timeout"], conv=float)
        load(conf, args, ["memory_limit", "memlimit"])

        # Builds the parameters that depend on the task type
        args["managers"] = []
        infile_param = conf.get("infile", "input.txt")
        outfile_param = conf.get("outfile", "output.txt")

        # If there is sol/grader.%l for some language %l, then,
        # presuming that the task type is Batch, we retrieve graders
        # in the form sol/grader.%l
        graders = False
        for lang in LANGUAGES:
            if os.path.exists(
                    os.path.join(self.path, "sol",
                                 "grader%s" % lang.source_extension)):
                graders = True
                break
        if graders:
            # Read grader for each language
            for lang in LANGUAGES:
                extension = lang.source_extension
                grader_filename = os.path.join(self.path, "sol",
                                               "grader%s" % extension)
                if os.path.exists(grader_filename):
                    digest = self.file_cacher.put_file_from_path(
                        grader_filename, "Grader for task %s and language %s" %
                        (task.name, lang))
                    args["managers"] += [
                        Manager("grader%s" % extension, digest)
                    ]
                else:
                    logger.warning("Grader for language %s not found ", lang)
            # Read managers with other known file extensions
            for other_filename in os.listdir(os.path.join(self.path, "sol")):
                if any(
                        other_filename.endswith(header)
                        for header in HEADER_EXTS):
                    digest = self.file_cacher.put_file_from_path(
                        os.path.join(self.path, "sol", other_filename),
                        "Manager %s for task %s" % (other_filename, task.name))
                    args["managers"] += [Manager(other_filename, digest)]
            compilation_param = "grader"
        else:
            compilation_param = "alone"

        # If there is check/checker (or equivalent), then, presuming
        # that the task type is Batch or OutputOnly, we retrieve the
        # comparator
        paths = [
            os.path.join(self.path, "check", "checker"),
            os.path.join(self.path, "corrector.exe"),
            os.path.join(self.path, "cor", "correttore")
        ]
        for path in paths:
            if os.path.exists(path):
                digest = self.file_cacher.put_file_from_path(
                    path, "Manager for task %s" % task.name)
                args["managers"] += [Manager("checker", digest)]
                evaluation_param = "comparator"
                break
        else:
            evaluation_param = "diff"

        # Detect subtasks by checking GEN
        gen_filename = os.path.join(self.path, 'gen', 'GEN')
        try:
            with io.open(gen_filename, "rt", encoding="utf-8") as gen_file:
                subtasks = []
                testcases = 0
                points = None
                for line in gen_file:
                    line = line.strip()
                    splitted = line.split('#', 1)

                    if len(splitted) == 1:
                        # This line represents a testcase, otherwise
                        # it's just a blank
                        if splitted[0] != '':
                            testcases += 1

                    else:
                        testcase, comment = splitted
                        testcase = testcase.strip()
                        comment = comment.strip()
                        testcase_detected = testcase != ''
                        copy_testcase_detected = comment.startswith("COPY:")
                        subtask_detected = comment.startswith('ST:')

                        flags = [
                            testcase_detected, copy_testcase_detected,
                            subtask_detected
                        ]
                        if len([x for x in flags if x]) > 1:
                            raise Exception("No testcase and command in"
                                            " the same line allowed")

                        # This line represents a testcase and contains a
                        # comment, but the comment doesn't start a new
                        # subtask
                        if testcase_detected or copy_testcase_detected:
                            testcases += 1

                        # This line starts a new subtask
                        if subtask_detected:
                            # Close the previous subtask
                            if points is None:
                                assert (testcases == 0)
                            else:
                                subtasks.append([points, testcases])
                            # Open the new one
                            testcases = 0
                            points = int(comment[3:].strip())

                # Close last subtask (if no subtasks were defined, just
                # fallback to Sum)
                if points is None:
                    args["score_type"] = "Sum"
                    total_value = float(conf.get("total_value", 100.0))
                    input_value = 0.0
                    n_input = testcases
                    if n_input != 0:
                        input_value = total_value / n_input
                    args["score_type_parameters"] = "%s" % input_value
                else:
                    subtasks.append([points, testcases])
                    assert (100 == sum([int(st[0]) for st in subtasks]))
                    n_input = sum([int(st[1]) for st in subtasks])
                    args["score_type"] = "GroupMin"
                    args["score_type_parameters"] = "%s" % subtasks

                if "n_input" in conf:
                    assert int(conf['n_input']) == n_input

        # If gen/GEN doesn't exist, just fallback to Sum
        except IOError:
            if 'n_input' not in conf:
                conf['n_input'] = 0
            n_input = int(conf['n_input'])
            if "score_type" in conf:
                args["score_type"] = conf["score_type"]
                if "score_type_parameters" in conf:
                    args["score_type_parameters"] = (
                        "%s" % conf["score_type_parameters"])
                    args["score_type_parameters"] = re.sub(
                        r'u\'([^\']+)\'', '\"\g<1>\"',
                        args["score_type_parameters"])
            else:
                args["score_type"] = "Sum"
                total_value = float(conf.get("total_value", 100.0))
                input_value = 0.0

                def count_testcases(folder):
                    c = 0
                    if os.path.isdir(folder):
                        for filename in sorted(os.listdir(folder)):
                            nombre, ext = os.path.splitext(filename)
                            if ext == ".in":
                                c += 1
                    return c

                casos = n_input + count_testcases(
                    os.path.join(self.path, "casos")) + count_testcases(
                        os.path.join(self.path, "casos", "generados"))
                if casos != 0:
                    input_value = total_value / casos
                args["score_type_parameters"] = "%s" % input_value

        # If output_only is set, then the task type is OutputOnly
        if conf.get('output_only', False):
            args["task_type"] = "OutputOnly"
            args["time_limit"] = None
            args["memory_limit"] = None
            args["task_type_parameters"] = '["%s"]' % evaluation_param
            task.submission_format = [
                SubmissionFormatElement("output_%03d.txt" % i)
                for i in xrange(n_input)
            ]

        # If there is check/manager (or equivalent), then the task
        # type is Communication
        else:
            paths = [
                os.path.join(self.path, "check", "manager"),
                os.path.join(self.path, "cor", "manager")
            ]
            for path in paths:
                if os.path.exists(path):
                    num_processes = load(conf, None, "num_processes")
                    if num_processes is None:
                        num_processes = 1
                    logger.info("Task type Communication")
                    args["task_type"] = "Communication"
                    args["task_type_parameters"] = '[%d]' % num_processes
                    digest = self.file_cacher.put_file_from_path(
                        path, "Manager for task %s" % task.name)
                    args["managers"] += [Manager("manager", digest)]
                    for lang in LANGUAGES:
                        stub_name = os.path.join(
                            self.path, "sol", "stub%s" % lang.source_extension)
                        if os.path.exists(stub_name):
                            digest = self.file_cacher.put_file_from_path(
                                stub_name, "Stub for task %s and language %s" %
                                (task.name, lang.name))
                            args["managers"] += [
                                Manager("stub%s" % lang.source_extension,
                                        digest)
                            ]
                        else:
                            logger.warning(
                                "Stub for language %s not "
                                "found.", lang.name)
                    for other_filename in os.listdir(
                            os.path.join(self.path, "sol")):
                        if any(
                                other_filename.endswith(header)
                                for header in HEADER_EXTS):
                            digest = self.file_cacher.put_file_from_path(
                                os.path.join(self.path, "sol", other_filename),
                                "Stub %s for task %s" %
                                (other_filename, task.name))
                            args["managers"] += [
                                Manager(other_filename, digest)
                            ]
                    break

            # Otherwise, the task type is Batch
            else:
                args["task_type"] = "Batch"
                args["task_type_parameters"] = \
                    '["%s", ["%s", "%s"], "%s"]' % \
                    (compilation_param, infile_param, outfile_param,
                     evaluation_param)

        args["testcases"] = []
        for i in xrange(n_input):
            input_digest = self.file_cacher.put_file_from_path(
                os.path.join(self.path, "input", "input%d.txt" % i),
                "Input %d for task %s" % (i, task.name))
            output_digest = self.file_cacher.put_file_from_path(
                os.path.join(self.path, "output", "output%d.txt" % i),
                "Output %d for task %s" % (i, task.name))
            args["testcases"] += [
                Testcase("%03d" % i, False, input_digest, output_digest)
            ]
            if args["task_type"] == "OutputOnly":
                task.attachments += [
                    Attachment("input_%03d.txt" % i, input_digest)
                ]

        def add_testcases_dir(folder):
            if os.path.isdir(folder):
                for filename in sorted(os.listdir(folder)):
                    nombre, ext = os.path.splitext(filename)
                    if ext == ".in":
                        input_digest = self.file_cacher.put_file_from_path(
                            os.path.join(folder, filename),
                            "Input %s for task %s" % (nombre, task.name))
                        output_digest = self.file_cacher.put_file_from_path(
                            os.path.join(folder, nombre + ".dat"),
                            "Output %s for task %s" % (nombre, task.name))
                        args["testcases"] += [
                            Testcase(nombre, False, input_digest,
                                     output_digest)
                        ]
                        if args["task_type"] == "OutputOnly":
                            task.attachments += [
                                Attachment(filename, input_digest)
                            ]

        add_testcases_dir(os.path.join(self.path, "casos"))
        add_testcases_dir(os.path.join(self.path, "casos", "generados"))

        public_testcases = load(conf,
                                None, ["public_testcases", "risultati"],
                                conv=lambda x: "" if x is None else x)
        if public_testcases == "all":
            for t in args["testcases"]:
                t.public = True
        elif public_testcases != "":
            for x in public_testcases.split(","):
                args["testcases"][int(x.strip())].public = True

        dataset = Dataset(**args)
        task.active_dataset = dataset

        # Import was successful
        os.remove(os.path.join(self.path, ".import_error"))

        logger.info("Task parameters loaded.")

        return task
Ejemplo n.º 22
0
    def get_task(self, name):
        """See docstring in class Loader.

        """
        try:
            num = self.tasks_order[name]

        # Here we expose an undocumented behavior, so that cmsMake can
        # import a task even without the whole contest; this is not to
        # be relied upon in general
        except AttributeError:
            num = 1

        task_path = os.path.join(self.path, name)

        # We first look for the yaml file inside the task folder,
        # and eventually fallback to a yaml file in its parent folder.
        try:
            conf = yaml.safe_load(
                io.open(os.path.join(task_path, "task.yaml"),
                        "rt", encoding="utf-8"))
        except IOError:
            conf = yaml.safe_load(
                io.open(os.path.join(self.path, name + ".yaml"),
                        "rt", encoding="utf-8"))

        logger.info("Loading parameters for task %s." % name)

        # Here we update the time of the last import
        touch(os.path.join(task_path, ".itime"))
        # If this file is not deleted, then the import failed
        touch(os.path.join(task_path, ".import_error"))

        args = {}

        args["num"] = num
        load(conf, args, ["name", "nome_breve"])
        load(conf, args, ["title", "nome"])

        assert name == args["name"]

        if args["name"] == args["title"]:
            logger.warning("Short name equals long name (title). "
                           "Please check.")

        primary_language = load(conf, None, "primary_language")
        if primary_language is None:
            primary_language = 'it'
        paths = [os.path.join(task_path, "statement", "statement.pdf"),
                 os.path.join(task_path, "testo", "testo.pdf")]
        for path in paths:
            if os.path.exists(path):
                digest = self.file_cacher.put_file_from_path(
                    path,
                    "Statement for task %s (lang: %s)" % (name,
                                                          primary_language))
                break
        else:
            logger.critical("Couldn't find any task statement, aborting...")
            sys.exit(1)
        args["statements"] = [Statement(primary_language, digest)]

        args["primary_statements"] = '["%s"]' % (primary_language)

        args["attachments"] = []  # FIXME Use auxiliary

        args["submission_format"] = [
            SubmissionFormatElement("%s.%%l" % name)]

        # Use the new token settings format if detected.
        if "token_mode" in conf:
            load(conf, args, "token_mode")
            load(conf, args, "token_max_number")
            load(conf, args, "token_min_interval", conv=make_timedelta)
            load(conf, args, "token_gen_initial")
            load(conf, args, "token_gen_number")
            load(conf, args, "token_gen_interval", conv=make_timedelta)
            load(conf, args, "token_gen_max")
        # Otherwise fall back on the old one.
        else:
            logger.warning(
                "%s.yaml uses a deprecated format for token settings which "
                "will soon stop being supported, you're advised to update it.",
                name)
            # Determine the mode.
            if conf.get("token_initial", None) is None:
                args["token_mode"] = "disabled"
            elif conf.get("token_gen_number", 0) > 0 and \
                    conf.get("token_gen_time", 0) == 0:
                args["token_mode"] = "infinite"
            else:
                args["token_mode"] = "finite"
            # Set the old default values.
            args["token_gen_initial"] = 0
            args["token_gen_number"] = 0
            args["token_gen_interval"] = timedelta()
            # Copy the parameters to their new names.
            load(conf, args, "token_total", "token_max_number")
            load(conf, args, "token_min_interval", conv=make_timedelta)
            load(conf, args, "token_initial", "token_gen_initial")
            load(conf, args, "token_gen_number")
            load(conf, args, "token_gen_time", "token_gen_interval",
                 conv=make_timedelta)
            load(conf, args, "token_max", "token_gen_max")
            # Remove some corner cases.
            if args["token_gen_initial"] is None:
                args["token_gen_initial"] = 0
            if args["token_gen_interval"].total_seconds() == 0:
                args["token_gen_interval"] = timedelta(minutes=1)

        load(conf, args, "max_submission_number")
        load(conf, args, "max_user_test_number")
        load(conf, args, "min_submission_interval", conv=make_timedelta)
        load(conf, args, "min_user_test_interval", conv=make_timedelta)

        # Attachments
        args["attachments"] = []
        if os.path.exists(os.path.join(task_path, "att")):
            for filename in os.listdir(os.path.join(task_path, "att")):
                digest = self.file_cacher.put_file_from_path(
                    os.path.join(task_path, "att", filename),
                    "Attachment %s for task %s" % (filename, name))
                args["attachments"] += [Attachment(filename, digest)]

        task = Task(**args)

        args = {}
        args["task"] = task
        args["description"] = conf.get("version", "Default")
        args["autojudge"] = False

        load(conf, args, ["time_limit", "timeout"], conv=float)
        load(conf, args, ["memory_limit", "memlimit"])

        # Builds the parameters that depend on the task type
        args["managers"] = []
        infile_param = conf.get("infile", "input.txt")
        outfile_param = conf.get("outfile", "output.txt")

        # If there is sol/grader.%l for some language %l, then,
        # presuming that the task type is Batch, we retrieve graders
        # in the form sol/grader.%l
        graders = False
        for lang in LANGUAGES:
            if os.path.exists(os.path.join(
                    task_path, "sol", "grader.%s" % lang)):
                graders = True
                break
        if graders:
            # Read grader for each language
            for lang in LANGUAGES:
                grader_filename = os.path.join(
                    task_path, "sol", "grader.%s" % lang)
                if os.path.exists(grader_filename):
                    digest = self.file_cacher.put_file_from_path(
                        grader_filename,
                        "Grader for task %s and language %s" % (name, lang))
                    args["managers"] += [
                        Manager("grader.%s" % lang, digest)]
                else:
                    logger.warning("Grader for language %s not found " % lang)
            # Read managers with other known file extensions
            for other_filename in os.listdir(os.path.join(task_path, "sol")):
                if other_filename.endswith('.h') or \
                        other_filename.endswith('lib.pas'):
                    digest = self.file_cacher.put_file_from_path(
                        os.path.join(task_path, "sol", other_filename),
                        "Manager %s for task %s" % (other_filename, name))
                    args["managers"] += [
                        Manager(other_filename, digest)]
            compilation_param = "grader"
        else:
            compilation_param = "alone"

        # If there is check/checker (or equivalent), then, presuming
        # that the task type is Batch or OutputOnly, we retrieve the
        # comparator
        paths = [os.path.join(task_path, "check", "checker"),
                 os.path.join(task_path, "cor", "correttore")]
        for path in paths:
            if os.path.exists(path):
                digest = self.file_cacher.put_file_from_path(
                    path,
                    "Manager for task %s" % name)
                args["managers"] += [
                    Manager("checker", digest)]
                evaluation_param = "comparator"
                break
        else:
            evaluation_param = "diff"

        # Detect subtasks by checking GEN
        gen_filename = os.path.join(task_path, 'gen', 'GEN')
        try:
            with io.open(gen_filename, "rt", encoding="utf-8") as gen_file:
                subtasks = []
                testcases = 0
                points = None
                for line in gen_file:
                    line = line.strip()
                    splitted = line.split('#', 1)

                    if len(splitted) == 1:
                        # This line represents a testcase, otherwise it's
                        # just a blank
                        if splitted[0] != '':
                            testcases += 1

                    else:
                        testcase, comment = splitted
                        testcase_detected = False
                        subtask_detected = False
                        if testcase.strip() != '':
                            testcase_detected = True
                        comment = comment.strip()
                        if comment.startswith('ST:'):
                            subtask_detected = True

                        if testcase_detected and subtask_detected:
                            raise Exception("No testcase and subtask in the"
                                            " same line allowed")

                        # This line represents a testcase and contains a
                        # comment, but the comment doesn't start a new
                        # subtask
                        if testcase_detected:
                            testcases += 1

                        # This line starts a new subtask
                        if subtask_detected:
                            # Close the previous subtask
                            if points is None:
                                assert(testcases == 0)
                            else:
                                subtasks.append([points, testcases])
                            # Open the new one
                            testcases = 0
                            points = int(comment[3:].strip())

                # Close last subtask (if no subtasks were defined, just
                # fallback to Sum)
                if points is None:
                    args["score_type"] = "Sum"
                    total_value = float(conf.get("total_value", 100.0))
                    input_value = 0.0
                    n_input = testcases
                    if n_input != 0:
                        input_value = total_value / n_input
                    args["score_type_parameters"] = "%s" % input_value
                else:
                    subtasks.append([points, testcases])
                    assert(100 == sum([int(st[0]) for st in subtasks]))
                    n_input = sum([int(st[1]) for st in subtasks])
                    args["score_type"] = "GroupMin"
                    args["score_type_parameters"] = "%s" % subtasks

                if "n_input" in conf:
                    assert int(conf['n_input']) == n_input

        # If gen/GEN doesn't exist, just fallback to Sum
        except IOError:
            args["score_type"] = "Sum"
            total_value = float(conf.get("total_value", 100.0))
            input_value = 0.0
            n_input = int(conf['n_input'])
            if n_input != 0:
                input_value = total_value / n_input
            args["score_type_parameters"] = "%s" % input_value

        # If output_only is set, then the task type is OutputOnly
        if conf.get('output_only', False):
            args["task_type"] = "OutputOnly"
            args["time_limit"] = None
            args["memory_limit"] = None
            args["task_type_parameters"] = '["%s"]' % evaluation_param
            task.submission_format = [
                SubmissionFormatElement("output_%03d.txt" % i)
                for i in xrange(n_input)]

        # If there is check/manager (or equivalent), then the task
        # type is Communication
        else:
            paths = [os.path.join(task_path, "check", "manager"),
                     os.path.join(task_path, "cor", "manager")]
            for path in paths:
                if os.path.exists(path):
                    args["task_type"] = "Communication"
                    args["task_type_parameters"] = '[]'
                    digest = self.file_cacher.put_file_from_path(
                        path,
                        "Manager for task %s" % name)
                    args["managers"] += [
                        Manager("manager", digest)]
                    for lang in LANGUAGES:
                        stub_name = os.path.join(
                            task_path, "sol", "stub.%s" % lang)
                        if os.path.exists(stub_name):
                            digest = self.file_cacher.put_file_from_path(
                                stub_name,
                                "Stub for task %s and language %s" % (name,
                                                                      lang))
                            args["managers"] += [
                                Manager("stub.%s" % lang, digest)]
                        else:
                            logger.warning("Stub for language %s not "
                                           "found." % lang)
                    break

            # Otherwise, the task type is Batch
            else:
                args["task_type"] = "Batch"
                args["task_type_parameters"] = \
                    '["%s", ["%s", "%s"], "%s"]' % \
                    (compilation_param, infile_param, outfile_param,
                     evaluation_param)

        args["testcases"] = []
        for i in xrange(n_input):
            input_digest = self.file_cacher.put_file_from_path(
                os.path.join(task_path, "input", "input%d.txt" % i),
                "Input %d for task %s" % (i, name))
            output_digest = self.file_cacher.put_file_from_path(
                os.path.join(task_path, "output", "output%d.txt" % i),
                "Output %d for task %s" % (i, name))
            args["testcases"] += [
                Testcase("%03d" % i, False, input_digest, output_digest)]
            if args["task_type"] == "OutputOnly":
                task.attachments += [
                    Attachment("input_%03d.txt" % i, input_digest)]
        public_testcases = load(conf, None, ["public_testcases", "risultati"],
                                conv=lambda x: "" if x is None else x)
        if public_testcases != "":
            for x in public_testcases.split(","):
                args["testcases"][int(x.strip())].public = True

        dataset = Dataset(**args)
        task.active_dataset = dataset

        # Import was successful
        os.remove(os.path.join(task_path, ".import_error"))

        logger.info("Task parameters loaded.")

        return task
Ejemplo n.º 23
0
    def execute(self, entry):
        """Assign a score to a submission result.

        This is the core of ScoringService: here we retrieve the result
        from the database, check if it is in the correct status,
        instantiate its ScoreType, compute its score, store it back in
        the database and tell ProxyService to update RWS if needed.

        entry (QueueEntry): entry containing the operation to perform.

        """
        operation = entry.item
        with SessionGen() as session:
            # Obtain submission.
            submission = Submission.get_from_id(operation.submission_id,
                                                session)
            if submission is None:
                raise ValueError("Submission %d not found in the database." %
                                 operation.submission_id)

            # Obtain dataset.
            dataset = Dataset.get_from_id(operation.dataset_id, session)
            if dataset is None:
                raise ValueError("Dataset %d not found in the database." %
                                 operation.dataset_id)

            # Obtain submission result.
            submission_result = submission.get_result(dataset)

            # It means it was not even compiled (for some reason).
            if submission_result is None:
                raise ValueError("Submission result %d(%d) was not found." %
                                 (operation.submission_id,
                                  operation.dataset_id))

            # Check if it's ready to be scored.
            if not submission_result.needs_scoring():
                if submission_result.scored():
                    logger.info("Submission result %d(%d) is already scored.",
                                operation.submission_id, operation.dataset_id)
                    return
                else:
                    raise ValueError("The state of the submission result "
                                     "%d(%d) doesn't allow scoring." %
                                     (operation.submission_id,
                                      operation.dataset_id))

            # For Codebreaker, your score depends on your previous submissions
            # to this task. So, let's get the previous submisisons for this task
            previous_submissions = session.query(Submission)\
                .filter(Submission.user_id == submission.user_id,
                        Submission.task_id == submission.task_id)\
                .order_by(asc(Submission.timestamp))\
                .all()
            # Counterintuitively, because we're nice people, we don't care how
            # these submissions were scored. We only care about their
            # evaluations, which will tell us how to score them.
            # For a codebreaker, this will be in one-to-one correspondence with
            # previous submissions, since each "task" should only have the one
            # "testcase".
            previous_evaluations = [
                session.query(Evaluation)
                .filter(Evaluation.submission_id == sub.id).first()
                for sub in previous_submissions]

            assert(len(previous_evaluations) == len(previous_submissions))

            # Now that we have the evaluations, we can pass these as parameters
            # to our score type
            params = [evaluation.outcome for evaluation in previous_evaluations]

            # Instantiate the score type.
            # We don't want to use the dataset since we have to pass in custom
            # params. Instead we'll just hardcode the name of the class in,
            # which is unfortunate.
            # TODO (bgbn): work out a way to make this more generic.
            score_type = get_score_type(name="AIOCCodebreakerScoreType",
                                        parameters=json.dumps(params),
                                        public_testcases=dict((k, tc.public)
                                            for k, tc in
                                            dataset.testcases.iteritems()))

            # Compute score and fill it in the database.
            submission_result.score, \
                submission_result.score_details, \
                submission_result.public_score, \
                submission_result.public_score_details, \
                submission_result.ranking_score_details = \
                score_type.compute_score(submission_result)

            # Store it.
            session.commit()

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                self.proxy_service.submission_scored(
                    submission_id=submission.id)
Ejemplo n.º 24
0
    def get(self, submission_id):
        """Retrieve a single submission.

        Query the database for the submission with the given ID, and
        the dataset given as query parameter (or the active one).

        submission_id (int): the ID of a submission.

        """
        # If it's not an integer we will ignore it. But if it's an
        # integer of a dataset that doesn't exist we'll raise a 404.
        dataset_id = local.request.args.get("dataset_id", type=int)

        with SessionGen() as local.session:
            # Load the submission, and check for existence.
            submission = Submission.get_from_id(submission_id, local.session)

            if submission is None:
                raise NotFound()

            # Load the dataset.
            if dataset_id is not None:
                dataset = Dataset.get_from_id(dataset_id, local.session)
                if dataset is None:
                    raise NotFound()
            else:
                q = local.session.query(Dataset)
                q = q.join(Task, Dataset.id == Task.active_dataset_id)
                q = q.filter(Task.id == submission.task_id)
                dataset = q.one()

            # Get the result (will fire a query).
            submission_result = submission.get_result(dataset)

            # Get the ScoreType (will fire a query for testcases).
            score_type = get_score_type(dataset=dataset)

            # Produce the data structure.
            s = submission
            sr = submission_result

            result = {
                '_ref': "%s" % s.id,
                'dataset': '%s' % dataset.id,
                'user': "******" % s.user_id,
                'task': "%s" % s.task_id,
                'timestamp': make_timestamp(s.timestamp),
                'language': s.language,
                # No files, no token: AWS doesn't need them.
            }

            if sr is not None:
                result.update({
                    'compilation_outcome':
                        {"ok": True,
                         "fail": False}.get(sr.compilation_outcome),
                    'compilation_text':
                        format_status_text(sr.compilation_text),
                    'compilation_tries': sr.compilation_tries,
                    'compilation_stdout': sr.compilation_stdout,
                    'compilation_stderr': sr.compilation_stderr,
                    'compilation_time': sr.compilation_time,
                    'compilation_wall_clock_time':
                        sr.compilation_wall_clock_time,
                    'compilation_memory': sr.compilation_memory,
                    'compilation_shard': sr.compilation_shard,
                    'compilation_sandbox': sr.compilation_sandbox,
                    'evaluation_outcome':
                        {"ok": True}.get(sr.evaluation_outcome),
                    'evaluation_tries': sr.evaluation_tries,
                    'evaluations': dict((ev.codename, {
                        'codename': ev.codename,
                        'outcome': ev.outcome,
                        'text': format_status_text(ev.text),
                        'execution_time': ev.execution_time,
                        'execution_wall_clock_time':
                            ev.execution_wall_clock_time,
                        'execution_memory': ev.execution_memory,
                        'evaluation_shard': ev.evaluation_shard,
                        'evaluation_sandbox': ev.evaluation_sandbox,
                    }) for ev in sr.evaluations),
                    'score': sr.score,
                    'max_score': score_type.max_score,
                    'score_details':
                        score_type.get_html_details(sr.score_details)
                        if sr.score is not None else None,
                })
            else:
                # Just copy all fields with None.
                result.update({
                    'compilation_outcome': None,
                    'compilation_text': None,
                    'compilation_tries': 0,
                    'compilation_stdout': None,
                    'compilation_stderr': None,
                    'compilation_time': None,
                    'compilation_wall_clock_time': None,
                    'compilation_memory': None,
                    'compilation_shard': None,
                    'compilation_sandbox': None,
                    'evaluation_outcome': None,
                    'evaluation_tries': 0,
                    'evaluations': {},
                    'score': None,
                    'max_score': score_type.max_score,
                    'score_details': None,
                })

        # Encode and send.
        local.response.mimetype = "application/json"
        local.response.data = json.dumps(result)
Ejemplo n.º 25
0
    def execute(self, entry):
        """Assign a score to a submission result.

        This is the core of ScoringService: here we retrieve the result
        from the database, check if it is in the correct status,
        instantiate its ScoreType, compute its score, store it back in
        the database and tell ProxyService to update RWS if needed.

        entry (QueueEntry): entry containing the operation to perform.

        """
        operation = entry.item
        with SessionGen() as session:
            # Obtain submission.
            submission = Submission.get_from_id(operation.submission_id,
                                                session)
            if submission is None:
                raise ValueError("Submission %d not found in the database." %
                                 operation.submission_id)

            # Obtain dataset.
            dataset = Dataset.get_from_id(operation.dataset_id, session)
            if dataset is None:
                raise ValueError("Dataset %d not found in the database." %
                                 operation.dataset_id)

            # Obtain submission result.
            submission_result = submission.get_result(dataset)

            # It means it was not even compiled (for some reason).
            if submission_result is None:
                raise ValueError("Submission result %d(%d) was not found." %
                                 (operation.submission_id,
                                  operation.dataset_id))

            # Check if it's ready to be scored.
            if not submission_result.needs_scoring():
                if submission_result.scored():
                    logger.info("Submission result %d(%d) is already scored.",
                                operation.submission_id, operation.dataset_id)
                    return
                else:
                    raise ValueError("The state of the submission result "
                                     "%d(%d) doesn't allow scoring." %
                                     (operation.submission_id,
                                      operation.dataset_id))

            # Instantiate the score type.
            score_type = get_score_type(dataset=dataset)

            # Compute score and fill it in the database.
            submission_result.score, \
                submission_result.score_details, \
                submission_result.public_score, \
                submission_result.public_score_details, \
                submission_result.ranking_score_details = \
                score_type.compute_score(submission_result)

            # Round submission score to 2 decimal places
            submission_result.score = round(submission_result.score, 2)

            # Store it.
            session.commit()

            # Update statistics and access level
            score = submission_result.score
            taskscore = session.query(TaskScore)\
                .filter(TaskScore.user_id == submission.user_id)\
                .filter(TaskScore.task_id == submission.task_id).first()
            if taskscore is None:
                taskscore = TaskScore()
                taskscore.task_id = submission.task_id
                taskscore.user_id = submission.user_id
                session.add(taskscore)
            mtime = max([0] + [e.execution_time
                               for e in submission_result.evaluations])
            if score > taskscore.score:
                taskscore.score = score
                taskscore.time = mtime
            elif score == taskscore.score and mtime < taskscore.time:
                taskscore.time = mtime
            submission.task.nsubscorrect = session.query(Submission)\
                .filter(Submission.task_id == submission.task_id)\
                .filter(Submission.results.any(
                    SubmissionResult.score == 100)).count()
            submission.task.nuserscorrect = session.query(TaskScore)\
                .filter(TaskScore.task_id == submission.task_id)\
                .filter(TaskScore.score == 100).count()
            submission.user.score = sum([
                t.score for t in session.query(TaskScore)
                .filter(TaskScore.user_id == submission.user_id).all()])
            if submission.user.score >= 300 and \
               submission.user.access_level == 6:
                submission.user.access_level = 5
            submission.task.nsubs = session.query(Submission)\
                .filter(Submission.task_id == submission.task_id).count()
            submission.task.nusers = session.query(TaskScore)\
                .filter(TaskScore.task_id == submission.task_id).count()
            session.commit()

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                self.proxy_service.submission_scored(
                    submission_id=submission.id)
Ejemplo n.º 26
0
    def invalidate_submission(self,
                              contest_id=None,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              level="compilation"):
        """Request to invalidate some computed data.

        Invalidate the compilation and/or evaluation data of the
        SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of the contest asked for, or, if all three are
          None, the contest this service is running for (or all contests).
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        The data is cleared, the operations involving the submissions
        currently enqueued are deleted, and the ones already assigned to
        the workers are ignored. New appropriate operations are
        enqueued.

        submission_id (int|None): id of the submission to invalidate,
            or None.
        dataset_id (int|None): id of the dataset to invalidate, or
            None.
        participation_id (int|None): id of the participation to
            invalidate, or None.
        task_id (int|None): id of the task to invalidate, or None.
        level (string): 'compilation' or 'evaluation'

        """
        logger.info("Invalidation request received.")

        # Validate arguments
        # TODO Check that all these objects belong to this contest.
        if level not in ("compilation", "evaluation"):
            raise ValueError("Unexpected invalidation level `%s'." % level)

        if contest_id is None:
            contest_id = self.contest_id

        with SessionGen() as session:
            # When invalidating a dataset we need to know the task_id, otherwise
            # get_submissions will return all the submissions of the contest.
            if dataset_id is not None and task_id is None \
                    and submission_id is None:
                task_id = Dataset.get_from_id(dataset_id, session).task_id
            # First we load all involved submissions.
            submissions = get_submissions(
                # Give contest_id only if all others are None.
                contest_id if {participation_id, task_id, submission_id}
                == {None} else None,
                participation_id,
                task_id,
                submission_id,
                session)

            # Then we get all relevant operations, and we remove them
            # both from the queue and from the pool (i.e., we ignore
            # the workers involved in those operations).
            operations = get_relevant_operations(level, submissions,
                                                 dataset_id)
            for operation in operations:
                try:
                    self.dequeue(operation)
                except KeyError:
                    pass  # Ok, the operation wasn't in the queue.
                try:
                    self.get_executor().pool.ignore_operation(operation)
                except LookupError:
                    pass  # Ok, the operation wasn't in the pool.

            # Then we find all existing results in the database, and
            # we remove them.
            submission_results = get_submission_results(
                # Give contest_id only if all others are None.
                contest_id if {
                    participation_id, task_id, submission_id, dataset_id
                } == {None} else None,
                participation_id,
                # Provide the task_id only if the entire task has to be
                # reevaluated and not only a specific dataset.
                task_id if dataset_id is None else None,
                submission_id,
                dataset_id,
                session)
            logger.info("Submission results to invalidate %s for: %d.", level,
                        len(submission_results))
            for submission_result in submission_results:
                # We invalidate the appropriate data and queue the
                # operations to recompute those data.
                if level == "compilation":
                    submission_result.invalidate_compilation()
                elif level == "evaluation":
                    submission_result.invalidate_evaluation()

            # Finally, we re-enqueue the operations for the
            # submissions.
            for submission in submissions:
                self.submission_enqueue_operations(submission)

            session.commit()
        logger.info("Invalidate successfully completed.")
Ejemplo n.º 27
0
    def get_task(self, get_statement=True):
        """See docstring in class Loader.

        """

        logger.info("Checking dos2unix presence")
        i = os.system('dos2unix -V 2>/dev/null')
        self.dos2unix_found = (i == 0)
        if not self.dos2unix_found:
            logger.error("dos2unix not found - tests will not be converted!")

        name = os.path.basename(self.path)
        logger.info("Loading parameters for task %s.", name)

        args = {}

        # Here we update the time of the last import.
        touch(os.path.join(self.path, ".itime"))
        # If this file is not deleted, then the import failed.
        touch(os.path.join(self.path, ".import_error"))

        # Get alphabetical task index for use in title.

        tree = ET.parse(os.path.join(self.path, "problem.xml"))
        root = tree.getroot()

        args["name"] = name
        args["title"] = str(root.find('names').find("name").attrib['value'])

        if get_statement:
            args["statements"] = {}
            args["primary_statements"] = []
            for language, lang in iteritems(LANGUAGE_MAP):
                path = os.path.join(self.path, 'statements', '.pdf', language,
                                    'problem.pdf')
                if os.path.exists(path):
                    digest = self.file_cacher.put_file_from_path(
                        path,
                        "Statement for task %s (lang: %s)" % (name, language))
                    args["statements"][lang] = Statement(lang, digest)
                    args["primary_statements"].append(lang)

        args["submission_format"] = ["%s.%%l" % name]

        # These options cannot be configured in the Polygon format.
        # Uncomment the following to set specific values for them.

        # args['max_submission_number'] = 100
        # args['max_user_test_number'] = 100
        # args['min_submission_interval'] = make_timedelta(60)
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['max_user_test_number'] = 10
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['token_mode'] = 'infinite'
        # args['token_max_number'] = 100
        # args['token_min_interval'] = make_timedelta(60)
        # args['token_gen_initial'] = 1
        # args['token_gen_number'] = 1
        # args['token_gen_interval'] = make_timedelta(1800)
        # args['token_gen_max'] = 2

        task_cms_conf_path = os.path.join(self.path, 'files', 'cms_conf.py')
        task_cms_conf = None
        if os.path.exists(task_cms_conf_path):
            logger.info("Found additional CMS options for task %s.", name)
            with io.open(task_cms_conf_path, 'rb') as f:
                task_cms_conf = imp.load_module('cms_conf', f,
                                                task_cms_conf_path,
                                                ('.py', 'r', imp.PY_SOURCE))
        if task_cms_conf is not None and hasattr(task_cms_conf, "general"):
            args.update(task_cms_conf.general)

        task = Task(**args)

        judging = root.find('judging')
        testset = None
        for testset in judging:
            testset_name = testset.attrib["name"]

            args = {}
            args["task"] = task
            args["description"] = str(testset_name)
            args["autojudge"] = False

            tl = float(testset.find('time-limit').text)
            ml = int(testset.find('memory-limit').text)
            args["time_limit"] = tl * 0.001
            args["memory_limit"] = ml // (1024 * 1024)

            args["managers"] = {}
            infile_param = judging.attrib['input-file']
            outfile_param = judging.attrib['output-file']

            # Checker can be in any of these two locations.
            checker_src = os.path.join(self.path, "files", "check.cpp")
            if not os.path.exists(checker_src):
                checker_src = os.path.join(self.path, "check.cpp")

            if os.path.exists(checker_src):
                logger.info("Checker found, compiling")
                checker_exe = os.path.join(os.path.dirname(checker_src),
                                           "checker")
                testlib_path = "/usr/local/include/cms"
                testlib_include = os.path.join(testlib_path, "testlib.h")
                if not config.installed:
                    testlib_path = os.path.join(os.path.dirname(__file__),
                                                "polygon")
                code = subprocess.call([
                    "g++", "-x", "c++", "-O2", "-static", "-DCMS", "-I",
                    testlib_path, "-include", testlib_include, "-o",
                    checker_exe, checker_src
                ])
                if code != 0:
                    logger.critical("Could not compile checker")
                    return None
                digest = self.file_cacher.put_file_from_path(
                    checker_exe, "Manager for task %s" % name)
                args["managers"]["checker"] = Manager("checker", digest)
                evaluation_param = "comparator"
            else:
                logger.info("Checker not found, using diff")
                evaluation_param = "diff"

            args["task_type"] = "Batch"
            args["task_type_parameters"] = \
                ["alone", [infile_param, outfile_param], evaluation_param]

            args["score_type"] = "Sum"
            total_value = 100.0
            input_value = 0.0

            testcases = int(testset.find('test-count').text)

            n_input = testcases
            if n_input != 0:
                input_value = total_value / n_input
            args["score_type_parameters"] = input_value

            args["testcases"] = {}

            for i in range(testcases):
                infile = os.path.join(self.path, testset_name,
                                      "%02d" % (i + 1))
                outfile = os.path.join(self.path, testset_name,
                                       "%02d.a" % (i + 1))
                if self.dos2unix_found:
                    os.system('dos2unix -q %s' % (infile, ))
                    os.system('dos2unix -q %s' % (outfile, ))
                input_digest = self.file_cacher.put_file_from_path(
                    infile, "Input %d for task %s" % (i, name))
                output_digest = self.file_cacher.put_file_from_path(
                    outfile, "Output %d for task %s" % (i, name))
                testcase = Testcase("%03d" % (i, ), False, input_digest,
                                    output_digest)
                testcase.public = True
                args["testcases"][testcase.codename] = testcase

            if task_cms_conf is not None and \
               hasattr(task_cms_conf, "datasets") and \
               testset_name in task_cms_conf.datasets:
                args.update(task_cms_conf.datasets[testset_name])

            dataset = Dataset(**args)
            if testset_name == "tests":
                task.active_dataset = dataset

        os.remove(os.path.join(self.path, ".import_error"))

        logger.info("Task parameters loaded.")
        return task
Ejemplo n.º 28
0
    def action_finished(self, data, plus, error=None):
        """Callback from a worker, to signal that is finished some
        action (compilation or evaluation).

        data (dict): a dictionary that describes a Job instance.
        plus (tuple): the tuple (type_,
                                 object_id,
                                 dataset_id,
                                 testcase_codename,
                                 side_data=(priority, timestamp),
                                 shard_of_worker)

        """
        # Unpack the plus tuple. It's built in the RPC call to Worker's
        # execute_job method inside WorkerPool.acquire_worker.
        type_, object_id, dataset_id, testcase_codename, _, \
            shard = plus

        # Restore operation from its fields.
        operation = ESOperation(type_, object_id, dataset_id,
                                testcase_codename)

        # We notify the pool that the worker is available again for
        # further work (no matter how the current request turned out,
        # even if the worker encountered an error). If the pool
        # informs us that the data produced by the worker has to be
        # ignored (by returning True) we interrupt the execution of
        # this method and do nothing because in that case we know the
        # operation has returned to the queue and perhaps already been
        # reassigned to another worker.
        if self.get_executor().pool.release_worker(shard):
            logger.info("Ignored result from worker %s as requested.", shard)
            return

        job_success = True
        if error is not None:
            logger.error("Received error from Worker: `%s'.", error)
            job_success = False

        else:
            try:
                job = Job.import_from_dict_with_type(data)
            except:
                logger.error("Couldn't build Job for data %s.",
                             data,
                             exc_info=True)
                job_success = False

            else:
                if not job.success:
                    logger.error("Worker %s signaled action not successful.",
                                 shard)
                    job_success = False

        logger.info("`%s' completed. Success: %s.", operation, job_success)

        # We get the submission from DB and update it.
        with SessionGen() as session:
            dataset = Dataset.get_from_id(dataset_id, session)
            if dataset is None:
                logger.error("Could not find dataset %d in the database.",
                             dataset_id)
                return

            # TODO Try to move this 4-cases if-clause into a method of
            # ESOperation: I'd really like ES itself not to care about
            # which type of operation it's handling.
            if type_ == ESOperation.COMPILATION:
                submission = Submission.get_from_id(object_id, session)
                if submission is None:
                    logger.error(
                        "Could not find submission %d "
                        "in the database.", object_id)
                    return

                submission_result = submission.get_result(dataset)
                if submission_result is None:
                    logger.info(
                        "Couldn't find submission %d(%d) "
                        "in the database. Creating it.", object_id, dataset_id)
                    submission_result = \
                        submission.get_result_or_create(dataset)

                if job_success:
                    job.to_submission(submission_result)
                else:
                    submission_result.compilation_tries += 1

                session.commit()

                self.compilation_ended(submission_result)

            elif type_ == ESOperation.EVALUATION:
                submission = Submission.get_from_id(object_id, session)
                if submission is None:
                    logger.error(
                        "Could not find submission %d "
                        "in the database.", object_id)
                    return

                submission_result = submission.get_result(dataset)
                if submission_result is None:
                    logger.error(
                        "Couldn't find submission %d(%d) "
                        "in the database.", object_id, dataset_id)
                    return

                if job_success:
                    job.to_submission(submission_result)
                else:
                    submission_result.evaluation_tries += 1

                # Submission evaluation will be ended only when
                # evaluation for each testcase is available.
                evaluation_complete = (len(
                    submission_result.evaluations) == len(dataset.testcases))
                if evaluation_complete:
                    submission_result.set_evaluation_outcome()

                session.commit()

                if evaluation_complete:
                    self.evaluation_ended(submission_result)

            elif type_ == ESOperation.USER_TEST_COMPILATION:
                user_test = UserTest.get_from_id(object_id, session)
                if user_test is None:
                    logger.error(
                        "Could not find user test %d "
                        "in the database.", object_id)
                    return

                user_test_result = user_test.get_result(dataset)
                if user_test_result is None:
                    logger.error(
                        "Couldn't find user test %d(%d) "
                        "in the database. Creating it.", object_id, dataset_id)
                    user_test_result = \
                        user_test.get_result_or_create(dataset)

                if job_success:
                    job.to_user_test(user_test_result)
                else:
                    user_test_result.compilation_tries += 1

                session.commit()

                self.user_test_compilation_ended(user_test_result)

            elif type_ == ESOperation.USER_TEST_EVALUATION:
                user_test = UserTest.get_from_id(object_id, session)
                if user_test is None:
                    logger.error(
                        "Could not find user test %d "
                        "in the database.", object_id)
                    return

                user_test_result = user_test.get_result(dataset)
                if user_test_result is None:
                    logger.error(
                        "Couldn't find user test %d(%d) "
                        "in the database.", object_id, dataset_id)
                    return

                if job_success:
                    job.to_user_test(user_test_result)
                else:
                    user_test_result.evaluation_tries += 1

                session.commit()

                self.user_test_evaluation_ended(user_test_result)

            else:
                logger.error("Invalid operation type %r.", type_)
                return
Ejemplo n.º 29
0
    def post(self):
        fallback_page = "/tasks/add"

        try:
            attrs = dict()

            self.get_string(attrs, "name", empty=None)
            self.get_string(attrs, "title")

            assert attrs.get("name") is not None, "No task name specified."

            self.get_string(attrs, "primary_statements")

            self.get_submission_format(attrs)

            self.get_string(attrs, "token_mode")
            self.get_int(attrs, "token_max_number")
            self.get_timedelta_sec(attrs, "token_min_interval")
            self.get_int(attrs, "token_gen_initial")
            self.get_int(attrs, "token_gen_number")
            self.get_timedelta_min(attrs, "token_gen_interval")
            self.get_int(attrs, "token_gen_max")

            self.get_int(attrs, "max_submission_number")
            self.get_int(attrs, "max_user_test_number")
            self.get_timedelta_sec(attrs, "min_submission_interval")
            self.get_timedelta_sec(attrs, "min_user_test_interval")

            self.get_int(attrs, "score_precision")

            self.get_string(attrs, "score_mode")

            # Create the task.
            task = Task(**attrs)
            self.sql_session.add(task)

        except Exception as error:
            self.application.service.add_notification(make_datetime(),
                                                      "Invalid field(s)",
                                                      repr(error))
            self.redirect(fallback_page)
            return

        try:
            attrs = dict()

            self.get_time_limit(attrs, "time_limit")
            self.get_memory_limit(attrs, "memory_limit")
            self.get_task_type(attrs, "task_type", "TaskTypeOptions_")
            self.get_score_type(attrs, "score_type", "score_type_parameters")

            # Create its first dataset.
            attrs["description"] = "Default"
            attrs["autojudge"] = True
            attrs["task"] = task
            dataset = Dataset(**attrs)
            self.sql_session.add(dataset)

            # Make the dataset active. Life works better that way.
            task.active_dataset = dataset

        except Exception as error:
            self.application.service.add_notification(make_datetime(),
                                                      "Invalid field(s)",
                                                      repr(error))
            self.redirect(fallback_page)
            return

        if self.try_commit():
            # Create the task on RWS.
            self.application.service.proxy_service.reinitialize()
            self.redirect("/task/%s" % task.id)
        else:
            self.redirect(fallback_page)
Ejemplo n.º 30
0
    def new_evaluation(self, submission_id, dataset_id):
        """This RPC inform ScoringService that ES finished the work on
        a submission (either because it has been evaluated, or because
        the compilation failed).

        submission_id (int): the id of the submission that changed.
        dataset_id (int): the id of the dataset to use.

        """
        with SessionGen(commit=True) as session:
            submission = Submission.get_from_id(submission_id, session)

            if submission is None:
                logger.error("[new_evaluation] Couldn't find submission %d "
                             "in the database." % submission_id)
                raise ValueError

            if submission.user.hidden:
                logger.info("[new_evaluation] Submission %d not scored "
                            "because user is hidden." % submission_id)
                return

            dataset = Dataset.get_from_id(dataset_id, session)

            if dataset is None:
                logger.error("[new_evaluation] Couldn't find dataset %d "
                             "in the database." % dataset_id)
                raise ValueError

            submission_result = submission.get_result(dataset)

            # We'll accept only submissions that either didn't compile
            # at all or that did evaluate successfully.
            if submission_result is None or not submission_result.compiled():
                logger.warning("[new_evaluation] Submission %d(%d) is "
                               "not compiled." % (submission_id, dataset_id))
                return
            elif submission_result.compilation_outcome == "ok" and \
                    not submission_result.evaluated():
                logger.warning("[new_evaluation] Submission %d(%d) is "
                               "compiled but is not evaluated." %
                               (submission_id, dataset_id))
                return

            # Assign score to the submission.
            score_type = get_score_type(dataset=dataset)
            score, details, public_score, public_details, ranking_details = \
                score_type.compute_score(submission_result)

            # Mark submission as scored.
            self.submission_results_scored.add((submission_id, dataset_id))

            # Filling submission's score info in the db.
            submission_result.score = score
            submission_result.public_score = public_score

            # And details.
            submission_result.score_details = details
            submission_result.public_score_details = public_details
            submission_result.ranking_score_details = ranking_details

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                self.rankings_send_score(submission)
Ejemplo n.º 31
0
    def execute(self, entry):
        """Assign a score to a submission result.

        This is the core of ScoringService: here we retrieve the result
        from the database, check if it is in the correct status,
        instantiate its ScoreType, compute its score, store it back in
        the database and tell ProxyService to update RWS if needed.

        entry (QueueEntry): entry containing the operation to perform.

        """
        operation = entry.item
        with SessionGen() as session:
            # Obtain submission.
            submission = Submission.get_from_id(operation.submission_id,
                                                session)
            if submission is None:
                raise ValueError("Submission %d not found in the database." %
                                 operation.submission_id)

            # Obtain dataset.
            dataset = Dataset.get_from_id(operation.dataset_id, session)
            if dataset is None:
                raise ValueError("Dataset %d not found in the database." %
                                 operation.dataset_id)

            # Obtain submission result.
            submission_result = submission.get_result(dataset)

            # It means it was not even compiled (for some reason).
            if submission_result is None:
                raise ValueError(
                    "Submission result %d(%d) was not found." %
                    (operation.submission_id, operation.dataset_id))

            # Check if it's ready to be scored.
            if not submission_result.needs_scoring():
                if submission_result.scored():
                    logger.info("Submission result %d(%d) is already scored.",
                                operation.submission_id, operation.dataset_id)
                    return
                else:
                    raise ValueError(
                        "The state of the submission result "
                        "%d(%d) doesn't allow scoring." %
                        (operation.submission_id, operation.dataset_id))

            # For Codebreaker, your score depends on your previous submissions
            # to this task. So, let's get the previous submisisons for this task
            previous_submissions = session.query(Submission)\
                .filter(Submission.user_id == submission.user_id,
                        Submission.task_id == submission.task_id)\
                .order_by(asc(Submission.timestamp))\
                .all()
            # Counterintuitively, because we're nice people, we don't care how
            # these submissions were scored. We only care about their
            # evaluations, which will tell us how to score them.
            # For a codebreaker, this will be in one-to-one correspondence with
            # previous submissions, since each "task" should only have the one
            # "testcase".
            previous_evaluations = [
                session.query(Evaluation).filter(
                    Evaluation.submission_id == sub.id).first()
                for sub in previous_submissions
            ]

            assert (len(previous_evaluations) == len(previous_submissions))

            # Now that we have the evaluations, we can pass these as parameters
            # to our score type
            params = [
                evaluation.outcome for evaluation in previous_evaluations
            ]

            # Instantiate the score type.
            # We don't want to use the dataset since we have to pass in custom
            # params. Instead we'll just hardcode the name of the class in,
            # which is unfortunate.
            # TODO (bgbn): work out a way to make this more generic.
            score_type = get_score_type(
                name="AIOCCodebreakerScoreType",
                parameters=json.dumps(params),
                public_testcases=dict(
                    (k, tc.public) for k, tc in dataset.testcases.iteritems()))

            # Compute score and fill it in the database.
            submission_result.score, \
                submission_result.score_details, \
                submission_result.public_score, \
                submission_result.public_score_details, \
                submission_result.ranking_score_details = \
                score_type.compute_score(submission_result)

            # Store it.
            session.commit()

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                self.proxy_service.submission_scored(
                    submission_id=submission.id)
Ejemplo n.º 32
0
    def invalidate_submission(self,
                              contest_id=None,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              level="compilation"):
        """Request to invalidate some computed data.

        Invalidate the compilation and/or evaluation data of the
        SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of the contest asked for, or, if all three are
          None, the contest this service is running for (or all contests).
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        The data is cleared, the operations involving the submissions
        currently enqueued are deleted, and the ones already assigned to
        the workers are ignored. New appropriate operations are
        enqueued.

        submission_id (int|None): id of the submission to invalidate,
            or None.
        dataset_id (int|None): id of the dataset to invalidate, or
            None.
        participation_id (int|None): id of the participation to
            invalidate, or None.
        task_id (int|None): id of the task to invalidate, or None.
        level (string): 'compilation' or 'evaluation'

        """
        logger.info("Invalidation request received.")

        # Validate arguments
        # TODO Check that all these objects belong to this contest.
        if level not in ("compilation", "evaluation"):
            raise ValueError(
                "Unexpected invalidation level `%s'." % level)

        if contest_id is None:
            contest_id = self.contest_id

        with SessionGen() as session:
            # When invalidating a dataset we need to know the task_id,
            # otherwise get_submissions will return all the submissions of
            # the contest.
            if dataset_id is not None and task_id is None \
                    and submission_id is None:
                task_id = Dataset.get_from_id(dataset_id, session).task_id
            # First we load all involved submissions.
            submissions = get_submissions(
                session,
                # Give contest_id only if all others are None.
                contest_id
                if {participation_id, task_id, submission_id} == {None}
                else None,
                participation_id, task_id, submission_id).all()

            # Then we get all relevant operations, and we remove them
            # both from the queue and from the pool (i.e., we ignore
            # the workers involved in those operations).
            operations = get_relevant_operations(
                level, submissions, dataset_id)
            for operation in operations:
                try:
                    self.dequeue(operation)
                except KeyError:
                    pass  # Ok, the operation wasn't in the queue.
                try:
                    self.get_executor().pool.ignore_operation(operation)
                except LookupError:
                    pass  # Ok, the operation wasn't in the pool.

            # Then we find all existing results in the database, and
            # we remove them.
            submission_results = get_submission_results(
                session,
                # Give contest_id only if all others are None.
                contest_id
                if {participation_id,
                    task_id,
                    submission_id,
                    dataset_id} == {None}
                else None,
                participation_id,
                # Provide the task_id only if the entire task has to be
                # reevaluated and not only a specific dataset.
                task_id if dataset_id is None else None,
                submission_id, dataset_id).all()
            logger.info("Submission results to invalidate %s for: %d.",
                        level, len(submission_results))
            for submission_result in submission_results:
                # We invalidate the appropriate data and queue the
                # operations to recompute those data.
                if level == "compilation":
                    submission_result.invalidate_compilation()
                elif level == "evaluation":
                    submission_result.invalidate_evaluation()

            # Finally, we re-enqueue the operations for the
            # submissions.
            for submission in submissions:
                self.submission_enqueue_operations(submission)

            session.commit()
        logger.info("Invalidate successfully completed.")
Ejemplo n.º 33
0
    def get_task(self, get_statement=True):
        """See docstring in class Loader.

        """

        json_src = os.path.join(self.path, 'problem.json')
        if not os.path.exists(json_src):
            logger.critical('No task found.')
            raise OSError('No task found at path %s' % json_src)
        with open(json_src, 'rt', encoding='utf-8') as json_file:
            data = json.load(json_file)

        name = data['code']
        logger.info("Loading parameters for task %s.", name)

        args = {}

        args["name"] = name
        args["title"] = data['name']

        # Statements
        if get_statement:
            statements_dir = os.path.join(self.path, 'statements')
            if os.path.exists(statements_dir):
                statements = [
                    filename
                    for filename in os.listdir(statements_dir)
                    if filename[-4:] == ".pdf"]
                if len(statements) > 0:
                    args['statements'] = dict()
                    logger.info('Statements found')
                for statement in statements:
                    language = statement[:-4]
                    if language == "en_US":
                        args["primary_statements"] = ["en_US"]
                    digest = self.file_cacher.put_file_from_path(
                        os.path.join(statements_dir, statement),
                        "Statement for task %s (lang: %s)" %
                        (name, language))
                    args['statements'][language] = Statement(language, digest)

        # Attachments
        args["attachments"] = dict()
        attachments_dir = os.path.join(self.path, 'attachments')
        if os.path.exists(attachments_dir):
            logger.info("Attachments found")
            for filename in os.listdir(attachments_dir):
                digest = self.file_cacher.put_file_from_path(
                    os.path.join(attachments_dir, filename),
                    "Attachment %s for task %s" % (filename, name))
                args["attachments"][filename] = Attachment(filename, digest)

        data["task_type"] = \
            data["task_type"][0].upper() + data["task_type"][1:]

        # Setting the submission format
        # Obtaining testcases' codename
        testcases_dir = os.path.join(self.path, 'tests')
        if not os.path.exists(testcases_dir):
            logger.warning('Testcase folder was not found')
            testcase_codenames = []
        else:
            testcase_codenames = sorted([
                filename[:-3]
                for filename in os.listdir(testcases_dir)
                if filename[-3:] == '.in'])
        if data["task_type"] == 'OutputOnly':
            args["submission_format"] = list()
            for codename in testcase_codenames:
                args["submission_format"].append("%s.out" % codename)
        elif data["task_type"] == 'Notice':
            args["submission_format"] = list()
        else:
            args["submission_format"] = ["%s.%%l" % name]

        # These options cannot be configured in the TPS format.
        # Uncomment the following to set specific values for them.

        # args['max_user_test_number'] = 10
        # args['min_user_test_interval'] = make_timedelta(60)

        # args['token_mode'] = 'infinite'
        # args['token_max_number'] = 100
        # args['token_min_interval'] = make_timedelta(60)
        # args['token_gen_initial'] = 1
        # args['token_gen_number'] = 1
        # args['token_gen_interval'] = make_timedelta(1800)
        # args['token_gen_max'] = 2

        if "score_precision" in data:
            args['score_precision'] = int(data["score_precision"])
        else:
            args['score_precision'] = 2
        args['max_submission_number'] = 50
        args['max_user_test_number'] = 50
        if data["task_type"] == 'OutputOnly':
            args['max_submission_number'] = 100
            args['max_user_test_number'] = 100

        args['min_submission_interval'] = make_timedelta(60)
        args['min_user_test_interval'] = make_timedelta(60)

        task = Task(**args)

        args = dict()

        args["task"] = task
        args["description"] = "Default"
        args["autojudge"] = True

        if data['task_type'] != 'OutputOnly' \
                and data['task_type'] != 'Notice':
            args["time_limit"] = float(data['time_limit'])
            args["memory_limit"] = int(data['memory_limit'])

        args["managers"] = {}

        # Checker
        checker_dir = os.path.join(self.path, "checker")
        checker_src = os.path.join(checker_dir, "checker.cpp")

        if os.path.exists(checker_src):
            logger.info("Checker found, compiling")
            checker_exe = os.path.join(checker_dir, "checker")
            subprocess.call([
                "g++", "-x", "c++", "-std=gnu++14", "-O2", "-static",
                "-o", checker_exe, checker_src
            ])
            digest = self.file_cacher.put_file_from_path(
                checker_exe,
                "Manager for task %s" % name)
            args["managers"]['checker'] = Manager("checker", digest)
            evaluation_param = "comparator"
        else:
            logger.info("Checker not found, using diff if necessary")
            evaluation_param = "diff"

        # Note that the original TPS worked with custom task type Batch2017
        # and Communication2017 instead of Batch and Communication.
        args["task_type"] = data['task_type']
        args["task_type_parameters"] = \
            self._get_task_type_parameters(
                data, data['task_type'], evaluation_param)

        # Graders
        graders_dir = os.path.join(self.path, 'graders')

        if data['task_type'] == 'TwoSteps':
            pas_manager = name + 'lib.pas'
            pas_manager_path = os.path.join(graders_dir, pas_manager)
            if not os.path.exists(pas_manager_path):
                digest = self.file_cacher.put_file_content(
                    ''.encode('utf-8'), 'Pascal manager for task %s' % name)
                args["managers"][pas_manager] = Manager(pas_manager, digest)

        if not os.path.exists(graders_dir):
            logger.warning('Grader folder was not found')
            graders_list = []
        else:
            graders_list = \
                [filename
                 for filename in os.listdir(graders_dir)
                 if filename != 'manager.cpp']
        for grader_name in graders_list:
            grader_src = os.path.join(graders_dir, grader_name)
            digest = self.file_cacher.put_file_from_path(
                grader_src,
                "Manager for task %s" % name)
            if data['task_type'] == 'Communication' \
                    and os.path.splitext(grader_name)[0] == 'grader':
                grader_name = 'stub' + os.path.splitext(grader_name)[1]
            args["managers"][grader_name] = Manager(grader_name, digest)

        # Manager
        manager_src = os.path.join(graders_dir, 'manager.cpp')

        if os.path.exists(manager_src):
            logger.info("Manager found, compiling")
            manager_exe = os.path.join(graders_dir, "manager")
            subprocess.call([
                "g++", "-x", "c++", "-O2", "-static",
                "-o", manager_exe, manager_src
            ])
            digest = self.file_cacher.put_file_from_path(
                manager_exe,
                "Manager for task %s" % name)
            args["managers"]["manager"] = Manager("manager", digest)

        # Testcases
        args["testcases"] = {}

        for codename in testcase_codenames:
            infile = os.path.join(testcases_dir, "%s.in" % codename)
            outfile = os.path.join(testcases_dir, "%s.out" % codename)
            if not os.path.exists(outfile):
                logger.critical(
                    'Could not find the output file for testcase %s', codename)
                logger.critical('Aborting...')
                return

            input_digest = self.file_cacher.put_file_from_path(
                infile,
                "Input %s for task %s" % (codename, name))
            output_digest = self.file_cacher.put_file_from_path(
                outfile,
                "Output %s for task %s" % (codename, name))
            testcase = Testcase(codename, True,
                                input_digest, output_digest)
            args["testcases"][codename] = testcase

        # Score Type
        subtasks_dir = os.path.join(self.path, 'subtasks')
        if not os.path.exists(subtasks_dir):
            logger.warning('Subtask folder was not found')
            subtasks = []
        else:
            subtasks = sorted(os.listdir(subtasks_dir))

        if len(subtasks) == 0:
            number_tests = max(len(testcase_codenames), 1)
            args["score_type"] = "Sum"
            args["score_type_parameters"] = 100 / number_tests
        else:
            args["score_type"] = "GroupMin"
            parsed_data = []
            subtask_no = -1
            add_optional_name = False
            for subtask in subtasks:
                subtask_no += 1
                with open(os.path.join(subtasks_dir, subtask), 'rt',
                          encoding='utf-8') as subtask_json:
                    subtask_data = json.load(subtask_json)
                    score = int(subtask_data["score"])
                    testcases = "|".join(
                        re.escape(testcase)
                        for testcase in subtask_data["testcases"]
                    )
                    optional_name = "Subtask %d" % subtask_no
                    if subtask_no == 0 and score == 0:
                        add_optional_name = True
                        optional_name = "Samples"
                    if add_optional_name:
                        parsed_data.append([score, testcases, optional_name])
                    else:
                        parsed_data.append([score, testcases])
            args["score_type_parameters"] = parsed_data

        dataset = Dataset(**args)
        task.active_dataset = dataset

        logger.info("Task parameters loaded.")

        return task
Ejemplo n.º 34
0
            args["score_type"] = "GroupMin"
            parsed_data = []
            subtask_no = -1
            add_optional_name = False
            for subtask in subtasks:
                subtask_no += 1
                with io.open(os.path.join(subtasks_dir, subtask), 'rt',
                             encoding='utf-8') as subtask_json:
                    subtask_data = json.load(subtask_json)
                    score = int(subtask_data["score"])
                    testcases = "|".join(
                        re.escape(testcase)
                        for testcase in subtask_data["testcases"]
                    )
                    optional_name = "Subtask %d" % subtask_no
                    if subtask_no == 0 and score == 0:
                        add_optional_name = True
                        optional_name = "Samples"
                    if add_optional_name:
                        parsed_data.append([score, testcases, optional_name])
                    else:
                        parsed_data.append([score, testcases])
            args["score_type_parameters"] = parsed_data

        dataset = Dataset(**args)
        task.active_dataset = dataset

        logger.info("Task parameters loaded.")

        return task
Ejemplo n.º 35
0
    def write_results(self, items):
        """Receive worker results from the cache and writes them to the DB.

        Grouping results together by object (i.e., submission result
        or user test result) and type (compilation or evaluation)
        allows this method to talk less to the DB, for example by
        retrieving datasets and submission results only once instead
        of once for every result.

        items ([(operation, Result)]): the results received by ES but
            not yet written to the db.

        """
        logger.info("Starting commit process...")

        # Reorganize the results by submission/usertest result and
        # operation type (i.e., group together the testcase
        # evaluations for the same submission and dataset).
        by_object_and_type = defaultdict(list)
        for operation, result in items:
            t = (operation.type_, operation.object_id, operation.dataset_id)
            by_object_and_type[t].append((operation, result))

        with SessionGen() as session:
            for key, operation_results in by_object_and_type.items():
                type_, object_id, dataset_id = key

                dataset = Dataset.get_from_id(dataset_id, session)
                if dataset is None:
                    logger.error("Could not find dataset %d in the database.",
                                 dataset_id)
                    continue

                # Get submission or user test results.
                if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]:
                    object_ = Submission.get_from_id(object_id, session)
                    if object_ is None:
                        logger.error(
                            "Could not find submission %d "
                            "in the database.", object_id)
                        continue
                    object_result = object_.get_result_or_create(dataset)
                else:
                    object_ = UserTest.get_from_id(object_id, session)
                    if object_ is None:
                        logger.error(
                            "Could not find user test %d "
                            "in the database.", object_id)
                        continue
                    object_result = object_.get_result_or_create(dataset)

                self.write_results_one_object_and_type(session, object_result,
                                                       operation_results)

            logger.info("Committing evaluations...")
            session.commit()

            num_testcases_per_dataset = dict()
            for type_, object_id, dataset_id in by_object_and_type.keys():
                if type_ == ESOperation.EVALUATION:
                    if dataset_id not in num_testcases_per_dataset:
                        num_testcases_per_dataset[dataset_id] = session\
                            .query(func.count(Testcase.id))\
                            .filter(Testcase.dataset_id == dataset_id).scalar()
                    num_evaluations = session\
                        .query(func.count(Evaluation.id)) \
                        .filter(Evaluation.dataset_id == dataset_id) \
                        .filter(Evaluation.submission_id == object_id).scalar()
                    if num_evaluations == num_testcases_per_dataset[
                            dataset_id]:
                        submission_result = SubmissionResult.get_from_id(
                            (object_id, dataset_id), session)
                        submission_result.set_evaluation_outcome()

            logger.info("Committing evaluation outcomes...")
            session.commit()

            logger.info("Ending operations for %s objects...",
                        len(by_object_and_type))
            for type_, object_id, dataset_id in by_object_and_type.keys():
                if type_ == ESOperation.COMPILATION:
                    submission_result = SubmissionResult.get_from_id(
                        (object_id, dataset_id), session)
                    self.compilation_ended(submission_result)
                elif type_ == ESOperation.EVALUATION:
                    submission_result = SubmissionResult.get_from_id(
                        (object_id, dataset_id), session)
                    if submission_result.evaluated():
                        self.evaluation_ended(submission_result)
                elif type_ == ESOperation.USER_TEST_COMPILATION:
                    user_test_result = UserTestResult.get_from_id(
                        (object_id, dataset_id), session)
                    self.user_test_compilation_ended(user_test_result)
                elif type_ == ESOperation.USER_TEST_EVALUATION:
                    user_test_result = UserTestResult.get_from_id(
                        (object_id, dataset_id), session)
                    self.user_test_evaluation_ended(user_test_result)

        logger.info("Done")
Ejemplo n.º 36
0
    def action_finished(self, data, plus, error=None):
        """Callback from a worker, to signal that is finished some
        action (compilation or evaluation).

        data (dict): a dictionary that describes a Job instance.
        plus (tuple): the tuple (type_,
                                 object_id,
                                 dataset_id,
                                 testcase_codename,
                                 side_data=(priority, timestamp),
                                 shard_of_worker)

        """
        # Unpack the plus tuple. It's built in the RPC call to Worker's
        # execute_job method inside WorkerPool.acquire_worker.
        type_, object_id, dataset_id, testcase_codename, _, \
            shard = plus

        # Restore operation from its fields.
        operation = ESOperation(
            type_, object_id, dataset_id, testcase_codename)

        # We notify the pool that the worker is available again for
        # further work (no matter how the current request turned out,
        # even if the worker encountered an error). If the pool
        # informs us that the data produced by the worker has to be
        # ignored (by returning True) we interrupt the execution of
        # this method and do nothing because in that case we know the
        # operation has returned to the queue and perhaps already been
        # reassigned to another worker.
        if self.get_executor().pool.release_worker(shard):
            logger.info("Ignored result from worker %s as requested.", shard)
            return

        job_success = True
        if error is not None:
            logger.error("Received error from Worker: `%s'.", error)
            job_success = False

        else:
            try:
                job = Job.import_from_dict_with_type(data)
            except:
                logger.error("[action_finished] Couldn't build Job for "
                             "data %s.", data, exc_info=True)
                job_success = False

            else:
                if not job.success:
                    logger.error("Worker %s signaled action "
                                 "not successful.", shard)
                    job_success = False

        logger.info("Operation `%s' for submission %s completed. Success: %s.",
                    operation, object_id, job_success)

        # We get the submission from DB and update it.
        with SessionGen() as session:
            dataset = Dataset.get_from_id(dataset_id, session)
            if dataset is None:
                logger.error("[action_finished] Could not find "
                             "dataset %d in the database.",
                             dataset_id)
                return

            # TODO Try to move this 4-cases if-clause into a method of
            # ESOperation: I'd really like ES itself not to care about
            # which type of operation it's handling.
            if type_ == ESOperation.COMPILATION:
                submission = Submission.get_from_id(object_id, session)
                if submission is None:
                    logger.error("[action_finished] Could not find "
                                 "submission %d in the database.",
                                 object_id)
                    return

                submission_result = submission.get_result(dataset)
                if submission_result is None:
                    logger.info("[action_finished] Couldn't find "
                                "submission %d(%d) in the database. "
                                "Creating it.", object_id, dataset_id)
                    submission_result = \
                        submission.get_result_or_create(dataset)

                if job_success:
                    job.to_submission(submission_result)
                else:
                    submission_result.compilation_tries += 1

                session.commit()

                self.compilation_ended(submission_result)

            elif type_ == ESOperation.EVALUATION:
                submission = Submission.get_from_id(object_id, session)
                if submission is None:
                    logger.error("[action_finished] Could not find "
                                 "submission %d in the database.",
                                 object_id)
                    return

                submission_result = submission.get_result(dataset)
                if submission_result is None:
                    logger.error("[action_finished] Couldn't find "
                                 "submission %d(%d) in the database.",
                                 object_id, dataset_id)
                    return

                if job_success:
                    job.to_submission(submission_result)
                else:
                    submission_result.evaluation_tries += 1

                # Submission evaluation will be ended only when
                # evaluation for each testcase is available.
                evaluation_complete = (len(submission_result.evaluations) ==
                                       len(dataset.testcases))
                if evaluation_complete:
                    submission_result.set_evaluation_outcome()

                session.commit()

                if evaluation_complete:
                    self.evaluation_ended(submission_result)

            elif type_ == ESOperation.USER_TEST_COMPILATION:
                user_test = UserTest.get_from_id(object_id, session)
                if user_test is None:
                    logger.error("[action_finished] Could not find "
                                 "user test %d in the database.",
                                 object_id)
                    return

                user_test_result = user_test.get_result(dataset)
                if user_test_result is None:
                    logger.error("[action_finished] Couldn't find "
                                 "user test %d(%d) in the database. "
                                 "Creating it.", object_id, dataset_id)
                    user_test_result = \
                        user_test.get_result_or_create(dataset)

                if job_success:
                    job.to_user_test(user_test_result)
                else:
                    user_test_result.compilation_tries += 1

                session.commit()

                self.user_test_compilation_ended(user_test_result)

            elif type_ == ESOperation.USER_TEST_EVALUATION:
                user_test = UserTest.get_from_id(object_id, session)
                if user_test is None:
                    logger.error("[action_finished] Could not find "
                                 "user test %d in the database.",
                                 object_id)
                    return

                user_test_result = user_test.get_result(dataset)
                if user_test_result is None:
                    logger.error("[action_finished] Couldn't find "
                                 "user test %d(%d) in the database.",
                                 object_id, dataset_id)
                    return

                if job_success:
                    job.to_user_test(user_test_result)
                else:
                    user_test_result.evaluation_tries += 1

                session.commit()

                self.user_test_evaluation_ended(user_test_result)

            else:
                logger.error("Invalid operation type %r.", type_)
                return
Ejemplo n.º 37
0
    def execute(self, entry):
        """Assign a score to a submission result.

        This is the core of ScoringService: here we retrieve the result
        from the database, check if it is in the correct status,
        instantiate its ScoreType, compute its score, store it back in
        the database and tell ProxyService to update RWS if needed.

        entry (QueueEntry): entry containing the operation to perform.

        """
        operation = entry.item
        with SessionGen() as session:
            # Obtain submission.
            submission = Submission.get_from_id(operation.submission_id,
                                                session)
            if submission is None:
                raise ValueError("Submission %d not found in the database." %
                                 operation.submission_id)

            # Obtain dataset.
            dataset = Dataset.get_from_id(operation.dataset_id, session)
            if dataset is None:
                raise ValueError("Dataset %d not found in the database." %
                                 operation.dataset_id)

            # Obtain submission result.
            submission_result = submission.get_result(dataset)

            # It means it was not even compiled (for some reason).
            if submission_result is None:
                raise ValueError(
                    "Submission result %d(%d) was not found." %
                    (operation.submission_id, operation.dataset_id))

            # Check if it's ready to be scored.
            if not submission_result.needs_scoring():
                if submission_result.scored():
                    logger.info("Submission result %d(%d) is already scored.",
                                operation.submission_id, operation.dataset_id)
                    return
                else:
                    raise ValueError(
                        "The state of the submission result "
                        "%d(%d) doesn't allow scoring." %
                        (operation.submission_id, operation.dataset_id))

            # Instantiate the score type.
            score_type = get_score_type(dataset=dataset)

            # Compute score and fill it in the database.
            submission_result.score, \
                submission_result.score_details, \
                submission_result.public_score, \
                submission_result.public_score_details, \
                ranking_score_details = \
                score_type.compute_score(submission_result)
            submission_result.ranking_score_details = \
                json.dumps(ranking_score_details)

            # Store it.
            session.commit()

            # If dataset is the active one, update RWS.
            if dataset is submission.task.active_dataset:
                logger.info("Submission scored %.1f seconds after submission",
                            (make_datetime() -
                             submission.timestamp).total_seconds())
                self.proxy_service.submission_scored(
                    submission_id=submission.id)
Ejemplo n.º 38
0
    def invalidate_submission(self,
                              contest_id=None,
                              submission_id=None,
                              dataset_id=None,
                              participation_id=None,
                              task_id=None,
                              testcases=None,
                              overwrite=None,
                              force_priority=None,
                              level="compilation"):
        """Request to invalidate some computed data.

        Invalidate the compilation and/or evaluation data of the
        SubmissionResults that:
        - belong to submission_id or, if None, to any submission of
          participation_id and/or task_id or, if both None, to any
          submission of the contest asked for, or, if all three are
          None, the contest this service is running for (or all contests).
        - belong to dataset_id or, if None, to any dataset of task_id
          or, if None, to any dataset of any task of the contest this
          service is running for.

        The data is cleared, the operations involving the submissions
        currently enqueued are deleted, and the ones already assigned to
        the workers are ignored. New appropriate operations are
        enqueued.

        submission_id (int|None): id of the submission to invalidate,
            or None.
        dataset_id (int|None): id of the dataset to invalidate, or
            None.
        participation_id (int|None): id of the participation to
            invalidate, or None.
        task_id (int|None): id of the task to invalidate, or None.
        level (string): 'compilation' or 'evaluation'

        """
        logger.info("Invalidation request received.")

        # Avoid running the sweeper for the next 10-ish minutes, to
        # avoid race conditions between invalidate's requeuing of
        # submissions and the sweeper's.
        self.avoid_next_sweepers = 1

        # Validate arguments
        # TODO Check that all these objects belong to this contest.
        if level not in ("compilation", "evaluation"):
            raise ValueError("Unexpected invalidation level `%s'." % level)

        if contest_id is None:
            contest_id = self.contest_id

        with SessionGen() as session:
            # First we load all involved submissions.
            if (dataset_id is not None) and (submission_id is None):
                dataset = Dataset.get_from_id(dataset_id, session)
                task_id_for_submissions = dataset.task_id
            else:
                task_id_for_submissions = task_id
            submissions = get_submissions(
                # Give contest_id only if all others are None.
                contest_id if {
                    participation_id, task_id_for_submissions, submission_id
                } == {None} else None,
                participation_id,
                task_id_for_submissions,
                submission_id,
                session)

            # Then we get all relevant operations, and we remove them
            # both from the queue and from the pool (i.e., we ignore
            # the workers involved in those operations).
            operations = get_relevant_operations(level, submissions,
                                                 dataset_id, testcases)
            for operation in operations:
                try:
                    self.dequeue(operation)
                except KeyError:
                    pass  # Ok, the operation wasn't in the queue.
                try:
                    self.get_executor().pool.ignore_operation(operation)
                except LookupError:
                    pass  # Ok, the operation wasn't in the pool.

            # Then we find all existing results in the database, and
            # we remove them.
            submission_results = get_submission_results(
                # Give contest_id only if all others are None.
                contest_id if {
                    participation_id, task_id, submission_id, dataset_id
                } == {None} else None,
                participation_id,
                task_id,
                submission_id,
                dataset_id,
                session)
            logger.info("Submission results to invalidate %s for: %d.", level,
                        len(submission_results))
            for submission_result in submission_results:
                # We invalidate the appropriate data and queue the
                # operations to recompute those data.
                if level == "compilation":
                    submission_result.invalidate_compilation(
                        testcases if overwrite is not False else [])
                elif level == "evaluation":
                    submission_result.invalidate_evaluation(
                        testcases if overwrite is not False else [])

            # There is a small chance that an invalidate won't
            # succeed: if a result is in the pending structure, it
            # might be written after the commit here.
            session.commit()

            # Collecting the ids so that we can close the session
            # before the rpcs.
            submission_ids = [submission.id for submission in submissions]

        # Finally, we re-enqueue the operations for the submissions.
        for idx in range(0, len(submission_ids), 500):
            random_service(self.evaluation_services).new_submissions(
                submission_ids=submission_ids[idx:min(idx +
                                                      500, len(submission_ids
                                                               ))],
                dataset_id=dataset_id,
                force_priority=force_priority)

        logger.info("Invalidate successfully completed.")
Ejemplo n.º 39
0
    def post(self, dataset_id_to_copy):
        fallback_page = self.url("dataset", dataset_id_to_copy, "clone")

        dataset = self.safe_get_item(Dataset, dataset_id_to_copy)
        task = self.safe_get_item(Task, dataset.task_id)
        task_id = task.id

        try:
            original_dataset = \
                self.safe_get_item(Dataset, dataset_id_to_copy)
        except ValueError:
            raise tornado.web.HTTPError(404)

        try:
            attrs = dict()

            self.get_string(attrs, "description")

            # Ensure description is unique.
            if any(attrs["description"] == d.description
                   for d in task.datasets):
                self.application.service.add_notification(
                    make_datetime(),
                    "Dataset name %r is already taken." % attrs["description"],
                    "Please choose a unique name for this dataset.")
                self.redirect(fallback_page)
                return

            self.get_time_limit(attrs, "time_limit")
            self.get_memory_limit(attrs, "memory_limit")
            self.get_task_type(attrs, "task_type", "TaskTypeOptions_")
            self.get_score_type(attrs, "score_type", "score_type_parameters")

            # Create the dataset.
            attrs["autojudge"] = False
            attrs["task"] = task
            dataset = Dataset(**attrs)
            self.sql_session.add(dataset)

        except Exception as error:
            logger.warning("Invalid field.", exc_info=True)
            self.application.service.add_notification(make_datetime(),
                                                      "Invalid field(s)",
                                                      repr(error))
            self.redirect(fallback_page)
            return

        if original_dataset is not None:
            # If we were cloning the dataset, copy all managers and
            # testcases across too. If the user insists, clone all
            # evaluation information too.
            clone_results = bool(self.get_argument("clone_results", False))
            dataset.clone_from(original_dataset, True, True, clone_results)

        # If the task does not yet have an active dataset, make this
        # one active.
        if task.active_dataset is None:
            task.active_dataset = dataset

        if self.try_commit():
            self.redirect(self.url("task", task_id))
        else:
            self.redirect(fallback_page)
Ejemplo n.º 40
0
    def write_results(self, items):
        """Receive worker results from the cache and writes them to the DB.

        Grouping results together by object (i.e., submission result
        or user test result) and type (compilation or evaluation)
        allows this method to talk less to the DB, for example by
        retrieving datasets and submission results only once instead
        of once for every result.

        items ([(operation, Result)]): the results received by ES but
            not yet written to the db.

        """
        logger.info("Starting commit process...")

        # Reorganize the results by submission/usertest result and
        # operation type (i.e., group together the testcase
        # evaluations for the same submission and dataset).
        by_object_and_type = defaultdict(list)
        for operation, result in items:
            t = (operation.type_, operation.object_id, operation.dataset_id)
            by_object_and_type[t].append((operation, result))

        with SessionGen() as session:
            for key, operation_results in by_object_and_type.items():
                type_, object_id, dataset_id = key

                dataset = Dataset.get_from_id(dataset_id, session)
                if dataset is None:
                    logger.error("Could not find dataset %d in the database.",
                                 dataset_id)
                    continue

                # Get submission or user test results.
                if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]:
                    object_ = Submission.get_from_id(object_id, session)
                    if object_ is None:
                        logger.error("Could not find submission %d "
                                     "in the database.", object_id)
                        continue
                    object_result = object_.get_result_or_create(dataset)
                else:
                    object_ = UserTest.get_from_id(object_id, session)
                    if object_ is None:
                        logger.error("Could not find user test %d "
                                     "in the database.", object_id)
                        continue
                    object_result = object_.get_result_or_create(dataset)

                self.write_results_one_object_and_type(
                    session, object_result, operation_results)

            logger.info("Committing evaluations...")
            session.commit()

            num_testcases_per_dataset = dict()
            for type_, object_id, dataset_id in by_object_and_type.keys():
                if type_ == ESOperation.EVALUATION:
                    if dataset_id not in num_testcases_per_dataset:
                        num_testcases_per_dataset[dataset_id] = session\
                            .query(func.count(Testcase.id))\
                            .filter(Testcase.dataset_id == dataset_id).scalar()
                    num_evaluations = session\
                        .query(func.count(Evaluation.id)) \
                        .filter(Evaluation.dataset_id == dataset_id) \
                        .filter(Evaluation.submission_id == object_id).scalar()
                    if num_evaluations == num_testcases_per_dataset[dataset_id]:
                        submission_result = SubmissionResult.get_from_id(
                            (object_id, dataset_id), session)
                        submission_result.set_evaluation_outcome()

            logger.info("Committing evaluation outcomes...")
            session.commit()

            logger.info("Ending operations for %s objects...",
                        len(by_object_and_type))
            for type_, object_id, dataset_id in by_object_and_type.keys():
                if type_ == ESOperation.COMPILATION:
                    submission_result = SubmissionResult.get_from_id(
                        (object_id, dataset_id), session)
                    self.compilation_ended(submission_result)
                elif type_ == ESOperation.EVALUATION:
                    submission_result = SubmissionResult.get_from_id(
                        (object_id, dataset_id), session)
                    if submission_result.evaluated():
                        self.evaluation_ended(submission_result)
                elif type_ == ESOperation.USER_TEST_COMPILATION:
                    user_test_result = UserTestResult.get_from_id(
                        (object_id, dataset_id), session)
                    self.user_test_compilation_ended(user_test_result)
                elif type_ == ESOperation.USER_TEST_EVALUATION:
                    user_test_result = UserTestResult.get_from_id(
                        (object_id, dataset_id), session)
                    self.user_test_evaluation_ended(user_test_result)

        logger.info("Done")