def build_job(self, session): """Produce the Job for this operation. Return the Job object that has to be sent to Workers to have them perform the operation this object describes. session (Session): the database session to use to fetch objects if necessary. return (Job): the job encoding of the operation, as understood by Workers and TaskTypes. """ result = None dataset = Dataset.get_from_id(self.dataset_id, session) if self.type_ == ESOperation.COMPILATION: submission = Submission.get_from_id(self.object_id, session) result = CompilationJob.from_submission(submission, dataset) elif self.type_ == ESOperation.EVALUATION: submission = Submission.get_from_id(self.object_id, session) result = EvaluationJob.from_submission(submission, dataset, self.testcase_codename) elif self.type_ == ESOperation.USER_TEST_COMPILATION: user_test = UserTest.get_from_id(self.object_id, session) result = CompilationJob.from_user_test(user_test, dataset) elif self.type_ == ESOperation.USER_TEST_EVALUATION: user_test = UserTest.get_from_id(self.object_id, session) result = EvaluationJob.from_user_test(user_test, dataset) return result
def build_job(self, session): """Produce the Job for this operation. Return the Job object that has to be sent to Workers to have them perform the operation this object describes. session (Session): the database session to use to fetch objects if necessary. return (Job): the job encoding of the operation, as understood by Workers and TaskTypes. """ result = None dataset = Dataset.get_from_id(self.dataset_id, session) if self.type_ == ESOperation.COMPILATION: submission = Submission.get_from_id(self.object_id, session) result = CompilationJob.from_submission(submission, dataset) elif self.type_ == ESOperation.EVALUATION: submission = Submission.get_from_id(self.object_id, session) result = EvaluationJob.from_submission( submission, dataset, self.testcase_codename) elif self.type_ == ESOperation.USER_TEST_COMPILATION: user_test = UserTest.get_from_id(self.object_id, session) result = CompilationJob.from_user_test(user_test, dataset) elif self.type_ == ESOperation.USER_TEST_EVALUATION: user_test = UserTest.get_from_id(self.object_id, session) result = EvaluationJob.from_user_test(user_test, dataset) return result
def submission_scored(self, submission_id): """Notice that a submission has been scored. Usually called by ScoringService when it's done with scoring a submission result. Since we don't trust anyone we verify that, and then send data about the score to the rankings. submission_id (int): the id of the submission that changed. dataset_id (int): the id of the dataset to use. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error("[submission_scored] Received score request for " "unexistent submission id %s." % submission_id) raise KeyError("Submission not found.") if submission.user.hidden: logger.info("[submission_scored] Score for submission %d " "not sent because user is hidden." % submission_id) return # Update RWS. self.send_score(submission)
def submission_tokened(self, submission_id): """Notice that a submission has been tokened. Usually called by ContestWebServer when it's processing a token request of an user. Since we don't trust anyone we verify that, and then send data about the token to the rankings. submission_id (int): the id of the submission that changed. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error( "[submission_tokened] Received token request for " "unexistent submission id %s.", submission_id) raise KeyError("Submission not found.") if submission.participation.hidden: logger.info( "[submission_tokened] Token for submission %d " "not sent because participation is hidden.", submission_id) return if not submission.official: logger.info( "[submission_tokened] Token for submission %d " "not sent because the submission is not official.", submission_id) return # Update RWS. for operation in self.operations_for_token(submission): self.enqueue(operation)
def new_submission(self, submission_id, dataset_id=None, force_priority=None): """This RPC prompts ES of the existence of a new submission. ES takes the right countermeasures, i.e., it schedules it for compilation. submission_id (int): the id of the new submission. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if dataset_id is not None: dataset = Dataset.get_from_id(dataset_id, session) else: dataset = None if submission is None: logger.error( "[new_submission] Couldn't find submission " "%d in the database.", submission_id) return self.enqueue_all(self.get_submission_operations( submission, dataset), force_priority=force_priority) session.commit()
def enqueue(self, operation, priority, timestamp, job=None): """Push an operation in the queue. Push an operation in the operation queue if the submission is not already in the queue or assigned to a worker. operation (ESOperation): the operation to put in the queue. priority (int): the priority of the operation. timestamp (datetime): the time of the submission. job (dict|None): the job associated; if None will be computed return (bool): True if pushed, False if not. """ if job is None: with SessionGen() as session: dataset = Dataset.get_from_id(operation.dataset_id, session) if operation.for_submission(): object_ = Submission.get_from_id(operation.object_id, session) else: object_ = UserTest.get_from_id(operation.object_id, session) job = Job.from_operation(operation, object_, dataset).export_to_dict() return self.queue_service.enqueue( operation=operation.to_list(), priority=priority, timestamp=(timestamp - EvaluationService.EPOCH).total_seconds(), job=job)
def submission_tokened(self, submission_id): """Notice that a submission has been tokened. Usually called by ContestWebServer when it's processing a token request of an user. Since we don't trust anyone we verify that, and then send data about the token to the rankings. submission_id (int): the id of the submission that changed. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error("[submission_tokened] Received token request for " "unexistent submission id %s." % submission_id) raise KeyError("Submission not found.") if submission.user.hidden: logger.info("[submission_tokened] Token for submission %d " "not sent because user is hidden." % submission_id) return # Update RWS. self.send_token(submission)
def execute(self, entry): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. entry (QueueEntry): entry containing the operation to perform. """ operation = entry.item with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(operation.submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % operation.submission_id) # Obtain dataset. dataset = Dataset.get_from_id(operation.dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % operation.dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError( "Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id) ) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info( "Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id ) return else: raise ValueError( "The state of the submission result " "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id) ) # Instantiate the score type. score_type = get_score_type(dataset=dataset) # Compute score and fill it in the database. submission_result.score, submission_result.score_details, submission_result.public_score, submission_result.public_score_details, submission_result.ranking_score_details = score_type.compute_score( submission_result ) # Store it. session.commit() # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: self.proxy_service.submission_scored(submission_id=submission.id)
def acquire_worker(self, operations): """Tries to assign an operation to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. operations ([ESOperation]): the operations to assign to a worker. return (int|None): None if no workers are available, the worker assigned to the operation otherwise. """ # We look for an available worker. try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: self._workers_available_event.clear() return None # Then we fill the info for future memory. self._add_operations(shard, operations) logger.debug("Worker %s acquired.", shard) self._start_time[shard] = make_datetime() with SessionGen() as session: jobs = [] datasets = {} submissions = {} user_tests = {} for operation in operations: if operation.dataset_id not in datasets: datasets[operation.dataset_id] = Dataset.get_from_id( operation.dataset_id, session) object_ = None if operation.for_submission(): if operation.object_id not in submissions: submissions[operation.object_id] = \ Submission.get_from_id( operation.object_id, session) object_ = submissions[operation.object_id] else: if operation.object_id not in user_tests: user_tests[operation.object_id] = \ UserTest.get_from_id(operation.object_id, session) object_ = user_tests[operation.object_id] logger.info("Asking worker %s to `%s'.", shard, operation) jobs.append( Job.from_operation(operation, object_, datasets[operation.dataset_id])) job_group_dict = JobGroup(jobs).export_to_dict() self._worker[shard].execute_job_group( job_group_dict=job_group_dict, callback=self._service.action_finished, plus=shard) return shard
def acquire_worker(self, operations): """Tries to assign an operation to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. operations ([ESOperation]): the operations to assign to a worker. return (int|None): None if no workers are available, the worker assigned to the operation otherwise. """ # We look for an available worker. try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: self._workers_available_event.clear() return None # Then we fill the info for future memory. self._add_operations(shard, operations) logger.debug("Worker %s acquired.", shard) self._start_time[shard] = make_datetime() with SessionGen() as session: jobs = [] datasets = {} submissions = {} user_tests = {} for operation in operations: if operation.dataset_id not in datasets: datasets[operation.dataset_id] = Dataset.get_from_id( operation.dataset_id, session) object_ = None if operation.for_submission(): if operation.object_id not in submissions: submissions[operation.object_id] = \ Submission.get_from_id( operation.object_id, session) object_ = submissions[operation.object_id] else: if operation.object_id not in user_tests: user_tests[operation.object_id] = \ UserTest.get_from_id(operation.object_id, session) object_ = user_tests[operation.object_id] logger.info("Asking worker %s to `%s'.", shard, operation) jobs.append(Job.from_operation( operation, object_, datasets[operation.dataset_id])) job_group_dict = JobGroup(jobs).export_to_dict() self._worker[shard].execute_job_group( job_group_dict=job_group_dict, callback=self._service.action_finished, plus=shard) return shard
def from_operations(operations, session): jobs = [] for operation in operations: # The get_from_id method loads from the instance map (if the # object exists there), which thus acts as a cache. if operation.for_submission(): object_ = Submission.get_from_id(operation.object_id, session) else: object_ = UserTest.get_from_id(operation.object_id, session) dataset = Dataset.get_from_id(operation.dataset_id, session) jobs.append(Job.from_operation(operation, object_, dataset)) return JobGroup(jobs)
def submission_scored(self, submission_id): """Notice that a submission has been scored. Usually called by ScoringService when it's done with scoring a submission result. Since we don't trust anyone we verify that, and then send data about the score to the rankings. submission_id (int): the id of the submission that changed. dataset_id (int): the id of the dataset to use. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error( "[submission_scored] Received score request for " "unexistent submission id %s.", submission_id) raise KeyError("Submission not found.") # ScoringService sent us a submission of another contest, they # do not know about our contest_id in multicontest setup. if submission.task.contest_id != self.contest_id: logger.debug( "Ignoring submission %d of contest %d " "(this ProxyService considers contest %d only).", submission.id, submission.task.contest_id, self.contest_id) return if submission.participation.hidden: logger.info( "[submission_scored] Score for submission %d " "not sent because the participation is hidden.", submission_id) return if not submission.official: logger.info( "[submission_scored] Score for submission %d " "not sent because the submission is not official.", submission_id) return # Update RWS. for operation in self.operations_for_score(submission): self.enqueue(operation)
def new_submission(self, submission_id): """This RPC prompts ES of the existence of a new submission. ES takes the right countermeasures, i.e., it schedules it for compilation. submission_id (int): the id of the new submission. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error("[new_submission] Couldn't find submission " "%d in the database.", submission_id) return self.submission_enqueue_operations(submission) session.commit()
def submission_scored(self, submission_id): """Notice that a submission has been scored. Usually called by ScoringService when it's done with scoring a submission result. Since we don't trust anyone we verify that, and then send data about the score to the rankings. submission_id (int): the id of the submission that changed. dataset_id (int): the id of the dataset to use. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error( "[submission_scored] Received score request for " "unexistent submission id %s.", submission_id) raise KeyError("Submission not found.") if submission.task.score_mode == SCORE_MODE_MAX_OTHER_USERS: logger.info( "[submission_scored] Score for submission %d " "not sent because of the task score mode", submission_id) return if submission.participation.hidden: logger.info( "[submission_scored] Score for submission %d " "not sent because the participation is hidden.", submission_id) return if not submission.official: logger.info( "[submission_scored] Score for submission %d " "not sent because the submission is not official.", submission_id) return # Update RWS. for operation in self.operations_for_score(submission): self.enqueue(operation)
def get_submission_ops(self, submission_id, dataset_id=None): """This RPC returns the operations (including job) for the given submission and dataset. """ with SessionGen() as session: submission = Submission.get_from_id(submission_id, session) if dataset_id is not None: dataset = Dataset.get_from_id(dataset_id, session) else: dataset = None if submission is None: logger.error( "[get_submission_ops] Couldn't find submission " "%d in the database.", submission_id) return [] return [(operation.to_list(), priority, (timestamp - EvaluationService.EPOCH).total_seconds(), job) for operation, priority, timestamp, job in self.get_submission_operations(submission, dataset)]
def submission_tokened(self, submission_id): """This RPC inform ScoringService that the user has played the token on a submission. submission_id (int): the id of the submission that changed. timestamp (int): the time of the token. """ with SessionGen(commit=False) as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error("[submission_tokened] Received token request for " "unexistent submission id %s." % submission_id) raise KeyError if submission.user.hidden: logger.info("[submission_tokened] Token for submission %d " "not sent because user is hidden." % submission_id) return # Update RWS. self.rankings_send_token(submission)
def execute(self, entry): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. entry (QueueEntry): entry containing the operation to perform. """ operation = entry.item with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(operation.submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % operation.submission_id) # Obtain dataset. dataset = Dataset.get_from_id(operation.dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % operation.dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError("Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id)) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info("Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id) return else: raise ValueError("The state of the submission result " "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id)) # For Codebreaker, your score depends on your previous submissions # to this task. So, let's get the previous submisisons for this task previous_submissions = session.query(Submission)\ .filter(Submission.user_id == submission.user_id, Submission.task_id == submission.task_id)\ .order_by(asc(Submission.timestamp))\ .all() # Counterintuitively, because we're nice people, we don't care how # these submissions were scored. We only care about their # evaluations, which will tell us how to score them. # For a codebreaker, this will be in one-to-one correspondence with # previous submissions, since each "task" should only have the one # "testcase". previous_evaluations = [ session.query(Evaluation) .filter(Evaluation.submission_id == sub.id).first() for sub in previous_submissions] assert(len(previous_evaluations) == len(previous_submissions)) # Now that we have the evaluations, we can pass these as parameters # to our score type params = [evaluation.outcome for evaluation in previous_evaluations] # Instantiate the score type. # We don't want to use the dataset since we have to pass in custom # params. Instead we'll just hardcode the name of the class in, # which is unfortunate. # TODO (bgbn): work out a way to make this more generic. score_type = get_score_type(name="AIOCCodebreakerScoreType", parameters=json.dumps(params), public_testcases=dict((k, tc.public) for k, tc in dataset.testcases.iteritems())) # Compute score and fill it in the database. submission_result.score, \ submission_result.score_details, \ submission_result.public_score, \ submission_result.public_score_details, \ submission_result.ranking_score_details = \ score_type.compute_score(submission_result) # Store it. session.commit() # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: self.proxy_service.submission_scored( submission_id=submission.id)
def write_result(self, operation, job): """Receive worker results from QS and writes them to the DB. operation (dict): operation performed, exported as dict job (dict): job containing the result, exported as dict """ logger.debug("Starting commit process...") operation = ESOperation.from_dict(operation) job = Job.import_from_dict_with_type(job) with SessionGen() as session: type_ = operation.type_ object_id = operation.object_id dataset_id = operation.dataset_id dataset = session.query(Dataset)\ .filter(Dataset.id == dataset_id)\ .options(joinedload(Dataset.testcases))\ .first() if dataset is None: logger.error("Could not find dataset %d in the database.", dataset_id) return False, [] # Get submission or user test, and their results. if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]: object_ = Submission.get_from_id(object_id, session) if object_ is None: logger.error( "Could not find submission %d " "in the database.", object_id) return False, [] object_result = object_.get_result_or_create(dataset) else: object_ = UserTest.get_from_id(object_id, session) object_result = object_.get_result_or_create(dataset) logger.info("Writing result to db for %s", operation) new_operations = [] try: new_operations = self.write_results_one_row( session, object_result, operation, job) except IntegrityError: logger.warning( "Integrity error while inserting worker result.", exc_info=True) # This is not an error condition, as the result is already # in the DB. return True, [] logger.debug("Committing evaluations...") session.commit() # If we collected some new operations to do while writing # the results, it means we had to invalidate the submission. # We return immediately since we already have all the operations # we need to do next. if new_operations != []: return True, [[ op.to_dict(), priority, (timestamp - EvaluationService.EPOCH).total_seconds(), job_ ] for op, priority, timestamp, job_ in new_operations] if type_ == ESOperation.EVALUATION: if len(object_result.evaluations) == len(dataset.testcases): object_result.set_evaluation_outcome() logger.debug("Committing evaluation outcomes...") session.commit() logger.info("Ending operations...") if type_ == ESOperation.COMPILATION: new_operations = self.compilation_ended(object_result) elif type_ == ESOperation.EVALUATION: if object_result.evaluated(): new_operations = self.evaluation_ended(object_result) elif type_ == ESOperation.USER_TEST_COMPILATION: new_operations = \ self.user_test_compilation_ended(object_result) elif type_ == ESOperation.USER_TEST_EVALUATION: new_operations = self.user_test_evaluation_ended(object_result) logger.debug("Done") return True, [[ op.to_dict(), priority, (timestamp - EvaluationService.EPOCH).total_seconds(), job_ ] for op, priority, timestamp, job_ in new_operations]
def _score(self, submission_id, dataset_id): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. submission_id (int): the id of the submission that has to be scored. dataset_id (int): the id of the dataset to use. """ with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % submission_id) # Obtain dataset. dataset = Dataset.get_from_id(dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError("Submission result %d(%d) was not found." % (submission_id, dataset_id)) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info("Submission result %d(%d) is already scored.", submission_id, dataset_id) return else: raise ValueError("The state of the submission result " "%d(%d) doesn't allow scoring." % (submission_id, dataset_id)) # Instantiate the score type. score_type = get_score_type(dataset=dataset) # Compute score and fill it in the database. submission_result.score, \ submission_result.score_details, \ submission_result.public_score, \ submission_result.public_score_details, \ submission_result.ranking_score_details = \ score_type.compute_score(submission_result) # Store it. session.commit() # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: self.proxy_service.submission_scored( submission_id=submission.id)
def action_finished(self, data, plus, error=None): """Callback from a worker, to signal that is finished some action (compilation or evaluation). data (dict): a dictionary that describes a Job instance. plus (tuple): the tuple (type_, object_id, dataset_id, testcase_codename, side_data=(priority, timestamp), shard_of_worker) """ # Unpack the plus tuple. It's built in the RPC call to Worker's # execute_job method inside WorkerPool.acquire_worker. type_, object_id, dataset_id, testcase_codename, _, \ shard = plus # Restore operation from its fields. operation = ESOperation(type_, object_id, dataset_id, testcase_codename) # We notify the pool that the worker is available again for # further work (no matter how the current request turned out, # even if the worker encountered an error). If the pool # informs us that the data produced by the worker has to be # ignored (by returning True) we interrupt the execution of # this method and do nothing because in that case we know the # operation has returned to the queue and perhaps already been # reassigned to another worker. if self.get_executor().pool.release_worker(shard): logger.info("Ignored result from worker %s as requested.", shard) return job_success = True if error is not None: logger.error("Received error from Worker: `%s'.", error) job_success = False else: try: job = Job.import_from_dict_with_type(data) except: logger.error("Couldn't build Job for data %s.", data, exc_info=True) job_success = False else: if not job.success: logger.error("Worker %s signaled action not successful.", shard) job_success = False logger.info("`%s' completed. Success: %s.", operation, job_success) # We get the submission from DB and update it. with SessionGen() as session: dataset = Dataset.get_from_id(dataset_id, session) if dataset is None: logger.error("Could not find dataset %d in the database.", dataset_id) return # TODO Try to move this 4-cases if-clause into a method of # ESOperation: I'd really like ES itself not to care about # which type of operation it's handling. if type_ == ESOperation.COMPILATION: submission = Submission.get_from_id(object_id, session) if submission is None: logger.error( "Could not find submission %d " "in the database.", object_id) return submission_result = submission.get_result(dataset) if submission_result is None: logger.info( "Couldn't find submission %d(%d) " "in the database. Creating it.", object_id, dataset_id) submission_result = \ submission.get_result_or_create(dataset) if job_success: job.to_submission(submission_result) else: submission_result.compilation_tries += 1 session.commit() self.compilation_ended(submission_result) elif type_ == ESOperation.EVALUATION: submission = Submission.get_from_id(object_id, session) if submission is None: logger.error( "Could not find submission %d " "in the database.", object_id) return submission_result = submission.get_result(dataset) if submission_result is None: logger.error( "Couldn't find submission %d(%d) " "in the database.", object_id, dataset_id) return if job_success: job.to_submission(submission_result) else: submission_result.evaluation_tries += 1 # Submission evaluation will be ended only when # evaluation for each testcase is available. evaluation_complete = (len( submission_result.evaluations) == len(dataset.testcases)) if evaluation_complete: submission_result.set_evaluation_outcome() session.commit() if evaluation_complete: self.evaluation_ended(submission_result) elif type_ == ESOperation.USER_TEST_COMPILATION: user_test = UserTest.get_from_id(object_id, session) if user_test is None: logger.error( "Could not find user test %d " "in the database.", object_id) return user_test_result = user_test.get_result(dataset) if user_test_result is None: logger.error( "Couldn't find user test %d(%d) " "in the database. Creating it.", object_id, dataset_id) user_test_result = \ user_test.get_result_or_create(dataset) if job_success: job.to_user_test(user_test_result) else: user_test_result.compilation_tries += 1 session.commit() self.user_test_compilation_ended(user_test_result) elif type_ == ESOperation.USER_TEST_EVALUATION: user_test = UserTest.get_from_id(object_id, session) if user_test is None: logger.error( "Could not find user test %d " "in the database.", object_id) return user_test_result = user_test.get_result(dataset) if user_test_result is None: logger.error( "Couldn't find user test %d(%d) " "in the database.", object_id, dataset_id) return if job_success: job.to_user_test(user_test_result) else: user_test_result.evaluation_tries += 1 session.commit() self.user_test_evaluation_ended(user_test_result) else: logger.error("Invalid operation type %r.", type_) return
def execute(self, entry): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. entry (QueueEntry): entry containing the operation to perform. """ operation = entry.item with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(operation.submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % operation.submission_id) # Obtain dataset. dataset = Dataset.get_from_id(operation.dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % operation.dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError( "Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id)) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info("Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id) return else: raise ValueError( "The state of the submission result " "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id)) # Instantiate the score type. score_type = get_score_type(dataset=dataset) # Compute score and fill it in the database. submission_result.score, \ submission_result.score_details, \ submission_result.public_score, \ submission_result.public_score_details, \ ranking_score_details = \ score_type.compute_score(submission_result) submission_result.ranking_score_details = \ json.dumps(ranking_score_details) # Store it. session.commit() # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: logger.info("Submission scored %.1f seconds after submission", (make_datetime() - submission.timestamp).total_seconds()) self.proxy_service.submission_scored( submission_id=submission.id)
def write_results(self, items): """Receive worker results from the cache and writes them to the DB. Grouping results together by object (i.e., submission result or user test result) and type (compilation or evaluation) allows this method to talk less to the DB, for example by retrieving datasets and submission results only once instead of once for every result. items ([(operation, Result)]): the results received by ES but not yet written to the db. """ logger.info("Starting commit process...") # Reorganize the results by submission/usertest result and # operation type (i.e., group together the testcase # evaluations for the same submission and dataset). by_object_and_type = defaultdict(list) for operation, result in items: t = (operation.type_, operation.object_id, operation.dataset_id) by_object_and_type[t].append((operation, result)) with SessionGen() as session: # Dictionary holding the objects we use repeatedly, # indexed by id, to avoid querying them multiple times. # TODO: this pattern is used in WorkerPool and should be # abstracted away. datasets = dict() subs = dict() srs = dict() for key, operation_results in iteritems(by_object_and_type): type_, object_id, dataset_id = key # Get dataset. if dataset_id not in datasets: datasets[dataset_id] = session.query(Dataset)\ .filter(Dataset.id == dataset_id)\ .options(joinedload(Dataset.testcases))\ .first() dataset = datasets[dataset_id] if dataset is None: logger.error("Could not find dataset %d in the database.", dataset_id) continue # Get submission or user test, and their results. if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]: if object_id not in subs: subs[object_id] = \ Submission.get_from_id(object_id, session) object_ = subs[object_id] if object_ is None: logger.error("Could not find submission %d " "in the database.", object_id) continue result_id = (object_id, dataset_id) if result_id not in srs: srs[result_id] = object_.get_result_or_create(dataset) object_result = srs[result_id] else: # We do not cache user tests as they can come up # only once. object_ = UserTest.get_from_id(object_id, session) object_result = object_.get_result_or_create(dataset) self.write_results_one_object_and_type( session, object_result, operation_results) logger.info("Committing evaluations...") session.commit() for type_, object_id, dataset_id in by_object_and_type: if type_ == ESOperation.EVALUATION: submission_result = srs[(object_id, dataset_id)] dataset = datasets[dataset_id] if len(submission_result.evaluations) == \ len(dataset.testcases): submission_result.set_evaluation_outcome() logger.info("Committing evaluation outcomes...") session.commit() logger.info("Ending operations for %s objects...", len(by_object_and_type)) for type_, object_id, dataset_id in by_object_and_type: if type_ == ESOperation.COMPILATION: submission_result = srs[(object_id, dataset_id)] self.compilation_ended(submission_result) elif type_ == ESOperation.EVALUATION: submission_result = srs[(object_id, dataset_id)] if submission_result.evaluated(): self.evaluation_ended(submission_result) elif type_ == ESOperation.USER_TEST_COMPILATION: user_test_result = UserTest\ .get_from_id(object_id, session)\ .get_result(datasets[dataset_id]) self.user_test_compilation_ended(user_test_result) elif type_ == ESOperation.USER_TEST_EVALUATION: user_test_result = UserTest\ .get_from_id(object_id, session)\ .get_result(datasets[dataset_id]) self.user_test_evaluation_ended(user_test_result) logger.info("Done")
def execute(self, entry): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. entry (QueueEntry): entry containing the operation to perform. """ operation = entry.item with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(operation.submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % operation.submission_id) # Obtain dataset. dataset = Dataset.get_from_id(operation.dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % operation.dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError("Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id)) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info("Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id) return else: raise ValueError("The state of the submission result " "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id)) # Instantiate the score type. score_type = get_score_type(dataset=dataset) # Compute score and fill it in the database. submission_result.score, \ submission_result.score_details, \ submission_result.public_score, \ submission_result.public_score_details, \ submission_result.ranking_score_details = \ score_type.compute_score(submission_result) # Round submission score to 2 decimal places submission_result.score = round(submission_result.score, 2) # Store it. session.commit() # Update statistics and access level score = submission_result.score taskscore = session.query(TaskScore)\ .filter(TaskScore.user_id == submission.user_id)\ .filter(TaskScore.task_id == submission.task_id).first() if taskscore is None: taskscore = TaskScore() taskscore.task_id = submission.task_id taskscore.user_id = submission.user_id session.add(taskscore) mtime = max([0] + [e.execution_time for e in submission_result.evaluations]) if score > taskscore.score: taskscore.score = score taskscore.time = mtime elif score == taskscore.score and mtime < taskscore.time: taskscore.time = mtime submission.task.nsubscorrect = session.query(Submission)\ .filter(Submission.task_id == submission.task_id)\ .filter(Submission.results.any( SubmissionResult.score == 100)).count() submission.task.nuserscorrect = session.query(TaskScore)\ .filter(TaskScore.task_id == submission.task_id)\ .filter(TaskScore.score == 100).count() submission.user.score = sum([ t.score for t in session.query(TaskScore) .filter(TaskScore.user_id == submission.user_id).all()]) if submission.user.score >= 300 and \ submission.user.access_level == 6: submission.user.access_level = 5 submission.task.nsubs = session.query(Submission)\ .filter(Submission.task_id == submission.task_id).count() submission.task.nusers = session.query(TaskScore)\ .filter(TaskScore.task_id == submission.task_id).count() session.commit() # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: self.proxy_service.submission_scored( submission_id=submission.id)
def write_results(self, items): """Receive worker results from the cache and writes them to the DB. Grouping results together by object (i.e., submission result or user test result) and type (compilation or evaluation) allows this method to talk less to the DB, for example by retrieving datasets and submission results only once instead of once for every result. items ([(operation, Result)]): the results received by ES but not yet written to the db. """ logger.info("Starting commit process...") # Reorganize the results by submission/usertest result and # operation type (i.e., group together the testcase # evaluations for the same submission and dataset). by_object_and_type = defaultdict(list) for operation, result in items: t = (operation.type_, operation.object_id, operation.dataset_id) by_object_and_type[t].append((operation, result)) with SessionGen() as session: for key, operation_results in by_object_and_type.items(): type_, object_id, dataset_id = key dataset = Dataset.get_from_id(dataset_id, session) if dataset is None: logger.error("Could not find dataset %d in the database.", dataset_id) continue # Get submission or user test results. if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]: object_ = Submission.get_from_id(object_id, session) if object_ is None: logger.error( "Could not find submission %d " "in the database.", object_id) continue object_result = object_.get_result_or_create(dataset) else: object_ = UserTest.get_from_id(object_id, session) if object_ is None: logger.error( "Could not find user test %d " "in the database.", object_id) continue object_result = object_.get_result_or_create(dataset) self.write_results_one_object_and_type(session, object_result, operation_results) logger.info("Committing evaluations...") session.commit() num_testcases_per_dataset = dict() for type_, object_id, dataset_id in by_object_and_type.keys(): if type_ == ESOperation.EVALUATION: if dataset_id not in num_testcases_per_dataset: num_testcases_per_dataset[dataset_id] = session\ .query(func.count(Testcase.id))\ .filter(Testcase.dataset_id == dataset_id).scalar() num_evaluations = session\ .query(func.count(Evaluation.id)) \ .filter(Evaluation.dataset_id == dataset_id) \ .filter(Evaluation.submission_id == object_id).scalar() if num_evaluations == num_testcases_per_dataset[ dataset_id]: submission_result = SubmissionResult.get_from_id( (object_id, dataset_id), session) submission_result.set_evaluation_outcome() logger.info("Committing evaluation outcomes...") session.commit() logger.info("Ending operations for %s objects...", len(by_object_and_type)) for type_, object_id, dataset_id in by_object_and_type.keys(): if type_ == ESOperation.COMPILATION: submission_result = SubmissionResult.get_from_id( (object_id, dataset_id), session) self.compilation_ended(submission_result) elif type_ == ESOperation.EVALUATION: submission_result = SubmissionResult.get_from_id( (object_id, dataset_id), session) if submission_result.evaluated(): self.evaluation_ended(submission_result) elif type_ == ESOperation.USER_TEST_COMPILATION: user_test_result = UserTestResult.get_from_id( (object_id, dataset_id), session) self.user_test_compilation_ended(user_test_result) elif type_ == ESOperation.USER_TEST_EVALUATION: user_test_result = UserTestResult.get_from_id( (object_id, dataset_id), session) self.user_test_evaluation_ended(user_test_result) logger.info("Done")
def new_evaluation(self, submission_id, dataset_id): """This RPC inform ScoringService that ES finished the work on a submission (either because it has been evaluated, or because the compilation failed). submission_id (int): the id of the submission that changed. dataset_id (int): the id of the dataset to use. """ with SessionGen(commit=True) as session: submission = Submission.get_from_id(submission_id, session) if submission is None: logger.error("[new_evaluation] Couldn't find submission %d " "in the database." % submission_id) raise ValueError if submission.user.hidden: logger.info("[new_evaluation] Submission %d not scored " "because user is hidden." % submission_id) return dataset = Dataset.get_from_id(dataset_id, session) if dataset is None: logger.error("[new_evaluation] Couldn't find dataset %d " "in the database." % dataset_id) raise ValueError submission_result = submission.get_result(dataset) # We'll accept only submissions that either didn't compile # at all or that did evaluate successfully. if submission_result is None or not submission_result.compiled(): logger.warning("[new_evaluation] Submission %d(%d) is " "not compiled." % (submission_id, dataset_id)) return elif submission_result.compilation_outcome == "ok" and \ not submission_result.evaluated(): logger.warning("[new_evaluation] Submission %d(%d) is " "compiled but is not evaluated." % (submission_id, dataset_id)) return # Assign score to the submission. score_type = get_score_type(dataset=dataset) score, details, public_score, public_details, ranking_details = \ score_type.compute_score(submission_result) # Mark submission as scored. self.submission_results_scored.add((submission_id, dataset_id)) # Filling submission's score info in the db. submission_result.score = score submission_result.public_score = public_score # And details. submission_result.score_details = details submission_result.public_score_details = public_details submission_result.ranking_score_details = ranking_details # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: self.rankings_send_score(submission)
def write_results(self, items): """Receive worker results from the cache and writes them to the DB. Grouping results together by object (i.e., submission result or user test result) and type (compilation or evaluation) allows this method to talk less to the DB, for example by retrieving datasets and submission results only once instead of once for every result. items ([(operation, Result)]): the results received by ES but not yet written to the db. """ logger.info("Starting commit process...") # Reorganize the results by submission/usertest result and # operation type (i.e., group together the testcase # evaluations for the same submission and dataset). by_object_and_type = defaultdict(list) for operation, result in items: t = (operation.type_, operation.object_id, operation.dataset_id) by_object_and_type[t].append((operation, result)) with SessionGen() as session: for key, operation_results in by_object_and_type.items(): type_, object_id, dataset_id = key dataset = Dataset.get_from_id(dataset_id, session) if dataset is None: logger.error("Could not find dataset %d in the database.", dataset_id) continue # Get submission or user test results. if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]: object_ = Submission.get_from_id(object_id, session) if object_ is None: logger.error("Could not find submission %d " "in the database.", object_id) continue object_result = object_.get_result_or_create(dataset) else: object_ = UserTest.get_from_id(object_id, session) if object_ is None: logger.error("Could not find user test %d " "in the database.", object_id) continue object_result = object_.get_result_or_create(dataset) self.write_results_one_object_and_type( session, object_result, operation_results) logger.info("Committing evaluations...") session.commit() num_testcases_per_dataset = dict() for type_, object_id, dataset_id in by_object_and_type.keys(): if type_ == ESOperation.EVALUATION: if dataset_id not in num_testcases_per_dataset: num_testcases_per_dataset[dataset_id] = session\ .query(func.count(Testcase.id))\ .filter(Testcase.dataset_id == dataset_id).scalar() num_evaluations = session\ .query(func.count(Evaluation.id)) \ .filter(Evaluation.dataset_id == dataset_id) \ .filter(Evaluation.submission_id == object_id).scalar() if num_evaluations == num_testcases_per_dataset[dataset_id]: submission_result = SubmissionResult.get_from_id( (object_id, dataset_id), session) submission_result.set_evaluation_outcome() logger.info("Committing evaluation outcomes...") session.commit() logger.info("Ending operations for %s objects...", len(by_object_and_type)) for type_, object_id, dataset_id in by_object_and_type.keys(): if type_ == ESOperation.COMPILATION: submission_result = SubmissionResult.get_from_id( (object_id, dataset_id), session) self.compilation_ended(submission_result) elif type_ == ESOperation.EVALUATION: submission_result = SubmissionResult.get_from_id( (object_id, dataset_id), session) if submission_result.evaluated(): self.evaluation_ended(submission_result) elif type_ == ESOperation.USER_TEST_COMPILATION: user_test_result = UserTestResult.get_from_id( (object_id, dataset_id), session) self.user_test_compilation_ended(user_test_result) elif type_ == ESOperation.USER_TEST_EVALUATION: user_test_result = UserTestResult.get_from_id( (object_id, dataset_id), session) self.user_test_evaluation_ended(user_test_result) logger.info("Done")
def get(self, submission_id): """Retrieve a single submission. Query the database for the submission with the given ID, and the dataset given as query parameter (or the active one). submission_id (int): the ID of a submission. """ # If it's not an integer we will ignore it. But if it's an # integer of a dataset that doesn't exist we'll raise a 404. dataset_id = local.request.args.get("dataset_id", type=int) with SessionGen() as local.session: # Load the submission, and check for existence. submission = Submission.get_from_id(submission_id, local.session) if submission is None: raise NotFound() # Load the dataset. if dataset_id is not None: dataset = Dataset.get_from_id(dataset_id, local.session) if dataset is None: raise NotFound() else: q = local.session.query(Dataset) q = q.join(Task, Dataset.id == Task.active_dataset_id) q = q.filter(Task.id == submission.task_id) dataset = q.one() # Get the result (will fire a query). submission_result = submission.get_result(dataset) # Get the ScoreType (will fire a query for testcases). score_type = get_score_type(dataset=dataset) # Produce the data structure. s = submission sr = submission_result result = { '_ref': "%s" % s.id, 'dataset': '%s' % dataset.id, 'user': "******" % s.user_id, 'task': "%s" % s.task_id, 'timestamp': make_timestamp(s.timestamp), 'language': s.language, # No files, no token: AWS doesn't need them. } if sr is not None: result.update({ 'compilation_outcome': {"ok": True, "fail": False}.get(sr.compilation_outcome), 'compilation_text': format_status_text(sr.compilation_text), 'compilation_tries': sr.compilation_tries, 'compilation_stdout': sr.compilation_stdout, 'compilation_stderr': sr.compilation_stderr, 'compilation_time': sr.compilation_time, 'compilation_wall_clock_time': sr.compilation_wall_clock_time, 'compilation_memory': sr.compilation_memory, 'compilation_shard': sr.compilation_shard, 'compilation_sandbox': sr.compilation_sandbox, 'evaluation_outcome': {"ok": True}.get(sr.evaluation_outcome), 'evaluation_tries': sr.evaluation_tries, 'evaluations': dict((ev.codename, { 'codename': ev.codename, 'outcome': ev.outcome, 'text': format_status_text(ev.text), 'execution_time': ev.execution_time, 'execution_wall_clock_time': ev.execution_wall_clock_time, 'execution_memory': ev.execution_memory, 'evaluation_shard': ev.evaluation_shard, 'evaluation_sandbox': ev.evaluation_sandbox, }) for ev in sr.evaluations), 'score': sr.score, 'max_score': score_type.max_score, 'score_details': score_type.get_html_details(sr.score_details) if sr.score is not None else None, }) else: # Just copy all fields with None. result.update({ 'compilation_outcome': None, 'compilation_text': None, 'compilation_tries': 0, 'compilation_stdout': None, 'compilation_stderr': None, 'compilation_time': None, 'compilation_wall_clock_time': None, 'compilation_memory': None, 'compilation_shard': None, 'compilation_sandbox': None, 'evaluation_outcome': None, 'evaluation_tries': 0, 'evaluations': {}, 'score': None, 'max_score': score_type.max_score, 'score_details': None, }) # Encode and send. local.response.mimetype = "application/json" local.response.data = json.dumps(result)
def execute(self, entry): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. entry (QueueEntry): entry containing the operation to perform. """ operation = entry.item with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(operation.submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % operation.submission_id) # Obtain dataset. dataset = Dataset.get_from_id(operation.dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % operation.dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError( "Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id)) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info("Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id) return else: raise ValueError( "The state of the submission result " "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id)) # For Codebreaker, your score depends on your previous submissions # to this task. So, let's get the previous submisisons for this task previous_submissions = session.query(Submission)\ .filter(Submission.user_id == submission.user_id, Submission.task_id == submission.task_id)\ .order_by(asc(Submission.timestamp))\ .all() # Counterintuitively, because we're nice people, we don't care how # these submissions were scored. We only care about their # evaluations, which will tell us how to score them. # For a codebreaker, this will be in one-to-one correspondence with # previous submissions, since each "task" should only have the one # "testcase". previous_evaluations = [ session.query(Evaluation).filter( Evaluation.submission_id == sub.id).first() for sub in previous_submissions ] assert (len(previous_evaluations) == len(previous_submissions)) # Now that we have the evaluations, we can pass these as parameters # to our score type params = [ evaluation.outcome for evaluation in previous_evaluations ] # Instantiate the score type. # We don't want to use the dataset since we have to pass in custom # params. Instead we'll just hardcode the name of the class in, # which is unfortunate. # TODO (bgbn): work out a way to make this more generic. score_type = get_score_type( name="AIOCCodebreakerScoreType", parameters=json.dumps(params), public_testcases=dict( (k, tc.public) for k, tc in dataset.testcases.iteritems())) # Compute score and fill it in the database. submission_result.score, \ submission_result.score_details, \ submission_result.public_score, \ submission_result.public_score_details, \ submission_result.ranking_score_details = \ score_type.compute_score(submission_result) # Store it. session.commit() # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: self.proxy_service.submission_scored( submission_id=submission.id)
def action_finished(self, data, plus, error=None): """Callback from a worker, to signal that is finished some action (compilation or evaluation). data (dict): a dictionary that describes a Job instance. plus (tuple): the tuple (type_, object_id, dataset_id, testcase_codename, side_data=(priority, timestamp), shard_of_worker) """ # Unpack the plus tuple. It's built in the RPC call to Worker's # execute_job method inside WorkerPool.acquire_worker. type_, object_id, dataset_id, testcase_codename, _, \ shard = plus # Restore operation from its fields. operation = ESOperation( type_, object_id, dataset_id, testcase_codename) # We notify the pool that the worker is available again for # further work (no matter how the current request turned out, # even if the worker encountered an error). If the pool # informs us that the data produced by the worker has to be # ignored (by returning True) we interrupt the execution of # this method and do nothing because in that case we know the # operation has returned to the queue and perhaps already been # reassigned to another worker. if self.get_executor().pool.release_worker(shard): logger.info("Ignored result from worker %s as requested.", shard) return job_success = True if error is not None: logger.error("Received error from Worker: `%s'.", error) job_success = False else: try: job = Job.import_from_dict_with_type(data) except: logger.error("[action_finished] Couldn't build Job for " "data %s.", data, exc_info=True) job_success = False else: if not job.success: logger.error("Worker %s signaled action " "not successful.", shard) job_success = False logger.info("Operation `%s' for submission %s completed. Success: %s.", operation, object_id, job_success) # We get the submission from DB and update it. with SessionGen() as session: dataset = Dataset.get_from_id(dataset_id, session) if dataset is None: logger.error("[action_finished] Could not find " "dataset %d in the database.", dataset_id) return # TODO Try to move this 4-cases if-clause into a method of # ESOperation: I'd really like ES itself not to care about # which type of operation it's handling. if type_ == ESOperation.COMPILATION: submission = Submission.get_from_id(object_id, session) if submission is None: logger.error("[action_finished] Could not find " "submission %d in the database.", object_id) return submission_result = submission.get_result(dataset) if submission_result is None: logger.info("[action_finished] Couldn't find " "submission %d(%d) in the database. " "Creating it.", object_id, dataset_id) submission_result = \ submission.get_result_or_create(dataset) if job_success: job.to_submission(submission_result) else: submission_result.compilation_tries += 1 session.commit() self.compilation_ended(submission_result) elif type_ == ESOperation.EVALUATION: submission = Submission.get_from_id(object_id, session) if submission is None: logger.error("[action_finished] Could not find " "submission %d in the database.", object_id) return submission_result = submission.get_result(dataset) if submission_result is None: logger.error("[action_finished] Couldn't find " "submission %d(%d) in the database.", object_id, dataset_id) return if job_success: job.to_submission(submission_result) else: submission_result.evaluation_tries += 1 # Submission evaluation will be ended only when # evaluation for each testcase is available. evaluation_complete = (len(submission_result.evaluations) == len(dataset.testcases)) if evaluation_complete: submission_result.set_evaluation_outcome() session.commit() if evaluation_complete: self.evaluation_ended(submission_result) elif type_ == ESOperation.USER_TEST_COMPILATION: user_test = UserTest.get_from_id(object_id, session) if user_test is None: logger.error("[action_finished] Could not find " "user test %d in the database.", object_id) return user_test_result = user_test.get_result(dataset) if user_test_result is None: logger.error("[action_finished] Couldn't find " "user test %d(%d) in the database. " "Creating it.", object_id, dataset_id) user_test_result = \ user_test.get_result_or_create(dataset) if job_success: job.to_user_test(user_test_result) else: user_test_result.compilation_tries += 1 session.commit() self.user_test_compilation_ended(user_test_result) elif type_ == ESOperation.USER_TEST_EVALUATION: user_test = UserTest.get_from_id(object_id, session) if user_test is None: logger.error("[action_finished] Could not find " "user test %d in the database.", object_id) return user_test_result = user_test.get_result(dataset) if user_test_result is None: logger.error("[action_finished] Couldn't find " "user test %d(%d) in the database.", object_id, dataset_id) return if job_success: job.to_user_test(user_test_result) else: user_test_result.evaluation_tries += 1 session.commit() self.user_test_evaluation_ended(user_test_result) else: logger.error("Invalid operation type %r.", type_) return
def write_results(self, items): """Receive worker results from the cache and writes them to the DB. Grouping results together by object (i.e., submission result or user test result) and type (compilation or evaluation) allows this method to talk less to the DB, for example by retrieving datasets and submission results only once instead of once for every result. items ([(operation, Result)]): the results received by ES but not yet written to the db. """ logger.info("Starting commit process...") # Reorganize the results by submission/usertest result and # operation type (i.e., group together the testcase # evaluations for the same submission and dataset). by_object_and_type = defaultdict(list) for operation, result in items: t = (operation.type_, operation.object_id, operation.dataset_id) by_object_and_type[t].append((operation, result)) with SessionGen() as session: # Dictionary holding the objects we use repeatedly, # indexed by id, to avoid querying them multiple times. # TODO: this pattern is used in WorkerPool and should be # abstracted away. datasets = dict() subs = dict() srs = dict() for key, operation_results in iteritems(by_object_and_type): type_, object_id, dataset_id = key # Get dataset. if dataset_id not in datasets: datasets[dataset_id] = session.query(Dataset)\ .filter(Dataset.id == dataset_id)\ .options(joinedload(Dataset.testcases))\ .first() dataset = datasets[dataset_id] if dataset is None: logger.error("Could not find dataset %d in the database.", dataset_id) continue # Get submission or user test, and their results. if type_ in [ESOperation.COMPILATION, ESOperation.EVALUATION]: if object_id not in subs: subs[object_id] = \ Submission.get_from_id(object_id, session) object_ = subs[object_id] if object_ is None: logger.error( "Could not find submission %d " "in the database.", object_id) continue result_id = (object_id, dataset_id) if result_id not in srs: srs[result_id] = object_.get_result_or_create(dataset) object_result = srs[result_id] else: # We do not cache user tests as they can come up # only once. object_ = UserTest.get_from_id(object_id, session) object_result = object_.get_result_or_create(dataset) self.write_results_one_object_and_type(session, object_result, operation_results) logger.info("Committing evaluations...") session.commit() for type_, object_id, dataset_id in by_object_and_type: if type_ == ESOperation.EVALUATION: submission_result = srs[(object_id, dataset_id)] dataset = datasets[dataset_id] if len(submission_result.evaluations) == \ len(dataset.testcases): submission_result.set_evaluation_outcome() logger.info("Committing evaluation outcomes...") session.commit() logger.info("Ending operations for %s objects...", len(by_object_and_type)) for type_, object_id, dataset_id in by_object_and_type: if type_ == ESOperation.COMPILATION: submission_result = srs[(object_id, dataset_id)] self.compilation_ended(submission_result) elif type_ == ESOperation.EVALUATION: submission_result = srs[(object_id, dataset_id)] if submission_result.evaluated(): self.evaluation_ended(submission_result) elif type_ == ESOperation.USER_TEST_COMPILATION: user_test_result = UserTest\ .get_from_id(object_id, session)\ .get_result(datasets[dataset_id]) self.user_test_compilation_ended(user_test_result) elif type_ == ESOperation.USER_TEST_EVALUATION: user_test_result = UserTest\ .get_from_id(object_id, session)\ .get_result(datasets[dataset_id]) self.user_test_evaluation_ended(user_test_result) logger.info("Done")
def execute(self, entry): """Assign a score to a submission result. This is the core of ScoringService: here we retrieve the result from the database, check if it is in the correct status, instantiate its ScoreType, compute its score, store it back in the database and tell ProxyService to update RWS if needed. entry (QueueEntry): entry containing the operation to perform. """ operation = entry.item with SessionGen() as session: # Obtain submission. submission = Submission.get_from_id(operation.submission_id, session) if submission is None: raise ValueError("Submission %d not found in the database." % operation.submission_id) # Obtain dataset. dataset = Dataset.get_from_id(operation.dataset_id, session) if dataset is None: raise ValueError("Dataset %d not found in the database." % operation.dataset_id) # Obtain submission result. submission_result = submission.get_result(dataset) # It means it was not even compiled (for some reason). if submission_result is None: raise ValueError( "Submission result %d(%d) was not found." % (operation.submission_id, operation.dataset_id)) # Check if it's ready to be scored. if not submission_result.needs_scoring(): if submission_result.scored(): logger.info("Submission result %d(%d) is already scored.", operation.submission_id, operation.dataset_id) return else: raise ValueError( "The state of the submission result " "%d(%d) doesn't allow scoring." % (operation.submission_id, operation.dataset_id)) # Instantiate the score type. score_type = get_score_type(dataset=dataset) # Compute score and fill it in the database. submission_result.score, \ submission_result.score_details, \ submission_result.public_score, \ submission_result.public_score_details, \ ranking_score_details = \ score_type.compute_score(submission_result) submission_result.ranking_score_details = \ json.dumps(ranking_score_details) task = submission.task participation = submission.participation relevant_submissions = session.query(SubmissionResult)\ .join(SubmissionResult.submission)\ .filter(Submission.participation_id == participation.id)\ .filter(Submission.task_id == task.id) \ .filter(SubmissionResult.dataset_id == dataset.id) \ .filter(SubmissionResult.filter_scored())\ .order_by(Submission.timestamp.asc())\ .all() changed_task_results = [] official_submissions = [ s for s in relevant_submissions if s.submission.official ] official_ptr = 0 for i in range(len(relevant_submissions)): sr = relevant_submissions[i] if official_ptr < len(official_submissions) and \ sr == official_submissions[official_ptr]: official_ptr += 1 if sr.submission.timestamp >= submission.timestamp: old_data = (sr.task_score, sr.task_score_details, sr.task_public_score, sr.task_public_score_details, sr.task_ranking_score_details) new_data = score_type.\ compute_total_score( official_submissions[:official_ptr] ) new_data = new_data[:4] + (json.dumps(new_data[4]), ) if old_data != new_data: sr.task_score, \ sr.task_score_details, \ sr.task_public_score, \ sr.task_public_score_details, \ sr.task_ranking_score_details = \ new_data changed_task_results.append(sr.submission_id) # Store it. session.commit() logger.metric("submission_scoring_time", submission_id=submission.id, dataset_id=submission_result.dataset_id, language=submission.language, task=submission.task_id, participant=submission.participation_id, value=(make_datetime() - submission.timestamp).total_seconds()) logger.info("Submission scored %d seconds after submission", (make_datetime() - submission.timestamp).total_seconds()) # If dataset is the active one, update RWS. if dataset is submission.task.active_dataset: if submission.id not in changed_task_results: logger.error("Submission was recently scored but " "it isn't listed as submissions with " "a task score change") changed_task_results.append(submission.id) for changed_submission_id in changed_task_results: self.proxy_service.submission_scored( submission_id=changed_submission_id)