def enqueue(self, operation, priority, timestamp, job=None): """Push an operation in the queue. Push an operation in the operation queue if the submission is not already in the queue or assigned to a worker. operation (ESOperation): the operation to put in the queue. priority (int): the priority of the operation. timestamp (datetime): the time of the submission. job (dict|None): the job associated; if None will be computed return (bool): True if pushed, False if not. """ if job is None: with SessionGen() as session: dataset = Dataset.get_from_id(operation.dataset_id, session) if operation.for_submission(): object_ = Submission.get_from_id(operation.object_id, session) else: object_ = UserTest.get_from_id(operation.object_id, session) job = Job.from_operation(operation, object_, dataset).export_to_dict() return self.queue_service.enqueue( operation=operation.to_list(), priority=priority, timestamp=(timestamp - EvaluationService.EPOCH).total_seconds(), job=job)
def acquire_worker(self, operations): """Tries to assign an operation to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. operations ([ESOperation]): the operations to assign to a worker. return (int|None): None if no workers are available, the worker assigned to the operation otherwise. """ # We look for an available worker. try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: self._workers_available_event.clear() return None # Then we fill the info for future memory. self._add_operations(shard, operations) logger.debug("Worker %s acquired.", shard) self._start_time[shard] = make_datetime() with SessionGen() as session: jobs = [] datasets = {} submissions = {} user_tests = {} for operation in operations: if operation.dataset_id not in datasets: datasets[operation.dataset_id] = Dataset.get_from_id( operation.dataset_id, session) object_ = None if operation.for_submission(): if operation.object_id not in submissions: submissions[operation.object_id] = \ Submission.get_from_id( operation.object_id, session) object_ = submissions[operation.object_id] else: if operation.object_id not in user_tests: user_tests[operation.object_id] = \ UserTest.get_from_id(operation.object_id, session) object_ = user_tests[operation.object_id] logger.info("Asking worker %s to `%s'.", shard, operation) jobs.append( Job.from_operation(operation, object_, datasets[operation.dataset_id])) job_group_dict = JobGroup(jobs).export_to_dict() self._worker[shard].execute_job_group( job_group_dict=job_group_dict, callback=self._service.action_finished, plus=shard) return shard
def acquire_worker(self, operations): """Tries to assign an operation to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. operations ([ESOperation]): the operations to assign to a worker. return (int|None): None if no workers are available, the worker assigned to the operation otherwise. """ # We look for an available worker. try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: self._workers_available_event.clear() return None # Then we fill the info for future memory. self._add_operations(shard, operations) logger.debug("Worker %s acquired.", shard) self._start_time[shard] = make_datetime() with SessionGen() as session: jobs = [] datasets = {} submissions = {} user_tests = {} for operation in operations: if operation.dataset_id not in datasets: datasets[operation.dataset_id] = Dataset.get_from_id( operation.dataset_id, session) object_ = None if operation.for_submission(): if operation.object_id not in submissions: submissions[operation.object_id] = \ Submission.get_from_id( operation.object_id, session) object_ = submissions[operation.object_id] else: if operation.object_id not in user_tests: user_tests[operation.object_id] = \ UserTest.get_from_id(operation.object_id, session) object_ = user_tests[operation.object_id] logger.info("Asking worker %s to `%s'.", shard, operation) jobs.append(Job.from_operation( operation, object_, datasets[operation.dataset_id])) job_group_dict = JobGroup(jobs).export_to_dict() self._worker[shard].execute_job_group( job_group_dict=job_group_dict, callback=self._service.action_finished, plus=shard) return shard
def get_user_test_operations(self, user_test): """Push in queue the operations required by a user test. user_test (UserTest): a user test. return ([ESOperation, int, datetime]): operations to enqueue, together with priority and timestamp. """ operations = [] for dataset in get_datasets_to_judge(user_test.task): for operation, priority, timestamp in user_test_get_operations( user_test, dataset): job = Job.from_operation(operation, user_test, dataset).export_to_dict() operations.append([operation, priority, timestamp, job]) return operations
def get_submission_operations(self, submission, dataset=None): """Push in queue the operations required by a submission. submission (Submission): a submission. return ([ESOperation, int, datetime]): operations to enqueue, together with priority and timestamp. """ operations = [] if dataset is None: datasets = get_datasets_to_judge(submission.task) else: datasets = [dataset] for dataset in datasets: submission_result = submission.get_result(dataset) number_of_operations = 0 for operation, priority, timestamp in submission_get_operations( submission_result, submission, dataset): number_of_operations += 1 job = Job.from_operation(operation, submission, dataset).export_to_dict() operations.append([operation, priority, timestamp, job]) # If we got 0 operations, but the submission result is to # evaluate, it means that we just need to finalize the # evaluation. if number_of_operations == 0 and submission_to_evaluate( submission_result): logger.info( "Result %d(%d) has already all evaluations, " "finalizing it.", submission.id, dataset.id) submission_result.set_evaluation_outcome() submission_result.sa_session.commit() self.evaluation_ended(submission_result) return operations