def build_job(self, session): """Produce the Job for this operation. Return the Job object that has to be sent to Workers to have them perform the operation this object describes. session (Session): the database session to use to fetch objects if necessary. return (Job): the job encoding of the operation, as understood by Workers and TaskTypes. """ result = None dataset = Dataset.get_from_id(self.dataset_id, session) if self.type_ == ESOperation.COMPILATION: submission = Submission.get_from_id(self.object_id, session) result = CompilationJob.from_submission(submission, dataset) elif self.type_ == ESOperation.EVALUATION: submission = Submission.get_from_id(self.object_id, session) result = EvaluationJob.from_submission(submission, dataset, self.testcase_codename) elif self.type_ == ESOperation.USER_TEST_COMPILATION: user_test = UserTest.get_from_id(self.object_id, session) result = CompilationJob.from_user_test(user_test, dataset) elif self.type_ == ESOperation.USER_TEST_EVALUATION: user_test = UserTest.get_from_id(self.object_id, session) result = EvaluationJob.from_user_test(user_test, dataset) return result
def build_job(self, session): """Produce the Job for this operation. Return the Job object that has to be sent to Workers to have them perform the operation this object describes. session (Session): the database session to use to fetch objects if necessary. return (Job): the job encoding of the operation, as understood by Workers and TaskTypes. """ result = None dataset = Dataset.get_from_id(self.dataset_id, session) if self.type_ == ESOperation.COMPILATION: submission = Submission.get_from_id(self.object_id, session) result = CompilationJob.from_submission(submission, dataset) elif self.type_ == ESOperation.EVALUATION: submission = Submission.get_from_id(self.object_id, session) result = EvaluationJob.from_submission( submission, dataset, self.testcase_codename) elif self.type_ == ESOperation.USER_TEST_COMPILATION: user_test = UserTest.get_from_id(self.object_id, session) result = CompilationJob.from_user_test(user_test, dataset) elif self.type_ == ESOperation.USER_TEST_EVALUATION: user_test = UserTest.get_from_id(self.object_id, session) result = EvaluationJob.from_user_test(user_test, dataset) return result
def acquire_worker(self, job, side_data=None): """Tries to assign a job to an available worker. If no workers are available then this returns None, otherwise this returns the chosen worker. job (job): the job to assign to a worker side_data (object): object to attach to the worker for later use returns (int): None if no workers are available, the worker assigned to the job otherwise """ # We look for an available worker try: shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True) except LookupError: return None # Then we fill the info for future memory self._job[shard] = job self._start_time[shard] = make_datetime() self._side_data[shard] = side_data logger.debug("Worker %s acquired." % shard) # And finally we ask the worker to do the job action, object_id = job timestamp = side_data[1] queue_time = self._start_time[shard] - timestamp logger.info( "Asking worker %s to %s submission/user test %d " " (%s after submission)." % (shard, action, object_id, queue_time) ) with SessionGen(commit=False) as session: if action == EvaluationService.JOB_TYPE_COMPILATION: submission = Submission.get_from_id(object_id, session) job_ = CompilationJob.from_submission(submission) elif action == EvaluationService.JOB_TYPE_EVALUATION: submission = Submission.get_from_id(object_id, session) job_ = EvaluationJob.from_submission(submission) elif action == EvaluationService.JOB_TYPE_TEST_COMPILATION: user_test = UserTest.get_from_id(object_id, session) job_ = CompilationJob.from_user_test(user_test) elif action == EvaluationService.JOB_TYPE_TEST_EVALUATION: user_test = UserTest.get_from_id(object_id, session) job_ = EvaluationJob.from_user_test(user_test) job_.get_output = True job_.only_execution = True self._worker[shard].execute_job( job_dict=job_.export_to_dict(), callback=self._service.action_finished.im_func, plus=(action, object_id, side_data, shard), ) return shard
def job(files): operation = ESOperation(ESOperation.EVALUATION, 1, 1, "023") return EvaluationJob(input="digest of input", output="digest of correct output", files=files, operation=operation, multithreaded_sandbox=True)
def job(executables): return EvaluationJob(language="L1", input="digest of input", output="digest of correct output", time_limit=2.5, memory_limit=123, executables=executables, multithreaded_sandbox=True)
def new_jobgroup(number_of_jobs, prefix=None): prefix = prefix if prefix is not None else "" jobgroup_dict = {} calls = [] for i in xrange(number_of_jobs): job_params = ("fake_task_type", "fake_parameters_%s" % i) job = EvaluationJob(*job_params, info="%s%d" % (prefix, i)) jobgroup_dict["%s" % i] = job calls.append(call(*job_params)) return JobGroup(jobgroup_dict), calls
def debugSubmission(submission_id, dataset_id, testcase_codename): config.keep_sandbox = True file_cacher = FileCacher() with SessionGen() as session: submission = session.query(Submission)\ .filter(Submission.id == submission_id)\ .first() if submission is None: logger.error("There's no submission with id %d" % submission_id) return False if dataset_id is None: dataset = submission.task.active_dataset dataset_id = submission.task.active_dataset_id else: dataset = session.query(Dataset)\ .filter(Dataset.id == dataset_id)\ .first() # Compilation operation = ESOperation(ESOperation.COMPILATION, submission_id, dataset_id) comp_job = CompilationJob.from_submission(operation, submission, dataset) task_type = get_task_type(comp_job.task_type, comp_job.task_type_parameters) task_type.execute_job(comp_job, file_cacher) for sandbox_path in comp_job.sandboxes: logger.info("Compilation sandbox created in %s" % sandbox_path) # Check if the compilation is successful result = submission.get_result(dataset) if result is None or result.compilation_failed(): logger.error("Compilatoin Failed") return True # Evaluation operation = ESOperation(ESOperation.EVALUATION, submission_id, dataset_id, testcase_codename) eval_job = EvaluationJob.from_submission(operation, submission, dataset) task_type = get_task_type(eval_job.task_type, eval_job.task_type_parameters) task_type.execute_job(eval_job, file_cacher) for sandbox_path in eval_job.sandboxes: logger.info("Evaluation sandbox created in %s" % sandbox_path) return True
def new_jobs(number_of_jobs, prefix=None): prefix = prefix if prefix is not None else "" jobs = [] calls = [] for i in xrange(number_of_jobs): job_params = ("fake_task_type", "fake_parameters_%s%d" % (prefix, i)) job = EvaluationJob(*job_params, info="%s%d" % (prefix, i)) jobs.append(job) calls.append(call(*job_params)) return jobs, calls
def new_jobs(number_of_jobs, prefix=None): prefix = prefix if prefix is not None else "" jobs = [] calls = [] for i in range(number_of_jobs): job_params = [ ESOperation(ESOperation.EVALUATION, unique_long_id(), unique_long_id(), unique_unicode_id()).to_dict(), "fake_task_type", "fake_parameters_%s%d" % (prefix, i) ] job = EvaluationJob(*job_params, info="%s%d" % (prefix, i)) jobs.append(job) # Arguments to get_task_type are the same as for the job, # but omitting the operation. calls.append(call(*job_params[1:])) return jobs, calls
def test_testcases(base_dir, soluzione, language, assume=None): global task, file_cacher # Use a disabled FileCacher with a FSBackend in order to avoid to fill # the database with junk and to save up space. if file_cacher is None: file_cacher = FileCacher(path=os.path.join(config.cache_dir, 'cmsMake'), enabled=False) # Load the task if task is None: loader = YamlLoader(os.path.realpath(os.path.join(base_dir, "..")), file_cacher) # Normally we should import the contest before, but YamlLoader # accepts get_task() even without previous get_contest() calls task = loader.get_task(os.path.split(os.path.realpath(base_dir))[1]) # Prepare the EvaluationJob dataset = task.active_dataset if dataset.task_type != "OutputOnly": digest = file_cacher.put_file_from_path( os.path.join(base_dir, soluzione), "Solution %s for task %s" % (soluzione, task.name)) executables = { task.name: Executable(filename=task.name, digest=digest) } jobs = [(t, EvaluationJob(language=language, task_type=dataset.task_type, task_type_parameters=json.loads( dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] tasktype = get_task_type(dataset=dataset) else: print("Generating outputs...", end='') files = {} for t in sorted(dataset.testcases.keys()): with file_cacher.get_file(dataset.testcases[t].input) as fin: with TemporaryFile() as fout: print(str(t), end='') call(soluzione, stdin=fin, stdout=fout, cwd=base_dir) fout.seek(0) digest = file_cacher.put_file_from_fobj(fout) outname = "output_%s.txt" % t files[outname] = File(filename=outname, digest=digest) jobs = [(t, EvaluationJob(task_type=dataset.task_type, task_type_parameters=json.loads( dataset.task_type_parameters), managers=dict(dataset.managers), files=files, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] for k, job in jobs: job._key = k tasktype = get_task_type(dataset=dataset) print() ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0], end='') sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) if dataset.task_type != "OutputOnly": status = job.plus["exit_status"] info.append("Time: %5.3f Wall: %5.3f Memory: %s" % (job.plus["execution_time"], job.plus["execution_wall_clock_time"], mem_human(job.plus["execution_memory"]))) else: status = "ok" info.append("N/A") points.append(float(job.outcome)) comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print() print("Want to stop and consider everything to timeout? [y/N]", end='') if assume is not None: print(assume) tmp = assume else: tmp = raw_input().lower() if tmp in ['y', 'yes']: stop = True else: ask_again = False # Result pretty printing print() clen = max(len(c) for c in comments) ilen = max(len(i) for i in info) for (i, p, c, b) in zip(tcnames, points, comments, info): print("%s) %5.2lf --- %s [%s]" % (i, p, c.ljust(clen), b.center(ilen))) return zip(points, comments, info)
def test_testcases(base_dir, solution, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) cmscontrib.loaders.italy_yaml.logger = NullLogger() # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = cmscontrib.loaders.italy_yaml.YamlLoader( base_dir, file_cacher) task = loader.get_task(get_statement=False) # Prepare the EvaluationJob dataset = task.active_dataset digest = file_cacher.put_file_from_path( os.path.join(base_dir, solution), "Solution %s for task %s" % (solution, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [ (t, EvaluationJob( operation=ESOperation(ESOperation.EVALUATION, None, dataset.id, dataset.testcases[t].codename).to_dict(), language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases ] tasktype = get_task_type(dataset=dataset) ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0]) sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") move_cursor(directions.UP, erase=True) continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) status = job.plus.get("exit_status") info.append( (job.plus.get("execution_time"), job.plus.get("execution_memory"))) points.append(float(job.outcome)) # Avoid printing unneeded newline job.text = [t.rstrip() for t in job.text] comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print("Want to stop and consider everything to timeout? [y/N] ", end='') sys.stdout.flush() if assume is not None: tmp = assume print(tmp) else: # User input with a timeout of 5 seconds, at the end of which # we automatically say "n". ready will be a list of input ready # for reading, or an empty list if the timeout expired. # See: http://stackoverflow.com/a/2904057 ready, _, _ = select.select([sys.stdin], [], [], 5) if ready: tmp = sys.stdin.readline().strip().lower() else: tmp = 'n' print(tmp) if tmp in ['y', 'yes']: stop = True else: ask_again = False print() move_cursor(directions.UP, erase=True) # Subtasks scoring subtasks = json.loads(dataset.score_type_parameters) if not isinstance(subtasks, list) or len(subtasks) == 0: subtasks = [[100, len(info)]] if dataset.score_type == 'GroupMin': scoreFun = min else: if dataset.score_type != 'Sum': logger.warning("Score type %s not yet supported! Using Sum" % dataset.score_type) def scoreFun(x): return sum(x) / len(x) pos = 0 sts = [] # For each subtask generate a list of testcase it owns, the score gained # and the highest time and memory usage. for i in subtasks: stscores = [] stsdata = [] worst = [0, 0] try: for _ in xrange(i[1]): stscores.append(points[pos]) stsdata.append( (tcnames[pos], points[pos], comments[pos], info[pos])) if info[pos][0] > worst[0]: worst[0] = info[pos][0] if info[pos][1] > worst[1]: worst[1] = info[pos][1] pos += 1 sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst)) except: sts.append((0, i[0], stsdata, [0, 0])) # Result pretty printing # Strips sol/ and _EVAL from the solution's name solution = solution[4:-5] print() clen = max(len(c) for c in comments) for st, d in enumerate(sts): print( "Subtask %d:" % st, add_color_to_string( "%5.2f/%d" % (d[0], d[1]), colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN, bold=True)) for (i, p, c, w) in d[2]: print("%s)" % i, add_color_to_string( "%5.2lf" % p, colors.RED if abs(p - 1) > 0.01 else colors.BLACK), "--- %s [Time:" % c.ljust(clen), add_color_to_string( ("%5.3f" % w[0]) if w[0] is not None else "N/A", colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0] else colors.BLACK), "Memory:", add_color_to_string( "%5s" % mem_human(w[1]) if w[1] is not None else "N/A", colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1] else colors.BLACK, ), end="]") move_cursor(directions.RIGHT, 1000) move_cursor(directions.LEFT, len(solution) - 1) print(add_color_to_string(solution, colors.BLACK, bold=True)) print() sols.append((solution, sum([st[0] for st in sts]))) global tested_something if not tested_something: tested_something = True atexit.register(print_at_exit) return zip(points, comments, info)
def test_testcases(base_dir, soluzione, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = YamlLoader( os.path.realpath(os.path.join(base_dir, "..")), file_cacher) # Normally we should import the contest before, but YamlLoader # accepts get_task() even without previous get_contest() calls task = loader.get_task(os.path.split(os.path.realpath(base_dir))[1]) # Prepare the EvaluationJob dataset = task.active_dataset digest = file_cacher.put_file_from_path( os.path.join(base_dir, soluzione), "Solution %s for task %s" % (soluzione, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [(t, EvaluationJob( language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] tasktype = get_task_type(dataset=dataset) ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0], end='') sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) status = job.plus["exit_status"] info.append("Time: %5.3f Wall: %5.3f Memory: %s" % (job.plus["execution_time"], job.plus["execution_wall_clock_time"], mem_human(job.plus["execution_memory"]))) points.append(float(job.outcome)) comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print() print("Want to stop and consider everything to timeout? [y/N]", end='') if assume is not None: print(assume) tmp = assume else: tmp = raw_input().lower() if tmp in ['y', 'yes']: stop = True else: ask_again = False # Result pretty printing print() clen = max(len(c) for c in comments) ilen = max(len(i) for i in info) for (i, p, c, b) in zip(tcnames, points, comments, info): print("%s) %5.2lf --- %s [%s]" % (i, p, c.ljust(clen), b.center(ilen))) return zip(points, comments, info)