def job(files):
operation = ESOperation(ESOperation.EVALUATION, 1, 1, "023")
return EvaluationJob(input="digest of input",
output="digest of correct output",
files=files,
operation=operation.to_dict(),
multithreaded_sandbox=True)
def debugSubmission(submission_id, dataset_id, testcase_codename):
config.keep_sandbox = True
file_cacher = FileCacher()
with SessionGen() as session:
submission = session.query(Submission)\
.filter(Submission.id == submission_id)\
.first()
if submission is None:
logger.error("There's no submission with id %d" % submission_id)
return False
if dataset_id is None:
dataset = submission.task.active_dataset
dataset_id = submission.task.active_dataset_id
else:
dataset = session.query(Dataset)\
.filter(Dataset.id == dataset_id)\
.first()
# Compilation
operation = ESOperation(ESOperation.COMPILATION, submission_id,
dataset_id)
comp_job = CompilationJob.from_submission(operation, submission,
dataset)
task_type = get_task_type(comp_job.task_type,
comp_job.task_type_parameters)
task_type.execute_job(comp_job, file_cacher)
for sandbox_path in comp_job.sandboxes:
logger.info("Compilation sandbox created in %s" % sandbox_path)
# Check if the compilation is successful
result = submission.get_result(dataset)
if result is None or result.compilation_failed():
logger.error("Compilatoin Failed")
return True
# Evaluation
operation = ESOperation(ESOperation.EVALUATION, submission_id,
dataset_id, testcase_codename)
eval_job = EvaluationJob.from_submission(operation, submission,
dataset)
task_type = get_task_type(eval_job.task_type,
eval_job.task_type_parameters)
task_type.execute_job(eval_job, file_cacher)
for sandbox_path in eval_job.sandboxes:
logger.info("Evaluation sandbox created in %s" % sandbox_path)
return True
def user_test_evaluation_operation(self, result):
active_priority = PriorityQueue.PRIORITY_MEDIUM \
if result.evaluation_tries == 0 else PriorityQueue.PRIORITY_LOW
return (ESOperation(ESOperation.USER_TEST_EVALUATION,
result.user_test.id, result.dataset.id),
active_priority if result.dataset.active else
PriorityQueue.PRIORITY_EXTRA_LOW, result.user_test.timestamp)
def user_test_compilation_operation(self, user_test, dataset, result=None):
active_priority = PriorityQueue.PRIORITY_HIGH \
if result is None or result.compilation_tries == 0 \
else PriorityQueue.PRIORITY_MEDIUM
return (ESOperation(ESOperation.USER_TEST_COMPILATION, user_test.id,
dataset.id), active_priority if dataset.active else
PriorityQueue.PRIORITY_EXTRA_LOW, user_test.timestamp)
def submission_evaluation_operation(self, result, codename):
active_priority = PriorityQueue.PRIORITY_MEDIUM \
if result.evaluation_tries == 0 else PriorityQueue.PRIORITY_LOW
return (ESOperation(ESOperation.EVALUATION, result.submission.id,
result.dataset.id, codename), active_priority
if result.dataset.active else PriorityQueue.PRIORITY_EXTRA_LOW,
result.submission.timestamp)
def submission_compilation_operation(self,
submission,
dataset,
result=None):
active_priority = PriorityQueue.PRIORITY_HIGH \
if result is None or result.compilation_tries == 0 \
else PriorityQueue.PRIORITY_MEDIUM
return (ESOperation(ESOperation.COMPILATION, submission.id,
dataset.id), active_priority if dataset.active else
PriorityQueue.PRIORITY_EXTRA_LOW, submission.timestamp)
def import_from_dict(cls, data):
"""Create a Job from the output of export_to_dict."""
if data['operation'] is not None:
data['operation'] = ESOperation.from_dict(data['operation'])
data['files'] = dict((k, File(k, v)) for k, v in data['files'].items())
data['managers'] = dict(
(k, Manager(k, v)) for k, v in data['managers'].items())
data['executables'] = dict(
(k, Executable(k, v)) for k, v in data['executables'].items())
return cls(**data)
def new_jobs(number_of_jobs, prefix=None):
prefix = prefix if prefix is not None else ""
jobs = []
calls = []
for i in range(number_of_jobs):
job_params = [
ESOperation(ESOperation.EVALUATION, unique_long_id(),
unique_long_id(), unique_unicode_id()).to_dict(),
"fake_task_type",
"fake_parameters_%s%d" % (prefix, i)
]
job = EvaluationJob(*job_params, info="%s%d" % (prefix, i))
jobs.append(job)
# Arguments to get_task_type are the same as for the job,
# but omitting the operation.
calls.append(call(*job_params[1:]))
return jobs, calls
def test_testcases(base_dir, solution, language, assume=None):
global task, file_cacher
# Use a FileCacher with a NullBackend in order to avoid to fill
# the database with junk
if file_cacher is None:
file_cacher = FileCacher(null=True)
cmscontrib.loaders.italy_yaml.logger = NullLogger()
# Load the task
# TODO - This implies copying a lot of data to the FileCacher,
# which is annoying if you have to do it continuously; it would be
# better to use a persistent cache (although local, possibly
# filesystem-based instead of database-based) and somehow detect
# when the task has already been loaded
if task is None:
loader = cmscontrib.loaders.italy_yaml.YamlLoader(
base_dir, file_cacher)
task = loader.get_task(get_statement=False)
# Prepare the EvaluationJob
dataset = task.active_dataset
digest = file_cacher.put_file_from_path(
os.path.join(base_dir, solution),
"Solution %s for task %s" % (solution, task.name))
executables = {task.name: Executable(filename=task.name, digest=digest)}
jobs = [
(t,
EvaluationJob(
operation=ESOperation(ESOperation.EVALUATION, None, dataset.id,
dataset.testcases[t].codename).to_dict(),
language=language,
task_type=dataset.task_type,
task_type_parameters=json.loads(dataset.task_type_parameters),
managers=dict(dataset.managers),
executables=executables,
input=dataset.testcases[t].input,
output=dataset.testcases[t].output,
time_limit=dataset.time_limit,
memory_limit=dataset.memory_limit)) for t in dataset.testcases
]
tasktype = get_task_type(dataset=dataset)
ask_again = True
last_status = "ok"
status = "ok"
stop = False
info = []
points = []
comments = []
tcnames = []
for jobinfo in sorted(jobs):
print(jobinfo[0])
sys.stdout.flush()
job = jobinfo[1]
# Skip the testcase if we decide to consider everything to
# timeout
if stop:
info.append("Time limit exceeded")
points.append(0.0)
comments.append("Timeout.")
move_cursor(directions.UP, erase=True)
continue
# Evaluate testcase
last_status = status
tasktype.evaluate(job, file_cacher)
status = job.plus.get("exit_status")
info.append(
(job.plus.get("execution_time"), job.plus.get("execution_memory")))
points.append(float(job.outcome))
# Avoid printing unneeded newline
job.text = [t.rstrip() for t in job.text]
comments.append(format_status_text(job.text))
tcnames.append(jobinfo[0])
# If we saw two consecutive timeouts, ask wether we want to
# consider everything to timeout
if ask_again and status == "timeout" and last_status == "timeout":
print("Want to stop and consider everything to timeout? [y/N] ",
end='')
sys.stdout.flush()
if assume is not None:
tmp = assume
print(tmp)
else:
# User input with a timeout of 5 seconds, at the end of which
# we automatically say "n". ready will be a list of input ready
# for reading, or an empty list if the timeout expired.
# See: http://stackoverflow.com/a/2904057
ready, _, _ = select.select([sys.stdin], [], [], 5)
if ready:
tmp = sys.stdin.readline().strip().lower()
else:
tmp = 'n'
print(tmp)
if tmp in ['y', 'yes']:
stop = True
else:
ask_again = False
print()
move_cursor(directions.UP, erase=True)
# Subtasks scoring
subtasks = json.loads(dataset.score_type_parameters)
if not isinstance(subtasks, list) or len(subtasks) == 0:
subtasks = [[100, len(info)]]
if dataset.score_type == 'GroupMin':
scoreFun = min
else:
if dataset.score_type != 'Sum':
logger.warning("Score type %s not yet supported! Using Sum" %
dataset.score_type)
def scoreFun(x):
return sum(x) / len(x)
pos = 0
sts = []
# For each subtask generate a list of testcase it owns, the score gained
# and the highest time and memory usage.
for i in subtasks:
stscores = []
stsdata = []
worst = [0, 0]
try:
for _ in xrange(i[1]):
stscores.append(points[pos])
stsdata.append(
(tcnames[pos], points[pos], comments[pos], info[pos]))
if info[pos][0] > worst[0]:
worst[0] = info[pos][0]
if info[pos][1] > worst[1]:
worst[1] = info[pos][1]
pos += 1
sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst))
except:
sts.append((0, i[0], stsdata, [0, 0]))
# Result pretty printing
# Strips sol/ and _EVAL from the solution's name
solution = solution[4:-5]
print()
clen = max(len(c) for c in comments)
for st, d in enumerate(sts):
print(
"Subtask %d:" % st,
add_color_to_string(
"%5.2f/%d" % (d[0], d[1]),
colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN,
bold=True))
for (i, p, c, w) in d[2]:
print("%s)" % i,
add_color_to_string(
"%5.2lf" % p,
colors.RED if abs(p - 1) > 0.01 else colors.BLACK),
"--- %s [Time:" % c.ljust(clen),
add_color_to_string(
("%5.3f" % w[0]) if w[0] is not None else "N/A",
colors.BLUE if w[0] is not None
and w[0] >= 0.95 * d[3][0] else colors.BLACK),
"Memory:",
add_color_to_string(
"%5s" % mem_human(w[1]) if w[1] is not None else "N/A",
colors.BLUE if w[1] is not None
and w[1] >= 0.95 * d[3][1] else colors.BLACK,
),
end="]")
move_cursor(directions.RIGHT, 1000)
move_cursor(directions.LEFT, len(solution) - 1)
print(add_color_to_string(solution, colors.BLACK, bold=True))
print()
sols.append((solution, sum([st[0] for st in sts])))
global tested_something
if not tested_something:
tested_something = True
atexit.register(print_at_exit)
return zip(points, comments, info)
