def make_input(assume=None):
n = 0
try:
os.makedirs(input_dir)
except OSError:
pass
for (is_copy, line, st) in testcases:
print(
"Generating",
add_color_to_string("input # %d" % n, colors.BLACK,
stream=sys.stderr, bold=True),
file=sys.stderr
)
new_input = os.path.join(input_dir, 'input%d.txt' % (n))
if is_copy:
# Copy the file
print("> Copy input file from:", line)
copy_input = os.path.join(base_dir, line)
shutil.copyfile(copy_input, new_input)
else:
# Call the generator
with open(new_input, 'wb') as fout:
call(base_dir,
[gen_exe] + line.split(),
stdout=fout)
command = [validator_exe, new_input]
if st != 0:
command.append("%s" % st)
call(base_dir, command)
n += 1
for _ in range(3):
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_input(assume=None):
n = 0
try:
os.makedirs(input_dir)
except OSError:
pass
for (is_copy, line, st) in testcases:
print("Generating",
add_color_to_string("input # %d" % n,
colors.BLACK,
stream=sys.stderr,
bold=True),
file=sys.stderr)
new_input = os.path.join(input_dir, 'input%d.txt' % (n))
if is_copy:
# Copy the file
print("> Copy input file from:", line)
copy_input = os.path.join(base_dir, line)
shutil.copyfile(copy_input, new_input)
else:
# Call the generator
with open(new_input, 'wb') as fout:
call(base_dir, [gen_exe] + line.split(), stdout=fout)
command = [validator_exe, new_input]
if st != 0:
command.append("%s" % st)
call(base_dir, command)
n += 1
for _ in range(3):
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def compile_src(srcs, exe, for_evaluation, lang, assume=None):
if lang != 'pas' or len(srcs) == 1:
compilation_commands = get_compilation_commands(
lang, srcs, exe, for_evaluation=for_evaluation)
for command in compilation_commands:
call(base_dir, command)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
# When using Pascal with graders, file naming conventions
# require us to do a bit of trickery, i.e., performing the
# compilation in a separate temporary directory
else:
tempdir = tempfile.mkdtemp()
task_name = detect_task_name(base_dir)
new_srcs = [os.path.split(srcs[0])[1], '%s.pas' % (task_name)]
new_exe = os.path.split(srcs[1])[1][:-4]
shutil.copyfile(os.path.join(base_dir, srcs[0]),
os.path.join(tempdir, new_srcs[0]))
shutil.copyfile(os.path.join(base_dir, srcs[1]),
os.path.join(tempdir, new_srcs[1]))
lib_filename = '%slib.pas' % (task_name)
if os.path.exists(os.path.join(SOL_DIRNAME, lib_filename)):
shutil.copyfile(os.path.join(SOL_DIRNAME, lib_filename),
os.path.join(tempdir, lib_filename))
compilation_commands = get_compilation_commands(
lang, new_srcs, new_exe, for_evaluation=for_evaluation)
for command in compilation_commands:
call(tempdir, command)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
shutil.copyfile(os.path.join(tempdir, new_exe),
os.path.join(base_dir, exe))
shutil.copymode(os.path.join(tempdir, new_exe),
os.path.join(base_dir, exe))
shutil.rmtree(tempdir)
def compile_src(srcs, exe, for_evaluation, lang, assume=None):
# We put everything in a temporary directory to reproduce
# the same conditions that we have when compiling a
# submission.
tempdir = tempfile.mkdtemp()
try:
task_name = detect_task_name(base_dir)
grader_num = 1 if len(srcs) > 1 else 0
new_srcs = []
for grader in srcs[:grader_num]:
grader_name = os.path.basename(grader)
shutil.copyfile(os.path.join(base_dir, grader), os.path.join(tempdir, grader_name))
new_srcs.append(os.path.join(tempdir, grader_name))
# For now, we assume we only have one non-grader source.
source_name = task_name + LANGUAGE_TO_SOURCE_EXT_MAP[lang]
shutil.copyfile(os.path.join(base_dir, srcs[grader_num]), os.path.join(tempdir, source_name))
new_srcs.append(source_name)
# Libraries are needed/used only for C/C++ and Pascal
if lang in LANGUAGE_TO_HEADER_EXT_MAP:
lib_template = "%s" + LANGUAGE_TO_HEADER_EXT_MAP[lang]
lib_filename = lib_template % (task_name)
lib_path = os.path.join(base_dir, SOL_DIRNAME, lib_filename)
if os.path.exists(lib_path):
shutil.copyfile(lib_path, os.path.join(tempdir, lib_filename))
new_exe = os.path.join(tempdir, task_name)
compilation_commands = get_compilation_commands(lang, new_srcs, new_exe, for_evaluation=for_evaluation)
for command in compilation_commands:
call(tempdir, command)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
shutil.copyfile(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe))
shutil.copymode(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe))
finally:
shutil.rmtree(tempdir)
def make_output(n, assume=None):
try:
os.makedirs(output_dir)
except OSError:
pass
print(
"Generating",
add_color_to_string("output # %d" % n, colors.BLACK,
stream=sys.stderr, bold=True),
file=sys.stderr
)
temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp"))
use_stdin = yaml_conf.get("infile") in {None, ""}
use_stdout = yaml_conf.get("outfile") in {None, ""}
# Names of the actual source and destination.
infile = os.path.join(input_dir, 'input%d.txt' % (n))
outfile = os.path.join(output_dir, 'output%d.txt' % (n))
# Names of the input and output in temp directory.
copied_infile = os.path.join(
temp_dir,
"input.txt" if use_stdin else yaml_conf.get("infile"))
copied_outfile = os.path.join(
temp_dir,
"output.txt" if use_stdout else yaml_conf.get("outfile"))
os.symlink(infile, copied_infile)
fin = None
fout = None
try:
if use_stdin:
fin = open(copied_infile, "rb")
if use_stdout:
fout = open(copied_outfile, 'wb')
shutil.copy(sol_exe, temp_dir)
# If the task of of type Communication, then there is
# nothing to put in the output files
if task_type != ['Communication', '']:
call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)],
stdin=fin, stdout=fout)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
finally:
if fin is not None:
fin.close()
if fout is not None:
fout.close()
os.rename(copied_outfile, outfile)
shutil.rmtree(temp_dir)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_output(n, assume=None):
try:
os.makedirs(output_dir)
except OSError:
pass
print("Generating",
add_color_to_string("output # %d" % n,
colors.BLACK,
stream=sys.stderr,
bold=True),
file=sys.stderr)
temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp"))
use_stdin = yaml_conf.get("infile") in {None, ""}
use_stdout = yaml_conf.get("outfile") in {None, ""}
# Names of the actual source and destination.
infile = os.path.join(input_dir, 'input%d.txt' % (n))
outfile = os.path.join(output_dir, 'output%d.txt' % (n))
# Names of the input and output in temp directory.
copied_infile = os.path.join(
temp_dir, "input.txt" if use_stdin else yaml_conf.get("infile"))
copied_outfile = os.path.join(
temp_dir, "output.txt" if use_stdout else yaml_conf.get("outfile"))
os.symlink(infile, copied_infile)
fin = None
fout = None
try:
if use_stdin:
fin = open(copied_infile, "rb")
if use_stdout:
fout = open(copied_outfile, 'wb')
shutil.copy(sol_exe, temp_dir)
# If the task of of type Communication, then there is
# nothing to put in the output files
if task_type != ['Communication', '']:
call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)],
stdin=fin,
stdout=fout)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
finally:
if fin is not None:
fin.close()
if fout is not None:
fout.close()
os.rename(copied_outfile, outfile)
shutil.rmtree(temp_dir)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def compile_src(srcs, exe, for_evaluation, lang, assume=None):
# We put everything in a temporary directory to reproduce
# the same conditions that we have when compiling a
# submission.
tempdir = tempfile.mkdtemp()
try:
task_name = detect_task_name(base_dir)
grader_num = 1 if len(srcs) > 1 else 0
new_srcs = []
for grader in srcs[:grader_num]:
grader_name = os.path.basename(grader)
shutil.copyfile(os.path.join(base_dir, grader),
os.path.join(tempdir, grader_name))
new_srcs.append(os.path.join(tempdir, grader_name))
# For now, we assume we only have one non-grader source.
source_name = task_name + LANGUAGE_TO_SOURCE_EXT_MAP[lang]
shutil.copyfile(os.path.join(base_dir, srcs[grader_num]),
os.path.join(tempdir, source_name))
new_srcs.append(source_name)
# Libraries are needed/used only for C/C++ and Pascal
if lang in LANGUAGE_TO_HEADER_EXT_MAP:
lib_template = "%s" + LANGUAGE_TO_HEADER_EXT_MAP[lang]
lib_filename = lib_template % (task_name)
lib_path = os.path.join(base_dir, SOL_DIRNAME, lib_filename)
if os.path.exists(lib_path):
shutil.copyfile(lib_path,
os.path.join(tempdir, lib_filename))
new_exe = os.path.join(tempdir, task_name)
compilation_commands = get_compilation_commands(
lang,
new_srcs,
new_exe,
for_evaluation=for_evaluation)
for command in compilation_commands:
call(tempdir, command)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
shutil.copyfile(os.path.join(tempdir, new_exe),
os.path.join(base_dir, exe))
shutil.copymode(os.path.join(tempdir, new_exe),
os.path.join(base_dir, exe))
finally:
shutil.rmtree(tempdir)
def compile_src(srcs, exe, for_evaluation, lang, assume=None):
if lang != 'pas' or len(srcs) == 1:
compilation_commands = get_compilation_commands(
lang,
srcs,
exe,
for_evaluation=for_evaluation)
for command in compilation_commands:
call(base_dir, command)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
# When using Pascal with graders, file naming conventions
# require us to do a bit of trickery, i.e., performing the
# compilation in a separate temporary directory
else:
tempdir = tempfile.mkdtemp()
task_name = detect_task_name(base_dir)
new_srcs = [os.path.split(srcs[0])[1],
'%s.pas' % (task_name)]
new_exe = os.path.split(srcs[1])[1][:-4]
shutil.copyfile(os.path.join(base_dir, srcs[0]),
os.path.join(tempdir, new_srcs[0]))
shutil.copyfile(os.path.join(base_dir, srcs[1]),
os.path.join(tempdir, new_srcs[1]))
lib_filename = '%slib.pas' % (task_name)
if os.path.exists(os.path.join(SOL_DIRNAME, lib_filename)):
shutil.copyfile(os.path.join(SOL_DIRNAME, lib_filename),
os.path.join(tempdir, lib_filename))
compilation_commands = get_compilation_commands(
lang,
new_srcs,
new_exe,
for_evaluation=for_evaluation)
for command in compilation_commands:
call(tempdir, command)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
shutil.copyfile(os.path.join(tempdir, new_exe),
os.path.join(base_dir, exe))
shutil.copymode(os.path.join(tempdir, new_exe),
os.path.join(base_dir, exe))
shutil.rmtree(tempdir)
def make_output(n, assume=None):
try:
os.makedirs(output_dir)
except OSError:
pass
print(
"Generating",
add_color_to_string("output # %d" % n, colors.BLACK,
stream=sys.stderr, bold=True),
file=sys.stderr
)
with io.open(os.path.join(input_dir,
'input%d.txt' % (n)), 'rb') as fin:
with io.open(os.path.join(output_dir,
'output%d.txt' % (n)), 'wb') as fout:
if task_type != ['Communication', '']:
call(base_dir, [sol_exe], stdin=fin, stdout=fout)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
# If the task of of type Communication, then there is
# nothing to put in the output files
else:
pass
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def test_testcases(base_dir, solution, language, assume=None):
global task, file_cacher
# Use a FileCacher with a NullBackend in order to avoid to fill
# the database with junk
if file_cacher is None:
file_cacher = FileCacher(null=True)
cmscontrib.loaders.italy_yaml.logger = NullLogger()
# Load the task
# TODO - This implies copying a lot of data to the FileCacher,
# which is annoying if you have to do it continuously; it would be
# better to use a persistent cache (although local, possibly
# filesystem-based instead of database-based) and somehow detect
# when the task has already been loaded
if task is None:
loader = cmscontrib.loaders.italy_yaml.YamlLoader(base_dir,
file_cacher)
task = loader.get_task(get_statement=False)
# Prepare the EvaluationJob
dataset = task.active_dataset
digest = file_cacher.put_file_from_path(
os.path.join(base_dir, solution),
"Solution %s for task %s" % (solution, task.name))
executables = {task.name: Executable(filename=task.name, digest=digest)}
jobs = [(t, EvaluationJob(
language=language,
task_type=dataset.task_type,
task_type_parameters=json.loads(dataset.task_type_parameters),
managers=dict(dataset.managers),
executables=executables,
input=dataset.testcases[t].input, output=dataset.testcases[t].output,
time_limit=dataset.time_limit,
memory_limit=dataset.memory_limit)) for t in dataset.testcases]
tasktype = get_task_type(dataset=dataset)
ask_again = True
last_status = "ok"
status = "ok"
stop = False
info = []
points = []
comments = []
tcnames = []
for jobinfo in sorted(jobs):
print(jobinfo[0])
sys.stdout.flush()
job = jobinfo[1]
# Skip the testcase if we decide to consider everything to
# timeout
if stop:
info.append("Time limit exceeded")
points.append(0.0)
comments.append("Timeout.")
move_cursor(directions.UP, erase=True)
continue
# Evaluate testcase
last_status = status
tasktype.evaluate(job, file_cacher)
status = job.plus.get("exit_status")
info.append((job.plus.get("execution_time"),
job.plus.get("execution_memory")))
points.append(float(job.outcome))
comments.append(format_status_text(job.text))
tcnames.append(jobinfo[0])
# If we saw two consecutive timeouts, ask wether we want to
# consider everything to timeout
if ask_again and status == "timeout" and last_status == "timeout":
print("Want to stop and consider everything to timeout? [y/N]",
end='')
if assume is not None:
print(assume)
tmp = assume
else:
tmp = raw_input().lower()
if tmp in ['y', 'yes']:
stop = True
else:
ask_again = False
print()
move_cursor(directions.UP, erase=True)
# Subtasks scoring
try:
subtasks = json.loads(dataset.score_type_parameters)
subtasks[0]
except:
subtasks = [[100, len(info)]]
if dataset.score_type == 'GroupMin':
scoreFun = min
else:
if dataset.score_type != 'Sum':
logger.warning("Score type %s not yet supported! Using Sum"
% dataset.score_type)
def scoreFun(x):
return sum(x) / len(x)
pos = 0
sts = []
# For each subtask generate a list of testcase it owns, the score gained
# and the highest time and memory usage.
for i in subtasks:
stscores = []
stsdata = []
worst = [0, 0]
try:
for _ in xrange(i[1]):
stscores.append(points[pos])
stsdata.append((tcnames[pos], points[pos],
comments[pos], info[pos]))
if info[pos][0] > worst[0]:
worst[0] = info[pos][0]
if info[pos][1] > worst[1]:
worst[1] = info[pos][1]
pos += 1
sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst))
except:
sts.append((0, i[0], stsdata, [0, 0]))
# Result pretty printing
# Strips sol/ and _EVAL from the solution's name
solution = solution[4:-5]
print()
clen = max(len(c) for c in comments)
for st, d in enumerate(sts):
print(
"Subtask %d:" % st,
add_color_to_string(
"%5.2f/%d" % (d[0], d[1]),
colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN,
bold=True
)
)
for (i, p, c, w) in d[2]:
print(
"%s)" % i,
add_color_to_string(
"%5.2lf" % p,
colors.RED if abs(p - 1) > 0.01 else colors.BLACK
),
"--- %s [Time:" % c.ljust(clen),
add_color_to_string(
("%5.3f" % w[0]) if w[0] is not None else "N/A",
colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0]
else colors.BLACK
),
"Memory:",
add_color_to_string(
"%5s" % mem_human(w[1]) if w[1] is not None else "N/A",
colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1]
else colors.BLACK,
),
end="]"
)
move_cursor(directions.RIGHT, 1000)
move_cursor(directions.LEFT, len(solution) - 1)
print(add_color_to_string(solution, colors.BLACK, bold=True))
print()
sols.append((solution, sum([st[0] for st in sts])))
global tested_something
if not tested_something:
tested_something = True
atexit.register(print_at_exit)
return zip(points, comments, info)
def test_testcases(base_dir, solution, language, assume=None):
global task, file_cacher
# Use a FileCacher with a NullBackend in order to avoid to fill
# the database with junk
if file_cacher is None:
file_cacher = FileCacher(null=True)
cmscontrib.loaders.italy_yaml.logger = NullLogger()
# Load the task
# TODO - This implies copying a lot of data to the FileCacher,
# which is annoying if you have to do it continuously; it would be
# better to use a persistent cache (although local, possibly
# filesystem-based instead of database-based) and somehow detect
# when the task has already been loaded
if task is None:
loader = cmscontrib.loaders.italy_yaml.YamlLoader(
base_dir, file_cacher)
task = loader.get_task(get_statement=False)
# Prepare the EvaluationJob
dataset = task.active_dataset
digest = file_cacher.put_file_from_path(
os.path.join(base_dir, solution),
"Solution %s for task %s" % (solution, task.name))
executables = {task.name: Executable(filename=task.name, digest=digest)}
jobs = [
(t,
EvaluationJob(
operation=ESOperation(ESOperation.EVALUATION, None, dataset.id,
dataset.testcases[t].codename).to_dict(),
language=language,
task_type=dataset.task_type,
task_type_parameters=json.loads(dataset.task_type_parameters),
managers=dict(dataset.managers),
executables=executables,
input=dataset.testcases[t].input,
output=dataset.testcases[t].output,
time_limit=dataset.time_limit,
memory_limit=dataset.memory_limit)) for t in dataset.testcases
]
tasktype = get_task_type(dataset=dataset)
ask_again = True
last_status = "ok"
status = "ok"
stop = False
info = []
points = []
comments = []
tcnames = []
for jobinfo in sorted(jobs):
print(jobinfo[0])
sys.stdout.flush()
job = jobinfo[1]
# Skip the testcase if we decide to consider everything to
# timeout
if stop:
info.append("Time limit exceeded")
points.append(0.0)
comments.append("Timeout.")
move_cursor(directions.UP, erase=True)
continue
# Evaluate testcase
last_status = status
tasktype.evaluate(job, file_cacher)
status = job.plus.get("exit_status")
info.append(
(job.plus.get("execution_time"), job.plus.get("execution_memory")))
points.append(float(job.outcome))
# Avoid printing unneeded newline
job.text = [t.rstrip() for t in job.text]
comments.append(format_status_text(job.text))
tcnames.append(jobinfo[0])
# If we saw two consecutive timeouts, ask wether we want to
# consider everything to timeout
if ask_again and status == "timeout" and last_status == "timeout":
print("Want to stop and consider everything to timeout? [y/N] ",
end='')
sys.stdout.flush()
if assume is not None:
tmp = assume
print(tmp)
else:
# User input with a timeout of 5 seconds, at the end of which
# we automatically say "n". ready will be a list of input ready
# for reading, or an empty list if the timeout expired.
# See: http://stackoverflow.com/a/2904057
ready, _, _ = select.select([sys.stdin], [], [], 5)
if ready:
tmp = sys.stdin.readline().strip().lower()
else:
tmp = 'n'
print(tmp)
if tmp in ['y', 'yes']:
stop = True
else:
ask_again = False
print()
move_cursor(directions.UP, erase=True)
# Subtasks scoring
subtasks = json.loads(dataset.score_type_parameters)
if not isinstance(subtasks, list) or len(subtasks) == 0:
subtasks = [[100, len(info)]]
if dataset.score_type == 'GroupMin':
scoreFun = min
else:
if dataset.score_type != 'Sum':
logger.warning("Score type %s not yet supported! Using Sum" %
dataset.score_type)
def scoreFun(x):
return sum(x) / len(x)
pos = 0
sts = []
# For each subtask generate a list of testcase it owns, the score gained
# and the highest time and memory usage.
for i in subtasks:
stscores = []
stsdata = []
worst = [0, 0]
try:
for _ in xrange(i[1]):
stscores.append(points[pos])
stsdata.append(
(tcnames[pos], points[pos], comments[pos], info[pos]))
if info[pos][0] > worst[0]:
worst[0] = info[pos][0]
if info[pos][1] > worst[1]:
worst[1] = info[pos][1]
pos += 1
sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst))
except:
sts.append((0, i[0], stsdata, [0, 0]))
# Result pretty printing
# Strips sol/ and _EVAL from the solution's name
solution = solution[4:-5]
print()
clen = max(len(c) for c in comments)
for st, d in enumerate(sts):
print(
"Subtask %d:" % st,
add_color_to_string(
"%5.2f/%d" % (d[0], d[1]),
colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN,
bold=True))
for (i, p, c, w) in d[2]:
print("%s)" % i,
add_color_to_string(
"%5.2lf" % p,
colors.RED if abs(p - 1) > 0.01 else colors.BLACK),
"--- %s [Time:" % c.ljust(clen),
add_color_to_string(
("%5.3f" % w[0]) if w[0] is not None else "N/A",
colors.BLUE if w[0] is not None
and w[0] >= 0.95 * d[3][0] else colors.BLACK),
"Memory:",
add_color_to_string(
"%5s" % mem_human(w[1]) if w[1] is not None else "N/A",
colors.BLUE if w[1] is not None
and w[1] >= 0.95 * d[3][1] else colors.BLACK,
),
end="]")
move_cursor(directions.RIGHT, 1000)
move_cursor(directions.LEFT, len(solution) - 1)
print(add_color_to_string(solution, colors.BLACK, bold=True))
print()
sols.append((solution, sum([st[0] for st in sts])))
global tested_something
if not tested_something:
tested_something = True
atexit.register(print_at_exit)
return zip(points, comments, info)
