def do_format(self, record):
"""Produce a human-readable message from the given record.
This is the "core" of the format method, but it has been split
out to allow the code to focus only on formatting, rather than
bookkeeping (putting args into their placeholders in msg and
formatting time and exception).
record (LogRecord): the data for the log message.
return (string): the formatted log message.
"""
# Determine the first part (time and severity) and its color.
severity_str = record.asctime + " - " + record.levelname
severity_col = self.SEVERITY_COLORS[record.levelno]
# Determine the second part (service coords) and its color.
if hasattr(record, "service_name") and \
hasattr(record, "service_shard"):
coord_str = "%s,%d" % (record.service_name, record.service_shard)
else:
coord_str = "None"
coord_col = get_color_hash(coord_str)
# Determine the third part (operation) and its color.
if hasattr(record, "operation"):
operation_str = record.operation
else:
operation_str = ""
operation_col = get_color_hash(operation_str)
# Colorize the strings.
if self.colors:
severity_str = add_color_to_string(severity_str, severity_col,
bold=True, force=True)
coord_str = add_color_to_string(coord_str, coord_col,
bold=True, force=True)
operation_str = add_color_to_string(operation_str, operation_col,
bold=True, force=True)
# Put them all together.
fmt = severity_str
fmt += " [" + coord_str
if hasattr(record, "operation"):
fmt += "/" + operation_str
fmt += "] " + record.message
return fmt
def make_input(assume=None):
n = 0
try:
os.makedirs(input_dir)
except OSError:
pass
for (is_copy, line, st) in testcases:
print(
"Generating",
add_color_to_string("input # %d" % n, colors.BLACK,
stream=sys.stderr, bold=True),
file=sys.stderr
)
new_input = os.path.join(input_dir, 'input%d.txt' % (n))
if is_copy:
# Copy the file
print("> Copy input file from:", line)
copy_input = os.path.join(base_dir, line)
shutil.copyfile(copy_input, new_input)
else:
# Call the generator
with open(new_input, 'wb') as fout:
call(base_dir,
[gen_exe] + line.split(),
stdout=fout)
command = [validator_exe, new_input]
if st != 0:
command.append("%s" % st)
call(base_dir, command)
n += 1
for _ in range(3):
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_input(assume=None):
n = 0
try:
os.makedirs(input_dir)
except OSError:
pass
for (is_copy, line, st) in testcases:
print("Generating",
add_color_to_string("input # %d" % n,
colors.BLACK,
stream=sys.stderr,
bold=True),
file=sys.stderr)
new_input = os.path.join(input_dir, 'input%d.txt' % (n))
if is_copy:
# Copy the file
print("> Copy input file from:", line)
copy_input = os.path.join(base_dir, line)
shutil.copyfile(copy_input, new_input)
else:
# Call the generator
with open(new_input, 'wb') as fout:
call(base_dir, [gen_exe] + line.split(), stdout=fout)
command = [validator_exe, new_input]
if st != 0:
command.append("%s" % st)
call(base_dir, command)
n += 1
for _ in range(3):
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_output(n, assume=None):
try:
os.makedirs(output_dir)
except OSError:
pass
print(
"Generating",
add_color_to_string("output # %d" % n, colors.BLACK,
stream=sys.stderr, bold=True),
file=sys.stderr
)
temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp"))
use_stdin = yaml_conf.get("infile") in {None, ""}
use_stdout = yaml_conf.get("outfile") in {None, ""}
# Names of the actual source and destination.
infile = os.path.join(input_dir, 'input%d.txt' % (n))
outfile = os.path.join(output_dir, 'output%d.txt' % (n))
# Names of the input and output in temp directory.
copied_infile = os.path.join(
temp_dir,
"input.txt" if use_stdin else yaml_conf.get("infile"))
copied_outfile = os.path.join(
temp_dir,
"output.txt" if use_stdout else yaml_conf.get("outfile"))
os.symlink(infile, copied_infile)
fin = None
fout = None
try:
if use_stdin:
fin = open(copied_infile, "rb")
if use_stdout:
fout = open(copied_outfile, 'wb')
shutil.copy(sol_exe, temp_dir)
# If the task of of type Communication, then there is
# nothing to put in the output files
if task_type != ['Communication', '']:
call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)],
stdin=fin, stdout=fout)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
finally:
if fin is not None:
fin.close()
if fout is not None:
fout.close()
os.rename(copied_outfile, outfile)
shutil.rmtree(temp_dir)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def print_at_exit():
print()
print()
for s in sols:
print("%s: %3d" % (
add_color_to_string("%30s" % s[0], colors.BLACK,
bold=True),
s[1])
)
def print_at_exit():
print()
print()
for s in sols:
print("%s: %3d" % (
add_color_to_string("%30s" % s[0], colors.BLACK,
bold=True),
s[1])
)
def make_output(n, assume=None):
try:
os.makedirs(output_dir)
except OSError:
pass
print("Generating",
add_color_to_string("output # %d" % n,
colors.BLACK,
stream=sys.stderr,
bold=True),
file=sys.stderr)
temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp"))
use_stdin = yaml_conf.get("infile") in {None, ""}
use_stdout = yaml_conf.get("outfile") in {None, ""}
# Names of the actual source and destination.
infile = os.path.join(input_dir, 'input%d.txt' % (n))
outfile = os.path.join(output_dir, 'output%d.txt' % (n))
# Names of the input and output in temp directory.
copied_infile = os.path.join(
temp_dir, "input.txt" if use_stdin else yaml_conf.get("infile"))
copied_outfile = os.path.join(
temp_dir, "output.txt" if use_stdout else yaml_conf.get("outfile"))
os.symlink(infile, copied_infile)
fin = None
fout = None
try:
if use_stdin:
fin = open(copied_infile, "rb")
if use_stdout:
fout = open(copied_outfile, 'wb')
shutil.copy(sol_exe, temp_dir)
# If the task of of type Communication, then there is
# nothing to put in the output files
if task_type != ['Communication', '']:
call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)],
stdin=fin,
stdout=fout)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
finally:
if fin is not None:
fin.close()
if fout is not None:
fout.close()
os.rename(copied_outfile, outfile)
shutil.rmtree(temp_dir)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def do_format(self, record):
"""Produce a human-readable message from the given record.
This is the "core" of the format method, but it has been split
out to allow the code to focus only on formatting, rather than
bookkeeping (putting args into their placeholders in msg and
formatting time and exception).
record (LogRecord): the data for the log message.
return (string): the formatted log message.
"""
severity = self.get_severity(record)
coordinates = self.get_coordinates(record)
operation = self.get_operation(record)
message = record.message
if self.colors:
severity_col = self.SEVERITY_COLORS[record.levelno]
severity = add_color_to_string(severity,
severity_col,
bold=True,
force=True)
coordinates_col = get_color_hash(coordinates)
if len(coordinates) > 0:
coordinates = add_color_to_string(coordinates,
coordinates_col,
bold=True,
force=True)
operation_col = get_color_hash(operation)
if len(operation) > 0:
operation = add_color_to_string(operation,
operation_col,
bold=True,
force=True)
fmt = severity
if coordinates.strip() != "":
fmt += " [%s]" % (coordinates.strip())
if operation.strip() != "":
fmt += " [%s]" % (operation.strip())
fmt += " %s" % message
return fmt
def test_src(exe, lang, assume=None):
# Solution names begin with sol/ and end with _EVAL, we strip that
print(
"Testing solution",
add_color_to_string(exe[4:-5], colors.BLACK, bold=True)
)
test_testcases(
base_dir,
exe,
language=lang,
assume=assume)
def test_src(exe, lang, assume=None):
# Solution names begin with sol/ and end with _EVAL, we strip that
print(
"Testing solution",
add_color_to_string(exe[4:-5], colors.BLACK, bold=True)
)
test_testcases(
base_dir,
exe,
language=lang,
assume=assume)
def do_format(self, record):
"""Produce a human-readable message from the given record.
This is the "core" of the format method, but it has been split
out to allow the code to focus only on formatting, rather than
bookkeeping (putting args into their placeholders in msg and
formatting time and exception).
record (LogRecord): the data for the log message.
return (string): the formatted log message.
"""
severity = self.get_severity(record)
coordinates = self.get_coordinates(record)
operation = self.get_operation(record)
message = record.message
if self.colors:
severity_col = self.SEVERITY_COLORS[record.levelno]
severity = add_color_to_string(severity, severity_col,
bold=True, force=True)
coordinates_col = get_color_hash(coordinates)
if coordinates != "":
coordinates = add_color_to_string(coordinates, coordinates_col,
bold=True, force=True)
operation_col = get_color_hash(operation)
if operation != "":
operation = add_color_to_string(operation, operation_col,
bold=True, force=True)
fmt = severity
if coordinates.strip() != "":
fmt += " [%s]" % (coordinates.strip())
if operation.strip() != "":
fmt += " [%s]" % (operation.strip())
fmt += " %s" % message
return fmt
def make_output(n, assume=None):
try:
os.makedirs(output_dir)
except OSError:
pass
print(
"Generating",
add_color_to_string("output # %d" % n, colors.BLACK,
stream=sys.stderr, bold=True),
file=sys.stderr
)
with io.open(os.path.join(input_dir,
'input%d.txt' % (n)), 'rb') as fin:
with io.open(os.path.join(output_dir,
'output%d.txt' % (n)), 'wb') as fout:
if task_type != ['Communication', '']:
call(base_dir, [sol_exe], stdin=fin, stdout=fout)
move_cursor(directions.UP, erase=True, stream=sys.stderr)
# If the task of of type Communication, then there is
# nothing to put in the output files
else:
pass
move_cursor(directions.UP, erase=True, stream=sys.stderr)
def test_testcases(base_dir, solution, language, assume=None):
global task, file_cacher
# Use a FileCacher with a NullBackend in order to avoid to fill
# the database with junk
if file_cacher is None:
file_cacher = FileCacher(null=True)
cmscontrib.loaders.italy_yaml.logger = NullLogger()
# Load the task
# TODO - This implies copying a lot of data to the FileCacher,
# which is annoying if you have to do it continuously; it would be
# better to use a persistent cache (although local, possibly
# filesystem-based instead of database-based) and somehow detect
# when the task has already been loaded
if task is None:
loader = cmscontrib.loaders.italy_yaml.YamlLoader(base_dir,
file_cacher)
task = loader.get_task(get_statement=False)
# Prepare the EvaluationJob
dataset = task.active_dataset
digest = file_cacher.put_file_from_path(
os.path.join(base_dir, solution),
"Solution %s for task %s" % (solution, task.name))
executables = {task.name: Executable(filename=task.name, digest=digest)}
jobs = [(t, EvaluationJob(
language=language,
task_type=dataset.task_type,
task_type_parameters=json.loads(dataset.task_type_parameters),
managers=dict(dataset.managers),
executables=executables,
input=dataset.testcases[t].input, output=dataset.testcases[t].output,
time_limit=dataset.time_limit,
memory_limit=dataset.memory_limit)) for t in dataset.testcases]
tasktype = get_task_type(dataset=dataset)
ask_again = True
last_status = "ok"
status = "ok"
stop = False
info = []
points = []
comments = []
tcnames = []
for jobinfo in sorted(jobs):
print(jobinfo[0])
sys.stdout.flush()
job = jobinfo[1]
# Skip the testcase if we decide to consider everything to
# timeout
if stop:
info.append("Time limit exceeded")
points.append(0.0)
comments.append("Timeout.")
move_cursor(directions.UP, erase=True)
continue
# Evaluate testcase
last_status = status
tasktype.evaluate(job, file_cacher)
status = job.plus.get("exit_status")
info.append((job.plus.get("execution_time"),
job.plus.get("execution_memory")))
points.append(float(job.outcome))
comments.append(format_status_text(job.text))
tcnames.append(jobinfo[0])
# If we saw two consecutive timeouts, ask wether we want to
# consider everything to timeout
if ask_again and status == "timeout" and last_status == "timeout":
print("Want to stop and consider everything to timeout? [y/N]",
end='')
if assume is not None:
print(assume)
tmp = assume
else:
tmp = raw_input().lower()
if tmp in ['y', 'yes']:
stop = True
else:
ask_again = False
print()
move_cursor(directions.UP, erase=True)
# Subtasks scoring
try:
subtasks = json.loads(dataset.score_type_parameters)
subtasks[0]
except:
subtasks = [[100, len(info)]]
if dataset.score_type == 'GroupMin':
scoreFun = min
else:
if dataset.score_type != 'Sum':
logger.warning("Score type %s not yet supported! Using Sum"
% dataset.score_type)
def scoreFun(x):
return sum(x) / len(x)
pos = 0
sts = []
# For each subtask generate a list of testcase it owns, the score gained
# and the highest time and memory usage.
for i in subtasks:
stscores = []
stsdata = []
worst = [0, 0]
try:
for _ in xrange(i[1]):
stscores.append(points[pos])
stsdata.append((tcnames[pos], points[pos],
comments[pos], info[pos]))
if info[pos][0] > worst[0]:
worst[0] = info[pos][0]
if info[pos][1] > worst[1]:
worst[1] = info[pos][1]
pos += 1
sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst))
except:
sts.append((0, i[0], stsdata, [0, 0]))
# Result pretty printing
# Strips sol/ and _EVAL from the solution's name
solution = solution[4:-5]
print()
clen = max(len(c) for c in comments)
for st, d in enumerate(sts):
print(
"Subtask %d:" % st,
add_color_to_string(
"%5.2f/%d" % (d[0], d[1]),
colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN,
bold=True
)
)
for (i, p, c, w) in d[2]:
print(
"%s)" % i,
add_color_to_string(
"%5.2lf" % p,
colors.RED if abs(p - 1) > 0.01 else colors.BLACK
),
"--- %s [Time:" % c.ljust(clen),
add_color_to_string(
("%5.3f" % w[0]) if w[0] is not None else "N/A",
colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0]
else colors.BLACK
),
"Memory:",
add_color_to_string(
"%5s" % mem_human(w[1]) if w[1] is not None else "N/A",
colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1]
else colors.BLACK,
),
end="]"
)
move_cursor(directions.RIGHT, 1000)
move_cursor(directions.LEFT, len(solution) - 1)
print(add_color_to_string(solution, colors.BLACK, bold=True))
print()
sols.append((solution, sum([st[0] for st in sts])))
global tested_something
if not tested_something:
tested_something = True
atexit.register(print_at_exit)
return zip(points, comments, info)
def test_testcases(base_dir, solution, language, assume=None):
global task, file_cacher
# Use a FileCacher with a NullBackend in order to avoid to fill
# the database with junk
if file_cacher is None:
file_cacher = FileCacher(null=True)
cmscontrib.loaders.italy_yaml.logger = NullLogger()
# Load the task
# TODO - This implies copying a lot of data to the FileCacher,
# which is annoying if you have to do it continuously; it would be
# better to use a persistent cache (although local, possibly
# filesystem-based instead of database-based) and somehow detect
# when the task has already been loaded
if task is None:
loader = cmscontrib.loaders.italy_yaml.YamlLoader(
base_dir, file_cacher)
task = loader.get_task(get_statement=False)
# Prepare the EvaluationJob
dataset = task.active_dataset
digest = file_cacher.put_file_from_path(
os.path.join(base_dir, solution),
"Solution %s for task %s" % (solution, task.name))
executables = {task.name: Executable(filename=task.name, digest=digest)}
jobs = [
(t,
EvaluationJob(
operation=ESOperation(ESOperation.EVALUATION, None, dataset.id,
dataset.testcases[t].codename).to_dict(),
language=language,
task_type=dataset.task_type,
task_type_parameters=json.loads(dataset.task_type_parameters),
managers=dict(dataset.managers),
executables=executables,
input=dataset.testcases[t].input,
output=dataset.testcases[t].output,
time_limit=dataset.time_limit,
memory_limit=dataset.memory_limit)) for t in dataset.testcases
]
tasktype = get_task_type(dataset=dataset)
ask_again = True
last_status = "ok"
status = "ok"
stop = False
info = []
points = []
comments = []
tcnames = []
for jobinfo in sorted(jobs):
print(jobinfo[0])
sys.stdout.flush()
job = jobinfo[1]
# Skip the testcase if we decide to consider everything to
# timeout
if stop:
info.append("Time limit exceeded")
points.append(0.0)
comments.append("Timeout.")
move_cursor(directions.UP, erase=True)
continue
# Evaluate testcase
last_status = status
tasktype.evaluate(job, file_cacher)
status = job.plus.get("exit_status")
info.append(
(job.plus.get("execution_time"), job.plus.get("execution_memory")))
points.append(float(job.outcome))
# Avoid printing unneeded newline
job.text = [t.rstrip() for t in job.text]
comments.append(format_status_text(job.text))
tcnames.append(jobinfo[0])
# If we saw two consecutive timeouts, ask wether we want to
# consider everything to timeout
if ask_again and status == "timeout" and last_status == "timeout":
print("Want to stop and consider everything to timeout? [y/N] ",
end='')
sys.stdout.flush()
if assume is not None:
tmp = assume
print(tmp)
else:
# User input with a timeout of 5 seconds, at the end of which
# we automatically say "n". ready will be a list of input ready
# for reading, or an empty list if the timeout expired.
# See: http://stackoverflow.com/a/2904057
ready, _, _ = select.select([sys.stdin], [], [], 5)
if ready:
tmp = sys.stdin.readline().strip().lower()
else:
tmp = 'n'
print(tmp)
if tmp in ['y', 'yes']:
stop = True
else:
ask_again = False
print()
move_cursor(directions.UP, erase=True)
# Subtasks scoring
subtasks = json.loads(dataset.score_type_parameters)
if not isinstance(subtasks, list) or len(subtasks) == 0:
subtasks = [[100, len(info)]]
if dataset.score_type == 'GroupMin':
scoreFun = min
else:
if dataset.score_type != 'Sum':
logger.warning("Score type %s not yet supported! Using Sum" %
dataset.score_type)
def scoreFun(x):
return sum(x) / len(x)
pos = 0
sts = []
# For each subtask generate a list of testcase it owns, the score gained
# and the highest time and memory usage.
for i in subtasks:
stscores = []
stsdata = []
worst = [0, 0]
try:
for _ in xrange(i[1]):
stscores.append(points[pos])
stsdata.append(
(tcnames[pos], points[pos], comments[pos], info[pos]))
if info[pos][0] > worst[0]:
worst[0] = info[pos][0]
if info[pos][1] > worst[1]:
worst[1] = info[pos][1]
pos += 1
sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst))
except:
sts.append((0, i[0], stsdata, [0, 0]))
# Result pretty printing
# Strips sol/ and _EVAL from the solution's name
solution = solution[4:-5]
print()
clen = max(len(c) for c in comments)
for st, d in enumerate(sts):
print(
"Subtask %d:" % st,
add_color_to_string(
"%5.2f/%d" % (d[0], d[1]),
colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN,
bold=True))
for (i, p, c, w) in d[2]:
print("%s)" % i,
add_color_to_string(
"%5.2lf" % p,
colors.RED if abs(p - 1) > 0.01 else colors.BLACK),
"--- %s [Time:" % c.ljust(clen),
add_color_to_string(
("%5.3f" % w[0]) if w[0] is not None else "N/A",
colors.BLUE if w[0] is not None
and w[0] >= 0.95 * d[3][0] else colors.BLACK),
"Memory:",
add_color_to_string(
"%5s" % mem_human(w[1]) if w[1] is not None else "N/A",
colors.BLUE if w[1] is not None
and w[1] >= 0.95 * d[3][1] else colors.BLACK,
),
end="]")
move_cursor(directions.RIGHT, 1000)
move_cursor(directions.LEFT, len(solution) - 1)
print(add_color_to_string(solution, colors.BLACK, bold=True))
print()
sols.append((solution, sum([st[0] for st in sts])))
global tested_something
if not tested_something:
tested_something = True
atexit.register(print_at_exit)
return zip(points, comments, info)
