def display_side_by_side_color(answer: str, expected: str) -> None:
max_chars = shutil.get_terminal_size()[0] // 2 - 2
logger.info(utils.NO_HEADER + 'output:' + " " * (max_chars - 7) + "|" +
"expected:")
logger.info(utils.NO_HEADER + '%s',
"-" * max_chars + "|" + "-" * max_chars)
for _, diff_found, ans_line, exp_line, ans_chars, exp_chars in _side_by_side_diff(
answer, expected):
if diff_found:
logger.info(
utils.NO_HEADER + '%s',
utils.red(_space_padding(ans_line, max_chars - ans_chars)) +
"|" + utils.green(exp_line))
else:
logger.info(
utils.NO_HEADER + '%s',
_space_padding(ans_line, max_chars - ans_chars) + "|" +
exp_line)
def downloadSingleCase(url: str, target_dir: str, out: bool = False) -> None:
os.makedirs(target_dir, exist_ok=True)
URL = re.sub(r'dl=0', 'dl=1', url)
path = urlparse(URL).path
suffix = PurePath(path).suffix
filename = PurePath(path).name
if suffix:
filename = filename[:-len(suffix)] # Remove suffix
filename += ".out" if out else ".in"
log.info(f"Download: {URL}")
r = session.get(URL, headers=agent)
if r.status_code != requests.codes.ok:
log.error(utils.FAILURE + utils.red("Download failed."))
exit(1)
else:
log.info(utils.SUCCESS + "Download Succeeded")
target_file = target_dir + '/' + filename
log.info(f"Save to: {target_file}")
with open(target_file, "w") as fw:
fw.write(r.text)
def display_snipped_side_by_side_color(answer: str, expected: str) -> None:
"""
Display first differ line and its previous 3 lines and its next 3 lines.
"""
max_chars = shutil.get_terminal_size()[0] // 2 - 2
deq: Deque[Tuple[Optional[int], bool, str, str, int, int]] = collections.deque(maxlen=7)
count_from_first_difference = 0
i = 0
for flag, diff_found, ans_line, exp_line, ans_chars, exp_chars in _side_by_side_diff(answer, expected):
if flag:
i += 1
if count_from_first_difference > 0:
count_from_first_difference += 1
line_num = i if flag else None
deq.append((line_num, diff_found, ans_line, exp_line, ans_chars, exp_chars))
if diff_found:
if count_from_first_difference == 0:
count_from_first_difference = 1
if count_from_first_difference == 4:
break
max_line_num_digits = max([len(str(entry[0])) for entry in deq if entry[0] is not None])
logger.info(utils.NO_HEADER + '%s', " " * max_line_num_digits + "|output:" + " " * (max_chars - 7 - max_line_num_digits - 1) + "|" + "expected:")
logger.info(utils.NO_HEADER + '%s', "-" * max_chars + "|" + "-" * max_chars)
last_line_number = 0
for (line_number, diff_found, ans_line, exp_line, ans_chars, exp_chars) in deq:
num_spaces_after_output = max_chars - ans_chars - max_line_num_digits - 1
line_number_str = str(line_number) if line_number is not None else ""
line_num_display = _space_padding(line_number_str, max_line_num_digits - len(line_number_str)) + "|"
if not diff_found:
logger.info(utils.NO_HEADER + '%s', line_num_display + _space_padding(ans_line, num_spaces_after_output) + "|" + exp_line)
else:
logger.info(utils.NO_HEADER + '%s', line_num_display + utils.red(_space_padding(ans_line, num_spaces_after_output)) + "|" + utils.green(exp_line))
if line_number is not None:
last_line_number = line_number
num_snipped_lines = answer.count('\n') + 1 - last_line_number
if num_snipped_lines > 0:
logger.info(utils.NO_HEADER + '... (%s lines) ...', num_snipped_lines)
def try_hack_once(
generator: str,
command: str,
hack: str,
*,
tle: Optional[float],
attempt: int,
lock: Optional[threading.Lock] = None
) -> Optional[Tuple[bytes, bytes]]:
with BufferedExecutor(lock) as submit:
# print the header
submit(logger.info, '')
submit(logger.info, '%d-th attempt', attempt)
# generate input
submit(logger.info, 'generate input...')
info, proc = utils.exec_command(generator, stdin=None, timeout=tle)
input_data = info['answer'] # type: Optional[bytes]
if not check_status(info, proc, submit=submit):
return None
assert input_data is not None
# generate output
submit(logger.info, 'generate output...')
info, proc = utils.exec_command(command, input=input_data, timeout=tle)
output_data = info['answer'] # type: Optional[bytes]
if not check_status(info, proc, submit=submit):
return None
assert output_data is not None
# hack
submit(logger.info, 'hack...')
info, proc = utils.exec_command(hack, input=input_data, timeout=tle)
answer = (info['answer'] or b'').decode() # type: str
elapsed = info['elapsed'] # type: float
memory = info['memory'] # type: Optional[float]
# compare
status = 'AC'
if proc.returncode is None:
submit(logger.info, 'FAILURE: ' + utils.red('TLE'))
status = 'TLE'
elif proc.returncode != 0:
logger.info(
utils.FAILURE + '' + utils.red('RE') + ': return code %d',
proc.returncode)
status = 'RE'
expected = output_data.decode()
if not simple_match(answer, expected):
logger.info(utils.FAILURE + '' + utils.red('WA'))
logger.info(
'NO_HDEADER: input:\n%s',
utils.make_pretty_large_file_content(input_data,
limit=40,
head=20,
tail=10,
bold=True))
logger.info(
'NO_HDEADER: output:\n%s',
utils.make_pretty_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
logger.info(
'NO_HDEADER: expected:\n%s',
utils.make_pretty_large_file_content(output_data,
limit=40,
head=20,
tail=10,
bold=True))
status = 'WA'
if status == 'AC':
return None
else:
return (input_data, output_data)
def test(args: 'argparse.Namespace') -> None:
# list tests
if not args.test:
args.test = fmtutils.glob_with_format(args.directory,
args.format) # by default
if args.ignore_backup:
args.test = fmtutils.drop_backup_or_hidden_files(args.test)
tests = fmtutils.construct_relationship_of_files(args.test, args.directory,
args.format)
# check wheather GNU time is available
if not check_gnu_time(args.gnu_time):
logger.warning('GNU time is not available: %s', args.gnu_time)
args.gnu_time = None
if args.mle is not None and args.gnu_time is None:
raise RuntimeError('--mle is used but GNU time does not exist')
# run tests
history: List[Dict[str, Any]] = []
if args.jobs is None:
for name, paths in sorted(tests.items()):
history += [
test_single_case(name,
paths['in'],
paths.get('out'),
args=args)
]
else:
if os.name == 'nt':
logger.warning(
"-j/--jobs opiton is unstable on Windows environmet")
with concurrent.futures.ThreadPoolExecutor(
max_workers=args.jobs) as executor:
lock = threading.Lock()
futures: List[concurrent.futures.Future] = []
for name, paths in sorted(tests.items()):
futures += [
executor.submit(test_single_case,
name,
paths['in'],
paths.get('out'),
lock=lock,
args=args)
]
for future in futures:
history += [future.result()]
# summarize
slowest: float = -1.0
slowest_name = ''
heaviest: float = -1.0
heaviest_name = ''
ac_count = 0
for result in history:
if result['status'] == 'AC':
ac_count += 1
if slowest < result['elapsed']:
slowest = result['elapsed']
slowest_name = result['testcase']['name']
if result['memory'] is not None and heaviest < result['memory']:
heaviest = result['memory']
heaviest_name = result['testcase']['name']
# print the summary
logger.info('')
logger.info('slowest: %f sec (for %s)', slowest, slowest_name)
if heaviest >= 0:
if heaviest < MEMORY_WARNING:
logger.info('max memory: %f MB (for %s)', heaviest, heaviest_name)
else:
logger.warning('max memory: %f MB (for %s)', heaviest,
heaviest_name)
if ac_count == len(tests):
logger.info(
utils.SUCCESS + 'test ' + utils.green('success') + ': %d cases',
len(tests))
else:
logger.info(
utils.FAILURE + 'test ' + utils.red('failed') +
': %d AC / %d cases', ac_count, len(tests))
if args.json:
print(json.dumps(history))
if ac_count != len(tests):
sys.exit(1)
def display_result(proc: subprocess.Popen, answer: str,
memory: Optional[float], test_input_path: pathlib.Path,
test_output_path: Optional[pathlib.Path], *,
mle: Optional[float], display_mode: DisplayMode,
does_print_input: bool, silent: bool,
match_result: Optional[bool]) -> JudgeStatus:
"""display_result prints the result of the test and its statistics.
This function prints many logs and does some I/O.
"""
# prepare the function to print the input
is_input_printed = False
def print_input() -> None:
nonlocal is_input_printed
if does_print_input and not is_input_printed:
is_input_printed = True
with test_input_path.open('rb') as inf:
logger.info(
utils.NO_HEADER + 'input:\n%s',
pretty_printers.make_pretty_large_file_content(inf.read(),
limit=40,
head=20,
tail=10,
bold=True))
# check TLE, RE or not
status = JudgeStatus.AC
if proc.returncode is None:
logger.info(utils.FAILURE + '' + utils.red('TLE'))
status = JudgeStatus.TLE
if not silent:
print_input()
elif memory is not None and mle is not None and memory > mle:
logger.info(utils.FAILURE + '' + utils.red('MLE'))
status = JudgeStatus.MLE
if not silent:
print_input()
elif proc.returncode != 0:
logger.info(utils.FAILURE + '' + utils.red('RE') + ': return code %d',
proc.returncode)
status = JudgeStatus.RE
if not silent:
print_input()
# check WA or not
if match_result is not None and not match_result:
if status == JudgeStatus.AC:
logger.info(utils.FAILURE + '' + utils.red('WA'))
status = JudgeStatus.WA
if not silent:
print_input()
if test_output_path is not None:
with test_output_path.open('rb') as outf:
expected = outf.read().decode()
else:
expected = ''
if display_mode == DisplayMode.SUMMARY:
logger.info(
utils.NO_HEADER + 'output:\n%s',
pretty_printers.make_pretty_large_file_content(
answer.encode(), limit=40, head=20, tail=10,
bold=True))
logger.info(
utils.NO_HEADER + 'expected:\n%s',
pretty_printers.make_pretty_large_file_content(
expected.encode(),
limit=40,
head=20,
tail=10,
bold=True))
elif display_mode == DisplayMode.DIFF:
if max(answer.count('\n'), expected.count('\n')) <= 40:
pretty_printers.display_side_by_side_color(
answer, expected)
else:
pretty_printers.display_snipped_side_by_side_color(
answer, expected)
else:
assert False
if match_result is None:
if not silent:
print_input()
logger.info(
utils.NO_HEADER + 'output:\n%s',
pretty_printers.make_pretty_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
if status == JudgeStatus.AC:
logger.info(utils.SUCCESS + '' + utils.green('AC'))
return status
def try_hack_once(
generator: str,
command: str,
hack: str,
*,
tle: Optional[float],
attempt: int,
lock: Optional[threading.Lock] = None,
generated_input_hashes: Dict[bytes,
str]) -> Optional[Tuple[bytes, bytes]]:
with BufferedExecutor(lock) as submit:
# print the header
submit(logger.info, '')
submit(logger.info, '%d-th attempt', attempt)
# generate input
submit(logger.info, 'generate input...')
info, proc = utils.exec_command(generator, stdin=None, timeout=tle)
input_data: Optional[bytes] = info['answer']
if not check_status(info, proc, submit=submit):
return None
assert input_data is not None
# check the randomness of generator
name = '{}-th attempt'.format(attempt)
conflicted_name = check_randomness_of_generator(
input_data,
name=name,
lock=lock,
generated_input_hashes=generated_input_hashes)
if conflicted_name is not None:
submit(
logger.warning,
'The same input is already generated at %s. Please use a random input generator.',
conflicted_name)
submit(logger.info, utils.NO_HEADER + 'input:')
submit(
logger.info, utils.NO_HEADER + '%s',
pretty_printers.make_pretty_large_file_content(input_data,
limit=40,
head=20,
tail=10,
bold=True))
# generate output
submit(logger.info, 'generate output...')
info, proc = utils.exec_command(command, input=input_data, timeout=tle)
output_data: Optional[bytes] = info['answer']
if not check_status(info, proc, submit=submit):
return None
assert output_data is not None
# hack
submit(logger.info, 'hack...')
info, proc = utils.exec_command(hack, input=input_data, timeout=tle)
answer: str = (info['answer'] or b'').decode()
# compare
status = 'AC'
if proc.returncode is None:
submit(logger.info, 'FAILURE: ' + utils.red('TLE'))
status = 'TLE'
elif proc.returncode != 0:
logger.info(
utils.FAILURE + '' + utils.red('RE') + ': return code %d',
proc.returncode)
status = 'RE'
expected = output_data.decode()
if not simple_match(answer, expected):
logger.info(utils.FAILURE + '' + utils.red('WA'))
logger.info(
utils.NO_HEADER + 'input:\n%s',
pretty_printers.make_pretty_large_file_content(input_data,
limit=40,
head=20,
tail=10,
bold=True))
logger.info(
utils.NO_HEADER + 'output:\n%s',
pretty_printers.make_pretty_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
logger.info(
utils.NO_HEADER + 'expected:\n%s',
pretty_printers.make_pretty_large_file_content(output_data,
limit=40,
head=20,
tail=10,
bold=True))
status = 'WA'
if status == 'AC':
return None
else:
return (input_data, output_data)
def compare_and_report(proc: subprocess.Popen, answer: str,
memory: Optional[float], test_input_path: pathlib.Path,
test_output_path: Optional[pathlib.Path], *,
mle: Optional[float], mode: str, error: Optional[float],
does_print_input: bool, silent: bool, rstrip: bool,
judge: Optional[str]) -> str:
rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf
# prepare the comparing function
if error is not None: # float mode
match = lambda a, b: compare_as_floats(a, b, error)
elif judge is not None: # special judge mode
def match(a: str, b: str) -> bool:
# On Windows, a temp file is not created if we use "with" statement,
user_output = tempfile.NamedTemporaryFile(delete=False)
judge_result = False
try:
if rstrip:
user_output.write(a.rstrip(rstrip_targets).encode())
else:
user_output.write(a.encode())
user_output.close()
arg0 = judge
arg1 = str(test_input_path.resolve())
arg2 = user_output.name
arg3 = str((str(test_output_path.resolve())
if test_output_path is not None else ''))
actual_command = '{} {} {} {}'.format(
arg0, arg1, arg2, arg3
) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584
logger.info('$ %s', actual_command)
info, proc = utils.exec_command(actual_command)
if not silent:
logger.info(
utils.NO_HEADER + 'judge\'s output:\n%s',
utils.make_pretty_large_file_content(info['answer']
or b'',
limit=40,
head=20,
tail=10,
bold=True))
judge_result = (proc.returncode == 0)
finally:
os.unlink(user_output.name)
return judge_result
else:
def match(a: str, b: str) -> bool:
if a == b:
return True
if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets):
logger.warning('WA if no rstrip')
return True
if a == b.replace('\n', '\r\n'):
logger.warning(r'WA if not replacing "\r\n" with "\n"')
return True
if rstrip and a.rstrip(rstrip_targets) == b.replace(
'\n', '\r\n').rstrip(rstrip_targets):
logger.warning('WA if no rstrip')
logger.warning(r'WA if not replacing "\r\n" with "\n"')
return True
if a.replace('\n', '\r\n') == b:
logger.warning(r'WA if not replacing "\n" with "\r\n"')
return True
if rstrip and a.replace(
'\n',
'\r\n').rstrip(rstrip_targets) == b.rstrip(rstrip_targets):
# TODO: use a smart way if you need more equality patterns
logger.warning('WA if no rstrip')
logger.warning(r'WA if not replacing "\n" with "\r\n"')
return True
return False
# prepare the function to print the input
is_input_printed = False
def print_input() -> None:
nonlocal is_input_printed
if does_print_input and not is_input_printed:
is_input_printed = True
with test_input_path.open('rb') as inf:
logger.info(
utils.NO_HEADER + 'input:\n%s',
utils.make_pretty_large_file_content(inf.read(),
limit=40,
head=20,
tail=10,
bold=True))
# check TLE, RE or not
status = 'AC'
if proc.returncode is None:
logger.info(utils.FAILURE + '' + utils.red('TLE'))
status = 'TLE'
print_input()
elif memory is not None and mle is not None and memory > mle:
logger.info(utils.FAILURE + '' + utils.red('MLE'))
status = 'MLE'
print_input()
elif proc.returncode != 0:
logger.info(utils.FAILURE + '' + utils.red('RE') + ': return code %d',
proc.returncode)
status = 'RE'
print_input()
# check WA or not
if (test_output_path is not None) or (judge is not None):
if test_output_path is not None:
with test_output_path.open('rb') as outf:
expected = outf.read().decode()
else: # only if --judge-command option
expected = ''
logger.warning('expected output is not found')
# compare
if not match(answer, expected):
logger.info(utils.FAILURE + '' + utils.red('WA'))
print_input()
if not silent:
if mode == 'simple':
logger.info(
utils.NO_HEADER + 'output:\n%s',
utils.make_pretty_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
logger.info(
utils.NO_HEADER + 'expected:\n%s',
utils.make_pretty_large_file_content(expected.encode(),
limit=40,
head=20,
tail=10,
bold=True))
elif mode == 'side-by-side':
if max(answer.count('\n'), expected.count('\n')) <= 40:
display_side_by_side_color(answer, expected)
else:
display_snipped_side_by_side_color(answer, expected)
else:
assert False
status = 'WA'
else:
if not silent:
header = ('output:\n' if is_input_printed else '')
logger.info(
utils.NO_HEADER + '%s%s', header,
utils.make_pretty_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
if status == 'AC':
logger.info(utils.SUCCESS + '' + utils.green('AC'))
return status
def generate_output_single_case(test_name: str,
test_input_path: pathlib.Path,
*,
lock: Optional[threading.Lock] = None,
args: 'argparse.Namespace') -> None:
# print the header
if lock is None:
logger.info('')
logger.info('%s', test_name)
# run the command
with test_input_path.open() as inf:
info, proc = utils.exec_command(args.command,
stdin=inf,
timeout=args.tle)
answer = info['answer'] # type: Optional[bytes]
elapsed = info['elapsed'] # type: float
# acquire lock to print logs properly, if in parallel
nullcontext = contextlib.ExitStack()
with lock or nullcontext:
if lock is not None:
logger.info('')
logger.info('%s', test_name)
# check the result
logger.info('time: %f sec', elapsed)
if proc.returncode is None:
logger.info(utils.red('TLE'))
logger.info('skipped.')
return
elif proc.returncode != 0:
logger.info('FIALURE: ' + utils.red('RE') + ': return code %d',
proc.returncode)
logger.info('skipped.')
return
assert answer is not None
logger.info(utils.NO_HEADER + '' +
utils.make_pretty_large_file_content(
answer, limit=40, head=20, tail=10, bold=True))
# find the destination path
match_result = fmtutils.match_with_format(
args.directory, args.format,
test_input_path) # type: Optional[Match[Any]]
if match_result is not None:
matched_name = match_result.groupdict()['name'] # type: str
else:
assert False
test_output_path = fmtutils.path_from_format(args.directory,
args.format,
name=matched_name,
ext='out')
# write the result to the file
if not test_output_path.parent.is_dir():
os.makedirs(str(test_output_path.parent), exist_ok=True)
with test_output_path.open('wb') as fh:
fh.write(answer)
logger.info(utils.SUCCESS + 'saved to: %s', test_output_path)
