def test_side_by_side7(self):
display_lines = [
logging.red(logging.red_diff('0 2') + ' ' * (self.max_chars - 3)) + '|' + logging.green(''),
logging.red(logging.red_diff('1 2') + ' ' * (self.max_chars - 3)) + '|' + logging.green(''),
'2 2' + ' ' * (self.max_chars - 3) + '|2 2',
'3 2' + ' ' * (self.max_chars - 3) + '|3 2',
'4 2' + ' ' * (self.max_chars - 3) + '|4 2',
logging.red(' ' * (self.max_chars)) + '|' + logging.green(logging.green_diff('5 2')),
logging.red(' ' * (self.max_chars)) + '|' + logging.green(logging.green_diff('6 2')),
]
self.snippet_call_test('0 2\n1 2\n2 2\n3 2\n4 2', '2 2\n3 2\n4 2\n5 2\n6 2', display_lines)
def test_side_by_side8(self):
display_lines = [
logging.red(
logging.red_diff('1') + ' 0 2' + ' ' * (self.max_chars - 5)) +
'|' + logging.green(logging.green_diff('2') + ' 0 2'),
logging.red('1 ' + logging.red_diff('0') + ' 2' + ' ' *
(self.max_chars - 5)) + '|' +
logging.green('1 ' + logging.green_diff('1') + ' 2'),
logging.red('1 0 ' + logging.red_diff('2') + ' ' *
(self.max_chars - 5)) + '|' +
logging.green('1 0 ' + logging.green_diff('3')),
]
self.snippet_call_test('1 0 2\n1 0 2\n1 0 2', '2 0 2\n1 1 2\n1 0 3',
display_lines)
def test_side_by_side4(self):
display_lines = [
'Alice' + ' ' * (self.max_chars - 5) + '|Alice',
logging.red('B' + logging.red_diff('0') + 'b' + ' ' *
(self.max_chars - 3)) + '|' +
logging.green('B' + logging.green_diff('o') + 'b'),
'Alice' + ' ' * (self.max_chars - 5) + '|Alice',
]
self.snippet_call_test('Alice\nB0b\nAlice', 'Alice\nBob\nAlice',
display_lines)
def display_snipped_side_by_side_color(answer: str, expected: str):
"""
Display first differ line and its previous 3 lines and its next 3 lines.
"""
max_chars = shutil.get_terminal_size()[0] // 2 - 2
deq = collections.deque(
maxlen=7
) # type: Deque[Tuple[Optional[int], bool, str, str, int, int]]
count_from_first_difference = 0
i = 0
for flag, diff_found, ans_line, exp_line, ans_chars, exp_chars in side_by_side_diff(
answer, expected):
if flag: i += 1
if count_from_first_difference > 0:
count_from_first_difference += 1
line_num = i if flag else None
deq.append(
(line_num, diff_found, ans_line, exp_line, ans_chars, exp_chars))
if diff_found:
if count_from_first_difference == 0:
count_from_first_difference = 1
if count_from_first_difference == 4:
break
max_line_num_digits = max(
[len(str(entry[0])) for entry in deq if entry[0] is not None])
log.emit(" " * max_line_num_digits + "|output:" + " " *
(max_chars - 7 - max_line_num_digits - 1) + "|" + "expected:")
log.emit("-" * max_chars + "|" + "-" * max_chars)
last_line_number = 0
for (line_number, diff_found, ans_line, exp_line, ans_chars,
exp_chars) in deq:
num_spaces_after_output = max_chars - ans_chars - max_line_num_digits - 1
line_number_str = str(line_number) if line_number is not None else ""
line_num_display = space_padding(
line_number_str, max_line_num_digits - len(line_number_str)) + "|"
if not diff_found:
log.emit(line_num_display +
space_padding(ans_line, num_spaces_after_output) + "|" +
exp_line)
else:
log.emit(
line_num_display +
log.red(space_padding(ans_line, num_spaces_after_output)) +
"|" + log.green(exp_line))
if line_number is not None:
last_line_number = line_number
num_snipped_lines = answer.count('\n') + 1 - last_line_number
if num_snipped_lines > 0:
log.emit('... ({} lines) ...'.format(num_snipped_lines))
def display_side_by_side_color(answer: str, expected: str):
max_chars = shutil.get_terminal_size()[0] // 2 - 2
log.emit("output:" + " " * (max_chars - 7) + "|" + "expected:")
log.emit("-" * max_chars + "|" + "-" * max_chars)
for _, diff_found, ans_line, exp_line, ans_chars, exp_chars in side_by_side_diff(
answer, expected):
if diff_found:
log.emit(
log.red(space_padding(ans_line, max_chars - ans_chars)) + "|" +
log.green(exp_line))
else:
log.emit(
space_padding(ans_line, max_chars - ans_chars) + "|" +
exp_line)
def display_side_by_side_color(answer: str, expected: str):
def space_padding(s: str, max_length: int) -> str:
return s + " " * max_length
max_chars = shutil.get_terminal_size()[0] // 2 - 2
log.emit("output:" + " " * (max_chars - 7) + "|" + "expected:")
log.emit("-" * max_chars + "|" + "-" * max_chars)
for i, (diff_found, ans_line, exp_line, ans_chars,
exp_chars) in enumerate(side_by_side_diff(answer, expected)):
if diff_found:
log.emit(
log.red(space_padding(ans_line, max_chars - ans_chars)) + "|" +
log.green(exp_line))
else:
log.emit(
space_padding(ans_line, max_chars - ans_chars) + "|" +
exp_line)
def compare_and_report(proc: subprocess.Popen, answer: str,
memory: Optional[float], test_input_path: pathlib.Path,
test_output_path: Optional[pathlib.Path], *,
mle: Optional[float], mode: str, error: Optional[float],
does_print_input: bool, silent: bool, rstrip: bool,
judge: Optional[str]) -> str:
rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf
# prepare the comparing function
if error is not None: # float mode
match = lambda a, b: compare_as_floats(a, b, error)
elif judge is not None: # special judge mode
def match(a, b):
# On Windows, a temp file is not created if we use "with" statement,
user_output = tempfile.NamedTemporaryFile(delete=False)
judge_result = False
try:
if rstrip:
user_output.write(a.rstrip(rstrip_targets).encode())
else:
user_output.write(a.encode())
user_output.close()
arg0 = judge
arg1 = str(test_input_path.resolve())
arg2 = user_output.name
arg3 = str((str(test_output_path.resolve())
if test_output_path is not None else ''))
actual_command = '{} {} {} {}'.format(
arg0, arg1, arg2, arg3
) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584
log.status('$ %s', actual_command)
info, proc = utils.exec_command(actual_command)
if not silent:
log.emit(
'judge\'s output:\n%s',
utils.snip_large_file_content(info['answer'] or b'',
limit=40,
head=20,
tail=10,
bold=True))
judge_result = (proc.returncode == 0)
finally:
os.unlink(user_output.name)
return judge_result
else:
def match(a, b):
if a == b:
return True
if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets):
log.warning('WA if no rstrip')
return True
return False
# prepare the function to print the input
is_input_printed = False
def print_input():
nonlocal is_input_printed
if does_print_input and not is_input_printed:
is_input_printed = True
with test_input_path.open('rb') as inf:
log.emit(
'input:\n%s',
utils.snip_large_file_content(inf.read(),
limit=40,
head=20,
tail=10,
bold=True))
# check TLE, RE or not
status = 'AC'
if proc.returncode is None:
log.failure(log.red('TLE'))
status = 'TLE'
print_input()
elif memory is not None and mle is not None and memory > mle:
log.failure(log.red('MLE'))
status = 'MLE'
print_input()
elif proc.returncode != 0:
log.failure(log.red('RE') + ': return code %d', proc.returncode)
status = 'RE'
print_input()
# check WA or not
if (test_output_path is not None) or (judge is not None):
if test_output_path is not None:
with test_output_path.open('rb') as outf:
expected = outf.read().decode()
else: # only if --judge-command option
expected = ''
log.warning('expected output is not found')
# compare
if not match(answer, expected):
log.failure(log.red('WA'))
print_input()
if not silent:
if mode == "simple":
log.emit(
'output:\n%s',
utils.snip_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
log.emit(
'expected:\n%s',
utils.snip_large_file_content(expected.encode(),
limit=40,
head=20,
tail=10,
bold=True))
elif mode == "side-by-side":
display_side_by_side_color(answer, expected)
else:
assert False
status = 'WA'
else:
if not silent:
log.emit(('output:\n%s' if is_input_printed else '%s'),
utils.snip_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
if status == 'AC':
log.success(log.green('AC'))
return status
def test(args: 'argparse.Namespace') -> None:
# list tests
if not args.test:
args.test = fmtutils.glob_with_format(args.directory,
args.format) # by default
if args.ignore_backup:
args.test = fmtutils.drop_backup_or_hidden_files(args.test)
tests = fmtutils.construct_relationship_of_files(args.test, args.directory,
args.format)
# check wheather GNU time is available
if not check_gnu_time(args.gnu_time):
log.warning('GNU time is not available: %s', args.gnu_time)
args.gnu_time = None
if args.mle is not None and args.gnu_time is None:
raise RuntimeError('--mle is used but GNU time does not exist')
# run tests
history = [] # type: List[Dict[str, Any]]
if args.jobs is None:
for name, paths in sorted(tests.items()):
history += [
test_single_case(name,
paths['in'],
paths.get('out'),
args=args)
]
else:
if os.name == 'nt':
log.warning("-j/--jobs opiton is unstable on Windows environmet")
with concurrent.futures.ThreadPoolExecutor(
max_workers=args.jobs) as executor:
lock = threading.Lock()
futures = [] # type: List[concurrent.futures.Future]
for name, paths in sorted(tests.items()):
futures += [
executor.submit(test_single_case,
name,
paths['in'],
paths.get('out'),
lock=lock,
args=args)
]
for future in futures:
history += [future.result()]
# summarize
slowest = -1.0 # type: float
slowest_name = ''
heaviest = -1.0 # type: float
heaviest_name = ''
ac_count = 0
for result in history:
if result['status'] == 'AC':
ac_count += 1
if slowest < result['elapsed']:
slowest = result['elapsed']
slowest_name = result['testcase']['name']
if result['memory'] is not None and heaviest < result['memory']:
heaviest = result['memory']
heaviest_name = result['testcase']['name']
# print the summary
log.emit('')
log.status('slowest: %f sec (for %s)', slowest, slowest_name)
if heaviest >= 0:
if heaviest < MEMORY_WARNING:
log.status('max memory: %f MB (for %s)', heaviest, heaviest_name)
else:
log.warning('max memory: %f MB (for %s)', heaviest, heaviest_name)
if ac_count == len(tests):
log.success('test ' + log.green('success') + ': %d cases', len(tests))
else:
log.failure('test ' + log.red('failed') + ': %d AC / %d cases',
ac_count, len(tests))
if args.json:
print(json.dumps(history))
if ac_count != len(tests):
sys.exit(1)
def test(args: 'argparse.Namespace') -> None:
# prepare
if not args.test:
args.test = cutils.glob_with_format(args.directory, args.format) # by default
if args.ignore_backup:
args.test = cutils.drop_backup_or_hidden_files(args.test)
tests = cutils.construct_relationship_of_files(args.test, args.directory, args.format)
if args.error: # float mode
match = lambda a, b: compare_as_floats(a, b, args.error)
else:
def match(a, b):
if a == b:
return True
if args.rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets):
log.warning('WA if no rstrip')
return True
return False
rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf
slowest = -1 # type: Union[int, float]
slowest_name = ''
ac_count = 0
history = [] # type: List[Dict[str, Any]]
for name, it in sorted(tests.items()):
is_input_printed = False
def print_input():
nonlocal is_input_printed
if args.print_input and not is_input_printed:
is_input_printed = True
with open(it['in'], 'rb') as inf:
log.emit('input:\n%s', utils.snip_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True))
log.emit('')
log.info('%s', name)
# run the binary
with it['in'].open() as inf:
begin = time.perf_counter()
answer_byte, proc = utils.exec_command(args.command, shell=True, stdin=inf, timeout=args.tle)
end = time.perf_counter()
elapsed = end - begin
answer = answer_byte.decode() # TODO: the `answer` should be bytes, not str
if slowest < elapsed:
slowest = elapsed
slowest_name = name
log.status('time: %f sec', elapsed)
proc.terminate()
# check TLE, RE or not
result = 'AC'
if proc.returncode is None:
log.failure(log.red('TLE'))
result = 'TLE'
print_input()
elif proc.returncode != 0:
log.failure(log.red('RE') + ': return code %d', proc.returncode)
result = 'RE'
print_input()
# check WA or not
if 'out' in it:
with it['out'].open() as outf:
correct = outf.read()
# compare
if args.mode == 'all':
if not match(answer, correct):
log.failure(log.red('WA'))
print_input()
if not args.silent:
log.emit('output:\n%s', utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True))
log.emit('expected:\n%s', utils.snip_large_file_content(correct.encode(), limit=40, head=20, tail=10, bold=True))
result = 'WA'
elif args.mode == 'line':
answer_words = answer.splitlines()
correct_words = correct.splitlines()
for i, (x, y) in enumerate(zip(answer_words + [None] * len(correct_words), correct_words + [None] * len(answer_words))): # type: ignore
if x is None and y is None:
break
elif x is None:
print_input()
log.failure(log.red('WA') + ': line %d: line is nothing: expected "%s"', i + 1, log.bold(y))
result = 'WA'
elif y is None:
print_input()
log.failure(log.red('WA') + ': line %d: unexpected line: output "%s"', i + 1, log.bold(x))
result = 'WA'
elif not match(x, y):
print_input()
log.failure(log.red('WA') + ': line %d: output "%s": expected "%s"', i + 1, log.bold(x), log.bold(y))
result = 'WA'
else:
assert False
else:
if not args.silent:
log.emit(('output:\n%s' if is_input_printed else '%s'), utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True))
if result == 'AC':
log.success(log.green('AC'))
ac_count += 1
# push the result
testcase = {
'name': name,
'input': str(it['in'].resolve()),
}
if 'out' in it:
testcase['output'] = str(it['out'].resolve())
history += [{
'result': result,
'testcase': testcase,
'output': answer,
'exitcode': proc.returncode,
'elapsed': elapsed,
}]
# summarize
log.emit('')
log.status('slowest: %f sec (for %s)', slowest, slowest_name)
if ac_count == len(tests):
log.success('test ' + log.green('success') + ': %d cases', len(tests))
else:
log.failure('test ' + log.red('failed') + ': %d AC / %d cases', ac_count, len(tests))
if args.json:
print(json.dumps(history))
if ac_count != len(tests):
sys.exit(1)
def test_side_by_side2(self):
display_lines = [
'98 |Alice' + ' ' * (self.max_chars - 9) + '|Alice',
'99 |Bob' + ' ' * (self.max_chars - 7) + '|Bob',
'100|Alice' + ' ' * (self.max_chars - 9) + '|Alice',
'101|' + logging.red(logging.red_diff('Bob') + ' ' * (self.max_chars - 7)) + '|' + logging.green(logging.green_diff('John')),
'102|' + logging.red(logging.red_diff('Alice') + ' ' * (self.max_chars - 9)) + '|' + logging.green(logging.green_diff('John')),
'103|Bob' + ' ' * (self.max_chars - 7) + '|Bob',
'104|Alice' + ' ' * (self.max_chars - 9) + '|Alice',
'... (1 lines) ...',
]
output = ('\n' * 97 + 'Alice\nBob\nAlice\nBob\nAlice\nBob\nAlice\nBob').replace('\n', os.linesep)
expect = ('\n' * 97 + 'Alice\nBob\nAlice\nJohn\nJohn\nBob\nAlice\nBob').replace('\n', os.linesep)
self.snippet_call_test(output, expect, display_lines, 3)
def test_side_by_side1(self):
display_lines = [
'41|Alice' + ' ' * (self.max_chars - 8) + '|Alice',
'42|Bob' + ' ' * (self.max_chars - 6) + '|Bob',
'43|Alice' + ' ' * (self.max_chars - 8) + '|Alice',
'44|' + logging.red(logging.red_diff('Bob') + ' ' * (self.max_chars - 6)) + '|' + logging.green(logging.green_diff('John')),
'45|' + logging.red(logging.red_diff('Alice') + ' ' * (self.max_chars - 8)) + '|' + logging.green(logging.green_diff('John')),
'46|Bob' + ' ' * (self.max_chars - 6) + '|Bob',
'47|Alice' + ' ' * (self.max_chars - 8) + '|Alice',
'... (1 lines) ...',
]
output = ('\n' * 40 + 'Alice\nBob\nAlice\nBob\nAlice\nBob\nAlice\nBob').replace('\n', os.linesep)
expect = ('\n' * 40 + 'Alice\nBob\nAlice\nJohn\nJohn\nBob\nAlice\nBob').replace('\n', os.linesep)
self.snippet_call_test(output, expect, display_lines, 2)
def test_side_by_side2(self):
self.snippet_call_test('kmy', 'kmv', (logging.red('km' + logging.red_diff('y') + ' ' * (self.max_chars - 3)) + '|' + logging.green('km' + logging.green_diff('v')), ))
def compare_and_report(proc: subprocess.Popen, answer: str, elapsed: float,
memory: Optional[float], test_input_path: pathlib.Path,
test_output_path: Optional[pathlib.Path], *,
mle: Optional[float], mode: str, error: Optional[float],
does_print_input: bool, silent: bool,
rstrip: bool) -> str:
# prepare the comparing function
if error: # float mode
match = lambda a, b: compare_as_floats(a, b, error)
else:
rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf
def match(a, b):
if a == b:
return True
if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets):
log.warning('WA if no rstrip')
return True
return False
# prepare the function to print the input
is_input_printed = False
def print_input():
nonlocal is_input_printed
if does_print_input and not is_input_printed:
is_input_printed = True
with test_input_path.open('rb') as inf:
log.emit(
'input:\n%s',
utils.snip_large_file_content(inf.read(),
limit=40,
head=20,
tail=10,
bold=True))
# check TLE, RE or not
status = 'AC'
if proc.returncode is None:
log.failure(log.red('TLE'))
status = 'TLE'
print_input()
elif memory is not None and mle is not None and memory > mle:
log.failure(log.red('MLE'))
status = 'MLE'
print_input()
elif proc.returncode != 0:
log.failure(log.red('RE') + ': return code %d', proc.returncode)
status = 'RE'
print_input()
# check WA or not
if test_output_path is not None:
with test_output_path.open() as outf:
expected = outf.read()
# compare
if mode == 'all':
if not match(answer, expected):
log.failure(log.red('WA'))
print_input()
if not silent:
log.emit(
'output:\n%s',
utils.snip_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
log.emit(
'expected:\n%s',
utils.snip_large_file_content(expected.encode(),
limit=40,
head=20,
tail=10,
bold=True))
status = 'WA'
elif mode == 'line':
answer_words = answer.splitlines()
correct_words = expected.splitlines()
for i, (x, y) in enumerate(
zip(answer_words + [None] * len(correct_words),
correct_words +
[None] * len(answer_words))): # type: ignore
if x is None and y is None:
break
elif x is None:
print_input()
log.failure(
log.red('WA') +
': line %d: line is nothing: expected "%s"', i + 1,
log.bold(y))
status = 'WA'
elif y is None:
print_input()
log.failure(
log.red('WA') +
': line %d: unexpected line: output "%s"', i + 1,
log.bold(x))
status = 'WA'
elif not match(x, y):
print_input()
log.failure(
log.red('WA') +
': line %d: output "%s": expected "%s"', i + 1,
log.bold(x), log.bold(y))
status = 'WA'
else:
assert False
else:
if not silent:
log.emit(('output:\n%s' if is_input_printed else '%s'),
utils.snip_large_file_content(answer.encode(),
limit=40,
head=20,
tail=10,
bold=True))
if status == 'AC':
log.success(log.green('AC'))
return status
