def test_side_by_side7(self): display_lines = [ logging.red(logging.red_diff('0 2') + ' ' * (self.max_chars - 3)) + '|' + logging.green(''), logging.red(logging.red_diff('1 2') + ' ' * (self.max_chars - 3)) + '|' + logging.green(''), '2 2' + ' ' * (self.max_chars - 3) + '|2 2', '3 2' + ' ' * (self.max_chars - 3) + '|3 2', '4 2' + ' ' * (self.max_chars - 3) + '|4 2', logging.red(' ' * (self.max_chars)) + '|' + logging.green(logging.green_diff('5 2')), logging.red(' ' * (self.max_chars)) + '|' + logging.green(logging.green_diff('6 2')), ] self.snippet_call_test('0 2\n1 2\n2 2\n3 2\n4 2', '2 2\n3 2\n4 2\n5 2\n6 2', display_lines)
def test_side_by_side8(self): display_lines = [ logging.red( logging.red_diff('1') + ' 0 2' + ' ' * (self.max_chars - 5)) + '|' + logging.green(logging.green_diff('2') + ' 0 2'), logging.red('1 ' + logging.red_diff('0') + ' 2' + ' ' * (self.max_chars - 5)) + '|' + logging.green('1 ' + logging.green_diff('1') + ' 2'), logging.red('1 0 ' + logging.red_diff('2') + ' ' * (self.max_chars - 5)) + '|' + logging.green('1 0 ' + logging.green_diff('3')), ] self.snippet_call_test('1 0 2\n1 0 2\n1 0 2', '2 0 2\n1 1 2\n1 0 3', display_lines)
def test_side_by_side4(self): display_lines = [ 'Alice' + ' ' * (self.max_chars - 5) + '|Alice', logging.red('B' + logging.red_diff('0') + 'b' + ' ' * (self.max_chars - 3)) + '|' + logging.green('B' + logging.green_diff('o') + 'b'), 'Alice' + ' ' * (self.max_chars - 5) + '|Alice', ] self.snippet_call_test('Alice\nB0b\nAlice', 'Alice\nBob\nAlice', display_lines)
def display_snipped_side_by_side_color(answer: str, expected: str): """ Display first differ line and its previous 3 lines and its next 3 lines. """ max_chars = shutil.get_terminal_size()[0] // 2 - 2 deq = collections.deque( maxlen=7 ) # type: Deque[Tuple[Optional[int], bool, str, str, int, int]] count_from_first_difference = 0 i = 0 for flag, diff_found, ans_line, exp_line, ans_chars, exp_chars in side_by_side_diff( answer, expected): if flag: i += 1 if count_from_first_difference > 0: count_from_first_difference += 1 line_num = i if flag else None deq.append( (line_num, diff_found, ans_line, exp_line, ans_chars, exp_chars)) if diff_found: if count_from_first_difference == 0: count_from_first_difference = 1 if count_from_first_difference == 4: break max_line_num_digits = max( [len(str(entry[0])) for entry in deq if entry[0] is not None]) log.emit(" " * max_line_num_digits + "|output:" + " " * (max_chars - 7 - max_line_num_digits - 1) + "|" + "expected:") log.emit("-" * max_chars + "|" + "-" * max_chars) last_line_number = 0 for (line_number, diff_found, ans_line, exp_line, ans_chars, exp_chars) in deq: num_spaces_after_output = max_chars - ans_chars - max_line_num_digits - 1 line_number_str = str(line_number) if line_number is not None else "" line_num_display = space_padding( line_number_str, max_line_num_digits - len(line_number_str)) + "|" if not diff_found: log.emit(line_num_display + space_padding(ans_line, num_spaces_after_output) + "|" + exp_line) else: log.emit( line_num_display + log.red(space_padding(ans_line, num_spaces_after_output)) + "|" + log.green(exp_line)) if line_number is not None: last_line_number = line_number num_snipped_lines = answer.count('\n') + 1 - last_line_number if num_snipped_lines > 0: log.emit('... ({} lines) ...'.format(num_snipped_lines))
def display_side_by_side_color(answer: str, expected: str): max_chars = shutil.get_terminal_size()[0] // 2 - 2 log.emit("output:" + " " * (max_chars - 7) + "|" + "expected:") log.emit("-" * max_chars + "|" + "-" * max_chars) for _, diff_found, ans_line, exp_line, ans_chars, exp_chars in side_by_side_diff( answer, expected): if diff_found: log.emit( log.red(space_padding(ans_line, max_chars - ans_chars)) + "|" + log.green(exp_line)) else: log.emit( space_padding(ans_line, max_chars - ans_chars) + "|" + exp_line)
def display_side_by_side_color(answer: str, expected: str): def space_padding(s: str, max_length: int) -> str: return s + " " * max_length max_chars = shutil.get_terminal_size()[0] // 2 - 2 log.emit("output:" + " " * (max_chars - 7) + "|" + "expected:") log.emit("-" * max_chars + "|" + "-" * max_chars) for i, (diff_found, ans_line, exp_line, ans_chars, exp_chars) in enumerate(side_by_side_diff(answer, expected)): if diff_found: log.emit( log.red(space_padding(ans_line, max_chars - ans_chars)) + "|" + log.green(exp_line)) else: log.emit( space_padding(ans_line, max_chars - ans_chars) + "|" + exp_line)
def compare_and_report(proc: subprocess.Popen, answer: str, memory: Optional[float], test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, mle: Optional[float], mode: str, error: Optional[float], does_print_input: bool, silent: bool, rstrip: bool, judge: Optional[str]) -> str: rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf # prepare the comparing function if error is not None: # float mode match = lambda a, b: compare_as_floats(a, b, error) elif judge is not None: # special judge mode def match(a, b): # On Windows, a temp file is not created if we use "with" statement, user_output = tempfile.NamedTemporaryFile(delete=False) judge_result = False try: if rstrip: user_output.write(a.rstrip(rstrip_targets).encode()) else: user_output.write(a.encode()) user_output.close() arg0 = judge arg1 = str(test_input_path.resolve()) arg2 = user_output.name arg3 = str((str(test_output_path.resolve()) if test_output_path is not None else '')) actual_command = '{} {} {} {}'.format( arg0, arg1, arg2, arg3 ) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584 log.status('$ %s', actual_command) info, proc = utils.exec_command(actual_command) if not silent: log.emit( 'judge\'s output:\n%s', utils.snip_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) judge_result = (proc.returncode == 0) finally: os.unlink(user_output.name) return judge_result else: def match(a, b): if a == b: return True if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets): log.warning('WA if no rstrip') return True return False # prepare the function to print the input is_input_printed = False def print_input(): nonlocal is_input_printed if does_print_input and not is_input_printed: is_input_printed = True with test_input_path.open('rb') as inf: log.emit( 'input:\n%s', utils.snip_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) # check TLE, RE or not status = 'AC' if proc.returncode is None: log.failure(log.red('TLE')) status = 'TLE' print_input() elif memory is not None and mle is not None and memory > mle: log.failure(log.red('MLE')) status = 'MLE' print_input() elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) status = 'RE' print_input() # check WA or not if (test_output_path is not None) or (judge is not None): if test_output_path is not None: with test_output_path.open('rb') as outf: expected = outf.read().decode() else: # only if --judge-command option expected = '' log.warning('expected output is not found') # compare if not match(answer, expected): log.failure(log.red('WA')) print_input() if not silent: if mode == "simple": log.emit( 'output:\n%s', utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit( 'expected:\n%s', utils.snip_large_file_content(expected.encode(), limit=40, head=20, tail=10, bold=True)) elif mode == "side-by-side": display_side_by_side_color(answer, expected) else: assert False status = 'WA' else: if not silent: log.emit(('output:\n%s' if is_input_printed else '%s'), utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if status == 'AC': log.success(log.green('AC')) return status
def test(args: 'argparse.Namespace') -> None: # list tests if not args.test: args.test = fmtutils.glob_with_format(args.directory, args.format) # by default if args.ignore_backup: args.test = fmtutils.drop_backup_or_hidden_files(args.test) tests = fmtutils.construct_relationship_of_files(args.test, args.directory, args.format) # check wheather GNU time is available if not check_gnu_time(args.gnu_time): log.warning('GNU time is not available: %s', args.gnu_time) args.gnu_time = None if args.mle is not None and args.gnu_time is None: raise RuntimeError('--mle is used but GNU time does not exist') # run tests history = [] # type: List[Dict[str, Any]] if args.jobs is None: for name, paths in sorted(tests.items()): history += [ test_single_case(name, paths['in'], paths.get('out'), args=args) ] else: if os.name == 'nt': log.warning("-j/--jobs opiton is unstable on Windows environmet") with concurrent.futures.ThreadPoolExecutor( max_workers=args.jobs) as executor: lock = threading.Lock() futures = [] # type: List[concurrent.futures.Future] for name, paths in sorted(tests.items()): futures += [ executor.submit(test_single_case, name, paths['in'], paths.get('out'), lock=lock, args=args) ] for future in futures: history += [future.result()] # summarize slowest = -1.0 # type: float slowest_name = '' heaviest = -1.0 # type: float heaviest_name = '' ac_count = 0 for result in history: if result['status'] == 'AC': ac_count += 1 if slowest < result['elapsed']: slowest = result['elapsed'] slowest_name = result['testcase']['name'] if result['memory'] is not None and heaviest < result['memory']: heaviest = result['memory'] heaviest_name = result['testcase']['name'] # print the summary log.emit('') log.status('slowest: %f sec (for %s)', slowest, slowest_name) if heaviest >= 0: if heaviest < MEMORY_WARNING: log.status('max memory: %f MB (for %s)', heaviest, heaviest_name) else: log.warning('max memory: %f MB (for %s)', heaviest, heaviest_name) if ac_count == len(tests): log.success('test ' + log.green('success') + ': %d cases', len(tests)) else: log.failure('test ' + log.red('failed') + ': %d AC / %d cases', ac_count, len(tests)) if args.json: print(json.dumps(history)) if ac_count != len(tests): sys.exit(1)
def test(args: 'argparse.Namespace') -> None: # prepare if not args.test: args.test = cutils.glob_with_format(args.directory, args.format) # by default if args.ignore_backup: args.test = cutils.drop_backup_or_hidden_files(args.test) tests = cutils.construct_relationship_of_files(args.test, args.directory, args.format) if args.error: # float mode match = lambda a, b: compare_as_floats(a, b, args.error) else: def match(a, b): if a == b: return True if args.rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets): log.warning('WA if no rstrip') return True return False rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf slowest = -1 # type: Union[int, float] slowest_name = '' ac_count = 0 history = [] # type: List[Dict[str, Any]] for name, it in sorted(tests.items()): is_input_printed = False def print_input(): nonlocal is_input_printed if args.print_input and not is_input_printed: is_input_printed = True with open(it['in'], 'rb') as inf: log.emit('input:\n%s', utils.snip_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) log.emit('') log.info('%s', name) # run the binary with it['in'].open() as inf: begin = time.perf_counter() answer_byte, proc = utils.exec_command(args.command, shell=True, stdin=inf, timeout=args.tle) end = time.perf_counter() elapsed = end - begin answer = answer_byte.decode() # TODO: the `answer` should be bytes, not str if slowest < elapsed: slowest = elapsed slowest_name = name log.status('time: %f sec', elapsed) proc.terminate() # check TLE, RE or not result = 'AC' if proc.returncode is None: log.failure(log.red('TLE')) result = 'TLE' print_input() elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) result = 'RE' print_input() # check WA or not if 'out' in it: with it['out'].open() as outf: correct = outf.read() # compare if args.mode == 'all': if not match(answer, correct): log.failure(log.red('WA')) print_input() if not args.silent: log.emit('output:\n%s', utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit('expected:\n%s', utils.snip_large_file_content(correct.encode(), limit=40, head=20, tail=10, bold=True)) result = 'WA' elif args.mode == 'line': answer_words = answer.splitlines() correct_words = correct.splitlines() for i, (x, y) in enumerate(zip(answer_words + [None] * len(correct_words), correct_words + [None] * len(answer_words))): # type: ignore if x is None and y is None: break elif x is None: print_input() log.failure(log.red('WA') + ': line %d: line is nothing: expected "%s"', i + 1, log.bold(y)) result = 'WA' elif y is None: print_input() log.failure(log.red('WA') + ': line %d: unexpected line: output "%s"', i + 1, log.bold(x)) result = 'WA' elif not match(x, y): print_input() log.failure(log.red('WA') + ': line %d: output "%s": expected "%s"', i + 1, log.bold(x), log.bold(y)) result = 'WA' else: assert False else: if not args.silent: log.emit(('output:\n%s' if is_input_printed else '%s'), utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if result == 'AC': log.success(log.green('AC')) ac_count += 1 # push the result testcase = { 'name': name, 'input': str(it['in'].resolve()), } if 'out' in it: testcase['output'] = str(it['out'].resolve()) history += [{ 'result': result, 'testcase': testcase, 'output': answer, 'exitcode': proc.returncode, 'elapsed': elapsed, }] # summarize log.emit('') log.status('slowest: %f sec (for %s)', slowest, slowest_name) if ac_count == len(tests): log.success('test ' + log.green('success') + ': %d cases', len(tests)) else: log.failure('test ' + log.red('failed') + ': %d AC / %d cases', ac_count, len(tests)) if args.json: print(json.dumps(history)) if ac_count != len(tests): sys.exit(1)
def test_side_by_side2(self): display_lines = [ '98 |Alice' + ' ' * (self.max_chars - 9) + '|Alice', '99 |Bob' + ' ' * (self.max_chars - 7) + '|Bob', '100|Alice' + ' ' * (self.max_chars - 9) + '|Alice', '101|' + logging.red(logging.red_diff('Bob') + ' ' * (self.max_chars - 7)) + '|' + logging.green(logging.green_diff('John')), '102|' + logging.red(logging.red_diff('Alice') + ' ' * (self.max_chars - 9)) + '|' + logging.green(logging.green_diff('John')), '103|Bob' + ' ' * (self.max_chars - 7) + '|Bob', '104|Alice' + ' ' * (self.max_chars - 9) + '|Alice', '... (1 lines) ...', ] output = ('\n' * 97 + 'Alice\nBob\nAlice\nBob\nAlice\nBob\nAlice\nBob').replace('\n', os.linesep) expect = ('\n' * 97 + 'Alice\nBob\nAlice\nJohn\nJohn\nBob\nAlice\nBob').replace('\n', os.linesep) self.snippet_call_test(output, expect, display_lines, 3)
def test_side_by_side1(self): display_lines = [ '41|Alice' + ' ' * (self.max_chars - 8) + '|Alice', '42|Bob' + ' ' * (self.max_chars - 6) + '|Bob', '43|Alice' + ' ' * (self.max_chars - 8) + '|Alice', '44|' + logging.red(logging.red_diff('Bob') + ' ' * (self.max_chars - 6)) + '|' + logging.green(logging.green_diff('John')), '45|' + logging.red(logging.red_diff('Alice') + ' ' * (self.max_chars - 8)) + '|' + logging.green(logging.green_diff('John')), '46|Bob' + ' ' * (self.max_chars - 6) + '|Bob', '47|Alice' + ' ' * (self.max_chars - 8) + '|Alice', '... (1 lines) ...', ] output = ('\n' * 40 + 'Alice\nBob\nAlice\nBob\nAlice\nBob\nAlice\nBob').replace('\n', os.linesep) expect = ('\n' * 40 + 'Alice\nBob\nAlice\nJohn\nJohn\nBob\nAlice\nBob').replace('\n', os.linesep) self.snippet_call_test(output, expect, display_lines, 2)
def test_side_by_side2(self): self.snippet_call_test('kmy', 'kmv', (logging.red('km' + logging.red_diff('y') + ' ' * (self.max_chars - 3)) + '|' + logging.green('km' + logging.green_diff('v')), ))
def compare_and_report(proc: subprocess.Popen, answer: str, elapsed: float, memory: Optional[float], test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, mle: Optional[float], mode: str, error: Optional[float], does_print_input: bool, silent: bool, rstrip: bool) -> str: # prepare the comparing function if error: # float mode match = lambda a, b: compare_as_floats(a, b, error) else: rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf def match(a, b): if a == b: return True if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets): log.warning('WA if no rstrip') return True return False # prepare the function to print the input is_input_printed = False def print_input(): nonlocal is_input_printed if does_print_input and not is_input_printed: is_input_printed = True with test_input_path.open('rb') as inf: log.emit( 'input:\n%s', utils.snip_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) # check TLE, RE or not status = 'AC' if proc.returncode is None: log.failure(log.red('TLE')) status = 'TLE' print_input() elif memory is not None and mle is not None and memory > mle: log.failure(log.red('MLE')) status = 'MLE' print_input() elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) status = 'RE' print_input() # check WA or not if test_output_path is not None: with test_output_path.open() as outf: expected = outf.read() # compare if mode == 'all': if not match(answer, expected): log.failure(log.red('WA')) print_input() if not silent: log.emit( 'output:\n%s', utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit( 'expected:\n%s', utils.snip_large_file_content(expected.encode(), limit=40, head=20, tail=10, bold=True)) status = 'WA' elif mode == 'line': answer_words = answer.splitlines() correct_words = expected.splitlines() for i, (x, y) in enumerate( zip(answer_words + [None] * len(correct_words), correct_words + [None] * len(answer_words))): # type: ignore if x is None and y is None: break elif x is None: print_input() log.failure( log.red('WA') + ': line %d: line is nothing: expected "%s"', i + 1, log.bold(y)) status = 'WA' elif y is None: print_input() log.failure( log.red('WA') + ': line %d: unexpected line: output "%s"', i + 1, log.bold(x)) status = 'WA' elif not match(x, y): print_input() log.failure( log.red('WA') + ': line %d: output "%s": expected "%s"', i + 1, log.bold(x), log.bold(y)) status = 'WA' else: assert False else: if not silent: log.emit(('output:\n%s' if is_input_printed else '%s'), utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if status == 'AC': log.success(log.green('AC')) return status