def display_side_by_side_color(answer: str, expected: str) -> None: max_chars = shutil.get_terminal_size()[0] // 2 - 2 logger.info(utils.NO_HEADER + 'output:' + " " * (max_chars - 7) + "|" + "expected:") logger.info(utils.NO_HEADER + '%s', "-" * max_chars + "|" + "-" * max_chars) for _, diff_found, ans_line, exp_line, ans_chars, exp_chars in _side_by_side_diff( answer, expected): if diff_found: logger.info( utils.NO_HEADER + '%s', utils.red(_space_padding(ans_line, max_chars - ans_chars)) + "|" + utils.green(exp_line)) else: logger.info( utils.NO_HEADER + '%s', _space_padding(ans_line, max_chars - ans_chars) + "|" + exp_line)
def downloadSingleCase(url: str, target_dir: str, out: bool = False) -> None: os.makedirs(target_dir, exist_ok=True) URL = re.sub(r'dl=0', 'dl=1', url) path = urlparse(URL).path suffix = PurePath(path).suffix filename = PurePath(path).name if suffix: filename = filename[:-len(suffix)] # Remove suffix filename += ".out" if out else ".in" log.info(f"Download: {URL}") r = session.get(URL, headers=agent) if r.status_code != requests.codes.ok: log.error(utils.FAILURE + utils.red("Download failed.")) exit(1) else: log.info(utils.SUCCESS + "Download Succeeded") target_file = target_dir + '/' + filename log.info(f"Save to: {target_file}") with open(target_file, "w") as fw: fw.write(r.text)
def display_snipped_side_by_side_color(answer: str, expected: str) -> None: """ Display first differ line and its previous 3 lines and its next 3 lines. """ max_chars = shutil.get_terminal_size()[0] // 2 - 2 deq: Deque[Tuple[Optional[int], bool, str, str, int, int]] = collections.deque(maxlen=7) count_from_first_difference = 0 i = 0 for flag, diff_found, ans_line, exp_line, ans_chars, exp_chars in _side_by_side_diff(answer, expected): if flag: i += 1 if count_from_first_difference > 0: count_from_first_difference += 1 line_num = i if flag else None deq.append((line_num, diff_found, ans_line, exp_line, ans_chars, exp_chars)) if diff_found: if count_from_first_difference == 0: count_from_first_difference = 1 if count_from_first_difference == 4: break max_line_num_digits = max([len(str(entry[0])) for entry in deq if entry[0] is not None]) logger.info(utils.NO_HEADER + '%s', " " * max_line_num_digits + "|output:" + " " * (max_chars - 7 - max_line_num_digits - 1) + "|" + "expected:") logger.info(utils.NO_HEADER + '%s', "-" * max_chars + "|" + "-" * max_chars) last_line_number = 0 for (line_number, diff_found, ans_line, exp_line, ans_chars, exp_chars) in deq: num_spaces_after_output = max_chars - ans_chars - max_line_num_digits - 1 line_number_str = str(line_number) if line_number is not None else "" line_num_display = _space_padding(line_number_str, max_line_num_digits - len(line_number_str)) + "|" if not diff_found: logger.info(utils.NO_HEADER + '%s', line_num_display + _space_padding(ans_line, num_spaces_after_output) + "|" + exp_line) else: logger.info(utils.NO_HEADER + '%s', line_num_display + utils.red(_space_padding(ans_line, num_spaces_after_output)) + "|" + utils.green(exp_line)) if line_number is not None: last_line_number = line_number num_snipped_lines = answer.count('\n') + 1 - last_line_number if num_snipped_lines > 0: logger.info(utils.NO_HEADER + '... (%s lines) ...', num_snipped_lines)
def try_hack_once( generator: str, command: str, hack: str, *, tle: Optional[float], attempt: int, lock: Optional[threading.Lock] = None ) -> Optional[Tuple[bytes, bytes]]: with BufferedExecutor(lock) as submit: # print the header submit(logger.info, '') submit(logger.info, '%d-th attempt', attempt) # generate input submit(logger.info, 'generate input...') info, proc = utils.exec_command(generator, stdin=None, timeout=tle) input_data = info['answer'] # type: Optional[bytes] if not check_status(info, proc, submit=submit): return None assert input_data is not None # generate output submit(logger.info, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data = info['answer'] # type: Optional[bytes] if not check_status(info, proc, submit=submit): return None assert output_data is not None # hack submit(logger.info, 'hack...') info, proc = utils.exec_command(hack, input=input_data, timeout=tle) answer = (info['answer'] or b'').decode() # type: str elapsed = info['elapsed'] # type: float memory = info['memory'] # type: Optional[float] # compare status = 'AC' if proc.returncode is None: submit(logger.info, 'FAILURE: ' + utils.red('TLE')) status = 'TLE' elif proc.returncode != 0: logger.info( utils.FAILURE + '' + utils.red('RE') + ': return code %d', proc.returncode) status = 'RE' expected = output_data.decode() if not simple_match(answer, expected): logger.info(utils.FAILURE + '' + utils.red('WA')) logger.info( 'NO_HDEADER: input:\n%s', utils.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) logger.info( 'NO_HDEADER: output:\n%s', utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) logger.info( 'NO_HDEADER: expected:\n%s', utils.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) status = 'WA' if status == 'AC': return None else: return (input_data, output_data)
def test(args: 'argparse.Namespace') -> None: # list tests if not args.test: args.test = fmtutils.glob_with_format(args.directory, args.format) # by default if args.ignore_backup: args.test = fmtutils.drop_backup_or_hidden_files(args.test) tests = fmtutils.construct_relationship_of_files(args.test, args.directory, args.format) # check wheather GNU time is available if not check_gnu_time(args.gnu_time): logger.warning('GNU time is not available: %s', args.gnu_time) args.gnu_time = None if args.mle is not None and args.gnu_time is None: raise RuntimeError('--mle is used but GNU time does not exist') # run tests history: List[Dict[str, Any]] = [] if args.jobs is None: for name, paths in sorted(tests.items()): history += [ test_single_case(name, paths['in'], paths.get('out'), args=args) ] else: if os.name == 'nt': logger.warning( "-j/--jobs opiton is unstable on Windows environmet") with concurrent.futures.ThreadPoolExecutor( max_workers=args.jobs) as executor: lock = threading.Lock() futures: List[concurrent.futures.Future] = [] for name, paths in sorted(tests.items()): futures += [ executor.submit(test_single_case, name, paths['in'], paths.get('out'), lock=lock, args=args) ] for future in futures: history += [future.result()] # summarize slowest: float = -1.0 slowest_name = '' heaviest: float = -1.0 heaviest_name = '' ac_count = 0 for result in history: if result['status'] == 'AC': ac_count += 1 if slowest < result['elapsed']: slowest = result['elapsed'] slowest_name = result['testcase']['name'] if result['memory'] is not None and heaviest < result['memory']: heaviest = result['memory'] heaviest_name = result['testcase']['name'] # print the summary logger.info('') logger.info('slowest: %f sec (for %s)', slowest, slowest_name) if heaviest >= 0: if heaviest < MEMORY_WARNING: logger.info('max memory: %f MB (for %s)', heaviest, heaviest_name) else: logger.warning('max memory: %f MB (for %s)', heaviest, heaviest_name) if ac_count == len(tests): logger.info( utils.SUCCESS + 'test ' + utils.green('success') + ': %d cases', len(tests)) else: logger.info( utils.FAILURE + 'test ' + utils.red('failed') + ': %d AC / %d cases', ac_count, len(tests)) if args.json: print(json.dumps(history)) if ac_count != len(tests): sys.exit(1)
def display_result(proc: subprocess.Popen, answer: str, memory: Optional[float], test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, mle: Optional[float], display_mode: DisplayMode, does_print_input: bool, silent: bool, match_result: Optional[bool]) -> JudgeStatus: """display_result prints the result of the test and its statistics. This function prints many logs and does some I/O. """ # prepare the function to print the input is_input_printed = False def print_input() -> None: nonlocal is_input_printed if does_print_input and not is_input_printed: is_input_printed = True with test_input_path.open('rb') as inf: logger.info( utils.NO_HEADER + 'input:\n%s', pretty_printers.make_pretty_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) # check TLE, RE or not status = JudgeStatus.AC if proc.returncode is None: logger.info(utils.FAILURE + '' + utils.red('TLE')) status = JudgeStatus.TLE if not silent: print_input() elif memory is not None and mle is not None and memory > mle: logger.info(utils.FAILURE + '' + utils.red('MLE')) status = JudgeStatus.MLE if not silent: print_input() elif proc.returncode != 0: logger.info(utils.FAILURE + '' + utils.red('RE') + ': return code %d', proc.returncode) status = JudgeStatus.RE if not silent: print_input() # check WA or not if match_result is not None and not match_result: if status == JudgeStatus.AC: logger.info(utils.FAILURE + '' + utils.red('WA')) status = JudgeStatus.WA if not silent: print_input() if test_output_path is not None: with test_output_path.open('rb') as outf: expected = outf.read().decode() else: expected = '' if display_mode == DisplayMode.SUMMARY: logger.info( utils.NO_HEADER + 'output:\n%s', pretty_printers.make_pretty_large_file_content( answer.encode(), limit=40, head=20, tail=10, bold=True)) logger.info( utils.NO_HEADER + 'expected:\n%s', pretty_printers.make_pretty_large_file_content( expected.encode(), limit=40, head=20, tail=10, bold=True)) elif display_mode == DisplayMode.DIFF: if max(answer.count('\n'), expected.count('\n')) <= 40: pretty_printers.display_side_by_side_color( answer, expected) else: pretty_printers.display_snipped_side_by_side_color( answer, expected) else: assert False if match_result is None: if not silent: print_input() logger.info( utils.NO_HEADER + 'output:\n%s', pretty_printers.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if status == JudgeStatus.AC: logger.info(utils.SUCCESS + '' + utils.green('AC')) return status
def try_hack_once( generator: str, command: str, hack: str, *, tle: Optional[float], attempt: int, lock: Optional[threading.Lock] = None, generated_input_hashes: Dict[bytes, str]) -> Optional[Tuple[bytes, bytes]]: with BufferedExecutor(lock) as submit: # print the header submit(logger.info, '') submit(logger.info, '%d-th attempt', attempt) # generate input submit(logger.info, 'generate input...') info, proc = utils.exec_command(generator, stdin=None, timeout=tle) input_data: Optional[bytes] = info['answer'] if not check_status(info, proc, submit=submit): return None assert input_data is not None # check the randomness of generator name = '{}-th attempt'.format(attempt) conflicted_name = check_randomness_of_generator( input_data, name=name, lock=lock, generated_input_hashes=generated_input_hashes) if conflicted_name is not None: submit( logger.warning, 'The same input is already generated at %s. Please use a random input generator.', conflicted_name) submit(logger.info, utils.NO_HEADER + 'input:') submit( logger.info, utils.NO_HEADER + '%s', pretty_printers.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) # generate output submit(logger.info, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data: Optional[bytes] = info['answer'] if not check_status(info, proc, submit=submit): return None assert output_data is not None # hack submit(logger.info, 'hack...') info, proc = utils.exec_command(hack, input=input_data, timeout=tle) answer: str = (info['answer'] or b'').decode() # compare status = 'AC' if proc.returncode is None: submit(logger.info, 'FAILURE: ' + utils.red('TLE')) status = 'TLE' elif proc.returncode != 0: logger.info( utils.FAILURE + '' + utils.red('RE') + ': return code %d', proc.returncode) status = 'RE' expected = output_data.decode() if not simple_match(answer, expected): logger.info(utils.FAILURE + '' + utils.red('WA')) logger.info( utils.NO_HEADER + 'input:\n%s', pretty_printers.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) logger.info( utils.NO_HEADER + 'output:\n%s', pretty_printers.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) logger.info( utils.NO_HEADER + 'expected:\n%s', pretty_printers.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) status = 'WA' if status == 'AC': return None else: return (input_data, output_data)
def compare_and_report(proc: subprocess.Popen, answer: str, memory: Optional[float], test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, mle: Optional[float], mode: str, error: Optional[float], does_print_input: bool, silent: bool, rstrip: bool, judge: Optional[str]) -> str: rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf # prepare the comparing function if error is not None: # float mode match = lambda a, b: compare_as_floats(a, b, error) elif judge is not None: # special judge mode def match(a: str, b: str) -> bool: # On Windows, a temp file is not created if we use "with" statement, user_output = tempfile.NamedTemporaryFile(delete=False) judge_result = False try: if rstrip: user_output.write(a.rstrip(rstrip_targets).encode()) else: user_output.write(a.encode()) user_output.close() arg0 = judge arg1 = str(test_input_path.resolve()) arg2 = user_output.name arg3 = str((str(test_output_path.resolve()) if test_output_path is not None else '')) actual_command = '{} {} {} {}'.format( arg0, arg1, arg2, arg3 ) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584 logger.info('$ %s', actual_command) info, proc = utils.exec_command(actual_command) if not silent: logger.info( utils.NO_HEADER + 'judge\'s output:\n%s', utils.make_pretty_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) judge_result = (proc.returncode == 0) finally: os.unlink(user_output.name) return judge_result else: def match(a: str, b: str) -> bool: if a == b: return True if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets): logger.warning('WA if no rstrip') return True if a == b.replace('\n', '\r\n'): logger.warning(r'WA if not replacing "\r\n" with "\n"') return True if rstrip and a.rstrip(rstrip_targets) == b.replace( '\n', '\r\n').rstrip(rstrip_targets): logger.warning('WA if no rstrip') logger.warning(r'WA if not replacing "\r\n" with "\n"') return True if a.replace('\n', '\r\n') == b: logger.warning(r'WA if not replacing "\n" with "\r\n"') return True if rstrip and a.replace( '\n', '\r\n').rstrip(rstrip_targets) == b.rstrip(rstrip_targets): # TODO: use a smart way if you need more equality patterns logger.warning('WA if no rstrip') logger.warning(r'WA if not replacing "\n" with "\r\n"') return True return False # prepare the function to print the input is_input_printed = False def print_input() -> None: nonlocal is_input_printed if does_print_input and not is_input_printed: is_input_printed = True with test_input_path.open('rb') as inf: logger.info( utils.NO_HEADER + 'input:\n%s', utils.make_pretty_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) # check TLE, RE or not status = 'AC' if proc.returncode is None: logger.info(utils.FAILURE + '' + utils.red('TLE')) status = 'TLE' print_input() elif memory is not None and mle is not None and memory > mle: logger.info(utils.FAILURE + '' + utils.red('MLE')) status = 'MLE' print_input() elif proc.returncode != 0: logger.info(utils.FAILURE + '' + utils.red('RE') + ': return code %d', proc.returncode) status = 'RE' print_input() # check WA or not if (test_output_path is not None) or (judge is not None): if test_output_path is not None: with test_output_path.open('rb') as outf: expected = outf.read().decode() else: # only if --judge-command option expected = '' logger.warning('expected output is not found') # compare if not match(answer, expected): logger.info(utils.FAILURE + '' + utils.red('WA')) print_input() if not silent: if mode == 'simple': logger.info( utils.NO_HEADER + 'output:\n%s', utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) logger.info( utils.NO_HEADER + 'expected:\n%s', utils.make_pretty_large_file_content(expected.encode(), limit=40, head=20, tail=10, bold=True)) elif mode == 'side-by-side': if max(answer.count('\n'), expected.count('\n')) <= 40: display_side_by_side_color(answer, expected) else: display_snipped_side_by_side_color(answer, expected) else: assert False status = 'WA' else: if not silent: header = ('output:\n' if is_input_printed else '') logger.info( utils.NO_HEADER + '%s%s', header, utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if status == 'AC': logger.info(utils.SUCCESS + '' + utils.green('AC')) return status
def generate_output_single_case(test_name: str, test_input_path: pathlib.Path, *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> None: # print the header if lock is None: logger.info('') logger.info('%s', test_name) # run the command with test_input_path.open() as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle) answer = info['answer'] # type: Optional[bytes] elapsed = info['elapsed'] # type: float # acquire lock to print logs properly, if in parallel nullcontext = contextlib.ExitStack() with lock or nullcontext: if lock is not None: logger.info('') logger.info('%s', test_name) # check the result logger.info('time: %f sec', elapsed) if proc.returncode is None: logger.info(utils.red('TLE')) logger.info('skipped.') return elif proc.returncode != 0: logger.info('FIALURE: ' + utils.red('RE') + ': return code %d', proc.returncode) logger.info('skipped.') return assert answer is not None logger.info(utils.NO_HEADER + '' + utils.make_pretty_large_file_content( answer, limit=40, head=20, tail=10, bold=True)) # find the destination path match_result = fmtutils.match_with_format( args.directory, args.format, test_input_path) # type: Optional[Match[Any]] if match_result is not None: matched_name = match_result.groupdict()['name'] # type: str else: assert False test_output_path = fmtutils.path_from_format(args.directory, args.format, name=matched_name, ext='out') # write the result to the file if not test_output_path.parent.is_dir(): os.makedirs(str(test_output_path.parent), exist_ok=True) with test_output_path.open('wb') as fh: fh.write(answer) logger.info(utils.SUCCESS + 'saved to: %s', test_output_path)