def generate_output(args: 'argparse.Namespace') -> None: if not args.test: args.test = cutils.glob_with_format(args.directory, args.format) # by default if args.ignore_backup: args.test = cutils.drop_backup_or_hidden_files(args.test) tests = cutils.construct_relationship_of_files(args.test, args.directory, args.format) for name, it in sorted(tests.items()): log.emit('') log.info('%s', name) if 'out' in it: log.info('output file already exists.') log.info('skipped.') continue with it['in'].open() as inf: begin = time.perf_counter() answer, proc = utils.exec_command(args.command, shell=True, stdin=inf) end = time.perf_counter() log.status('time: %f sec', end - begin) if proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) log.info('skipped.') continue log.emit(utils.snip_large_file_content(answer, limit=40, head=20, tail=10, bold=True)) match_result = cutils.match_with_format(args.directory, args.format, it['in']) # type: Optional[Match[Any]] if match_result is not None: matched_name = match_result.groupdict()['name'] # type: str else: assert False path = cutils.path_from_format(args.directory, args.format, name=matched_name, ext='out') if not path.parent.is_dir(): os.makedirs(str(path.parent), exist_ok=True) with path.open('wb') as fh: fh.write(answer) log.success('saved to: %s', path)
def match(a, b): # On Windows, a temp file is not created if we use "with" statement, user_output = tempfile.NamedTemporaryFile(delete=False) judge_result = False try: if rstrip: user_output.write(a.rstrip(rstrip_targets).encode()) else: user_output.write(a.encode()) user_output.close() arg0 = judge arg1 = str(test_input_path.resolve()) arg2 = user_output.name arg3 = str((str(test_output_path.resolve()) if test_output_path is not None else '')) actual_command = '{} {} {} {}'.format( arg0, arg1, arg2, arg3 ) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584 log.status('$ %s', actual_command) info, proc = utils.exec_command(actual_command) if not silent: log.emit( 'judge\'s output:\n%s', utils.snip_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) judge_result = (proc.returncode == 0) finally: os.unlink(user_output.name) return judge_result
def generate_input_single_case(generator: str, *, input_path: pathlib.Path, output_path: pathlib.Path, command: Optional[str], tle: Optional[float], name: str, lock: Optional[threading.Lock] = None) -> None: with BufferedExecutor(lock) as submit: # print the header submit(log.emit, '') submit(log.info, '%s', name) # generate input submit(log.status, 'generate input...') info, proc = utils.exec_command(generator, timeout=tle) input_data = info['answer'] # type: bytes if not check_status(info, proc, submit=submit): return None # generate output if command is None: output_data = None # type: Optional[bytes] else: submit(log.status, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data = info['answer'] if not check_status(info, proc, submit=submit): return None # write result submit(write_result, input_data=input_data, output_data=output_data, input_path=input_path, output_path=output_path, print_data=True)
def test_single_case(test_name: str, test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> Dict[str, Any]: # print the header earlier if not in parallel if lock is None: log.emit('') log.info('%s', test_name) # run the binary with test_input_path.open() as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle, gnu_time=args.gnu_time) # TODO: the `answer` should be bytes, not str answer = (info['answer'] or b'').decode(errors='replace') # type: str elapsed = info['elapsed'] # type: float memory = info['memory'] # type: Optional[float] # lock is require to avoid mixing logs if in parallel nullcontext = contextlib.ExitStack() # TODO: use contextlib.nullcontext() after updating Python to 3.7 with lock or nullcontext: if lock is not None: log.emit('') log.info('%s', test_name) log.status('time: %f sec', elapsed) if memory: if memory < MEMORY_PRINT: if args.print_memory: log.status('memory: %f MB', memory) elif memory < MEMORY_WARNING: log.status('memory: %f MB', memory) else: log.warning('memory: %f MB', memory) status = compare_and_report(proc, answer, elapsed, memory, test_input_path, test_output_path, mle=args.mle, mode=args.mode, error=args.error, does_print_input=args.print_input, silent=args.silent, rstrip=args.rstrip, judge=args.judge) # return the result testcase = { 'name': test_name, 'input': str(test_input_path.resolve()), } if test_output_path: testcase['output'] = str(test_output_path.resolve()) return { 'status': status, 'testcase': testcase, 'output': answer, 'exitcode': proc.returncode, 'elapsed': elapsed, 'memory': memory, }
def generate_output_single_case(test_name: str, test_input_path: pathlib.Path, *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> None: # print the header if lock is None: log.emit('') log.info('%s', test_name) # run the command with test_input_path.open() as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle) answer = info['answer'] # type: Optional[bytes] elapsed = info['elapsed'] # type: float # acquire lock to print logs properly, if in parallel nullcontext = contextlib.ExitStack() with lock or nullcontext: if lock is not None: log.emit('') log.info('%s', test_name) # check the result log.status('time: %f sec', elapsed) if proc.returncode is None: log.failure(log.red('TLE')) log.info('skipped.') return elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) log.info('skipped.') return assert answer is not None log.emit(utils.snip_large_file_content(answer, limit=40, head=20, tail=10, bold=True)) # find the destination path match_result = fmtutils.match_with_format(args.directory, args.format, test_input_path) # type: Optional[Match[Any]] if match_result is not None: matched_name = match_result.groupdict()['name'] # type: str else: assert False test_output_path = fmtutils.path_from_format(args.directory, args.format, name=matched_name, ext='out') # write the result to the file if not test_output_path.parent.is_dir(): os.makedirs(str(test_output_path.parent), exist_ok=True) with test_output_path.open('wb') as fh: fh.write(answer) log.success('saved to: %s', test_output_path)
def test(args: 'argparse.Namespace') -> None: # prepare if not args.test: args.test = cutils.glob_with_format(args.directory, args.format) # by default if args.ignore_backup: args.test = cutils.drop_backup_or_hidden_files(args.test) tests = cutils.construct_relationship_of_files(args.test, args.directory, args.format) if args.error: # float mode match = lambda a, b: compare_as_floats(a, b, args.error) else: def match(a, b): if a == b: return True if args.rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets): log.warning('WA if no rstrip') return True return False rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf slowest = -1 # type: Union[int, float] slowest_name = '' ac_count = 0 history = [] # type: List[Dict[str, Any]] for name, it in sorted(tests.items()): is_input_printed = False def print_input(): nonlocal is_input_printed if args.print_input and not is_input_printed: is_input_printed = True with open(it['in'], 'rb') as inf: log.emit('input:\n%s', utils.snip_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) log.emit('') log.info('%s', name) # run the binary with it['in'].open() as inf: begin = time.perf_counter() answer_byte, proc = utils.exec_command(args.command, shell=True, stdin=inf, timeout=args.tle) end = time.perf_counter() elapsed = end - begin answer = answer_byte.decode() # TODO: the `answer` should be bytes, not str if slowest < elapsed: slowest = elapsed slowest_name = name log.status('time: %f sec', elapsed) proc.terminate() # check TLE, RE or not result = 'AC' if proc.returncode is None: log.failure(log.red('TLE')) result = 'TLE' print_input() elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) result = 'RE' print_input() # check WA or not if 'out' in it: with it['out'].open() as outf: correct = outf.read() # compare if args.mode == 'all': if not match(answer, correct): log.failure(log.red('WA')) print_input() if not args.silent: log.emit('output:\n%s', utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit('expected:\n%s', utils.snip_large_file_content(correct.encode(), limit=40, head=20, tail=10, bold=True)) result = 'WA' elif args.mode == 'line': answer_words = answer.splitlines() correct_words = correct.splitlines() for i, (x, y) in enumerate(zip(answer_words + [None] * len(correct_words), correct_words + [None] * len(answer_words))): # type: ignore if x is None and y is None: break elif x is None: print_input() log.failure(log.red('WA') + ': line %d: line is nothing: expected "%s"', i + 1, log.bold(y)) result = 'WA' elif y is None: print_input() log.failure(log.red('WA') + ': line %d: unexpected line: output "%s"', i + 1, log.bold(x)) result = 'WA' elif not match(x, y): print_input() log.failure(log.red('WA') + ': line %d: output "%s": expected "%s"', i + 1, log.bold(x), log.bold(y)) result = 'WA' else: assert False else: if not args.silent: log.emit(('output:\n%s' if is_input_printed else '%s'), utils.snip_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if result == 'AC': log.success(log.green('AC')) ac_count += 1 # push the result testcase = { 'name': name, 'input': str(it['in'].resolve()), } if 'out' in it: testcase['output'] = str(it['out'].resolve()) history += [{ 'result': result, 'testcase': testcase, 'output': answer, 'exitcode': proc.returncode, 'elapsed': elapsed, }] # summarize log.emit('') log.status('slowest: %f sec (for %s)', slowest, slowest_name) if ac_count == len(tests): log.success('test ' + log.green('success') + ': %d cases', len(tests)) else: log.failure('test ' + log.red('failed') + ': %d AC / %d cases', ac_count, len(tests)) if args.json: print(json.dumps(history)) if ac_count != len(tests): sys.exit(1)
def try_hack_once( generator: str, command: str, hack: str, *, tle: Optional[float], attempt: int, lock: Optional[threading.Lock] = None ) -> Optional[Tuple[bytes, bytes]]: with BufferedExecutor(lock) as submit: # print the header submit(log.emit, '') submit(log.info, '%d-th attempt', attempt) # generate input submit(log.status, 'generate input...') info, proc = utils.exec_command(generator, stdin=None, timeout=tle) input_data = info['answer'] # type: Optional[bytes] if not check_status(info, proc, submit=submit): return None assert input_data is not None # generate output submit(log.status, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data = info['answer'] # type: Optional[bytes] if not check_status(info, proc, submit=submit): return None assert output_data is not None # hack submit(log.status, 'hack...') info, proc = utils.exec_command(hack, input=input_data, timeout=tle) answer = (info['answer'] or b'').decode() # type: str elapsed = info['elapsed'] # type: float memory = info['memory'] # type: Optional[float] # compare status = 'AC' if proc.returncode is None: submit(log.failure, log.red('TLE')) status = 'TLE' elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) status = 'RE' expected = output_data.decode() if not simple_match(answer, expected): log.failure(log.red('WA')) log.emit( 'input:\n%s', utils.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) log.emit( 'output:\n%s', utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit( 'expected:\n%s', utils.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) status = 'WA' if status == 'AC': return None else: return (input_data, output_data)