def generate_input_single_case(generator: str, *, input_path: pathlib.Path, output_path: pathlib.Path, command: Optional[str], tle: Optional[float], name: str, lock: Optional[threading.Lock] = None, generated_input_hashes: Dict[bytes, str]) -> None: with BufferedExecutor(lock) as submit: # print the header submit(logger.info, '') submit(logger.info, '%s', name) # generate input submit(logger.info, 'generate input...') info, proc = utils.exec_command(generator, timeout=tle) input_data: bytes = info['answer'] if not check_status(info, proc, submit=submit): return # check the randomness of generator conflicted_name = check_randomness_of_generator(input_data, name=name, lock=lock, generated_input_hashes=generated_input_hashes) if conflicted_name is not None: submit(logger.warning, 'The same input is already generated at %s. Please use a random input generator.', conflicted_name) # generate output if command is None: output_data: Optional[bytes] = None else: submit(logger.info, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data = info['answer'] if not check_status(info, proc, submit=submit): return # write result submit(write_result, input_data=input_data, output_data=output_data, input_path=input_path, output_path=output_path, print_data=True)
def generate_input_single_case(generator: str, *, input_path: pathlib.Path, output_path: pathlib.Path, command: Optional[str], tle: Optional[float], name: str, lock: Optional[threading.Lock] = None) -> None: with BufferedExecutor(lock) as submit: # print the header submit(logger.info, '') submit(logger.info, '%s', name) # generate input submit(logger.info, 'generate input...') info, proc = utils.exec_command(generator, timeout=tle) input_data: bytes = info['answer'] if not check_status(info, proc, submit=submit): return None # generate output if command is None: output_data: Optional[bytes] = None else: submit(logger.info, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data = info['answer'] if not check_status(info, proc, submit=submit): return None # write result submit(write_result, input_data=input_data, output_data=output_data, input_path=input_path, output_path=output_path, print_data=True)
def try_hack_once(generator: str, command: str, hack: str, *, tle: Optional[float], attempt: int, lock: Optional[threading.Lock] = None, generated_input_hashes: Dict[bytes, str]) -> Optional[Tuple[bytes, bytes]]: with BufferedExecutor(lock) as submit: # print the header submit(logger.info, '') submit(logger.info, '%d-th attempt', attempt) # generate input submit(logger.info, 'generate input...') info, proc = utils.exec_command(generator, stdin=None, timeout=tle) input_data: Optional[bytes] = info['answer'] if not check_status(info, proc, submit=submit): return None assert input_data is not None # check the randomness of generator name = '{}-th attempt' conflicted_name = check_randomness_of_generator(input_data, name=name, lock=lock, generated_input_hashes=generated_input_hashes) if conflicted_name is not None: submit(logger.warning, 'The same input is already generated at %s. Please use a random input generator.', conflicted_name) submit(logger.info, utils.NO_HEADER + 'input:') submit(logger.info, utils.NO_HEADER + '%s', pretty_printers.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) # generate output submit(logger.info, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data: Optional[bytes] = info['answer'] if not check_status(info, proc, submit=submit): return None assert output_data is not None # hack submit(logger.info, 'hack...') info, proc = utils.exec_command(hack, input=input_data, timeout=tle) answer: str = (info['answer'] or b'').decode() # compare status = 'AC' if proc.returncode is None: submit(logger.info, 'FAILURE: ' + utils.red('TLE')) status = 'TLE' elif proc.returncode != 0: logger.info(utils.FAILURE + '' + utils.red('RE') + ': return code %d', proc.returncode) status = 'RE' expected = output_data.decode() if not simple_match(answer, expected): logger.info(utils.FAILURE + '' + utils.red('WA')) logger.info(utils.NO_HEADER + 'input:\n%s', pretty_printers.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) logger.info(utils.NO_HEADER + 'output:\n%s', pretty_printers.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) logger.info(utils.NO_HEADER + 'expected:\n%s', pretty_printers.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) status = 'WA' if status == 'AC': return None else: return (input_data, output_data)
def try_hack_once(generator: str, command: str, hack: str, *, tle: Optional[float], attempt: int, lock: Optional[threading.Lock] = None) -> Optional[Tuple[bytes, bytes]]: with BufferedExecutor(lock) as submit: # print the header submit(logger.info, '') submit(logger.info, '%d-th attempt', attempt) # generate input submit(logger.info, 'generate input...') info, proc = utils.exec_command(generator, stdin=None, timeout=tle) input_data: Optional[bytes] = info['answer'] if not check_status(info, proc, submit=submit): return None assert input_data is not None # generate output submit(logger.info, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data: Optional[bytes] = info['answer'] if not check_status(info, proc, submit=submit): return None assert output_data is not None # hack submit(logger.info, 'hack...') info, proc = utils.exec_command(hack, input=input_data, timeout=tle) answer: str = (info['answer'] or b'').decode() elapsed: float = info['elapsed'] memory: Optional[float] = info['memory'] # compare status = 'AC' if proc.returncode is None: submit(logger.info, 'FAILURE: ' + utils.red('TLE')) status = 'TLE' elif proc.returncode != 0: logger.info(utils.FAILURE + '' + utils.red('RE') + ': return code %d', proc.returncode) status = 'RE' expected = output_data.decode() if not simple_match(answer, expected): logger.info(utils.FAILURE + '' + utils.red('WA')) logger.info(utils.NO_HEADER + 'input:\n%s', pretty_printers.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) logger.info(utils.NO_HEADER + 'output:\n%s', pretty_printers.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) logger.info(utils.NO_HEADER + 'expected:\n%s', pretty_printers.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) status = 'WA' if status == 'AC': return None else: return (input_data, output_data)
def match(a, b): # On Windows, a temp file is not created if we use "with" statement, user_output = tempfile.NamedTemporaryFile(delete=False) judge_result = False try: if rstrip: user_output.write(a.rstrip(rstrip_targets).encode()) else: user_output.write(a.encode()) user_output.close() arg0 = judge arg1 = str(test_input_path.resolve()) arg2 = user_output.name arg3 = str((str(test_output_path.resolve()) if test_output_path is not None else '')) actual_command = '{} {} {} {}'.format(arg0, arg1, arg2, arg3) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584 log.status('$ %s', actual_command) info, proc = utils.exec_command(actual_command) if not silent: log.emit('judge\'s output:\n%s', utils.make_pretty_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) judge_result = (proc.returncode == 0) finally: os.unlink(user_output.name) return judge_result
def generate_output_single_case(test_name: str, test_input_path: pathlib.Path, *, lock: Optional[threading.Lock] = None, args: argparse.Namespace) -> None: # print the header if lock is None: logger.info('') logger.info('%s', test_name) # run the command with test_input_path.open('rb') as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle) answer: Optional[bytes] = info['answer'] elapsed: float = info['elapsed'] # acquire lock to print logs properly, if in parallel nullcontext = contextlib.ExitStack() with lock or nullcontext: if lock is not None: logger.info('') logger.info('%s', test_name) # check the result logger.info('time: %f sec', elapsed) if proc.returncode is None: logger.info(utils.red('TLE')) logger.info('skipped.') return elif proc.returncode != 0: logger.info('FIALURE: ' + utils.red('RE') + ': return code %d', proc.returncode) logger.info('skipped.') return assert answer is not None logger.info(utils.NO_HEADER + '' + pretty_printers.make_pretty_large_file_content( answer, limit=40, head=20, tail=10, bold=True)) # find the destination path match_result: Optional[Match[Any]] = fmtutils.match_with_format( args.directory, args.format, test_input_path) if match_result is not None: matched_name: str = match_result.groupdict()['name'] else: assert False test_output_path = fmtutils.path_from_format(args.directory, args.format, name=matched_name, ext='out') # write the result to the file if not test_output_path.parent.is_dir(): os.makedirs(str(test_output_path.parent), exist_ok=True) with test_output_path.open('wb') as fh: fh.write(answer) logger.info(utils.SUCCESS + 'saved to: %s', test_output_path)
def run(self, *, actual_output: bytes, input_path: pathlib.Path, expected_output_path: Optional[pathlib.Path]) -> bool: with tempfile.TemporaryDirectory() as tempdir: actual_output_path = pathlib.Path(tempdir) / 'actual.out' with open(actual_output_path, 'wb') as fh: fh.write(actual_output) # if you use shlex.quote, it fails on Windows. why? command = ' '.join([ self.judge_command, # already quoted and joined command str(input_path.resolve()), str(actual_output_path.resolve()), str(expected_output_path.resolve( ) if expected_output_path is not None else ''), ]) logger.info('$ %s', command) info, proc = utils.exec_command(command) if not self.is_silent: logger.info( utils.NO_HEADER + 'judge\'s output:\n%s', pretty_printers.make_pretty_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) return proc.returncode == 0
def test_single_case(test_name: str, test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, lock: Optional[threading.Lock] = None, args: argparse.Namespace) -> Dict[str, Any]: # print the header earlier if not in parallel if lock is None: logger.info('') logger.info('%s', test_name) # run the binary with test_input_path.open('rb') as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle, gnu_time=args.gnu_time) # TODO: the `answer` should be bytes, not str answer: str = (info['answer'] or b'').decode(errors='replace') elapsed: float = info['elapsed'] memory: Optional[float] = info['memory'] # lock is require to avoid mixing logs if in parallel nullcontext = contextlib.ExitStack() # TODO: use contextlib.nullcontext() after updating Python to 3.7 with lock or nullcontext: if lock is not None: logger.info('') logger.info('%s', test_name) logger.info('time: %f sec', elapsed) if memory: if memory < MEMORY_PRINT: if args.print_memory: logger.info('memory: %f MB', memory) elif memory < MEMORY_WARNING: logger.info('memory: %f MB', memory) else: logger.warning('memory: %f MB', memory) match_function = build_match_function(compare_mode=CompareMode(args.compare_mode), error=args.error, judge_command=args.judge, silent=args.silent, test_input_path=test_input_path, test_output_path=test_output_path) match_result = run_checking_output(answer=answer.encode(), test_output_path=test_output_path, is_special_judge=args.judge is not None, match_function=match_function) status = display_result(proc, answer, memory, test_input_path, test_output_path, mle=args.mle, display_mode=DisplayMode(args.display_mode), does_print_input=args.print_input, silent=args.silent, match_result=match_result) # return the result testcase = { 'name': test_name, 'input': str(test_input_path.resolve()), } if test_output_path: testcase['output'] = str(test_output_path.resolve()) return { 'status': status.value, 'testcase': testcase, 'output': answer, 'exitcode': proc.returncode, 'elapsed': elapsed, 'memory': memory, }
def test_single_case(test_name: str, test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> Dict[str, Any]: # print the header earlier if not in parallel if lock is None: log.emit('') log.info('%s', test_name) # run the binary with test_input_path.open() as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle, gnu_time=args.gnu_time) # TODO: the `answer` should be bytes, not str answer = (info['answer'] or b'').decode(errors='replace') # type: str elapsed = info['elapsed'] # type: float memory = info['memory'] # type: Optional[float] # lock is require to avoid mixing logs if in parallel nullcontext = contextlib.ExitStack() # TODO: use contextlib.nullcontext() after updating Python to 3.7 with lock or nullcontext: if lock is not None: log.emit('') log.info('%s', test_name) log.status('time: %f sec', elapsed) if memory: if memory < MEMORY_PRINT: if args.print_memory: log.status('memory: %f MB', memory) elif memory < MEMORY_WARNING: log.status('memory: %f MB', memory) else: log.warning('memory: %f MB', memory) status = compare_and_report(proc, answer, memory, test_input_path, test_output_path, mle=args.mle, mode=args.display_mode, error=args.error, does_print_input=args.print_input, silent=args.silent, rstrip=args.rstrip, judge=args.judge) # return the result testcase = { 'name': test_name, 'input': str(test_input_path.resolve()), } if test_output_path: testcase['output'] = str(test_output_path.resolve()) return { 'status': status, 'testcase': testcase, 'output': answer, 'exitcode': proc.returncode, 'elapsed': elapsed, 'memory': memory, }