def match(a, b): # On Windows, a temp file is not created if we use "with" statement, user_output = tempfile.NamedTemporaryFile(delete=False) judge_result = False try: if rstrip: user_output.write(a.rstrip(rstrip_targets).encode()) else: user_output.write(a.encode()) user_output.close() arg0 = judge arg1 = str(test_input_path.resolve()) arg2 = user_output.name arg3 = str((str(test_output_path.resolve()) if test_output_path is not None else '')) actual_command = '{} {} {} {}'.format(arg0, arg1, arg2, arg3) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584 log.status('$ %s', actual_command) info, proc = utils.exec_command(actual_command) if not silent: log.emit('judge\'s output:\n%s', utils.make_pretty_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) judge_result = (proc.returncode == 0) finally: os.unlink(user_output.name) return judge_result
def test_single_case(test_name: str, test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> Dict[str, Any]: # print the header earlier if not in parallel if lock is None: log.emit('') log.info('%s', test_name) # run the binary with test_input_path.open() as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle, gnu_time=args.gnu_time) # TODO: the `answer` should be bytes, not str answer = (info['answer'] or b'').decode(errors='replace') # type: str elapsed = info['elapsed'] # type: float memory = info['memory'] # type: Optional[float] # lock is require to avoid mixing logs if in parallel nullcontext = contextlib.ExitStack() # TODO: use contextlib.nullcontext() after updating Python to 3.7 with lock or nullcontext: if lock is not None: log.emit('') log.info('%s', test_name) log.status('time: %f sec', elapsed) if memory: if memory < MEMORY_PRINT: if args.print_memory: log.status('memory: %f MB', memory) elif memory < MEMORY_WARNING: log.status('memory: %f MB', memory) else: log.warning('memory: %f MB', memory) status = compare_and_report(proc, answer, memory, test_input_path, test_output_path, mle=args.mle, mode=args.display_mode, error=args.error, does_print_input=args.print_input, silent=args.silent, rstrip=args.rstrip, judge=args.judge) # return the result testcase = { 'name': test_name, 'input': str(test_input_path.resolve()), } if test_output_path: testcase['output'] = str(test_output_path.resolve()) return { 'status': status, 'testcase': testcase, 'output': answer, 'exitcode': proc.returncode, 'elapsed': elapsed, 'memory': memory, }
def display_side_by_side_color(answer: str, expected: str): max_chars = shutil.get_terminal_size()[0] // 2 - 2 log.emit("output:" + " " * (max_chars - 7) + "|" + "expected:") log.emit("-" * max_chars + "|" + "-" * max_chars) for _, diff_found, ans_line, exp_line, ans_chars, exp_chars in side_by_side_diff(answer, expected): if diff_found: log.emit(log.red(space_padding(ans_line, max_chars - ans_chars)) + "|" + log.green(exp_line)) else: log.emit(space_padding(ans_line, max_chars - ans_chars) + "|" + exp_line)
def generate_output_single_case_exists_ok( test_name: str, test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> None: if test_output_path is not None: nullcontext = contextlib.ExitStack() with lock or nullcontext: log.emit('') log.info('%s', test_name) log.info('output file already exists.') log.info('skipped.') else: generate_output_single_case(test_name, test_input_path, lock=lock, args=args)
def display_snipped_side_by_side_color(answer: str, expected: str): """ Display first differ line and its previous 3 lines and its next 3 lines. """ max_chars = shutil.get_terminal_size()[0] // 2 - 2 deq = collections.deque(maxlen=7) # type: Deque[Tuple[Optional[int], bool, str, str, int, int]] count_from_first_difference = 0 i = 0 for flag, diff_found, ans_line, exp_line, ans_chars, exp_chars in side_by_side_diff(answer, expected): if flag: i += 1 if count_from_first_difference > 0: count_from_first_difference += 1 line_num = i if flag else None deq.append((line_num, diff_found, ans_line, exp_line, ans_chars, exp_chars)) if diff_found: if count_from_first_difference == 0: count_from_first_difference = 1 if count_from_first_difference == 4: break max_line_num_digits = max([len(str(entry[0])) for entry in deq if entry[0] is not None]) log.emit(" " * max_line_num_digits + "|output:" + " " * (max_chars - 7 - max_line_num_digits - 1) + "|" + "expected:") log.emit("-" * max_chars + "|" + "-" * max_chars) last_line_number = 0 for (line_number, diff_found, ans_line, exp_line, ans_chars, exp_chars) in deq: num_spaces_after_output = max_chars - ans_chars - max_line_num_digits - 1 line_number_str = str(line_number) if line_number is not None else "" line_num_display = space_padding(line_number_str, max_line_num_digits - len(line_number_str)) + "|" if not diff_found: log.emit(line_num_display + space_padding(ans_line, num_spaces_after_output) + "|" + exp_line) else: log.emit(line_num_display + log.red(space_padding(ans_line, num_spaces_after_output)) + "|" + log.green(exp_line)) if line_number is not None: last_line_number = line_number num_snipped_lines = answer.count('\n') + 1 - last_line_number if num_snipped_lines > 0: log.emit('... ({} lines) ...'.format(num_snipped_lines))
def write_result(input_data: bytes, output_data: Optional[bytes], *, input_path: pathlib.Path, output_path: pathlib.Path, print_data: bool, lock: Optional[threading.Lock] = None) -> None: # acquire lock to print logs properly, if in parallel nullcontext = contextlib.ExitStack() with lock or nullcontext: if not input_path.parent.is_dir(): os.makedirs(str(input_path.parent), exist_ok=True) if print_data: log.emit('input:') log.emit( utils.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) with input_path.open('wb') as fh: fh.write(input_data) log.success('saved to: %s', input_path) if output_data is not None: if print_data: log.emit('output:') log.emit( utils.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) with output_path.open('wb') as fh: fh.write(output_data) log.success('saved to: %s', output_path)
def submit(args: 'argparse.Namespace') -> None: # guess url history = onlinejudge_command.download_history.DownloadHistory() if args.file.parent.resolve() == pathlib.Path.cwd(): guessed_urls = history.get(directory=pathlib.Path.cwd()) else: log.warning('cannot guess URL since the given file is not in the current directory') guessed_urls = [] if args.url is None: if len(guessed_urls) == 1: args.url = guessed_urls[0] log.info('guessed problem: %s', args.url) else: log.error('failed to guess the URL to submit') log.info('please manually specify URL as: $ oj submit URL FILE') sys.exit(1) # parse url problem = dispatch.problem_from_url(args.url) if problem is None: sys.exit(1) # read code with args.file.open('rb') as fh: code = fh.read() # type: bytes format_config = { 'dos2unix': args.format_dos2unix or args.golf, 'rstrip': args.format_dos2unix or args.golf, } code = format_code(code, **format_config) # report code log.info('code (%d byte):', len(code)) log.emit(utils.make_pretty_large_file_content(code, limit=30, head=10, tail=10, bold=True)) with utils.new_session_with_our_user_agent(path=args.cookie) as sess: # guess or select language ids language_dict = {language.id: language.name for language in problem.get_available_languages(session=sess)} # type: Dict[LanguageId, str] matched_lang_ids = None # type: Optional[List[str]] if args.language in language_dict: matched_lang_ids = [args.language] else: if args.guess: kwargs = { 'language_dict': language_dict, 'cxx_latest': args.guess_cxx_latest, 'cxx_compiler': args.guess_cxx_compiler, 'python_version': args.guess_python_version, 'python_interpreter': args.guess_python_interpreter, } matched_lang_ids = guess_lang_ids_of_file(args.file, code, **kwargs) if not matched_lang_ids: log.info('failed to guess languages from the file name') matched_lang_ids = list(language_dict.keys()) if args.language is not None: log.info('you can use `--no-guess` option if you want to do an unusual submission') matched_lang_ids = select_ids_of_matched_languages(args.language.split(), matched_lang_ids, language_dict=language_dict) else: if args.language is None: matched_lang_ids = None else: matched_lang_ids = select_ids_of_matched_languages(args.language.split(), list(language_dict.keys()), language_dict=language_dict) # report selected language ids if matched_lang_ids is not None and len(matched_lang_ids) == 1: args.language = matched_lang_ids[0] log.info('chosen language: %s (%s)', args.language, language_dict[LanguageId(args.language)]) else: if matched_lang_ids is None: log.error('language is unknown') log.info('supported languages are:') elif len(matched_lang_ids) == 0: log.error('no languages are matched') log.info('supported languages are:') else: log.error('Matched languages were not narrowed down to one.') log.info('You have to choose:') for lang_id in sorted(matched_lang_ids or language_dict.keys()): log.emit('%s (%s)', lang_id, language_dict[LanguageId(lang_id)]) sys.exit(1) # confirm guessed_unmatch = ([problem.get_url()] != guessed_urls) if guessed_unmatch: samples_text = ('samples of "{}'.format('", "'.join(guessed_urls)) if guessed_urls else 'no samples') log.warning('the problem "%s" is specified to submit, but %s were downloaded in this directory. this may be mis-operation', problem.get_url(), samples_text) if args.wait: log.status('sleep(%.2f)', args.wait) time.sleep(args.wait) if not args.yes: if guessed_unmatch: problem_id = problem.get_url().rstrip('/').split('/')[-1].split('?')[-1] # this is too ad-hoc key = problem_id[:3] + (problem_id[-1] if len(problem_id) >= 4 else '') sys.stdout.write('Are you sure? Please type "{}" '.format(key)) sys.stdout.flush() c = sys.stdin.readline().rstrip() if c != key: log.info('terminated.') return else: sys.stdout.write('Are you sure? [y/N] ') sys.stdout.flush() c = sys.stdin.read(1) if c.lower() != 'y': log.info('terminated.') return # submit try: submission = problem.submit_code(code, language_id=LanguageId(args.language), session=sess) except NotLoggedInError: log.failure('login required') sys.exit(1) except SubmissionError: log.failure('submission failed') sys.exit(1) # show result if args.open: browser = webbrowser.get() log.status('open the submission page with browser') opened = browser.open_new_tab(submission.get_url()) if not opened: log.failure('failed to open the url. please set the $BROWSER envvar')
def compare_and_report(proc: subprocess.Popen, answer: str, memory: Optional[float], test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, mle: Optional[float], mode: str, error: Optional[float], does_print_input: bool, silent: bool, rstrip: bool, judge: Optional[str]) -> str: rstrip_targets = ' \t\r\n\f\v\0' # ruby's one, follow AnarchyGolf # prepare the comparing function if error is not None: # float mode match = lambda a, b: compare_as_floats(a, b, error) elif judge is not None: # special judge mode def match(a, b): # On Windows, a temp file is not created if we use "with" statement, user_output = tempfile.NamedTemporaryFile(delete=False) judge_result = False try: if rstrip: user_output.write(a.rstrip(rstrip_targets).encode()) else: user_output.write(a.encode()) user_output.close() arg0 = judge arg1 = str(test_input_path.resolve()) arg2 = user_output.name arg3 = str((str(test_output_path.resolve()) if test_output_path is not None else '')) actual_command = '{} {} {} {}'.format(arg0, arg1, arg2, arg3) # TODO: quote arguments for paths including spaces; see https://github.com/kmyk/online-judge-tools/pull/584 log.status('$ %s', actual_command) info, proc = utils.exec_command(actual_command) if not silent: log.emit('judge\'s output:\n%s', utils.make_pretty_large_file_content(info['answer'] or b'', limit=40, head=20, tail=10, bold=True)) judge_result = (proc.returncode == 0) finally: os.unlink(user_output.name) return judge_result else: def match(a, b): if a == b: return True if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets): log.warning('WA if no rstrip') return True if a == b.replace('\n', '\r\n'): log.warning(r'WA if not replacing "\r\n" with "\n"') return True if rstrip and a.rstrip(rstrip_targets) == b.replace('\n', '\r\n').rstrip(rstrip_targets): log.warning('WA if no rstrip') log.warning(r'WA if not replacing "\r\n" with "\n"') return True if a.replace('\n', '\r\n') == b: log.warning(r'WA if not replacing "\n" with "\r\n"') return True if rstrip and a.replace('\n', '\r\n').rstrip(rstrip_targets) == b.rstrip(rstrip_targets): # TODO: use a smart way if you need more equality patterns log.warning('WA if no rstrip') log.warning(r'WA if not replacing "\n" with "\r\n"') return True return False # prepare the function to print the input is_input_printed = False def print_input(): nonlocal is_input_printed if does_print_input and not is_input_printed: is_input_printed = True with test_input_path.open('rb') as inf: log.emit('input:\n%s', utils.make_pretty_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True)) # check TLE, RE or not status = 'AC' if proc.returncode is None: log.failure(log.red('TLE')) status = 'TLE' print_input() elif memory is not None and mle is not None and memory > mle: log.failure(log.red('MLE')) status = 'MLE' print_input() elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) status = 'RE' print_input() # check WA or not if (test_output_path is not None) or (judge is not None): if test_output_path is not None: with test_output_path.open('rb') as outf: expected = outf.read().decode() else: # only if --judge-command option expected = '' log.warning('expected output is not found') # compare if not match(answer, expected): log.failure(log.red('WA')) print_input() if not silent: if mode == 'simple': log.emit('output:\n%s', utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit('expected:\n%s', utils.make_pretty_large_file_content(expected.encode(), limit=40, head=20, tail=10, bold=True)) elif mode == 'side-by-side': if max(answer.count('\n'), expected.count('\n')) <= 40: display_side_by_side_color(answer, expected) else: display_snipped_side_by_side_color(answer, expected) else: assert False status = 'WA' else: if not silent: log.emit(('output:\n%s' if is_input_printed else '%s'), utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) if status == 'AC': log.success(log.green('AC')) return status
def test(args: 'argparse.Namespace') -> None: # list tests if not args.test: args.test = fmtutils.glob_with_format(args.directory, args.format) # by default if args.ignore_backup: args.test = fmtutils.drop_backup_or_hidden_files(args.test) tests = fmtutils.construct_relationship_of_files(args.test, args.directory, args.format) # check wheather GNU time is available if not check_gnu_time(args.gnu_time): log.warning('GNU time is not available: %s', args.gnu_time) args.gnu_time = None if args.mle is not None and args.gnu_time is None: raise RuntimeError('--mle is used but GNU time does not exist') # run tests history = [] # type: List[Dict[str, Any]] if args.jobs is None: for name, paths in sorted(tests.items()): history += [test_single_case(name, paths['in'], paths.get('out'), args=args)] else: if os.name == 'nt': log.warning("-j/--jobs opiton is unstable on Windows environmet") with concurrent.futures.ThreadPoolExecutor(max_workers=args.jobs) as executor: lock = threading.Lock() futures = [] # type: List[concurrent.futures.Future] for name, paths in sorted(tests.items()): futures += [executor.submit(test_single_case, name, paths['in'], paths.get('out'), lock=lock, args=args)] for future in futures: history += [future.result()] # summarize slowest = -1.0 # type: float slowest_name = '' heaviest = -1.0 # type: float heaviest_name = '' ac_count = 0 for result in history: if result['status'] == 'AC': ac_count += 1 if slowest < result['elapsed']: slowest = result['elapsed'] slowest_name = result['testcase']['name'] if result['memory'] is not None and heaviest < result['memory']: heaviest = result['memory'] heaviest_name = result['testcase']['name'] # print the summary log.emit('') log.status('slowest: %f sec (for %s)', slowest, slowest_name) if heaviest >= 0: if heaviest < MEMORY_WARNING: log.status('max memory: %f MB (for %s)', heaviest, heaviest_name) else: log.warning('max memory: %f MB (for %s)', heaviest, heaviest_name) if ac_count == len(tests): log.success('test ' + log.green('success') + ': %d cases', len(tests)) else: log.failure('test ' + log.red('failed') + ': %d AC / %d cases', ac_count, len(tests)) if args.json: print(json.dumps(history)) if ac_count != len(tests): sys.exit(1)
def print_input(): nonlocal is_input_printed if does_print_input and not is_input_printed: is_input_printed = True with test_input_path.open('rb') as inf: log.emit('input:\n%s', utils.make_pretty_large_file_content(inf.read(), limit=40, head=20, tail=10, bold=True))
def try_hack_once( generator: str, command: str, hack: str, *, tle: Optional[float], attempt: int, lock: Optional[threading.Lock] = None ) -> Optional[Tuple[bytes, bytes]]: with BufferedExecutor(lock) as submit: # print the header submit(log.emit, '') submit(log.info, '%d-th attempt', attempt) # generate input submit(log.status, 'generate input...') info, proc = utils.exec_command(generator, stdin=None, timeout=tle) input_data = info['answer'] # type: Optional[bytes] if not check_status(info, proc, submit=submit): return None assert input_data is not None # generate output submit(log.status, 'generate output...') info, proc = utils.exec_command(command, input=input_data, timeout=tle) output_data = info['answer'] # type: Optional[bytes] if not check_status(info, proc, submit=submit): return None assert output_data is not None # hack submit(log.status, 'hack...') info, proc = utils.exec_command(hack, input=input_data, timeout=tle) answer = (info['answer'] or b'').decode() # type: str elapsed = info['elapsed'] # type: float memory = info['memory'] # type: Optional[float] # compare status = 'AC' if proc.returncode is None: submit(log.failure, log.red('TLE')) status = 'TLE' elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) status = 'RE' expected = output_data.decode() if not simple_match(answer, expected): log.failure(log.red('WA')) log.emit( 'input:\n%s', utils.make_pretty_large_file_content(input_data, limit=40, head=20, tail=10, bold=True)) log.emit( 'output:\n%s', utils.make_pretty_large_file_content(answer.encode(), limit=40, head=20, tail=10, bold=True)) log.emit( 'expected:\n%s', utils.make_pretty_large_file_content(output_data, limit=40, head=20, tail=10, bold=True)) status = 'WA' if status == 'AC': return None else: return (input_data, output_data)
def generate_output_single_case(test_name: str, test_input_path: pathlib.Path, *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> None: # print the header if lock is None: log.emit('') log.info('%s', test_name) # run the command with test_input_path.open() as inf: info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle) answer = info['answer'] # type: Optional[bytes] elapsed = info['elapsed'] # type: float # acquire lock to print logs properly, if in parallel nullcontext = contextlib.ExitStack() with lock or nullcontext: if lock is not None: log.emit('') log.info('%s', test_name) # check the result log.status('time: %f sec', elapsed) if proc.returncode is None: log.failure(log.red('TLE')) log.info('skipped.') return elif proc.returncode != 0: log.failure(log.red('RE') + ': return code %d', proc.returncode) log.info('skipped.') return assert answer is not None log.emit( utils.make_pretty_large_file_content(answer, limit=40, head=20, tail=10, bold=True)) # find the destination path match_result = fmtutils.match_with_format( args.directory, args.format, test_input_path) # type: Optional[Match[Any]] if match_result is not None: matched_name = match_result.groupdict()['name'] # type: str else: assert False test_output_path = fmtutils.path_from_format(args.directory, args.format, name=matched_name, ext='out') # write the result to the file if not test_output_path.parent.is_dir(): os.makedirs(str(test_output_path.parent), exist_ok=True) with test_output_path.open('wb') as fh: fh.write(answer) log.success('saved to: %s', test_output_path)
def download(args: 'argparse.Namespace') -> None: # prepare values problem = dispatch.problem_from_url(args.url) if problem is None: raise requests.exceptions.InvalidURL( 'The contest "%s" is not supported' % args.url) is_default_format = args.format is None and args.directory is None # must be here since args.directory and args.format are overwritten if args.directory is None: args.directory = pathlib.Path('test') if args.format is None: args.format = '%b.%e' # get samples from the server with utils.new_session_with_our_user_agent(path=args.cookie) as sess: if args.yukicoder_token and isinstance(problem, YukicoderProblem): sess.headers['Authorization'] = 'Bearer {}'.format( args.yukicoder_token) if args.system: samples = problem.download_system_cases(session=sess) else: samples = problem.download_sample_cases(session=sess) if not samples: raise SampleParseError("Sample not found") # append the history for submit subcommand if not args.dry_run and is_default_format: history = onlinejudge_command.download_history.DownloadHistory() if not list(args.directory.glob('*')): # reset the history to help users who use only one directory for many problems history.remove(directory=pathlib.Path.cwd()) history.add(problem, directory=pathlib.Path.cwd()) # prepare files to write def iterate_files_to_write( sample: TestCase, *, i: int) -> Iterator[Tuple[str, pathlib.Path, bytes]]: for ext in ['in', 'out']: data = getattr(sample, ext + 'put_data') if data is None: continue name = sample.name table = {} table['i'] = str(i + 1) table['e'] = ext table['n'] = name table['b'] = os.path.basename(name) table['d'] = os.path.dirname(name) path = args.directory / format_utils.percentformat( args.format, table) # type: pathlib.Path yield ext, path, data for i, sample in enumerate(samples): for _, path, _ in iterate_files_to_write(sample, i=i): if path.exists(): raise FileExistsError( 'Failed to download since file already exists: ' + str(path)) # write samples to files for i, sample in enumerate(samples): log.emit('') log.info('sample %d', i) for ext, path, data in iterate_files_to_write(sample, i=i): log.status('%sput: %s', ext, sample.name) if not args.silent: log.emit( utils.make_pretty_large_file_content(data, limit=40, head=20, tail=10, bold=True)) if not args.dry_run: path.parent.mkdir(parents=True, exist_ok=True) with path.open('wb') as fh: fh.write(data) log.success('saved to: %s', path) # print json if args.json: print(json.dumps(list(map(convert_sample_to_dict, samples))))