def predict_judge_method(html: str) -> Judge:
def normalize(sentence):
return sentence.replace('\\', '').replace("{", "").replace("}", "").replace(",", "").replace(" ", "").replace(
"−", "-").lower().strip()
soup = BeautifulSoup(html, "html.parser")
sentences = soup.get_text().split("\n")
interactive_sentences = []
for s in sentences:
for kw in INTERACTIVE_ANCHORS:
if kw in s:
interactive_sentences.append(s)
if len(interactive_sentences) > 0:
return InteractiveJudge()
decimal_sentences = [normalize(s)
for s in sentences if is_decimal_context(s)]
decimal_keyword_cands = set()
decimal_val_cands = set()
if len(decimal_sentences) > 0: # Decimal
is_absolute = False
is_relative = False
for s in decimal_sentences:
for regexp in DECIMAL_STRATEGY_RE_LIST_KEYWORD:
r = regexp.findall(s)
for t in r:
if t == "絶対誤差" or t == "absolute":
is_absolute = True
elif t == "相対誤差" or t == "relative":
is_relative = True
decimal_keyword_cands.add(t)
for s in decimal_sentences:
for regexp in DECIMAL_STRATEGY_RE_LIST_VAL:
r = regexp.findall(s)
for t in r:
decimal_val_cands.add(int(t))
if len(decimal_val_cands) == 0:
# No error value candidate is found
return NormalJudge()
if len(decimal_val_cands) == 1:
if is_absolute and is_relative:
error_type = ErrorType.AbsoluteOrRelative
elif is_absolute:
error_type = ErrorType.Absolute
else:
assert is_relative
error_type = ErrorType.Relative
return DecimalJudge(error_type, 10.0**(int(list(decimal_val_cands)[0])))
raise MultipleDecimalCandidatesError(decimal_val_cands)
return NormalJudge()
def default_metadata():
return Metadata(problem=None,
code_filename=None,
sample_in_pattern=DEFAULT_IN_EXAMPLE_PATTERN,
sample_out_pattern=DEFAULT_OUT_EXAMPLE_PATTERN,
lang=None,
judge_method=NormalJudge())
def predict_constants(html: str) -> ProblemConstantSet:
try:
yes_str, no_str = predict_yes_no(html)
except YesNoPredictionFailedError:
yes_str = no_str = None
try:
mod = predict_modulo(html)
except MultipleModCandidatesError as e:
logger.warning(
"Modulo prediction failed -- "
"two or more candidates {} are detected as modulo values".format(
e.cands))
mod = None
try:
judge = predict_judge_method(html)
except MultipleModCandidatesError as e:
logger.warning(
"decimal prediction failed -- "
"two or more candidates {} are detected as decimal values".format(
e.cands))
judge = NormalJudge()
return ProblemConstantSet(mod=mod,
yes_str=yes_str,
no_str=no_str,
judge_method=judge)
def from_dict(cls, dic):
if "judge" in dic:
judge_type = dic["judge"]["judge_type"]
if judge_type == "normal":
judge_method = NormalJudge.from_dict(dic["judge"])
elif judge_type == "decimal":
judge_method = DecimalJudge.from_dict(dic["judge"])
else:
raise Exception("invalid judge type")
else:
judge_method = NormalJudge()
return Metadata(problem=Problem.from_dict(dic["problem"]),
code_filename=dic["code_filename"],
sample_in_pattern=dic["sample_in_pattern"],
sample_out_pattern=dic["sample_out_pattern"],
lang=Language.from_name(dic["lang"]),
judge_method=judge_method)
def get_sample_patterns_and_judge_method(
metadata_file: str) -> Tuple[str, str, Judge]:
try:
metadata = Metadata.load_from(metadata_file)
return metadata.sample_in_pattern, metadata.sample_out_pattern, metadata.judge_method
except IOError:
logger.warning(
"{} is not found. Assume the example file name patterns are {} and {}"
.format(metadata_file, DEFAULT_IN_EXAMPLE_PATTERN,
DEFAULT_OUT_EXAMPLE_PATTERN))
return DEFAULT_IN_EXAMPLE_PATTERN, DEFAULT_OUT_EXAMPLE_PATTERN, NormalJudge(
)
def run_for_samples(exec_file: str,
sample_pair_list: List[Tuple[str, str]],
timeout_sec: int,
judge_method: Judge = NormalJudge(),
knock_out: bool = False,
skip_io_on_success: bool = False) -> TestSummary:
success_count = 0
has_error_output = False
for in_sample_file, out_sample_file in sample_pair_list:
# Run program
exec_res = run_program(exec_file,
in_sample_file,
timeout_sec=timeout_sec)
# Output header
with open(out_sample_file, 'r') as f:
answer_text = f.read()
is_correct = exec_res.is_correct_output(answer_text, judge_method)
has_error_output = has_error_output or exec_res.has_stderr()
if is_correct:
if exec_res.has_stderr():
message = with_color(
"CORRECT but with stderr (Please remove stderr!)",
Fore.LIGHTYELLOW_EX)
else:
message = "{} {elapsed} ms".format(with_color(
"PASSED", Fore.LIGHTGREEN_EX),
elapsed=exec_res.elapsed_ms)
success_count += 1
else:
if exec_res.status == ExecStatus.NORMAL:
message = with_color("WA", Fore.LIGHTRED_EX)
else:
message = with_color(exec_res.status.name, Fore.LIGHTYELLOW_EX)
print("# {case_name} ... {message}".format(
case_name=os.path.basename(in_sample_file),
message=message,
))
# Output details for incorrect results or has stderr.
if not is_correct or (exec_res.has_stderr()
and not skip_io_on_success):
print('{}\n'.format(
build_details_str(exec_res, in_sample_file, out_sample_file)))
if knock_out and not is_correct:
print('Stop testing ...')
break
return TestSummary(success_count, has_error_output)
def __init__(self,
problem: Optional[Problem],
code_filename: Optional[str],
sample_in_pattern: Optional[str],
sample_out_pattern: Optional[str],
lang: Optional[Language],
judge_method: Judge = NormalJudge()):
self.problem = problem
self.code_filename = code_filename
self.sample_in_pattern = sample_in_pattern
self.sample_out_pattern = sample_out_pattern
self.lang = lang
self.judge_method = judge_method
def from_dict(cls, dic):
if "judge" in dic:
judge_type = dic["judge"]["judge_type"]
if judge_type == "normal":
judge_method = NormalJudge.from_dict(dic["judge"])
elif judge_type == "decimal":
judge_method = DecimalJudge.from_dict(dic["judge"])
elif judge_type == "multisolution":
judge_method = MultiSolutionJudge()
elif judge_type == "interactive":
judge_method = InteractiveJudge()
else:
raise NoJudgeTypeException()
else:
judge_method = NormalJudge()
return Metadata(problem=Problem.from_dict(dic["problem"]),
code_filename=dic["code_filename"],
sample_in_pattern=dic["sample_in_pattern"],
sample_out_pattern=dic["sample_out_pattern"],
lang=Language.from_name(dic["lang"]),
judge_method=judge_method)
def __init__(self,
problem: Problem,
code_filename: str,
sample_in_pattern: str,
sample_out_pattern: str,
lang: Language,
judge_method: Judge = NormalJudge()):
self.problem = problem
self.code_filename = code_filename
self.sample_in_pattern = sample_in_pattern
self.sample_out_pattern = sample_out_pattern
self.lang = lang
self.judge_method = judge_method
def _decide_judge_method(args: argparse.Namespace, metadata: Metadata,
lang: Optional[Language]):
def _decide_decimal_judge():
if args.error_value is not None:
diff = args.error_value
elif isinstance(metadata.judge_method, DecimalJudge):
diff = metadata.judge_method.diff
else:
diff = DEFAULT_EPS
if args.judge_type:
assert args.judge_type in [
"absolute", "relative", "absolute_or_relative"
]
error_type = ErrorType(args.judge_type)
elif isinstance(metadata.judge_method, DecimalJudge):
error_type = metadata.judge_method.error_type
else:
raise Exception("Must not reach")
return DecimalJudge(diff=diff, error_type=error_type)
if args.judge_type is not None:
if args.judge_type == "normal":
return NormalJudge()
elif args.judge_type in [
"absolute", "relative", "absolute_or_relative"
]:
return _decide_decimal_judge()
elif args.judge_type == "multisolution":
assert lang is not None
return MultiSolutionJudge()
elif args.judge_type == "interactive":
assert lang is not None
return InteractiveJudge()
else:
logger.error(
"Unknown judge type: {}. judge type must be one of [{}]".
format(args.judge_type,
", ".join(USER_FACING_JUDGE_TYPE_LIST)))
raise InvalidJudgeTypeError()
if isinstance(metadata.judge_method, DecimalJudge):
return _decide_decimal_judge()
return metadata.judge_method
def main(prog, args) -> bool:
parser = argparse.ArgumentParser(
prog=prog,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("--exec", '-e',
help="File path to the execution target. [Default] Automatically detected exec file",
default=None)
parser.add_argument("--num", '-n',
help="The case number to test (1-origin). All cases are tested if not specified.",
type=int,
default=None)
parser.add_argument("--dir", '-d',
help="Target directory to test. [Default] Current directory",
default=".")
parser.add_argument("--timeout", '-t',
help="Timeout for each test cases (sec) [Default] 1",
type=int,
default=1)
parser.add_argument("--knock-out", '-k',
help="Stop execution immediately after any example's failure [Default] False",
action="store_true",
default=False)
parser.add_argument('--skip-almost-ac-feedback', '-s',
help='Hide inputs and expected/actual outputs if result is correct and there are error outputs'
' [Default] False,',
action='store_true',
default=False)
parser.add_argument('--judge-type', '-j',
help='error type'
' must be one of [{}]'.format(
', '.join(USER_FACING_JUDGE_TYPE_LIST)),
type=str,
default=None)
parser.add_argument('--error-value', '-v',
help='error value for decimal number judge:'
' [Default] ' + str(DEFAULT_EPS),
type=float,
default=None)
parser.add_argument('--compile-before-testing', '-c',
help='compile source before testing [true, false]: '
' [Default]: false',
type=bool,
default=None)
parser.add_argument('--compile-only-when-diff-detected',
help='compile only when diff detected [true, false]'
' [Default]: true',
type=bool,
default=None)
parser.add_argument("--config",
help="File path to your config file\n{0}{1}".format("[Default (Primary)] {}\n".format(
USER_CONFIG_PATH),
"[Default (Secondary)] {}\n".format(
get_default_config_path()))
)
args = parser.parse_args(args)
config = get_config(args)
if config.etc_config.compile_before_testing is not None and args.compile_before_testing is None:
args.compile_before_testing = config.etc_config.compile_before_testing
if args.compile_before_testing:
if config.etc_config.compile_only_when_diff_detected is not None and args.compile_only_when_diff_detected is None:
args.compile_only_when_diff_detected = config.etc_config.compile_only_when_diff_detected
metadata_file = os.path.join(args.dir, "metadata.json")
metadata = get_metadata(metadata_file)
judge_method = metadata.judge_method
lang = metadata.lang
in_sample_file_list = sorted(
glob.glob(os.path.join(args.dir, metadata.sample_in_pattern)))
out_sample_file_list = sorted(
glob.glob(os.path.join(args.dir, metadata.sample_out_pattern)))
user_input_decimal_error_type = None
if args.judge_type is not None:
if args.judge_type == "normal":
judge_method = NormalJudge()
elif args.judge_type in ["absolute", "relative", "absolute_or_relative"]:
user_input_decimal_error_type = ErrorType(args.judge_type)
elif args.judge_type == "multisolution":
judge_method = MultiSolutionJudge(lang.name)
elif args.judge_type == "interactive":
judge_method = InteractiveJudge(lang.name)
else:
logger.error("Unknown judge type: {}. judge type must be one of [{}]".format(
args.judge_type, ", ".join(USER_FACING_JUDGE_TYPE_LIST)))
sys.exit(-1)
user_input_error_value = args.error_value
if isinstance(judge_method, DecimalJudge):
judge_method = DecimalJudge(error_type=user_input_decimal_error_type or judge_method.error_type,
diff=user_input_error_value or judge_method.diff)
elif user_input_decimal_error_type is not None:
judge_method = DecimalJudge(error_type=user_input_decimal_error_type,
diff=user_input_error_value or DEFAULT_EPS)
elif user_input_error_value is not None:
assert judge_method.judge_type == JudgeType.Normal
logger.warn("error_value {} is ignored because this is normal judge".format(
user_input_error_value))
if isinstance(judge_method, DecimalJudge):
logger.info("Decimal number judge is enabled. type={}, diff={}".format(
judge_method.error_type.value, judge_method.diff))
if metadata.code_filename is None or not args.compile_before_testing:
print("compile is skipped and infer exec file")
exclude_exec_files = []
if hasattr(judge_method, "judge_exec_filename"):
judge_method.judge_exec_filename = os.path.join(
args.dir, judge_method.judge_exec_filename)
exclude_exec_files.append(judge_method.judge_exec_filename)
exec_file = args.exec or infer_exec_file(
glob.glob(os.path.join(args.dir, '*')), exclude_exec_files)
else:
if args.compile_only_when_diff_detected:
force_compile = True
else:
force_compile = False
exec_file = lang.get_test_command('main', args.dir)
print("command: ", exec_file)
print("directory: ", args.dir)
# Compile
if not compile_main_and_judge_programs(metadata, args.dir, force_compile=force_compile):
exit()
if args.num is None:
return run_all_tests(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.knock_out,
args.skip_almost_ac_feedback, judge_method, args.dir)
else:
return run_single_test(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.num,
judge_method, args.dir)
def run_for_samples(exec_file: str, sample_pair_list: List[Tuple[str, str]], timeout_sec: int,
judge_method: Judge = NormalJudge(), knock_out: bool = False,
skip_io_on_success: bool = False, cwd: str = "./") -> TestSummary:
success_count = 0
has_error_output = False
for in_sample_file, out_sample_file in sample_pair_list:
if judge_method.judge_type == JudgeType.Interactive:
exec_res = run_interactive_program(exec_file,
judge_method.judge_code_lang.get_test_command(
'judge', cwd),
in_sample_file, out_sample_file,
timeout_sec=timeout_sec,
current_working_dir=cwd
)
is_correct = exec_res.is_correct_output(judge_method=judge_method)
else:
# Run program
exec_res = run_program(exec_file, in_sample_file,
timeout_sec=timeout_sec, current_working_dir=cwd)
if judge_method.judge_type == JudgeType.MultiSolution:
is_correct = exec_res.is_correct_output(
judge_method=judge_method, sample_input_file=in_sample_file, sample_output_file=out_sample_file, cwd=cwd)
else:
# Output header
with open(out_sample_file, 'r') as f:
expected_answer_text = f.read()
is_correct = exec_res.is_correct_output(
expected_answer_text, judge_method)
if exec_res.output is None:
exec_res.output = ""
elif isinstance(exec_res.output, bytes):
exec_res.output = exec_res.output.decode()
if exec_res.stderr is None:
exec_res.stderr = ""
elif isinstance(exec_res.stderr, bytes):
exec_res.stderr = exec_res.stderr.decode()
has_error_output = has_error_output or exec_res.has_stderr()
if is_correct:
if exec_res.has_stderr():
message = with_color(
"CORRECT but with stderr (Please remove stderr!)", Fore.LIGHTYELLOW_EX)
else:
message = "{} {elapsed} ms".format(
with_color("PASSED", Fore.LIGHTGREEN_EX),
elapsed=exec_res.elapsed_ms)
success_count += 1
else:
if exec_res.status == ExecStatus.NORMAL:
message = with_color("WA", Fore.LIGHTRED_EX)
else:
message = with_color(
exec_res.status.name, Fore.LIGHTYELLOW_EX)
print("# {case_name} ... {message}".format(
case_name=os.path.basename(in_sample_file),
message=message,
))
# Output details for incorrect results or has stderr.
if not is_correct or (exec_res.has_stderr() and not skip_io_on_success):
print('{}\n'.format(build_details_str(
exec_res, in_sample_file, out_sample_file)))
if knock_out and not is_correct:
print('Stop testing ...')
break
return TestSummary(success_count, has_error_output)
def main(prog, args) -> None:
if len(args) == 0:
print("Usage: atcoder tools set [options]")
return
parser = argparse.ArgumentParser(
prog=prog, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--judge-type',
'-j',
help='error type'
' must be one of [{}]'.format(
', '.join(USER_FACING_JUDGE_TYPE_LIST)),
type=str,
default=None)
parser.add_argument('--error-value',
'-v',
help='error value for decimal number judge:'
' [Default] ' + str(DEFAULT_EPS),
type=float,
default=None)
parser.add_argument(
"--lang",
help="Programming language of your template code, {}.\n".format(
" or ".join([lang.name for lang in ALL_LANGUAGES])),
default=None)
parser.add_argument(
"--dir",
'-d',
help="Target directory to test. [Default] Current directory",
default=".")
args = parser.parse_args(args)
old_metadata = Metadata.load_from(os.path.join(args.dir, "metadata.json"))
# Use the old metadata as base metadata.
output_metadata = Metadata.load_from(
os.path.join(args.dir, "metadata.json"))
if args.judge_type in ["absolute", "relative", "absolute_or_relative"]:
new_metadata_judge_type = "decimal"
else:
new_metadata_judge_type = args.judge_type
old_metadata_judge_type = old_metadata.judge_method.judge_type.value
if new_metadata_judge_type is not None and new_metadata_judge_type != old_metadata_judge_type:
if new_metadata_judge_type == JudgeType.Normal.value:
output_metadata.judge_method = NormalJudge()
elif new_metadata_judge_type == JudgeType.Decimal.value:
output_metadata.judge_method = DecimalJudge()
elif new_metadata_judge_type == JudgeType.MultiSolution.value:
output_metadata.judge_method = MultiSolutionJudge()
elif new_metadata_judge_type == JudgeType.Interactive.value:
output_metadata.judge_method = InteractiveJudge()
else:
raise NoJudgeTypeException()
judge_code_filename = os.path.join(args.dir, "judge.cpp")
if new_metadata_judge_type == JudgeType.Decimal.value:
if args.error_value is not None:
output_metadata.judge_method.diff = args.error_value
else:
logger.warn(
"Error-value is not specified. Default value will be set.")
output_metadata.judge_method.error_type = ErrorType(args.judge_type)
elif new_metadata_judge_type == JudgeType.MultiSolution.value:
if not os.path.exists(judge_code_filename):
print("Creating {} (multi-solution)".format(judge_code_filename))
judge_template_path = get_default_judge_template_path('cpp')
shutil.copy(judge_template_path, judge_code_filename)
else:
print("Judge code exists. Skipping creating judge code...")
elif new_metadata_judge_type == JudgeType.Interactive.value:
if not os.path.exists(judge_code_filename):
print("Creating {} (interactive)".format(judge_code_filename))
judge_template_path = get_default_judge_template_path('cpp')
shutil.copy(judge_template_path, judge_code_filename)
else:
print("Judge code exists. Skipping creating judge code...")
if args.lang is not None:
if args.lang != output_metadata.lang.name:
output_metadata.lang = Language.from_name(args.lang)
output_metadata.code_filename = output_metadata.lang.get_code_filename(
'main')
url = "https://atcoder.jp/contests/{}/tasks/{}".format(
output_metadata.problem.contest.contest_id,
output_metadata.problem.problem_id)
main_code_filename = os.path.join(args.dir,
output_metadata.code_filename)
if not os.path.exists(main_code_filename):
codegen_main("", ["--lang", output_metadata.lang.name, url],
open(main_code_filename, 'w'))
else:
print("File exists: ", output_metadata.code_filename)
else:
print("Already set to {}. Skipping changing language...".format(
args.lang))
output_metadata.save_to(os.path.join(args.dir, "metadata.json"))
@classmethod
def update_from_source(cls, old, program):
dic = old.to_dict() if old else {}
with open(program, 'r') as f:
source = f.read()
m = re.search(
r'^# contest: (\S+?), problem: (\S+?), alphabet: (\S+?)$', source, re.MULTILINE)
dic['code_filename'] = program
dic['problem'] = {'alphabet': m.group(3),
'contest': {'contest_id': m.group(1)},
'problem_id': m.group(2)}
ext = os.path.splitext(program)[1][1:]
langs = [language for language in ALL_LANGUAGES if language.extension == ext]
if len(langs) == 1:
dic['lang'] = langs[0].name
if 'sample_in_pattern' not in dic:
dic['sample_in_pattern'] = DEFAULT_IN_EXAMPLE_PATTERN
if 'sample_out_pattern' not in dic:
dic['sample_out_pattern'] = DEFAULT_IN_EXAMPLE_PATTERN
return cls.from_dict(dic)
DEFAULT_METADATA = Metadata(
problem=None,
code_filename=None,
sample_in_pattern=DEFAULT_IN_EXAMPLE_PATTERN,
sample_out_pattern=DEFAULT_OUT_EXAMPLE_PATTERN,
lang=CPP,
judge_method=NormalJudge()
)
def main(prog, args):
if len(args) == 0:
print("Usage: atcoder tools set [options]")
return
parser = argparse.ArgumentParser(
prog=prog,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--judge-type', '-j',
help='error type'
' must be one of [{}]'.format(
', '.join(USER_FACING_JUDGE_TYPE_LIST)),
type=str,
default=None)
parser.add_argument('--error-value', '-v',
help='error value for decimal number judge:'
' [Default] ' + str(DEFAULT_EPS),
type=float,
default=None)
parser.add_argument("--lang",
help="Programming language of your template code, {}.\n".format(
" or ".join([lang.name for lang in ALL_LANGUAGES])),
default=None)
parser.add_argument("--dir", '-d',
help="Target directory to test. [Default] Current directory",
default=".")
args = parser.parse_args(args)
metadata = Metadata.load_from(args.dir + "/metadata.json")
new_judge_type = args.judge_type
if new_judge_type in ["decimal", "absolute", "relative", "absolute_or_relative"]:
new_judge_type = "decimal"
if args.judge_type == "decimal":
args.judge_type = "absolute_or_relative"
old_judge_type = metadata.judge_method.judge_type.value
if new_judge_type is not None and new_judge_type != old_judge_type:
if new_judge_type == JudgeType.Normal.value:
metadata.judge_method = NormalJudge()
elif new_judge_type == JudgeType.Decimal.value:
metadata.judge_method = DecimalJudge()
elif new_judge_type == JudgeType.MultiSolution.value:
metadata.judge_method = MultiSolutionJudge()
elif new_judge_type == JudgeType.Interactive.value:
metadata.judge_method = InteractiveJudge()
else:
raise NoJudgeTypeException()
if new_judge_type == JudgeType.Decimal.value:
if args.error_value is not None:
metadata.judge_method.diff = args.error_value
else:
print("Warning: error-value is not specified default value is set. ")
metadata.judge_method.error_type = ErrorType(args.judge_type)
elif new_judge_type == JudgeType.MultiSolution.value:
if not os.path.exists("./judge.cpp"):
print("touch ./judge.cpp (multi sotlution)")
judge_template_path = get_default_judge_template_path('cpp')
shutil.copy(judge_template_path, "./judge.cpp")
else:
print("Judge Code exists")
elif new_judge_type == JudgeType.Interactive.value:
if not os.path.exists("/judge.cpp"):
print("touch ./judge.cpp (interactive)")
judge_template_path = get_default_judge_template_path('cpp')
shutil.copy(judge_template_path, "./judge.cpp")
else:
print("Judge Code exists")
if args.lang is not None:
if args.lang != metadata.lang.name:
metadata.lang = Language.from_name(args.lang)
metadata.code_filename = metadata.lang.get_code_filename('main')
url = "https://atcoder.jp/contests/{}/tasks/{}".format(
metadata.problem.contest.contest_id, metadata.problem.problem_id)
if not os.path.exists(metadata.code_filename):
codegen_main("", ["--lang", metadata.lang.name,
url], open(metadata.code_filename, 'w'))
else:
print("file exists: ", metadata.code_filename)
else:
print("already set to {}".format(args.lang))
metadata.save_to(args.dir + "/metadata.json")
return metadata
def main(prog, args) -> bool:
parser = argparse.ArgumentParser(
prog=prog, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
"--exec",
'-e',
help=
"File path to the execution target. [Default] Automatically detected exec file",
default=None)
parser.add_argument(
"--num",
'-n',
help=
"The case number to test (1-origin). All cases are tested if not specified.",
type=int,
default=None)
parser.add_argument(
"--dir",
'-d',
help="Target directory to test. [Default] Current directory",
default=".")
parser.add_argument("--timeout",
'-t',
help="Timeout for each test cases (sec) [Default] 1",
type=int,
default=1)
parser.add_argument(
"--knock-out",
'-k',
help=
"Stop execution immediately after any example's failure [Default] False",
action="store_true",
default=False)
parser.add_argument(
'--skip-almost-ac-feedback',
'-s',
help=
'Hide inputs and expected/actual outputs if result is correct and there are error outputs'
' [Default] False,',
action='store_true',
default=False)
parser.add_argument('--judge-type',
'-j',
help='error type'
' must be one of [{}]'.format(
', '.join(USER_FACING_JUDGE_TYPE_LIST)),
type=str,
default=None)
parser.add_argument('--error-value',
'-v',
help='error value for decimal number judge:'
' [Default] ' + str(DEFAULT_EPS),
type=float,
default=None)
args = parser.parse_args(args)
exec_file = args.exec or infer_exec_file(
glob.glob(os.path.join(args.dir, '*')))
metadata_file = os.path.join(args.dir, "metadata.json")
in_ex_pattern, out_ex_pattern, judge_method = get_sample_patterns_and_judge_method(
metadata_file)
in_sample_file_list = sorted(
glob.glob(os.path.join(args.dir, in_ex_pattern)))
out_sample_file_list = sorted(
glob.glob(os.path.join(args.dir, out_ex_pattern)))
user_input_decimal_error_type = None
if args.judge_type is not None:
if args.judge_type == "normal":
judge_method = NormalJudge()
elif args.judge_type in [
"absolute", "relative", "absolute_or_relative"
]:
user_input_decimal_error_type = ErrorType(args.judge_type)
else:
logger.error(
"Unknown judge type: {}. judge type must be one of [{}]".
format(args.judge_type,
", ".join(USER_FACING_JUDGE_TYPE_LIST)))
sys.exit(-1)
user_input_error_value = args.error_value
if isinstance(judge_method, DecimalJudge):
judge_method = DecimalJudge(error_type=user_input_decimal_error_type
or judge_method.error_type,
diff=user_input_error_value
or judge_method.diff)
elif user_input_decimal_error_type is not None:
judge_method = DecimalJudge(error_type=user_input_decimal_error_type,
diff=user_input_error_value or DEFAULT_EPS)
elif user_input_error_value is not None:
assert judge_method.judge_type == JudgeType.Normal
logger.warn(
"error_value {} is ignored because this is normal judge".format(
user_input_error_value))
if isinstance(judge_method, DecimalJudge):
logger.info("Decimal number judge is enabled. type={}, diff={}".format(
judge_method.error_type.value, judge_method.diff))
if args.num is None:
return run_all_tests(exec_file, in_sample_file_list,
out_sample_file_list, args.timeout,
args.knock_out, args.skip_almost_ac_feedback,
judge_method)
else:
return run_single_test(exec_file, in_sample_file_list,
out_sample_file_list, args.timeout, args.num,
judge_method)