def login(self, credential_supplier=None, use_local_session_cache=True, save_session_cache=True): if credential_supplier is None: credential_supplier = default_credential_supplier if use_local_session_cache: load_cookie_to(self._session) if self.check_logging_in(): logger.info( "Successfully Logged in using the previous session cache.") logger.info( "If you'd like to invalidate the cache, delete {}.".format(default_cookie_path)) return username, password = credential_supplier() resp = self._request("https://arc001.contest.atcoder.jp/login", data={ 'name': username, "password": password }, method='POST') if resp.text.find("パスワードを忘れた方はこちら") != -1: raise LoginError if use_local_session_cache and save_session_cache: save_cookie(self._session)
def main(prog, args, output_file=sys.stdout): parser = argparse.ArgumentParser( prog=prog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( "url", help="URL (e.g. https://atcoder.jp/contests/abc012/tasks/abc012_3)") parser.add_argument("--without-login", action="store_true", help="Download data without login") parser.add_argument( "--lang", help="Programming language of your template code, {}.\n".format( " or ".join([lang.name for lang in ALL_LANGUAGES])) + "[Default] {}".format(CPP.name)) parser.add_argument("--template", help="File path to your template code\n{}".format( "\n".join([ "[Default ({dname})] {path}".format( dname=lang.display_name, path=lang.default_template_path) for lang in ALL_LANGUAGES ]))) parser.add_argument("--save-no-session-cache", action="store_true", help="Save no session cache to avoid security risk", default=None) parser.add_argument( "--config", help="File path to your config file\n{0}{1}".format( "[Default (Primary)] {}\n".format(USER_CONFIG_PATH), "[Default (Secondary)] {}\n".format(get_default_config_path()))) args = parser.parse_args(args) args.workspace = DEFAULT_WORKSPACE_DIR_PATH # dummy for get_config() args.parallel = False # dummy for get_config() config = get_config(args) client = AtCoderClient() if not config.etc_config.download_without_login: try: client.login( save_session_cache=not config.etc_config.save_no_session_cache) logger.info("Login successful.") except LoginError: logger.error( "Failed to login (maybe due to wrong username/password combination?)" ) sys.exit(-1) else: logger.info("Downloading data without login.") generate_code(client, args.url, config, output_file=output_file)
def load_cookie_to(session: requests.Session, cookie_path: Optional[str] = None): cookie_path = cookie_path or default_cookie_path session.cookies = LWPCookieJar(cookie_path) if os.path.exists(cookie_path): session.cookies.load() logger.info( "Loaded session from {}".format(os.path.abspath(cookie_path))) return True return False
def _load(path: str) -> Config: logger.info("Going to load {} as config".format(path)) with open(path, 'r') as f: program_args = ProgramArgs.load(args) if language is not None: assert program_args.lang is None program_args.lang = language.name return Config.load(f, program_args)
def prepare_contest(atcoder_client: AtCoderClient, contest_id: str, config: Config, retry_delay_secs: float = 1.5, retry_max_delay_secs: float = 60, retry_max_tries: int = 10): attempt_count = 1 while True: problem_list = atcoder_client.download_problem_list( Contest(contest_id=contest_id)) if problem_list: break if 0 < retry_max_tries < attempt_count: raise EnvironmentInitializationError logger.warning( "Failed to fetch. Will retry in {} seconds. (Attempt {})".format( retry_delay_secs, attempt_count)) time.sleep(retry_delay_secs) retry_delay_secs = min(retry_delay_secs * 2, retry_max_delay_secs) attempt_count += 1 atcoder_client.download_contest_languages(problem_list) tasks = [(atcoder_client, problem, config) for problem in problem_list] output_splitter() if config.etc_config.parallel_download: thread_pool = Pool(processes=cpu_count()) thread_pool.map(func, tasks) else: for argv in tasks: try: func(argv) except Exception: # Prevent the script from stopping print(traceback.format_exc(), file=sys.stderr) pass if config.postprocess_config.exec_cmd_on_contest_dir is not None: contest_dir_path = os.path.join(config.code_style_config.workspace_dir, contest_id) logger.info( _message_on_execution( contest_dir_path, config.postprocess_config.exec_cmd_on_contest_dir)) config.postprocess_config.execute_on_contest_dir(contest_dir_path)
def prepare_contest(atcoder_client: AtCoderClient, contest_id: str, config: Config): retry_duration = 1.5 while True: problem_list = atcoder_client.download_problem_list( Contest(contest_id=contest_id)) if problem_list: break sleep(retry_duration) logger.warning( "Failed to fetch. Will retry in {} seconds".format(retry_duration)) tasks = [(atcoder_client, problem, config) for problem in problem_list] output_splitter() if config.etc_config.parallel_download: thread_pool = Pool(processes=cpu_count()) thread_pool.map(func, tasks) else: for argv in tasks: try: func(argv) except Exception: # Prevent the script from stopping print(traceback.format_exc(), file=sys.stderr) pass if config.postprocess_config.exec_cmd_on_contest_dir is not None: contest_dir_path = os.path.join( config.code_style_config.workspace_dir, contest_id) logger.info(_message_on_execution(contest_dir_path, config.postprocess_config.exec_cmd_on_contest_dir)) config.postprocess_config.execute_on_contest_dir( contest_dir_path)
def submit_source_code(self, contest: Contest, problem: Problem, lang: Union[str, Language], source: str) -> Submission: if isinstance(lang, str): warnings.warn( "Parameter lang as a str object is deprecated. " "Please use 'atcodertools.common.language.Language' object instead", UserWarning) lang_option_pattern = lang else: lang_option_pattern = lang.submission_lang_pattern resp = self._request(contest.get_submit_url()) soup = BeautifulSoup(resp.text, "html.parser") session_id = soup.find("input", attrs={"type": "hidden"}).get("value") task_select_area = soup.find( 'select', attrs={"id": "submit-task-selector"}) task_field_name = task_select_area.get("name") task_number = task_select_area.find( "option", text=re.compile('{} -'.format(problem.get_alphabet()))).get("value") language_select_area = soup.find( 'select', attrs={"id": "submit-language-selector-{}".format(task_number)}) language_field_name = language_select_area.get("name") language_number = language_select_area.find( "option", text=lang_option_pattern).get("value") postdata = { "__session": session_id, task_field_name: task_number, language_field_name: language_number, "source_code": source } logger.info("Do Post") resp = self._request( contest.get_submit_url(), data=postdata, method='POST') return Submission.make_submissions_from(resp.text)[0]
def login(self, credential_supplier=None, use_local_session_cache=True, save_session_cache=True): if credential_supplier is None: credential_supplier = default_credential_supplier if use_local_session_cache: load_cookie_to(self._session) if self.check_logging_in(): logger.info( "Successfully Logged in using the previous session cache.") logger.info( "If you'd like to invalidate the cache, delete {}.".format( default_cookie_path)) return username, password = credential_supplier() soup = BeautifulSoup( self._session.get("https://atcoder.jp/login").text, "html.parser") token = soup.find_all("form")[1].find("input", type="hidden").get("value") resp = self._request("https://atcoder.jp/login", data={ 'username': username, "password": password, "csrf_token": token }, method='POST') if resp.text.find("パスワードを忘れた方はこちら") != -1: raise LoginError if use_local_session_cache and save_session_cache: save_cookie(self._session)
def main(prog, args) -> bool: parser = argparse.ArgumentParser( prog=prog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( "--exec", '-e', help= "File path to the execution target. [Default] Automatically detected exec file", default=None) parser.add_argument( "--num", '-n', help= "The case number to test (1-origin). All cases are tested if not specified.", type=int, default=None) parser.add_argument( "--dir", '-d', help="Target directory to test. [Default] Current directory", default=".") parser.add_argument("--timeout", '-t', help="Timeout for each test cases (sec) [Default] 1", type=int, default=1) parser.add_argument( "--knock-out", '-k', help= "Stop execution immediately after any example's failure [Default] False", action="store_true", default=False) parser.add_argument( '--skip-almost-ac-feedback', '-s', help= 'Hide inputs and expected/actual outputs if result is correct and there are error outputs' ' [Default] False,', action='store_true', default=False) parser.add_argument('--judge-type', '-j', help='error type' ' must be one of [{}]'.format( ', '.join(USER_FACING_JUDGE_TYPE_LIST)), type=str, default=None) parser.add_argument('--error-value', '-v', help='error value for decimal number judge:' ' [Default] ' + str(DEFAULT_EPS), type=float, default=None) parser.add_argument('--compile-before-testing', '-c', help='compile source before testing [true, false]: ' ' [Default]: false', type=bool, default=None) parser.add_argument('--compile-only-when-diff-detected', help='compile only when diff detected [true, false]' ' [Default]: true', type=bool, default=None) parser.add_argument( "--config", help="File path to your config file\n{0}{1}".format( "[Default (Primary)] {}\n".format(USER_CONFIG_PATH), "[Default (Secondary)] {}\n".format(get_default_config_path()))) args = parser.parse_args(args) metadata_file = os.path.join(args.dir, "metadata.json") metadata = get_metadata(metadata_file) lang = metadata.lang # TODO: Stop loading language-specific config because tester doesn't have and shouldn't have --lang params. # TODO: All information required to run tester should be from metadata.json except for etc config # TODO: https://github.com/kyuridenamida/atcoder-tools/issues/177 config = get_config(args, lang) in_sample_file_list = sorted( glob.glob(os.path.join(args.dir, metadata.sample_in_pattern))) out_sample_file_list = sorted( glob.glob(os.path.join(args.dir, metadata.sample_out_pattern))) judge_method = _decide_judge_method(args, metadata, lang) if isinstance(judge_method, DecimalJudge): logger.info("Decimal number judge is enabled. type={}, diff={}".format( judge_method.error_type.value, judge_method.diff)) if args.exec is not None: exec_file = args.exec elif config.etc_config.compile_before_testing: # Use atcoder-tools's functionality to compile source code try: compile_main_and_judge_programs( metadata, args.dir, force_compile=not config.etc_config. compile_only_when_diff_detected) except BadStatusCodeException as e: raise e exec_file = lang.get_test_command('main', args.dir) else: # TODO Have a smarter strategy to detect judge program excluded_exec_files = [ os.path.join(args.dir, "judge"), os.path.join(args.dir, "judge.exe") ] exec_file = infer_exec_file(glob.glob(os.path.join(args.dir, '*')), excluded_exec_files) logger.info("Inferred exec file: {}".format(exec_file)) if args.num is None: return run_all_tests(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.knock_out, args.skip_almost_ac_feedback, judge_method, args.dir, lang) # TODO: pass judge_lang instead else: return run_single_test(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.num, judge_method, args.dir, lang)
def main(prog, args, credential_supplier=None, use_local_session_cache=True) -> bool: parser = argparse.ArgumentParser( prog=prog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( "--exec", '-e', help= "File path to the execution target. [Default] Automatically detected exec file", default=None) parser.add_argument( "--dir", '-d', help="Target directory to test. [Default] Current directory", default=".") parser.add_argument("--timeout", '-t', help="Timeout for each test cases (sec) [Default] 1", type=int, default=1) parser.add_argument( "--code", '-c', help= "Path to the source code to submit [Default] Code path written in metadata.json", type=str, default=None) parser.add_argument( "--force", "-f", action="store_true", help= "Submit the code regardless of the local test result [Default] False", default=False) parser.add_argument("--save-no-session-cache", action="store_true", help="Save no session cache to avoid security risk", default=False) parser.add_argument( "--unlock-safety", "-u", action="store_true", help= "By default, this script only submits the first code per problem. However, you can remove" " the safety by this option in order to submit codes twice or more.", default=False) args = parser.parse_args(args) metadata_file = os.path.join(args.dir, "metadata.json") try: metadata = Metadata.load_from(metadata_file) except IOError: logger.error( "{0} is not found! You need {0} to use this submission functionality." .format(metadata_file)) return False try: client = AtCoderClient() client.login( save_session_cache=args.save_no_session_cache, credential_supplier=credential_supplier, use_local_session_cache=use_local_session_cache, ) except LoginError: logger.error("Login failed. Try again.") return False tester_args = [] if args.exec: tester_args += ["-e", args.exec] if args.dir: tester_args += ["-d", args.dir] if args.timeout: tester_args += ["-t", str(args.timeout)] if args.force or tester.main("", tester_args): submissions = client.download_submission_list(metadata.problem.contest) if not args.unlock_safety: for submission in submissions: if submission.problem_id == metadata.problem.problem_id: logger.error( with_color( "Cancel submitting because you already sent some code to the problem. Please " "specify -u to send the code. {}".format( metadata.problem.contest.get_submissions_url( submission)), Fore.LIGHTRED_EX)) return False code_path = args.code or os.path.join(args.dir, metadata.code_filename) with open(code_path, 'r') as f: source = f.read() logger.info("Submitting {} as {}".format(code_path, metadata.lang.name)) submission = client.submit_source_code(metadata.problem.contest, metadata.problem, metadata.lang, source) logger.info("{} {}".format( with_color("Done!", Fore.LIGHTGREEN_EX), metadata.problem.contest.get_submissions_url(submission)))
def _load(path: str) -> Config: logger.info("Going to load {} as config".format(path)) with open(path, 'r') as f: return Config.load(f, args)
def emit_info(text): logger.info(text)
def emit_info(text): logger.info("Problem {}: {}".format(pid, text))
def main(prog, args): parser = argparse.ArgumentParser( prog=prog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("contest_id", help="Contest ID (e.g. arc001)") parser.add_argument("--without-login", action="store_true", help="Download data without login") parser.add_argument( "--workspace", help="Path to workspace's root directory. This script will create files" " in {{WORKSPACE}}/{{contest_name}}/{{alphabet}}/ e.g. ./your-workspace/arc001/A/\n" "[Default] {}".format(DEFAULT_WORKSPACE_DIR_PATH)) parser.add_argument( "--lang", help="Programming language of your template code, {}.\n".format( " or ".join([lang.name for lang in ALL_LANGUAGES])) + "[Default] {}".format(CPP.name)) parser.add_argument("--template", help="File path to your template code\n{}".format( "\n".join([ "[Default ({dname})] {path}".format( dname=lang.display_name, path=lang.default_template_path) for lang in ALL_LANGUAGES ]))) # Deleted functionality parser.add_argument('--replacement', help=argparse.SUPPRESS) parser.add_argument( "--parallel", action="store_true", help= "Prepare problem directories asynchronously using multi processors.", default=None) parser.add_argument("--save-no-session-cache", action="store_true", help="Save no session cache to avoid security risk", default=None) parser.add_argument( "--config", help="File path to your config file\n{0}{1}".format( "[Default (Primary)] {}\n".format(USER_CONFIG_PATH), "[Default (Secondary)] {}\n".format(get_default_config_path()))) args = parser.parse_args(args) if args.replacement is not None: logger.error( with_color( "Sorry! --replacement argument no longer exists" " and you can only use --template." " See the official document for details.", Fore.LIGHTRED_EX)) raise DeletedFunctionalityError config = get_config(args) try: import AccountInformation # noqa raise BannedFileDetectedError( "We abolished the logic with AccountInformation.py. Please delete the file." ) except ImportError: pass client = AtCoderClient() if not config.etc_config.download_without_login: try: client.login( save_session_cache=not config.etc_config.save_no_session_cache) logger.info("Login successful.") except LoginError: logger.error( "Failed to login (maybe due to wrong username/password combination?)" ) sys.exit(-1) else: logger.info("Downloading data without login.") prepare_contest(client, args.contest_id, config)
def main(prog, args) -> bool: parser = argparse.ArgumentParser( prog=prog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument( "--exec", '-e', help= "File path to the execution target. [Default] Automatically detected exec file", default=None) parser.add_argument( "--num", '-n', help= "The case number to test (1-origin). All cases are tested if not specified.", type=int, default=None) parser.add_argument( "--dir", '-d', help="Target directory to test. [Default] Current directory", default=".") parser.add_argument("--timeout", '-t', help="Timeout for each test cases (sec) [Default] 1", type=int, default=1) parser.add_argument( "--knock-out", '-k', help= "Stop execution immediately after any example's failure [Default] False", action="store_true", default=False) parser.add_argument( '--skip-almost-ac-feedback', '-s', help= 'Hide inputs and expected/actual outputs if result is correct and there are error outputs' ' [Default] False,', action='store_true', default=False) parser.add_argument('--judge-type', '-j', help='error type' ' must be one of [{}]'.format( ', '.join(USER_FACING_JUDGE_TYPE_LIST)), type=str, default=None) parser.add_argument('--error-value', '-v', help='error value for decimal number judge:' ' [Default] ' + str(DEFAULT_EPS), type=float, default=None) args = parser.parse_args(args) exec_file = args.exec or infer_exec_file( glob.glob(os.path.join(args.dir, '*'))) metadata_file = os.path.join(args.dir, "metadata.json") in_ex_pattern, out_ex_pattern, judge_method = get_sample_patterns_and_judge_method( metadata_file) in_sample_file_list = sorted( glob.glob(os.path.join(args.dir, in_ex_pattern))) out_sample_file_list = sorted( glob.glob(os.path.join(args.dir, out_ex_pattern))) user_input_decimal_error_type = None if args.judge_type is not None: if args.judge_type == "normal": judge_method = NormalJudge() elif args.judge_type in [ "absolute", "relative", "absolute_or_relative" ]: user_input_decimal_error_type = ErrorType(args.judge_type) else: logger.error( "Unknown judge type: {}. judge type must be one of [{}]". format(args.judge_type, ", ".join(USER_FACING_JUDGE_TYPE_LIST))) sys.exit(-1) user_input_error_value = args.error_value if isinstance(judge_method, DecimalJudge): judge_method = DecimalJudge(error_type=user_input_decimal_error_type or judge_method.error_type, diff=user_input_error_value or judge_method.diff) elif user_input_decimal_error_type is not None: judge_method = DecimalJudge(error_type=user_input_decimal_error_type, diff=user_input_error_value or DEFAULT_EPS) elif user_input_error_value is not None: assert judge_method.judge_type == JudgeType.Normal logger.warn( "error_value {} is ignored because this is normal judge".format( user_input_error_value)) if isinstance(judge_method, DecimalJudge): logger.info("Decimal number judge is enabled. type={}, diff={}".format( judge_method.error_type.value, judge_method.diff)) if args.num is None: return run_all_tests(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.knock_out, args.skip_almost_ac_feedback, judge_method) else: return run_single_test(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.num, judge_method)
def save_cookie(session: requests.Session, cookie_path: Optional[str] = None): cookie_path = cookie_path or default_cookie_path os.makedirs(os.path.dirname(cookie_path), exist_ok=True) session.cookies.save() logger.info("Saved session into {}".format(os.path.abspath(cookie_path))) os.chmod(cookie_path, 0o600)
def main(prog, args) -> bool: parser = argparse.ArgumentParser( prog=prog, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--exec", '-e', help="File path to the execution target. [Default] Automatically detected exec file", default=None) parser.add_argument("--num", '-n', help="The case number to test (1-origin). All cases are tested if not specified.", type=int, default=None) parser.add_argument("--dir", '-d', help="Target directory to test. [Default] Current directory", default=".") parser.add_argument("--timeout", '-t', help="Timeout for each test cases (sec) [Default] 1", type=int, default=1) parser.add_argument("--knock-out", '-k', help="Stop execution immediately after any example's failure [Default] False", action="store_true", default=False) parser.add_argument('--skip-almost-ac-feedback', '-s', help='Hide inputs and expected/actual outputs if result is correct and there are error outputs' ' [Default] False,', action='store_true', default=False) parser.add_argument('--judge-type', '-j', help='error type' ' must be one of [{}]'.format( ', '.join(USER_FACING_JUDGE_TYPE_LIST)), type=str, default=None) parser.add_argument('--error-value', '-v', help='error value for decimal number judge:' ' [Default] ' + str(DEFAULT_EPS), type=float, default=None) parser.add_argument('--compile-before-testing', '-c', help='compile source before testing [true, false]: ' ' [Default]: false', type=bool, default=None) parser.add_argument('--compile-only-when-diff-detected', help='compile only when diff detected [true, false]' ' [Default]: true', type=bool, default=None) parser.add_argument("--config", help="File path to your config file\n{0}{1}".format("[Default (Primary)] {}\n".format( USER_CONFIG_PATH), "[Default (Secondary)] {}\n".format( get_default_config_path())) ) args = parser.parse_args(args) config = get_config(args) if config.etc_config.compile_before_testing is not None and args.compile_before_testing is None: args.compile_before_testing = config.etc_config.compile_before_testing if args.compile_before_testing: if config.etc_config.compile_only_when_diff_detected is not None and args.compile_only_when_diff_detected is None: args.compile_only_when_diff_detected = config.etc_config.compile_only_when_diff_detected metadata_file = os.path.join(args.dir, "metadata.json") metadata = get_metadata(metadata_file) judge_method = metadata.judge_method lang = metadata.lang in_sample_file_list = sorted( glob.glob(os.path.join(args.dir, metadata.sample_in_pattern))) out_sample_file_list = sorted( glob.glob(os.path.join(args.dir, metadata.sample_out_pattern))) user_input_decimal_error_type = None if args.judge_type is not None: if args.judge_type == "normal": judge_method = NormalJudge() elif args.judge_type in ["absolute", "relative", "absolute_or_relative"]: user_input_decimal_error_type = ErrorType(args.judge_type) elif args.judge_type == "multisolution": judge_method = MultiSolutionJudge(lang.name) elif args.judge_type == "interactive": judge_method = InteractiveJudge(lang.name) else: logger.error("Unknown judge type: {}. judge type must be one of [{}]".format( args.judge_type, ", ".join(USER_FACING_JUDGE_TYPE_LIST))) sys.exit(-1) user_input_error_value = args.error_value if isinstance(judge_method, DecimalJudge): judge_method = DecimalJudge(error_type=user_input_decimal_error_type or judge_method.error_type, diff=user_input_error_value or judge_method.diff) elif user_input_decimal_error_type is not None: judge_method = DecimalJudge(error_type=user_input_decimal_error_type, diff=user_input_error_value or DEFAULT_EPS) elif user_input_error_value is not None: assert judge_method.judge_type == JudgeType.Normal logger.warn("error_value {} is ignored because this is normal judge".format( user_input_error_value)) if isinstance(judge_method, DecimalJudge): logger.info("Decimal number judge is enabled. type={}, diff={}".format( judge_method.error_type.value, judge_method.diff)) if metadata.code_filename is None or not args.compile_before_testing: print("compile is skipped and infer exec file") exclude_exec_files = [] if hasattr(judge_method, "judge_exec_filename"): judge_method.judge_exec_filename = os.path.join( args.dir, judge_method.judge_exec_filename) exclude_exec_files.append(judge_method.judge_exec_filename) exec_file = args.exec or infer_exec_file( glob.glob(os.path.join(args.dir, '*')), exclude_exec_files) else: if args.compile_only_when_diff_detected: force_compile = True else: force_compile = False exec_file = lang.get_test_command('main', args.dir) print("command: ", exec_file) print("directory: ", args.dir) # Compile if not compile_main_and_judge_programs(metadata, args.dir, force_compile=force_compile): exit() if args.num is None: return run_all_tests(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.knock_out, args.skip_almost_ac_feedback, judge_method, args.dir) else: return run_single_test(exec_file, in_sample_file_list, out_sample_file_list, args.timeout, args.num, judge_method, args.dir)