def test_contest_from_url(self): contest = dispatch.contest_from_url( 'https://codeforces.com/contest/1244') self.assertIsInstance(contest, service.codeforces.CodeforcesContest) self.assertIsInstance(contest.get_service(), service.codeforces.CodeforcesService) self.assertEqual(contest.get_url(), 'https://codeforces.com/contest/1244')
def generate_contest_directory(contest_id): contest = contest_from_url(ATCODER_URL + contest_id) contest_dir = Path(contest_id).resolve() contest_dir.mkdir() chdir(contest_dir) with with_cookiejar(requests.Session()) as sess: problems = contest.list_problems(session=sess) for i, problem in enumerate(problems): problem_url = problem.get_url() problem_dir = contest_dir.joinpath(chr(ord('a') + i)) problem_dir.mkdir() chdir(problem_dir) command = ['pipenv', 'run', 'oj', 'download', problem_url] subprocess.run(command).check_returncode() problem_dir.joinpath('main.py').touch() time.sleep(1) chdir(contest_dir)
def main(args: Optional[List[str]] = None, *, debug: bool = False) -> Dict[str, Any]: parser = get_parser() parsed = parser.parse_args(args=args) if parsed.verbose: basicConfig(level=DEBUG) else: basicConfig(level=INFO) # do sleep to prevent impolite scraping logger.info('sleep %f sec', parsed.wait) time.sleep(parsed.wait) # parse the URL problem = dispatch.problem_from_url(getattr(parsed, 'url', '')) contest = dispatch.contest_from_url(getattr(parsed, 'url', '')) service = dispatch.service_from_url(getattr(parsed, 'url', '')) # prepare a session session = requests.Session() session.headers['User-Agent'] = parsed.user_agent # set yukicoder's token if parsed.yukicoder_token is not None: parser.error("don't use --yukicoder-token. use $YUKICODER_TOKEN") else: parsed.yukicoder_token = os.environ.get('YUKICODER_TOKEN') is_yukicoder = isinstance(problem, YukicoderProblem) or isinstance( service, YukicoderService) if parsed.yukicoder_token and is_yukicoder: session.headers['Authorization'] = 'Bearer {}'.format( parsed.yukicoder_token) # set password to login from the environment variable if parsed.subcommand == 'login-service': if parsed.password is not None: parser.error("don't use --password. use $PASSWORD") else: parsed.password = os.environ.get('PASSWORD') try: with utils.with_cookiejar(session, path=parsed.cookie) as session: result = None # type: Optional[Dict[str, Any]] schema = {} # type: Dict[str, Any] if parsed.subcommand == 'get-problem': if problem is None: raise ValueError("unsupported URL: {}".format( repr(parsed.url))) result = get_problem.main( problem, is_system=parsed.system, is_full=parsed.full, is_compatibility=parsed.compatibility, session=session) if parsed.compatibility: schema = get_problem.schema_compatibility else: schema = get_problem.schema elif parsed.subcommand == 'get-contest': if contest is None: raise ValueError("unsupported URL: {}".format( repr(parsed.url))) result = get_contest.main(contest, is_full=parsed.full, session=session) schema = get_contest.schema elif parsed.subcommand == 'get-service': if service is None: raise ValueError("unsupported URL: {}".format( repr(parsed.url))) result = get_service.main( service, does_list_contests=parsed.list_contests, session=session) schema = get_service.schema elif parsed.subcommand == 'login-service': if service is None: raise ValueError("unsupported URL: {}".format( repr(parsed.url))) result = login_service.main(service, username=parsed.username, password=parsed.password, check_only=parsed.check, session=session) schema = login_service.schema elif parsed.subcommand == 'submit-code': if problem is None: raise ValueError("unsupported URL: {}".format( repr(parsed.url))) result = submit_code.main(problem, file=parsed.file, language_id=parsed.language, session=session) schema = submit_code.schema elif parsed.subcommand is None: parser.print_help() result = None else: assert False except: etype, evalue, _ = sys.exc_info() logger.exception('%s', evalue) wrapped = { "status": "error", "messages": [ *map(lambda line: line.strip(), traceback.format_exception_only(etype, evalue)) ], "result": None, } # type: Dict[str, Any] if debug: return wrapped else: print(json.dumps(wrapped)) raise SystemExit(1) else: if result is None: # no subcommand given if debug: return { "status": "ok", "messages": [], "result": None, } else: raise SystemExit(0) wrapped = { "status": "ok", "messages": [], "result": result, } if not debug: print(json.dumps(wrapped)) try: jsonschema.validate(result, schema) except jsonschema.exceptions.ValidationError as e: logger.debug('%s', e) if debug: return wrapped else: raise SystemExit(0)
def download(args: argparse.Namespace) -> None: # prepare values problem = dispatch.problem_from_url(args.url) if problem is None: if dispatch.contest_from_url(args.url) is not None: logger.warning('You specified a URL for a contest instead of a problem. If you want to download for all problems of a contest at once, please try to use `oj-prepare` command of https://github.com/online-judge-tools/template-generator') raise requests.exceptions.InvalidURL('The URL "%s" is not supported' % args.url) is_default_format = args.format is None and args.directory is None # must be here since args.directory and args.format are overwritten if args.directory is None: args.directory = pathlib.Path('test') if args.format is None: args.format = '%b.%e' # get samples from the server with utils.new_session_with_our_user_agent(path=args.cookie) as sess: if args.yukicoder_token and isinstance(problem, YukicoderProblem): sess.headers['Authorization'] = 'Bearer {}'.format(args.yukicoder_token) if args.system: samples = problem.download_system_cases(session=sess) else: samples = problem.download_sample_cases(session=sess) if not samples: raise SampleParseError("Sample not found") # append the history for submit subcommand if not args.dry_run and is_default_format: history = onlinejudge_command.download_history.DownloadHistory() if not list(args.directory.glob('*')): # reset the history to help users who use only one directory for many problems history.remove(directory=pathlib.Path.cwd()) history.add(problem, directory=pathlib.Path.cwd()) # prepare files to write def iterate_files_to_write(sample: TestCase, *, i: int) -> Iterator[Tuple[str, pathlib.Path, bytes]]: for ext in ['in', 'out']: data = getattr(sample, ext + 'put_data') if data is None: continue name = sample.name table = {} table['i'] = str(i + 1) table['e'] = ext table['n'] = name table['b'] = os.path.basename(name) table['d'] = os.path.dirname(name) path: pathlib.Path = args.directory / format_utils.percentformat(args.format, table) yield ext, path, data for i, sample in enumerate(samples): for _, path, _ in iterate_files_to_write(sample, i=i): if path.exists(): raise FileExistsError('Failed to download since file already exists: ' + str(path)) # write samples to files for i, sample in enumerate(samples): logger.info('') logger.info('sample %d', i) for ext, path, data in iterate_files_to_write(sample, i=i): content = '' if not args.silent: content = '\n' + pretty_printers.make_pretty_large_file_content(data, limit=40, head=20, tail=10, bold=True) logger.info('%sput: %s%s', ext, sample.name, content) if not args.dry_run: path.parent.mkdir(parents=True, exist_ok=True) with path.open('wb') as fh: fh.write(data) logger.info(utils.SUCCESS + 'saved to: %s', path) if args.log_file: with args.log_file.open(mode='w') as fhs: json.dump(list(map(convert_sample_to_dict, samples)), fhs)
def main(): cache_dir = './.wrong_answer' target_dir = '.' # Parse input arguments parser = argparse.ArgumentParser( description="""Wrong Answer!! - Download AtCoder system test data""", epilog="""This program downloads the data by using the result of its own crawler. Because of this, you might not be able to download the latest contest's test data. If you think the data already opened was not registered to this program's DB, please leave a issue at the github page. Thank you.""", usage='%(prog)s [contest] problem [test_case]') parser.add_argument( '-c', '--contest', help= 'specify contest name explicitly (You can omit this when you use triple)' ) parser.add_argument('-u', '--updatedb', action='store_true', help='update database') parser.add_argument('-l', '--list', action='store_true', help='print contest list') parser.add_argument('cases', nargs='*') args = parser.parse_args() contest = args.contest if args.updatedb: download(GITHUB_URL + '/folders.txt', BASE_URLS) log.info(f"{BASE_URLS} updated.") if len(args.cases) == 0: exit(0) if args.list: with open(BASE_URLS, "r") as f: for X in f: X = X.split()[0] print(X) exit(0) argc = len(args.cases) if argc == 0: if args.contest is not None: url = findContest(args.contest).split()[1] print( """Downloading whole test sets for a contest could be very huge and slow. If you really want to do that, use below URL with a browser. But this program does not support whole contest downloading on purpose. """) URL = re.sub(r'dl=0', 'dl=1', url) print(URL) exit(0) else: parser.print_help() exit(1) elif argc == 1: # Download whole test cases of the specified problem. problem = args.cases[0].upper() MODE = Mode.PROBLEM elif argc == 2: # Download the test case of the problem. problem = args.cases[0].upper() case = args.cases[1] MODE = Mode.TESTCASE elif argc == 3: # Download the test case of the problem, of the contest contest = args.cases[0].upper() problem = args.cases[1].upper() case = args.cases[2] MODE = Mode.TESTCASE else: parser.print_help() exit(1) # If contest is not set, use CWD as contest name # Old ABC has some test data in ARC. # You can find ARC contest name from the problem's page URL if contest is None: contest = os.path.basename(os.getcwd()).upper() log.warning( f"Contest name not specified. Use current dir '{contest}' as contest name." ) if (contest[0:3] == "ABC"): # This is my rule. Each problem's web page urls are in '.problems' if Path('.problems').exists(): with open('.problems') as f: url = f.readlines()[ord(problem) - ord('A')].rsplit()[0] comtest = url.split('/')[-1].split('_')[0].upper() else: url = f"https://atcoder.jp/contests/{contest}" result = get_contest.main(dispatch.contest_from_url(url), is_full=False, session=session) ps = result['problems'] for i in ps: if i['context']['alphabet'] == problem: url = i['url'] break else: log.error( f"Specified problem '{problem}' not found in the contest page" ) exit(1) comtest = url.split('/')[-1].split('_')[0].upper() if (contest != comtest): log.warning(f"Look like the data are in '{comtest}'.") log.warning( f"Download data from '{comtest}' instead of '{contest}'") log.warning( f"If you see errors, please use --contest option next time." ) contest = comtest # This URL is for whole test cases of a contest. Ignore. [contest, URL] = findContest(contest).split() if MODE == Mode.TESTCASE: testcases_path = f"{cache_dir}/{problem}.json" URL = GITHUB_URL + f"/contests/{contest}/{problem}.json" if not os.path.exists(testcases_path): os.makedirs(cache_dir, exist_ok=True) download(URL, testcases_path) with open(testcases_path, "r") as f: J = json.load(f) res = '.*' + case + '.*' for i in J: if re.match(res, i, re.I): URL1 = J[i]['in'] URL2 = J[i]['out'] break else: log.error(f"Test case '{case}' not found in {testcases_path}") with open(testcases_path, "r") as f: print() log.info(f"Problem {problem} has these test cases.") for i in J: print(f"{i}", end=" ") print() exit(1) target_dir += '/' + problem downloadSingleCase(URL1, target_dir) downloadSingleCase(URL2, target_dir, out=True) log.info(utils.SUCCESS + "Succeeded") exit(0) elif MODE == Mode.PROBLEM: cases = f"{cache_dir}/testcases.txt" URL = GITHUB_URL + f"/contests/{contest}/testcases.txt" if not os.path.exists(cases): os.makedirs(cache_dir, exist_ok=True) download(URL, cases) purls = {} found = False with open(cases, "r") as f: lines = f.readlines() for i in lines: [p, url] = i.split() if p == problem: break else: log.error( f"Specified problem '{problem}' not found in {cases}.") exit(1) url = re.sub(r'dl=0', 'dl=1', url) log.info(f"Downloading: {url}") log.warning( "This could be long! They make archive file each time by request.") log.info("Waiting a responce...") r = session.get(url, headers=agent, stream=True) if r.status_code != requests.codes.ok: log.error(f"Failed. {r.status_code}") exit(1) log.info("Got a responce.") siz = sizeof_fmt(float(r.headers['Content-Length'])) log.info(f"Downloading... Size: {siz}") b = r.raw.read(40960) bs = bytearray() while b: bs += b b = r.raw.read(40960) sys.stdout.write('\r') sys.stdout.write(str(len(bs))) sys.stdout.flush() sys.stdout.write("\n") sys.stdout.flush() zf = zipfile.ZipFile(io.BytesIO(bs)) target_dir += f"/{problem}" Path(target_dir).mkdir(parents=True, exist_ok=True) for i in zf.infolist(): if i.is_dir(): continue path = PurePath(i.filename) fn = path.stem if path.match("*.nkftmpjKHWPL"): continue if path.match("*.swp"): continue if path.match("out/*"): nfn = f"{target_dir}/{fn}.out" else: nfn = f"{target_dir}/{fn}.in" log.info(f"Create: {nfn}") with zf.open(i.filename) as fr: with open(nfn, "w") as fw: for line in fr: fw.write(line.decode('utf-8').replace('\r', '')) log.info(utils.green('AC')) exit(0)
def test_contest_from_url(self): self.assertIsNone(dispatch.contest_from_url('https://atcoder.jp/')) self.assertIsNone(dispatch.contest_from_url('https://'))
def test_contest_from_url(self): contest = dispatch.contest_from_url( 'https://atcoder.jp/contests/agc030') self.assertIsInstance(contest, service.atcoder.AtCoderContest) self.assertIsInstance(contest.get_service(), service.atcoder.AtCoderService)