def test_contest_from_url(self):
contest = dispatch.contest_from_url(
'https://codeforces.com/contest/1244')
self.assertIsInstance(contest, service.codeforces.CodeforcesContest)
self.assertIsInstance(contest.get_service(),
service.codeforces.CodeforcesService)
self.assertEqual(contest.get_url(),
'https://codeforces.com/contest/1244')
def generate_contest_directory(contest_id):
contest = contest_from_url(ATCODER_URL + contest_id)
contest_dir = Path(contest_id).resolve()
contest_dir.mkdir()
chdir(contest_dir)
with with_cookiejar(requests.Session()) as sess:
problems = contest.list_problems(session=sess)
for i, problem in enumerate(problems):
problem_url = problem.get_url()
problem_dir = contest_dir.joinpath(chr(ord('a') + i))
problem_dir.mkdir()
chdir(problem_dir)
command = ['pipenv', 'run', 'oj', 'download', problem_url]
subprocess.run(command).check_returncode()
problem_dir.joinpath('main.py').touch()
time.sleep(1)
chdir(contest_dir)
def main(args: Optional[List[str]] = None,
*,
debug: bool = False) -> Dict[str, Any]:
parser = get_parser()
parsed = parser.parse_args(args=args)
if parsed.verbose:
basicConfig(level=DEBUG)
else:
basicConfig(level=INFO)
# do sleep to prevent impolite scraping
logger.info('sleep %f sec', parsed.wait)
time.sleep(parsed.wait)
# parse the URL
problem = dispatch.problem_from_url(getattr(parsed, 'url', ''))
contest = dispatch.contest_from_url(getattr(parsed, 'url', ''))
service = dispatch.service_from_url(getattr(parsed, 'url', ''))
# prepare a session
session = requests.Session()
session.headers['User-Agent'] = parsed.user_agent
# set yukicoder's token
if parsed.yukicoder_token is not None:
parser.error("don't use --yukicoder-token. use $YUKICODER_TOKEN")
else:
parsed.yukicoder_token = os.environ.get('YUKICODER_TOKEN')
is_yukicoder = isinstance(problem, YukicoderProblem) or isinstance(
service, YukicoderService)
if parsed.yukicoder_token and is_yukicoder:
session.headers['Authorization'] = 'Bearer {}'.format(
parsed.yukicoder_token)
# set password to login from the environment variable
if parsed.subcommand == 'login-service':
if parsed.password is not None:
parser.error("don't use --password. use $PASSWORD")
else:
parsed.password = os.environ.get('PASSWORD')
try:
with utils.with_cookiejar(session, path=parsed.cookie) as session:
result = None # type: Optional[Dict[str, Any]]
schema = {} # type: Dict[str, Any]
if parsed.subcommand == 'get-problem':
if problem is None:
raise ValueError("unsupported URL: {}".format(
repr(parsed.url)))
result = get_problem.main(
problem,
is_system=parsed.system,
is_full=parsed.full,
is_compatibility=parsed.compatibility,
session=session)
if parsed.compatibility:
schema = get_problem.schema_compatibility
else:
schema = get_problem.schema
elif parsed.subcommand == 'get-contest':
if contest is None:
raise ValueError("unsupported URL: {}".format(
repr(parsed.url)))
result = get_contest.main(contest,
is_full=parsed.full,
session=session)
schema = get_contest.schema
elif parsed.subcommand == 'get-service':
if service is None:
raise ValueError("unsupported URL: {}".format(
repr(parsed.url)))
result = get_service.main(
service,
does_list_contests=parsed.list_contests,
session=session)
schema = get_service.schema
elif parsed.subcommand == 'login-service':
if service is None:
raise ValueError("unsupported URL: {}".format(
repr(parsed.url)))
result = login_service.main(service,
username=parsed.username,
password=parsed.password,
check_only=parsed.check,
session=session)
schema = login_service.schema
elif parsed.subcommand == 'submit-code':
if problem is None:
raise ValueError("unsupported URL: {}".format(
repr(parsed.url)))
result = submit_code.main(problem,
file=parsed.file,
language_id=parsed.language,
session=session)
schema = submit_code.schema
elif parsed.subcommand is None:
parser.print_help()
result = None
else:
assert False
except:
etype, evalue, _ = sys.exc_info()
logger.exception('%s', evalue)
wrapped = {
"status":
"error",
"messages": [
*map(lambda line: line.strip(),
traceback.format_exception_only(etype, evalue))
],
"result":
None,
} # type: Dict[str, Any]
if debug:
return wrapped
else:
print(json.dumps(wrapped))
raise SystemExit(1)
else:
if result is None:
# no subcommand given
if debug:
return {
"status": "ok",
"messages": [],
"result": None,
}
else:
raise SystemExit(0)
wrapped = {
"status": "ok",
"messages": [],
"result": result,
}
if not debug:
print(json.dumps(wrapped))
try:
jsonschema.validate(result, schema)
except jsonschema.exceptions.ValidationError as e:
logger.debug('%s', e)
if debug:
return wrapped
else:
raise SystemExit(0)
def download(args: argparse.Namespace) -> None:
# prepare values
problem = dispatch.problem_from_url(args.url)
if problem is None:
if dispatch.contest_from_url(args.url) is not None:
logger.warning('You specified a URL for a contest instead of a problem. If you want to download for all problems of a contest at once, please try to use `oj-prepare` command of https://github.com/online-judge-tools/template-generator')
raise requests.exceptions.InvalidURL('The URL "%s" is not supported' % args.url)
is_default_format = args.format is None and args.directory is None # must be here since args.directory and args.format are overwritten
if args.directory is None:
args.directory = pathlib.Path('test')
if args.format is None:
args.format = '%b.%e'
# get samples from the server
with utils.new_session_with_our_user_agent(path=args.cookie) as sess:
if args.yukicoder_token and isinstance(problem, YukicoderProblem):
sess.headers['Authorization'] = 'Bearer {}'.format(args.yukicoder_token)
if args.system:
samples = problem.download_system_cases(session=sess)
else:
samples = problem.download_sample_cases(session=sess)
if not samples:
raise SampleParseError("Sample not found")
# append the history for submit subcommand
if not args.dry_run and is_default_format:
history = onlinejudge_command.download_history.DownloadHistory()
if not list(args.directory.glob('*')):
# reset the history to help users who use only one directory for many problems
history.remove(directory=pathlib.Path.cwd())
history.add(problem, directory=pathlib.Path.cwd())
# prepare files to write
def iterate_files_to_write(sample: TestCase, *, i: int) -> Iterator[Tuple[str, pathlib.Path, bytes]]:
for ext in ['in', 'out']:
data = getattr(sample, ext + 'put_data')
if data is None:
continue
name = sample.name
table = {}
table['i'] = str(i + 1)
table['e'] = ext
table['n'] = name
table['b'] = os.path.basename(name)
table['d'] = os.path.dirname(name)
path: pathlib.Path = args.directory / format_utils.percentformat(args.format, table)
yield ext, path, data
for i, sample in enumerate(samples):
for _, path, _ in iterate_files_to_write(sample, i=i):
if path.exists():
raise FileExistsError('Failed to download since file already exists: ' + str(path))
# write samples to files
for i, sample in enumerate(samples):
logger.info('')
logger.info('sample %d', i)
for ext, path, data in iterate_files_to_write(sample, i=i):
content = ''
if not args.silent:
content = '\n' + pretty_printers.make_pretty_large_file_content(data, limit=40, head=20, tail=10, bold=True)
logger.info('%sput: %s%s', ext, sample.name, content)
if not args.dry_run:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open('wb') as fh:
fh.write(data)
logger.info(utils.SUCCESS + 'saved to: %s', path)
if args.log_file:
with args.log_file.open(mode='w') as fhs:
json.dump(list(map(convert_sample_to_dict, samples)), fhs)
def main():
cache_dir = './.wrong_answer'
target_dir = '.'
# Parse input arguments
parser = argparse.ArgumentParser(
description="""Wrong Answer!! - Download AtCoder system test data""",
epilog="""This program downloads the data by using the result of
its own crawler. Because of this, you might not be able to download the
latest contest's test data. If you think the data already opened was not
registered to this program's DB, please leave a issue at the github page.
Thank you.""",
usage='%(prog)s [contest] problem [test_case]')
parser.add_argument(
'-c',
'--contest',
help=
'specify contest name explicitly (You can omit this when you use triple)'
)
parser.add_argument('-u',
'--updatedb',
action='store_true',
help='update database')
parser.add_argument('-l',
'--list',
action='store_true',
help='print contest list')
parser.add_argument('cases', nargs='*')
args = parser.parse_args()
contest = args.contest
if args.updatedb:
download(GITHUB_URL + '/folders.txt', BASE_URLS)
log.info(f"{BASE_URLS} updated.")
if len(args.cases) == 0:
exit(0)
if args.list:
with open(BASE_URLS, "r") as f:
for X in f:
X = X.split()[0]
print(X)
exit(0)
argc = len(args.cases)
if argc == 0:
if args.contest is not None:
url = findContest(args.contest).split()[1]
print(
"""Downloading whole test sets for a contest could be very huge and slow.
If you really want to do that, use below URL with a browser.
But this program does not support whole contest downloading on purpose.
""")
URL = re.sub(r'dl=0', 'dl=1', url)
print(URL)
exit(0)
else:
parser.print_help()
exit(1)
elif argc == 1:
# Download whole test cases of the specified problem.
problem = args.cases[0].upper()
MODE = Mode.PROBLEM
elif argc == 2:
# Download the test case of the problem.
problem = args.cases[0].upper()
case = args.cases[1]
MODE = Mode.TESTCASE
elif argc == 3:
# Download the test case of the problem, of the contest
contest = args.cases[0].upper()
problem = args.cases[1].upper()
case = args.cases[2]
MODE = Mode.TESTCASE
else:
parser.print_help()
exit(1)
# If contest is not set, use CWD as contest name
# Old ABC has some test data in ARC.
# You can find ARC contest name from the problem's page URL
if contest is None:
contest = os.path.basename(os.getcwd()).upper()
log.warning(
f"Contest name not specified. Use current dir '{contest}' as contest name."
)
if (contest[0:3] == "ABC"):
# This is my rule. Each problem's web page urls are in '.problems'
if Path('.problems').exists():
with open('.problems') as f:
url = f.readlines()[ord(problem) - ord('A')].rsplit()[0]
comtest = url.split('/')[-1].split('_')[0].upper()
else:
url = f"https://atcoder.jp/contests/{contest}"
result = get_contest.main(dispatch.contest_from_url(url),
is_full=False,
session=session)
ps = result['problems']
for i in ps:
if i['context']['alphabet'] == problem:
url = i['url']
break
else:
log.error(
f"Specified problem '{problem}' not found in the contest page"
)
exit(1)
comtest = url.split('/')[-1].split('_')[0].upper()
if (contest != comtest):
log.warning(f"Look like the data are in '{comtest}'.")
log.warning(
f"Download data from '{comtest}' instead of '{contest}'")
log.warning(
f"If you see errors, please use --contest option next time."
)
contest = comtest
# This URL is for whole test cases of a contest. Ignore.
[contest, URL] = findContest(contest).split()
if MODE == Mode.TESTCASE:
testcases_path = f"{cache_dir}/{problem}.json"
URL = GITHUB_URL + f"/contests/{contest}/{problem}.json"
if not os.path.exists(testcases_path):
os.makedirs(cache_dir, exist_ok=True)
download(URL, testcases_path)
with open(testcases_path, "r") as f:
J = json.load(f)
res = '.*' + case + '.*'
for i in J:
if re.match(res, i, re.I):
URL1 = J[i]['in']
URL2 = J[i]['out']
break
else:
log.error(f"Test case '{case}' not found in {testcases_path}")
with open(testcases_path, "r") as f:
print()
log.info(f"Problem {problem} has these test cases.")
for i in J:
print(f"{i}", end=" ")
print()
exit(1)
target_dir += '/' + problem
downloadSingleCase(URL1, target_dir)
downloadSingleCase(URL2, target_dir, out=True)
log.info(utils.SUCCESS + "Succeeded")
exit(0)
elif MODE == Mode.PROBLEM:
cases = f"{cache_dir}/testcases.txt"
URL = GITHUB_URL + f"/contests/{contest}/testcases.txt"
if not os.path.exists(cases):
os.makedirs(cache_dir, exist_ok=True)
download(URL, cases)
purls = {}
found = False
with open(cases, "r") as f:
lines = f.readlines()
for i in lines:
[p, url] = i.split()
if p == problem: break
else:
log.error(
f"Specified problem '{problem}' not found in {cases}.")
exit(1)
url = re.sub(r'dl=0', 'dl=1', url)
log.info(f"Downloading: {url}")
log.warning(
"This could be long! They make archive file each time by request.")
log.info("Waiting a responce...")
r = session.get(url, headers=agent, stream=True)
if r.status_code != requests.codes.ok:
log.error(f"Failed. {r.status_code}")
exit(1)
log.info("Got a responce.")
siz = sizeof_fmt(float(r.headers['Content-Length']))
log.info(f"Downloading... Size: {siz}")
b = r.raw.read(40960)
bs = bytearray()
while b:
bs += b
b = r.raw.read(40960)
sys.stdout.write('\r')
sys.stdout.write(str(len(bs)))
sys.stdout.flush()
sys.stdout.write("\n")
sys.stdout.flush()
zf = zipfile.ZipFile(io.BytesIO(bs))
target_dir += f"/{problem}"
Path(target_dir).mkdir(parents=True, exist_ok=True)
for i in zf.infolist():
if i.is_dir(): continue
path = PurePath(i.filename)
fn = path.stem
if path.match("*.nkftmpjKHWPL"): continue
if path.match("*.swp"): continue
if path.match("out/*"):
nfn = f"{target_dir}/{fn}.out"
else:
nfn = f"{target_dir}/{fn}.in"
log.info(f"Create: {nfn}")
with zf.open(i.filename) as fr:
with open(nfn, "w") as fw:
for line in fr:
fw.write(line.decode('utf-8').replace('\r', ''))
log.info(utils.green('AC'))
exit(0)
def test_contest_from_url(self):
self.assertIsNone(dispatch.contest_from_url('https://atcoder.jp/'))
self.assertIsNone(dispatch.contest_from_url('https://'))
def test_contest_from_url(self):
contest = dispatch.contest_from_url(
'https://atcoder.jp/contests/agc030')
self.assertIsInstance(contest, service.atcoder.AtCoderContest)
self.assertIsInstance(contest.get_service(),
service.atcoder.AtCoderService)