def simple_match(a: str, b: str) -> bool:
if a == b:
return True
if a.rstrip() == b.rstrip():
log.warning('WA if no rstrip')
return True
return False
def download_sample_cases(
self,
session: Optional[requests.Session] = None
) -> List[onlinejudge.type.TestCase]:
session = session or utils.new_default_session()
# get
url = self.get_url(contests=False) + '/file/statement/samples.zip'
resp = utils.request('GET',
url,
session=session,
raise_for_status=False)
if resp.status_code == 404:
log.warning('samples.zip not found')
log.info(
'this 404 happens in both cases: 1. no sample cases as intended; 2. just an error'
)
return []
resp.raise_for_status()
# parse
with zipfile.ZipFile(io.BytesIO(resp.content)) as fh:
samples = [] # type: List[TestCase]
for filename in sorted(fh.namelist()):
log.debug('filename: %s', filename)
if filename.endswith('.in'):
inpath = filename
outpath = filename[:-3] + '.ans'
indata = fh.read(inpath).decode()
outdata = fh.read(outpath).decode()
samples += [
TestCase(LabeledString(inpath, indata),
LabeledString(outpath, outdata))
]
return samples
def generate_scanner(args: 'argparse.Namespace') -> None:
if not args.silent:
log.warning('This feature is ' + log.red('experimental') + '.')
if args.silent:
for handler in log.logger.handlers:
log.removeHandler(handler)
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
with utils.with_cookiejar(utils.new_default_session(),
path=args.cookie) as sess:
it = problem.get_input_format(session=sess) # type: Any
if not it:
log.error('input format not found')
sys.exit(1)
try:
log.debug('original data: %s', repr(it))
it = list(tokenize(it))
log.debug('tokenized: %s', str(it))
it = list(parse(it))
log.debug('parsed: %s', str(it))
it = postprocess(it)
log.debug('postprocessed: %s', str(it))
it = export(it, use_scanf=args.scanf, repeat_macro=args.repeat_macro)
log.debug('result: %s', repr(it))
except:
log.error('something wrong')
raise
log.success('success:')
print(log.bold(it.rstrip())) # to stdout
def get_input_format(self,
session: Optional[requests.Session] = None) -> str:
"""
:raises Exception: if no such problem exists
"""
session = session or utils.new_default_session()
# get
resp = _request('GET',
self.get_url(type='beta'),
raise_for_status=False,
session=session)
if _list_alert(resp):
log.warning('are you logged in?')
resp.raise_for_status()
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
for h3 in soup.find_all('h3'):
if h3.string in ('入力', 'Input'):
tag = h3
for _ in range(3):
tag = utils.next_sibling_tag(tag)
if tag is None:
break
if tag.name in ('pre', 'blockquote'):
s = ''
for it in tag:
s += it.string or it # AtCoder uses <var>...</var> for math symbols
return s
return ''
def match(a, b):
if a == b:
return True
if rstrip and a.rstrip(rstrip_targets) == b.rstrip(rstrip_targets):
log.warning('WA if no rstrip')
return True
return False
def download_sample_cases(
self,
session: Optional[requests.Session] = None
) -> List[onlinejudge.type.TestCase]:
session = session or utils.new_default_session()
# get
resp = _request('GET', self.get_url(), session=session)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
if AtCoderService._report_messages(msgs, unexpected=True):
# example message: "message: You cannot see this page."
log.warning('are you logged in?')
return []
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
samples = onlinejudge._implementation.testcase_zipper.SampleZipper()
lang = None
for pre, h3 in self._find_sample_tags(soup):
s = utils.textfile(utils.dos2unix(pre.string.lstrip()))
name = h3.string
l = self._get_tag_lang(pre)
if lang is None:
lang = l
elif lang != l:
log.info(
'skipped due to language: current one is %s, not %s: %s ',
lang, l, name)
continue
samples.add(s.encode(), name)
return samples.get()
def download_sample_cases(
self,
session: Optional[requests.Session] = None) -> List[TestCase]:
log.warning(
"most of problems in arena have no registered sample cases.")
return AOJProblem(
self.get_problem_id()).download_sample_cases(session=session)
def drop_backup_or_hidden_files(paths: List[pathlib.Path]) -> List[pathlib.Path]:
result = [] # type: List[pathlib.Path]
for path in paths:
if is_backup_or_hidden_file(path):
log.warning('ignore a backup file: %s', path)
else:
result += [path]
return result
def version_check() -> None:
if utils.is_update_available_on_pypi():
log.warning('update available: %s -> %s', version.__version__,
utils.get_latest_version_from_pypi())
log.info('run: $ pip3 install -U %s', version.__package_name__)
log.info(
'see: https://github.com/kmyk/online-judge-tools/blob/master/CHANGELOG.md'
)
def download_sample_cases(
self,
session: Optional[requests.Session] = None) -> List[TestCase]:
"""
:raises NotImplementedError:
"""
log.warning('use --system option')
raise NotImplementedError
def f(x):
try:
y = float(x)
if not math.isfinite(y):
log.warning('not an real number found: %f', y)
return y
except ValueError:
return x
def is_logged_in_with_message(service: onlinejudge.type.Service, *,
session: requests.Session) -> bool:
if service.is_logged_in(session=session):
log.info('You have already signed in.')
return True
else:
log.warning('You are not signed in.')
return False
def download(args: 'argparse.Namespace') -> None:
# prepare values
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
is_default_format = args.format is None and args.directory is None # must be here since args.directory and args.format are overwritten
if args.directory is None:
args.directory = pathlib.Path('test')
if args.format is None:
args.format = '%b.%e'
# get samples from the server
with utils.with_cookiejar(utils.new_session_with_our_user_agent(), path=args.cookie) as sess:
if args.system:
samples = problem.download_system_cases(session=sess) # type: ignore
else:
samples = problem.download_sample_cases(session=sess) # type: ignore
# append the history for submit command
if not args.dry_run and is_default_format:
history = onlinejudge._implementation.download_history.DownloadHistory()
history.add(problem)
# write samples to files
for i, sample in enumerate(samples):
log.emit('')
log.info('sample %d', i)
for ext in ['in', 'out']:
data = getattr(sample, ext + 'put_data')
if data is None:
continue
name = sample.name
table = {}
table['i'] = str(i + 1)
table['e'] = ext
table['n'] = name
table['b'] = os.path.basename(name)
table['d'] = os.path.dirname(name)
path = args.directory / format_utils.percentformat(args.format, table) # type: pathlib.Path
log.status('%sput: %s', ext, name)
if not args.silent:
log.emit(utils.snip_large_file_content(data, limit=40, head=20, tail=10, bold=True))
if args.dry_run:
continue
if path.exists():
log.warning('file already exists: %s', path)
if not args.overwrite:
log.warning('skipped')
continue
path.parent.mkdir(parents=True, exist_ok=True)
with path.open('wb') as fh:
fh.write(data)
log.success('saved to: %s', path)
# print json
if args.json:
print(json.dumps(list(map(convert_sample_to_dict, samples))))
def download_sample_cases(
self,
*,
session: Optional[requests.Session] = None) -> List[TestCase]:
session = session or utils.get_default_session()
if self.domain == 'codingcompetitions.withgoogle.com':
url = 'https://codejam.googleapis.com/dashboard/{}/poll?p=e30'.format(
self.contest_id)
resp = utils.request('GET', url, session=session)
data = json.loads(
base64.urlsafe_b64decode(resp.content + b'=' *
((-len(resp.content)) % 4)).decode())
log.debug('%s', data)
# parse JSON
for task in data['challenge']['tasks']:
if task['id'] == self.problem_id:
statement = task['statement']
break
else:
raise SampleParseError(
"the problem {} is not found in the challenge {}".format(
repr(self.problem_id), repr(self.contest_id)))
elif self.domain == 'code.google.com':
url = 'https://{}/{}/contest/{}/dashboard/ContestInfo'.format(
self.domain, self.kind, self.contest_id)
resp = utils.request('GET', url, session=session)
data = json.loads(resp.content.decode())
# parse JSON
assert self.problem_id.startswith('p')
i = int(self.problem_id[1:])
statement = data['problems'][i]['body']
else:
assert False
# parse HTML
soup = bs4.BeautifulSoup(statement, utils.html_parser)
io_contents = soup.find_all('pre', class_='io-content')
if len(io_contents) != 2:
raise SampleParseError(
"""the number of <pre class="io-content"> is not two""")
if io_contents[0].text.startswith('Case #'):
log.warning('''the sample input starts with "Case #"''')
if not io_contents[1].text.startswith('Case #'):
log.warning('''the sample output doesn't start with "Case #"''')
sample = TestCase(
'sample',
'Input',
utils.textfile(io_contents[0].text.rstrip()).encode(),
'Output',
utils.textfile(io_contents[1].text.rstrip()).encode(),
)
return [sample]
def _list_alert(resp: requests.Response, soup: Optional[bs4.BeautifulSoup] = None, print_: bool = False) -> List[str]:
if soup is None:
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
msgs = [] # type: List[str]
for alert in soup.find_all('div', attrs={'role': 'alert'}):
msg = ' '.join([s.strip() for s in alert.strings if s.strip()])
if print_:
log.warning('AtCoder says: %s', msg)
msgs += [msg]
return msgs
def download_sample_cases(self, session: Optional[requests.Session] = None) -> List[onlinejudge.type.TestCase]:
session = session or utils.get_default_session()
# get
url = self.get_url(contests=False) + '/file/statement/samples.zip'
resp = utils.request('GET', url, session=session, raise_for_status=False)
if resp.status_code == 404:
log.warning('samples.zip not found')
log.info('this 404 happens in both cases: 1. no sample cases as intended; 2. just an error')
return []
resp.raise_for_status()
# parse
return onlinejudge._implementation.testcase_zipper.extract_from_zip(resp.content, '%s.%e', out='ans')
def download_data(self, *, session: Optional[requests.Session] = None) -> AtCoderProblemDetailedData:
"""
:raises Exception: if no such problem exists
"""
session = session or utils.get_default_session()
resp = _request('GET', self.get_url(type='beta'), raise_for_status=False, session=session)
timestamp = datetime.datetime.now(datetime.timezone.utc).astimezone()
if _list_alert(resp):
log.warning('are you logged in?')
resp.raise_for_status()
html = resp.content.decode(resp.encoding).encode() # ensure UTF-8
return AtCoderProblemDetailedData.from_html(html, problem=self, session=session, response=resp, timestamp=timestamp)
def login(args: 'argparse.Namespace') -> None:
service = onlinejudge.dispatch.service_from_url(args.url)
if service is None:
sys.exit(1)
with utils.with_cookiejar(utils.new_session_with_our_user_agent(),
path=args.cookie) as session:
if is_logged_in_with_message(service, session=session):
return
else:
if args.check:
sys.exit(1)
if args.use_browser in ('always', 'auto'):
try:
login_with_browser(service, session=session)
except ImportError:
log.error(
'Selenium is not installed: try $ pip3 install selenium')
pass
except WebDriverException as e:
log.error('%s', e)
pass
else:
if is_logged_in_with_message(service, session=session):
return
else:
sys.exit(1)
if args.use_browser in ('never', 'auto'):
if args.use_browser == 'auto':
log.warning('use CUI login since Selenium fails')
try:
login_with_password(service,
username=args.username,
password=args.password,
session=session)
except NotImplementedError as e:
log.error('%s', e)
pass
except onlinejudge.type.LoginError:
sys.exit(1)
else:
if is_logged_in_with_message(service, session=session):
return
else:
sys.exit(1)
sys.exit(1)
def download_sample_cases(
self,
*,
session: Optional[requests.Session] = None) -> List[TestCase]:
session = session or utils.get_default_session()
# get samples via the official API
# reference: http://developers.u-aizu.ac.jp/api?key=judgedat%2Ftestcases%2Fsamples%2F%7BproblemId%7D_GET
url = 'https://judgedat.u-aizu.ac.jp/testcases/samples/{}'.format(
self.problem_id)
resp = utils.request('GET', url, session=session)
samples = [] # type: List[TestCase]
for i, sample in enumerate(
json.loads(resp.content.decode(resp.encoding))):
samples += [
TestCase(
'sample-{}'.format(i + 1),
str(sample['serial']),
sample['in'].encode(),
str(sample['serial']),
sample['out'].encode(),
)
]
# parse HTML if no samples are registered
# see: https://github.com/kmyk/online-judge-tools/issues/207
if not samples:
log.warning("sample cases are not registered in the official API")
log.status("fallback: parsing HTML")
# reference: http://developers.u-aizu.ac.jp/api?key=judgeapi%2Fresources%2Fdescriptions%2F%7Blang%7D%2F%7Bproblem_id%7D_GET
url = 'https://judgeapi.u-aizu.ac.jp/resources/descriptions/ja/{}'.format(
self.problem_id)
resp = utils.request('GET', url, session=session)
html = json.loads(resp.content.decode(resp.encoding))['html']
# list h3+pre
zipper = onlinejudge._implementation.testcase_zipper.SampleZipper()
expected_strings = ('入力例', '出力例', 'Sample Input', 'Sample Output')
soup = bs4.BeautifulSoup(html, utils.html_parser)
for pre in soup.find_all('pre'):
tag = pre.find_previous_sibling()
if tag and tag.name == 'h3' and tag.string and any(
s in tag.string for s in expected_strings):
s = utils.textfile(utils.parse_content(pre).lstrip())
zipper.add(s.encode(), tag.string)
samples = zipper.get()
return samples
def get_available_languages(
self,
session: Optional[requests.Session] = None) -> List[Language]:
session = session or utils.new_default_session()
info = self._get_model(session=session)
lang_display_mapping = self._get_lang_display_mapping()
result = [] # type: List[Language]
for lang in info['languages']:
descr = lang_display_mapping.get(lang)
if descr is None:
log.warning('display mapping for language `%s\' not found',
lang)
descr = lang
result += [Language(lang, descr)]
return result
def _parse_input_format(cls, soup: bs4.BeautifulSoup) -> Optional[str]:
for h3 in soup.find_all('h3', text=re.compile(r'^(入力|Input)$')):
if h3.parent.name == 'section':
section = h3.parent
else:
section = h3.find_next_sibling('section')
if section is None:
section = soup.find(class_='io-style')
if section is None:
log.warning('<section> tag not found. something wrong')
return None
pre = section.find('pre')
if pre is not None:
return pre.decode_contents(formatter=None)
return None
def add(self, s: str, name: str = '') -> None:
if self.dangling is None:
if re.search('output', name, re.IGNORECASE) or re.search(
'出力', name):
log.warning('strange name for input string: %s', name)
self.dangling = LabeledString(name, s)
else:
if re.search('input', name, re.IGNORECASE) or re.search(
'入力', name):
if not (re.search('output', name, re.IGNORECASE)
or re.search('出力', name)
): # to ignore titles like "Output for Sample Input 1"
log.warning('strange name for output string: %s', name)
self.data += [TestCase(self.dangling, LabeledString(name, s))]
self.dangling = None
def test_single_case(test_name: str, test_input_path: pathlib.Path, test_output_path: Optional[pathlib.Path], *, lock: Optional[threading.Lock] = None, args: 'argparse.Namespace') -> Dict[str, Any]:
# print the header earlier if not in parallel
if lock is None:
log.emit('')
log.info('%s', test_name)
# run the binary
with test_input_path.open() as inf:
info, proc = utils.exec_command(args.command, stdin=inf, timeout=args.tle, gnu_time=args.gnu_time)
# TODO: the `answer` should be bytes, not str
answer = (info['answer'] or b'').decode(errors='replace') # type: str
elapsed = info['elapsed'] # type: float
memory = info['memory'] # type: Optional[float]
# lock is require to avoid mixing logs if in parallel
nullcontext = contextlib.ExitStack() # TODO: use contextlib.nullcontext() after updating Python to 3.7
with lock or nullcontext:
if lock is not None:
log.emit('')
log.info('%s', test_name)
log.status('time: %f sec', elapsed)
if memory:
if memory < MEMORY_PRINT:
if args.print_memory:
log.status('memory: %f MB', memory)
elif memory < MEMORY_WARNING:
log.status('memory: %f MB', memory)
else:
log.warning('memory: %f MB', memory)
status = compare_and_report(proc, answer, elapsed, memory, test_input_path, test_output_path, mle=args.mle, mode=args.mode, error=args.error, does_print_input=args.print_input, silent=args.silent, rstrip=args.rstrip, judge=args.judge)
# return the result
testcase = {
'name': test_name,
'input': str(test_input_path.resolve()),
}
if test_output_path:
testcase['output'] = str(test_output_path.resolve())
return {
'status': status,
'testcase': testcase,
'output': answer,
'exitcode': proc.returncode,
'elapsed': elapsed,
'memory': memory,
}
def submit_code(
self,
code: bytes,
language_id: LanguageId,
*,
filename: Optional[str] = None,
session: Optional[requests.Session] = None
) -> onlinejudge.type.Submission:
"""
:raises NotLoggedInError:
:raises SubmissionError:
"""
session = session or utils.get_default_session()
# get
resp = utils.request('GET', self.get_url(), session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form', class_='submitForm')
if form is None:
log.error('not logged in')
raise NotLoggedInError
log.debug('form: %s', str(form))
# make data
form = utils.FormSender(form, url=resp.url)
form.set('programTypeId', language_id)
form.set_file('sourceFile', filename or 'code', code)
resp = form.request(session=session)
resp.raise_for_status()
# result
if resp.url.endswith('/my'):
# example: https://codeforces.com/contest/598/my
log.success('success: result: %s', resp.url)
return utils.DummySubmission(resp.url, problem=self)
else:
log.failure('failure')
# parse error messages
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
msgs = [] # type: List[str]
for span in soup.findAll('span', class_='error'):
msgs += [span.string]
log.warning('Codeforces says: "%s"', span.string)
raise SubmissionError(
'it may be the "You have submitted exactly the same code before" error: '
+ str(msgs))
def _generate_test_cases_in_cloned_repository(self):
path = self._get_cloned_repository_path()
try:
subprocess.check_call(['git', '--version'],
stdout=sys.stdout,
stderr=sys.stderr)
except FileNotFoundError:
log.error('git command not found')
raise
# init the problem repository
if not path.exists():
url = 'https://github.com/yosupo06/library-checker-problems'
log.status('$ git clone %s %s', url, path)
subprocess.check_call(
['git', 'clone', url, str(path)],
stdout=sys.stdout,
stderr=sys.stderr)
log.status('$ cd %s', path)
with utils.chdir(path):
# sync the problem repository
log.status('$ git pull')
subprocess.check_call(['git', 'pull'],
stdout=sys.stdout,
stderr=sys.stderr)
# generate test cases
if sys.version_info < (3, 6):
log.warning("generate.py may not work on Python 3.5 or older")
if os.name == 'nt':
log.warning("generate.py may not work on Windows")
log.status('$ ./generate.py problems.toml -p %s', self.problem_id)
try:
subprocess.check_call([
sys.executable, 'generate.py', 'problems.toml', '-p',
self.problem_id
],
stdout=sys.stdout,
stderr=sys.stderr)
except subprocess.CalledProcessError:
log.error(
"the generate.py failed: check https://github.com/yosupo06/library-checker-problems/issues"
)
raise
def login(args: 'argparse.Namespace') -> None:
# get service
service = onlinejudge.dispatch.service_from_url(args.url)
if service is None:
sys.exit(1)
# configure
kwargs = {}
if isinstance(service, onlinejudge.service.yukicoder.YukicoderService):
if not args.method:
args.method = 'github'
if args.method not in ['github', 'twitter']:
log.failure('login for yukicoder: invalid option: --method %s', args.method)
sys.exit(1)
kwargs['method'] = args.method
else:
if args.method:
log.failure('login for %s: invalid option: --method %s', service.get_name(), args.method)
sys.exit(1)
with utils.with_cookiejar(utils.new_session_with_our_user_agent(), path=args.cookie) as sess:
if args.check:
if service.is_logged_in(session=sess):
log.info('You have already signed in.')
else:
log.info('You are not signed in.')
sys.exit(1)
else:
# login
def get_credentials() -> Tuple[str, str]:
if args.username is None:
args.username = input('Username: '******'If you don\'t want to give your password to this program, you can give only your session tokens.')
log.info('see: https://github.com/kmyk/online-judge-tools/blob/master/LOGIN_WITH_COOKIES.md')
try:
service.login(get_credentials, session=sess, **kwargs) # type: ignore
except onlinejudge.type.LoginError:
pass
def get(self, directory: pathlib.Path = pathlib.Path.cwd()) -> List[str]:
if not self.path.exists():
return []
log.status('read history from: %s', self.path)
found = set()
with open(str(self.path)) as fh:
for line in fh:
try:
data = json.loads(line)
except json.decoder.JSONDecodeError as e:
log.warning('corrupted line found in: %s', self.path)
log.debug('%s', traceback.format_exc())
continue
if pathlib.Path(data['directory']) == directory:
found.add(data['url'])
log.status('found urls in history:\n%s', '\n'.join(found))
return list(found)
def _generate_test_cases_in_cloned_repository(self, compile_checker: bool = False) -> None:
LibraryCheckerService._update_cloned_repository()
path = LibraryCheckerService._get_cloned_repository_path()
if sys.version_info < (3, 6):
log.warning("generate.py may not work on Python 3.5 or older")
if os.name == 'nt':
log.warning("generate.py may not work on Windows")
problem_spec = str(self._get_problem_directory_path() / 'info.toml')
command = [sys.executable, str(path / 'generate.py'), problem_spec]
if compile_checker:
command.append('--compile-checker')
log.status('$ %s', ' '.join(command))
try:
subprocess.check_call(command, stdout=sys.stderr, stderr=sys.stderr)
except subprocess.CalledProcessError:
log.error("the generate.py failed: check https://github.com/yosupo06/library-checker-problems/issues")
raise
def submit_code(
self,
code: bytes,
language_id: LanguageId,
*,
filename: Optional[str] = None,
session: Optional[requests.Session] = None
) -> onlinejudge.type.Submission:
"""
:raises NotLoggedInError:
"""
session = session or utils.get_default_session()
# get
url = self.get_url() + '/submit'
resp = utils.request('GET', url, session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form', id='submit_form')
if not form:
log.error('form not found')
raise NotLoggedInError
# post
form = utils.FormSender(form, url=resp.url)
form.set('lang', language_id)
form.set_file('file', filename or 'code', code)
form.unset('custom_test')
resp = form.request(session=session)
resp.raise_for_status()
# result
if 'submissions' in resp.url:
# example: https://yukicoder.me/submissions/314087
log.success('success: result: %s', resp.url)
return utils.DummySubmission(resp.url, problem=self)
else:
log.failure('failure')
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
for div in soup.findAll('div', attrs={'role': 'alert'}):
log.warning('yukicoder says: "%s"', div.string)
raise SubmissionError
def add(self, content: bytes, name: str) -> None:
if self._dangling is None:
if re.search('output', name, re.IGNORECASE) or re.search(
'出力', name):
log.warning('strange name for input string: %s', name)
self._dangling = (name, content)
else:
if re.search('input', name, re.IGNORECASE) or re.search(
'入力', name):
if not (re.search('output', name, re.IGNORECASE)
or re.search('出力', name)
): # to ignore titles like "Output for Sample Input 1"
log.warning('strange name for output string: %s', name)
index = len(self._testcases)
input_name, input_content = self._dangling
self._testcases += [
TestCase('sample-{}'.format(index + 1), input_name,
input_content, name, content)
]
self._dangling = None