def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: Optional[requests.Session] = None) -> bool:
session = session or utils.new_default_session()
url = 'https://www.hackerrank.com/auth/login'
# get
resp = utils.request('GET', url, session=session)
if resp.url != url:
log.debug('redirected: %s', resp.url)
log.info('You have already signed in.')
return True
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
csrftoken = soup.find('meta', attrs={'name': 'csrf-token'}).attrs['content']
tag = soup.find('input', attrs={'name': 'username'})
while tag.name != 'form':
tag = tag.parent
form = tag
# post
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set('login', username)
form.set('password', password)
form.set('remember_me', 'true')
form.set('fallback', 'true')
resp = form.request(session, method='POST', action='/rest/auth/login', headers={'X-CSRF-Token': csrftoken})
resp.raise_for_status()
log.debug('redirected: %s', resp.url)
# result
if '/auth' not in resp.url:
log.success('You signed in.')
return True
else:
log.failure('You failed to sign in. Wrong user ID or password.')
return False
def download_system_cases(self, session: Optional[requests.Session] = None) -> List[TestCase]:
session = session or utils.new_default_session()
# get
# example: https://www.hackerrank.com/rest/contests/hourrank-1/challenges/beautiful-array/download_testcases
url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/download_testcases'.format(self.contest_slug, self.challenge_slug)
resp = utils.request('GET', url, session=session, raise_for_status=False)
if resp.status_code != 200:
log.error('response: %s', resp.content.decode())
return []
# parse
with zipfile.ZipFile(io.BytesIO(resp.content)) as fh:
# list names
names = [] # type: List[str]
pattern = re.compile(r'(in|out)put/\1put(\d+).txt')
for filename in sorted(fh.namelist()): # "input" < "output"
if filename.endswith('/'):
continue
log.debug('filename: %s', filename)
m = pattern.match(filename)
assert m
if m.group(1) == 'in':
names += [m.group(2)]
# zip samples
samples = [] # type: List[TestCase]
for name in names:
inpath = 'input/input{}.txt'.format(name)
outpath = 'output/output{}.txt'.format(name)
indata = fh.read(inpath).decode()
outdata = fh.read(outpath).decode()
samples += [TestCase(LabeledString(inpath, indata), LabeledString(outpath, outdata))]
return samples
def generate_scanner(args: 'argparse.Namespace') -> None:
if not args.silent:
log.warning('This feature is ' + log.red('experimental') + '.')
if args.silent:
for handler in log.logger.handlers:
log.removeHandler(handler)
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
with utils.with_cookiejar(utils.new_default_session(), path=args.cookie) as sess:
it: Any = problem.get_input_format(session=sess)
if not it:
log.error('input format not found')
sys.exit(1)
try:
log.debug('original data: %s', repr(it))
it = list(tokenize(it))
log.debug('tokenized: %s', str(it))
it = list(parse(it))
log.debug('parsed: %s', str(it))
it = postprocess(it)
log.debug('postprocessed: %s', str(it))
it = export(it, use_scanf=args.scanf, repeat_macro=args.repeat_macro)
log.debug('result: %s', repr(it))
except:
log.error('something wrong')
raise
log.success('success:')
print(log.bold(it.rstrip())) # to stdout
def login_with_github(
self,
get_credentials: onlinejudge.type.CredentialsProvider,
session: Optional[requests.Session] = None) -> bool:
session = session or utils.new_default_session()
url = 'https://yukicoder.me/auth/github'
# get
resp = utils.request('GET', url, session=session)
if urllib.parse.urlparse(resp.url).hostname == 'yukicoder.me':
log.info('You have already signed in.')
return True
# redirect to github.com
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form')
if not form:
log.error('form not found')
log.info('Did you logged in?')
return False
log.debug('form: %s', str(form))
# post
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set('login', username)
form.set('password', password)
resp = form.request(session)
resp.raise_for_status()
if urllib.parse.urlparse(resp.url).hostname == 'yukicoder.me':
log.success('You signed in.')
return True
else:
log.failure('You failed to sign in. Wrong user ID or password.')
return False
def download_sample_cases(
self,
session: Optional[requests.Session] = None
) -> List[onlinejudge.type.TestCase]:
session = session or utils.new_default_session()
# get
url = self.get_url(contests=False) + '/file/statement/samples.zip'
resp = utils.request('GET',
url,
session=session,
raise_for_status=False)
if resp.status_code == 404:
log.warning('samples.zip not found')
log.info(
'this 404 happens in both cases: 1. no sample cases as intended; 2. just an error'
)
return []
resp.raise_for_status()
# parse
with zipfile.ZipFile(io.BytesIO(resp.content)) as fh:
samples = [] # type: List[TestCase]
for filename in sorted(fh.namelist()):
log.debug('filename: %s', filename)
if filename.endswith('.in'):
inpath = filename
outpath = filename[:-3] + '.ans'
indata = fh.read(inpath).decode()
outdata = fh.read(outpath).decode()
samples += [
TestCase(LabeledString(inpath, indata),
LabeledString(outpath, outdata))
]
return samples
def submit(self, code, language, session=None):
session = session or utils.new_default_session()
# get
resp = utils.request('GET', self.get_url(), session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
csrftoken = soup.find('meta', attrs={
'name': 'csrf-token'
}).attrs['content']
# post
url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/submissions'.format(
self.contest_slug, self.challenge_slug)
payload = {'code': code, 'language': language}
log.debug('payload: %s', payload)
resp = utils.request('POST',
url,
session=session,
json=payload,
headers={'X-CSRF-Token': csrftoken})
# parse
it = json.loads(resp.content.decode())
log.debug('json: %s', it)
if not it['status']:
log.failure('Submit Code: failed')
return None
model_id = it['model']['id']
url = self.get_url().rstrip('/') + '/submissions/code/{}'.format(
model_id)
log.success('success: result: %s', url)
return onlinejudge.submission.CompatibilitySubmission(url,
problem=self)
def login(self,
get_credentials: onlinejudge.type.CredentialsProvider,
session: Optional[requests.Session] = None) -> bool:
session = session or utils.new_default_session()
url = 'https://toph.co/login'
# get
resp = utils.request('GET', url, session=session)
if resp.url != url: # redirected
log.info('You are already logged in.')
return True
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form', class_='login-form')
log.debug('form: %s', str(form))
username, password = get_credentials()
form[
'action'] = '/login' # to avoid KeyError inside form.request method as Toph does not have any defined action
form = utils.FormSender(form, url=resp.url)
form.set('handle', username)
form.set('password', password)
# post
resp = form.request(session)
resp.raise_for_status()
resp = utils.request(
'GET', url, session=session
) # Toph's Location header is not getting the expected value
if resp.url != url:
log.success('Welcome, %s.', username)
return True
else:
log.failure('Invalid handle/email or password.')
return False
def login(self, get_credentials, session=None):
session = session or utils.new_default_session()
url = "http://codeforces.com/enter"
# get
resp = utils.request("GET", url, session=session)
if resp.url != url: # redirected
log.info("You have already signed in.")
return True
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find("form", id="enterForm")
log.debug("form: %s", str(form))
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set("handleOrEmail", username)
form.set("password", password)
form.set("remember", "on")
# post
resp = form.request(session)
resp.raise_for_status()
if resp.url != url: # redirected
log.success("Welcome, %s.", username)
return True
else:
log.failure("Invalid handle or password.")
return False
def run_program(args: argparse.Namespace,
parser: argparse.ArgumentParser) -> None:
if args.version:
print('online-judge-tools {}'.format(onlinejudge.__version__))
exit(0)
if args.verbose:
log.setLevel(log.logging.DEBUG)
log.debug('args: %s', str(args))
if args.subcommand in ['download', 'd', 'dl']:
download(args)
elif args.subcommand in ['login', 'l']:
login(args)
elif args.subcommand in ['submit', 's']:
submit(args)
elif args.subcommand in ['test', 't']:
test(args)
elif args.subcommand in ['test-reactive', 't/r']:
test_reactive(args)
elif args.subcommand in ['generate-scanner', 'g/s']:
generate_scanner(args)
elif args.subcommand in ['generate-output', 'g/o']:
generate_output(args)
elif args.subcommand in ['split-input', 's/i']:
split_input(args)
elif args.subcommand in ['code-statistics', 'c/s']:
code_statistics(args)
elif args.subcommand == 'get-standings':
get_standings(args)
else:
parser.print_help(file=sys.stderr)
sys.exit(1)
def _find_sample_tags(self, soup) -> Generator[Tuple[bs4.Tag, bs4.Tag], None, None]:
for pre in soup.find_all('pre'):
log.debug('pre tag: %s', str(pre))
if not pre.string:
continue
prv = utils.previous_sibling_tag(pre)
# the first format: h3+pre
if prv and prv.name == 'h3' and prv.string:
yield ( pre, prv )
else:
# ignore tags which are not samples
# example: https://atcoder.jp/contests/abc003/tasks/abc003_4
while prv is not None:
if prv.name == 'pre':
break
prv = utils.previous_sibling_tag(prv)
if prv is not None:
continue
# the second format: h3+section pre
if pre.parent and pre.parent.name == 'section':
prv = pre.parent and utils.previous_sibling_tag(pre.parent)
if prv and prv.name == 'h3' and prv.string:
yield ( pre, prv )
def submit(self, code, language, session=None):
assert language in self.get_language_dict(session=session)
session = session or requests.Session()
url = self.get_url() + '/submit'
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
form = soup.find('form', action=re.compile(r'/submit$'))
if not form:
log.error('form not found')
return None
log.debug('form: %s', str(form))
# post
form = utils.FormSender(form, url=resp.url)
if False:
form.set('source', code)
else:
form.set_file('file', ('source', code))
form.set('lang', language)
resp = form.request(session=session)
resp.raise_for_status()
# result
if '/submissions/' in resp.url:
log.success('success: result: %s', resp.url)
return resp.url
else:
log.failure('failure')
return None
def login(self,
get_credentials: onlinejudge.service.CredentialsProvider,
session: Optional[requests.Session] = None) -> bool:
session = session or utils.new_default_session()
url = 'https://codeforces.com/enter'
# get
resp = utils.request('GET', url, session=session)
if resp.url != url: # redirected
log.info('You have already signed in.')
return True
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form', id='enterForm')
log.debug('form: %s', str(form))
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set('handleOrEmail', username)
form.set('password', password)
form.set('remember', 'on')
# post
resp = form.request(session)
resp.raise_for_status()
if resp.url != url: # redirected
log.success('Welcome, %s.', username)
return True
else:
log.failure('Invalid handle or password.')
return False
def download(self, session=None):
session = session or requests.Session()
url = self.get_url()
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
samples = utils.SampleZipper()
for tag in soup.find_all('div', class_=re.compile(
'^(in|out)put$')): # Codeforces writes very nice HTML :)
log.debug('tag: %s', str(tag))
assert len(list(tag.children))
title, pre = list(tag.children)
assert 'title' in title.attrs['class']
assert pre.name == 'pre'
s = ''
for it in pre.children:
if it.name == 'br':
s += '\n'
else:
s += it.string
samples.add(s, title.string)
return samples.get()
def login(self, get_credentials, session=None):
session = session or requests.Session()
url = 'http://codeforces.com/enter'
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
if resp.url != url: # redirected
log.info('You have already signed in.')
return True
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form', id='enterForm')
log.debug('form: %s', str(form))
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set('handle', username)
form.set('password', password)
form.set('remember', 'on')
# post
resp = form.request(session)
resp.raise_for_status()
if resp.url != url: # redirected
log.success('Welcome, %s.', username)
return True
else:
log.failure('Invalid handle or password.')
return False
def submit_code(self, code: bytes, language: str, session: Optional['requests.Session'] = None) -> onlinejudge.type.Submission: # or SubmissionError
session = session or utils.new_default_session()
# get
resp = utils.request('GET', self.get_url(), session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
form = soup.find('form', class_='submitForm')
if form is None:
log.error('not logged in')
raise SubmissionError
log.debug('form: %s', str(form))
# make data
form = utils.FormSender(form, url=resp.url)
form.set('programTypeId', language)
form.set_file('sourceFile', 'code', code)
resp = form.request(session=session)
resp.raise_for_status()
# result
if resp.url.endswith('/my'):
# example: https://codeforces.com/contest/598/my
log.success('success: result: %s', resp.url)
return onlinejudge.type.DummySubmission(resp.url)
else:
log.failure('failure')
log.debug('redirected to %s', resp.url)
# parse error messages
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
for span in soup.findAll('span', class_='error'):
log.warning('Codeforces says: "%s"', span.string)
raise SubmissionError
def run_program(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
# logging
log_level = log.logging.INFO
if args.verbose:
log_level = log.logging.DEBUG
log.setLevel(log_level)
handler = log.logging.StreamHandler(sys.stderr)
handler.setLevel(log_level)
log.addHandler(handler)
log.debug('args: %s', str(args))
if args.subcommand in [ 'download', 'd', 'dl' ]:
download(args)
elif args.subcommand in [ 'login', 'l' ]:
login(args)
elif args.subcommand in [ 'submit', 's' ]:
submit(args)
elif args.subcommand in [ 'test', 't' ]:
test(args)
elif args.subcommand in [ 'test-reactive', 't/r' ]:
test_reactive(args)
elif args.subcommand in [ 'generate-scanner', 'g/s' ]:
generate_scanner(args)
elif args.subcommand in [ 'generate-output', 'g/o' ]:
generate_output(args)
elif args.subcommand in [ 'split-input', 's/i' ]:
split_input(args)
elif args.subcommand in [ 'code-statistics', 'c/s' ]:
code_statistics(args)
elif args.subcommand == 'get-standings':
get_standings(args)
else:
parser.print_help(file=sys.stderr)
sys.exit(1)
def download(
self,
session: Optional[requests.Session] = None
) -> List[onlinejudge.problem.TestCase]:
session = session or utils.new_default_session()
# get
resp = utils.request('GET', self.get_url(), session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
samples = utils.SampleZipper()
for tag in soup.find_all('div', class_=re.compile(
'^(in|out)put$')): # Codeforces writes very nice HTML :)
log.debug('tag: %s', str(tag))
assert len(list(tag.children))
title, pre = list(tag.children)
assert 'title' in title.attrs['class']
assert pre.name == 'pre'
s = ''
for it in pre.children:
if it.name == 'br':
s += '\n'
else:
s += it.string
samples.add(s, title.string)
return samples.get()
def generate_scanner(args):
if not args.silent:
log.warning("This feature is " + log.red("experimental") + ".")
if args.silent:
for handler in log.logger.handlers:
log.removeHandler(handler)
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
with utils.with_cookiejar(utils.new_default_session(), path=args.cookie) as sess:
it = problem.get_input_format(session=sess)
if not it:
log.error("input format not found")
sys.exit(1)
try:
log.debug("original data: %s", repr(it))
it = list(tokenize(it))
log.debug("tokenized: %s", str(it))
it = list(parse(it))
log.debug("parsed: %s", str(it))
it = postprocess(it)
log.debug("postprocessed: %s", str(it))
it = export(it, use_scanf=args.scanf, repeat_macro=args.repeat_macro)
log.debug("result: %s", repr(it))
except:
log.error("something wrong")
raise
log.success("success:")
print(log.bold(it.rstrip())) # to stdout
def submit(self, code, language, session=None):
session = session or utils.new_default_session()
# get
resp = utils.request("GET", self.get_url(), session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
csrftoken = soup.find("meta", attrs={
"name": "csrf-token"
}).attrs["content"]
# post
url = "https://www.hackerrank.com/rest/contests/{}/challenges/{}/submissions".format(
self.contest_slug, self.challenge_slug)
payload = {"code": code, "language": language}
log.debug("payload: %s", payload)
resp = utils.request(
"POST",
url,
session=session,
json=payload,
headers={"X-CSRF-Token": csrftoken},
)
# parse
it = json.loads(resp.content.decode())
log.debug("json: %s", it)
if not it["status"]:
log.failure("Submit Code: failed")
return None
model_id = it["model"]["id"]
url = self.get_url().rstrip("/") + "/submissions/code/{}".format(
model_id)
log.success("success: result: %s", url)
return onlinejudge.submission.CompatibilitySubmission(url,
problem=self)
def run_webdriver(webdriver, target_url, cookie_path, headless=True):
# launch
if webdriver is None:
log.error('webdriver is not specified')
sys.exit(1)
elif 'phantomjs' in os.path.basename(webdriver):
kwargs = {}
if '/' in webdriver:
kwargs['executable_path'] = webdriver
driver = selenium.webdriver.PhantomJS(**kwargs)
elif 'chrom' in os.path.basename(webdriver):
kwargs = {}
if '/' in webdriver:
kwargs['executable_path'] = webdriver
kwargs['chrome_options'] = selenium.webdriver.ChromeOptions()
if headless:
kwargs['chrome_options'].add_argument('--headless')
kwargs['chrome_options'].add_argument('--disable-gpu')
driver = selenium.webdriver.Chrome(**kwargs)
else:
parser.error('unknown webdriver: %s', webdriver)
# workaround
# NOTE: selenium can read/write only cookies of the current domain
domain = '.'.join(urllib.parse.urlparse(target_url).netloc.split('.')[-2:])
# default cookie path
default_selenium_cookie_path = os.path.join(
default_data_dir, 'cookie-' + driver.name + '-' + domain + '.jar')
cookie_path = cookie_path or default_selenium_cookie_path
# load cookie
if os.path.exists(cookie_path):
log.info('load cookie for %s from: %s', driver.name, cookie_path)
driver.get(target_url)
time.sleep(1)
with open(cookie_path) as fh:
cookies = ast.literal_eval(fh.read())
for cookie in cookies:
log.debug('cookie: %s', repr(cookie))
try:
driver.add_cookie(cookie)
except selenium.common.exceptions.WebDriverException as e:
log.debug('exception:\n%s', str(e))
yield driver
# save cookie
log.info('save cookie for %s to: %s', driver.name, cookie_path)
driver.get(target_url)
time.sleep(1)
if os.path.dirname(cookie_path):
os.makedirs(os.path.dirname(cookie_path), exist_ok=True)
with open(cookie_path, 'w') as fh:
fh.write(repr(driver.get_cookies()) + '\n')
os.chmod(cookie_path, 0o600) # NOTE: to make secure a little bit
driver.close()
def glob_with_format(directory, format):
table = {}
table["s"] = "*"
table["e"] = "*"
pattern = os.path.join(directory, utils.parcentformat(format, table))
paths = glob.glob(pattern)
for path in paths:
log.debug("testcase globbed: %s", path)
return paths
def glob_with_format(format):
table = {}
table['s'] = '*'
table['e'] = '*'
pattern = utils.parcentformat(format, table)
paths = glob.glob(pattern)
for path in paths:
log.debug('testcase globbed: %s', path)
return paths
def glob_with_format(directory: pathlib.Path, format: str) -> List[pathlib.Path]:
table = {}
table['s'] = '*'
table['e'] = '*'
pattern = str(directory / utils.percentformat(format, table))
paths = list(map(pathlib.Path, glob.glob(pattern)))
for path in paths:
log.debug('testcase globbed: %s', path)
return paths
def glob_with_format(directory: str, format: str) -> List[str]:
table = {}
table['s'] = '*'
table['e'] = '*'
pattern = os.path.join(directory, utils.parcentformat(format, table))
paths = glob.glob(pattern)
for path in paths:
log.debug('testcase globbed: %s', path)
return paths
def _parse_sample_tag(self, tag):
assert isinstance(tag, bs4.Tag)
assert tag.name == 'pre'
prv = utils.previous_sibling_tag(tag)
pprv = tag.parent and utils.previous_sibling_tag(tag.parent)
if prv.name == 'h6' and tag.parent.name == 'div' and tag.parent['class'] == ['paragraph'] and pprv.name == 'h5':
log.debug('h6: %s', str(prv))
log.debug('name.encode(): %s', prv.string.encode())
s = tag.string or '' # tag.string for the tag "<pre></pre>" returns None
return utils.textfile(s.lstrip()), pprv.string + ' ' + prv.string
def glob_with_format(directory: pathlib.Path,
format: str) -> List[pathlib.Path]:
table = {}
table['s'] = '*'
table['e'] = '*'
pattern = (
glob.escape(str(directory)) + '/' +
utils.percentformat(glob.escape(format).replace('\\%', '%'), table))
paths = list(map(pathlib.Path, glob.glob(pattern)))
for path in paths:
log.debug('testcase globbed: %s', path)
return paths
def _get_model(self, session: Optional[requests.Session] = None) -> Dict[str, Any]:
session = session or utils.new_default_session()
# get
url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug)
resp = utils.request('GET', url, session=session)
# parse
it = json.loads(resp.content.decode())
log.debug('json: %s', it)
if not it['status']:
log.error('get model: failed')
raise onlinejudge.type.SubmissionError
return it['model']
def request(self, session, action=None, **kwargs):
action = action or self.form['action']
url = urllib.parse.urljoin(self.url, action)
method = self.form['method'].upper()
log.status('%s: %s', method, url)
log.debug('payload: %s', str(self.payload))
resp = session.request(method,
url,
data=self.payload,
files=self.files,
**kwargs)
log.status(describe_status_code(resp.status_code))
return resp
def __init__(self, form, url):
assert isinstance(form, bs4.Tag)
assert form.name == 'form'
self.form = form
self.url = url
self.payload = {}
self.files = {}
for input in self.form.find_all('input'):
log.debug('input: %s', str(input))
if input.attrs.get('type') in ['checkbox', 'radio']:
continue
if 'name' in input.attrs and 'value' in input.attrs:
self.payload[input['name']] = input['value']
def __init__(self, form, url):
assert isinstance(form, bs4.Tag)
assert form.name == "form"
self.form = form
self.url = url
self.payload = {}
self.files = {}
for input in self.form.find_all("input"):
log.debug("input: %s", str(input))
if input.attrs.get("type") in ["checkbox", "radio"]:
continue
if "name" in input.attrs and "value" in input.attrs:
self.payload[input["name"]] = input["value"]
