def login(self, get_credentials, session=None):
session = session or utils.new_default_session()
url = 'https://practice.contest.atcoder.jp/login'
# get
resp = utils.request('GET',
url,
session=session,
allow_redirects=False)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
for msg in msgs:
log.status('message: %s', msg)
if msgs:
return 'login' not in resp.url
# post
username, password = get_credentials()
resp = utils.request('POST',
url,
session=session,
data={
'name': username,
'password': password
},
allow_redirects=False)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
AtCoderService._report_messages(msgs)
return 'login' not in resp.url # AtCoder redirects to the top page if success
def login(self, get_credentials, session=None):
session = session or requests.Session()
url = 'http://codeforces.com/enter'
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
if resp.url != url: # redirected
log.info('You have already signed in.')
return True
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
form = soup.find('form', id='enterForm')
log.debug('form: %s', str(form))
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set('handle', username)
form.set('password', password)
form.set('remember', 'on')
# post
resp = form.request(session)
resp.raise_for_status()
if resp.url != url: # redirected
log.success('Welcome, %s.', username)
return True
else:
log.failure('Invalid handle or password.')
return False
def generate_output(args: 'argparse.Namespace') -> None:
if not args.test:
args.test = cutils.glob_with_format(args.directory, args.format) # by default
if args.ignore_backup:
args.test = cutils.drop_backup_or_hidden_files(args.test)
tests = cutils.construct_relationship_of_files(args.test, args.directory, args.format)
for name, it in sorted(tests.items()):
log.emit('')
log.info('%s', name)
if 'out' in it:
log.info('output file already exists.')
log.info('skipped.')
continue
with it['in'].open() as inf:
begin = time.perf_counter()
answer, proc = utils.exec_command(args.command, shell=True, stdin=inf)
end = time.perf_counter()
log.status('time: %f sec', end - begin)
if proc.returncode != 0:
log.failure(log.red('RE') + ': return code %d', proc.returncode)
log.info('skipped.')
continue
log.emit(log.bold(answer.decode().rstrip()))
match_result = cutils.match_with_format(args.directory, args.format, it['in']) # type: Optional[Match[Any]]
if match_result is not None:
matched_name = match_result.groupdict()['name'] # type: str
else:
assert False
path = cutils.path_from_format(args.directory, args.format, name=matched_name, ext='out')
if not path.parent.is_dir():
os.makedirs(str(path.parent), exist_ok=True)
with path.open('wb') as fh:
fh.write(answer)
log.success('saved to: %s', path)
def submit(self, code, language, session=None):
assert language in self.get_language_dict(session=session)
session = session or requests.Session()
url = self.get_url() + '/submit'
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
form = soup.find('form', action=re.compile(r'/submit$'))
if not form:
log.error('form not found')
return None
log.debug('form: %s', str(form))
# post
form = utils.FormSender(form, url=resp.url)
if False:
form.set('source', code)
else:
form.set_file('file', ('source', code))
form.set('lang', language)
resp = form.request(session=session)
resp.raise_for_status()
# result
if '/submissions/' in resp.url:
log.success('success: result: %s', resp.url)
return resp.url
else:
log.failure('failure')
return None
def login(self, get_credentials, session=None):
session = session or utils.new_default_session()
url = "https://practice.contest.atcoder.jp/login"
# get
resp = utils.request("GET",
url,
session=session,
allow_redirects=False)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
for msg in msgs:
log.status("message: %s", msg)
if msgs:
return "login" not in resp.url
# post
username, password = get_credentials()
resp = utils.request(
"POST",
url,
session=session,
data={
"name": username,
"password": password
},
allow_redirects=False,
)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
AtCoderService._report_messages(msgs)
return "login" not in resp.url # AtCoder redirects to the top page if success
def download_with_running_code(
self,
session: Optional[requests.Session] = None) -> List[TestCase]:
session = session or utils.new_default_session()
# get
resp = utils.request('GET', self.get_url(), session=session)
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
csrftoken = soup.find('meta', attrs={
'name': 'csrf-token'
}).attrs['content']
# post
url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests'.format(
self.contest_slug, self.challenge_slug)
payload = {'code': ':', 'language': 'bash', 'customtestcase': False}
log.debug('payload: %s', payload)
resp = utils.request('POST',
url,
session=session,
json=payload,
headers={'X-CSRF-Token': csrftoken})
# parse
it = json.loads(resp.content.decode())
log.debug('json: %s', it)
if not it['status']:
log.error('Run Code: failed')
return []
model_id = it['model']['id']
now = datetime.datetime.now()
unixtime = int(datetime.datetime.now().timestamp() * 10**3)
url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests/{}?_={}'.format(
self.contest_slug, self.challenge_slug, it['model']['id'],
unixtime)
# sleep
log.status('sleep(3)')
time.sleep(3)
# get
resp = utils.request('GET',
url,
session=session,
headers={'X-CSRF-Token': csrftoken})
# parse
it = json.loads(resp.content.decode())
log.debug('json: %s', it)
if not it['status']:
log.error('Run Code: failed')
return []
samples: List[TestCase] = []
for i, (inf, outf) in enumerate(
zip(it['model']['stdin'], it['model']['expected_output'])):
inname = 'Testcase {} Input'.format(i)
outname = 'Testcase {} Expected Output'.format(i)
samples += [
TestCase(
LabeledString(inname, utils.textfile(inf)),
LabeledString(outname, utils.textfile(outf)),
)
]
return samples
def login(self,
get_credentials: onlinejudge.type.CredentialsProvider,
session: Optional[requests.Session] = None) -> bool:
session = session or utils.new_default_session()
# 自分で書き換えた箇所
# url = 'https://practice.contest.atcoder.jp/login'
url = 'https://atcoder.jp/login'
# get
resp = _request('GET', url, session=session, allow_redirects=False)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
for msg in msgs:
log.status('message: %s', msg)
if msgs:
return 'login' not in resp.url
# post
username, password = get_credentials()
# 自分で書き換えた箇所(CSRF対策)
soup = bs4.BeautifulSoup(resp.text, 'lxml')
csrftoken = soup.find_all('input')[0]['value']
resp = _request('POST',
url,
session=session,
data={
'csrf_token': csrftoken,
'username': username,
'password': password
},
allow_redirects=False)
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
AtCoderService._report_messages(msgs)
return 'login' not in resp.url # AtCoder redirects to the top page if success
def download(self, session=None):
session = session or requests.Session()
url = self.get_url()
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
samples = utils.SampleZipper()
for pre in soup.find_all('pre'):
log.debug('pre: %s', str(pre))
hn = utils.previous_sibling_tag(pre)
log.debug('hN: %s', str(hn))
log.debug(hn)
if hn and hn.name in [
'h2', 'h3'
] and hn.string and 'ample' in hn.string.lower(
): # 'ample' is the suffix of 'sample', 'example'
s = utils.textfile(pre.string.lstrip())
name = hn.string
samples.add(s, name)
return samples.get()
def generate_output(args):
if not args.test:
args.test = glob_with_format(args.format) # by default
if args.ignore_backup:
args.test = drop_backup_or_hidden_files(args.test)
tests = construct_relationship_of_files(args.test, args.format)
for name, it in sorted(tests.items()):
log.emit('')
log.info('%s', name)
if 'out' in it:
log.info('output file already exists.')
log.info('skipped.')
continue
with open(it['in']) as inf:
begin = time.perf_counter()
answer, proc = utils.exec_command(args.command, shell=args.shell, stdin=inf)
end = time.perf_counter()
log.status('time: %f sec', end - begin)
if proc.returncode != 0:
log.failure(log.red('RE') + ': return code %d', proc.returncode)
log.info('skipped.')
continue
log.emit(log.bold(answer.decode().rstrip()))
path = path_from_format(args.format, match_with_format(args.format, it['in']).groupdict()['name'], 'out')
with open(path, 'w') as fh:
fh.buffer.write(answer)
log.success('saved to: %s', path)
def download(self, session=None):
session = session or requests.Session()
url = self.get_url()
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
msgs = AtCoderService._get_messages_from_cookie(resp.cookies)
if AtCoderService._report_messages(msgs, unexpected=True):
return []
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
samples = utils.SampleZipper()
lang = None
for pre, h3 in self._find_sample_tags(soup):
s = utils.textfile(utils.dos2unix(pre.string.lstrip()))
name = h3.string
l = self._get_tag_lang(pre)
if lang is None:
lang = l
elif lang != l:
log.info(
'skipped due to language: current one is %s, not %s: %s ',
lang, l, name)
continue
samples.add(s, name)
return samples.get()
def login(self, get_credentials, session=None):
driver = session # rename
assert isinstance(driver,
selenium.webdriver.remote.webdriver.WebDriver)
# go to the login page
url = 'https://accounts.topcoder.com/member'
driver.get(url)
log.status('wait for redirect...')
time.sleep(3)
if driver.current_url != url:
log.info('You have already logged in.')
return True
# type credentials and click login
username, password = get_credentials()
driver.find_element_by_xpath(
'//form[@name="vm.loginForm"]//input[@name="username"]').send_keys(
username)
driver.find_element_by_xpath(
'//form[@name="vm.loginForm"]//input[@id="current-password-input"]'
).send_keys(password)
driver.find_element_by_xpath(
'//form[@name="vm.loginForm"]//button[@type="submit" and text()="Log In"]'
).click()
# wait a minute
log.info('Logging in...')
time.sleep(6)
if driver.current_url != url:
log.success('Success')
return True
else:
log.failure('Failure')
return False
def login_with_github(self, get_credentials, session=None):
session = session or requests.Session()
url = 'https://yukicoder.me/auth/github'
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
if urllib.parse.urlparse(resp.url).hostname == 'yukicoder.me':
log.info('You have already signed in.')
return True
# redirect to github.com
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser)
form = soup.find('form')
if not form:
log.error('form not found')
log.info('Did you logged in?')
return False
log.debug('form: %s', str(form))
# post
username, password = get_credentials()
form = utils.FormSender(form, url=resp.url)
form.set('login', username)
form.set('password', password)
resp = form.request(session)
resp.raise_for_status()
if urllib.parse.urlparse(resp.url).hostname == 'yukicoder.me':
log.success('You signed in.')
return True
else:
log.failure('You failed to sign in. Wrong user ID or password.')
return False
def download(self, session=None):
session = session or requests.Session()
url = self.get_url()
# get
log.status('GET: %s', url)
resp = session.get(url)
log.status(utils.describe_status_code(resp.status_code))
resp.raise_for_status()
# parse
soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding),
utils.html_parser)
samples = utils.SampleZipper()
for tag in soup.find_all('div', class_=re.compile(
'^(in|out)put$')): # Codeforces writes very nice HTML :)
log.debug('tag: %s', str(tag))
assert len(list(tag.children))
title, pre = list(tag.children)
assert 'title' in title.attrs['class']
assert pre.name == 'pre'
s = ''
for it in pre.children:
if it.name == 'br':
s += '\n'
else:
s += it.string
samples.add(s, title.string)
return samples.get()
def submission_from_url(s):
for cls in submissions:
it = cls.from_url(s)
if it is not None:
log.status('submission recognized: %s: %s', str(it), s)
return it
log.failure('unknown submission: %s', s)
def submission_from_url(s: str) -> Optional['Submission']:
for cls in submissions:
submission = cls.from_url(s)
if submission is not None:
log.status('submission recognized: %s: %s', str(submission), s)
return submission
log.failure('unknown submission: %s', s)
return None
def format_code(code: bytes, dos2unix: bool = False, rstrip: bool = False) -> bytes:
if dos2unix:
log.status('dos2unix...')
code = code.replace(b'\r\n', b'\n')
if rstrip:
log.status('rstrip...')
code = code.rstrip()
return code
def call_generate_scanner(self, url, expected, options=[]):
cmd = [self.ojtools, '-v', 'generate-scanner', url] + options
output = subprocess.check_output(cmd, stderr=sys.stderr).decode()
log.status('result:\n%s', output)
if expected != output:
log.error('expected:\n%s' % expected)
self.assertEqual(expected, output)
time.sleep(1)
def _report_messages(cls,
msgs: List[str],
unexpected: bool = False) -> bool:
for msg in msgs:
log.status('message: %s', msg)
if msgs and unexpected:
log.failure('unexpected messages found')
return bool(msgs)
def _flush(self) -> None:
# halve the size if it is more than 1MiB
if self.path.stat().st_size >= 1024 * 1024:
with open(str(self.path)) as fh:
history_lines = fh.readlines()
with open(str(self.path), 'w') as fh:
fh.write(''.join(history_lines[:-len(history_lines) // 2]))
log.status('halve history at: %s', self.path)
def request(method, url, session=None, raise_for_status=True, **kwargs):
assert method in ['GET', 'POST']
kwargs.setdefault('allow_redirects', True)
log.status('%s: %s', method, url)
resp = session.request(method, url, **kwargs)
log.status(describe_status_code(resp.status_code))
if raise_for_status:
resp.raise_for_status()
return resp
def download(args):
# prepare values
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
kwargs = {}
if args.system:
supported_service_names = ["aoj", "yukicoder"]
if problem.get_service().get_name() not in supported_service_names:
log.error("--system for %s is not supported",
problem.get_service().get_name())
sys.exit(1)
kwargs["is_system"] = True
if args.format is None:
if kwargs.get("is_system"):
if problem.get_service().get_name() == "yukicoder":
args.format = "%b.%e"
else:
args.format = "%i.%e"
else:
args.format = "sample-%i.%e"
# get samples from the server
with utils.with_cookiejar(utils.new_default_session(),
path=args.cookie) as sess:
samples = problem.download(session=sess, **kwargs)
# write samples to files
for i, sample in enumerate(samples):
log.emit("")
log.info("sample %d", i)
for kind in ["input", "output"]:
ext = kind[:-3]
data = sample[kind]["data"]
name = sample[kind]["name"]
table = {}
table["i"] = str(i + 1)
table["e"] = ext
table["n"] = name
table["b"] = os.path.basename(name)
table["d"] = os.path.dirname(name)
path = os.path.join(args.directory,
utils.parcentformat(args.format, table))
log.status("%sput: %s", ext, name)
log.emit(colorama.Style.BRIGHT + data.rstrip() +
colorama.Style.RESET_ALL)
if args.dry_run:
continue
if os.path.exists(path):
log.warning("file already exists: %s", path)
if not args.overwrite:
log.warning("skipped")
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as fh:
fh.write(data)
log.success("saved to: %s", path)
def download(args: 'argparse.Namespace') -> None:
# prepare values
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
kwargs = {}
if args.system:
supported_service_names = ['aoj', 'yukicoder']
if problem.get_service().get_name() not in supported_service_names:
log.error('--system for %s is not supported',
problem.get_service().get_name())
sys.exit(1)
kwargs['is_system'] = True
if args.format is None:
if kwargs.get('is_system'):
if problem.get_service().get_name() == 'yukicoder':
args.format = '%b.%e'
else:
args.format = '%i.%e'
else:
args.format = 'sample-%i.%e'
# get samples from the server
with utils.with_cookiejar(utils.new_default_session(),
path=args.cookie) as sess:
samples = problem.download(session=sess, **kwargs) # type: ignore
# write samples to files
for i, sample in enumerate(samples):
log.emit('')
log.info('sample %d', i)
for kind in ['input', 'output']:
ext = kind[:-3]
data = getattr(sample, kind).data
name = getattr(sample, kind).name
table = {}
table['i'] = str(i + 1)
table['e'] = ext
table['n'] = name
table['b'] = os.path.basename(name)
table['d'] = os.path.dirname(name)
path = os.path.join(args.directory,
utils.parcentformat(args.format, table))
log.status('%sput: %s', ext, name)
log.emit(colorama.Style.BRIGHT + data.rstrip() +
colorama.Style.RESET_ALL)
if args.dry_run:
continue
if os.path.exists(path):
log.warning('file already exists: %s', path)
if not args.overwrite:
log.warning('skipped')
continue
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w') as fh:
fh.write(data)
log.success('saved to: %s', path)
def request(method, url, session=None, raise_for_status=True, **kwargs):
assert method in ["GET", "POST"]
kwargs.setdefault("allow_redirects", True)
log.status("%s: %s", method, url)
resp = session.request(method, url, **kwargs)
log.status(describe_status_code(resp.status_code))
if raise_for_status:
resp.raise_for_status()
return resp
def problem_from_url(s):
for cls in problems:
it = cls.from_url(s)
if it is not None:
log.status('problem recognized: %s: %s', str(it), s)
return it
it = submission_from_url(s)
if it is not None:
return it.get_problem()
log.failure('unknown problem: %s', s)
def problem_from_url(s: str) -> Optional['Problem']:
for cls in problems:
problem = cls.from_url(s)
if problem is not None:
log.status('problem recognized: %s: %s', str(problem), s)
return problem
submission = submission_from_url(s)
if submission is not None:
return submission.get_problem()
log.failure('unknown problem: %s', s)
return None
def submit(args):
problem = onlinejudge.dispatch.problem_from_url(args.url)
if problem is None:
sys.exit(1)
# code
with open(args.file) as fh:
code = fh.buffer.read()
try:
s = code.decode() # for logging
except UnicodeDecodeError as e:
log.failure('%s: %s', e.__class__.__name__, str(e))
s = repr(code)[1:]
log.info('code:')
log.emit(log.bold(s))
# session
with utils.session(cookiejar=args.cookie) as sess:
# language
langs = problem.get_language_dict(session=sess)
if args.language not in langs:
log.error('language is unknown')
log.info('supported languages are:')
for lang in sorted(langs.keys()):
log.emit('%s (%s)', lang, langs[lang]['description'])
sys.exit(1)
# confirm
if args.wait:
log.status('sleep(%.2f)', args.wait)
time.sleep(args.wait)
if not args.yes:
sys.stdout.write('Are you sure? [y/N] ')
sys.stdout.flush()
c = sys.stdin.read(1)
if c != 'y':
log.info('terminated.')
return
# submit
url = problem.submit(code, language=args.language, session=sess)
if url and args.open:
if not isinstance(args.open, str):
args.open = None
for browser in default_url_opener:
args.open = shutil.which(browser)
if args.open:
break
if not args.open:
log.failure('couldn\'t open the url. please specify a browser')
else:
log.info('open the submission page with: %s', args.open)
subprocess.check_call([args.open, url],
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr)
def service_from_url(s):
for cls in services:
it = cls.from_url(s)
if it is not None:
log.status('service recognized: %s: %s', str(it), s)
return it
it = submission_from_url(s)
if it is not None:
return it.get_service()
it = problem_from_url(s)
if it is not None:
return it.get_service()
log.failure('unknown service: %s', s)
def request(self, session, action=None, **kwargs):
action = action or self.form['action']
url = urllib.parse.urljoin(self.url, action)
method = self.form['method'].upper()
log.status('%s: %s', method, url)
log.debug('payload: %s', str(self.payload))
resp = session.request(method,
url,
data=self.payload,
files=self.files,
**kwargs)
log.status(describe_status_code(resp.status_code))
return resp
def with_cookiejar(
session: requests.Session,
path: pathlib.Path = default_cookie_path
) -> Generator[requests.Session, None, None]:
session.cookies = http.cookiejar.LWPCookieJar(str(path)) # type: ignore
if path.exists():
log.status('load cookie from: %s', path)
session.cookies.load() # type: ignore
yield session
log.status('save cookie to: %s', path)
path.parent.mkdir(parents=True, exist_ok=True)
session.cookies.save() # type: ignore
path.chmod(0o600) # NOTE: to make secure a little bit
def request(method: str,
url: str,
session: requests.Session,
raise_for_status: bool = True,
**kwargs) -> requests.Response:
assert method in ['GET', 'POST']
kwargs.setdefault('allow_redirects', True)
log.status('%s: %s', method, url)
resp = session.request(method, url, **kwargs)
log.status(describe_status_code(resp.status_code))
if raise_for_status:
resp.raise_for_status()
return resp