def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: Optional[requests.Session] = None) -> bool: session = session or utils.new_default_session() url = 'https://www.hackerrank.com/auth/login' # get resp = utils.request('GET', url, session=session) if resp.url != url: log.debug('redirected: %s', resp.url) log.info('You have already signed in.') return True # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) csrftoken = soup.find('meta', attrs={'name': 'csrf-token'}).attrs['content'] tag = soup.find('input', attrs={'name': 'username'}) while tag.name != 'form': tag = tag.parent form = tag # post username, password = get_credentials() form = utils.FormSender(form, url=resp.url) form.set('login', username) form.set('password', password) form.set('remember_me', 'true') form.set('fallback', 'true') resp = form.request(session, method='POST', action='/rest/auth/login', headers={'X-CSRF-Token': csrftoken}) resp.raise_for_status() log.debug('redirected: %s', resp.url) # result if '/auth' not in resp.url: log.success('You signed in.') return True else: log.failure('You failed to sign in. Wrong user ID or password.') return False
def download_system_cases(self, session: Optional[requests.Session] = None) -> List[TestCase]: session = session or utils.new_default_session() # get # example: https://www.hackerrank.com/rest/contests/hourrank-1/challenges/beautiful-array/download_testcases url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/download_testcases'.format(self.contest_slug, self.challenge_slug) resp = utils.request('GET', url, session=session, raise_for_status=False) if resp.status_code != 200: log.error('response: %s', resp.content.decode()) return [] # parse with zipfile.ZipFile(io.BytesIO(resp.content)) as fh: # list names names = [] # type: List[str] pattern = re.compile(r'(in|out)put/\1put(\d+).txt') for filename in sorted(fh.namelist()): # "input" < "output" if filename.endswith('/'): continue log.debug('filename: %s', filename) m = pattern.match(filename) assert m if m.group(1) == 'in': names += [m.group(2)] # zip samples samples = [] # type: List[TestCase] for name in names: inpath = 'input/input{}.txt'.format(name) outpath = 'output/output{}.txt'.format(name) indata = fh.read(inpath).decode() outdata = fh.read(outpath).decode() samples += [TestCase(LabeledString(inpath, indata), LabeledString(outpath, outdata))] return samples
def generate_scanner(args: 'argparse.Namespace') -> None: if not args.silent: log.warning('This feature is ' + log.red('experimental') + '.') if args.silent: for handler in log.logger.handlers: log.removeHandler(handler) problem = onlinejudge.dispatch.problem_from_url(args.url) if problem is None: sys.exit(1) with utils.with_cookiejar(utils.new_default_session(), path=args.cookie) as sess: it: Any = problem.get_input_format(session=sess) if not it: log.error('input format not found') sys.exit(1) try: log.debug('original data: %s', repr(it)) it = list(tokenize(it)) log.debug('tokenized: %s', str(it)) it = list(parse(it)) log.debug('parsed: %s', str(it)) it = postprocess(it) log.debug('postprocessed: %s', str(it)) it = export(it, use_scanf=args.scanf, repeat_macro=args.repeat_macro) log.debug('result: %s', repr(it)) except: log.error('something wrong') raise log.success('success:') print(log.bold(it.rstrip())) # to stdout
def login_with_github( self, get_credentials: onlinejudge.type.CredentialsProvider, session: Optional[requests.Session] = None) -> bool: session = session or utils.new_default_session() url = 'https://yukicoder.me/auth/github' # get resp = utils.request('GET', url, session=session) if urllib.parse.urlparse(resp.url).hostname == 'yukicoder.me': log.info('You have already signed in.') return True # redirect to github.com # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find('form') if not form: log.error('form not found') log.info('Did you logged in?') return False log.debug('form: %s', str(form)) # post username, password = get_credentials() form = utils.FormSender(form, url=resp.url) form.set('login', username) form.set('password', password) resp = form.request(session) resp.raise_for_status() if urllib.parse.urlparse(resp.url).hostname == 'yukicoder.me': log.success('You signed in.') return True else: log.failure('You failed to sign in. Wrong user ID or password.') return False
def download_sample_cases( self, session: Optional[requests.Session] = None ) -> List[onlinejudge.type.TestCase]: session = session or utils.new_default_session() # get url = self.get_url(contests=False) + '/file/statement/samples.zip' resp = utils.request('GET', url, session=session, raise_for_status=False) if resp.status_code == 404: log.warning('samples.zip not found') log.info( 'this 404 happens in both cases: 1. no sample cases as intended; 2. just an error' ) return [] resp.raise_for_status() # parse with zipfile.ZipFile(io.BytesIO(resp.content)) as fh: samples = [] # type: List[TestCase] for filename in sorted(fh.namelist()): log.debug('filename: %s', filename) if filename.endswith('.in'): inpath = filename outpath = filename[:-3] + '.ans' indata = fh.read(inpath).decode() outdata = fh.read(outpath).decode() samples += [ TestCase(LabeledString(inpath, indata), LabeledString(outpath, outdata)) ] return samples
def submit(self, code, language, session=None): session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) csrftoken = soup.find('meta', attrs={ 'name': 'csrf-token' }).attrs['content'] # post url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/submissions'.format( self.contest_slug, self.challenge_slug) payload = {'code': code, 'language': language} log.debug('payload: %s', payload) resp = utils.request('POST', url, session=session, json=payload, headers={'X-CSRF-Token': csrftoken}) # parse it = json.loads(resp.content.decode()) log.debug('json: %s', it) if not it['status']: log.failure('Submit Code: failed') return None model_id = it['model']['id'] url = self.get_url().rstrip('/') + '/submissions/code/{}'.format( model_id) log.success('success: result: %s', url) return onlinejudge.submission.CompatibilitySubmission(url, problem=self)
def login(self, get_credentials: onlinejudge.type.CredentialsProvider, session: Optional[requests.Session] = None) -> bool: session = session or utils.new_default_session() url = 'https://toph.co/login' # get resp = utils.request('GET', url, session=session) if resp.url != url: # redirected log.info('You are already logged in.') return True # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find('form', class_='login-form') log.debug('form: %s', str(form)) username, password = get_credentials() form[ 'action'] = '/login' # to avoid KeyError inside form.request method as Toph does not have any defined action form = utils.FormSender(form, url=resp.url) form.set('handle', username) form.set('password', password) # post resp = form.request(session) resp.raise_for_status() resp = utils.request( 'GET', url, session=session ) # Toph's Location header is not getting the expected value if resp.url != url: log.success('Welcome, %s.', username) return True else: log.failure('Invalid handle/email or password.') return False
def login(self, get_credentials, session=None): session = session or utils.new_default_session() url = "http://codeforces.com/enter" # get resp = utils.request("GET", url, session=session) if resp.url != url: # redirected log.info("You have already signed in.") return True # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find("form", id="enterForm") log.debug("form: %s", str(form)) username, password = get_credentials() form = utils.FormSender(form, url=resp.url) form.set("handleOrEmail", username) form.set("password", password) form.set("remember", "on") # post resp = form.request(session) resp.raise_for_status() if resp.url != url: # redirected log.success("Welcome, %s.", username) return True else: log.failure("Invalid handle or password.") return False
def run_program(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: if args.version: print('online-judge-tools {}'.format(onlinejudge.__version__)) exit(0) if args.verbose: log.setLevel(log.logging.DEBUG) log.debug('args: %s', str(args)) if args.subcommand in ['download', 'd', 'dl']: download(args) elif args.subcommand in ['login', 'l']: login(args) elif args.subcommand in ['submit', 's']: submit(args) elif args.subcommand in ['test', 't']: test(args) elif args.subcommand in ['test-reactive', 't/r']: test_reactive(args) elif args.subcommand in ['generate-scanner', 'g/s']: generate_scanner(args) elif args.subcommand in ['generate-output', 'g/o']: generate_output(args) elif args.subcommand in ['split-input', 's/i']: split_input(args) elif args.subcommand in ['code-statistics', 'c/s']: code_statistics(args) elif args.subcommand == 'get-standings': get_standings(args) else: parser.print_help(file=sys.stderr) sys.exit(1)
def _find_sample_tags(self, soup) -> Generator[Tuple[bs4.Tag, bs4.Tag], None, None]: for pre in soup.find_all('pre'): log.debug('pre tag: %s', str(pre)) if not pre.string: continue prv = utils.previous_sibling_tag(pre) # the first format: h3+pre if prv and prv.name == 'h3' and prv.string: yield ( pre, prv ) else: # ignore tags which are not samples # example: https://atcoder.jp/contests/abc003/tasks/abc003_4 while prv is not None: if prv.name == 'pre': break prv = utils.previous_sibling_tag(prv) if prv is not None: continue # the second format: h3+section pre if pre.parent and pre.parent.name == 'section': prv = pre.parent and utils.previous_sibling_tag(pre.parent) if prv and prv.name == 'h3' and prv.string: yield ( pre, prv )
def submit(self, code, language, session=None): assert language in self.get_language_dict(session=session) session = session or requests.Session() url = self.get_url() + '/submit' # get log.status('GET: %s', url) resp = session.get(url) log.status(utils.describe_status_code(resp.status_code)) resp.raise_for_status() # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find('form', action=re.compile(r'/submit$')) if not form: log.error('form not found') return None log.debug('form: %s', str(form)) # post form = utils.FormSender(form, url=resp.url) if False: form.set('source', code) else: form.set_file('file', ('source', code)) form.set('lang', language) resp = form.request(session=session) resp.raise_for_status() # result if '/submissions/' in resp.url: log.success('success: result: %s', resp.url) return resp.url else: log.failure('failure') return None
def login(self, get_credentials: onlinejudge.service.CredentialsProvider, session: Optional[requests.Session] = None) -> bool: session = session or utils.new_default_session() url = 'https://codeforces.com/enter' # get resp = utils.request('GET', url, session=session) if resp.url != url: # redirected log.info('You have already signed in.') return True # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find('form', id='enterForm') log.debug('form: %s', str(form)) username, password = get_credentials() form = utils.FormSender(form, url=resp.url) form.set('handleOrEmail', username) form.set('password', password) form.set('remember', 'on') # post resp = form.request(session) resp.raise_for_status() if resp.url != url: # redirected log.success('Welcome, %s.', username) return True else: log.failure('Invalid handle or password.') return False
def download(self, session=None): session = session or requests.Session() url = self.get_url() # get log.status('GET: %s', url) resp = session.get(url) log.status(utils.describe_status_code(resp.status_code)) resp.raise_for_status() # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() for tag in soup.find_all('div', class_=re.compile( '^(in|out)put$')): # Codeforces writes very nice HTML :) log.debug('tag: %s', str(tag)) assert len(list(tag.children)) title, pre = list(tag.children) assert 'title' in title.attrs['class'] assert pre.name == 'pre' s = '' for it in pre.children: if it.name == 'br': s += '\n' else: s += it.string samples.add(s, title.string) return samples.get()
def login(self, get_credentials, session=None): session = session or requests.Session() url = 'http://codeforces.com/enter' # get log.status('GET: %s', url) resp = session.get(url) log.status(utils.describe_status_code(resp.status_code)) resp.raise_for_status() if resp.url != url: # redirected log.info('You have already signed in.') return True # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find('form', id='enterForm') log.debug('form: %s', str(form)) username, password = get_credentials() form = utils.FormSender(form, url=resp.url) form.set('handle', username) form.set('password', password) form.set('remember', 'on') # post resp = form.request(session) resp.raise_for_status() if resp.url != url: # redirected log.success('Welcome, %s.', username) return True else: log.failure('Invalid handle or password.') return False
def submit_code(self, code: bytes, language: str, session: Optional['requests.Session'] = None) -> onlinejudge.type.Submission: # or SubmissionError session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) form = soup.find('form', class_='submitForm') if form is None: log.error('not logged in') raise SubmissionError log.debug('form: %s', str(form)) # make data form = utils.FormSender(form, url=resp.url) form.set('programTypeId', language) form.set_file('sourceFile', 'code', code) resp = form.request(session=session) resp.raise_for_status() # result if resp.url.endswith('/my'): # example: https://codeforces.com/contest/598/my log.success('success: result: %s', resp.url) return onlinejudge.type.DummySubmission(resp.url) else: log.failure('failure') log.debug('redirected to %s', resp.url) # parse error messages soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) for span in soup.findAll('span', class_='error'): log.warning('Codeforces says: "%s"', span.string) raise SubmissionError
def run_program(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: # logging log_level = log.logging.INFO if args.verbose: log_level = log.logging.DEBUG log.setLevel(log_level) handler = log.logging.StreamHandler(sys.stderr) handler.setLevel(log_level) log.addHandler(handler) log.debug('args: %s', str(args)) if args.subcommand in [ 'download', 'd', 'dl' ]: download(args) elif args.subcommand in [ 'login', 'l' ]: login(args) elif args.subcommand in [ 'submit', 's' ]: submit(args) elif args.subcommand in [ 'test', 't' ]: test(args) elif args.subcommand in [ 'test-reactive', 't/r' ]: test_reactive(args) elif args.subcommand in [ 'generate-scanner', 'g/s' ]: generate_scanner(args) elif args.subcommand in [ 'generate-output', 'g/o' ]: generate_output(args) elif args.subcommand in [ 'split-input', 's/i' ]: split_input(args) elif args.subcommand in [ 'code-statistics', 'c/s' ]: code_statistics(args) elif args.subcommand == 'get-standings': get_standings(args) else: parser.print_help(file=sys.stderr) sys.exit(1)
def download( self, session: Optional[requests.Session] = None ) -> List[onlinejudge.problem.TestCase]: session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() for tag in soup.find_all('div', class_=re.compile( '^(in|out)put$')): # Codeforces writes very nice HTML :) log.debug('tag: %s', str(tag)) assert len(list(tag.children)) title, pre = list(tag.children) assert 'title' in title.attrs['class'] assert pre.name == 'pre' s = '' for it in pre.children: if it.name == 'br': s += '\n' else: s += it.string samples.add(s, title.string) return samples.get()
def generate_scanner(args): if not args.silent: log.warning("This feature is " + log.red("experimental") + ".") if args.silent: for handler in log.logger.handlers: log.removeHandler(handler) problem = onlinejudge.dispatch.problem_from_url(args.url) if problem is None: sys.exit(1) with utils.with_cookiejar(utils.new_default_session(), path=args.cookie) as sess: it = problem.get_input_format(session=sess) if not it: log.error("input format not found") sys.exit(1) try: log.debug("original data: %s", repr(it)) it = list(tokenize(it)) log.debug("tokenized: %s", str(it)) it = list(parse(it)) log.debug("parsed: %s", str(it)) it = postprocess(it) log.debug("postprocessed: %s", str(it)) it = export(it, use_scanf=args.scanf, repeat_macro=args.repeat_macro) log.debug("result: %s", repr(it)) except: log.error("something wrong") raise log.success("success:") print(log.bold(it.rstrip())) # to stdout
def submit(self, code, language, session=None): session = session or utils.new_default_session() # get resp = utils.request("GET", self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) csrftoken = soup.find("meta", attrs={ "name": "csrf-token" }).attrs["content"] # post url = "https://www.hackerrank.com/rest/contests/{}/challenges/{}/submissions".format( self.contest_slug, self.challenge_slug) payload = {"code": code, "language": language} log.debug("payload: %s", payload) resp = utils.request( "POST", url, session=session, json=payload, headers={"X-CSRF-Token": csrftoken}, ) # parse it = json.loads(resp.content.decode()) log.debug("json: %s", it) if not it["status"]: log.failure("Submit Code: failed") return None model_id = it["model"]["id"] url = self.get_url().rstrip("/") + "/submissions/code/{}".format( model_id) log.success("success: result: %s", url) return onlinejudge.submission.CompatibilitySubmission(url, problem=self)
def run_webdriver(webdriver, target_url, cookie_path, headless=True): # launch if webdriver is None: log.error('webdriver is not specified') sys.exit(1) elif 'phantomjs' in os.path.basename(webdriver): kwargs = {} if '/' in webdriver: kwargs['executable_path'] = webdriver driver = selenium.webdriver.PhantomJS(**kwargs) elif 'chrom' in os.path.basename(webdriver): kwargs = {} if '/' in webdriver: kwargs['executable_path'] = webdriver kwargs['chrome_options'] = selenium.webdriver.ChromeOptions() if headless: kwargs['chrome_options'].add_argument('--headless') kwargs['chrome_options'].add_argument('--disable-gpu') driver = selenium.webdriver.Chrome(**kwargs) else: parser.error('unknown webdriver: %s', webdriver) # workaround # NOTE: selenium can read/write only cookies of the current domain domain = '.'.join(urllib.parse.urlparse(target_url).netloc.split('.')[-2:]) # default cookie path default_selenium_cookie_path = os.path.join( default_data_dir, 'cookie-' + driver.name + '-' + domain + '.jar') cookie_path = cookie_path or default_selenium_cookie_path # load cookie if os.path.exists(cookie_path): log.info('load cookie for %s from: %s', driver.name, cookie_path) driver.get(target_url) time.sleep(1) with open(cookie_path) as fh: cookies = ast.literal_eval(fh.read()) for cookie in cookies: log.debug('cookie: %s', repr(cookie)) try: driver.add_cookie(cookie) except selenium.common.exceptions.WebDriverException as e: log.debug('exception:\n%s', str(e)) yield driver # save cookie log.info('save cookie for %s to: %s', driver.name, cookie_path) driver.get(target_url) time.sleep(1) if os.path.dirname(cookie_path): os.makedirs(os.path.dirname(cookie_path), exist_ok=True) with open(cookie_path, 'w') as fh: fh.write(repr(driver.get_cookies()) + '\n') os.chmod(cookie_path, 0o600) # NOTE: to make secure a little bit driver.close()
def glob_with_format(directory, format): table = {} table["s"] = "*" table["e"] = "*" pattern = os.path.join(directory, utils.parcentformat(format, table)) paths = glob.glob(pattern) for path in paths: log.debug("testcase globbed: %s", path) return paths
def glob_with_format(format): table = {} table['s'] = '*' table['e'] = '*' pattern = utils.parcentformat(format, table) paths = glob.glob(pattern) for path in paths: log.debug('testcase globbed: %s', path) return paths
def glob_with_format(directory: pathlib.Path, format: str) -> List[pathlib.Path]: table = {} table['s'] = '*' table['e'] = '*' pattern = str(directory / utils.percentformat(format, table)) paths = list(map(pathlib.Path, glob.glob(pattern))) for path in paths: log.debug('testcase globbed: %s', path) return paths
def glob_with_format(directory: str, format: str) -> List[str]: table = {} table['s'] = '*' table['e'] = '*' pattern = os.path.join(directory, utils.parcentformat(format, table)) paths = glob.glob(pattern) for path in paths: log.debug('testcase globbed: %s', path) return paths
def _parse_sample_tag(self, tag): assert isinstance(tag, bs4.Tag) assert tag.name == 'pre' prv = utils.previous_sibling_tag(tag) pprv = tag.parent and utils.previous_sibling_tag(tag.parent) if prv.name == 'h6' and tag.parent.name == 'div' and tag.parent['class'] == ['paragraph'] and pprv.name == 'h5': log.debug('h6: %s', str(prv)) log.debug('name.encode(): %s', prv.string.encode()) s = tag.string or '' # tag.string for the tag "<pre></pre>" returns None return utils.textfile(s.lstrip()), pprv.string + ' ' + prv.string
def glob_with_format(directory: pathlib.Path, format: str) -> List[pathlib.Path]: table = {} table['s'] = '*' table['e'] = '*' pattern = ( glob.escape(str(directory)) + '/' + utils.percentformat(glob.escape(format).replace('\\%', '%'), table)) paths = list(map(pathlib.Path, glob.glob(pattern))) for path in paths: log.debug('testcase globbed: %s', path) return paths
def _get_model(self, session: Optional[requests.Session] = None) -> Dict[str, Any]: session = session or utils.new_default_session() # get url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}'.format(self.contest_slug, self.challenge_slug) resp = utils.request('GET', url, session=session) # parse it = json.loads(resp.content.decode()) log.debug('json: %s', it) if not it['status']: log.error('get model: failed') raise onlinejudge.type.SubmissionError return it['model']
def request(self, session, action=None, **kwargs): action = action or self.form['action'] url = urllib.parse.urljoin(self.url, action) method = self.form['method'].upper() log.status('%s: %s', method, url) log.debug('payload: %s', str(self.payload)) resp = session.request(method, url, data=self.payload, files=self.files, **kwargs) log.status(describe_status_code(resp.status_code)) return resp
def __init__(self, form, url): assert isinstance(form, bs4.Tag) assert form.name == 'form' self.form = form self.url = url self.payload = {} self.files = {} for input in self.form.find_all('input'): log.debug('input: %s', str(input)) if input.attrs.get('type') in ['checkbox', 'radio']: continue if 'name' in input.attrs and 'value' in input.attrs: self.payload[input['name']] = input['value']
def __init__(self, form, url): assert isinstance(form, bs4.Tag) assert form.name == "form" self.form = form self.url = url self.payload = {} self.files = {} for input in self.form.find_all("input"): log.debug("input: %s", str(input)) if input.attrs.get("type") in ["checkbox", "radio"]: continue if "name" in input.attrs and "value" in input.attrs: self.payload[input["name"]] = input["value"]