def download_with_running_code( self, session: Optional[requests.Session] = None) -> List[TestCase]: session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) csrftoken = soup.find('meta', attrs={ 'name': 'csrf-token' }).attrs['content'] # post url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests'.format( self.contest_slug, self.challenge_slug) payload = {'code': ':', 'language': 'bash', 'customtestcase': False} log.debug('payload: %s', payload) resp = utils.request('POST', url, session=session, json=payload, headers={'X-CSRF-Token': csrftoken}) # parse it = json.loads(resp.content.decode()) log.debug('json: %s', it) if not it['status']: log.error('Run Code: failed') return [] model_id = it['model']['id'] now = datetime.datetime.now() unixtime = int(datetime.datetime.now().timestamp() * 10**3) url = 'https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests/{}?_={}'.format( self.contest_slug, self.challenge_slug, it['model']['id'], unixtime) # sleep log.status('sleep(3)') time.sleep(3) # get resp = utils.request('GET', url, session=session, headers={'X-CSRF-Token': csrftoken}) # parse it = json.loads(resp.content.decode()) log.debug('json: %s', it) if not it['status']: log.error('Run Code: failed') return [] samples: List[TestCase] = [] for i, (inf, outf) in enumerate( zip(it['model']['stdin'], it['model']['expected_output'])): inname = 'Testcase {} Input'.format(i) outname = 'Testcase {} Expected Output'.format(i) samples += [ TestCase( LabeledString(inname, utils.textfile(inf)), LabeledString(outname, utils.textfile(outf)), ) ] return samples
def download(self, session=None): session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) msgs = AtCoderService._get_messages_from_cookie(resp.cookies) if AtCoderService._report_messages(msgs, unexpected=True): # example message: "message: You cannot see this page." log.warning('are you logged in?') return [] # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() lang = None for pre, h3 in self._find_sample_tags(soup): s = utils.textfile(utils.dos2unix(pre.string.lstrip())) name = h3.string l = self._get_tag_lang(pre) if lang is None: lang = l elif lang != l: log.info( 'skipped due to language: current one is %s, not %s: %s ', lang, l, name) continue samples.add(s, name) return samples.get()
def download(self, session=None): session = session or requests.Session() url = self.get_url() # get log.status('GET: %s', url) resp = session.get(url) log.status(utils.describe_status_code(resp.status_code)) resp.raise_for_status() msgs = AtCoderService._get_messages_from_cookie(resp.cookies) if AtCoderService._report_messages(msgs, unexpected=True): return [] # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() lang = None for pre, h3 in self._find_sample_tags(soup): s = utils.textfile(utils.dos2unix(pre.string.lstrip())) name = h3.string l = self._get_tag_lang(pre) if lang is None: lang = l elif lang != l: log.info( 'skipped due to language: current one is %s, not %s: %s ', lang, l, name) continue samples.add(s, name) return samples.get()
def download_samples(self, session=None): session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() for pre in soup.find_all('pre'): log.debug('pre: %s', str(pre)) hn = utils.previous_sibling_tag(pre) if hn is None: div = pre.parent if div is not None: log.debug('div: %s', str(hn)) hn = utils.previous_sibling_tag(div) log.debug('hN: %s', str(hn)) log.debug(hn) keywords = ['sample', 'example', '入力例', '出力例'] if hn and hn.name in ['h2', 'h3'] and hn.string and any( filter(lambda keyword: keyword in hn.string.lower(), keywords)): s = utils.textfile(pre.string.lstrip()) name = hn.string samples.add(s, name) return samples.get()
def download_samples( self, session: Optional[requests.Session] = None) -> List[TestCase]: session = session or utils.new_default_session() # get resp = utils.request('GET', self.get_url(), session=session) # parse soup = bs4.BeautifulSoup( resp.content, utils.html_parser ) # NOTE: resp.content is not decoded for workaround, see https://github.com/kmyk/online-judge-tools/pull/186 samples = utils.SampleZipper() for pre in soup.find_all('pre'): log.debug('pre: %s', str(pre)) hn = utils.previous_sibling_tag(pre) if hn is None: div = pre.parent if div is not None: log.debug('div: %s', str(hn)) hn = utils.previous_sibling_tag(div) log.debug('hN: %s', str(hn)) log.debug(hn) keywords = ['sample', 'example', '入力例', '出力例'] if hn and hn.name in ['h2', 'h3'] and hn.string and any( filter(lambda keyword: keyword in hn.string.lower(), keywords)): s = utils.textfile(pre.string.lstrip()) name = hn.string samples.add(s, name) return samples.get()
def download(self, session=None): session = session or requests.Session() url = self.get_url() # get log.status('GET: %s', url) resp = session.get(url) log.status(utils.describe_status_code(resp.status_code)) resp.raise_for_status() # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() for pre in soup.find_all('pre'): log.debug('pre: %s', str(pre)) hn = utils.previous_sibling_tag(pre) log.debug('hN: %s', str(hn)) log.debug(hn) if hn and hn.name in [ 'h2', 'h3' ] and hn.string and 'ample' in hn.string.lower( ): # 'ample' is the suffix of 'sample', 'example' s = utils.textfile(pre.string.lstrip()) name = hn.string samples.add(s, name) return samples.get()
def _parse_sample_tag(self, tag): assert isinstance(tag, bs4.Tag) assert tag.name == 'pre' prv = utils.previous_sibling_tag(tag) pprv = tag.parent and utils.previous_sibling_tag(tag.parent) if prv.name == 'h6' and tag.parent.name == 'div' and tag.parent['class'] == ['paragraph'] and pprv.name == 'h5': log.debug('h6: %s', str(prv)) log.debug('name.encode(): %s', prv.string.encode()) s = tag.string or '' # tag.string for the tag "<pre></pre>" returns None return utils.textfile(s.lstrip()), pprv.string + ' ' + prv.string
def _parse_sample_tag(self, tag): assert isinstance(tag, bs4.Tag) assert tag.name == "h2" name = tag.contents[0] if ":" in name: name = name[:name.find(":")] if name in ["Sample input", "Sample output"]: nxt = tag.next_sibling while nxt and nxt.string.strip() == "": nxt = nxt.next_sibling if nxt.name == "pre": s = utils.textfile(utils.dos2unix(nxt.string.lstrip())) else: s = "" return s, name
def _parse_sample_tag(self, tag): assert isinstance(tag, bs4.Tag) assert tag.name == "pre" prv = utils.previous_sibling_tag(tag) pprv = tag.parent and utils.previous_sibling_tag(tag.parent) if ( prv.name == "h6" and tag.parent.name == "div" and tag.parent["class"] == ["paragraph"] and pprv.name == "h5" ): log.debug("h6: %s", str(prv)) log.debug("name.encode(): %s", prv.string.encode()) s = tag.string or "" # tag.string for the tag "<pre></pre>" returns None return utils.textfile(s.lstrip()), pprv.string + " " + prv.string
def _parse_sample_tag(self, tag): assert isinstance(tag, bs4.Tag) assert tag.name == 'h2' name = tag.contents[0] if ':' in name: name = name[: name.find(':') ] if name in [ 'Sample input', 'Sample output' ]: nxt = tag.next_sibling while nxt and nxt.string.strip() == '': nxt = nxt.next_sibling if nxt.name == 'pre': s = utils.textfile(utils.dos2unix(nxt.string.lstrip())) else: s = '' return s, name
def download_sample_cases( self, session: Optional[requests.Session] = None ) -> List[onlinejudge.type.TestCase]: session = session or utils.new_default_session() # get # 自分で書き換えた箇所(初期 AtCoder の問題 URl は末尾が数字になっているため) url = self.get_url() for _ in range(2): resp = _request('GET', url, session=session) msgs = AtCoderService._get_messages_from_cookie(resp.cookies) if AtCoderService._report_messages(msgs, unexpected=True): # example message: "message: You cannot see this page." log.warning('are you logged in?') return [] # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) samples = utils.SampleZipper() if len(list(self._find_sample_tags(soup))) > 0: break else: url = url[:-1] + chr(ord(url[-1]) - ord('a') + ord('1')) lang = None for pre, h3 in self._find_sample_tags(soup): s = utils.textfile(utils.dos2unix(pre.string.lstrip())) name = h3.string l = self._get_tag_lang(pre) if lang is None: lang = l elif lang != l: log.info( 'skipped due to language: current one is %s, not %s: %s ', lang, l, name) continue samples.add(s, name) return samples.get()
def download_with_running_code(self, session=None): session = session or utils.new_default_session() # get resp = utils.request("GET", self.get_url(), session=session) # parse soup = bs4.BeautifulSoup(resp.content.decode(resp.encoding), utils.html_parser) csrftoken = soup.find("meta", attrs={ "name": "csrf-token" }).attrs["content"] # post url = "https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests".format( self.contest_slug, self.challenge_slug) payload = {"code": ":", "language": "bash", "customtestcase": False} log.debug("payload: %s", payload) resp = utils.request( "POST", url, session=session, json=payload, headers={"X-CSRF-Token": csrftoken}, ) # parse it = json.loads(resp.content.decode()) log.debug("json: %s", it) if not it["status"]: log.error("Run Code: failed") return [] model_id = it["model"]["id"] now = datetime.datetime.now() unixtime = int(datetime.datetime.now().timestamp() * 10**3) url = "https://www.hackerrank.com/rest/contests/{}/challenges/{}/compile_tests/{}?_={}".format( self.contest_slug, self.challenge_slug, it["model"]["id"], unixtime) # sleep log.status("sleep(3)") time.sleep(3) # get resp = utils.request("GET", url, session=session, headers={"X-CSRF-Token": csrftoken}) # parse it = json.loads(resp.content.decode()) log.debug("json: %s", it) if not it["status"]: log.error("Run Code: failed") return [] samples = [] for i, (inf, outf) in enumerate( zip(it["model"]["stdin"], it["model"]["expected_output"])): inname = "Testcase {} Input".format(i) outname = "Testcase {} Expected Output".format(i) samples += [{ "input": { "data": utils.textfile(inf), "name": inname }, "output": { "data": utils.textfile(outf), "name": outname }, }] return samples