def download_source_code(self) -> enums.DownloadStatus:
r = requests.get(
f'https://codeforces.com/contest/{self.contest_id}/submission/{self.id}',
headers=Config()['GET-Headers'],
allow_redirects=False)
time.sleep(Config()['Request-Delay'])
if check_for_redirecting(r):
status = enums.DownloadStatus.FAILED
self.download_status = status.value
return status
if check_for_status(r):
status = enums.DownloadStatus.FAILED
self.download_status = status.value
return status
text = get_html_content(r, id='program-source-text')
if not text:
status = enums.DownloadStatus.FAILED
self.download_status = status.value
return status
reset_timeout_counter(reset_consecutive=True)
self.__dump_code(text)
status = enums.DownloadStatus.FINISHED
self.download_status = status.value
return status
def __is_eligible(raw_submission) -> bool:
try:
return \
not any(map(lambda tag: tag in Config()['Excluded-Tags'], raw_submission['problem']['tags'])) and \
raw_submission['verdict'] in Config()['Supported-Verdicts'] \
and (raw_submission['programmingLanguage'] in Config()['Supported-Languages'] or
len(Config()['Supported-Languages']) == 0)
except:
return False
def download_source_code(self) -> None:
max_submissions = Config()['Max-Submissions']
submissions = list(
filter(lambda s: not Contest.__validate_submission(s),
self.submissions.values()))
size = len(submissions)
size = size if max_submissions <= 0 else min(size, max_submissions)
submissions = list(submissions)[:size]
now = datetime.datetime.now()
for submission in tqdm(
submissions,
total=len(submissions),
position=0,
leave=True,
desc=
f'[{self.id}][{now.hour:02d}:{now.minute:02d}] Downloading code'
):
if check_consecutive_timeouts():
reset_timeout_counter(reset_consecutive=True)
break
self.submissions[
submission.
id].download_status = submission.download_source_code().value
self.__dump()
def get_path(contest_id: int, **kwargs) -> Optional[str]:
if not contest_id:
return None
path = [str(contest_id)]
if kwargs.get('meta', False):
path = [*path, 'meta.json']
return Config().combine_path(*path)
def __request_raw_submissions(self, contest_id: int, start: int,
count: int) -> List[dict]:
payload = {'contestId': contest_id, 'from': start, 'count': count}
r = requests.get(
'https://codeforces.com/api/contest.status',
# headers=Config()['GET-Headers'],
# allow_redirects=False,
params=payload)
try:
r.raise_for_status()
except requests.HTTPError:
time.sleep(Config()['Request-Delay'])
return []
time.sleep(Config()['Request-Delay'])
return r.json()['result']
def get_code_path_extensionless(submission_id: int, contest_id: int,
language: str) -> Optional[str]:
path = [
contest_id,
shorten_programming_language(language), submission_id
]
if not any(path):
return None
return Config().combine_path(*path)
def __eligible_raw_contests(self) -> List[dict]:
result = self.__filter_raw_contests(self.__request_raw_contests(),
apply_config_constraints=True)
self.__print_eligible_contests(contests=result)
if len(result) > 0:
time.sleep(Config()['Request-Delay'])
return result
def __filter_raw_contests(self, *args, **kwargs) -> List[dict]:
data = args[0]
if not isinstance(data, list):
data = list(data)
result = list(filter(lambda x: Yanker.__is_eligible(x), data))
if kwargs.get('apply_config_constraints', False):
max_contests = Config()['Max-Contests']
initial_contest_id = Config()['Initial-Contest-Id']
initial_contest_index = 0
for i, v in enumerate(result):
if v['id'] == initial_contest_id:
initial_contest_index = i
break
if max_contests >= 0:
start = min(initial_contest_index, len(result))
end = min(start + max_contests, len(result))
result = result[start:end]
return result
def __filter_raw_submissions(self, *args, **kwargs) -> List[dict]:
data = args[0]
if not isinstance(data, list):
data = list(data)
result = list(filter(lambda x: Contest.__is_eligible(x), data))
if kwargs.get('apply_config_constraints', False):
max_submissions = Config()['Max-Submissions']
if max_submissions >= 0:
result = result[:min(max_submissions, len(result))]
return result
def __eligible_raw_submissions(self) -> List[dict]:
result = []
current_index = 1
batch_size = 20000
max_submissions = Config()['Max-Submissions']
progress_bar = None
indent = '\t' * TqdmControl().indent
last = 0
if max_submissions > 0:
progress_bar = tqdm(
total=max_submissions,
position=TqdmControl().pos,
leave=True,
bar_format=f'{indent}[{self.id}] Updating metadata')
progress_bar.desc = f'{indent}\tFiltering submissions'
progress_bar.bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}, {rate_fmt}{postfix}]'
while len(result) < max_submissions or max_submissions == -1:
raw_submissions = self.__request_raw_submissions(
self.id, current_index, batch_size)
if len(raw_submissions) == 0:
break
raw_submissions = self.__filter_raw_submissions(
raw_submissions, apply_config_constraints=True)
for submission in raw_submissions:
result.append(submission)
if progress_bar:
current = min(len(result), max_submissions)
progress_bar.update(current - last)
time.sleep(0.1)
last = current
current_index += batch_size
if progress_bar:
progress_bar.bar_format = f'{indent}Finished updating'
progress_bar.refresh()
time.sleep(0.1)
progress_bar.close()
return result
def __ensure_data(self) -> None:
progress_bar = tqdm(
total=len(self.__eligible_raw_contests),
position=TqdmControl().pos,
leave=True,
desc=f'Contests',
bar_format=
'{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}, {rate_fmt}{postfix}]'
)
for raw_contest in self.__eligible_raw_contests:
contest_id = raw_contest['id']
contest_path = Contest.get_path(contest_id, meta=True)
TqdmControl().inc_indent()
TqdmControl().inc_pos()
contest_instance = Contest.deserialize(
download=Config()['After-Update'], path=contest_path)
if not contest_instance:
contest_instance = Contest(download=True, info=raw_contest)
TqdmControl().dec_pos()
TqdmControl().dec_indent()
self.contests[contest_id] = contest_instance
progress_bar.update()
progress_bar.close()
def __validate_submission(submission):
languages = Config()['Supported-Languages']
return submission.language not in languages or \
any(map(lambda tag: tag in Config()['Excluded-Tags'], submission.tags)) or \
submission.download_status == enums.DownloadStatus.FINISHED.value and \
submission.validate_code(fixup=True)
def __ensure_directories(self) -> None:
path = Config().combine_path(
self.contest_id, shorten_programming_language(self.language))
if path and not OPE(path):
OMD(path)
def __is_eligible(raw_contest) -> bool:
try:
return raw_contest['phase'] in Config()['Supported-Phases'] and \
raw_contest['type'] in Config()['Supported-Contest-Formats']
except:
return False