Ejemplo n.º 1
0
    def download_source_code(self) -> enums.DownloadStatus:
        r = requests.get(
            f'https://codeforces.com/contest/{self.contest_id}/submission/{self.id}',
            headers=Config()['GET-Headers'],
            allow_redirects=False)
        time.sleep(Config()['Request-Delay'])

        if check_for_redirecting(r):
            status = enums.DownloadStatus.FAILED
            self.download_status = status.value
            return status

        if check_for_status(r):
            status = enums.DownloadStatus.FAILED
            self.download_status = status.value
            return status

        text = get_html_content(r, id='program-source-text')
        if not text:
            status = enums.DownloadStatus.FAILED
            self.download_status = status.value
            return status

        reset_timeout_counter(reset_consecutive=True)
        self.__dump_code(text)
        status = enums.DownloadStatus.FINISHED
        self.download_status = status.value
        return status
Ejemplo n.º 2
0
 def __is_eligible(raw_submission) -> bool:
     try:
         return \
             not any(map(lambda tag: tag in Config()['Excluded-Tags'], raw_submission['problem']['tags'])) and \
             raw_submission['verdict'] in Config()['Supported-Verdicts'] \
             and (raw_submission['programmingLanguage'] in Config()['Supported-Languages'] or
                  len(Config()['Supported-Languages']) == 0)
     except:
         return False
Ejemplo n.º 3
0
    def download_source_code(self) -> None:
        max_submissions = Config()['Max-Submissions']
        submissions = list(
            filter(lambda s: not Contest.__validate_submission(s),
                   self.submissions.values()))
        size = len(submissions)
        size = size if max_submissions <= 0 else min(size, max_submissions)
        submissions = list(submissions)[:size]
        now = datetime.datetime.now()
        for submission in tqdm(
                submissions,
                total=len(submissions),
                position=0,
                leave=True,
                desc=
                f'[{self.id}][{now.hour:02d}:{now.minute:02d}] Downloading code'
        ):

            if check_consecutive_timeouts():
                reset_timeout_counter(reset_consecutive=True)
                break

            self.submissions[
                submission.
                id].download_status = submission.download_source_code().value
            self.__dump()
Ejemplo n.º 4
0
    def get_path(contest_id: int, **kwargs) -> Optional[str]:
        if not contest_id:
            return None

        path = [str(contest_id)]
        if kwargs.get('meta', False):
            path = [*path, 'meta.json']
        return Config().combine_path(*path)
Ejemplo n.º 5
0
    def __request_raw_submissions(self, contest_id: int, start: int,
                                  count: int) -> List[dict]:
        payload = {'contestId': contest_id, 'from': start, 'count': count}
        r = requests.get(
            'https://codeforces.com/api/contest.status',
            # headers=Config()['GET-Headers'],
            # allow_redirects=False,
            params=payload)

        try:
            r.raise_for_status()
        except requests.HTTPError:
            time.sleep(Config()['Request-Delay'])
            return []

        time.sleep(Config()['Request-Delay'])
        return r.json()['result']
Ejemplo n.º 6
0
 def get_code_path_extensionless(submission_id: int, contest_id: int,
                                 language: str) -> Optional[str]:
     path = [
         contest_id,
         shorten_programming_language(language), submission_id
     ]
     if not any(path):
         return None
     return Config().combine_path(*path)
Ejemplo n.º 7
0
    def __eligible_raw_contests(self) -> List[dict]:
        result = self.__filter_raw_contests(self.__request_raw_contests(),
                                            apply_config_constraints=True)

        self.__print_eligible_contests(contests=result)
        if len(result) > 0:
            time.sleep(Config()['Request-Delay'])

        return result
Ejemplo n.º 8
0
    def __filter_raw_contests(self, *args, **kwargs) -> List[dict]:
        data = args[0]
        if not isinstance(data, list):
            data = list(data)

        result = list(filter(lambda x: Yanker.__is_eligible(x), data))

        if kwargs.get('apply_config_constraints', False):
            max_contests = Config()['Max-Contests']
            initial_contest_id = Config()['Initial-Contest-Id']
            initial_contest_index = 0
            for i, v in enumerate(result):
                if v['id'] == initial_contest_id:
                    initial_contest_index = i
                    break
            if max_contests >= 0:
                start = min(initial_contest_index, len(result))
                end = min(start + max_contests, len(result))
                result = result[start:end]

        return result
Ejemplo n.º 9
0
    def __filter_raw_submissions(self, *args, **kwargs) -> List[dict]:
        data = args[0]
        if not isinstance(data, list):
            data = list(data)

        result = list(filter(lambda x: Contest.__is_eligible(x), data))

        if kwargs.get('apply_config_constraints', False):
            max_submissions = Config()['Max-Submissions']
            if max_submissions >= 0:
                result = result[:min(max_submissions, len(result))]

        return result
Ejemplo n.º 10
0
    def __eligible_raw_submissions(self) -> List[dict]:
        result = []
        current_index = 1
        batch_size = 20000
        max_submissions = Config()['Max-Submissions']
        progress_bar = None
        indent = '\t' * TqdmControl().indent
        last = 0
        if max_submissions > 0:
            progress_bar = tqdm(
                total=max_submissions,
                position=TqdmControl().pos,
                leave=True,
                bar_format=f'{indent}[{self.id}] Updating metadata')
            progress_bar.desc = f'{indent}\tFiltering submissions'
            progress_bar.bar_format = '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}, {rate_fmt}{postfix}]'
        while len(result) < max_submissions or max_submissions == -1:
            raw_submissions = self.__request_raw_submissions(
                self.id, current_index, batch_size)
            if len(raw_submissions) == 0:
                break

            raw_submissions = self.__filter_raw_submissions(
                raw_submissions, apply_config_constraints=True)
            for submission in raw_submissions:
                result.append(submission)

            if progress_bar:
                current = min(len(result), max_submissions)
                progress_bar.update(current - last)
                time.sleep(0.1)
                last = current

            current_index += batch_size

        if progress_bar:
            progress_bar.bar_format = f'{indent}Finished updating'
            progress_bar.refresh()
            time.sleep(0.1)
            progress_bar.close()
        return result
Ejemplo n.º 11
0
    def __ensure_data(self) -> None:
        progress_bar = tqdm(
            total=len(self.__eligible_raw_contests),
            position=TqdmControl().pos,
            leave=True,
            desc=f'Contests',
            bar_format=
            '{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}, {rate_fmt}{postfix}]'
        )
        for raw_contest in self.__eligible_raw_contests:
            contest_id = raw_contest['id']
            contest_path = Contest.get_path(contest_id, meta=True)
            TqdmControl().inc_indent()
            TqdmControl().inc_pos()
            contest_instance = Contest.deserialize(
                download=Config()['After-Update'], path=contest_path)
            if not contest_instance:
                contest_instance = Contest(download=True, info=raw_contest)
            TqdmControl().dec_pos()
            TqdmControl().dec_indent()
            self.contests[contest_id] = contest_instance
            progress_bar.update()

        progress_bar.close()
Ejemplo n.º 12
0
 def __validate_submission(submission):
     languages = Config()['Supported-Languages']
     return submission.language not in languages or \
            any(map(lambda tag: tag in Config()['Excluded-Tags'], submission.tags)) or \
            submission.download_status == enums.DownloadStatus.FINISHED.value and \
            submission.validate_code(fixup=True)
Ejemplo n.º 13
0
 def __ensure_directories(self) -> None:
     path = Config().combine_path(
         self.contest_id, shorten_programming_language(self.language))
     if path and not OPE(path):
         OMD(path)
Ejemplo n.º 14
0
 def __is_eligible(raw_contest) -> bool:
     try:
         return raw_contest['phase'] in Config()['Supported-Phases'] and \
                raw_contest['type'] in Config()['Supported-Contest-Formats']
     except:
         return False