async def get_participations( self, contest=None, user=None, is_disqualified=None, virtual_participation_number=None) -> List[Participation_DB]: a = API() page = 1 await a.get_participations( contest=contest, user=user, is_disqualified=is_disqualified, virtual_participation_number=virtual_participation_number, page=page) # why the hell are these names so long? cond_contest = self.parse(Contest_DB.key, contest) if not cond_contest: cond_contest = Participation_DB.contest.any(cond_contest) cond_user = self.parse(func.lower(User_DB.username), func.lower(user)) if not cond_user: cond_user = Participation_DB.user.any(cond_user) q = session.query(Participation_DB).\ filter(cond_contest).\ filter(cond_user).\ filter(self.parse(Participation_DB.is_disqualified, is_disqualified)).\ filter(self.parse(Participation_DB.virtual_participation_number, virtual_participation_number)) if a.data.total_objects == q.count(): return q.all() def get_id(participation): return participation.id participation_id = list(map(get_id, a.data.objects)) qq = session.query(Submission_DB.id).\ filter(Submission_DB.id.in_(participation_id)).all() qq = list(map(itemgetter(0), qq)) for submission in a.data.objects: if submission.id not in qq: session.add(Submission_DB(submission)) total_pages = a.data.total_pages for participation in a.data.objects: if participation.id not in participation_id: session.add(Participation_DB(participation)) apis = [] to_gather = [] for _ in range(2, total_pages + 1): page += 1 api = API() to_await = await a.get_participations( contest=contest, user=user, is_disqualified=is_disqualified, virtual_participation_number=virtual_participation_number, page=page) apis.append(api) to_gather.append(to_await) await asyncio.gather(*to_gather) for api in apis: participation_id = list(map(get_id, api.data.objects)) qq = session.query(Submission_DB.id).\ filter(Submission_DB.id.in_(participation_id)).all() qq = list(map(itemgetter(0), qq)) for submission in api.data.objects: if submission.id not in qq: session.add(Submission_DB(submission)) total_pages = api.data.total_pages for participation in api.data.objects: if participation.id not in participation_id: session.add(Participation_DB(participation)) session.commit() return q.all()
async def get_submissions(self, user=None, problem=None, language=None, result=None) -> List[Submission_DB]: # This function is the only one which might take a while to run and # has data that is added reguarly. asyncio.gather can apply to all # functions but this one is the only one which really needs it a = API() page = 1 import time start = time.time() await a.get_submissions(user=user, problem=problem, language=language, result=result, page=page) print("Done Api Call", time.time() - start) start = time.time() q = session.query(Submission_DB) q = q.filter(Submission_DB._user == user) cond_user = self.parse(func.lower(User_DB.username), func.lower(user)) if not cond_user: q = q.join(User_DB, cond_user, aliased=True) cond_problem = self.parse(Problem_DB.code, problem) if not cond_problem: q = q.join(Problem_DB, cond_problem, aliased=True) cond_lang = self.parse(Language_DB.key, language) if not cond_lang: q = q.join(Language_DB, cond_lang, aliased=True) q = q.filter(self.parse(Submission_DB.result, result)) if a.data.total_objects == q.count(): return q.all() def get_id(submission): return submission.id submission_ids = list(map(get_id, a.data.objects)) qq = session.query(Submission_DB.id).\ filter(Submission_DB.id.in_(submission_ids)).all() qq = list(map(itemgetter(0), qq)) for submission in a.data.objects: if submission.id not in qq: session.add(Submission_DB(submission)) total_pages = a.data.total_pages apis = [] to_gather = [] async def get_submission(api, user, problem, language, result, page): try: api.get_submissions(user=user, problem=problem, language=language, result=result, page=page) except: # Sometimes when caching a user with many pages one might not return correctly # this will silently return nothing # Perhaps I should implement some sort of error catching in the cogs pass for _ in range(2, total_pages + 1): page += 1 api = API() to_await = api.get_submissions(user=user, problem=problem, language=language, result=result, page=page) apis.append(api) to_gather.append(to_await) await asyncio.gather(*to_gather) for api in apis: if api.data.objects is None: continue submission_ids = list(map(get_id, api.data.objects)) qq = session.query(Submission_DB.id).\ filter(Submission_DB.id.in_(submission_ids)).all() qq = list(map(itemgetter(0), qq)) for submission in api.data.objects: if submission.id not in qq: session.add(Submission_DB(submission)) session.commit() return q.all()
async def get_submissions(self, user: str = None, problem: str = None, language: str = None, result: str = None) -> List[Submission_DB]: # This function is the only one which might take a while to run and # has data that is added reguarly. asyncio.gather can apply to all # functions but this one is the only one which really needs it a = API() page = 1 import time start = time.time() await a.get_submissions(user=user, problem=problem, language=language, result=result, page=page) logger.info("Got submissions for %s, time elasped %s", user, time.time() - start) start = time.time() q = session.query(Submission_DB) q = q.filter(Submission_DB._user == user) cond_user = self.parse(func.lower(User_DB.username), func.lower(user)) if not cond_user: q = q.join(User_DB, cond_user, aliased=True) cond_problem = self.parse(Problem_DB.code, problem) if not cond_problem: q = q.join(Problem_DB, cond_problem, aliased=True) cond_lang = self.parse(Language_DB.key, language) if not cond_lang: q = q.join(Language_DB, cond_lang, aliased=True) q = q.filter(self.parse(Submission_DB.result, result)) if a.data.total_objects == q.count(): return q.all() submission_ids = list(map(attrgetter('id'), a.data.objects)) qq = session.query(Submission_DB.id).\ filter(Submission_DB.id.in_(submission_ids)).all() qq = list(map(itemgetter(0), qq)) for submission in a.data.objects: if submission.id not in qq: session.add(Submission_DB(submission)) total_pages = a.data.total_pages apis = [] to_gather = [] for _ in range(2, total_pages + 1): page += 1 api = API() to_await = api.get_submissions(user=user, problem=problem, language=language, result=result, page=page) apis.append(api) to_gather.append(to_await) await asyncio.gather(*to_gather) for api in apis: if api.data.objects is None: continue submission_ids = list(map(attrgetter('id'), api.data.objects)) qq = session.query(Submission_DB.id).\ filter(Submission_DB.id.in_(submission_ids)).all() qq = list(map(itemgetter(0), qq)) for submission in api.data.objects: if submission.id not in qq: session.add(Submission_DB(submission)) session.commit() return q.all()