def init_title_path(apps, schema_editor): Contest = apps.get_model('clist', 'Contest') qs = Contest.objects.all() for c in tqdm.tqdm(qs.iterator(), total=qs.count()): c.slug = slug(c.title) c.title_path = c.slug.strip('-').replace('-', '.') c.save()
def standings(request, title_slug=None, contest_id=None, template='standings.html', extra_context=None): context = {} groupby = request.GET.get('groupby') if groupby == 'none': groupby = None search = request.GET.get('search') if search == '': url = request.get_full_path() url = re.sub('search=&?', '', url) url = re.sub(r'\?$', '', url) return redirect(url) orderby = request.GET.getlist('orderby') if orderby: if '--' in orderby: updated_orderby = [] else: orderby_set = set() unique_orderby = reversed([ f for k, f in [(f.lstrip('-'), f) for f in reversed(orderby)] if k not in orderby_set and not orderby_set.add(k) ]) updated_orderby = [ f for f in unique_orderby if not f.startswith('--') ] if updated_orderby != orderby: query = request.GET.copy() query.setlist('orderby', updated_orderby) return redirect(f'{request.path}?{query.urlencode()}') contests = Contest.objects to_redirect = False contest = None if contest_id is not None: contest = contests.filter(pk=contest_id).first() if title_slug is None: to_redirect = True else: if contest is None or slug(contest.title) != title_slug: contest = None title_slug += f'-{contest_id}' if contest is None and title_slug is not None: contests_iterator = contests.filter(slug=title_slug).iterator() contest = None try: contest = next(contests_iterator) another = next(contests_iterator) except StopIteration: another = None if contest is None: return HttpResponseNotFound() if another is None: to_redirect = True else: return redirect( reverse('ranking:standings_list') + f'?search=slug:{title_slug}') if contest is None: return HttpResponseNotFound() if to_redirect: query = query_transform(request) url = reverse('ranking:standings', kwargs={ 'title_slug': slug(contest.title), 'contest_id': str(contest.pk) }) if query: query = '?' + query return redirect(url + query) with_detail = request.GET.get('detail', 'true') in ['true', 'on'] if request.user.is_authenticated: coder = request.user.coder if 'detail' in request.GET: coder.settings['standings_with_detail'] = with_detail coder.save() else: with_detail = coder.settings.get('standings_with_detail', False) else: coder = None with_row_num = False contest_fields = list(contest.info.get('fields', [])) hidden_fields = list(contest.info.get('hidden_fields', [])) statistics = Statistics.objects.filter(contest=contest) options = contest.info.get('standings', {}) order = None resource_standings = contest.resource.info.get('standings', {}) order = copy.copy(options.get('order', resource_standings.get('order'))) if order: for f in order: if f.startswith('addition__') and f.split( '__', 1)[1] not in contest_fields: order = None break if order is None: order = ['place_as_int', '-solving'] # fixed fields fixed_fields = ( ('penalty', 'Penalty'), ('total_time', 'Time'), ('advanced', 'Advance'), ) fixed_fields += tuple(options.get('fixed_fields', [])) if not with_detail: fixed_fields += (('rating_change', 'Rating change'), ) statistics = statistics \ .select_related('account') \ .select_related('account__resource') \ .prefetch_related('account__coders') has_country = ('country' in contest_fields or '_countries' in contest_fields or statistics.filter(account__country__isnull=False).exists()) division = request.GET.get('division') if division == 'any': with_row_num = True if 'place_as_int' in order: order.remove('place_as_int') order.append('place_as_int') fixed_fields += (('division', 'Division'), ) if 'team_id' in contest_fields and not groupby: order.append('addition__name') statistics = statistics.distinct(*[f.lstrip('-') for f in order]) # host = resource_standings.get('account_team_resource', contest.resource.host) # account_team_resource = Resource.objects.get(host=host) # context['account_team_resource'] = account_team_resource # statistics = statistics.annotate( # accounts=RawSQL( # ''' # SELECT array_agg(array[u2.key, u3.rating::text, u3.url]) # FROM "ranking_statistics" U0 # INNER JOIN "ranking_account" U2 # ON (u0."account_id" = u2."id") # INNER JOIN "ranking_account" U3 # ON (u2."key" = u3."key" AND u3."resource_id" = %s) # WHERE ( # u0."contest_id" = %s # AND ("u0"."addition" -> 'team_id') = ("ranking_statistics"."addition" -> 'team_id') # ) # ''', # [account_team_resource.pk, contest.pk] # ) # ) order.append('pk') statistics = statistics.order_by(*order) fields = OrderedDict() for k, v in fixed_fields: if k in contest_fields: fields[k] = v n_highlight_context = _standings_highlight(statistics, options) # field to select fields_to_select_defaults = { 'rating': { 'options': ['rated', 'unrated'], 'noajax': True, 'nomultiply': True, 'nourl': True }, 'advanced': { 'options': ['true', 'false'], 'noajax': True, 'nomultiply': True }, 'highlight': { 'options': ['true', 'false'], 'noajax': True, 'nomultiply': True }, } fields_to_select = OrderedDict() map_fields_to_select = {'rating_change': 'rating'} for f in sorted(contest_fields): f = f.strip('_') if f.lower() in [ 'institution', 'room', 'affiliation', 'city', 'languages', 'school', 'class', 'job', 'region', 'rating_change', 'advanced', 'company', 'language', 'league', 'onsite', 'degree', 'university', 'list', ]: f = map_fields_to_select.get(f, f) field_to_select = fields_to_select.setdefault(f, {}) field_to_select['values'] = [ v for v in request.GET.getlist(f) if v ] field_to_select.update(fields_to_select_defaults.get(f, {})) if n_highlight_context.get('statistics_ids'): f = 'highlight' field_to_select = fields_to_select.setdefault(f, {}) field_to_select['values'] = [v for v in request.GET.getlist(f) if v] field_to_select.update(fields_to_select_defaults.get(f, {})) chats = coder.chats.all() if coder else None if chats: options_values = {c.chat_id: c.title for c in chats} fields_to_select['chat'] = { 'values': [ v for v in request.GET.getlist('chat') if v and v in options_values ], 'options': options_values, 'noajax': True, 'nogroupby': True, 'nourl': True, } hidden_fields_values = [v for v in request.GET.getlist('field') if v] for v in hidden_fields_values: if v not in hidden_fields: hidden_fields.append(v) for k in contest_fields: if (k in fields or k in [ 'problems', 'team_id', 'solved', 'hack', 'challenges', 'url', 'participant_type', 'division' ] or k == 'medal' and '_medal_title_field' in contest_fields or 'country' in k and k not in hidden_fields_values or k in ['name'] and k not in hidden_fields_values or k.startswith('_') or k in hidden_fields and k not in hidden_fields_values): continue if with_detail or k in hidden_fields_values: fields[k] = k else: hidden_fields.append(k) for k, field in fields.items(): if k != field: continue field = ' '.join(k.split('_')) if field and not field[0].isupper(): field = field.title() fields[k] = field if hidden_fields: fields_to_select['field'] = { 'values': hidden_fields_values, 'options': hidden_fields, 'noajax': True, 'nogroupby': True, 'nourl': True, 'nofilter': True, } per_page = options.get('per_page', 50) if per_page is None: per_page = 100500 elif contest.n_statistics and contest.n_statistics < 500: per_page = contest.n_statistics mod_penalty = {} first = statistics.first() if first and all('time' not in k for k in contest_fields): penalty = first.addition.get('penalty') if penalty and isinstance(penalty, int) and 'solved' not in first.addition: mod_penalty.update({'solving': first.solving, 'penalty': penalty}) params = {} problems = contest.info.get('problems', {}) if 'division' in problems: divisions_order = list( problems.get('divisions_order', sorted(contest.info['problems']['division'].keys()))) elif 'divisions_order' in contest.info: divisions_order = contest.info['divisions_order'] else: divisions_order = [] if divisions_order: divisions_order.append('any') if division not in divisions_order: division = divisions_order[0] params['division'] = division if 'division' in problems: if division == 'any': _problems = OrderedDict() for div in reversed(divisions_order): for p in problems['division'].get(div, []): k = get_problem_short(p) if k not in _problems: _problems[k] = p else: for f in 'n_accepted', 'n_teams', 'n_partial', 'n_total': if f in p: _problems[k][f] = _problems[k].get( f, 0) + p[f] problems = list(_problems.values()) else: problems = problems['division'][division] if division != 'any': statistics = statistics.filter(addition__division=division) for p in problems: if 'full_score' in p and isinstance( p['full_score'], (int, float)) and abs(p['full_score'] - 1) > 1e-9: mod_penalty = {} break last = None merge_problems = False for p in problems: if last and (last.get('full_score') or last.get('subname')) and ( 'name' in last and last.get('name') == p.get('name') or 'group' in last and last.get('group') == p.get('group')): merge_problems = True last['colspan'] = last.get('colspan', 1) + 1 p['skip'] = True else: last = p last['colspan'] = 1 # own_stat = statistics.filter(account__coders=coder).first() if coder else None # filter by search search = request.GET.get('search') if search: with_row_num = True if search.startswith('party:'): _, party_slug = search.split(':') party = get_object_or_404(Party.objects.for_user(request.user), slug=party_slug) statistics = statistics.filter( Q(account__coders__in=party.coders.all()) | Q(account__coders__in=party.admins.all()) | Q(account__coders=party.author)) else: cond = get_iregex_filter(search, 'account__key', 'addition__name', logger=request.logger) statistics = statistics.filter(cond) # filter by country countries = request.GET.getlist('country') countries = set([c for c in countries if c]) if countries: with_row_num = True cond = Q(account__country__in=countries) if 'None' in countries: cond |= Q(account__country__isnull=True) if '_countries' in contest_fields: for code in countries: name = get_country_name(code) if name: cond |= Q(addition___countries__icontains=name) statistics = statistics.filter(cond) params['countries'] = countries # filter by field to select for field, field_to_select in fields_to_select.items(): values = field_to_select.get('values') if not values or field_to_select.get('nofilter'): continue with_row_num = True filt = Q() if field == 'languages': for lang in values: if lang == 'any': filt = Q(**{'addition___languages__isnull': False}) break filt |= Q(**{'addition___languages__contains': [lang]}) elif field == 'rating': for q in values: if q not in field_to_select['options']: continue q = q == 'unrated' if q: filt |= Q(addition__rating_change__isnull=True) & Q( addition__new_rating__isnull=True) else: filt |= Q(addition__rating_change__isnull=False) | Q( addition__new_rating__isnull=False) elif field == 'advanced': for q in values: if q not in field_to_select['options']: continue filt |= Q(addition__advanced=q == 'true') elif field == 'highlight': for q in values: if q not in field_to_select['options']: continue filt = Q(pk__in=n_highlight_context.get('statistics_ids', {})) if q == 'false': filt = ~filt elif field == 'chat': for q in values: if q not in field_to_select['options']: continue chat = Chat.objects.filter(chat_id=q, is_group=True).first() if chat: filt |= Q(account__coders__in=chat.coders.all()) # subquery = Chat.objects.filter(coder=OuterRef('account__coders'), is_group=False).values('name')[:1] # statistics = statistics.annotate(chat_name=Subquery(subquery)) else: query_field = f'addition__{field}' statistics = statistics.annotate(**{ f'{query_field}_str': Cast(JSONF(query_field), models.TextField()) }) for q in values: if q == 'None': filt |= Q(**{f'{query_field}__isnull': True}) else: filt |= Q(**{f'{query_field}_str': q}) statistics = statistics.filter(filt) # groupby if groupby == 'country' or groupby in fields_to_select: statistics = statistics.order_by('pk') participants_info = n_highlight_context.get('participants_info') n_highlight = options.get('n_highlight') advanced_by_participants_info = participants_info and n_highlight and groupby != 'languages' fields = OrderedDict() fields['groupby'] = groupby.title() fields['n_accounts'] = 'Num' fields['avg_score'] = 'Avg' medals = {m['name']: m for m in options.get('medals', [])} if 'medal' in contest_fields: for medal in settings.ORDERED_MEDALS_: fields[f'n_{medal}'] = medals.get(medal, {}).get( 'value', medal[0].upper()) if 'advanced' in contest_fields or advanced_by_participants_info: fields['n_advanced'] = 'Adv' orderby = [f for f in orderby if f.lstrip('-') in fields ] or ['-n_accounts', '-avg_score'] if groupby == 'languages': _, before_params = statistics.query.sql_with_params() querysets = [] for problem in problems: key = get_problem_short(problem) field = f'addition__problems__{key}__language' score = f'addition__problems__{key}__result' qs = statistics \ .filter(**{f'{field}__isnull': False, f'{score}__isnull': False}) \ .annotate(language=Cast(JSONF(field), models.TextField())) \ .annotate(score=Case( When(**{f'{score}__startswith': '+'}, then=1), When(**{f'{score}__startswith': '-'}, then=0), When(**{f'{score}__startswith': '?'}, then=0), default=Cast(JSONF(score), models.FloatField()), output_field=models.FloatField(), )) \ .annotate(sid=F('pk')) querysets.append(qs) merge_statistics = querysets[0].union(*querysets[1:], all=True) language_query, language_params = merge_statistics.query.sql_with_params( ) field = 'solving' statistics = statistics.annotate(groupby=F(field)) elif groupby == 'rating': statistics = statistics.annotate(groupby=Case( When(addition__rating_change__isnull=False, then=Value('Rated')), default=Value('Unrated'), output_field=models.TextField(), )) elif groupby == 'country': if '_countries' in contest_fields: statistics = statistics.annotate(country=RawSQL( '''json_array_elements((("addition" ->> '_countries'))::json)::jsonb''', [])) field = 'country' else: field = 'account__country' statistics = statistics.annotate(groupby=F(field)) else: field = f'addition__{groupby}' types = contest.info.get('fields_types', {}).get(groupby, []) if 'int' in types: field_type = models.IntegerField() elif 'float' in types: field_type = models.FloatField() else: field_type = models.TextField() statistics = statistics.annotate( groupby=Cast(JSONF(field), field_type)) statistics = statistics.order_by('groupby') statistics = statistics.values('groupby') statistics = statistics.annotate(n_accounts=Count('id')) statistics = statistics.annotate(avg_score=Avg('solving')) if 'medal' in contest_fields: for medal in settings.ORDERED_MEDALS_: n_medal = f'n_{medal}' statistics = statistics.annotate( **{ f'{n_medal}': Count(Case(When(addition__medal__iexact=medal, then=1))) }) if 'advanced' in contest_fields: statistics = statistics.annotate(n_advanced=Count( Case( When(addition__advanced=True, then=1), When(~Q(addition__advanced=False) & ~Q(addition__advanced=''), then=1), ))) elif advanced_by_participants_info: pks = list() for pk, info in participants_info.items(): if 'n' not in info or info['n'] > info.get( 'n_highlight', n_highlight): continue pks.append(pk) statistics = statistics.annotate( n_advanced=Count(Case(When(pk__in=set(pks), then=1)))) statistics = statistics.order_by(*orderby) if groupby == 'languages': query, sql_params = statistics.query.sql_with_params() query = query.replace( f'"ranking_statistics"."{field}" AS "groupby"', '"language" AS "groupby"') query = query.replace(f'GROUP BY "ranking_statistics"."{field}"', 'GROUP BY "language"') query = query.replace('"ranking_statistics".', '') query = query.replace('AVG("solving") AS "avg_score"', 'AVG("score") AS "avg_score"') query = query.replace('COUNT("id") AS "n_accounts"', 'COUNT("sid") AS "n_accounts"') query = re.sub('FROM "ranking_statistics".*GROUP BY', f'FROM ({language_query}) t1 GROUP BY', query) sql_params = sql_params[:-len(before_params)] + language_params with connection.cursor() as cursor: cursor.execute(query, sql_params) columns = [col[0] for col in cursor.description] statistics = [ dict(zip(columns, row)) for row in cursor.fetchall() ] statistics = ListAsQueryset(statistics) problems = [] labels_groupby = { 'n_accounts': 'Number of participants', 'avg_score': 'Average score', 'n_advanced': 'Number of advanced', } for medal in settings.ORDERED_MEDALS_: labels_groupby[f'n_{medal}'] = 'Number of ' + medals.get( medal, {}).get('value', medal) num_rows_groupby = statistics.count() map_colors_groupby = { s['groupby']: idx for idx, s in enumerate(statistics) } else: groupby = 'none' labels_groupby = None num_rows_groupby = None map_colors_groupby = None my_statistics = [] if groupby == 'none' and coder: statistics = statistics.annotate( my_stat=SubqueryExists('account__coders', filter=Q(coder=coder))) my_statistics = statistics.filter(account__coders=coder).extra( select={'floating': True}) context.update({ 'standings_options': options, 'mod_penalty': mod_penalty, 'colored_by_group_score': mod_penalty or options.get('colored_by_group_score'), 'contest': contest, 'statistics': statistics, 'my_statistics': my_statistics, 'problems': problems, 'params': params, 'fields': fields, 'fields_types': contest.info.get('fields_types', {}), 'divisions_order': divisions_order, 'has_country': has_country, 'per_page': per_page, 'with_row_num': with_row_num, 'merge_problems': merge_problems, 'fields_to_select': fields_to_select, 'truncatechars_name_problem': 10 * (2 if merge_problems else 1), 'with_detail': with_detail, 'groupby': groupby, 'pie_limit_rows_groupby': 50, 'labels_groupby': labels_groupby, 'num_rows_groupby': num_rows_groupby, 'map_colors_groupby': map_colors_groupby, 'advance': contest.info.get('advance'), 'timezone': get_timezone(request), 'timeformat': get_timeformat(request), 'with_neighbors': request.GET.get('neighbors') == 'on', 'with_table_inner_scroll': not request.user_agent.is_mobile, }) context.update(n_highlight_context) if extra_context is not None: context.update(extra_context) return render(request, template, context)
def location(self, contest): return reverse('ranking:standings', args=(slug(contest.title), contest.pk))
def update(self): stage = self.contest contests = Contest.objects.filter( resource=self.contest.resource, start_time__gte=self.contest.start_time, end_time__lte=self.contest.end_time, **self.filter_params, ).exclude(pk=self.contest.pk) contests = contests.order_by('start_time') placing = self.score_params.get('place') n_best = self.score_params.get('n_best') fields = self.score_params.get('fields', []) detail_problems = self.score_params.get('detail_problems') order_by = self.score_params['order_by'] advances = self.score_params.get('advances', {}) results = collections.defaultdict(collections.OrderedDict) problems_infos = collections.OrderedDict() for idx, contest in enumerate(tqdm.tqdm( contests, desc=f'getting contests for stage {stage}'), start=1): info = { 'code': str(contest.pk), 'name': contest.title, 'url': reverse('ranking:standings', kwargs={ 'title_slug': slug(contest.title), 'contest_id': str(contest.pk) }), 'n_accepted': 0, 'n_teams': 0, } problems = contest.info.get('problems', []) if not detail_problems: full_score = None if placing: if 'division' in placing: full_score = max([ max(p.values()) for p in placing['division'].values() ]) else: full_score = max(placing.values()) elif 'division' in problems: full_scores = [] for ps in problems['division'].values(): full = 0 for problem in ps: full += problem.get('full_score', 1) full_scores.append(full) info['full_score'] = max(full_scores) else: full_score = 0 for problem in problems: full_score += problem.get('full_score', 1) if full_score is not None: info['full_score'] = full_score if self.score_params.get('abbreviation_problem_name'): info['short'] = ''.join( re.findall(r'(\b[A-Z]|[0-9])', contest.title)) problems_infos[str(contest.pk)] = info else: for problem in problems: problem = dict(problem) add_prefix_to_problem_key(problem, f'{idx}.') problem['group'] = info['name'] problem['url'] = info['url'] problems_infos[get_problem_key(problem)] = problem exclude_advances = {} if advances and advances.get('exclude_stages'): qs = Statistics.objects \ .filter(contest__stage__in=advances['exclude_stages'], addition___advance__isnull=False) \ .values('account__key', 'addition___advance', 'contest__title') \ .order_by('contest__end_time') for r in qs: d = r['addition___advance'] if 'contest' not in d: d['contest'] = r['contest__title'] exclude_advances[r['account__key']] = d statistics = Statistics.objects.select_related('account') filter_statistics = self.score_params.get('filter_statistics') if filter_statistics: statistics = statistics.filter(**filter_statistics) def get_placing(placing, stat): return placing['division'][ stat. addition['division']] if 'division' in placing else placing account_keys = dict() total = statistics.filter(contest__in=contests).count() with tqdm.tqdm(total=total, desc=f'getting statistics for stage {stage}') as pbar: for idx, contest in enumerate(contests, start=1): problem_info_key = str(contest.pk) problem_key = get_problem_key(problems_infos[problem_info_key]) pbar.set_postfix(contest=contest) stats = statistics.filter(contest_id=contest.pk) if placing: placing_scores = deepcopy(placing) n_rows = 0 for s in stats: n_rows += 1 placing_ = get_placing(placing_scores, s) key = str(s.place_as_int) if key in placing_: placing_.setdefault('scores', {}) placing_['scores'][key] = placing_.pop(key) scores = [] for place in reversed(range(1, n_rows + 1)): placing_ = get_placing(placing_scores, s) key = str(place) if key in placing_: scores.append(placing_.pop(key)) else: if scores: placing_['scores'][key] += sum(scores) placing_['scores'][key] /= len(scores) + 1 scores = [] for s in stats: if not detail_problems: problems_infos[problem_info_key]['n_teams'] += 1 if s.solving < 1e-9: score = 0 else: if not detail_problems: problems_infos[problem_info_key]['n_accepted'] += 1 if placing: placing_ = get_placing(placing_scores, s) score = placing_['scores'].get( str(s.place_as_int), placing_.get('default')) if score is None: continue else: score = s.solving row = results[s.account] row['member'] = s.account account_keys[s.account.key] = s.account problems = row.setdefault('problems', {}) if detail_problems: for key, problem in s.addition.get('problems', {}).items(): problems[f'{idx}.' + key] = problem else: problem = problems.setdefault(problem_key, {}) problem['result'] = score url = s.addition.get('url') if url: problem['url'] = url if n_best and not detail_problems: row.setdefault('scores', []).append((score, problem)) else: row['score'] = row.get('score', 0) + score field_values = {} for field in fields: inp = field['field'] out = field.get('out', inp) if 'type' in field: continue if field.get('first' ) and out in row or inp not in s.addition: continue if field.get('safe'): val = s.addition[inp] else: val = ast.literal_eval(str(s.addition[inp])) field_values[out] = val if field.get('accumulate'): val = round( val + ast.literal_eval(str(row.get(out, 0))), 2) row[out] = val if 'solved' in s.addition: solved = row.setdefault('solved', {}) for k, v in s.addition['solved'].items(): solved[k] = solved.get(k, 0) + v if 'status' in self.score_params: field = self.score_params['status'] problem['status'] = field_values.get( field, row.get(field)) else: for field in order_by: field = field.lstrip('-') if field in ['score', 'rating']: continue status = field_values.get(field, row.get(field)) if status is None: continue problem['status'] = status break pbar.update() if self.score_params.get('writers_proportionally_score'): total = sum( [len(contest.info.get('writers', [])) for contest in contests]) with tqdm.tqdm(total=total, desc=f'getting writers for stage {stage}') as pbar: writers = set() for contest in contests: problem_info_key = str(contest.pk) problem_key = get_problem_key( problems_infos[problem_info_key]) for writer in contest.info.get('writers', []): if writer in account_keys: account = account_keys[writer] else: account = Account.objects.filter( resource_id=contest.resource_id, key__iexact=writer).first() pbar.update() if not account: continue writers.add(account) row = results[account] row['member'] = account row.setdefault('score', 0) row.setdefault('writer', 0) row['writer'] += 1 problems = row.setdefault('problems', {}) problem = problems.setdefault(problem_key, {}) problem['status'] = 'W' n_contests = len(contests) for account in writers: row = results[account] if n_contests == row['writer'] or 'score' not in row: continue row['score'] = row['score'] / (n_contests - row['writer']) * n_contests for field in fields: t = field.get('type') if t == 'points_for_common_problems': group = field['group'] inp = field['field'] out = field.get('out', inp) common_problems = dict() for account, row in results.items(): problems = set(row['problems'].keys()) key = row[group] common_problems[ key] = problems if key not in common_problems else ( problems & common_problems[key]) for account, row in results.items(): key = row[group] problems = common_problems[key] value = 0 for k in problems: value += float(row['problems'].get(k, {}).get(inp, 0)) for k, v in row['problems'].items(): if k not in problems: v['status_tag'] = 'strike' row[out] = round(value, 2) results = list(results.values()) if n_best: for row in results: scores = row.pop('scores') for index, (score, problem) in enumerate( sorted(scores, key=lambda s: s[0], reverse=True)): if index < n_best: row['score'] = row.get('score', 0) + score else: problem['status'] = problem.pop('result') results = [r for r in results if r['score'] > 1e-9 or r.get('writer')] results = sorted( results, key=lambda r: tuple( r.get(k.lstrip('-'), 0) * (-1 if k.startswith('-') else 1) for k in order_by), reverse=True, ) with transaction.atomic(): fields_set = set() fields = list() pks = set() placing_infos = {} score_advance = None place_advance = 0 for row in tqdm.tqdm(results, desc=f'update statistics for stage {stage}'): division = row.get('division', 'none') placing_info = placing_infos.setdefault(division, {}) placing_info['index'] = placing_info.get('index', 0) + 1 curr_score = tuple(row.get(k.lstrip('-'), 0) for k in order_by) if curr_score != placing_info.get('last_score'): placing_info['last_score'] = curr_score placing_info['place'] = placing_info['index'] if advances and ('divisions' not in advances or division in advances['divisions']): tmp = score_advance, place_advance if curr_score != score_advance: score_advance = curr_score place_advance += 1 for advance in advances.get('options', []): handle = row['member'].key if handle in exclude_advances and advance[ 'next'] == exclude_advances[handle]['next']: advance = exclude_advances[handle] if 'class' in advance and not advance[ 'class'].startswith('text-'): advance['class'] = f'text-{advance["class"]}' row['_advance'] = advance break if 'places' in advance and place_advance in advance[ 'places']: row['_advance'] = advance for field in advance.get('inplace_fields', []): row[field] = advance[field] tmp = None break if tmp is not None: score_advance, place_advance = tmp account = row.pop('member') solving = row.pop('score') stat, created = Statistics.objects.update_or_create( account=account, contest=stage, defaults={ 'place': str(placing_info['place']), 'place_as_int': placing_info['place'], 'solving': solving, 'addition': row, }, ) pks.add(stat.pk) for k in row.keys(): if k not in fields_set: fields_set.add(k) fields.append(k) stage.statistics_set.exclude(pk__in=pks).delete() stage.info['fields'] = list(fields) standings_info = self.score_params.get('info', {}) standings_info['fixed_fields'] = [(f.lstrip('-'), f.lstrip('-')) for f in order_by] stage.info['standings'] = standings_info stage.info['problems'] = list(problems_infos.values()) stage.save()
def save(self, *args, **kwargs): if self.duration_in_secs is None: self.duration_in_secs = (self.end_time - self.start_time).total_seconds() self.slug = slug(self.title).strip('-') self.title_path = self.slug.replace('-', '.') return super(Contest, self).save(*args, **kwargs)
def actual_url(self): if self.n_statistics: return reverse('ranking:standings', args=(slug(self.title), self.pk)) if self.standings_url: return self.standings_url return self.url
def get_events(request): if request.user.is_authenticated: coder = request.user.coder else: coder = None categories = request.POST.getlist('categories') ignore_filters = request.POST.getlist('ignore_filters') has_filter = False referer = request.META.get('HTTP_REFERER') if referer: parsed = urlparse(referer) query_dict = parse_qs(parsed.query) as_coder = query_dict.get('as_coder') if as_coder and request.user.has_perm('as_coder'): coder = Coder.objects.get(user__username=as_coder[0]) has_filter = 'filter' in query_dict categories = query_dict.get('filter', categories) tzname = get_timezone(request) offset = get_timezone_offset(tzname) query = Q() if coder: query = coder.get_contest_filter(categories, ignore_filters) elif has_filter: query = Coder.get_contest_filter(None, categories, ignore_filters) if not coder or coder.settings.get('calendar_filter_long', True): if categories == ['calendar'] and '0' not in ignore_filters: query &= Q(duration_in_secs__lt=timedelta(days=1).total_seconds()) start_time = arrow.get(request.POST.get('start', timezone.now())).datetime end_time = arrow.get( request.POST.get('end', timezone.now() + timedelta(days=31))).datetime query = query & Q(end_time__gte=start_time) & Q(start_time__lte=end_time) search_query = request.POST.get('search_query', None) if search_query: search_query_re = verify_regex(search_query) query &= Q(host__iregex=search_query_re) | Q( title__iregex=search_query_re) party_slug = request.POST.get('party') if party_slug: party = get_object_or_404(Party.objects.for_user(request.user), slug=party_slug) query = Q(rating__party=party) & query contests = Contest.objects if party_slug else Contest.visible contests = contests.select_related('resource') contests = contests.annotate(has_statistics=Exists('statistics')) try: result = [] for contest in contests.filter(query): c = { 'id': contest.pk, 'title': contest.title, 'host': contest.host, 'url': (reverse('ranking:standings', args=(slug(contest.title), contest.pk)) if contest.has_statistics else contest.standings_url or contest.url), 'start': (contest.start_time + timedelta(minutes=offset)).strftime("%Y-%m-%dT%H:%M:%S"), 'end': (contest.end_time + timedelta(minutes=offset)).strftime("%Y-%m-%dT%H:%M:%S"), 'countdown': contest.next_time, 'hr_duration': contest.hr_duration, 'color': contest.resource.color, 'icon': contest.resource.icon, } result.append(c) except Exception as e: return JsonResponse( {'message': f'query = `{search_query}`, error = {e}'}, safe=False, status=400) return JsonResponse(result, safe=False)
def update(self): stage = self.contest contests = Contest.objects.filter( resource=self.contest.resource, start_time__gte=self.contest.start_time, end_time__lte=self.contest.end_time, **self.filter_params, ).exclude(pk=self.contest.pk) contests = contests.order_by('start_time') placing = self.score_params.get('place') n_best = self.score_params.get('n_best') fields = self.score_params.get('fields', []) order_by = self.score_params['order_by'] advances = self.score_params.get('advances', {}) results = collections.defaultdict(collections.OrderedDict) problems_infos = collections.OrderedDict() for contest in tqdm.tqdm(contests, desc=f'getting contests for stage {stage}'): info = { 'code': str(contest.pk), 'name': contest.title, 'url': reverse('ranking:standings', kwargs={ 'title_slug': slug(contest.title), 'contest_id': str(contest.pk) }), 'n_accepted': 0, 'n_teams': 0, } problems = contest.info.get('problems', []) full_score = None if placing: if 'division' in placing: full_score = max([ max(p.values()) for p in placing['division'].values() ]) else: full_score = max(placing.values()) elif 'division' in problems: full_scores = [] for ps in problems['division'].values(): full = 0 for problem in ps: full += problem.get('full_score', 1) full_scores.append(full) info['full_score'] = max(full_scores) else: full_score = 0 for problem in problems: full_score += problem.get('full_score', 1) if full_score is not None: info['full_score'] = full_score problems_infos[contest.pk] = info exclude_advances = {} if advances and advances.get('exclude_stages'): qs = Statistics.objects \ .filter(contest__stage__in=advances['exclude_stages'], addition___advance__isnull=False) \ .values('account__key', 'addition___advance', 'contest__title') \ .order_by('contest__end_time') for r in qs: d = r['addition___advance'] d['contest'] = r['contest__title'] exclude_advances[r['account__key']] = d statistics = Statistics.objects.select_related('account') filter_statistics = self.score_params.get('filter_statistics') if filter_statistics: statistics = statistics.filter(**filter_statistics) total = statistics.filter(contest__in=contests).count() with tqdm.tqdm(total=total, desc=f'getting statistics for stage {stage}') as pbar: for contest in contests: pbar.set_postfix(contest=contest) stats = statistics.filter(contest_id=contest.pk) for s in stats.iterator(): row = results[s.account] row['member'] = s.account problems_infos[contest.pk]['n_teams'] += 1 if s.solving < 1e-9: score = 0 else: problems_infos[contest.pk]['n_accepted'] += 1 if placing: placing_ = placing['division'][s.addition[ 'division']] if 'division' in placing else placing score = placing_.get(str(s.place_as_int), placing_['default']) else: score = s.solving problems = row.setdefault('problems', {}) problem = problems.setdefault(str(contest.pk), {}) problem['result'] = score url = s.addition.get('url') if url: problem['url'] = url if n_best: row.setdefault('scores', []).append((score, problem)) else: row['score'] = row.get('score', 0) + score field_values = {} for field in fields: inp = field['field'] out = field.get('out', inp) if 'type' in field: continue if field.get('first' ) and out in row or inp not in s.addition: continue val = ast.literal_eval(str(s.addition[inp])) field_values[out] = val if field.get('accumulate'): val = round( val + ast.literal_eval(str(row.get(out, 0))), 2) row[out] = val if 'solved' in s.addition: solved = row.setdefault('solved', {}) for k, v in s.addition['solved'].items(): solved[k] = solved.get(k, 0) + v if 'status' in self.score_params: field = self.score_params['status'] problem['status'] = field_values.get( field, row.get(field)) else: for field in order_by: field = field.lstrip('-') if field == 'score': continue status = field_values.get(field, row.get(field)) if status is None: continue problem['status'] = status break pbar.update() for field in fields: t = field.get('type') if t == 'points_for_common_problems': group = field['group'] inp = field['field'] out = field.get('out', inp) common_problems = dict() for account, row in results.items(): problems = set(row['problems'].keys()) key = row[group] common_problems[ key] = problems if key not in common_problems else ( problems & common_problems[key]) for account, row in results.items(): key = row[group] problems = common_problems[key] value = 0 for k in problems: value += float(row['problems'].get(k, {}).get(inp, 0)) for k, v in row['problems'].items(): if k not in problems: v['status_tag'] = 'strike' row[out] = round(value, 2) results = list(results.values()) if n_best: for row in results: scores = row.pop('scores') for index, (score, problem) in enumerate( sorted(scores, key=lambda s: s[0], reverse=True)): if index < n_best: row['score'] = row.get('score', 0) + score else: problem['status'] = problem.pop('result') results = [r for r in results if r['score'] > 1e-9] results = sorted( results, key=lambda r: tuple(r[k.lstrip('-')] * (-1 if k.startswith('-') else 1) for k in order_by), reverse=True, ) with transaction.atomic(): fields_set = set() fields = list() pks = set() last_score = None place = None score_advance = None place_advance = 0 for index, row in enumerate(tqdm.tqdm( results, desc=f'update statistics for stage {stage}'), start=1): curr_score = tuple(row[k.lstrip('-')] for k in order_by) if curr_score != last_score: last_score = curr_score place = index if advances: tmp = score_advance, place_advance if curr_score != score_advance: score_advance = curr_score place_advance += 1 for advance in advances.get('options', []): handle = row['member'].key if handle in exclude_advances and advance[ 'next'] == exclude_advances[handle]['next']: advance = exclude_advances[handle] advance.pop('class', None) row['_advance'] = advance break if 'places' in advance and place_advance in advance[ 'places']: row['_advance'] = advance tmp = None break if tmp is not None: score_advance, place_advance = tmp account = row.pop('member') solving = row.pop('score') stat, created = Statistics.objects.update_or_create( account=account, contest=stage, defaults={ 'place': str(place), 'place_as_int': place, 'solving': solving, 'addition': row, }, ) pks.add(stat.pk) for k in row.keys(): if k not in fields_set: fields_set.add(k) fields.append(k) stage.statistics_set.exclude(pk__in=pks).delete() stage.info['fields'] = list(fields) standings_info = self.score_params.get('info', {}) standings_info['fixed_fields'] = [(f.lstrip('-'), f.lstrip('-')) for f in order_by] stage.info['standings'] = standings_info stage.info['problems'] = list(problems_infos.values()) stage.save()
def standings(request, title_slug, contest_id, template='standings.html', extra_context=None): groupby = request.GET.get('groupby') search = request.GET.get('search') if search == '': url = request.get_full_path() url = re.sub('search=&?', '', url) url = re.sub(r'\?$', '', url) return redirect(url) orderby = request.GET.getlist('orderby') if orderby: if '--' in orderby: updated_orderby = [] else: orderby_set = set() unique_orderby = reversed([ f for k, f in [(f.lstrip('-'), f) for f in reversed(orderby)] if k not in orderby_set and not orderby_set.add(k) ]) updated_orderby = [ f for f in unique_orderby if not f.startswith('--') ] if updated_orderby != orderby: query = request.GET.copy() query.setlist('orderby', updated_orderby) return redirect(f'{request.path}?{query.urlencode()}') contest = get_object_or_404(Contest.objects.select_related('resource'), pk=contest_id) if slug(contest.title) != title_slug: return HttpResponseNotFound(f'Not found {slug(contest.title)} slug') contest_fields = contest.info.get('fields', []) statistics = Statistics.objects.filter(contest=contest) order = None resource_standings = contest.resource.info.get('standings', {}) order = copy.copy(resource_standings.get('order')) if order: for f in order: if f.startswith('addition__') and f.split( '__', 1)[1] not in contest_fields: order = None break if order is None: order = ['place_as_int', '-solving'] options = contest.info.get('standings', {}) # fixed fields fixed_fields = ( ('penalty', 'Penalty'), ('total_time', 'Time'), ('advanced', 'Advanced')) + tuple(options.get('fixed_fields', [])) statistics = statistics \ .select_related('account') \ .select_related('account__resource') \ .prefetch_related('account__coders') has_country = 'country' in contest_fields or statistics.filter( account__country__isnull=False).exists() division = request.GET.get('division') if division == 'any': if 'place_as_int' in order: order.remove('place_as_int') order.append('place_as_int') fixed_fields += (('division', 'Division'), ) if 'team_id' in contest_fields and not groupby: order.append('addition__name') statistics = statistics.distinct(*[f.lstrip('-') for f in order]) order.append('pk') statistics = statistics.order_by(*order) fields = OrderedDict() for k, v in fixed_fields: if k in contest_fields: fields[k] = v # field to select fields_to_select = {} for f in contest_fields: f = f.strip('_') if f.lower() in [ 'institution', 'room', 'affiliation', 'city', 'languages' ]: fields_to_select[f] = request.GET.getlist(f) with_detail = request.GET.get('detail') in ['true', 'on'] if request.user.is_authenticated: coder = request.user.coder if 'detail' in request.GET: coder.settings['standings_with_detail'] = with_detail coder.save() else: with_detail = coder.settings.get('standings_with_detail', False) else: coder = None if with_detail: for k in contest_fields: if (k not in fields and k not in [ 'problems', 'name', 'team_id', 'solved', 'hack', 'challenges', 'url', 'participant_type', 'division' ] and 'country' not in k and not k.startswith('_')): field = ' '.join(k.split('_')) if field and not field[0].isupper(): field = field.title() fields[k] = field per_page = options.get('per_page', 200) if per_page is None: per_page = 100500 data_1st_u = options.get('1st_u') participants_info = {} if data_1st_u: seen = {} last_hl = None for s in statistics: match = re.search(data_1st_u['regex'], s.account.key) k = match.group('key') solving = s.solving penalty = s.addition.get('penalty') info = participants_info.setdefault(s.id, {}) info['search'] = rf'^{k}' if k in seen or last_hl: p_info = participants_info.get(seen.get(k)) if (not p_info or last_hl and (-last_hl['solving'], last_hl['penalty']) < (-p_info['solving'], p_info['penalty'])): p_info = last_hl info.update({ 't_solving': p_info['solving'] - solving, 't_penalty': p_info['penalty'] - penalty if penalty is not None else None, }) if k not in seen: seen[k] = s.id info.update({ 'n': len(seen), 'solving': solving, 'penalty': penalty }) if len(seen) == options.get('n_highlight'): last_hl = info elif 'n_highlight' in options: for idx, s in enumerate(statistics[:options['n_highlight']], 1): participants_info[s.id] = {'n': idx} medals = options.get('medals') if medals: names = [m['name'] for m in medals] counts = [m['count'] for m in medals] medals = list(zip(names, accumulate(counts))) mod_penalty = {} first = statistics.first() if first and all('time' not in k for k in contest_fields): penalty = first.addition.get('penalty') if penalty and isinstance(penalty, int) and 'solved' not in first.addition: mod_penalty.update({'solving': first.solving, 'penalty': penalty}) params = {} problems = contest.info.get('problems', {}) if 'division' in problems: divisions_order = list( problems.get('divisions_order', sorted(contest.info['problems']['division'].keys()))) divisions_order.append('any') if division not in divisions_order: division = divisions_order[0] params['division'] = division if division == 'any': _problems = OrderedDict() for div in reversed(divisions_order): for p in problems['division'].get(div, []): k = get_problem_key(p) if k not in _problems: _problems[k] = p else: for f in 'n_accepted', 'n_teams': if f in p: _problems[k][f] = _problems[k].get(f, 0) + p[f] problems = list(_problems.values()) else: statistics = statistics.filter(addition__division=division) problems = problems['division'][division] else: divisions_order = [] for p in problems: if 'full_score' in p and isinstance( p['full_score'], (int, float)) and abs(p['full_score'] - 1) > 1e-9: mod_penalty = {} break last = None merge_problems = False for p in problems: if last and 'name' in last and last.get('name') == p.get( 'name') and last.get('full_score'): merge_problems = True last['colspan'] = last.get('colspan', 1) + 1 p['skip'] = True else: last = p # own_stat = statistics.filter(account__coders=coder).first() if coder else None # filter by search search = request.GET.get('search') if search: if search.startswith('party:'): _, party_slug = search.split(':') party = get_object_or_404(Party.objects.for_user(request.user), slug=party_slug) statistics = statistics.filter( Q(account__coders__in=party.coders.all()) | Q(account__coders__in=party.admins.all()) | Q(account__coders=party.author)) else: search_re = verify_regex(search) statistics = statistics.filter( Q(account__key__iregex=search_re) | Q(addition__name__iregex=search_re)) # filter by country countries = request.GET.getlist('country') countries = set([c for c in countries if c]) if countries: cond = Q(account__country__in=countries) if 'None' in countries: cond |= Q(account__country__isnull=True) statistics = statistics.filter(cond) params['countries'] = countries # filter by field to select for field, values in fields_to_select.items(): if not values: continue filt = Q() if field == 'languages': for lang in values: filt |= Q(**{f'addition___languages__contains': [lang]}) else: for q in values: if q == 'None': filt |= Q(**{f'addition__{field}__isnull': True}) else: filt |= Q(**{f'addition__{field}': q}) statistics = statistics.filter(filt) # groupby if groupby == 'country' or groupby in fields_to_select: fields = OrderedDict() fields['groupby'] = groupby fields['n_accounts'] = 'num' fields['avg_score'] = 'avg' if 'medal' in contest_fields: for medal in settings.ORDERED_MEDALS_: fields[f'n_{medal}'] = medal[0] if 'advanced' in contest_fields: fields['n_advanced'] = 'adv' orderby = [f for f in orderby if f.lstrip('-') in fields ] or ['-n_accounts', '-avg_score'] if groupby == 'languages': _, before_params = statistics.query.sql_with_params() querysets = [] for problem in problems: key = get_problem_key(problem) field = f'addition__problems__{key}__language' score = f'addition__problems__{key}__result' qs = statistics \ .filter(**{f'{field}__isnull': False}) \ .annotate(language=Cast(JSONF(field), models.TextField())) \ .annotate(score=Case( When(**{f'{score}__startswith': '+'}, then=1), When(**{f'{score}__startswith': '-'}, then=0), When(**{f'{score}__startswith': '?'}, then=0), default=Cast(JSONF(score), models.FloatField()), output_field=models.FloatField(), )) \ .annotate(sid=F('pk')) querysets.append(qs) merge_statistics = querysets[0].union(*querysets[1:], all=True) language_query, language_params = merge_statistics.query.sql_with_params( ) field = 'solving' statistics = statistics.annotate(groupby=F(field)) elif groupby == 'country': field = 'account__country' statistics = statistics.annotate(groupby=F(field)) else: field = f'addition__{groupby}' statistics = statistics.annotate( groupby=Cast(JSONF(field), models.TextField())) statistics = statistics.order_by('groupby') statistics = statistics.values('groupby') statistics = statistics.annotate(n_accounts=Count('id')) statistics = statistics.annotate(avg_score=Avg('solving')) if 'medal' in contest_fields: for medal in settings.ORDERED_MEDALS_: n_medal = f'n_{medal}' statistics = statistics.annotate( **{ f'{n_medal}': Count(Case(When(addition__medal__iexact=medal, then=1))) }) if 'advanced' in contest_fields: statistics = statistics.annotate( n_advanced=Count(Case(When(addition__advanced=True, then=1)))) statistics = statistics.order_by(*orderby) if groupby == 'languages': query, params = statistics.query.sql_with_params() query = query.replace( f'"ranking_statistics"."{field}" AS "groupby"', '"language" AS "groupby"') query = query.replace(f'GROUP BY "ranking_statistics"."{field}"', 'GROUP BY "language"') query = query.replace(f'"ranking_statistics".', '') query = query.replace(f'AVG("solving") AS "avg_score"', 'AVG("score") AS "avg_score"') query = query.replace(f'COUNT("id") AS "n_accounts"', 'COUNT("sid") AS "n_accounts"') query = re.sub('FROM "ranking_statistics".*GROUP BY', f'FROM ({language_query}) t1 GROUP BY', query) params = params[:-len(before_params)] + language_params with connection.cursor() as cursor: cursor.execute(query, params) columns = [col[0] for col in cursor.description] statistics = [ dict(zip(columns, row)) for row in cursor.fetchall() ] statistics = ListAsQueryset(statistics) problems = [] labels_groupby = { 'n_accounts': 'Number of participants', 'avg_score': 'Average score', 'n_gold': 'Number of gold', 'n_silver': 'Number of silver', 'n_bronze': 'Number of bronze', 'n_advanced': 'Number of advanced', } else: groupby = None labels_groupby = None context = { 'data_1st_u': data_1st_u, 'participants_info': participants_info, 'standings_options': options, 'medals': medals, 'mod_penalty': mod_penalty, 'contest': contest, 'statistics': statistics, 'problems': problems, 'params': params, 'fields': fields, 'divisions_order': divisions_order, 'has_country': has_country, 'per_page': per_page, 'with_row_num': bool(search or countries), 'merge_problems': merge_problems, 'fields_to_select': fields_to_select, 'truncatechars_name_problem': 10 * (2 if merge_problems else 1), 'with_detail': with_detail, 'groupby': groupby, 'pie_limit_rows_groupby': 50, 'labels_groupby': labels_groupby, 'num_rows': statistics.count(), 'advance': contest.info.get('advance'), } if extra_context is not None: context.update(extra_context) return render(request, template, context)
def save(self, *args, **kwargs): self.slug = slug(self.name) return super().save(*args, **kwargs)