def solutions(request, sid, problem_key):
statistic = get_object_or_404(Statistics.objects.select_related(
'account', 'contest', 'contest__resource'),
pk=sid)
problems = statistic.addition.get('problems', {})
if problem_key not in problems:
return HttpResponseNotFound()
contest_problems = statistic.contest.info['problems']
if 'division' in contest_problems:
contest_problems = contest_problems['division'][
statistic.addition['division']]
for problem in contest_problems:
if get_problem_short(problem) == problem_key:
break
else:
problem = None
stat = problems[problem_key]
if 'solution' not in stat:
if stat.get('external_solution'):
plugin = statistic.contest.resource.plugin
try:
source_code = plugin.Statistic.get_source_code(
statistic.contest, stat)
stat.update(source_code)
except (NotImplementedError, ExceptionParseStandings,
FailOnGetResponse):
return HttpResponseNotFound()
else:
return HttpResponseNotFound()
return render(
request,
'solution-source.html' if request.is_ajax() else 'solution.html', {
'is_modal': request.is_ajax(),
'statistic': statistic,
'account': statistic.account,
'contest': statistic.contest,
'problem': problem,
'stat': stat,
'fields': ['time', 'status', 'language'],
})
def standings(request,
title_slug=None,
contest_id=None,
template='standings.html',
extra_context=None):
context = {}
groupby = request.GET.get('groupby')
if groupby == 'none':
groupby = None
search = request.GET.get('search')
if search == '':
url = request.get_full_path()
url = re.sub('search=&?', '', url)
url = re.sub(r'\?$', '', url)
return redirect(url)
orderby = request.GET.getlist('orderby')
if orderby:
if '--' in orderby:
updated_orderby = []
else:
orderby_set = set()
unique_orderby = reversed([
f for k, f in [(f.lstrip('-'), f) for f in reversed(orderby)]
if k not in orderby_set and not orderby_set.add(k)
])
updated_orderby = [
f for f in unique_orderby if not f.startswith('--')
]
if updated_orderby != orderby:
query = request.GET.copy()
query.setlist('orderby', updated_orderby)
return redirect(f'{request.path}?{query.urlencode()}')
contests = Contest.objects
to_redirect = False
contest = None
if contest_id is not None:
contest = contests.filter(pk=contest_id).first()
if title_slug is None:
to_redirect = True
else:
if contest is None or slug(contest.title) != title_slug:
contest = None
title_slug += f'-{contest_id}'
if contest is None and title_slug is not None:
contests_iterator = contests.filter(slug=title_slug).iterator()
contest = None
try:
contest = next(contests_iterator)
another = next(contests_iterator)
except StopIteration:
another = None
if contest is None:
return HttpResponseNotFound()
if another is None:
to_redirect = True
else:
return redirect(
reverse('ranking:standings_list') +
f'?search=slug:{title_slug}')
if contest is None:
return HttpResponseNotFound()
if to_redirect:
query = query_transform(request)
url = reverse('ranking:standings',
kwargs={
'title_slug': slug(contest.title),
'contest_id': str(contest.pk)
})
if query:
query = '?' + query
return redirect(url + query)
with_detail = request.GET.get('detail', 'true') in ['true', 'on']
if request.user.is_authenticated:
coder = request.user.coder
if 'detail' in request.GET:
coder.settings['standings_with_detail'] = with_detail
coder.save()
else:
with_detail = coder.settings.get('standings_with_detail', False)
else:
coder = None
with_row_num = False
contest_fields = list(contest.info.get('fields', []))
hidden_fields = list(contest.info.get('hidden_fields', []))
statistics = Statistics.objects.filter(contest=contest)
options = contest.info.get('standings', {})
order = None
resource_standings = contest.resource.info.get('standings', {})
order = copy.copy(options.get('order', resource_standings.get('order')))
if order:
for f in order:
if f.startswith('addition__') and f.split(
'__', 1)[1] not in contest_fields:
order = None
break
if order is None:
order = ['place_as_int', '-solving']
# fixed fields
fixed_fields = (
('penalty', 'Penalty'),
('total_time', 'Time'),
('advanced', 'Advance'),
)
fixed_fields += tuple(options.get('fixed_fields', []))
if not with_detail:
fixed_fields += (('rating_change', 'Rating change'), )
statistics = statistics \
.select_related('account') \
.select_related('account__resource') \
.prefetch_related('account__coders')
has_country = ('country' in contest_fields
or '_countries' in contest_fields or
statistics.filter(account__country__isnull=False).exists())
division = request.GET.get('division')
if division == 'any':
with_row_num = True
if 'place_as_int' in order:
order.remove('place_as_int')
order.append('place_as_int')
fixed_fields += (('division', 'Division'), )
if 'team_id' in contest_fields and not groupby:
order.append('addition__name')
statistics = statistics.distinct(*[f.lstrip('-') for f in order])
# host = resource_standings.get('account_team_resource', contest.resource.host)
# account_team_resource = Resource.objects.get(host=host)
# context['account_team_resource'] = account_team_resource
# statistics = statistics.annotate(
# accounts=RawSQL(
# '''
# SELECT array_agg(array[u2.key, u3.rating::text, u3.url])
# FROM "ranking_statistics" U0
# INNER JOIN "ranking_account" U2
# ON (u0."account_id" = u2."id")
# INNER JOIN "ranking_account" U3
# ON (u2."key" = u3."key" AND u3."resource_id" = %s)
# WHERE (
# u0."contest_id" = %s
# AND ("u0"."addition" -> 'team_id') = ("ranking_statistics"."addition" -> 'team_id')
# )
# ''',
# [account_team_resource.pk, contest.pk]
# )
# )
order.append('pk')
statistics = statistics.order_by(*order)
fields = OrderedDict()
for k, v in fixed_fields:
if k in contest_fields:
fields[k] = v
n_highlight_context = _standings_highlight(statistics, options)
# field to select
fields_to_select_defaults = {
'rating': {
'options': ['rated', 'unrated'],
'noajax': True,
'nomultiply': True,
'nourl': True
},
'advanced': {
'options': ['true', 'false'],
'noajax': True,
'nomultiply': True
},
'highlight': {
'options': ['true', 'false'],
'noajax': True,
'nomultiply': True
},
}
fields_to_select = OrderedDict()
map_fields_to_select = {'rating_change': 'rating'}
for f in sorted(contest_fields):
f = f.strip('_')
if f.lower() in [
'institution',
'room',
'affiliation',
'city',
'languages',
'school',
'class',
'job',
'region',
'rating_change',
'advanced',
'company',
'language',
'league',
'onsite',
'degree',
'university',
'list',
]:
f = map_fields_to_select.get(f, f)
field_to_select = fields_to_select.setdefault(f, {})
field_to_select['values'] = [
v for v in request.GET.getlist(f) if v
]
field_to_select.update(fields_to_select_defaults.get(f, {}))
if n_highlight_context.get('statistics_ids'):
f = 'highlight'
field_to_select = fields_to_select.setdefault(f, {})
field_to_select['values'] = [v for v in request.GET.getlist(f) if v]
field_to_select.update(fields_to_select_defaults.get(f, {}))
chats = coder.chats.all() if coder else None
if chats:
options_values = {c.chat_id: c.title for c in chats}
fields_to_select['chat'] = {
'values': [
v for v in request.GET.getlist('chat')
if v and v in options_values
],
'options':
options_values,
'noajax':
True,
'nogroupby':
True,
'nourl':
True,
}
hidden_fields_values = [v for v in request.GET.getlist('field') if v]
for v in hidden_fields_values:
if v not in hidden_fields:
hidden_fields.append(v)
for k in contest_fields:
if (k in fields or k in [
'problems', 'team_id', 'solved', 'hack', 'challenges', 'url',
'participant_type', 'division'
] or k == 'medal' and '_medal_title_field' in contest_fields
or 'country' in k and k not in hidden_fields_values
or k in ['name'] and k not in hidden_fields_values
or k.startswith('_')
or k in hidden_fields and k not in hidden_fields_values):
continue
if with_detail or k in hidden_fields_values:
fields[k] = k
else:
hidden_fields.append(k)
for k, field in fields.items():
if k != field:
continue
field = ' '.join(k.split('_'))
if field and not field[0].isupper():
field = field.title()
fields[k] = field
if hidden_fields:
fields_to_select['field'] = {
'values': hidden_fields_values,
'options': hidden_fields,
'noajax': True,
'nogroupby': True,
'nourl': True,
'nofilter': True,
}
per_page = options.get('per_page', 50)
if per_page is None:
per_page = 100500
elif contest.n_statistics and contest.n_statistics < 500:
per_page = contest.n_statistics
mod_penalty = {}
first = statistics.first()
if first and all('time' not in k for k in contest_fields):
penalty = first.addition.get('penalty')
if penalty and isinstance(penalty,
int) and 'solved' not in first.addition:
mod_penalty.update({'solving': first.solving, 'penalty': penalty})
params = {}
problems = contest.info.get('problems', {})
if 'division' in problems:
divisions_order = list(
problems.get('divisions_order',
sorted(contest.info['problems']['division'].keys())))
elif 'divisions_order' in contest.info:
divisions_order = contest.info['divisions_order']
else:
divisions_order = []
if divisions_order:
divisions_order.append('any')
if division not in divisions_order:
division = divisions_order[0]
params['division'] = division
if 'division' in problems:
if division == 'any':
_problems = OrderedDict()
for div in reversed(divisions_order):
for p in problems['division'].get(div, []):
k = get_problem_short(p)
if k not in _problems:
_problems[k] = p
else:
for f in 'n_accepted', 'n_teams', 'n_partial', 'n_total':
if f in p:
_problems[k][f] = _problems[k].get(
f, 0) + p[f]
problems = list(_problems.values())
else:
problems = problems['division'][division]
if division != 'any':
statistics = statistics.filter(addition__division=division)
for p in problems:
if 'full_score' in p and isinstance(
p['full_score'],
(int, float)) and abs(p['full_score'] - 1) > 1e-9:
mod_penalty = {}
break
last = None
merge_problems = False
for p in problems:
if last and (last.get('full_score') or last.get('subname')) and (
'name' in last and last.get('name') == p.get('name')
or 'group' in last and last.get('group') == p.get('group')):
merge_problems = True
last['colspan'] = last.get('colspan', 1) + 1
p['skip'] = True
else:
last = p
last['colspan'] = 1
# own_stat = statistics.filter(account__coders=coder).first() if coder else None
# filter by search
search = request.GET.get('search')
if search:
with_row_num = True
if search.startswith('party:'):
_, party_slug = search.split(':')
party = get_object_or_404(Party.objects.for_user(request.user),
slug=party_slug)
statistics = statistics.filter(
Q(account__coders__in=party.coders.all())
| Q(account__coders__in=party.admins.all())
| Q(account__coders=party.author))
else:
cond = get_iregex_filter(search,
'account__key',
'addition__name',
logger=request.logger)
statistics = statistics.filter(cond)
# filter by country
countries = request.GET.getlist('country')
countries = set([c for c in countries if c])
if countries:
with_row_num = True
cond = Q(account__country__in=countries)
if 'None' in countries:
cond |= Q(account__country__isnull=True)
if '_countries' in contest_fields:
for code in countries:
name = get_country_name(code)
if name:
cond |= Q(addition___countries__icontains=name)
statistics = statistics.filter(cond)
params['countries'] = countries
# filter by field to select
for field, field_to_select in fields_to_select.items():
values = field_to_select.get('values')
if not values or field_to_select.get('nofilter'):
continue
with_row_num = True
filt = Q()
if field == 'languages':
for lang in values:
if lang == 'any':
filt = Q(**{'addition___languages__isnull': False})
break
filt |= Q(**{'addition___languages__contains': [lang]})
elif field == 'rating':
for q in values:
if q not in field_to_select['options']:
continue
q = q == 'unrated'
if q:
filt |= Q(addition__rating_change__isnull=True) & Q(
addition__new_rating__isnull=True)
else:
filt |= Q(addition__rating_change__isnull=False) | Q(
addition__new_rating__isnull=False)
elif field == 'advanced':
for q in values:
if q not in field_to_select['options']:
continue
filt |= Q(addition__advanced=q == 'true')
elif field == 'highlight':
for q in values:
if q not in field_to_select['options']:
continue
filt = Q(pk__in=n_highlight_context.get('statistics_ids', {}))
if q == 'false':
filt = ~filt
elif field == 'chat':
for q in values:
if q not in field_to_select['options']:
continue
chat = Chat.objects.filter(chat_id=q, is_group=True).first()
if chat:
filt |= Q(account__coders__in=chat.coders.all())
# subquery = Chat.objects.filter(coder=OuterRef('account__coders'), is_group=False).values('name')[:1]
# statistics = statistics.annotate(chat_name=Subquery(subquery))
else:
query_field = f'addition__{field}'
statistics = statistics.annotate(**{
f'{query_field}_str':
Cast(JSONF(query_field), models.TextField())
})
for q in values:
if q == 'None':
filt |= Q(**{f'{query_field}__isnull': True})
else:
filt |= Q(**{f'{query_field}_str': q})
statistics = statistics.filter(filt)
# groupby
if groupby == 'country' or groupby in fields_to_select:
statistics = statistics.order_by('pk')
participants_info = n_highlight_context.get('participants_info')
n_highlight = options.get('n_highlight')
advanced_by_participants_info = participants_info and n_highlight and groupby != 'languages'
fields = OrderedDict()
fields['groupby'] = groupby.title()
fields['n_accounts'] = 'Num'
fields['avg_score'] = 'Avg'
medals = {m['name']: m for m in options.get('medals', [])}
if 'medal' in contest_fields:
for medal in settings.ORDERED_MEDALS_:
fields[f'n_{medal}'] = medals.get(medal, {}).get(
'value', medal[0].upper())
if 'advanced' in contest_fields or advanced_by_participants_info:
fields['n_advanced'] = 'Adv'
orderby = [f for f in orderby if f.lstrip('-') in fields
] or ['-n_accounts', '-avg_score']
if groupby == 'languages':
_, before_params = statistics.query.sql_with_params()
querysets = []
for problem in problems:
key = get_problem_short(problem)
field = f'addition__problems__{key}__language'
score = f'addition__problems__{key}__result'
qs = statistics \
.filter(**{f'{field}__isnull': False, f'{score}__isnull': False}) \
.annotate(language=Cast(JSONF(field), models.TextField())) \
.annotate(score=Case(
When(**{f'{score}__startswith': '+'}, then=1),
When(**{f'{score}__startswith': '-'}, then=0),
When(**{f'{score}__startswith': '?'}, then=0),
default=Cast(JSONF(score), models.FloatField()),
output_field=models.FloatField(),
)) \
.annotate(sid=F('pk'))
querysets.append(qs)
merge_statistics = querysets[0].union(*querysets[1:], all=True)
language_query, language_params = merge_statistics.query.sql_with_params(
)
field = 'solving'
statistics = statistics.annotate(groupby=F(field))
elif groupby == 'rating':
statistics = statistics.annotate(groupby=Case(
When(addition__rating_change__isnull=False,
then=Value('Rated')),
default=Value('Unrated'),
output_field=models.TextField(),
))
elif groupby == 'country':
if '_countries' in contest_fields:
statistics = statistics.annotate(country=RawSQL(
'''json_array_elements((("addition" ->> '_countries'))::json)::jsonb''',
[]))
field = 'country'
else:
field = 'account__country'
statistics = statistics.annotate(groupby=F(field))
else:
field = f'addition__{groupby}'
types = contest.info.get('fields_types', {}).get(groupby, [])
if 'int' in types:
field_type = models.IntegerField()
elif 'float' in types:
field_type = models.FloatField()
else:
field_type = models.TextField()
statistics = statistics.annotate(
groupby=Cast(JSONF(field), field_type))
statistics = statistics.order_by('groupby')
statistics = statistics.values('groupby')
statistics = statistics.annotate(n_accounts=Count('id'))
statistics = statistics.annotate(avg_score=Avg('solving'))
if 'medal' in contest_fields:
for medal in settings.ORDERED_MEDALS_:
n_medal = f'n_{medal}'
statistics = statistics.annotate(
**{
f'{n_medal}':
Count(Case(When(addition__medal__iexact=medal,
then=1)))
})
if 'advanced' in contest_fields:
statistics = statistics.annotate(n_advanced=Count(
Case(
When(addition__advanced=True, then=1),
When(~Q(addition__advanced=False)
& ~Q(addition__advanced=''),
then=1),
)))
elif advanced_by_participants_info:
pks = list()
for pk, info in participants_info.items():
if 'n' not in info or info['n'] > info.get(
'n_highlight', n_highlight):
continue
pks.append(pk)
statistics = statistics.annotate(
n_advanced=Count(Case(When(pk__in=set(pks), then=1))))
statistics = statistics.order_by(*orderby)
if groupby == 'languages':
query, sql_params = statistics.query.sql_with_params()
query = query.replace(
f'"ranking_statistics"."{field}" AS "groupby"',
'"language" AS "groupby"')
query = query.replace(f'GROUP BY "ranking_statistics"."{field}"',
'GROUP BY "language"')
query = query.replace('"ranking_statistics".', '')
query = query.replace('AVG("solving") AS "avg_score"',
'AVG("score") AS "avg_score"')
query = query.replace('COUNT("id") AS "n_accounts"',
'COUNT("sid") AS "n_accounts"')
query = re.sub('FROM "ranking_statistics".*GROUP BY',
f'FROM ({language_query}) t1 GROUP BY', query)
sql_params = sql_params[:-len(before_params)] + language_params
with connection.cursor() as cursor:
cursor.execute(query, sql_params)
columns = [col[0] for col in cursor.description]
statistics = [
dict(zip(columns, row)) for row in cursor.fetchall()
]
statistics = ListAsQueryset(statistics)
problems = []
labels_groupby = {
'n_accounts': 'Number of participants',
'avg_score': 'Average score',
'n_advanced': 'Number of advanced',
}
for medal in settings.ORDERED_MEDALS_:
labels_groupby[f'n_{medal}'] = 'Number of ' + medals.get(
medal, {}).get('value', medal)
num_rows_groupby = statistics.count()
map_colors_groupby = {
s['groupby']: idx
for idx, s in enumerate(statistics)
}
else:
groupby = 'none'
labels_groupby = None
num_rows_groupby = None
map_colors_groupby = None
my_statistics = []
if groupby == 'none' and coder:
statistics = statistics.annotate(
my_stat=SubqueryExists('account__coders', filter=Q(coder=coder)))
my_statistics = statistics.filter(account__coders=coder).extra(
select={'floating': True})
context.update({
'standings_options':
options,
'mod_penalty':
mod_penalty,
'colored_by_group_score':
mod_penalty or options.get('colored_by_group_score'),
'contest':
contest,
'statistics':
statistics,
'my_statistics':
my_statistics,
'problems':
problems,
'params':
params,
'fields':
fields,
'fields_types':
contest.info.get('fields_types', {}),
'divisions_order':
divisions_order,
'has_country':
has_country,
'per_page':
per_page,
'with_row_num':
with_row_num,
'merge_problems':
merge_problems,
'fields_to_select':
fields_to_select,
'truncatechars_name_problem':
10 * (2 if merge_problems else 1),
'with_detail':
with_detail,
'groupby':
groupby,
'pie_limit_rows_groupby':
50,
'labels_groupby':
labels_groupby,
'num_rows_groupby':
num_rows_groupby,
'map_colors_groupby':
map_colors_groupby,
'advance':
contest.info.get('advance'),
'timezone':
get_timezone(request),
'timeformat':
get_timeformat(request),
'with_neighbors':
request.GET.get('neighbors') == 'on',
'with_table_inner_scroll':
not request.user_agent.is_mobile,
})
context.update(n_highlight_context)
if extra_context is not None:
context.update(extra_context)
return render(request, template, context)
def parse_statistic(
self,
contests,
previous_days=None,
freshness_days=None,
limit=None,
with_check=True,
stop_on_error=False,
random_order=False,
no_update_results=False,
limit_duration_in_secs=7 * 60 * 60, # 7 hours
title_regex=None,
users=None,
with_stats=True,
update_without_new_rating=None,
):
now = timezone.now()
if with_check:
if previous_days is not None:
contests = contests.filter(end_time__gt=now - timedelta(days=previous_days), end_time__lt=now)
else:
contests = contests.filter(Q(timing__statistic__isnull=True) | Q(timing__statistic__lt=now))
started = contests.filter(start_time__lt=now, end_time__gt=now, statistics__isnull=False)
query = Q()
query &= (
Q(end_time__gt=now - F('resource__module__max_delay_after_end'))
| Q(timing__statistic__isnull=True)
)
query &= Q(end_time__lt=now - F('resource__module__min_delay_after_end'))
ended = contests.filter(query)
contests = started.union(ended)
contests = contests.distinct('id')
elif title_regex:
contests = contests.filter(title__iregex=title_regex)
else:
contests = contests.filter(end_time__lt=now - F('resource__module__min_delay_after_end'))
if freshness_days is not None:
contests = contests.filter(updated__lt=now - timedelta(days=freshness_days))
if limit:
contests = contests.order_by('-start_time')[:limit]
with transaction.atomic():
for c in contests:
module = c.resource.module
delay_on_success = module.delay_on_success or module.max_delay_after_end
if now < c.end_time:
if c.end_time - c.start_time <= timedelta(seconds=limit_duration_in_secs):
delay_on_success = timedelta(minutes=0)
elif c.end_time < now + delay_on_success:
delay_on_success = c.end_time - now + timedelta(seconds=5)
TimingContest.objects.update_or_create(
contest=c,
defaults={'statistic': now + delay_on_success}
)
if random_order:
contests = list(contests)
shuffle(contests)
countrier = Countrier()
count = 0
total = 0
n_upd_account_time = 0
progress_bar = tqdm(contests)
stages_ids = []
for contest in progress_bar:
resource = contest.resource
if not hasattr(resource, 'module'):
self.logger.error('Not found module contest = %s' % c)
continue
progress_bar.set_description(f'contest = {contest.title}')
progress_bar.refresh()
total += 1
parsed = False
user_info = None
user_info_has_rating = None
try:
r = {}
if hasattr(contest, 'stage'):
contest.stage.update()
count += 1
parsed = True
continue
plugin = resource.plugin.Statistic(contest=contest)
with REQ:
statistics_by_key = {} if with_stats else None
statistics_ids = set()
if not no_update_results and (users or users is None):
statistics = Statistics.objects.filter(contest=contest).select_related('account')
if users:
statistics = statistics.filter(account__key__in=users)
for s in tqdm(statistics.iterator(), 'getting parsed statistics'):
if with_stats:
statistics_by_key[s.account.key] = s.addition
statistics_ids.add(s.pk)
standings = plugin.get_standings(users=users, statistics=statistics_by_key)
with transaction.atomic():
if 'url' in standings and standings['url'] != contest.standings_url:
contest.standings_url = standings['url']
contest.save()
if 'title' in standings and standings['title'] != contest.title:
contest.title = standings['title']
contest.save()
if 'options' in standings:
contest_options = contest.info.get('standings', {})
standings_options = dict(contest_options)
standings_options.update(standings.pop('options'))
if canonize(standings_options) != canonize(contest_options):
contest.info['standings'] = standings_options
contest.save()
info_fields = standings.pop('info_fields', []) + ['divisions_order']
for field in info_fields:
if standings.get(field) is not None and contest.info.get(field) != standings[field]:
contest.info[field] = standings[field]
contest.save()
update_writers(contest, standings.pop('writers', None))
problems_time_format = standings.pop('problems_time_format', '{M}:{s:02d}')
result = standings.get('result', {})
if not no_update_results and (result or users is not None):
fields_set = set()
fields_types = {}
fields = list()
calculate_time = False
d_problems = {}
teams_viewed = set()
has_hidden = False
languages = set()
additions = contest.info.get('additions', {})
if additions:
for k, v in result.items():
for field in [r.get('member'), r.get('name')]:
r.update(OrderedDict(additions.pop(field, [])))
for k, v in additions.items():
result[k] = dict(v)
for r in tqdm(list(result.values()), desc=f'update results {contest}'):
for k, v in r.items():
if isinstance(v, str) and chr(0x00) in v:
r[k] = v.replace(chr(0x00), '')
member = r.pop('member')
account_action = r.pop('action', None)
if account_action == 'delete':
Account.objects.filter(resource=resource, key=member).delete()
continue
account, created = Account.objects.get_or_create(resource=resource, key=member)
if not contest.info.get('_no_update_account_time'):
stats = (statistics_by_key or {}).get(member, {})
no_rating = with_stats and 'new_rating' not in stats and 'rating_change' not in stats
updated = now + timedelta(days=1)
wait_rating = contest.resource.info.get('statistics', {}).get('wait_rating')
if no_rating and wait_rating:
updated = now + timedelta(hours=1)
title_re = wait_rating.get('title_re')
if (
contest.end_time + timedelta(days=wait_rating['days']) > now
and (not title_re or re.search(title_re, contest.title))
and updated < account.updated
):
if user_info is None:
generator = plugin.get_users_infos([member], contest.resource, [account])
try:
user_info = next(generator)
params = user_info.get('contest_addition_update_params', {})
field = user_info.get('contest_addition_update_by') or params.get('by') or 'key' # noqa
updates = user_info.get('contest_addition_update') or params.get('update') or {} # noqa
user_info_has_rating = getattr(contest, field) in updates
except StopIteration:
user_info = False
user_info_has_rating = False
if user_info_has_rating:
n_upd_account_time += 1
account.updated = updated
account.save()
elif (
created
or (not statistics_ids and updated < account.updated)
or (update_without_new_rating and updated < account.updated and no_rating)
):
n_upd_account_time += 1
account.updated = updated
account.save()
if r.get('name'):
while True:
name = unescape(r['name'])
if name == r['name']:
break
r['name'] = name
if len(r['name']) > 1024:
r['name'] = r['name'][:1020] + '...'
no_update_name = r.pop('_no_update_name', False)
if not no_update_name and account.name != r['name'] and member.find(r['name']) == -1:
account.name = r['name']
account.save()
country = r.get('country', None)
if country:
country = countrier.get(country)
if country and country != account.country:
account.country = country
account.save()
contest_addition_update = r.pop('contest_addition_update', {})
if contest_addition_update:
account_update_contest_additions(
account,
contest_addition_update,
timedelta(days=31) if with_check else None
)
account_info = r.pop('info', {})
if account_info:
if 'rating' in account_info:
account_info['rating_ts'] = int(now.timestamp())
account.info.update(account_info)
account.save()
default_division = contest.info.get('default_division')
if default_division and 'division' not in r:
r['division'] = default_division
problems = r.get('problems', {})
_languages = set()
for problem in problems.values():
if problem.get('language'):
languages.add(problem['language'])
_languages.add(problem['language'])
if '_languages' not in r and _languages:
r['_languages'] = list(sorted(_languages))
if 'team_id' not in r or r['team_id'] not in teams_viewed:
if 'team_id' in r:
teams_viewed.add(r['team_id'])
solved = {'solving': 0}
for k, v in problems.items():
if 'result' not in v:
continue
p = d_problems
if 'division' in standings.get('problems', {}):
p = p.setdefault(r['division'], {})
p = p.setdefault(k, {})
if 'default_problem_full_score' in contest.info:
full_score = contest.info['default_problem_full_score']
if 'partial' not in v and full_score - float(v['result']) > 1e-9:
v['partial'] = True
if not v.get('partial'):
solved['solving'] += 1
if 'full_score' not in p:
p['full_score'] = full_score
if contest.info.get('without_problem_first_ac'):
v.pop('first_ac', None)
v.pop('first_ac_of_all', None)
if contest.info.get('without_problem_time'):
v.pop('time', None)
if r.get('_skip_for_problem_stat'):
continue
p['n_teams'] = p.get('n_teams', 0) + 1
ac = str(v['result']).startswith('+')
try:
result = float(v['result'])
ac = ac or result > 0 and not v.get('partial', False)
except Exception:
pass
if ac:
p['n_accepted'] = p.get('n_accepted', 0) + 1
if 'default_problem_full_score' in contest.info and solved and 'solved' not in r:
r['solved'] = solved
calc_time = contest.calculate_time or (
contest.start_time <= now < contest.end_time and
not contest.resource.info.get('parse', {}).get('no_calculate_time', False)
)
advance = contest.info.get('advance')
if advance:
k = 'advanced'
r.pop(k, None)
for cond in advance['filter']:
field = cond['field']
value = r.get(field)
value = get_number_from_str(value)
if value is None:
continue
r[k] = getattr(operator, cond['operator'])(value, cond['threshold'])
medals = contest.info.get('standings', {}).get('medals')
if medals:
k = 'medal'
r.pop(k, None)
if 'place' in r:
place = get_number_from_str(r['place'])
if place:
for medal in medals:
if place <= medal['count']:
r[k] = medal['name']
if 'field' in medal:
r[medal['field']] = medal['value']
r[f'_{k}_title_field'] = medal['field']
break
place -= medal['count']
medal_fields = [m['field'] for m in medals if 'field' in m] or [k]
for f in medal_fields:
if f not in fields_set:
fields_set.add(f)
fields.append(f)
defaults = {
'place': r.pop('place', None),
'solving': r.pop('solving', 0),
'upsolving': r.pop('upsolving', 0),
}
defaults = {k: v for k, v in defaults.items() if v != '__unchanged__'}
addition = type(r)()
for k, v in r.items():
if k[0].isalpha() and not re.match('^[A-Z]+$', k):
k = k[0].upper() + k[1:]
k = '_'.join(map(str.lower, re.findall('[A-ZА-Я][^A-ZА-Я]*', k)))
if k not in fields_set:
fields_set.add(k)
fields.append(k)
if (k in Resource.RATING_FIELDS or k == 'rating_change') and v is None:
continue
fields_types.setdefault(k, set()).add(type(v).__name__)
addition[k] = v
if (
addition.get('rating_change') is None
and addition.get('new_rating') is not None
and addition.get('old_rating') is not None
):
delta = addition['new_rating'] - addition['old_rating']
f = 'rating_change'
addition[f] = f'{"+" if delta > 0 else ""}{delta}'
if f not in fields_set:
fields_set.add(f)
fields.insert(-1, f)
if 'is_rated' in addition and not addition['is_rated']:
addition.pop('old_rating', None)
if not calc_time:
defaults['addition'] = addition
rating_ts = int(min(contest.end_time, now).timestamp())
if 'new_rating' in addition and (
'rating_ts' not in account.info or account.info['rating_ts'] <= rating_ts
):
account.info['rating_ts'] = rating_ts
account.info['rating'] = addition['new_rating']
account.save()
statistic, created = Statistics.objects.update_or_create(
account=account,
contest=contest,
defaults=defaults,
)
if not created:
statistics_ids.remove(statistic.pk)
if calc_time:
p_problems = statistic.addition.get('problems', {})
ts = min(int((now - contest.start_time).total_seconds()), contest.duration_in_secs)
values = {
'D': ts // (24 * 60 * 60),
'H': ts // (60 * 60),
'h': ts // (60 * 60) % 24,
'M': ts // 60,
'm': ts // 60 % 60,
'S': ts,
's': ts % 60,
}
time = problems_time_format.format(**values)
for k, v in problems.items():
v_result = v.get('result', '')
if isinstance(v_result, str) and '?' in v_result:
calculate_time = True
p = p_problems.get(k, {})
if 'time' in v:
continue
has_change = v.get('result') != p.get('result')
if (not has_change or contest.end_time < now) and 'time' in p:
v['time'] = p['time']
else:
v['time'] = time
for p in problems.values():
p_result = p.get('result', '')
if isinstance(p_result, str) and '?' in p_result:
has_hidden = True
if calc_time:
statistic.addition = addition
statistic.save()
if users is None:
timing_statistic_delta = standings.get(
'timing_statistic_delta',
timedelta(minutes=30) if has_hidden and contest.end_time < now else None,
)
if timing_statistic_delta is not None:
contest.timing.statistic = timezone.now() + timing_statistic_delta
contest.timing.save()
if contest.start_time <= now:
if now < contest.end_time:
contest.info['last_parse_statistics'] = now.strftime('%Y-%m-%d %H:%M:%S.%f+%Z')
elif 'last_parse_statistics' in contest.info:
contest.info.pop('last_parse_statistics')
if fields_set and not isinstance(addition, OrderedDict):
fields.sort()
fields_types = {k: list(v) for k, v in fields_types.items()}
if statistics_ids:
first = Statistics.objects.filter(pk__in=statistics_ids).first()
if first:
self.logger.info(f'First deleted: {first}, account = {first.account}')
delete_info = Statistics.objects.filter(pk__in=statistics_ids).delete()
self.logger.info(f'Delete info: {delete_info}')
progress_bar.set_postfix(deleted=str(delete_info))
if canonize(fields) != canonize(contest.info.get('fields')):
contest.info['fields'] = fields
contest.info['fields_types'] = fields_types
if calculate_time and not contest.calculate_time:
contest.calculate_time = True
problems = standings.pop('problems', None)
if problems is not None:
if 'division' in problems:
for d, ps in problems['division'].items():
for p in ps:
k = get_problem_short(p)
if k:
p.update(d_problems.get(d, {}).get(k, {}))
if isinstance(problems['division'], OrderedDict):
problems['divisions_order'] = list(problems['division'].keys())
else:
for p in problems:
k = get_problem_short(p)
if k:
p.update(d_problems.get(k, {}))
update_problems(contest, problems=problems)
if languages:
languages = list(sorted(languages))
if canonize(languages) != canonize(contest.info.get('languages')):
contest.info['languages'] = languages
contest.save()
progress_bar.set_postfix(n_fields=len(fields))
else:
problems = standings.pop('problems', None)
if problems is not None:
problems = plugin.merge_dict(problems, contest.info.get('problems'))
if not users:
contest.info['problems'] = {}
update_problems(contest, problems=problems)
action = standings.get('action')
if action is not None:
args = []
if isinstance(action, tuple):
action, *args = action
self.logger.info(f'Action {action} with {args}, contest = {contest}, url = {contest.url}')
if action == 'delete':
if Statistics.objects.filter(contest=contest).exists():
self.logger.info(f'No deleted. Contest have statistics')
elif now < contest.end_time:
self.logger.info(f'No deleted. Try after = {contest.end_time - now}')
else:
delete_info = contest.delete()
self.logger.info(f'Delete info contest: {delete_info}')
elif action == 'url':
contest.url = args[0]
contest.save()
if 'result' in standings:
count += 1
parsed = True
except (ExceptionParseStandings, InitModuleException) as e:
progress_bar.set_postfix(exception=str(e), cid=str(contest.pk))
except Exception as e:
self.logger.error(f'contest = {contest.pk}, error = {e}, row = {r}')
if stop_on_error:
self.logger.error(format_exc())
break
if not parsed:
if now < c.end_time and c.duration_in_secs <= limit_duration_in_secs:
delay = timedelta(minutes=0)
else:
delay = resource.module.delay_on_error
contest.timing.statistic = timezone.now() + delay
contest.timing.save()
elif not no_update_results and (users is None or users):
stages = Stage.objects.filter(
~Q(pk__in=stages_ids),
contest__start_time__lte=contest.start_time,
contest__end_time__gte=contest.end_time,
contest__resource=contest.resource,
)
for stage in stages:
if Contest.objects.filter(pk=contest.pk, **stage.filter_params).exists():
stages_ids.append(stage.pk)
for stage in tqdm(Stage.objects.filter(pk__in=stages_ids),
total=len(stages_ids),
desc='getting stages'):
stage.update()
progress_bar.close()
self.logger.info(f'Parsed statistic: {count} of {total}. Updated account time: {n_upd_account_time}')
return count, total
def update(self):
stage = self.contest
contests = Contest.objects.filter(
resource=self.contest.resource,
start_time__gte=self.contest.start_time,
end_time__lte=self.contest.end_time,
**self.filter_params,
).exclude(pk=self.contest.pk)
contests = contests.order_by('start_time')
contests = contests.prefetch_related('writers')
placing = self.score_params.get('place')
n_best = self.score_params.get('n_best')
fields = self.score_params.get('fields', [])
detail_problems = self.score_params.get('detail_problems')
order_by = self.score_params['order_by']
advances = self.score_params.get('advances', {})
results = collections.defaultdict(collections.OrderedDict)
mapping_account_by_coder = {}
problems_infos = collections.OrderedDict()
divisions_order = []
for idx, contest in enumerate(tqdm.tqdm(
contests, desc=f'getting contests for stage {stage}'),
start=1):
info = {
'code':
str(contest.pk),
'name':
contest.title,
'url':
reverse('ranking:standings',
kwargs={
'title_slug': slug(contest.title),
'contest_id': str(contest.pk)
}),
}
for division in contest.info.get('divisions_order', []):
if division not in divisions_order:
divisions_order.append(division)
problems = contest.info.get('problems', [])
if not detail_problems:
full_score = None
if placing:
if 'division' in placing:
full_score = max([
max(p.values())
for p in placing['division'].values()
])
else:
full_score = max(placing.values())
elif 'division' in problems:
full_scores = []
for ps in problems['division'].values():
full = 0
for problem in ps:
full += problem.get('full_score', 1)
full_scores.append(full)
info['full_score'] = max(full_scores)
else:
full_score = 0
for problem in problems:
full_score += problem.get('full_score', 1)
if full_score is not None:
info['full_score'] = full_score
if self.score_params.get('regex_problem_name'):
match = re.search(
self.score_params.get('regex_problem_name'),
contest.title)
if match:
info['short'] = match.group(1)
if self.score_params.get('abbreviation_problem_name'):
info['short'] = ''.join(
re.findall(r'(\b[A-Z]|[0-9])',
info.get('short', contest.title)))
problems_infos[str(contest.pk)] = info
else:
for problem in problems:
problem = dict(problem)
add_prefix_to_problem_short(problem, f'{idx}.')
problem['group'] = info['name']
problem['url'] = info['url']
problems_infos[get_problem_short(problem)] = problem
exclude_advances = {}
if advances and advances.get('exclude_stages'):
qs = Statistics.objects \
.filter(contest__stage__in=advances['exclude_stages'], addition___advance__isnull=False) \
.values('account__key', 'addition___advance', 'contest__title') \
.order_by('contest__end_time')
for r in qs:
d = r['addition___advance']
if 'contest' not in d:
d['contest'] = r['contest__title']
exclude_advances[r['account__key']] = d
statistics = Statistics.objects \
.select_related('account', 'account__duplicate') \
.prefetch_related('account__coders')
filter_statistics = self.score_params.get('filter_statistics')
if filter_statistics:
statistics = statistics.filter(**filter_statistics)
def get_placing(placing, stat):
return placing['division'][
stat.
addition['division']] if 'division' in placing else placing
account_keys = dict()
total = statistics.filter(contest__in=contests).count()
with tqdm.tqdm(total=total,
desc=f'getting statistics for stage {stage}'
) as pbar, print_sql(count_only=True):
for idx, contest in enumerate(contests, start=1):
skip_problem_stat = '_skip_for_problem_stat' in contest.info.get(
'fields', [])
contest_unrated = contest.info.get('unrated')
if not detail_problems:
problem_info_key = str(contest.pk)
problem_short = get_problem_short(
problems_infos[problem_info_key])
pbar.set_postfix(contest=contest)
stats = statistics.filter(contest_id=contest.pk)
if placing:
placing_scores = deepcopy(placing)
n_rows = 0
for s in stats:
n_rows += 1
placing_ = get_placing(placing_scores, s)
key = str(s.place_as_int)
if key in placing_:
placing_.setdefault('scores', {})
placing_['scores'][key] = placing_.pop(key)
scores = []
for place in reversed(range(1, n_rows + 1)):
placing_ = get_placing(placing_scores, s)
key = str(place)
if key in placing_:
scores.append(placing_.pop(key))
else:
if scores:
placing_['scores'][key] += sum(scores)
placing_['scores'][key] /= len(scores) + 1
scores = []
for s in stats:
if not detail_problems and not skip_problem_stat:
problems_infos[problem_info_key].setdefault(
'n_total', 0)
problems_infos[problem_info_key]['n_total'] += 1
if s.solving < 1e-9:
score = 0
else:
if placing:
placing_ = get_placing(placing_scores, s)
score = placing_['scores'].get(
str(s.place_as_int), placing_.get('default'))
if score is None:
continue
else:
score = s.solving
if not detail_problems and not skip_problem_stat:
problems_infos[problem_info_key].setdefault(
'n_teams', 0)
problems_infos[problem_info_key]['n_teams'] += 1
if score:
problems_infos[problem_info_key].setdefault(
'n_accepted', 0)
problems_infos[problem_info_key]['n_accepted'] += 1
account = s.account
if account.duplicate is not None:
account = account.duplicate
coders = account.coders.all()
has_mapping_account_by_coder = False
if len(coders) == 1:
coder = coders[0]
if coder not in mapping_account_by_coder:
mapping_account_by_coder[coder] = account
else:
account = mapping_account_by_coder[coder]
has_mapping_account_by_coder = True
row = results[account]
row['member'] = account
account_keys[account.key] = account
problems = row.setdefault('problems', {})
if detail_problems:
for key, problem in s.addition.get('problems',
{}).items():
problems[f'{idx}.' + key] = problem
else:
problem = problems.setdefault(problem_short, {})
if contest_unrated:
problem = problem.setdefault('upsolving', {})
problem['result'] = score
url = s.addition.get('url')
if url:
problem['url'] = url
if contest_unrated:
score = 0
if n_best and not detail_problems:
row.setdefault('scores', []).append((score, problem))
else:
row['score'] = row.get('score', 0) + score
field_values = {}
for field in fields:
inp = field['field']
out = field.get('out', inp)
if field.get('skip_on_mapping_account_by_coder'
) and has_mapping_account_by_coder:
continue
if 'type' in field:
continue
if field.get('first') and out in row or (
inp not in s.addition and not hasattr(s, inp)):
continue
val = s.addition[
inp] if inp in s.addition else getattr(s, inp)
if not field.get('safe') and isinstance(val, str):
val = ast.literal_eval(val)
if 'cast' in field:
val = locate(field['cast'])(val)
field_values[out] = val
if field.get('accumulate'):
val = round(
val + ast.literal_eval(str(row.get(out, 0))),
2)
row[out] = val
if 'solved' in s.addition:
solved = row.setdefault('solved', {})
for k, v in s.addition['solved'].items():
solved[k] = solved.get(k, 0) + v
if 'status' in self.score_params:
field = self.score_params['status']
val = field_values.get(field, row.get(field))
if val is None:
val = getattr(s, field)
if val:
problem['status'] = val
else:
for field in order_by:
field = field.lstrip('-')
if field in ['score', 'rating']:
continue
status = field_values.get(field, row.get(field))
if status is None:
continue
problem['status'] = status
break
pbar.update()
for writer in contest.writers.all():
account_keys[writer.key] = writer
total = sum(
[len(contest.info.get('writers', [])) for contest in contests])
with tqdm.tqdm(
total=total,
desc=f'getting writers for stage {stage}') as pbar, print_sql(
count_only=True):
writers = set()
for contest in contests:
contest_writers = contest.info.get('writers', [])
if not contest_writers or detail_problems:
continue
problem_info_key = str(contest.pk)
problem_short = get_problem_short(
problems_infos[problem_info_key])
for writer in contest_writers:
if writer in account_keys:
account = account_keys[writer]
else:
try:
account = Account.objects.get(
resource_id=contest.resource_id,
key__iexact=writer)
except Account.DoesNotExist:
account = None
pbar.update()
if not account:
continue
writers.add(account)
row = results[account]
row['member'] = account
row.setdefault('score', 0)
if n_best:
row.setdefault('scores', [])
row.setdefault('writer', 0)
row['writer'] += 1
problems = row.setdefault('problems', {})
problem = problems.setdefault(problem_short, {})
problem['status'] = 'W'
if self.score_params.get('writers_proportionally_score'):
n_contests = len(contests)
for account in writers:
row = results[account]
if n_contests == row['writer'] or 'score' not in row:
continue
row['score'] = row['score'] / (n_contests -
row['writer']) * n_contests
for field in fields:
t = field.get('type')
if t == 'points_for_common_problems':
group = field['group']
inp = field['field']
out = field.get('out', inp)
common_problems = dict()
for account, row in results.items():
problems = {
k
for k, p in row['problems'].items()
if p.get('status') != 'W'
}
key = row[group]
common_problems[
key] = problems if key not in common_problems else (
problems & common_problems[key])
for account, row in results.items():
key = row[group]
problems = common_problems[key]
value = 0
for k in problems:
value += float(row['problems'].get(k, {}).get(inp, 0))
for k, v in row['problems'].items():
if k not in problems and v.get('status') != 'W':
v['status_tag'] = 'strike'
row[out] = round(value, 2)
results = list(results.values())
if n_best:
for row in results:
scores = row.pop('scores')
for index, (score, problem) in enumerate(
sorted(scores, key=lambda s: s[0], reverse=True)):
if index < n_best:
row['score'] = row.get('score', 0) + score
else:
problem['status'] = problem.pop('result')
filtered_results = []
for r in results:
if r['score'] > 1e-9 or r.get('writer'):
filtered_results.append(r)
continue
if detail_problems:
continue
problems = r.setdefault('problems', {})
for idx, contest in enumerate(contests, start=1):
skip_problem_stat = '_skip_for_problem_stat' in contest.info.get(
'fields', [])
if skip_problem_stat:
continue
problem_info_key = str(contest.pk)
problem_short = get_problem_short(
problems_infos[problem_info_key])
if problem_short in problems:
problems_infos[problem_info_key].setdefault('n_teams', 0)
problems_infos[problem_info_key]['n_teams'] -= 1
results = filtered_results
results = sorted(
results,
key=lambda r: tuple(
r.get(k.lstrip('-'), 0) * (-1 if k.startswith('-') else 1)
for k in order_by),
reverse=True,
)
with transaction.atomic():
fields_set = set()
fields = list()
pks = set()
placing_infos = {}
score_advance = None
place_advance = 0
for row in tqdm.tqdm(results,
desc=f'update statistics for stage {stage}'):
row['_no_update_n_contests'] = True
division = row.get('division', 'none')
placing_info = placing_infos.setdefault(division, {})
placing_info['index'] = placing_info.get('index', 0) + 1
curr_score = tuple(row.get(k.lstrip('-'), 0) for k in order_by)
if curr_score != placing_info.get('last_score'):
placing_info['last_score'] = curr_score
placing_info['place'] = placing_info['index']
if advances and ('divisions' not in advances
or division in advances['divisions']):
tmp = score_advance, place_advance
if curr_score != score_advance:
score_advance = curr_score
place_advance += 1
for advance in advances.get('options', []):
handle = row['member'].key
if handle in exclude_advances and advance[
'next'] == exclude_advances[handle]['next']:
advance = exclude_advances[handle]
if 'class' in advance and not advance[
'class'].startswith('text-'):
advance['class'] = f'text-{advance["class"]}'
row['_advance'] = advance
break
if 'places' in advance and place_advance in advance[
'places']:
row['_advance'] = advance
for field in advance.get('inplace_fields', []):
row[field] = advance[field]
tmp = None
break
if tmp is not None:
score_advance, place_advance = tmp
account = row.pop('member')
solving = row.pop('score')
stat, created = Statistics.objects.update_or_create(
account=account,
contest=stage,
defaults={
'place': str(placing_info['place']),
'place_as_int': placing_info['place'],
'solving': solving,
'addition': row,
},
)
pks.add(stat.pk)
for k in row.keys():
if k not in fields_set:
fields_set.add(k)
fields.append(k)
stage.statistics_set.exclude(pk__in=pks).delete()
stage.n_statistics = len(results)
stage.info['fields'] = list(fields)
standings_info = self.score_params.get('info', {})
standings_info['fixed_fields'] = [(f.lstrip('-'), f.lstrip('-'))
for f in order_by]
stage.info['standings'] = standings_info
if divisions_order and self.score_params.get('divisions_ordering'):
stage.info['divisions_order'] = divisions_order
stage.info['problems'] = list(problems_infos.values())
stage.save()
def update_problems(contest, problems=None):
if problems is not None:
if canonize(problems) == canonize(contest.info.get('problems')):
return
contest.info['problems'] = problems
contest.save()
problems = contest.info.get('problems')
if not problems or hasattr(contest, 'stage'):
return
if 'division' in problems:
problem_sets = problems['division'].items()
else:
problem_sets = [(None, problems)]
old_problem_ids = set(contest.problem_set.values_list('id', flat=True))
added_problems = dict()
for division, problem_set in problem_sets:
for index, problem_info in enumerate(problem_set, start=1):
key = get_problem_key(problem_info)
short = get_problem_short(problem_info)
name = get_problem_name(problem_info)
if short == name or short == key:
short = None
added_problem = added_problems.get(key)
defaults = {
'index':
index if not added_problem else None,
'short':
short,
'name':
name,
'divisions':
getattr(added_problem, 'divisions', []) +
[division] if division else None,
'url':
problem_info.get('url'),
'n_tries':
problem_info.get('n_teams', 0) +
getattr(added_problem, 'n_tries', 0),
'n_accepted':
problem_info.get('n_accepted', 0) +
getattr(added_problem, 'n_accepted', 0),
'time':
contest.start_time,
}
problem, created = Problem.objects.update_or_create(
contest=contest,
key=key,
defaults=defaults,
)
old_tags = set(problem.tags.all())
if 'tags' in problem_info:
if '' in problem_info['tags']:
problem_info['tags'].remove('')
contest.save()
for name in problem_info['tags']:
tag, _ = ProblemTag.objects.get_or_create(name=name)
if tag in old_tags:
old_tags.discard(tag)
else:
problem.tags.add(tag)
for tag in old_tags:
problem.tags.remove(tag)
added_problems[key] = problem
if problem.id in old_problem_ids:
old_problem_ids.remove(problem.id)
if old_problem_ids:
Problem.objects.filter(id__in=old_problem_ids).delete()
def handle(self, *args, **options):
self.stdout.write(str(options))
args = AttrDict(options)
bot = Bot()
if args.dump is not None and os.path.exists(args.dump):
with open(args.dump, 'r') as fo:
standings = json.load(fo)
else:
standings = {}
problems_info = standings.setdefault('__problems_info', {})
parser_command = ParserCommand()
iteration = 1 if args.dump else 0
while True:
subprocess.call('clear', shell=True)
print(now())
contest = Contest.objects.filter(pk=args.cid)
parser_command.parse_statistic(contest, without_contest_filter=True)
contest = contest.first()
resource = contest.resource
statistics = list(Statistics.objects.filter(contest=contest))
for p in problems_info.values():
if p.get('accepted') or not p.get('n_hidden'):
p.pop('show_hidden', None)
p['n_hidden'] = 0
updated = False
has_hidden = False
numbered = 0
statistics = [s for s in statistics if s.place_as_int is not None]
for stat in sorted(statistics, key=lambda s: s.place_as_int):
name_instead_key = resource.info.get('standings', {}).get('name_instead_key')
name_instead_key = stat.account.info.get('_name_instead_key', name_instead_key)
if name_instead_key:
name = stat.account.name
else:
name = stat.addition.get('name')
if not name or not has_season(stat.account.key, name):
name = stat.account.key
filtered = False
if args.query is not None and re.search(args.query, name, re.I):
filtered = True
if args.top and stat.place_as_int <= args.top:
filtered = True
contest_problems = contest.info.get('problems')
division = stat.addition.get('division')
if division and 'division' in contest_problems:
contest_problems = contest_problems['division'][division]
contest_problems = {get_problem_short(p): p for p in contest_problems}
message_id = None
key = str(stat.account.id)
if key in standings:
problems = standings[key]['problems']
message_id = standings[key].get('messageId')
def delete_message():
nonlocal message_id
if message_id:
for iteration in range(1, 5):
try:
bot.delete_message(chat_id=args.tid, message_id=message_id)
message_id = None
break
except telegram.error.TimedOut as e:
logger.warning(str(e))
time.sleep(iteration)
continue
p = []
has_update = False
has_first_ac = False
has_try_first_ac = False
has_new_accepted = False
has_top = False
for k, v in stat.addition.get('problems', {}).items():
p_info = problems_info.setdefault(k, {})
p_result = problems.get(k, {}).get('result')
result = v['result']
is_hidden = str(result).startswith('?')
is_accepted = str(result).startswith('+') or v.get('binary', False)
try:
is_accepted = is_accepted or float(result) > 0 and not v.get('partial')
except Exception:
pass
if is_hidden:
p_info['n_hidden'] = p_info.get('n_hidden', 0) + 1
if p_result != result or is_hidden:
has_new_accepted |= is_accepted
short = k
if k in contest_problems and k != get_problem_short(contest_problems[k]):
short = get_problem_name(contest_problems[k])
m = '%s%s %s' % (short, ('. ' + v['name']) if 'name' in v else '', result)
if v.get('verdict'):
m += ' ' + v['verdict']
if p_result != result:
m = '*%s*' % m
has_update = True
if iteration:
if p_info.get('show_hidden') == key:
delete_message()
if not is_hidden:
p_info.pop('show_hidden')
if not p_info.get('accepted'):
if is_accepted:
m += ' FIRST ACCEPTED'
has_first_ac = True
elif is_hidden and not p_info.get('show_hidden'):
p_info['show_hidden'] = key
m += ' TRY FIRST AC'
has_try_first_ac = True
if args.top and stat.place_as_int <= args.top:
has_top = True
p.append(m)
if is_accepted:
p_info['accepted'] = True
has_hidden = has_hidden or is_hidden
prev_place = standings[key].get('place')
place = stat.place
if has_new_accepted and prev_place:
place = '%s->%s' % (prev_place, place)
if args.numbered is not None and re.search(args.numbered, stat.account.key, re.I):
numbered += 1
place = '%s (%s)' % (place, numbered)
msg = '%s. _%s_' % (place, telegram.utils.helpers.escape_markdown(name.replace('_', ' ')))
if p:
msg = '%s, %s' % (', '.join(p), msg)
if has_top:
msg += f' TOP{args.top}'
if abs(standings[key]['solving'] - stat.solving) > 1e-9:
msg += ' = %d' % stat.solving
if 'penalty' in stat.addition:
msg += f' ({stat.addition["penalty"]})'
if has_update or has_first_ac or has_try_first_ac:
updated = True
if filtered:
print(stat.place, stat.solving, end=' | ')
if filtered:
print(msg)
if filtered and has_update or has_first_ac or has_try_first_ac:
if not args.dryrun:
delete_message()
for iteration in range(1, 5):
try:
message = bot.send_message(msg=msg, chat_id=args.tid)
message_id = message.message_id
break
except telegram.error.TimedOut as e:
logger.warning(str(e))
time.sleep(iteration * 3)
continue
except telegram.error.BadRequest as e:
logger.error(str(e))
break
standings[key] = {
'solving': stat.solving,
'place': stat.place,
'problems': stat.addition.get('problems', {}),
'messageId': message_id,
}
if args.dump is not None and (updated or not os.path.exists(args.dump)):
standings_dump = json.dumps(standings, indent=2)
with open(args.dump, 'w') as fo:
fo.write(standings_dump)
if iteration:
is_over = contest.end_time < now()
if is_over and not has_hidden:
break
tick = args.delay * 5 if is_over else args.delay
limit = now() + timedelta(seconds=tick)
size = 1
while now() < limit:
value = humanize.naturaldelta(limit - now())
out = f'{value:{size}s}'
size = len(value)
print(out, end='\r')
time.sleep(1)
print()
iteration += 1
