def parse_statistic(
self,
contests,
previous_days=None,
freshness_days=None,
limit=None,
with_check=True,
stop_on_error=False,
random_order=False,
no_update_results=False,
limit_duration_in_secs=7 * 60 * 60, # 7 hours
title_regex=None,
users=None,
with_stats=True,
):
now = timezone.now()
if with_check:
if previous_days is not None:
contests = contests.filter(end_time__gt=now - timedelta(days=previous_days), end_time__lt=now)
else:
contests = contests.filter(Q(timing__statistic__isnull=True) | Q(timing__statistic__lt=now))
started = contests.filter(start_time__lt=now, end_time__gt=now, statistics__isnull=False)
query = Q()
query &= (
Q(end_time__gt=now - F('resource__module__max_delay_after_end'))
| Q(timing__statistic__isnull=True)
)
query &= Q(end_time__lt=now - F('resource__module__min_delay_after_end'))
ended = contests.filter(query)
contests = started.union(ended)
contests = contests.distinct('id')
elif title_regex:
contests = contests.filter(title__iregex=title_regex)
else:
contests = contests.filter(end_time__lt=now - F('resource__module__min_delay_after_end'))
if freshness_days is not None:
contests = contests.filter(updated__lt=now - timedelta(days=freshness_days))
if limit:
contests = contests.order_by('-start_time')[:limit]
with transaction.atomic():
for c in contests:
module = c.resource.module
delay_on_success = module.delay_on_success or module.max_delay_after_end
if now < c.end_time:
if c.end_time - c.start_time <= timedelta(seconds=limit_duration_in_secs):
delay_on_success = timedelta(minutes=0)
elif c.end_time < now + delay_on_success:
delay_on_success = c.end_time - now + timedelta(seconds=5)
TimingContest.objects.update_or_create(
contest=c,
defaults={'statistic': now + delay_on_success}
)
if random_order:
contests = list(contests)
shuffle(contests)
countrier = Countrier()
count = 0
total = 0
progress_bar = tqdm(contests)
for contest in progress_bar:
resource = contest.resource
if not hasattr(resource, 'module'):
self.logger.error('Not found module contest = %s' % c)
continue
progress_bar.set_description(f'contest = {contest.title}')
progress_bar.refresh()
plugin = self._get_plugin(resource.module)
total += 1
parsed = False
try:
r = {}
if hasattr(contest, 'stage'):
contest.stage.update()
count += 1
parsed = True
continue
plugin = plugin.Statistic(contest=contest)
with REQ:
statistics_by_key = {}
statistics_ids = set()
if not no_update_results:
statistics = Statistics.objects.filter(contest=contest).select_related('account')
for s in tqdm(statistics.iterator(), 'getting parsed statistics'):
if with_stats:
statistics_by_key[s.account.key] = s.addition
statistics_ids.add(s.pk)
standings = plugin.get_standings(users=users, statistics=statistics_by_key)
with transaction.atomic():
if 'url' in standings and standings['url'] != contest.standings_url:
contest.standings_url = standings['url']
contest.save()
if 'title' in standings and standings['title'] != contest.title:
contest.title = standings['title']
contest.save()
if 'options' in standings:
contest_options = contest.info.get('standings', {})
standings_options = dict(contest_options)
standings_options.update(standings.pop('options'))
if self._canonize(standings_options) != self._canonize(contest_options):
contest.info['standings'] = standings_options
contest.save()
problems_time_format = standings.pop('problems_time_format', '{M}:{s:02d}')
result = standings.get('result', {})
if not no_update_results and result:
fields_set = set()
fields_types = {}
fields = list()
calculate_time = False
d_problems = {}
teams_viewed = set()
has_hidden = False
languages = set()
for r in tqdm(list(result.values()), desc=f'update results {contest}'):
for k, v in r.items():
if isinstance(v, str) and chr(0x00) in v:
r[k] = v.replace(chr(0x00), '')
member = r.pop('member')
account, created = Account.objects.get_or_create(resource=resource, key=member)
updated = now + timedelta(days=1)
if created or (not statistics_ids and updated < account.updated):
account.updated = updated
account.save()
if r.get('name'):
while True:
name = unescape(r['name'])
if name == r['name']:
break
r['name'] = name
no_update_name = r.pop('_no_update_name', False)
if not no_update_name and account.name != r['name'] and member.find(r['name']) == -1:
account.name = r['name']
account.save()
country = r.get('country', None)
if country:
country = countrier.get(country)
if country and country != account.country:
account.country = country
account.save()
contest_addition_update = r.pop('contest_addition_update', {})
if contest_addition_update:
account_update_contest_additions(
account,
contest_addition_update,
timedelta(days=31) if with_check else None
)
account_info = r.pop('info', {})
if account_info:
if 'rating' in account_info:
account_info['rating_ts'] = int(now.timestamp())
account.info.update(account_info)
account.save()
problems = r.get('problems', {})
_languages = set()
for problem in problems.values():
if problem.get('language'):
languages.add(problem['language'])
_languages.add(problem['language'])
if '_languages' not in r and _languages:
r['_languages'] = list(sorted(_languages))
if 'team_id' not in r or r['team_id'] not in teams_viewed:
if 'team_id' in r:
teams_viewed.add(r['team_id'])
for k, v in problems.items():
if 'result' not in v:
continue
p = d_problems
if 'division' in r:
p = p.setdefault(r['division'], {})
p = p.setdefault(k, {})
p['n_teams'] = p.get('n_teams', 0) + 1
ac = str(v['result']).startswith('+')
try:
result = float(v['result'])
ac = ac or result > 0 and not v.get('partial', False)
except Exception:
pass
if ac:
p['n_accepted'] = p.get('n_accepted', 0) + 1
calc_time = contest.calculate_time or contest.start_time <= now < contest.end_time
advance = contest.info.get('advance')
if advance:
k = 'advanced'
r.pop(k, None)
for cond in advance['filter']:
field = cond['field']
value = r.get(field)
value = get_number_from_str(value)
if value is None:
continue
r[k] = getattr(operator, cond['operator'])(value, cond['threshold'])
medals = contest.info.get('standings', {}).get('medals')
if medals:
k = 'medal'
r.pop(k, None)
if 'place' in r:
place = get_number_from_str(r['place'])
if place:
for medal in medals:
if place <= medal['count']:
r[k] = medal['name']
if 'field' in medal:
r[medal['field']] = medal['value']
r[f'_{k}_title_field'] = medal['field']
break
place -= medal['count']
medal_fields = [m['field'] for m in medals if 'field' in m] or [k]
for f in medal_fields:
if f not in fields_set:
fields_set.add(f)
fields.append(f)
additions = contest.info.get('additions', {})
if additions:
for field in [member, r.get('name')]:
r.update(OrderedDict(additions.get(field, [])))
defaults = {
'place': r.pop('place', None),
'solving': r.pop('solving', 0),
'upsolving': r.pop('upsolving', 0),
}
addition = type(r)()
for k, v in r.items():
if k[0].isalpha() and not re.match('^[A-Z]+$', k):
k = k[0].upper() + k[1:]
k = '_'.join(map(str.lower, re.findall('[A-ZА-Я][^A-ZА-Я]*', k)))
if k not in fields_set:
fields_set.add(k)
fields.append(k)
if (k in Resource.RATING_FIELDS or k == 'rating_change') and v is None:
continue
fields_types.setdefault(k, set()).add(type(v).__name__)
addition[k] = v
if (
addition.get('rating_change') is None
and addition.get('new_rating') is not None
and addition.get('old_rating') is not None
):
delta = addition['new_rating'] - addition['old_rating']
f = 'rating_change'
addition[f] = f'{"+" if delta > 0 else ""}{delta}'
if f not in fields_set:
fields_set.add(f)
fields.insert(-1, f)
if 'is_rated' in addition and not addition['is_rated']:
addition.pop('old_rating', None)
if not calc_time:
defaults['addition'] = addition
rating_ts = int(min(contest.end_time, now).timestamp())
if 'new_rating' in addition and (
'rating_ts' not in account.info or account.info['rating_ts'] <= rating_ts
):
account.info['rating_ts'] = rating_ts
account.info['rating'] = addition['new_rating']
account.save()
statistic, created = Statistics.objects.update_or_create(
account=account,
contest=contest,
defaults=defaults,
)
if not created:
statistics_ids.remove(statistic.pk)
if calc_time:
p_problems = statistic.addition.get('problems', {})
ts = min(int((now - contest.start_time).total_seconds()), contest.duration_in_secs)
values = {
'D': ts // (24 * 60 * 60),
'H': ts // (60 * 60),
'h': ts // (60 * 60) % 24,
'M': ts // 60,
'm': ts // 60 % 60,
'S': ts,
's': ts % 60,
}
time = problems_time_format.format(**values)
for k, v in problems.items():
v_result = v.get('result', '')
if isinstance(v_result, str) and '?' in v_result:
calculate_time = True
p = p_problems.get(k, {})
if 'time' in v:
continue
has_change = v.get('result') != p.get('result')
if (not has_change or contest.end_time < now) and 'time' in p:
v['time'] = p['time']
else:
v['time'] = time
for p in problems.values():
p_result = p.get('result', '')
if isinstance(p_result, str) and '?' in p_result:
has_hidden = True
if calc_time:
statistic.addition = addition
statistic.save()
if users is None:
timing_statistic_delta = standings.get('timing_statistic_delta',
timedelta(minutes=30) if has_hidden else None)
if timing_statistic_delta is not None:
contest.timing.statistic = timezone.now() + timing_statistic_delta
contest.timing.save()
if contest.start_time <= now:
if now < contest.end_time:
contest.info['last_parse_statistics'] = now.strftime('%Y-%m-%d %H:%M:%S.%f+%Z')
elif 'last_parse_statistics' in contest.info:
contest.info.pop('last_parse_statistics')
if fields_set and not isinstance(addition, OrderedDict):
fields.sort()
fields_types = {k: list(v) for k, v in fields_types.items()}
if statistics_ids:
first = Statistics.objects.filter(pk__in=statistics_ids).first()
self.logger.info(f'First deleted: {first}, account = {first.account}')
delete_info = Statistics.objects.filter(pk__in=statistics_ids).delete()
self.logger.info(f'Delete info: {delete_info}')
progress_bar.set_postfix(deleted=str(delete_info))
if self._canonize(fields) != self._canonize(contest.info.get('fields')):
contest.info['fields'] = fields
contest.info['fields_types'] = fields_types
if calculate_time and not contest.calculate_time:
contest.calculate_time = True
problems = standings.pop('problems', None)
if 'division' in problems:
for d, ps in problems['division'].items():
for p in ps:
k = get_problem_key(p)
if k:
p.update(d_problems.get(d, {}).get(k, {}))
if isinstance(problems['division'], OrderedDict):
problems['divisions_order'] = list(problems['division'].keys())
else:
for p in problems:
k = get_problem_key(p)
if k:
p.update(d_problems.get(k, {}))
if problems and self._canonize(problems) != self._canonize(contest.info.get('problems')):
contest.info['problems'] = problems
if languages:
languages = list(sorted(languages))
if self._canonize(languages) != self._canonize(contest.info.get('languages')):
contest.info['languages'] = languages
contest.save()
progress_bar.set_postfix(n_fields=len(fields))
action = standings.get('action')
if action is not None:
args = []
if isinstance(action, tuple):
action, *args = action
self.logger.info(f'Action {action} with {args}, contest = {contest}, url = {contest.url}')
if action == 'delete':
if Statistics.objects.filter(contest=contest).exists():
self.logger.info(f'No deleted. Contest have statistics')
elif now < contest.end_time:
self.logger.info(f'No deleted. Try after = {contest.end_time - now}')
else:
delete_info = contest.delete()
self.logger.info(f'Delete info contest: {delete_info}')
elif action == 'url':
contest.url = args[0]
contest.save()
if 'result' in standings:
count += 1
parsed = True
except (ExceptionParseStandings, InitModuleException) as e:
progress_bar.set_postfix(exception=str(e), cid=str(contest.pk))
except Exception as e:
self.logger.error(f'contest = {contest.pk}, error = {e}, row = {r}')
if stop_on_error:
self.logger.error(format_exc())
break
if not parsed:
if now < c.end_time and c.duration_in_secs <= limit_duration_in_secs:
delay = timedelta(minutes=0)
else:
delay = resource.module.delay_on_error
contest.timing.statistic = timezone.now() + delay
contest.timing.save()
else:
stages = Stage.objects.filter(
contest__start_time__lte=contest.start_time,
contest__end_time__gte=contest.end_time,
contest__resource=contest.resource,
)
for stage in stages:
if Contest.objects.filter(pk=contest.pk, **stage.filter_params).exists():
stage.update()
progress_bar.close()
self.logger.info(f'Parse statistic: {count} of {total}.')
return count, total
def parse_statistic(
self,
contests,
previous_days=None,
freshness_days=None,
limit=None,
with_check=True,
stop_on_error=False,
random_order=False,
no_update_results=False,
limit_duration_in_secs=7 * 60 * 60, # 7 hours
title_regex=None,
users=None,
with_stats=True,
update_without_new_rating=None,
without_contest_filter=False,
force_problems=False,
):
now = timezone.now()
contests = contests.select_related('resource__module')
if not without_contest_filter:
if with_check:
if previous_days is not None:
contests = contests.filter(end_time__gt=now -
timedelta(days=previous_days),
end_time__lt=now)
else:
contests = contests.filter(
Q(timing__statistic__isnull=True)
| Q(timing__statistic__lt=now))
started = contests.filter(start_time__lt=now,
end_time__gt=now,
statistics__isnull=False)
query = Q()
query &= (Q(end_time__gt=now -
F('resource__module__max_delay_after_end'))
| Q(timing__statistic__isnull=True))
query &= Q(end_time__lt=now -
F('resource__module__min_delay_after_end'))
ended = contests.filter(query)
contests = started.union(ended)
contests = contests.distinct('id')
elif title_regex:
contests = contests.filter(title__iregex=title_regex)
else:
contests = contests.filter(
end_time__lt=now -
F('resource__module__min_delay_after_end'))
if freshness_days is not None:
contests = contests.filter(updated__lt=now -
timedelta(days=freshness_days))
if limit:
contests = contests.order_by('-start_time')[:limit]
with transaction.atomic():
for c in contests:
module = c.resource.module
delay_on_success = module.delay_on_success or module.max_delay_after_end
if now < c.end_time:
if c.end_time - c.start_time <= timedelta(
seconds=limit_duration_in_secs):
delay_on_success = timedelta(minutes=0)
elif c.end_time < now + delay_on_success:
delay_on_success = c.end_time - now + timedelta(
seconds=5)
TimingContest.objects.update_or_create(
contest=c, defaults={'statistic': now + delay_on_success})
if random_order:
contests = list(contests)
shuffle(contests)
countrier = Countrier()
def canonize_name(name):
while True:
v = unescape(name)
if v == name:
break
name = v
if len(name) > 1024:
name = name[:1020] + '...'
return name
count = 0
total = 0
n_upd_account_time = 0
progress_bar = tqdm(contests)
stages_ids = []
for contest in progress_bar:
resource = contest.resource
if not hasattr(resource, 'module'):
self.logger.error('Not found module contest = %s' % c)
continue
progress_bar.set_description(f'contest = {contest.title}')
progress_bar.refresh()
total += 1
parsed = False
user_info_has_rating = {}
try:
r = {}
if hasattr(contest, 'stage'):
contest.stage.update()
count += 1
parsed = True
continue
plugin = resource.plugin.Statistic(contest=contest)
with REQ:
statistics_by_key = {} if with_stats else None
statistics_ids = set()
if not no_update_results and (users or users is None):
statistics = Statistics.objects.filter(
contest=contest).select_related('account')
if users:
statistics = statistics.filter(
account__key__in=users)
for s in tqdm(statistics.iterator(),
'getting parsed statistics'):
if with_stats:
statistics_by_key[s.account.key] = s.addition
statistics_ids.add(s.pk)
standings = plugin.get_standings(
users=users, statistics=statistics_by_key)
with transaction.atomic():
if 'url' in standings and standings[
'url'] != contest.standings_url:
contest.standings_url = standings['url']
contest.save()
if 'title' in standings and standings[
'title'] != contest.title:
contest.title = standings['title']
contest.save()
if 'options' in standings:
contest_options = contest.info.get('standings', {})
standings_options = dict(contest_options)
standings_options.update(standings.pop('options'))
canonized_fixed_fields = set([
canonize(f)
for f in standings_options.get('fixed_fields', [])
])
for field in contest_options.get('fixed_fields', []):
canonize_field = canonize(field)
if canonize_field not in canonized_fixed_fields:
canonized_fixed_fields.add(canonize_field)
standings_options['fixed_fields'].append(field)
if canonize(standings_options) != canonize(
contest_options):
contest.info['standings'] = standings_options
contest.save()
info_fields = standings.pop('info_fields',
[]) + ['divisions_order']
for field in info_fields:
if standings.get(
field) is not None and contest.info.get(
field) != standings[field]:
contest.info[field] = standings[field]
contest.save()
update_writers(contest, standings.pop('writers', None))
problems_time_format = standings.pop(
'problems_time_format', '{M}:{s:02d}')
standings_hidden_fields = standings.pop(
'hidden_fields', {})
result = standings.get('result', {})
if no_update_results:
continue
if result or users is not None:
fields_set = set()
fields_types = {}
fields = list()
addition_was_ordereddict = False
calculate_time = False
n_statistics = defaultdict(int)
d_problems = {}
teams_viewed = set()
has_hidden = False
languages = set()
hidden_fields = set()
medals_skip = set()
additions = copy.deepcopy(
contest.info.get('additions', {}))
if additions:
for k, v in result.items():
for field in [v.get('member'), v.get('name')]:
v.update(
OrderedDict(additions.pop(field, [])))
for k, v in additions.items():
result[k] = dict(v)
for r in tqdm(list(result.values()),
desc=f'update results {contest}'):
for k, v in r.items():
if isinstance(v, str) and chr(0x00) in v:
r[k] = v.replace(chr(0x00), '')
member = r.pop('member')
account_action = r.pop('action', None)
skip_result = r.get('_no_update_n_contests')
if account_action == 'delete':
Account.objects.filter(resource=resource,
key=member).delete()
continue
account, created = Account.objects.get_or_create(
resource=resource, key=member)
if not contest.info.get('_no_update_account_time'
) and not skip_result:
stats = (statistics_by_key
or {}).get(member, {})
no_rating = with_stats and 'new_rating' not in stats and 'rating_change' not in stats
resource_statistics = contest.resource.info.get(
'statistics', {})
updated_delta = resource_statistics.get(
'account_updated_delta', {'days': 1})
updated = now + timedelta(**updated_delta)
wait_rating = resource_statistics.get(
'wait_rating')
if no_rating and wait_rating:
updated = now + timedelta(hours=1)
title_re = wait_rating.get('title_re')
if ((contest.end_time + timedelta(
days=wait_rating['days']) > now
or update_without_new_rating) and
(not title_re
or re.search(title_re, contest.title))
and updated < account.updated):
division = r.get('division')
if division not in user_info_has_rating:
generator = plugin.get_users_infos(
[member], contest.resource,
[account])
try:
user_info = next(generator)
params = user_info.get(
'contest_addition_update_params',
{})
field = user_info.get(
'contest_addition_update_by'
) or params.get(
'by') or 'key' # noqa
updates = user_info.get(
'contest_addition_update'
) or params.get('update') or {
} # noqa
if not isinstance(
field, (list, tuple)):
field = [field]
user_info_has_rating[
division] = False
for f in field:
if getattr(contest,
f) in updates:
user_info_has_rating[
division] = True
break
except Exception:
self.logger.error(format_exc())
user_info_has_rating[
division] = False
if user_info_has_rating[division]:
n_upd_account_time += 1
account.updated = updated
account.save()
elif (created
or (not statistics_ids
and updated < account.updated)
or (update_without_new_rating
and updated < account.updated
and no_rating)):
n_upd_account_time += 1
account.updated = updated
account.save()
if contest.info.get('_push_name_instead_key'):
r['_name_instead_key'] = True
if contest.info.get(
'_push_name_instead_key_to_account'):
account.info['_name_instead_key'] = True
account.save()
no_update_name = r.pop('_no_update_name', False)
if r.get('name'):
r['name'] = canonize_name(r['name'])
if (not no_update_name
and account.name != r['name']):
account.name = r['name']
account.save()
country = r.get('country', None)
if country:
country = countrier.get(country)
if country and country != account.country:
account.country = country
account.save()
contest_addition_update = r.pop(
'contest_addition_update', {})
if contest_addition_update:
account_update_contest_additions(
account, contest_addition_update,
timedelta(days=31) if with_check else None)
account_info = r.pop('info', {})
if account_info:
if 'rating' in account_info:
account_info['rating_ts'] = int(
now.timestamp())
account.info.update(account_info)
account.save()
default_division = contest.info.get(
'default_division')
if default_division and 'division' not in r:
r['division'] = default_division
problems = r.get('problems', {})
_languages = set()
for problem in problems.values():
if problem.get('language'):
languages.add(problem['language'])
_languages.add(problem['language'])
if '_languages' not in r and _languages:
r['_languages'] = list(sorted(_languages))
if ('team_id' not in r or r['team_id']
not in teams_viewed) and not skip_result:
if 'team_id' in r:
teams_viewed.add(r['team_id'])
solved = {'solving': 0}
if r.get('division'):
n_statistics[r.get('division')] += 1
n_statistics['__total__'] += 1
for k, v in problems.items():
if 'result' not in v:
continue
p = d_problems
if 'division' in standings.get(
'problems', {}):
p = p.setdefault(r['division'], {})
p = p.setdefault(k, {})
scored = str(v['result']).startswith('+')
try:
scored = scored or float(
v['result']) > 0
except Exception:
pass
default_full_score = (
contest.info.get(
'default_problem_full_score')
or contest.resource.info.get(
'statistics', {}).get(
'default_problem_full_score'))
if default_full_score:
if 'partial' not in v and default_full_score - float(
v['result']) > 1e-9:
v['partial'] = True
if not v.get('partial'):
solved['solving'] += 1
if 'full_score' not in p:
p['full_score'] = default_full_score
ac = scored and not v.get('partial', False)
if contest.info.get('with_last_submit_time'
) and scored:
if '_last_submit_time' not in r or r[
'_last_submit_time'] < v[
'time']:
r['_last_submit_time'] = v['time']
if contest.info.get(
'without_problem_first_ac'):
v.pop('first_ac', None)
v.pop('first_ac_of_all', None)
if contest.info.get(
'without_problem_time'):
v.pop('time', None)
if r.get('_skip_for_problem_stat'):
continue
p['n_teams'] = p.get('n_teams', 0) + 1
if ac:
p['n_accepted'] = p.get(
'n_accepted', 0) + 1
elif scored and v.get('partial'):
p['n_partial'] = p.get('n_partial',
0) + 1
elif str(v['result']).startswith('?'):
p['n_hidden'] = p.get('n_hidden',
0) + 1
if 'default_problem_full_score' in contest.info and solved and 'solved' not in r:
r['solved'] = solved
calc_time = contest.calculate_time or (
contest.start_time <= now < contest.end_time
and
not contest.resource.info.get('parse', {}).get(
'no_calculate_time', False))
advance = contest.info.get('advance')
if advance:
k = 'advanced'
r.pop(k, None)
for cond in advance['filter']:
field = cond['field']
value = r.get(field)
value = get_number_from_str(value)
if value is None:
continue
r[k] = getattr(operator, cond['operator'])(
value, cond['threshold'])
medals = contest.info.get('standings',
{}).get('medals')
if medals:
k = 'medal'
r.pop(k, None)
if 'place' in r:
place = get_number_from_str(r['place'])
if member in contest.info.get(
'standings',
{}).get('medals_skip', []):
medals_skip.add(member)
elif place:
place -= len(medals_skip)
for medal in medals:
if place <= medal['count']:
r[k] = medal['name']
if 'field' in medal:
r[medal['field']] = medal[
'value']
r[f'_{k}_title_field'] = medal[
'field']
break
place -= medal['count']
medal_fields = [
m['field'] for m in medals if 'field' in m
] or [k]
for f in medal_fields:
if f not in fields_set:
fields_set.add(f)
fields.append(f)
defaults = {
'place': r.pop('place', None),
'solving': r.pop('solving', 0),
'upsolving': r.pop('upsolving', 0),
}
defaults = {
k: v
for k, v in defaults.items()
if v != '__unchanged__'
}
addition = type(r)()
addition_was_ordereddict |= isinstance(
addition, OrderedDict)
for k, v in r.items():
is_hidden_field = k in standings_hidden_fields
if k[0].isalpha() and not re.match(
'^[A-Z]+$', k):
k = k[0].upper() + k[1:]
k = '_'.join(
map(
str.lower,
re.findall('[A-ZА-Я][^A-ZА-Я]*',
k)))
if is_hidden_field:
hidden_fields.add(k)
if k not in fields_set:
fields_set.add(k)
fields.append(k)
if (k in Resource.RATING_FIELDS
or k == 'rating_change') and v is None:
continue
fields_types.setdefault(k, set()).add(
type(v).__name__)
addition[k] = v
if (addition.get('rating_change') is None
and addition.get('new_rating') is not None
and addition.get('old_rating')
is not None):
delta = addition['new_rating'] - addition[
'old_rating']
f = 'rating_change'
addition[
f] = f'{"+" if delta > 0 else ""}{delta}'
if f not in fields_set:
fields_set.add(f)
fields.append(f)
if 'is_rated' in addition and not addition[
'is_rated']:
addition.pop('old_rating', None)
if not calc_time:
defaults['addition'] = addition
rating_ts = int(
min(contest.end_time, now).timestamp())
if 'new_rating' in addition and (
'rating_ts' not in account.info
or account.info['rating_ts'] <= rating_ts):
account.info['rating_ts'] = rating_ts
account.info['rating'] = addition['new_rating']
account.save()
statistic, created = Statistics.objects.update_or_create(
account=account,
contest=contest,
defaults=defaults,
)
if not created:
statistics_ids.remove(statistic.pk)
if calc_time:
p_problems = statistic.addition.get(
'problems', {})
ts = min(
int((now - contest.start_time
).total_seconds()),
contest.duration_in_secs)
values = {
'D': ts // (24 * 60 * 60),
'H': ts // (60 * 60),
'h': ts // (60 * 60) % 24,
'M': ts // 60,
'm': ts // 60 % 60,
'S': ts,
's': ts % 60,
}
time = problems_time_format.format(
**values)
for k, v in problems.items():
v_result = v.get('result', '')
if isinstance(v_result,
str) and '?' in v_result:
calculate_time = True
p = p_problems.get(k, {})
if 'time' in v:
continue
has_change = v.get('result') != p.get(
'result')
if (not has_change or contest.end_time
< now) and 'time' in p:
v['time'] = p['time']
else:
v['time'] = time
for p in problems.values():
p_result = p.get('result', '')
if isinstance(p_result,
str) and '?' in p_result:
has_hidden = True
if calc_time:
statistic.addition = addition
statistic.save()
if users is None:
timing_statistic_delta = standings.get(
'timing_statistic_delta',
timedelta(minutes=30) if has_hidden
and contest.end_time < now else None,
)
if timing_statistic_delta is not None:
contest.timing.statistic = timezone.now(
) + timing_statistic_delta
contest.timing.save()
if contest.start_time <= now:
if now < contest.end_time:
contest.info[
'last_parse_statistics'] = now.strftime(
'%Y-%m-%d %H:%M:%S.%f+%Z')
elif 'last_parse_statistics' in contest.info:
contest.info.pop('last_parse_statistics')
if fields_set and not addition_was_ordereddict:
fields.sort()
for rating_field in ('old_rating', 'rating_change',
'new_rating'):
if rating_field in fields_set:
fields.remove(rating_field)
fields.append(rating_field)
if statistics_ids:
first = Statistics.objects.filter(
pk__in=statistics_ids).first()
if first:
self.logger.info(
f'First deleted: {first}, account = {first.account}'
)
delete_info = Statistics.objects.filter(
pk__in=statistics_ids).delete()
self.logger.info(f'Delete info: {delete_info}')
progress_bar.set_postfix(
deleted=str(delete_info))
if canonize(fields) != canonize(
contest.info.get('fields')):
contest.info['fields'] = fields
hidden_fields = list(hidden_fields)
if hidden_fields and canonize(
hidden_fields) != canonize(
contest.info.get('hidden_fields')):
contest.info['hidden_fields'] = hidden_fields
fields_types = {
k: list(v)
for k, v in fields_types.items()
}
for k, v in standings.get('fields_types',
{}).items():
fields_types.setdefault(k, []).extend(v)
contest.info['fields_types'] = fields_types
if calculate_time and not contest.calculate_time:
contest.calculate_time = True
contest.n_statistics = n_statistics.pop(
'__total__', 0)
problems = standings.pop('problems', None)
if problems is not None:
if 'division' in problems:
for d, ps in problems['division'].items():
for p in ps:
k = get_problem_short(p)
if k:
p.update(
d_problems.get(d, {}).get(
k, {}))
p['n_total'] = n_statistics[d]
if isinstance(problems['division'],
OrderedDict):
problems['divisions_order'] = list(
problems['division'].keys())
problems['n_statistics'] = n_statistics
else:
for p in problems:
k = get_problem_short(p)
if k:
p.update(d_problems.get(k, {}))
p['n_total'] = contest.n_statistics
update_problems(contest,
problems=problems,
force=force_problems)
if languages:
languages = list(sorted(languages))
if canonize(languages) != canonize(
contest.info.get('languages')):
contest.info['languages'] = languages
contest.save()
progress_bar.set_postfix(n_fields=len(fields))
else:
problems = standings.pop('problems', None)
if problems is not None and problems:
problems = plugin.merge_dict(
problems, contest.info.get('problems'))
if not users:
contest.info['problems'] = {}
update_problems(contest,
problems=problems,
force=force_problems)
action = standings.get('action')
if action is not None:
args = []
if isinstance(action, tuple):
action, *args = action
self.logger.info(
f'Action {action} with {args}, contest = {contest}, url = {contest.url}'
)
if action == 'delete':
if Statistics.objects.filter(
contest=contest).exists():
self.logger.info(
'No deleted. Contest have statistics')
elif now < contest.end_time:
self.logger.info(
f'No deleted. Try after = {contest.end_time - now}'
)
else:
delete_info = contest.delete()
self.logger.info(
f'Delete info contest: {delete_info}')
elif action == 'url':
contest.url = args[0]
contest.save()
if 'result' in standings:
count += 1
parsed = True
except (ExceptionParseStandings, InitModuleException) as e:
progress_bar.set_postfix(exception=str(e), cid=str(contest.pk))
except Exception as e:
self.logger.error(
f'contest = {contest.pk}, error = {e}, row = {r}')
if stop_on_error:
self.logger.error(format_exc())
break
if not parsed:
if now < c.end_time and c.duration_in_secs <= limit_duration_in_secs:
delay = timedelta(minutes=0)
else:
delay = resource.module.delay_on_error
contest.timing.statistic = timezone.now() + delay
contest.timing.save()
elif not no_update_results and (users is None or users):
stages = Stage.objects.filter(
~Q(pk__in=stages_ids),
contest__start_time__lte=contest.start_time,
contest__end_time__gte=contest.end_time,
contest__resource=contest.resource,
)
for stage in stages:
if Contest.objects.filter(pk=contest.pk,
**stage.filter_params).exists():
stages_ids.append(stage.pk)
for stage in tqdm(Stage.objects.filter(pk__in=stages_ids),
total=len(stages_ids),
desc='getting stages'):
stage.update()
progress_bar.close()
self.logger.info(
f'Parsed statistic: {count} of {total}. Updated account time: {n_upd_account_time}'
)
return count, total
def handle(self, *args, **options):
self.stdout.write(str(options))
args = AttrDict(options)
has_param = args.resources or args.query or args.limit
if args.resources:
filt = Q()
for r in args.resources:
filt |= Q(host__iregex=r)
resources = Resource.objects.filter(filt)
else:
resources = Resource.objects.filter(module__has_accounts_infos_update=True)
countrier = Countrier()
now = timezone.now()
for resource in resources:
with transaction.atomic():
plugin = self._get_plugin(resource.module)
accounts = resource.account_set
if args.query:
accounts = accounts.filter(key__iregex=args.query)
else:
accounts = accounts.filter(Q(updated__isnull=True) | Q(updated__lte=now))
total = accounts.count()
accounts = list(accounts[:args.limit])
if not accounts:
continue
try:
with tqdm(total=len(accounts), desc=f'getting {resource.host} (total = {total})') as pbar:
infos = plugin.Statistic.get_users_infos(
users=[a.key for a in accounts],
resource=resource,
accounts=accounts,
pbar=pbar,
)
for account, data in zip(accounts, infos):
info = data['info']
if info is None:
_, info = account.delete()
info = {k: v for k, v in info.items() if v}
pbar.set_postfix(warning=f'Remove user {account} = {info}')
continue
contest_addition_update = data.pop('contest_addition_update', {})
contest_addition_update_by = data.pop('contest_addition_update_by', None)
if contest_addition_update:
account_update_contest_additions(
account,
contest_addition_update,
timedelta_limit=timedelta(days=31) if account.info and not has_param else None,
by=contest_addition_update_by,
)
coders = data.pop('coders', [])
if coders:
qs = Coder.objects \
.filter(account__resource=resource, account__key__in=coders) \
.exclude(account=account)
for c in qs:
c.account_set.add(account)
if 'rename' in data:
other, created = Account.objects.get_or_create(resource=account.resource,
key=data['rename'])
if not created:
new = MergedModelInstance.create(other, [account])
account.delete()
else:
new = MergedModelInstance.create(account, [other])
other.delete()
account = new
if info.get('country'):
account.country = countrier.get(info['country'])
if info.get('rating'):
info['rating_ts'] = int(now.timestamp())
delta = info.pop('delta', timedelta(days=365))
account.info.update(info)
account.updated = now + delta
account.save()
except Exception:
if not has_param:
for account in tqdm(accounts, desc='changing update time'):
account.updated = now + timedelta(days=1)
account.save()
self.logger.error(format_exc())
self.logger.error(f'resource = {resource}')
def handle(self, *args, **options):
self.stdout.write(str(options))
args = AttrDict(options)
has_param = args.resources or args.query or args.limit
if args.resources:
filt = Q()
for r in args.resources:
filt |= Q(host__iregex=r)
resources = Resource.objects.filter(filt)
else:
resources = Resource.objects.filter(
module__has_accounts_infos_update=True)
countrier = Countrier()
now = timezone.now()
for resource in resources:
accounts = resource.account_set
if args.query:
accounts = accounts.filter(key__iregex=args.query)
elif args.force:
accounts = accounts.order_by('updated')
else:
accounts = accounts.filter(
Q(updated__isnull=True) | Q(updated__lte=now))
count, total = 0, accounts.count()
resource_info = resource.info.get('accounts', {})
if args.limit or not resource_info.get(
'nolimit', False) or resource_info.get('limit'):
limit = resource_info.get('limit') or args.limit or 1000
accounts = accounts[:limit]
accounts = list(accounts)
if not accounts:
continue
try:
with tqdm(total=len(accounts),
desc=f'getting {resource.host} (total = {total})'
) as pbar:
infos = resource.plugin.Statistic.get_users_infos(
users=[a.key for a in accounts],
resource=resource,
accounts=accounts,
pbar=pbar,
)
for account, data in zip(accounts, infos):
with transaction.atomic():
if data.get('skip'):
continue
count += 1
info = data['info']
if info is None:
_, info = account.delete()
info = {k: v for k, v in info.items() if v}
pbar.set_postfix(
warning=f'Remove user {account} = {info}')
continue
params = data.pop('contest_addition_update_params',
{})
contest_addition_update = data.pop(
'contest_addition_update',
params.pop('update', {}))
contest_addition_update_by = data.pop(
'contest_addition_update_by',
params.pop('by', None))
if contest_addition_update:
account_update_contest_additions(
account,
contest_addition_update,
timedelta_limit=timedelta(days=31) if
account.info and not has_param else None,
by=contest_addition_update_by,
**params,
)
if 'rename' in data:
other, created = Account.objects.get_or_create(
resource=account.resource,
key=data['rename'])
new = MergedModelInstance.create(
other, [account])
account.delete()
account = new
account.save()
coders = data.pop('coders', [])
if coders:
qs = Coder.objects \
.filter(account__resource=resource, account__key__in=coders) \
.exclude(account=account)
for c in qs:
account.coders.add(c)
if info.get('country'):
account.country = countrier.get(
info['country'])
if info.get('name'):
account.name = info['name']
if 'rating' in info:
info['rating_ts'] = int(now.timestamp())
delta = info.pop('delta', timedelta(days=365))
if data.get('replace_info'):
for k, v in account.info.items():
if k.endswith('_') and k not in info:
info[k] = v
account.info = info
else:
account.info.update(info)
account.updated = now + delta
account.save()
except Exception:
if not has_param:
for account in tqdm(accounts, desc='changing update time'):
account.updated = now + timedelta(days=1)
account.save()
self.logger.error(format_exc())
self.logger.error(f'resource = {resource}')
self.logger.info(
f'Parsed accounts infos (resource = {resource}): {count} of {total}'
)