def _fetch_user_problems(spojId, contest, database): url = SPOJ_URLS[contest] + '/status/' + spojId + '/signedlist/' html = tostring(_raw_fetch(url)) item = extract_submissions_data(spojId, html) parsedProblems = parseSignedlist(item['data']) if len(parsedProblems) > 0: database.update_submission_data(dict(item))
def build_solve_prob_div(self, database, userId, contest, body):
title = HtmlElement('h2').addAttr(Attr('class', 'center-align')).addNode('Problemas resolvidos pelo usuário')
body.addNode(title)
allProblems = parseSignedlist(database.get_problems_of_user_from_db(userId)['data'])
problems = sumarise_problems_by_column(allProblems, 'PROBLEM')
body.addNode(self.build_probs_table(problems, contest))
def _fetch_user_problems(spojId, contest, database):
url = SPOJ_URLS[contest] + '/status/' + spojId + '/signedlist/'
html = tostring(_raw_fetch(url))
item = extract_submissions_data(spojId, html)
parsedProblems = parseSignedlist(item['data'])
if len(parsedProblems) > 0:
database.update_submission_data(dict(item))
def _sumarize_user(submissions):
problems = parseSignedlist(submissions['data'])
accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC')
size = len(accepted)
if size not in ACCEPTED_CNT:
ACCEPTED_CNT[size] = 0
ACCEPTED_CNT[size] += 1
def get_solved_problems_by_user(self, user): subs = self.get_database().get_problems_of_user_from_db(user) probs = parseSignedlist(subs['data']) if probs is not None: p = sumarise_problems_by_column(probs, LABEL_COLUMN_ACCEPTED) if ACCEPTED in p: return [prob[LABEL_PROBLEM_COLUMN] for prob in p[ACCEPTED]] return []
def _sumarize_user(submissions): problems = parseSignedlist(submissions['data']) accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC') size = len(accepted) if size not in ACCEPTED_CNT: ACCEPTED_CNT[size] = 0 ACCEPTED_CNT[size] += 1
def process_submission(self, submission, dacus): subDict = parseSignedlist(submission['data']) probs = sumarise_problems_by_column(subDict, LABEL_COLUMN_ACCEPTED) try: distinctProbs = set([x[LABEL_PROBLEM_COLUMN] for x in probs[ACCEPTED]]) except: print submission['_id'], probs for prob in distinctProbs: dacus[prob] += 1
def process_submission(self, submission, acRate): subDict = parseSignedlist(submission['data']) probs = sumarise_problems_by_column(subDict, 'RESULT') cnt = 0 for prob in probs[ACCEPTED]: acRate[prob['PROBLEM']] += 1 cnt += 1 self.totalAcc += cnt
def process_submission(self, submission, acRate):
subDict = parseSignedlist(submission['data'])
probs = sumarise_problems_by_column(subDict, 'RESULT')
cnt = 0
for prob in probs[ACCEPTED]:
acRate[prob['PROBLEM']] += 1
cnt += 1
self.totalAcc += cnt
def process_submission(self, submission, dacus):
subDict = parseSignedlist(submission['data'])
probs = sumarise_problems_by_column(subDict, LABEL_COLUMN_ACCEPTED)
try:
distinctProbs = set(
[x[LABEL_PROBLEM_COLUMN] for x in probs[ACCEPTED]])
except:
print submission['_id'], probs
for prob in distinctProbs:
dacus[prob] += 1
def parseSubmissions(self, response):
identifier = response.url.split('/status/')[1].split('/signedlist/')[0]
log.msg('Crawling signedlist of user %s from %s.' % (identifier, response.url), level=log.INFO)
item = extract_submissions_data(identifier, response.body)
yield item
problems = parseSignedlist(item['data'])
for prob in problems:
url = str(self.allowed_domains[0]) + '/problems/' + prob[LABEL_PROBLEM_COLUMN] + '/'
yield Request(url=url)
def process_submission(self, submission): user = submission['_id'] subDict = parseSignedlist(submission['data']) probs = sumarise_problems_by_column(subDict, 'RESULT') distinctProbs = set([x['PROBLEM'] for x in probs[ACCEPTED]]) self.dacu[user] = self.dacuRec.compute_user_dacu(distinctProbs, self.topk) for problem in distinctProbs: if problem not in self.usersByProblems: self.usersByProblems[problem] = [] self.usersByProblems[problem].append(user)
def _sumarize_mean_dacu(submissions): problems = parseSignedlist(submissions['data']) accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC') for prob in accepted: MEAN_DACU[submissions[idField]] += DACU[prob] if len(accepted) == 0: MEAN_DACU[submissions[idField]] = 10000 else: MEAN_DACU[submissions[idField]] /= len(accepted)
def _sumarize_mean_dacu(submissions):
problems = parseSignedlist(submissions['data'])
accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC')
for prob in accepted:
MEAN_DACU[submissions[idField]] += DACU[prob]
if len(accepted) == 0:
MEAN_DACU[submissions[idField]] = 10000
else:
MEAN_DACU[submissions[idField]] /= len(accepted)
def process_submission(self, submission):
user = submission['_id']
subDict = parseSignedlist(submission['data'])
probs = sumarise_problems_by_column(subDict, 'RESULT')
distinctProbs = set([x['PROBLEM'] for x in probs[ACCEPTED]])
self.dacu[user] = self.dacuRec.compute_user_dacu(
distinctProbs, self.topk)
for problem in distinctProbs:
if problem not in self.usersByProblems:
self.usersByProblems[problem] = []
self.usersByProblems[problem].append(user)
def build_stats_div(self, database, userId, head, body):
allProblems = parseSignedlist(database.get_problems_of_user_from_db(userId)['data'])
problems = sumarise_problems_by_column(allProblems, 'RESULT')
subTable = self.build_submission_stats_table(problems)
probTable = self.build_problems_stats_table(allProblems)
table = HtmlElement('table')
table.addAttr(Attr('class', 'allStats'))
body.addNode(table)
tr = HtmlElement('tr')
table.addNode(tr)
tr.addNode(HtmlElement('td').addNode(probTable))
tr.addNode(HtmlElement('td').addNode(subTable))
self.build_submitions_chart(allProblems, head, body)
def process_submission(self, submission):
user = submission['_id']
subDict = parseSignedlist(submission['data'])
probs = sumarise_problems_by_column(subDict, 'RESULT')
distinctProbs = set([x['PROBLEM'] for x in probs[ACCEPTED]])
self.solvedProblemsByUser[user] = distinctProbs
self.hubs[user] = 1.0
self.auth[user] = 1.0
for problem in distinctProbs:
if problem not in self.usersByProblems:
self.usersByProblems[problem] = []
self.hubs[problem] = 1.0
self.auth[problem] = 1.0
self.usersByProblems[problem].append(user)
def process_submission(self, submission): user = submission['_id'] subDict = parseSignedlist(submission['data']) probs = sumarise_problems_by_column(subDict, 'RESULT') distinctProbs = set([x['PROBLEM'] for x in probs[ACCEPTED]]) self.solvedProblemsByUser[user] = distinctProbs self.hubs[user] = 1.0 self.auth[user] = 1.0 for problem in distinctProbs: if problem not in self.usersByProblems: self.usersByProblems[problem] = [] self.hubs[problem] = 1.0 self.auth[problem] = 1.0 self.usersByProblems[problem].append(user)
def gen_dacu_vs_prob_count(contest, plotName, sumarizeFunc, *args, **kwargs):
database = Database(contest)
database.iterate_over_submissions(_sumarize_dacu)
database.iterate_over_submissions(sumarizeFunc, None, *args, **kwargs)
with open(plotName + '.dat', 'w') as f:
print >>f, '# <# problems> <dacu> <user>'
for (user, dacu) in sorted(MEAN_DACU.items(), key=itemgetter(1), reverse=True):
subOfUser = database.get_problems_of_user_from_db(user)
if subOfUser is not None:
subs = parseSignedlist(subOfUser['data'])
accepted = set(x['PROBLEM'] for x in subs if x['RESULT'] == 'AC')
print >>f, len(accepted), dacu, user
else:
print sys.stderr, user + ' not found'
write_plot(plotName, "Prob count", "DACU", "DACU vs prob count", 13000, 'points')
print subprocess.Popen("gnuplot <" + plotName + ".plot", shell=True, stdout=PIPE).stdout.read()
def _sumarize_mean_top_dacu(submissions, topk): problems = parseSignedlist(submissions['data']) accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC') dacu = [(x, DACU[x]) for x in accepted] cnt = 0 for (prob, dacu) in sorted(dacu, key=itemgetter(1), reverse=False): if cnt >= topk: break MEAN_DACU[submissions[idField]] += dacu cnt += 1 if len(accepted) == 0: if idField in submissions: MEAN_DACU[submissions[idField]] = 10000 else: print >>sys.stderr, idField + ' not found at: ' + str(submissions) else: MEAN_DACU[submissions[idField]] /= topk
def _sumarize_mean_top_dacu(submissions, topk):
problems = parseSignedlist(submissions['data'])
accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC')
dacu = [(x, DACU[x]) for x in accepted]
cnt = 0
for (prob, dacu) in sorted(dacu, key=itemgetter(1), reverse=False):
if cnt >= topk:
break
MEAN_DACU[submissions[idField]] += dacu
cnt += 1
if len(accepted) == 0:
if idField in submissions:
MEAN_DACU[submissions[idField]] = 10000
else:
print >> sys.stderr, idField + ' not found at: ' + str(submissions)
else:
MEAN_DACU[submissions[idField]] /= topk
def gen_dacu_vs_prob_count(contest, plotName, sumarizeFunc, *args, **kwargs):
database = Database(contest)
database.iterate_over_submissions(_sumarize_dacu)
database.iterate_over_submissions(sumarizeFunc, None, *args, **kwargs)
with open(plotName + '.dat', 'w') as f:
print >> f, '# <# problems> <dacu> <user>'
for (user, dacu) in sorted(MEAN_DACU.items(),
key=itemgetter(1),
reverse=True):
subOfUser = database.get_problems_of_user_from_db(user)
if subOfUser is not None:
subs = parseSignedlist(subOfUser['data'])
accepted = set(x['PROBLEM'] for x in subs
if x['RESULT'] == 'AC')
print >> f, len(accepted), dacu, user
else:
print sys.stderr, user + ' not found'
write_plot(plotName, "Prob count", "DACU", "DACU vs prob count", 13000,
'points')
print subprocess.Popen("gnuplot <" + plotName + ".plot",
shell=True,
stdout=PIPE).stdout.read()
def _sumarize_dacu(submissions): problems = parseSignedlist(submissions['data']) accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC') for prob in accepted: DACU[prob] += 1
def _sumarize_dacu(submissions):
problems = parseSignedlist(submissions['data'])
accepted = set(x['PROBLEM'] for x in problems if x['RESULT'] == 'AC')
for prob in accepted:
DACU[prob] += 1