def ov(self, output_id):
output = DBSession.query(Output).get(int(output_id))
if output.group.competition_id != self.competition.id:
abort(404)
if not output.active:
abort(404, "This output is no longer active.")
user = request.identity and request.identity['user']
if user:
group = (DBSession.query(Group).filter(
Group.competition_id == self.competition.id).filter(
Group.users.any(id=user.id)).first())
else:
group = None
message = request.POST.get('message')
if group and message:
if output.use_ground_truth:
abort(403, "The instructor has already reviewed this output.")
output.use_ground_truth = True
protest = Protest(
message=message,
accepted=output.verification != output.ground_truth,
submitter=group,
output=output)
DBSession.add(output)
DBSession.add(protest)
flash('Your protest has been submitted.', 'success')
return {
'output': output,
'group': group,
'competition': self.competition
}
def submit_verification(self, output_id, status):
user = request.identity['user']
try:
output_id = int(output_id)
except ValueError:
return {
'status':
'error',
'msg':
'There was an error. Please try clearing your '
'browser cache and repeating this action.'
}
output = DBSession.query(Output).get(output_id)
if not output or output.input.group_id != self.group.id:
abort(404)
try:
status = VerificationStatus[status]
except KeyError:
abort(404)
if not user.admin and not self.group.competition.verification_open:
return {'status': 'error', 'msg': 'Verification closed'}
assert output.original is True
assert output.active is True
assert output.use_ground_truth is False
output.verification = status
return self.verification_data()
def verification_data(self):
data = {
k: str(v)
for k, v in (DBSession.query(Output.id, Output.verification).join(
Output.input).filter(Input.group_id == self.group.id))
}
return {'status': 'success', 'data': data}
def submit_evaluations(self):
me = request.identity['user']
students_allowed = set(user.id for user in self.group.users)
for user_id, score in request.POST.items():
user_id = int(user_id)
if user_id not in students_allowed:
abort(403, "You cannot submit an evaluation to this user")
score = float(score)
if score < 0:
abort(403, "Negative contributions are not allowed")
e = (DBSession.query(Evaluation)
.filter(Evaluation.from_student_id == me.id)
.filter(Evaluation.to_student_id == user_id)
.filter(Evaluation.group_id == self.group.id)
.one_or_none())
if not e:
e = Evaluation(
from_student_id=me.id,
to_student_id=user_id,
group=self.group,
score=score)
else:
e.score = score
DBSession.add(e)
flash('Your evaluations have been saved. Thank you.', 'success')
redirect('/group/{}'.format(self.group.id))
def index(self):
now = datetime.datetime.now()
comps = (DBSession.query(Competition).order_by(
Competition.input_upload_ends))
active, historical = [], []
for c in comps:
(active if c.end >= now else historical).append(c)
return {'active': active, 'historical': historical}
def index(self):
now = datetime.datetime.now()
comps = (DBSession.query(Competition).filter(
Competition.input_upload_begins <= now).filter(
Competition.open_verification_ends > now).order_by(
Competition.input_upload_ends).all())
if len(comps) == 1:
redirect('/competition/{}'.format(comps[0].id))
return {'competitions': comps}
def _lookup(self, comp_id, *args):
try:
comp_id = int(comp_id)
except ValueError:
abort(404, "Invalid input for competition id")
competition = DBSession.query(Competition).get(comp_id)
if not competition:
abort(404, "No such competition")
return CompetitionController(competition), args
def ov(self, output_id):
output = DBSession.query(Output).get(int(output_id))
if output.group.competition_id != self.competition.id:
abort(404)
if not output.active:
abort(404, "This output is no longer active.")
user = request.identity and request.identity['user']
if user:
group = (DBSession.query(Group).filter(
Group.competition_id == self.competition.id).filter(
Group.users.any(id=user.id)).first())
else:
group = None
problem = problemlib.load_problem(self.competition)
problem_module = problem.get_module()
show_input_downloads = ((user and user.admin)
or not output.group.competition.archived)
message = request.POST.get('message')
if group and message:
if output.use_ground_truth:
abort(403, "The instructor has already reviewed this output.")
output.use_ground_truth = True
protest = Protest(
message=message,
accepted=output.verification != output.ground_truth,
submitter=group,
output=output)
DBSession.add(output)
DBSession.add(protest)
flash('Your protest has been submitted.', 'success')
return {
"output": output,
"score": problem_module.Output.repr_score(output.score),
"group": group,
"show_input_downloads": show_input_downloads,
"competition": self.competition,
}
def authenticate(self, environ, identity):
if identity.get("identifier") != "token":
return None
client_id = identity["token"]
token = (DBSession.query(AuthToken).filter(
AuthToken.client_id == client_id).one_or_none())
if not token:
return None
return token.user.username
def _lookup(self, group_id, *args):
try:
group_id = int(group_id)
except ValueError:
abort(404, "Not a valid group identifier.")
user = request.identity['user']
group = DBSession.query(Group).get(group_id)
if not group:
abort(404, "No such group.")
if user not in group.users and not user.admin:
abort(403, "You are not a part of this group.")
return GroupController(group), args
def archive(self):
now = datetime.datetime.now()
comps = []
for comp in (DBSession.query(Competition).filter(
Competition.open_verification_ends <= now).order_by(
Competition.input_upload_ends.desc())):
if not comps or comps[-1].year != comp.input_upload_ends.year:
comps.append(
CompetitionYearTuple(comp.input_upload_ends.year, [comp]))
else:
comps[-1].competitions.append(comp)
return {'competitions': comps}
def resolution_protest(self, output_id):
if not self.group.competition.resolution_open:
return {'status': 'error',
'msg': 'Resolution stage is not open.'}
output = DBSession.query(Output).get(output_id)
if not output or output.group_id != self.group.id:
abort(404)
if not output.active:
return {'status': 'error',
'msg': 'Output has been replaced; cannot protest.'}
if output.use_ground_truth:
return {'status': 'error',
'msg': 'Output has already been protested.'}
assert output.original
output.use_ground_truth = True
return {'status': 'success'}
def verification_outputs(self):
user = request.identity['user']
if not user.admin and not self.group.competition.verification_open:
abort(403, "This file is only available during verification.")
f = BytesIO()
archive = zipfile.ZipFile(f, mode='w')
outputs = (DBSession.query(Output).join(
Output.input).filter(Input.group_id == self.group.id))
for output in outputs:
archive.writestr(
'verification_outputs/{}'.format(output.data.filename),
output.data.file.read())
archive.close()
f.seek(0)
response.content_type = 'application/zip'
return f.read()
def index(self):
user = request.identity['user']
submitted_outputs = {o.input.id: o for o in self.group.outputs}
d = {
'competition': self.group.competition,
'group': self.group,
'submitted_outputs': submitted_outputs
}
if self.group.competition.evaluation_open:
evals = dict(
DBSession.query(
Evaluation.to_student_id, Evaluation.score).filter(
Evaluation.group_id == self.group.id).filter(
Evaluation.from_student_id == user.id))
for member in self.group.users:
if member.id not in evals.keys():
evals[member.id] = 1.0
d['evals'] = evals
return d
def all_inputs(self):
user = request.identity and request.identity['user']
now = datetime.datetime.now()
comp = self.competition
if not (user and user.admin) and now < comp.output_upload_begins:
abort(
403, "Input downloading is not available until the output"
" upload stage begins.")
f = BytesIO()
archive = zipfile.ZipFile(f, mode='w')
inputs = (DBSession.query(Input).join(
Input.group).filter(Group.competition_id == comp.id))
for iput in inputs:
archive.writestr('inputs/{}'.format(iput.data.filename),
iput.data.file.read())
archive.close()
f.seek(0)
response.content_type = 'application/zip'
return f.read()
def authenticate(self, environ, identity):
try:
code = identity["code"]
except KeyError:
return None
if identity["identifier"] != "glogin":
return None
self.flow.fetch_token(code=code)
with gdiscovery.build("oauth2",
"v2",
credentials=self.flow.credentials) as service:
userinfo = service.userinfo().get().execute()
email = userinfo["email"]
user = DBSession.query(User).filter(User.email == email).one_or_none()
if not user:
# TODO: Handle unregistered users?
return None
return user.username
def authenticate(self, environ, identity):
try:
ticket = identity['login']
except KeyError:
return None
if identity['identifier'] != 'mpapi':
return None
r = requests.post(self.mpapi_fetch, data={'tkt': ticket})
r.raise_for_status()
data = r.json()
if data['result'] != 'success':
raise ValueError('MPAPI Failure')
username = data['uid']
attributes = data['attributes']
uid = attributes['uidNumber']
full_name = '{} {}'.format(attributes['first'], attributes['sn'])
email = attributes['mail']
user = DBSession.query(User).filter(User.id == uid).one_or_none()
if user:
# Update attributes in case of username/full name change
user.username = username
user.full_name = full_name
user.email = email
else:
user = User(id=uid,
username=username,
full_name=full_name,
email=email)
DBSession.add(user)
DBSession.flush()
transaction.commit()
return username
def grade(self):
rankings = self.index(ground_truth=True)
def new_gt(rankings_entry):
return GradingTuple(rankings_entry, GradingVerificationTuple(),
GradingInputTuple([], set()),
GradingContributionTuple(), defaultdict(dict))
groups = {k: new_gt(v) for k, v in rankings['groups'].items()}
compinfo = CompInfoTuple(len(rankings['inputs']), float("inf"), 0, 0)
# in the case a group submitted an input but has no outputs
# uploaded yet, they won't be in groups as they are off the
# rankings table. in this case, we need to make a
# GradingTuples for them now.
for group in self.competition.groups:
if group not in groups.keys():
rankings_entry = GroupEntry()
# If they submitted nothing, then everything is a "reject"
rankings_entry.reject_count = compinfo.inputs
groups[group] = new_gt(rankings_entry)
for group, gt in groups.items():
# compute verification correctness
q = (DBSession.query(Output).filter(Output.original == True).join(
Output.input).filter(Input.group_id == group.id))
for oput in q:
if oput.verification == oput.ground_truth:
gt.verification.correct += 1
elif oput.verification == VerificationStatus.accepted:
gt.verification.false_positives += 1
else:
gt.verification.false_negatives += 1
# compute input unique ranks
for iput, st in gt.rankings.input_ranks.items():
if st.rank is None:
groups[iput.group].input.scores_s.add('R{}'.format(id(st)))
else:
groups[iput.group].input.scores_s.add(st.score)
groups[iput.group].input.scores_l.append(st.score)
adj_score = gt.rankings.adj_score
if adj_score < compinfo.best_score:
compinfo.best_score = adj_score
if adj_score > compinfo.worst_score:
compinfo.worst_score = adj_score
compinfo.best_input_difference = max(
len(g.input.scores_s) for g in groups.values())
for group, gt in groups.items():
gt.contributions.ranking = (max(
16 - (gt.rankings.adj_score / compinfo.best_score), 0))
gt.contributions.verification = (gt.verification.correct /
sum(gt.verification) *
5) if any(gt.verification) else 0
gt.contributions.participation = (
(compinfo.inputs - gt.rankings.reject_count) /
compinfo.inputs * 70)
gt.contributions.input_difficulty = (
7 + len(gt.input.scores_s) / compinfo.best_input_difference *
3) if gt.input.scores_l else 0
for group, gt in groups.items():
for from_member in group.users:
q = (DBSession.query(Evaluation, Evaluation.score).filter(
Evaluation.group_id == group.id).filter(
Evaluation.from_student_id == from_member.id).all())
evals = {e.to_student: s for e, s in q}
for to_member in group.users:
if to_member not in evals.keys():
evals[to_member] = 1.0
for to_member, score in evals.items():
gt.evaluations[to_member][from_member] = (
score / sum(evals.values()))
return {'groups': groups, 'competition': self.competition}
def index(self, ground_truth=False, incognito=False):
user = request.identity and request.identity['user']
now = datetime.datetime.now()
admin = user and user.admin
comp = self.competition
problem = problemlib.load_problem(comp)
show_scores = admin or (comp.open_verification_begins
and now >= comp.open_verification_begins)
if user:
my_groups = (DBSession.query(Group).filter(
Group.competition_id == comp.id).all())
# non-sql filter, bleh
my_groups = [g for g in my_groups if user in g.users]
else:
my_groups = []
show_input_downloads = admin or not comp.archived
show_incognito_option = admin or any(g.incognito for g in my_groups)
if show_incognito_option:
incognito_teams = (DBSession.query(Group).filter(
Group.competition_id == comp.id).filter(
Group.incognito == True).all())
else:
incognito = False
incognito_teams = []
if not admin and now < comp.output_upload_begins:
flash("Rankings are not available yet", "info")
redirect("/competition")
if ground_truth and not admin:
abort(403, "You do not have permission for this option")
if ground_truth:
verification_column = Output.ground_truth
else:
verification_column = case(
[(Output.use_ground_truth, Output.ground_truth)],
else_=Output.verification)
open_verification = user and comp.open_verification_open
problem_module = problem.get_module()
score_sort = {
problemlib.RankSort.minimization: Output.score.asc(),
problemlib.RankSort.maximization: Output.score.desc(),
}[problem_module.Output.rank_sort]
groups = defaultdict(GroupEntry)
ir_query = (DBSession.query(
Input, Group, Output,
verification_column).join(Input.outputs).join(
Output.group).filter(Group.competition_id == comp.id).filter(
Output.active == True).order_by(
Input.group_id,
verification_column == VerificationStatus.rejected,
score_sort))
inputs = []
last_iput = None
accurate_count = 0
total_count = 0
for iput, ogroup, output, verif in ir_query:
if not incognito and ogroup.incognito:
continue
if iput is not last_iput:
inputs.append(iput)
potential_rank = 1
last_rank = 0
last_score = None
shown_score = None
shown_output = None
if show_scores:
shown_score = problem_module.Output.repr_score(output.score)
shown_output = output
if verif is VerificationStatus.rejected:
rank = None
shown_score = None
elif output.score == last_score:
rank = last_rank
else:
rank = potential_rank
vdiffer = False
if ground_truth:
if (output.use_ground_truth
or output.verification == output.ground_truth):
accurate_count += 1
else:
vdiffer = True
groups[ogroup].input_ranks[iput] = ScoreTuple(
shown_score, verif, rank, shown_output, vdiffer)
if verif is VerificationStatus.rejected:
groups[ogroup].reject_count += 1
else:
groups[ogroup].sum_of_ranks += rank
# add accepted resubmissions to penalty
if verif is VerificationStatus.accepted and not output.original:
groups[ogroup].penalties += 1
total_count += 1
potential_rank += 1
last_iput = iput
last_rank = rank
last_score = output.score
# no submission? this adds to reject count
for group in groups.values():
for iput in inputs:
if iput not in group.input_ranks.keys():
group.reject_count += 1
# add open verification protest rejections to penalty
rprotests = (DBSession.query(Protest).filter(
Protest.accepted == False).join(
Protest.submitter).filter(Group.competition_id == comp.id))
for protest in rprotests:
# technically, a group which failed to submit anything
# COULD protest... but this case is unlikely ;)
if protest.submitter in groups.keys():
groups[protest.submitter].penalties += 1
# compute places for groups
for this_ent in groups.values():
for other_ent in groups.values():
if other_ent < this_ent:
this_ent.place += 1
if request.response_type == 'application/json':
return {
'status': 'success',
'groups': {k.id: v.to_dict()
for k, v in groups.items()}
}
else:
return {
'groups':
groups,
'my_groups':
my_groups,
'competition':
comp,
'inputs':
inputs,
'admin':
admin,
'incognito':
incognito,
'show_incognito_option':
show_incognito_option,
'incognito_teams':
incognito_teams,
'show_input_downloads':
show_input_downloads,
'ground_truth':
ground_truth,
'verification_accuracy': (0 if not total_count else
(accurate_count / total_count)),
'open_verification':
open_verification
}
def index(self, ground_truth=False):
user = request.identity and request.identity['user']
now = datetime.datetime.now()
admin = user and user.admin
comp = self.competition
show_scores = admin or (comp.open_verification_begins
and now >= comp.open_verification_begins)
if not admin and now < comp.output_upload_begins:
flash("Rankings are not available yet", "info")
redirect("/competition")
if ground_truth and not admin:
abort(403, "You do not have permission for this option")
if ground_truth:
verification_column = Output.ground_truth
else:
verification_column = case(
[(Output.use_ground_truth, Output.ground_truth)],
else_=Output.verification)
open_verification = user and comp.open_verification_open
groups = defaultdict(GroupEntry)
ir_query = (DBSession.query(
Input, Group, Output,
verification_column).join(Input.outputs).join(
Output.group).filter(Group.competition_id == comp.id).filter(
Output.active == True).order_by(
Input.group_id,
verification_column == VerificationStatus.rejected,
Output.score))
inputs = []
last_iput = None
for iput, ogroup, output, verif in ir_query:
if iput is not last_iput:
inputs.append(iput)
potential_rank = 1
last_rank = 0
last_score = None
shown_score = output.score
shown_output = output
if not show_scores:
shown_score = None
shown_output = None
if verif is VerificationStatus.rejected:
rank = None
shown_score = None
elif output.score == last_score:
rank = last_rank
else:
rank = potential_rank
groups[ogroup].input_ranks[iput] = ScoreTuple(
shown_score, verif, rank, shown_output)
if verif is VerificationStatus.rejected:
groups[ogroup].reject_count += 1
else:
groups[ogroup].sum_of_ranks += rank
# add accepted resubmissions to penalty
if verif is VerificationStatus.accepted and not output.original:
groups[ogroup].penalties += 1
potential_rank += 1
last_iput = iput
last_rank = rank
last_score = output.score
# no submission? this adds to reject count
for group in groups.values():
for iput in inputs:
if iput not in group.input_ranks.keys():
group.reject_count += 1
# add open verification protest rejections to penalty
rprotests = (DBSession.query(Protest).filter(
Protest.accepted == False).join(
Protest.submitter).filter(Group.competition_id == comp.id))
for protest in rprotests:
# technically, a group which failed to submit anything
# COULD protest... but this case is unlikely ;)
if protest.submitter in groups.keys():
groups[protest.submitter].penalties += 1
# compute places for groups
for this_ent in groups.values():
for other_ent in groups.values():
if other_ent < this_ent:
this_ent.place += 1
if request.response_type == 'application/json':
return {
'status': 'success',
'groups': {k.id: v.to_dict()
for k, v in groups.items()}
}
else:
return {
'groups': groups,
'competition': comp,
'inputs': inputs,
'ground_truth': ground_truth,
'open_verification': open_verification
}
def submit_output(self, to_group, output_file=None):
user = request.identity['user']
to_group = DBSession.query(Group).get(to_group)
if not to_group:
abort(404, "No such group")
if to_group.competition_id != self.group.competition_id:
abort(400, "Cannot submit to a group in another competition")
if not hasattr(output_file, "file"):
abort(400, "Must include file in submission")
comp = self.group.competition
existing = (DBSession.query(Output)
.filter(Output.group_id == self.group.id)
.filter(Output.input_id == to_group.input.id)
.filter(Output.active == True)
.one_or_none())
if not (user.admin
or comp.output_upload_open
or (comp.resolution_open
and existing is not None
and not existing.use_ground_truth)):
abort(403, "Forbidden to upload this output at this time")
try:
contents = output_file.file.read().decode("utf-8")
except UnicodeDecodeError:
return {'status': 'error',
'msg': 'Output contains invalid characters.'}
problem = problemlib.load_problem(comp)
try:
output = problem.parse_output(to_group.input.data.file,
StringIO(contents))
except problemlib.FileFormatError as e:
return {
"status": "error",
"msg": f"Output has formatting errors: {e}",
}
new_contents = StringIO()
output.write(new_contents)
f = FileIntent(
BytesIO(new_contents.getvalue().encode("utf-8")),
'output_from_{}_to_{}.txt'.format(self.group.id, to_group.id),
'application/octet-stream')
try:
output.verify()
except problemlib.VerificationError:
ground_truth = VerificationStatus.rejected
except Exception:
ground_truth = VerificationStatus.waiting
else:
ground_truth = VerificationStatus.accepted
use_ground_truth = (
not comp.verification_begins
or datetime.datetime.now() >= comp.verification_begins
)
if existing:
if comp.resolution_open:
existing.active = False
else:
DBSession.delete(existing)
db_output = Output(
data=f,
group=self.group,
input=to_group.input,
score=output.score,
original=comp.output_upload_open,
ground_truth=ground_truth,
use_ground_truth=use_ground_truth,
)
DBSession.add(db_output)
DBSession.flush()
return {'status': 'success', 'url': db_output.data.url}
def get_user(self, identity, userid):
return DBSession.query(User).filter_by(username=userid).first()
def submit_output(self, to_group, output_file=None):
user = request.identity['user']
to_group = DBSession.query(Group).get(to_group)
if not to_group:
abort(404, "No such group")
if to_group.competition_id != self.group.competition_id:
abort(400, "Cannot submit to a group in another competition")
if not hasattr(output_file, "file"):
abort(400, "Must include file in submission")
comp = self.group.competition
existing = (DBSession.query(Output).filter(
Output.group_id == self.group.id).filter(
Output.input_id == to_group.input.id).filter(
Output.active == True).one_or_none())
if not (user.admin or comp.output_upload_open or
(comp.resolution_open and existing is not None
and not existing.use_ground_truth)):
abort(403, "Forbidden to upload this output at this time")
try:
contents = file_normalize(output_file.file.read())
except UnicodeDecodeError:
return {
'status': 'error',
'msg': 'Output contains invalid characters.'
}
if len(contents) > 1E6:
return {'status': 'error', 'msg': 'Output exceeds maximum size.'}
try:
score, _, _ = contents.partition('\n')
score = int(score)
except ValueError:
return {
'status': 'error',
'msg': 'First line must only contain an integer.'
}
f = FileIntent(
BytesIO(contents.encode('utf-8')),
'output_from_{}_to_{}.txt'.format(self.group.id, to_group.id),
'application/octet-stream')
if existing:
if comp.resolution_open:
existing.active = False
else:
DBSession.delete(existing)
output = Output(data=f,
group=self.group,
input=to_group.input,
score=score,
original=comp.output_upload_open)
verif_mod = self.group.competition.output_verifier.module
try:
verif_mod.verify(to_group.input.data.file, StringIO(contents))
except verif_mod.VerificationError:
output.ground_truth = VerificationStatus.rejected
except Exception:
output.ground_truth = VerificationStatus.waiting
else:
output.ground_truth = VerificationStatus.accepted
if comp.resolution_open or not comp.verification_begins:
output.use_ground_truth = True
DBSession.add(output)
DBSession.flush()
return {'status': 'success', 'url': output.data.url}