def download_file(request,
course_slug,
activity_slug,
component_slug=None,
submission_id=None,
userid=None):
course = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(course.activity_set,
slug=activity_slug,
deleted=False)
staff = False
if is_course_staff_by_slug(request, course_slug):
staff = True
# find the appropriate submission object
if submission_id:
# explicit request: get that one.
try:
submission_info = SubmissionInfo.from_submission_id(submission_id)
except ValueError:
return NotFoundResponse(request)
elif userid:
# userid specified: get their most recent submission
student = get_object_or_404(Person, find_userid_or_emplid(userid))
submission_info = SubmissionInfo(student=student,
activity=activity,
include_deleted=staff)
else:
return NotFoundResponse(request)
if not submission_info.have_submitted(
) or submission_info.activity != activity:
return NotFoundResponse(request)
if not submission_info.accessible_by(request):
return ForbiddenResponse(request)
# create the result
if component_slug:
# download single component if specified
submission_info.get_most_recent_components()
submitted_components = [
subcomp
for comp, subcomp in submission_info.components_and_submitted()
if subcomp and comp.slug == component_slug
]
if not submitted_components:
return NotFoundResponse(request)
submitted_component = submitted_components[0]
return submitted_component.download_response(
slug=submission_info.submissions[0].file_slug())
else:
# no component specified: give back the full ZIP file.
return submission_info.generate_student_zip()
def show_components_submission_history(request, course_slug, activity_slug, userid=None):
offering = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(offering.activity_set, slug=activity_slug, deleted=False)
staff = False
userid = userid or request.user.username
member = get_object_or_404(Member, find_member(userid), offering=offering)
submission_info = SubmissionInfo(student=member.person, activity=activity)
submission_info.get_all_components()
if not submission_info.accessible_by(request):
return ForbiddenResponse(request)
return render(request, "submission/submission_history_view.html",
{"offering":offering, "activity":activity, 'userid':userid, 'submission_info': submission_info})
def show_components_submission_history(request, course_slug, activity_slug, userid=None):
offering = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(offering.activity_set, slug=activity_slug, deleted=False)
staff = False
userid = userid or request.user.username
member = get_object_or_404(Member, find_member(userid), offering=offering)
submission_info = SubmissionInfo(student=member.person, activity=activity)
submission_info.get_all_components()
if not submission_info.accessible_by(request):
return ForbiddenResponse(request)
return render(request, "submission/submission_history_view.html",
{"offering":offering, "activity":activity, 'userid':userid, 'submission_info': submission_info})
def download_activity_files(request, course_slug, activity_slug):
offering = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(offering.activity_set,
slug=activity_slug,
deleted=False)
submission_info = SubmissionInfo.for_activity(activity)
submission_info.get_all_components()
return submission_info.generate_activity_zip()
def all_code_submissions(activity: Activity) -> List[SubmittedCodefile]:
"""
Return a list of all files (as SubmittedCodefile instances) for Codefile submissions in this activity.
"""
si = SubmissionInfo.for_activity(activity)
si.get_all_components()
found, individual_subcomps, last_submission = si.most_recent_submissions()
print(individual_subcomps)
# flatten submitted component list: https://stackoverflow.com/a/952952
sub_comps = [c for sublist in si.all_submitted_components for c in sublist]
# keep only SubmittedCodefiles
sub_comps = [c for c in sub_comps if c is not None and isinstance(c, SubmittedCodefile)]
return sub_comps
def download_file(request, course_slug, activity_slug, component_slug=None, submission_id=None, userid=None):
course = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(course.activity_set, slug = activity_slug, deleted=False)
staff = False
if is_course_staff_by_slug(request, course_slug):
staff = True
# find the appropriate submission object
if submission_id:
# explicit request: get that one.
try:
submission_info = SubmissionInfo.from_submission_id(submission_id)
except ValueError:
return NotFoundResponse(request)
elif userid:
# userid specified: get their most recent submission
student = get_object_or_404(Person, find_userid_or_emplid(userid))
submission_info = SubmissionInfo(student=student, activity=activity, include_deleted=staff)
else:
return NotFoundResponse(request)
if not submission_info.have_submitted() or submission_info.activity != activity:
return NotFoundResponse(request)
if not submission_info.accessible_by(request):
return ForbiddenResponse(request)
# create the result
if component_slug:
# download single component if specified
submission_info.get_most_recent_components()
submitted_components = [subcomp for comp, subcomp in submission_info.components_and_submitted() if subcomp and comp.slug == component_slug]
if not submitted_components:
return NotFoundResponse(request)
submitted_component = submitted_components[0]
return submitted_component.download_response(slug=submission_info.submissions[0].file_slug())
else:
# no component specified: give back the full ZIP file.
return submission_info.generate_student_zip()
def run_moss(main_activity: Activity, activities: List[Activity],
language: str, result: SimilarityResult) -> SimilarityResult:
"""
Run MOSS for the main_activity's submissions.
... comparing past submission from everything in the activities list.
... looking only at the given programming language.
... storing the results in result.
"""
assert language in MOSS_LANGUAGES
assert main_activity in activities
icon_url_path = reverse('dashboard:moss_icon', kwargs={'filename': ''})
tmpdir = tempfile.TemporaryDirectory()
tmp = tmpdir.name
code_dir = os.path.join(tmp, 'code')
moss_out_dir = os.path.join(tmp, 'moss')
# assemble tmp directory of submissions for MOSS
offering_slug = main_activity.offering.slug
extension = '.' + MOSS_LANGUAGES[language]
moss_files = [] # files that we will give to MOSS
file_submissions = {
} # MOSS input file to submission_id, so we can recover the source later
for a in activities:
si = SubmissionInfo.for_activity(a)
si.get_all_components()
_, individual_subcomps, _ = si.most_recent_submissions()
for userid, components in individual_subcomps.items():
prefix = os.path.join(code_dir, a.offering.slug, userid)
for comp, sub in components:
if not isinstance(sub, SubmittedCodefile):
# we can only deal with Codefile components
continue
if not isinstance(sub.code.storage, FileSystemStorage):
raise NotImplementedError(
'more work necessary to support non-filesystem file storage'
)
source_file = os.path.join(sub.code.storage.location,
sub.code.name)
moss_file = sub.file_filename(sub.code, prefix)
if not moss_file.endswith(extension):
# we only handle one language at a time
continue
dst_dir, _ = os.path.split(moss_file)
os.makedirs(dst_dir, exist_ok=True)
os.symlink(source_file, moss_file)
moss_files.append(moss_file)
file_submissions[moss_file] = sub.submission_id
if not moss_files:
raise MOSSError(
'No files found for that language to analyze with MOSS.')
# run MOSS
moss_pl = os.path.join(settings.MOSS_DISTRIBUTION_PATH, 'moss.pl')
cmd = [moss_pl, '-l', language, '-o', moss_out_dir] + moss_files
try:
res = subprocess.run(cmd, cwd=settings.MOSS_DISTRIBUTION_PATH)
except FileNotFoundError:
raise MOSSError(
'System not correctly configured with the MOSS executable.')
if res.returncode != 0:
raise MOSSError('MOSS command failed: ' + str(cmd))
# try to deal with MOSS' [profanity suppressed] HTML, and produce SimilarityData objects to represent everything
for f in os.listdir(moss_out_dir):
if f == 'index.html':
data = open(os.path.join(moss_out_dir, f), 'rt',
encoding='utf8').read()
soup = bs4.BeautifulSoup(data, 'lxml')
index_data = []
for tr in soup.find_all('tr'):
if tr.find('th'):
continue
m = []
for a in tr.find_all('a'):
label = a.get('href')
fn, perc = a.string.split(' ')
fn = _canonical_filename(fn, code_dir)
m.append((label, fn, perc))
# Only display if one side is from the main_activity: leave the past behind.
if any(fn.startswith(offering_slug + '/') for _, fn, _ in m):
index_data.append(m)
data = SimilarityData(result=result,
label='index.html',
file=None,
config={})
data.config['index_data'] = index_data
data.save()
elif match_base_re.match(f):
pass
elif match_top_re.match(f):
data = open(os.path.join(moss_out_dir, f), 'rt',
encoding='utf8').read()
soup = bs4.BeautifulSoup(data, 'lxml')
table = soup.find('table')
del table['bgcolor']
del table['border']
del table['cellspacing']
for th in table.find_all('th'):
if th.string is not None:
th.string = _canonical_filename(th.string, code_dir)
for img in table.find_all('img'):
src = img.get('src')
img['src'] = src.replace('../bitmaps/', icon_url_path)
file = File(file=io.BytesIO(str(table).encode('utf8')), name=f)
data = SimilarityData(result=result, label=f, file=file, config={})
data.save()
elif match_file_re.match(f):
try:
data = open(os.path.join(moss_out_dir, f),
'rt',
encoding='utf8').read()
except UnicodeDecodeError:
data = open(os.path.join(moss_out_dir, f),
'rt',
encoding='windows-1252').read()
soup = bs4.BeautifulSoup(data, 'lxml')
# find the input filename, which leads to the submission
for c in soup.find('body').children:
if isinstance(c, bs4.element.NavigableString):
c = str(c).strip()
if c.startswith(code_dir):
filename = c
break
submission_id = file_submissions[filename]
# the only <pre> is the real content we care about
pre = soup.find('pre')
for img in pre.find_all('img'):
src = img.get('src')
img['src'] = src.replace('../bitmaps/', icon_url_path)
file = File(file=io.BytesIO(str(pre).encode('utf8')), name=f)
data = SimilarityData(result=result,
label=f,
file=file,
submission_id=submission_id,
config={})
data.save()
else:
raise ValueError('unexpected file produced by MOSS')
result.config['complete'] = True
result.save()
return result
def render(self, request: HttpRequest, path: str) -> HttpResponse:
if not self.result.config.get('complete'):
# result still pending in Celery: no other data to find yet
context = {
'complete': False,
'offering': self.offering,
'activity': self.activity,
'result': self.result,
}
resp = render(request,
'submission/similarity-moss-result.html',
context=context)
return resp
base_match = match_base_re.match(path)
top_match = match_top_re.match(path)
file_match = match_file_re.match(path)
if base_match: # reconstruct matchX.html
match = base_match.group(2)
match0 = get_object_or_404(
SimilarityData.objects.select_related('submission'),
result=self.result,
label='match{}-0.html'.format(match))
match1 = get_object_or_404(
SimilarityData.objects.select_related('submission'),
result=self.result,
label='match{}-1.html'.format(match))
sub0, _ = SubmissionInfo._get_submission(match0.submission_id)
sub1, _ = SubmissionInfo._get_submission(match1.submission_id)
context = {
'complete': True,
'offering': self.offering,
'activity': self.activity,
'result': self.result,
'match_n': match,
'fn_top': 'match{}-top.html'.format(match),
'fn_left': 'match{}-0.html'.format(match),
'fn_right': 'match{}-1.html'.format(match),
'match0': match0,
'match1': match1,
'sub0': sub0,
'sub1': sub1,
}
resp = render(request,
'submission/similarity-moss-result.html',
context=context)
#resp.allow_frames_csp = True
#resp['X-Frame-Options'] = 'SAMEORIGIN'
return resp
elif top_match: # reconstruct matchX-top.html
match = top_match.group(1)
data = get_object_or_404(SimilarityData,
result=self.result,
label='match{}-top.html'.format(match))
elif file_match: # reconstruct matchX-[01].html
match = file_match.group(1)
side = file_match.group(2)
data = get_object_or_404(SimilarityData,
result=self.result,
label='match{}-{}.html'.format(
match, side))
else: # index page
data = SimilarityData.objects.get(result=self.result,
label='index.html')
context = {
'offering': self.offering,
'activity': self.activity,
'result': self.result,
'data': data,
}
resp = render(request,
'submission/similarity-moss.html',
context=context)
return resp
content = data.file.read().decode('utf8')
resp = HttpResponse(content)
resp.allow_frames_csp = True
resp['X-Frame-Options'] = 'SAMEORIGIN'
return resp
def _show_components_student(request,
course_slug,
activity_slug,
userid=None,
template="dashboard_student.html",
staff=False):
"""
Show all the component submission history of this activity
"""
if userid == None:
userid = request.user.username
course = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(course.activity_set,
slug=activity_slug,
deleted=False)
student = get_object_or_404(Person, find_userid_or_emplid(userid))
cansubmit = True
submission_configured = SubmissionComponent.objects.filter(
activity_id=activity.id).exists()
if not submission_configured:
return NotFoundResponse(request)
submission_info = SubmissionInfo(student=student, activity=activity)
submission_info.get_most_recent_components()
if activity.multisubmit():
submission_info.get_all_components()
any_submissions = bool(submission_info.submissions)
if submission_info.submissions and activity.due_date and activity.due_date < submission_info.latest(
).created_at:
late = submission_info.latest().created_at - activity.due_date
else:
late = 0
if activity.group:
gm = GroupMember.objects.filter(student__person=student,
activity=activity,
confirmed=True)
if gm:
group = gm[0].group
member = gm[0].student
else:
group = None
else:
group = None
# activity should be submitable
cansubmit = cansubmit and activity.submitable()
if not cansubmit:
messages.add_message(request, messages.ERROR,
"This activity is not submittable.")
return render(
request, "submission/" + template, {
"course": course,
"activity": activity,
"submission_info": submission_info,
'any_submissions': any_submissions,
"userid": userid,
"late": late,
"student": student,
"group": group,
"cansubmit": cansubmit
})
# get all components of activity
component_list = select_all_components(activity)
component_list.sort()
component_form_list = []
if request.method == 'POST':
component_form_list = make_form_from_list(component_list,
request=request)
submitted_comp = [
] # list all components which has content submitted in the POST
not_submitted_comp = [
] #list allcomponents which has no content submitted in the POST
if not activity.group:
new_sub = StudentSubmission(
) # the submission foreign key for newly submitted components
new_sub.member = get_object_or_404(
Member,
offering__slug=course_slug,
person__userid=request.user.username)
elif gm:
new_sub = GroupSubmission()
new_sub.group = group
new_sub.creator = member
else:
messages.add_message(
request, messages.ERROR,
"This is a group submission. You cannot submit since you aren't in a group."
)
return ForbiddenResponse(request)
new_sub.activity = activity
# begin validating uploaded data
submitted_comp = []
not_submitted_comp = []
# validate forms one by one
for data in component_form_list:
component = data['comp']
form = data['form']
if form.is_valid():
sub = form.save(commit=False)
sub.component = component
submitted_comp.append(sub)
else:
# hack to replace the "required" message to something more appropriate
for k, v in list(form.errors.items()):
for i, e in enumerate(v):
if e == "This field is required.":
v[i] = "Nothing submitted."
not_submitted_comp.append(component)
# check duplicate filenames here
all_ok = False
while not all_ok:
all_ok = True
d = {}
if not activity.multisubmit():
# single-submit logic: don't want to overrite filenames from earlier submissions that are still in-play
for c, s in submission_info.components_and_submitted():
d[c] = s and s.get_filename()
# filenames from this submission
for s in submitted_comp:
d[s.component] = s.get_filename()
# a list holding all file names
file_name_list = [
a[1] for a in list(d.items()) if a[1] is not None
]
to_be_removed = []
for (i, s) in enumerate(submitted_comp):
if file_name_list.count(s.get_filename()) > 1:
all_ok = False
to_be_removed.append(i)
not_submitted_comp.append(s.component)
#HACK: modify the 'errors' field in the form
for data in component_form_list:
if s.component == data['comp']:
# assume we have only one field for submission form
field_name = list(data['form'].fields.keys())[0]
data['form']._errors[field_name] = ErrorList([
"This file has the same name as another file in your submission."
])
# remove those has errors in submitted_comp
to_be_removed.reverse()
for t in to_be_removed:
submitted_comp.pop(t)
# all okay now
# end validating, begin saving
if len(submitted_comp) > 0:
new_sub.save()
for sub in submitted_comp:
sub.submission = new_sub
sub.save()
#LOG EVENT#
if activity.group:
group_str = " as a member of group %s" % new_sub.group.name
else:
group_str = ""
l = LogEntry(userid=request.user.username,
description="submitted for %s %s%s" %
(activity, sub.component.title, group_str),
related_object=sub)
l.save()
if len(not_submitted_comp) == 0:
messages.add_message(request, messages.SUCCESS,
"Your submission was successful.")
return HttpResponseRedirect(
reverse('offering:submission:show_components',
args=[course_slug, activity_slug]))
return render(
request, "submission/submission_error.html", {
"course": course,
"activity": activity,
"component_list": component_form_list,
"submitted_comp": submitted_comp,
"not_submitted_comp": not_submitted_comp
})
else: #not POST
component_form_list = make_form_from_list(component_list)
return render(
request, "submission/" + template, {
'component_form_list': component_form_list,
"course": course,
"activity": activity,
"submission_info": submission_info,
"userid": userid,
"late": late,
"student": student,
"group": group,
"cansubmit": cansubmit,
"is_staff": staff,
'any_submissions': any_submissions
})
def _show_components_student(request, course_slug, activity_slug, userid=None, template="dashboard_student.html", staff=False):
"""
Show all the component submission history of this activity
"""
if userid == None:
userid = request.user.username
course = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(course.activity_set,slug=activity_slug, deleted=False)
student = get_object_or_404(Person, find_userid_or_emplid(userid))
cansubmit = True
submission_configured = SubmissionComponent.objects.filter(activity_id=activity.id).exists()
if not submission_configured:
return NotFoundResponse(request)
submission_info = SubmissionInfo(student=student, activity=activity)
submission_info.get_most_recent_components()
if activity.multisubmit():
submission_info.get_all_components()
any_submissions = bool(submission_info.submissions)
if submission_info.submissions and activity.due_date and activity.due_date < submission_info.latest().created_at:
late = submission_info.latest().created_at - activity.due_date
else:
late = 0
if activity.group:
gm = GroupMember.objects.filter(student__person=student, activity=activity, confirmed=True)
if gm:
group = gm[0].group
member = gm[0].student
else:
group = None
else:
group = None
# activity should be submitable
cansubmit = cansubmit and activity.submitable()
if not cansubmit:
messages.add_message(request, messages.ERROR, "This activity is not submittable.")
return render(request, "submission/" + template,
{"course":course, "activity":activity, "submission_info": submission_info, 'any_submissions': any_submissions,
"userid":userid, "late":late, "student":student, "group":group, "cansubmit":cansubmit})
# get all components of activity
component_list = select_all_components(activity)
component_list.sort()
component_form_list=[]
if request.method == 'POST':
component_form_list = make_form_from_list(component_list, request=request)
submitted_comp = [] # list all components which has content submitted in the POST
not_submitted_comp = [] #list allcomponents which has no content submitted in the POST
if not activity.group:
new_sub = StudentSubmission() # the submission foreign key for newly submitted components
new_sub.member = get_object_or_404(Member, offering__slug=course_slug, person__userid=request.user.username)
elif gm:
new_sub = GroupSubmission()
new_sub.group = group
new_sub.creator = member
else:
messages.add_message(request, messages.ERROR, "This is a group submission. You cannot submit since you aren't in a group.")
return ForbiddenResponse(request)
new_sub.activity = activity
# begin validating uploaded data
submitted_comp = []
not_submitted_comp = []
# validate forms one by one
for data in component_form_list:
component = data['comp']
form = data['form']
if form.is_valid():
sub = form.save(commit=False)
sub.component = component
submitted_comp.append(sub)
else:
# hack to replace the "required" message to something more appropriate
for k,v in list(form.errors.items()):
for i,e in enumerate(v):
if e == "This field is required.":
v[i] = "Nothing submitted."
not_submitted_comp.append(component)
# check duplicate filenames here
all_ok = False
while not all_ok:
all_ok = True
d = {}
if not activity.multisubmit():
# single-submit logic: don't want to overrite filenames from earlier submissions that are still in-play
for c,s in submission_info.components_and_submitted():
d[c] = s and s.get_filename()
# filenames from this submission
for s in submitted_comp:
d[s.component] = s.get_filename()
# a list holding all file names
file_name_list = [a[1] for a in list(d.items()) if a[1] is not None]
to_be_removed = []
for (i, s) in enumerate(submitted_comp):
if file_name_list.count(s.get_filename()) > 1:
all_ok = False
to_be_removed.append(i)
not_submitted_comp.append(s.component)
#HACK: modify the 'errors' field in the form
for data in component_form_list:
if s.component == data['comp']:
# assume we have only one field for submission form
field_name = list(data['form'].fields.keys())[0]
data['form']._errors[field_name] = ErrorList(["This file has the same name as another file in your submission."])
# remove those has errors in submitted_comp
to_be_removed.reverse()
for t in to_be_removed:
submitted_comp.pop(t)
# all okay now
# end validating, begin saving
if len(submitted_comp) > 0:
new_sub.save()
for sub in submitted_comp:
sub.submission = new_sub
sub.save()
#LOG EVENT#
if activity.group:
group_str = " as a member of group %s" % new_sub.group.name
else:
group_str = ""
l = LogEntry(userid=request.user.username,
description="submitted for %s %s%s" % (activity, sub.component.title, group_str),
related_object=sub)
l.save()
if len(not_submitted_comp) == 0:
messages.add_message(request, messages.SUCCESS, "Your submission was successful.")
return HttpResponseRedirect(reverse('offering:submission:show_components', args=[course_slug, activity_slug]))
return render(request, "submission/submission_error.html",
{"course":course, "activity":activity, "component_list":component_form_list,
"submitted_comp":submitted_comp, "not_submitted_comp":not_submitted_comp})
else: #not POST
component_form_list = make_form_from_list(component_list)
return render(request, "submission/" + template,
{'component_form_list': component_form_list, "course": course, "activity": activity, "submission_info": submission_info,
"userid":userid, "late":late, "student":student, "group":group,
"cansubmit":cansubmit, "is_staff":staff, 'any_submissions': any_submissions})
def download_activity_files(request, course_slug, activity_slug):
offering = get_object_or_404(CourseOffering, slug=course_slug)
activity = get_object_or_404(offering.activity_set, slug=activity_slug, deleted=False)
submission_info = SubmissionInfo.for_activity(activity)
submission_info.get_all_components()
return submission_info.generate_activity_zip()