def download_file(request, course_slug, activity_slug, component_slug=None, submission_id=None, userid=None): course = get_object_or_404(CourseOffering, slug=course_slug) activity = get_object_or_404(course.activity_set, slug=activity_slug, deleted=False) staff = False if is_course_staff_by_slug(request, course_slug): staff = True # find the appropriate submission object if submission_id: # explicit request: get that one. try: submission_info = SubmissionInfo.from_submission_id(submission_id) except ValueError: return NotFoundResponse(request) elif userid: # userid specified: get their most recent submission student = get_object_or_404(Person, find_userid_or_emplid(userid)) submission_info = SubmissionInfo(student=student, activity=activity, include_deleted=staff) else: return NotFoundResponse(request) if not submission_info.have_submitted( ) or submission_info.activity != activity: return NotFoundResponse(request) if not submission_info.accessible_by(request): return ForbiddenResponse(request) # create the result if component_slug: # download single component if specified submission_info.get_most_recent_components() submitted_components = [ subcomp for comp, subcomp in submission_info.components_and_submitted() if subcomp and comp.slug == component_slug ] if not submitted_components: return NotFoundResponse(request) submitted_component = submitted_components[0] return submitted_component.download_response( slug=submission_info.submissions[0].file_slug()) else: # no component specified: give back the full ZIP file. return submission_info.generate_student_zip()
def show_components_submission_history(request, course_slug, activity_slug, userid=None): offering = get_object_or_404(CourseOffering, slug=course_slug) activity = get_object_or_404(offering.activity_set, slug=activity_slug, deleted=False) staff = False userid = userid or request.user.username member = get_object_or_404(Member, find_member(userid), offering=offering) submission_info = SubmissionInfo(student=member.person, activity=activity) submission_info.get_all_components() if not submission_info.accessible_by(request): return ForbiddenResponse(request) return render(request, "submission/submission_history_view.html", {"offering":offering, "activity":activity, 'userid':userid, 'submission_info': submission_info})
def download_activity_files(request, course_slug, activity_slug): offering = get_object_or_404(CourseOffering, slug=course_slug) activity = get_object_or_404(offering.activity_set, slug=activity_slug, deleted=False) submission_info = SubmissionInfo.for_activity(activity) submission_info.get_all_components() return submission_info.generate_activity_zip()
def all_code_submissions(activity: Activity) -> List[SubmittedCodefile]: """ Return a list of all files (as SubmittedCodefile instances) for Codefile submissions in this activity. """ si = SubmissionInfo.for_activity(activity) si.get_all_components() found, individual_subcomps, last_submission = si.most_recent_submissions() print(individual_subcomps) # flatten submitted component list: https://stackoverflow.com/a/952952 sub_comps = [c for sublist in si.all_submitted_components for c in sublist] # keep only SubmittedCodefiles sub_comps = [c for c in sub_comps if c is not None and isinstance(c, SubmittedCodefile)] return sub_comps
def download_file(request, course_slug, activity_slug, component_slug=None, submission_id=None, userid=None): course = get_object_or_404(CourseOffering, slug=course_slug) activity = get_object_or_404(course.activity_set, slug = activity_slug, deleted=False) staff = False if is_course_staff_by_slug(request, course_slug): staff = True # find the appropriate submission object if submission_id: # explicit request: get that one. try: submission_info = SubmissionInfo.from_submission_id(submission_id) except ValueError: return NotFoundResponse(request) elif userid: # userid specified: get their most recent submission student = get_object_or_404(Person, find_userid_or_emplid(userid)) submission_info = SubmissionInfo(student=student, activity=activity, include_deleted=staff) else: return NotFoundResponse(request) if not submission_info.have_submitted() or submission_info.activity != activity: return NotFoundResponse(request) if not submission_info.accessible_by(request): return ForbiddenResponse(request) # create the result if component_slug: # download single component if specified submission_info.get_most_recent_components() submitted_components = [subcomp for comp, subcomp in submission_info.components_and_submitted() if subcomp and comp.slug == component_slug] if not submitted_components: return NotFoundResponse(request) submitted_component = submitted_components[0] return submitted_component.download_response(slug=submission_info.submissions[0].file_slug()) else: # no component specified: give back the full ZIP file. return submission_info.generate_student_zip()
def run_moss(main_activity: Activity, activities: List[Activity], language: str, result: SimilarityResult) -> SimilarityResult: """ Run MOSS for the main_activity's submissions. ... comparing past submission from everything in the activities list. ... looking only at the given programming language. ... storing the results in result. """ assert language in MOSS_LANGUAGES assert main_activity in activities icon_url_path = reverse('dashboard:moss_icon', kwargs={'filename': ''}) tmpdir = tempfile.TemporaryDirectory() tmp = tmpdir.name code_dir = os.path.join(tmp, 'code') moss_out_dir = os.path.join(tmp, 'moss') # assemble tmp directory of submissions for MOSS offering_slug = main_activity.offering.slug extension = '.' + MOSS_LANGUAGES[language] moss_files = [] # files that we will give to MOSS file_submissions = { } # MOSS input file to submission_id, so we can recover the source later for a in activities: si = SubmissionInfo.for_activity(a) si.get_all_components() _, individual_subcomps, _ = si.most_recent_submissions() for userid, components in individual_subcomps.items(): prefix = os.path.join(code_dir, a.offering.slug, userid) for comp, sub in components: if not isinstance(sub, SubmittedCodefile): # we can only deal with Codefile components continue if not isinstance(sub.code.storage, FileSystemStorage): raise NotImplementedError( 'more work necessary to support non-filesystem file storage' ) source_file = os.path.join(sub.code.storage.location, sub.code.name) moss_file = sub.file_filename(sub.code, prefix) if not moss_file.endswith(extension): # we only handle one language at a time continue dst_dir, _ = os.path.split(moss_file) os.makedirs(dst_dir, exist_ok=True) os.symlink(source_file, moss_file) moss_files.append(moss_file) file_submissions[moss_file] = sub.submission_id if not moss_files: raise MOSSError( 'No files found for that language to analyze with MOSS.') # run MOSS moss_pl = os.path.join(settings.MOSS_DISTRIBUTION_PATH, 'moss.pl') cmd = [moss_pl, '-l', language, '-o', moss_out_dir] + moss_files try: res = subprocess.run(cmd, cwd=settings.MOSS_DISTRIBUTION_PATH) except FileNotFoundError: raise MOSSError( 'System not correctly configured with the MOSS executable.') if res.returncode != 0: raise MOSSError('MOSS command failed: ' + str(cmd)) # try to deal with MOSS' [profanity suppressed] HTML, and produce SimilarityData objects to represent everything for f in os.listdir(moss_out_dir): if f == 'index.html': data = open(os.path.join(moss_out_dir, f), 'rt', encoding='utf8').read() soup = bs4.BeautifulSoup(data, 'lxml') index_data = [] for tr in soup.find_all('tr'): if tr.find('th'): continue m = [] for a in tr.find_all('a'): label = a.get('href') fn, perc = a.string.split(' ') fn = _canonical_filename(fn, code_dir) m.append((label, fn, perc)) # Only display if one side is from the main_activity: leave the past behind. if any(fn.startswith(offering_slug + '/') for _, fn, _ in m): index_data.append(m) data = SimilarityData(result=result, label='index.html', file=None, config={}) data.config['index_data'] = index_data data.save() elif match_base_re.match(f): pass elif match_top_re.match(f): data = open(os.path.join(moss_out_dir, f), 'rt', encoding='utf8').read() soup = bs4.BeautifulSoup(data, 'lxml') table = soup.find('table') del table['bgcolor'] del table['border'] del table['cellspacing'] for th in table.find_all('th'): if th.string is not None: th.string = _canonical_filename(th.string, code_dir) for img in table.find_all('img'): src = img.get('src') img['src'] = src.replace('../bitmaps/', icon_url_path) file = File(file=io.BytesIO(str(table).encode('utf8')), name=f) data = SimilarityData(result=result, label=f, file=file, config={}) data.save() elif match_file_re.match(f): try: data = open(os.path.join(moss_out_dir, f), 'rt', encoding='utf8').read() except UnicodeDecodeError: data = open(os.path.join(moss_out_dir, f), 'rt', encoding='windows-1252').read() soup = bs4.BeautifulSoup(data, 'lxml') # find the input filename, which leads to the submission for c in soup.find('body').children: if isinstance(c, bs4.element.NavigableString): c = str(c).strip() if c.startswith(code_dir): filename = c break submission_id = file_submissions[filename] # the only <pre> is the real content we care about pre = soup.find('pre') for img in pre.find_all('img'): src = img.get('src') img['src'] = src.replace('../bitmaps/', icon_url_path) file = File(file=io.BytesIO(str(pre).encode('utf8')), name=f) data = SimilarityData(result=result, label=f, file=file, submission_id=submission_id, config={}) data.save() else: raise ValueError('unexpected file produced by MOSS') result.config['complete'] = True result.save() return result
def render(self, request: HttpRequest, path: str) -> HttpResponse: if not self.result.config.get('complete'): # result still pending in Celery: no other data to find yet context = { 'complete': False, 'offering': self.offering, 'activity': self.activity, 'result': self.result, } resp = render(request, 'submission/similarity-moss-result.html', context=context) return resp base_match = match_base_re.match(path) top_match = match_top_re.match(path) file_match = match_file_re.match(path) if base_match: # reconstruct matchX.html match = base_match.group(2) match0 = get_object_or_404( SimilarityData.objects.select_related('submission'), result=self.result, label='match{}-0.html'.format(match)) match1 = get_object_or_404( SimilarityData.objects.select_related('submission'), result=self.result, label='match{}-1.html'.format(match)) sub0, _ = SubmissionInfo._get_submission(match0.submission_id) sub1, _ = SubmissionInfo._get_submission(match1.submission_id) context = { 'complete': True, 'offering': self.offering, 'activity': self.activity, 'result': self.result, 'match_n': match, 'fn_top': 'match{}-top.html'.format(match), 'fn_left': 'match{}-0.html'.format(match), 'fn_right': 'match{}-1.html'.format(match), 'match0': match0, 'match1': match1, 'sub0': sub0, 'sub1': sub1, } resp = render(request, 'submission/similarity-moss-result.html', context=context) #resp.allow_frames_csp = True #resp['X-Frame-Options'] = 'SAMEORIGIN' return resp elif top_match: # reconstruct matchX-top.html match = top_match.group(1) data = get_object_or_404(SimilarityData, result=self.result, label='match{}-top.html'.format(match)) elif file_match: # reconstruct matchX-[01].html match = file_match.group(1) side = file_match.group(2) data = get_object_or_404(SimilarityData, result=self.result, label='match{}-{}.html'.format( match, side)) else: # index page data = SimilarityData.objects.get(result=self.result, label='index.html') context = { 'offering': self.offering, 'activity': self.activity, 'result': self.result, 'data': data, } resp = render(request, 'submission/similarity-moss.html', context=context) return resp content = data.file.read().decode('utf8') resp = HttpResponse(content) resp.allow_frames_csp = True resp['X-Frame-Options'] = 'SAMEORIGIN' return resp
def _show_components_student(request, course_slug, activity_slug, userid=None, template="dashboard_student.html", staff=False): """ Show all the component submission history of this activity """ if userid == None: userid = request.user.username course = get_object_or_404(CourseOffering, slug=course_slug) activity = get_object_or_404(course.activity_set, slug=activity_slug, deleted=False) student = get_object_or_404(Person, find_userid_or_emplid(userid)) cansubmit = True submission_configured = SubmissionComponent.objects.filter( activity_id=activity.id).exists() if not submission_configured: return NotFoundResponse(request) submission_info = SubmissionInfo(student=student, activity=activity) submission_info.get_most_recent_components() if activity.multisubmit(): submission_info.get_all_components() any_submissions = bool(submission_info.submissions) if submission_info.submissions and activity.due_date and activity.due_date < submission_info.latest( ).created_at: late = submission_info.latest().created_at - activity.due_date else: late = 0 if activity.group: gm = GroupMember.objects.filter(student__person=student, activity=activity, confirmed=True) if gm: group = gm[0].group member = gm[0].student else: group = None else: group = None # activity should be submitable cansubmit = cansubmit and activity.submitable() if not cansubmit: messages.add_message(request, messages.ERROR, "This activity is not submittable.") return render( request, "submission/" + template, { "course": course, "activity": activity, "submission_info": submission_info, 'any_submissions': any_submissions, "userid": userid, "late": late, "student": student, "group": group, "cansubmit": cansubmit }) # get all components of activity component_list = select_all_components(activity) component_list.sort() component_form_list = [] if request.method == 'POST': component_form_list = make_form_from_list(component_list, request=request) submitted_comp = [ ] # list all components which has content submitted in the POST not_submitted_comp = [ ] #list allcomponents which has no content submitted in the POST if not activity.group: new_sub = StudentSubmission( ) # the submission foreign key for newly submitted components new_sub.member = get_object_or_404( Member, offering__slug=course_slug, person__userid=request.user.username) elif gm: new_sub = GroupSubmission() new_sub.group = group new_sub.creator = member else: messages.add_message( request, messages.ERROR, "This is a group submission. You cannot submit since you aren't in a group." ) return ForbiddenResponse(request) new_sub.activity = activity # begin validating uploaded data submitted_comp = [] not_submitted_comp = [] # validate forms one by one for data in component_form_list: component = data['comp'] form = data['form'] if form.is_valid(): sub = form.save(commit=False) sub.component = component submitted_comp.append(sub) else: # hack to replace the "required" message to something more appropriate for k, v in list(form.errors.items()): for i, e in enumerate(v): if e == "This field is required.": v[i] = "Nothing submitted." not_submitted_comp.append(component) # check duplicate filenames here all_ok = False while not all_ok: all_ok = True d = {} if not activity.multisubmit(): # single-submit logic: don't want to overrite filenames from earlier submissions that are still in-play for c, s in submission_info.components_and_submitted(): d[c] = s and s.get_filename() # filenames from this submission for s in submitted_comp: d[s.component] = s.get_filename() # a list holding all file names file_name_list = [ a[1] for a in list(d.items()) if a[1] is not None ] to_be_removed = [] for (i, s) in enumerate(submitted_comp): if file_name_list.count(s.get_filename()) > 1: all_ok = False to_be_removed.append(i) not_submitted_comp.append(s.component) #HACK: modify the 'errors' field in the form for data in component_form_list: if s.component == data['comp']: # assume we have only one field for submission form field_name = list(data['form'].fields.keys())[0] data['form']._errors[field_name] = ErrorList([ "This file has the same name as another file in your submission." ]) # remove those has errors in submitted_comp to_be_removed.reverse() for t in to_be_removed: submitted_comp.pop(t) # all okay now # end validating, begin saving if len(submitted_comp) > 0: new_sub.save() for sub in submitted_comp: sub.submission = new_sub sub.save() #LOG EVENT# if activity.group: group_str = " as a member of group %s" % new_sub.group.name else: group_str = "" l = LogEntry(userid=request.user.username, description="submitted for %s %s%s" % (activity, sub.component.title, group_str), related_object=sub) l.save() if len(not_submitted_comp) == 0: messages.add_message(request, messages.SUCCESS, "Your submission was successful.") return HttpResponseRedirect( reverse('offering:submission:show_components', args=[course_slug, activity_slug])) return render( request, "submission/submission_error.html", { "course": course, "activity": activity, "component_list": component_form_list, "submitted_comp": submitted_comp, "not_submitted_comp": not_submitted_comp }) else: #not POST component_form_list = make_form_from_list(component_list) return render( request, "submission/" + template, { 'component_form_list': component_form_list, "course": course, "activity": activity, "submission_info": submission_info, "userid": userid, "late": late, "student": student, "group": group, "cansubmit": cansubmit, "is_staff": staff, 'any_submissions': any_submissions })
def _show_components_student(request, course_slug, activity_slug, userid=None, template="dashboard_student.html", staff=False): """ Show all the component submission history of this activity """ if userid == None: userid = request.user.username course = get_object_or_404(CourseOffering, slug=course_slug) activity = get_object_or_404(course.activity_set,slug=activity_slug, deleted=False) student = get_object_or_404(Person, find_userid_or_emplid(userid)) cansubmit = True submission_configured = SubmissionComponent.objects.filter(activity_id=activity.id).exists() if not submission_configured: return NotFoundResponse(request) submission_info = SubmissionInfo(student=student, activity=activity) submission_info.get_most_recent_components() if activity.multisubmit(): submission_info.get_all_components() any_submissions = bool(submission_info.submissions) if submission_info.submissions and activity.due_date and activity.due_date < submission_info.latest().created_at: late = submission_info.latest().created_at - activity.due_date else: late = 0 if activity.group: gm = GroupMember.objects.filter(student__person=student, activity=activity, confirmed=True) if gm: group = gm[0].group member = gm[0].student else: group = None else: group = None # activity should be submitable cansubmit = cansubmit and activity.submitable() if not cansubmit: messages.add_message(request, messages.ERROR, "This activity is not submittable.") return render(request, "submission/" + template, {"course":course, "activity":activity, "submission_info": submission_info, 'any_submissions': any_submissions, "userid":userid, "late":late, "student":student, "group":group, "cansubmit":cansubmit}) # get all components of activity component_list = select_all_components(activity) component_list.sort() component_form_list=[] if request.method == 'POST': component_form_list = make_form_from_list(component_list, request=request) submitted_comp = [] # list all components which has content submitted in the POST not_submitted_comp = [] #list allcomponents which has no content submitted in the POST if not activity.group: new_sub = StudentSubmission() # the submission foreign key for newly submitted components new_sub.member = get_object_or_404(Member, offering__slug=course_slug, person__userid=request.user.username) elif gm: new_sub = GroupSubmission() new_sub.group = group new_sub.creator = member else: messages.add_message(request, messages.ERROR, "This is a group submission. You cannot submit since you aren't in a group.") return ForbiddenResponse(request) new_sub.activity = activity # begin validating uploaded data submitted_comp = [] not_submitted_comp = [] # validate forms one by one for data in component_form_list: component = data['comp'] form = data['form'] if form.is_valid(): sub = form.save(commit=False) sub.component = component submitted_comp.append(sub) else: # hack to replace the "required" message to something more appropriate for k,v in list(form.errors.items()): for i,e in enumerate(v): if e == "This field is required.": v[i] = "Nothing submitted." not_submitted_comp.append(component) # check duplicate filenames here all_ok = False while not all_ok: all_ok = True d = {} if not activity.multisubmit(): # single-submit logic: don't want to overrite filenames from earlier submissions that are still in-play for c,s in submission_info.components_and_submitted(): d[c] = s and s.get_filename() # filenames from this submission for s in submitted_comp: d[s.component] = s.get_filename() # a list holding all file names file_name_list = [a[1] for a in list(d.items()) if a[1] is not None] to_be_removed = [] for (i, s) in enumerate(submitted_comp): if file_name_list.count(s.get_filename()) > 1: all_ok = False to_be_removed.append(i) not_submitted_comp.append(s.component) #HACK: modify the 'errors' field in the form for data in component_form_list: if s.component == data['comp']: # assume we have only one field for submission form field_name = list(data['form'].fields.keys())[0] data['form']._errors[field_name] = ErrorList(["This file has the same name as another file in your submission."]) # remove those has errors in submitted_comp to_be_removed.reverse() for t in to_be_removed: submitted_comp.pop(t) # all okay now # end validating, begin saving if len(submitted_comp) > 0: new_sub.save() for sub in submitted_comp: sub.submission = new_sub sub.save() #LOG EVENT# if activity.group: group_str = " as a member of group %s" % new_sub.group.name else: group_str = "" l = LogEntry(userid=request.user.username, description="submitted for %s %s%s" % (activity, sub.component.title, group_str), related_object=sub) l.save() if len(not_submitted_comp) == 0: messages.add_message(request, messages.SUCCESS, "Your submission was successful.") return HttpResponseRedirect(reverse('offering:submission:show_components', args=[course_slug, activity_slug])) return render(request, "submission/submission_error.html", {"course":course, "activity":activity, "component_list":component_form_list, "submitted_comp":submitted_comp, "not_submitted_comp":not_submitted_comp}) else: #not POST component_form_list = make_form_from_list(component_list) return render(request, "submission/" + template, {'component_form_list': component_form_list, "course": course, "activity": activity, "submission_info": submission_info, "userid":userid, "late":late, "student":student, "group":group, "cansubmit":cansubmit, "is_staff":staff, 'any_submissions': any_submissions})