def handle_uploaded_file(f, extract_path, request, user): result, mod_name = extract_file(f, extract_path, request, user) if not result: return result, mod_name, False # check there is at least a sub dir if mod_name == '': msg_text = _(u"Invalid zip file") messages.info(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="invalid_zip", data=msg_text).save() shutil.rmtree(extract_path, ignore_errors=True) return False, 400, False response = 200 try: course, response, is_new_course = process_course( extract_path, f, mod_name, request, user) except Exception as e: logger.error(e) messages.error(request, str(e), extra_tags="danger") CoursePublishingLog(user=user, action="upload_error", data=str(e)).save() return False, 500, False finally: # remove the temp upload files shutil.rmtree(extract_path, ignore_errors=True) return course, response, is_new_course
def upload_step2(request, course_id, editing=False): if (editing and not can_edit_course(request, course_id)): raise exceptions.PermissionDenied course = Course.objects.get(pk=course_id) if request.method == 'POST': form = UploadCourseStep2Form(request.POST, request.FILES) if form.is_valid() and course: # add the tags add_course_tags(form, course, request.user) redirect = 'oppia_course' if editing else 'oppia_upload_success' CoursePublishingLog( course=course, new_version=course.version, user=request.user, action="upload_course_published", data=_(u'Course published via file upload')).save() return HttpResponseRedirect(reverse(redirect)) else: form = UploadCourseStep2Form(initial={ 'tags': course.get_tags(), 'is_draft': course.is_draft, }) page_title = _(u'Upload Course - step 2') if not editing else _( u'Edit course') return render( request, 'course/form.html', { 'form': form, 'course_title': course.title, 'editing': editing, 'title': page_title })
def process_course_media(request, media_element, course, user): for file_element in media_element.findall('file'): media = Media() media.course = course media.filename = file_element.get("filename") url = file_element.get("download_url") media.digest = file_element.get("digest") if len(url) > Media.URL_MAX_LENGTH: msg_text = _(u'File %(filename)s has a download URL larger \ than the maximum length permitted. The media file \ has not been registered, so it won\'t be tracked. \ Please, fix this issue and upload the course \ again.') % {'filename': media.filename} messages.info(request, msg_text) CoursePublishingLog(course=course, user=user, action="media_url_too_long", data=msg_text).save() else: media.download_url = url # get any optional attributes for attr_name, attr_value in file_element.attrib.items(): if attr_name == "length": media.media_length = attr_value if attr_name == "filesize": media.filesize = attr_value media.save() # save gamification events gamification = file_element.find('gamification') events = parse_gamification_events(gamification) process_course_media_events(request, media, events, course, user)
def upload_step1(request): if request.method == 'POST': form = UploadCourseStep1Form(request.POST, request.FILES) if form.is_valid(): # All validation rules pass extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp', str(request.user.id)) course, resp = handle_uploaded_file(request.FILES['course_file'], extract_path, request, request.user) if course: CoursePublishingLog( course=course, user=request.user, action="file_uploaded", data=request.FILES['course_file'].name).save() return HttpResponseRedirect( reverse('oppia_upload2', args=[course.id])) # Redirect after POST else: os.remove( os.path.join(settings.COURSE_UPLOAD_DIR, request.FILES['course_file'].name)) else: form = UploadCourseStep1Form() # An unbound form return render(request, 'course/form.html', { 'form': form, 'title': _(u'Upload Course - step 1') })
def process_course_sections(request, structure, course, user, is_new_course): for index, section in enumerate(structure.findall("section")): activities = section.find('activities') # Check if the section contains any activity # (to avoid saving an empty one) if activities is None or len(activities.findall('activity')) == 0: msg_text = _("Section ") \ + str(index + 1) \ + _(" does not contain any activities.") messages.info(request, msg_text) CoursePublishingLog(course=course, user=user, action="no_activities", data=msg_text).save() continue title = {} for t in section.findall('title'): title[t.get('lang')] = t.text section = Section(course=course, title=json.dumps(title), order=section.get('order')) section.save() for act in activities.findall("activity"): parse_and_save_activity(request, user, course, section, act, is_new_course)
def handle_uploaded_file(f, extract_path, request, user): zipfilepath = os.path.join(settings.COURSE_UPLOAD_DIR, f.name) with open(zipfilepath, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) try: zip_file = ZipFile(zipfilepath) zip_file.extractall(path=extract_path) except (OSError, BadZipfile): msg_text = _(u"Invalid zip file") messages.error(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="invalid_zip", data=msg_text).save() shutil.rmtree(extract_path, ignore_errors=True) return False, 500 mod_name = '' for x in os.listdir(extract_path): if os.path.isdir(os.path.join(extract_path, x)): mod_name = x # check there is at least a sub dir if mod_name == '': msg_text = _(u"Invalid zip file") messages.info(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="invalid_zip", data=msg_text).save() shutil.rmtree(extract_path, ignore_errors=True) return False, 400 response = 200 try: course, response = process_course(extract_path, f, mod_name, request, user) except Exception as e: logger.error(e) messages.error(request, str(e), extra_tags="danger") CoursePublishingLog(user=user, action="upload_error", data=str(e)).save() return False, 500 finally: # remove the temp upload files shutil.rmtree(extract_path, ignore_errors=True) return course, response
def form_valid(self, form): CoursePublishingLog( course=self.course, new_version=self.course.version, user=self.request.user, action="upload_course_published", data=_(u'Course published via file upload')).save() return super().form_valid(form)
def check_upload_file_size(file, validation_errors): max_upload = SettingProperties.get_int(constants.MAX_UPLOAD_SIZE, settings.OPPIA_MAX_UPLOAD_SIZE) if file is not None and file.size > max_upload: size = int(math.floor(max_upload / 1024 / 1024)) validation_errors.append((_(u"Your file is larger than the maximum \ allowed (%(size)d Mb). You may want to \ check your course for large includes, \ such as images etc.") % {'size': size, })) msg_text = _(u"Maximum course file upload size exceeded") CoursePublishingLog(action="over_max_upload", data=msg_text).save() return validation_errors
def form_valid(self, form): user = self.request.user extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp', str(user.id)) course, resp = handle_uploaded_file(self.request.FILES['course_file'], extract_path, self.request, user) if course: CoursePublishingLog(course=course, user=user, action="file_uploaded", data=self.request.FILES['course_file'].name) \ .save() return HttpResponseRedirect( reverse('oppia:upload_step2', args=[course.id])) else: return super().form_invalid(form)
def process_course_media_events(request, media, events, course, user): for event in events: # Only add events if the didn't exist previously e, created = MediaGamificationEvent.objects \ .get_or_create(media=media, event=event['name'], defaults={'points': event['points'], 'user': request.user}) if created: msg_text = _(u'Gamification for "%(event)s" at course \ level added') % {'event': e.event} messages.info(request, msg_text) CoursePublishingLog(course=course, user=user, action="course_gamification_added", data=msg_text).save()
def clean_old_course(req, user, oldsections, old_course_filename, course): for section in oldsections: sec = Section.objects.get(pk=section) for act in sec.activities(): msg_text = _(u'Activity "%(act)s"(%(digest)s) is no longer in the course.') % {'act': act.title, 'digest': act.digest} messages.info(req, msg_text) CoursePublishingLog(course=course, user=user, action="activity_removed", data=msg_text).save() sec.delete() if old_course_filename is not None and old_course_filename != course.filename: try: os.remove(os.path.join(settings.COURSE_UPLOAD_DIR, old_course_filename)) except OSError: pass
def check_required_fields(request, validation_errors): required = ['username', 'password', 'tags', 'is_draft'] for field in required: if field not in request.POST or request.POST[field].strip() == '': validation_errors.append( "field '{0}' is missing or empty".format(field)) if api.COURSE_FILE_FIELD not in request.FILES: validation_errors.append("Course file not found") else: course_file = request.FILES[api.COURSE_FILE_FIELD] if course_file is not None and course_file.content_type != 'application/zip' and course_file.content_type != 'application/x-zip-compressed': validation_errors.append("You may only upload a zip file") msg_text = _(u"Invalid zip file") CoursePublishingLog(action="invalid_zip", data=msg_text).save() return validation_errors
def get_course_shortname(f, extract_path, request, user): result, mod_name = extract_file(f, extract_path, request, user) if not result: return result, mod_name xml_path = os.path.join(extract_path, mod_name, "module.xml") # check that the module.xml file exists if not os.path.isfile(xml_path): msg_text = _(u"Zip file does not contain a module.xml file") messages.info(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="no_module_xml", data=msg_text).save() return False, 400 # parse the module.xml file doc = ET.parse(xml_path) meta_info = parse_course_meta(doc) return True, meta_info['shortname']
def extract_file(f, extract_path, request, user): zipfilepath = os.path.join(settings.COURSE_UPLOAD_DIR, f.name) with open(zipfilepath, 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) try: zip_file = ZipFile(zipfilepath) zip_file.extractall(path=extract_path) except (OSError, BadZipfile): msg_text = _(u"Invalid zip file") messages.error(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="invalid_zip", data=msg_text).save() shutil.rmtree(extract_path, ignore_errors=True) return False, 500 mod_name = '' for x in os.listdir(extract_path): if os.path.isdir(os.path.join(extract_path, x)): mod_name = x return True, mod_name
def parse_course_contents(request, xml_doc, course, user, is_new_course): # add in any baseline activities parse_baseline_activities(request, xml_doc, course, user, is_new_course) # add all the sections and activities structure = xml_doc.find("structure") if len(structure.findall("section")) == 0: course.delete() msg_text = \ _(u"There don't appear to be any activities in this upload file.") messages.info(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="no_activities", data=msg_text).save() return False process_course_sections(request, structure, course, user, is_new_course) media_element = xml_doc.find('media') if media_element is not None: process_course_media(request, media_element, course, user) return True
def publish_view(request): # get the messages to clear possible previous unprocessed messages get_messages_array(request) if request.method != 'POST': return HttpResponse(status=405) validation_errors = [] validation_errors = check_required_fields(request, validation_errors) validation_errors = check_upload_file_size( request.FILES[api.COURSE_FILE_FIELD], validation_errors) if validation_errors: return JsonResponse({'errors': validation_errors}, status=400, ) # authenticate user authenticated, response_data, user = authenticate_user( request, request.POST['username'], request.POST['password']) if not authenticated: return JsonResponse(response_data, status=401) # check user has permissions to publish course if settings.OPPIA_STAFF_ONLY_UPLOAD is True \ and not user.is_staff \ and user.userprofile.can_upload is False: return HttpResponse(status=401) extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp', str(user.id)) course, status_code = handle_uploaded_file( request.FILES[api.COURSE_FILE_FIELD], extract_path, request, user) CoursePublishingLog(course=course if course else None, new_version=course.version if course else None, user=user, action="api_file_uploaded", data=request.FILES[api.COURSE_FILE_FIELD].name).save() if course is False: status = status_code if status_code is not None else 500 response_data = { 'messages': get_messages_array(request) } return JsonResponse(response_data, status=status) else: course.is_draft = (request.POST['is_draft'] == "True" or request.POST['is_draft'] == "true") course.save() # remove any existing tags CourseTag.objects.filter(course=course).delete() # add tags tags = request.POST['tags'].strip().split(",") add_course_tags(user, course, tags) msgs = get_messages_array(request) CoursePublishingLog(course=course, new_version=course.version, user=user, action="api_course_published", data=_(u'Course published via API')).save() if len(msgs) > 0: return JsonResponse({'messages': msgs}, status=201) else: return HttpResponse(status=201)
def parse_course_contents(req, xml_doc, course, user, new_course): # add in any baseline activities parse_baseline_activities(req, xml_doc, course, user, new_course) # add all the sections and activities structure = xml_doc.find("structure") if len(structure.findall("section")) == 0: course.delete() msg_text = _(u"There don't appear to be any activities in this upload file.") messages.info(req, msg_text) CoursePublishingLog(course=course, user=user, action="no_activities", data=msg_text).save() return for idx, s in enumerate(structure.findall("section")): activities = s.find('activities') # Check if the section contains any activity (to avoid saving an empty one) if activities is None or len(activities.findall('activity')) == 0: msg_text = _("Section ") + str(idx + 1) + _(" does not contain any activities.") messages.info(req, msg_text) CoursePublishingLog(course=course, user=user, action="no_activities", data=msg_text).save() continue title = {} for t in s.findall('title'): title[t.get('lang')] = t.text section = Section( course=course, title=json.dumps(title), order=s.get('order') ) section.save() for act in activities.findall("activity"): parse_and_save_activity(req, user, course, section, act, new_course) media_element = xml_doc.find('media') if media_element is not None: for file_element in media_element.findall('file'): media = Media() media.course = course media.filename = file_element.get("filename") url = file_element.get("download_url") media.digest = file_element.get("digest") if len(url) > Media.URL_MAX_LENGTH: msg_text = _(u'File %(filename)s has a download URL larger than the maximum length permitted. The media file has not been registered, so it won\'t be tracked. Please, fix this issue and upload the course again.') % {'filename': media.filename} messages.info(req, msg_text) CoursePublishingLog(course=course, user=user, action="media_url_too_long", data=msg_text).save() else: media.download_url = url # get any optional attributes for attr_name, attr_value in file_element.attrib.items(): if attr_name == "length": media.media_length = attr_value if attr_name == "filesize": media.filesize = attr_value media.save() # save gamification events gamification = file_element.find('gamification') events = parse_gamification_events(gamification) for event in events: # Only add events if the didn't exist previously e, created = MediaGamificationEvent.objects.get_or_create( media=media, event=event['name'], defaults={'points': event['points'], 'user': req.user}) if created: msg_text = _(u'Gamification for "%(event)s" at course level added') % {'event': e.event} messages.info(req, msg_text) CoursePublishingLog(course=course, user=user, action="course_gamification_added", data=msg_text).save()
def publish_view(request): # get the messages to clear possible previous unprocessed messages get_messages_array(request) if request.method != 'POST': return HttpResponse(status=405) course_file = request.FILES.get(api.COURSE_FILE_FIELD, None) validation_errors = [] validation_errors = check_required_fields(request, validation_errors) validation_errors = check_upload_file_size(course_file, validation_errors) if validation_errors: return JsonResponse( {'errors': validation_errors}, status=400, ) username = request.POST.get('username', None) password = request.POST.get('password', None) # authenticate user authenticated, response_data, user = authenticate_user( request, username, password) if not authenticated: return JsonResponse(response_data, status=401) extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp', str(user.id)) result, course_shortname = get_course_shortname(course_file, extract_path, request, user) if result: course_manager = CoursePermissions.objects.filter( user=user, course__shortname=course_shortname, role=CoursePermissions.MANAGER).count() else: course_manager = 0 # check user has permissions to publish course if not user.is_staff \ and user.userprofile.can_upload is False \ and course_manager == 0: msg_text = \ _(u"Sorry, only the original owner may update this course") messages.info(request, msg_text) CoursePublishingLog(user=user if user else None, action="permissions_error", data=msg_text).save() return HttpResponse(status=401) course, status_code = handle_uploaded_file(course_file, extract_path, request, user) CoursePublishingLog(course=course if course else None, new_version=course.version if course else None, user=user, action="api_file_uploaded", data=course_file.name).save() if course is False: status = status_code if status_code is not None else 500 response_data = {'messages': get_messages_array(request)} return JsonResponse(response_data, status=status) else: course.is_draft = (request.POST['is_draft'] == "True" or request.POST['is_draft'] == "true") course.save() # remove any existing tags CourseTag.objects.filter(course=course).delete() # add tags tags = request.POST['tags'].strip().split(",") add_course_tags(user, course, tags) msgs = get_messages_array(request) CoursePublishingLog(course=course, new_version=course.version, user=user, action="api_course_published", data=_(u'Course published via API')).save() if len(msgs) > 0: return JsonResponse({'messages': msgs}, status=201) else: return HttpResponse(status=201)
def parse_and_save_activity(request, user, course, section, activity_node, is_new_course, is_baseline=False): """ Parses an Activity XML and saves it to the DB :param section: section the activity belongs to :param act: a XML DOM element containing a single activity :param is_new_course: boolean indicating if it is a new course or existed previously :param is_baseline: is the activity part of the baseline? :return: None """ title = {} for t in activity_node.findall('title'): title[t.get('lang')] = t.text title = json.dumps(title) if title else None description = {} for t in activity_node.findall('description'): description[t.get('lang')] = t.text description = json.dumps(description) if description else None content, activity_type = get_activity_content(activity_node) image = None for i in activity_node.findall("image"): image = i.get('filename') digest = activity_node.get("digest") existed = False try: activity = Activity.objects.get( digest=digest, section__course__shortname=course.shortname) existed = True except Activity.DoesNotExist: activity = Activity() activity.section = section activity.title = title activity.type = activity_type activity.order = activity_node.get("order") activity.digest = digest activity.baseline = is_baseline activity.image = image activity.content = content activity.description = description if not existed and not is_new_course: msg_text = _(u'Activity "%(act)s"(%(digest)s) did not exist \ previously.') % { 'act': activity.title, 'digest': activity.digest } messages.warning(request, msg_text) CoursePublishingLog(course=course, user=user, action="activity_added", data=msg_text).save() else: msg_text = _(u'Activity "%(act)s"(%(digest)s) previously existed. \ Updated with new information' ) \ % {'act': activity.title, 'digest': activity.digest} ''' If we also want to show the activities that previously existed, uncomment this next line messages.info(req, msg_text) ''' CoursePublishingLog(course=course, user=user, action="activity_updated", data=msg_text).save() if (activity_type == "quiz") or (activity_type == "feedback"): updated_json = parse_and_save_quiz(user, activity) # we need to update the JSON contents both in the XML and in the # activity data activity_node.find("content").text = \ "<![CDATA[ " + updated_json + "]]>" activity.content = updated_json activity.save() # save gamification events gamification = activity_node.find('gamification') events = parse_gamification_events(gamification) for event in events: e, created = ActivityGamificationEvent.objects.get_or_create( activity=activity, event=event['name'], defaults={ 'points': event['points'], 'user': request.user }) if created: msg_text = _(u'Gamification for "%(event)s" at activity \ "%(act)s"(%(digest)s) added' ) \ % {'event': e.event, 'act': activity.title, 'digest': activity.digest} messages.info(request, msg_text) CoursePublishingLog(course=course, user=user, action="activity_gamification_added", data=msg_text).save()
def process_course(extract_path, f, mod_name, request, user): xml_path = os.path.join(extract_path, mod_name, "module.xml") # check that the module.xml file exists if not os.path.isfile(xml_path): msg_text = _(u"Zip file does not contain a module.xml file") messages.info(request, msg_text, extra_tags="danger") CoursePublishingLog(user=user, action="no_module_xml", data=msg_text).save() return False, 400, False # parse the module.xml file doc = ET.parse(xml_path) meta_info = parse_course_meta(doc) is_new_course = False oldsections = [] old_course_filename = None # Find if course already exists try: course = Course.objects.get(shortname=meta_info['shortname']) course_manager = CoursePermissions.objects.filter( user=user, course=course, role=CoursePermissions.MANAGER).count() # check that the current user is allowed to wipe out the other course if course.user != user and course_manager == 0: msg_text = \ _(u"Sorry, you do not have permissions to update this course.") messages.info(request, msg_text) CoursePublishingLog(course=course, new_version=meta_info['versionid'], old_version=course.version, user=user, action="permissions_error", data=msg_text).save() return False, 401, is_new_course # check if course version is older if course.version > meta_info['versionid']: msg_text = _(u"A newer version of this course already exists") messages.info(request, msg_text) CoursePublishingLog(course=course, new_version=meta_info['versionid'], old_version=course.version, user=user, action="newer_version_exists", data=msg_text).save() return False, 400, is_new_course # obtain the old sections oldsections = list( Section.objects.filter(course=course).values_list('pk', flat=True)) # wipe out old media oldmedia = Media.objects.filter(course=course) oldmedia.delete() old_course_filename = course.filename course.lastupdated_date = timezone.now() except Course.DoesNotExist: course = Course() course.status = CourseStatus.DRAFT is_new_course = True old_course_version = course.version course.shortname = meta_info['shortname'] course.title = meta_info['title'] course.description = meta_info['description'] course.version = meta_info['versionid'] course.priority = int(meta_info['priority']) course.user = user course.filename = f.name course.save() if not parse_course_contents(request, doc, course, user, is_new_course): return False, 500, is_new_course clean_old_course(request, user, oldsections, old_course_filename, course) # save gamification events if 'gamification' in meta_info: events = parse_gamification_events(meta_info['gamification']) for event in events: # Only add events if the didn't exist previously e, created = CourseGamificationEvent.objects.get_or_create( course=course, event=event['name'], defaults={ 'points': event['points'], 'user': user }) if created: msg_text = \ _(u'Gamification for "%(event)s" at course level added') \ % {'event': e.event} messages.info(request, msg_text) CoursePublishingLog(course=course, new_version=meta_info['versionid'], old_version=old_course_version, user=user, action="gamification_added", data=msg_text).save() tmp_path = replace_zip_contents(xml_path, doc, mod_name, extract_path) # Extract the final file into the courses area for preview zipfilepath = os.path.join(settings.COURSE_UPLOAD_DIR, f.name) shutil.copy(tmp_path + ".zip", zipfilepath) course_preview_path = os.path.join(settings.MEDIA_ROOT, "courses") ZipFile(zipfilepath).extractall(path=course_preview_path) writer = GamificationXMLWriter(course) writer.update_gamification(request.user) return course, 200, is_new_course