def handle_uploaded_file(f, extract_path, request, user):
result, mod_name = extract_file(f, extract_path, request, user)
if not result:
return result, mod_name, False
# check there is at least a sub dir
if mod_name == '':
msg_text = _(u"Invalid zip file")
messages.info(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="invalid_zip",
data=msg_text).save()
shutil.rmtree(extract_path, ignore_errors=True)
return False, 400, False
response = 200
try:
course, response, is_new_course = process_course(
extract_path, f, mod_name, request, user)
except Exception as e:
logger.error(e)
messages.error(request, str(e), extra_tags="danger")
CoursePublishingLog(user=user, action="upload_error",
data=str(e)).save()
return False, 500, False
finally:
# remove the temp upload files
shutil.rmtree(extract_path, ignore_errors=True)
return course, response, is_new_course
def upload_step2(request, course_id, editing=False):
if (editing and not can_edit_course(request, course_id)):
raise exceptions.PermissionDenied
course = Course.objects.get(pk=course_id)
if request.method == 'POST':
form = UploadCourseStep2Form(request.POST, request.FILES)
if form.is_valid() and course:
# add the tags
add_course_tags(form, course, request.user)
redirect = 'oppia_course' if editing else 'oppia_upload_success'
CoursePublishingLog(
course=course,
new_version=course.version,
user=request.user,
action="upload_course_published",
data=_(u'Course published via file upload')).save()
return HttpResponseRedirect(reverse(redirect))
else:
form = UploadCourseStep2Form(initial={
'tags': course.get_tags(),
'is_draft': course.is_draft,
})
page_title = _(u'Upload Course - step 2') if not editing else _(
u'Edit course')
return render(
request, 'course/form.html', {
'form': form,
'course_title': course.title,
'editing': editing,
'title': page_title
})
def process_course_media(request, media_element, course, user):
for file_element in media_element.findall('file'):
media = Media()
media.course = course
media.filename = file_element.get("filename")
url = file_element.get("download_url")
media.digest = file_element.get("digest")
if len(url) > Media.URL_MAX_LENGTH:
msg_text = _(u'File %(filename)s has a download URL larger \
than the maximum length permitted. The media file \
has not been registered, so it won\'t be tracked. \
Please, fix this issue and upload the course \
again.') % {'filename': media.filename}
messages.info(request, msg_text)
CoursePublishingLog(course=course,
user=user,
action="media_url_too_long",
data=msg_text).save()
else:
media.download_url = url
# get any optional attributes
for attr_name, attr_value in file_element.attrib.items():
if attr_name == "length":
media.media_length = attr_value
if attr_name == "filesize":
media.filesize = attr_value
media.save()
# save gamification events
gamification = file_element.find('gamification')
events = parse_gamification_events(gamification)
process_course_media_events(request, media, events, course, user)
def upload_step1(request):
if request.method == 'POST':
form = UploadCourseStep1Form(request.POST, request.FILES)
if form.is_valid(): # All validation rules pass
extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp',
str(request.user.id))
course, resp = handle_uploaded_file(request.FILES['course_file'],
extract_path, request,
request.user)
if course:
CoursePublishingLog(
course=course,
user=request.user,
action="file_uploaded",
data=request.FILES['course_file'].name).save()
return HttpResponseRedirect(
reverse('oppia_upload2',
args=[course.id])) # Redirect after POST
else:
os.remove(
os.path.join(settings.COURSE_UPLOAD_DIR,
request.FILES['course_file'].name))
else:
form = UploadCourseStep1Form() # An unbound form
return render(request, 'course/form.html', {
'form': form,
'title': _(u'Upload Course - step 1')
})
def process_course_sections(request, structure, course, user, is_new_course):
for index, section in enumerate(structure.findall("section")):
activities = section.find('activities')
# Check if the section contains any activity
# (to avoid saving an empty one)
if activities is None or len(activities.findall('activity')) == 0:
msg_text = _("Section ") \
+ str(index + 1) \
+ _(" does not contain any activities.")
messages.info(request, msg_text)
CoursePublishingLog(course=course,
user=user,
action="no_activities",
data=msg_text).save()
continue
title = {}
for t in section.findall('title'):
title[t.get('lang')] = t.text
section = Section(course=course,
title=json.dumps(title),
order=section.get('order'))
section.save()
for act in activities.findall("activity"):
parse_and_save_activity(request, user, course, section, act,
is_new_course)
def handle_uploaded_file(f, extract_path, request, user):
zipfilepath = os.path.join(settings.COURSE_UPLOAD_DIR, f.name)
with open(zipfilepath, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
try:
zip_file = ZipFile(zipfilepath)
zip_file.extractall(path=extract_path)
except (OSError, BadZipfile):
msg_text = _(u"Invalid zip file")
messages.error(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="invalid_zip",
data=msg_text).save()
shutil.rmtree(extract_path, ignore_errors=True)
return False, 500
mod_name = ''
for x in os.listdir(extract_path):
if os.path.isdir(os.path.join(extract_path, x)):
mod_name = x
# check there is at least a sub dir
if mod_name == '':
msg_text = _(u"Invalid zip file")
messages.info(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="invalid_zip",
data=msg_text).save()
shutil.rmtree(extract_path, ignore_errors=True)
return False, 400
response = 200
try:
course, response = process_course(extract_path, f, mod_name, request,
user)
except Exception as e:
logger.error(e)
messages.error(request, str(e), extra_tags="danger")
CoursePublishingLog(user=user, action="upload_error",
data=str(e)).save()
return False, 500
finally:
# remove the temp upload files
shutil.rmtree(extract_path, ignore_errors=True)
return course, response
def form_valid(self, form):
CoursePublishingLog(
course=self.course,
new_version=self.course.version,
user=self.request.user,
action="upload_course_published",
data=_(u'Course published via file upload')).save()
return super().form_valid(form)
def check_upload_file_size(file, validation_errors):
max_upload = SettingProperties.get_int(constants.MAX_UPLOAD_SIZE,
settings.OPPIA_MAX_UPLOAD_SIZE)
if file is not None and file.size > max_upload:
size = int(math.floor(max_upload / 1024 / 1024))
validation_errors.append((_(u"Your file is larger than the maximum \
allowed (%(size)d Mb). You may want to \
check your course for large includes, \
such as images etc.") % {'size': size, }))
msg_text = _(u"Maximum course file upload size exceeded")
CoursePublishingLog(action="over_max_upload", data=msg_text).save()
return validation_errors
def form_valid(self, form):
user = self.request.user
extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp',
str(user.id))
course, resp = handle_uploaded_file(self.request.FILES['course_file'],
extract_path, self.request, user)
if course:
CoursePublishingLog(course=course,
user=user,
action="file_uploaded",
data=self.request.FILES['course_file'].name) \
.save()
return HttpResponseRedirect(
reverse('oppia:upload_step2', args=[course.id]))
else:
return super().form_invalid(form)
def process_course_media_events(request, media, events, course, user):
for event in events:
# Only add events if the didn't exist previously
e, created = MediaGamificationEvent.objects \
.get_or_create(media=media,
event=event['name'],
defaults={'points': event['points'],
'user': request.user})
if created:
msg_text = _(u'Gamification for "%(event)s" at course \
level added') % {'event': e.event}
messages.info(request, msg_text)
CoursePublishingLog(course=course,
user=user,
action="course_gamification_added",
data=msg_text).save()
def clean_old_course(req, user, oldsections, old_course_filename, course):
for section in oldsections:
sec = Section.objects.get(pk=section)
for act in sec.activities():
msg_text = _(u'Activity "%(act)s"(%(digest)s) is no longer in the course.') % {'act': act.title, 'digest': act.digest}
messages.info(req, msg_text)
CoursePublishingLog(course=course,
user=user,
action="activity_removed",
data=msg_text).save()
sec.delete()
if old_course_filename is not None and old_course_filename != course.filename:
try:
os.remove(os.path.join(settings.COURSE_UPLOAD_DIR,
old_course_filename))
except OSError:
pass
def check_required_fields(request, validation_errors):
required = ['username', 'password', 'tags', 'is_draft']
for field in required:
if field not in request.POST or request.POST[field].strip() == '':
validation_errors.append(
"field '{0}' is missing or empty".format(field))
if api.COURSE_FILE_FIELD not in request.FILES:
validation_errors.append("Course file not found")
else:
course_file = request.FILES[api.COURSE_FILE_FIELD]
if course_file is not None and course_file.content_type != 'application/zip' and course_file.content_type != 'application/x-zip-compressed':
validation_errors.append("You may only upload a zip file")
msg_text = _(u"Invalid zip file")
CoursePublishingLog(action="invalid_zip", data=msg_text).save()
return validation_errors
def get_course_shortname(f, extract_path, request, user):
result, mod_name = extract_file(f, extract_path, request, user)
if not result:
return result, mod_name
xml_path = os.path.join(extract_path, mod_name, "module.xml")
# check that the module.xml file exists
if not os.path.isfile(xml_path):
msg_text = _(u"Zip file does not contain a module.xml file")
messages.info(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="no_module_xml",
data=msg_text).save()
return False, 400
# parse the module.xml file
doc = ET.parse(xml_path)
meta_info = parse_course_meta(doc)
return True, meta_info['shortname']
def extract_file(f, extract_path, request, user):
zipfilepath = os.path.join(settings.COURSE_UPLOAD_DIR, f.name)
with open(zipfilepath, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
try:
zip_file = ZipFile(zipfilepath)
zip_file.extractall(path=extract_path)
except (OSError, BadZipfile):
msg_text = _(u"Invalid zip file")
messages.error(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="invalid_zip",
data=msg_text).save()
shutil.rmtree(extract_path, ignore_errors=True)
return False, 500
mod_name = ''
for x in os.listdir(extract_path):
if os.path.isdir(os.path.join(extract_path, x)):
mod_name = x
return True, mod_name
def parse_course_contents(request, xml_doc, course, user, is_new_course):
# add in any baseline activities
parse_baseline_activities(request, xml_doc, course, user, is_new_course)
# add all the sections and activities
structure = xml_doc.find("structure")
if len(structure.findall("section")) == 0:
course.delete()
msg_text = \
_(u"There don't appear to be any activities in this upload file.")
messages.info(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="no_activities",
data=msg_text).save()
return False
process_course_sections(request, structure, course, user, is_new_course)
media_element = xml_doc.find('media')
if media_element is not None:
process_course_media(request, media_element, course, user)
return True
def publish_view(request):
# get the messages to clear possible previous unprocessed messages
get_messages_array(request)
if request.method != 'POST':
return HttpResponse(status=405)
validation_errors = []
validation_errors = check_required_fields(request, validation_errors)
validation_errors = check_upload_file_size(
request.FILES[api.COURSE_FILE_FIELD],
validation_errors)
if validation_errors:
return JsonResponse({'errors': validation_errors}, status=400, )
# authenticate user
authenticated, response_data, user = authenticate_user(
request,
request.POST['username'],
request.POST['password'])
if not authenticated:
return JsonResponse(response_data, status=401)
# check user has permissions to publish course
if settings.OPPIA_STAFF_ONLY_UPLOAD is True \
and not user.is_staff \
and user.userprofile.can_upload is False:
return HttpResponse(status=401)
extract_path = os.path.join(settings.COURSE_UPLOAD_DIR,
'temp',
str(user.id))
course, status_code = handle_uploaded_file(
request.FILES[api.COURSE_FILE_FIELD],
extract_path,
request,
user)
CoursePublishingLog(course=course if course else None,
new_version=course.version if course else None,
user=user,
action="api_file_uploaded",
data=request.FILES[api.COURSE_FILE_FIELD].name).save()
if course is False:
status = status_code if status_code is not None else 500
response_data = {
'messages': get_messages_array(request)
}
return JsonResponse(response_data, status=status)
else:
course.is_draft = (request.POST['is_draft'] == "True"
or request.POST['is_draft'] == "true")
course.save()
# remove any existing tags
CourseTag.objects.filter(course=course).delete()
# add tags
tags = request.POST['tags'].strip().split(",")
add_course_tags(user, course, tags)
msgs = get_messages_array(request)
CoursePublishingLog(course=course,
new_version=course.version,
user=user,
action="api_course_published",
data=_(u'Course published via API')).save()
if len(msgs) > 0:
return JsonResponse({'messages': msgs}, status=201)
else:
return HttpResponse(status=201)
def parse_course_contents(req, xml_doc, course, user, new_course):
# add in any baseline activities
parse_baseline_activities(req, xml_doc, course, user, new_course)
# add all the sections and activities
structure = xml_doc.find("structure")
if len(structure.findall("section")) == 0:
course.delete()
msg_text = _(u"There don't appear to be any activities in this upload file.")
messages.info(req, msg_text)
CoursePublishingLog(course=course,
user=user,
action="no_activities",
data=msg_text).save()
return
for idx, s in enumerate(structure.findall("section")):
activities = s.find('activities')
# Check if the section contains any activity (to avoid saving an empty one)
if activities is None or len(activities.findall('activity')) == 0:
msg_text = _("Section ") + str(idx + 1) + _(" does not contain any activities.")
messages.info(req, msg_text)
CoursePublishingLog(course=course,
user=user,
action="no_activities",
data=msg_text).save()
continue
title = {}
for t in s.findall('title'):
title[t.get('lang')] = t.text
section = Section(
course=course,
title=json.dumps(title),
order=s.get('order')
)
section.save()
for act in activities.findall("activity"):
parse_and_save_activity(req,
user,
course,
section,
act,
new_course)
media_element = xml_doc.find('media')
if media_element is not None:
for file_element in media_element.findall('file'):
media = Media()
media.course = course
media.filename = file_element.get("filename")
url = file_element.get("download_url")
media.digest = file_element.get("digest")
if len(url) > Media.URL_MAX_LENGTH:
msg_text = _(u'File %(filename)s has a download URL larger than the maximum length permitted. The media file has not been registered, so it won\'t be tracked. Please, fix this issue and upload the course again.') % {'filename': media.filename}
messages.info(req, msg_text)
CoursePublishingLog(course=course,
user=user,
action="media_url_too_long",
data=msg_text).save()
else:
media.download_url = url
# get any optional attributes
for attr_name, attr_value in file_element.attrib.items():
if attr_name == "length":
media.media_length = attr_value
if attr_name == "filesize":
media.filesize = attr_value
media.save()
# save gamification events
gamification = file_element.find('gamification')
events = parse_gamification_events(gamification)
for event in events:
# Only add events if the didn't exist previously
e, created = MediaGamificationEvent.objects.get_or_create(
media=media, event=event['name'],
defaults={'points': event['points'], 'user': req.user})
if created:
msg_text = _(u'Gamification for "%(event)s" at course level added') % {'event': e.event}
messages.info(req, msg_text)
CoursePublishingLog(course=course,
user=user,
action="course_gamification_added",
data=msg_text).save()
def publish_view(request):
# get the messages to clear possible previous unprocessed messages
get_messages_array(request)
if request.method != 'POST':
return HttpResponse(status=405)
course_file = request.FILES.get(api.COURSE_FILE_FIELD, None)
validation_errors = []
validation_errors = check_required_fields(request, validation_errors)
validation_errors = check_upload_file_size(course_file, validation_errors)
if validation_errors:
return JsonResponse(
{'errors': validation_errors},
status=400,
)
username = request.POST.get('username', None)
password = request.POST.get('password', None)
# authenticate user
authenticated, response_data, user = authenticate_user(
request, username, password)
if not authenticated:
return JsonResponse(response_data, status=401)
extract_path = os.path.join(settings.COURSE_UPLOAD_DIR, 'temp',
str(user.id))
result, course_shortname = get_course_shortname(course_file, extract_path,
request, user)
if result:
course_manager = CoursePermissions.objects.filter(
user=user,
course__shortname=course_shortname,
role=CoursePermissions.MANAGER).count()
else:
course_manager = 0
# check user has permissions to publish course
if not user.is_staff \
and user.userprofile.can_upload is False \
and course_manager == 0:
msg_text = \
_(u"Sorry, only the original owner may update this course")
messages.info(request, msg_text)
CoursePublishingLog(user=user if user else None,
action="permissions_error",
data=msg_text).save()
return HttpResponse(status=401)
course, status_code = handle_uploaded_file(course_file, extract_path,
request, user)
CoursePublishingLog(course=course if course else None,
new_version=course.version if course else None,
user=user,
action="api_file_uploaded",
data=course_file.name).save()
if course is False:
status = status_code if status_code is not None else 500
response_data = {'messages': get_messages_array(request)}
return JsonResponse(response_data, status=status)
else:
course.is_draft = (request.POST['is_draft'] == "True"
or request.POST['is_draft'] == "true")
course.save()
# remove any existing tags
CourseTag.objects.filter(course=course).delete()
# add tags
tags = request.POST['tags'].strip().split(",")
add_course_tags(user, course, tags)
msgs = get_messages_array(request)
CoursePublishingLog(course=course,
new_version=course.version,
user=user,
action="api_course_published",
data=_(u'Course published via API')).save()
if len(msgs) > 0:
return JsonResponse({'messages': msgs}, status=201)
else:
return HttpResponse(status=201)
def parse_and_save_activity(request,
user,
course,
section,
activity_node,
is_new_course,
is_baseline=False):
"""
Parses an Activity XML and saves it to the DB
:param section: section the activity belongs to
:param act: a XML DOM element containing a single activity
:param is_new_course: boolean indicating if it is a new course or existed
previously
:param is_baseline: is the activity part of the baseline?
:return: None
"""
title = {}
for t in activity_node.findall('title'):
title[t.get('lang')] = t.text
title = json.dumps(title) if title else None
description = {}
for t in activity_node.findall('description'):
description[t.get('lang')] = t.text
description = json.dumps(description) if description else None
content, activity_type = get_activity_content(activity_node)
image = None
for i in activity_node.findall("image"):
image = i.get('filename')
digest = activity_node.get("digest")
existed = False
try:
activity = Activity.objects.get(
digest=digest, section__course__shortname=course.shortname)
existed = True
except Activity.DoesNotExist:
activity = Activity()
activity.section = section
activity.title = title
activity.type = activity_type
activity.order = activity_node.get("order")
activity.digest = digest
activity.baseline = is_baseline
activity.image = image
activity.content = content
activity.description = description
if not existed and not is_new_course:
msg_text = _(u'Activity "%(act)s"(%(digest)s) did not exist \
previously.') % {
'act': activity.title,
'digest': activity.digest
}
messages.warning(request, msg_text)
CoursePublishingLog(course=course,
user=user,
action="activity_added",
data=msg_text).save()
else:
msg_text = _(u'Activity "%(act)s"(%(digest)s) previously existed. \
Updated with new information' ) \
% {'act': activity.title,
'digest': activity.digest}
'''
If we also want to show the activities that previously existed,
uncomment this next line
messages.info(req, msg_text)
'''
CoursePublishingLog(course=course,
user=user,
action="activity_updated",
data=msg_text).save()
if (activity_type == "quiz") or (activity_type == "feedback"):
updated_json = parse_and_save_quiz(user, activity)
# we need to update the JSON contents both in the XML and in the
# activity data
activity_node.find("content").text = \
"<![CDATA[ " + updated_json + "]]>"
activity.content = updated_json
activity.save()
# save gamification events
gamification = activity_node.find('gamification')
events = parse_gamification_events(gamification)
for event in events:
e, created = ActivityGamificationEvent.objects.get_or_create(
activity=activity,
event=event['name'],
defaults={
'points': event['points'],
'user': request.user
})
if created:
msg_text = _(u'Gamification for "%(event)s" at activity \
"%(act)s"(%(digest)s) added' ) \
% {'event': e.event,
'act': activity.title,
'digest': activity.digest}
messages.info(request, msg_text)
CoursePublishingLog(course=course,
user=user,
action="activity_gamification_added",
data=msg_text).save()
def process_course(extract_path, f, mod_name, request, user):
xml_path = os.path.join(extract_path, mod_name, "module.xml")
# check that the module.xml file exists
if not os.path.isfile(xml_path):
msg_text = _(u"Zip file does not contain a module.xml file")
messages.info(request, msg_text, extra_tags="danger")
CoursePublishingLog(user=user, action="no_module_xml",
data=msg_text).save()
return False, 400, False
# parse the module.xml file
doc = ET.parse(xml_path)
meta_info = parse_course_meta(doc)
is_new_course = False
oldsections = []
old_course_filename = None
# Find if course already exists
try:
course = Course.objects.get(shortname=meta_info['shortname'])
course_manager = CoursePermissions.objects.filter(
user=user, course=course, role=CoursePermissions.MANAGER).count()
# check that the current user is allowed to wipe out the other course
if course.user != user and course_manager == 0:
msg_text = \
_(u"Sorry, you do not have permissions to update this course.")
messages.info(request, msg_text)
CoursePublishingLog(course=course,
new_version=meta_info['versionid'],
old_version=course.version,
user=user,
action="permissions_error",
data=msg_text).save()
return False, 401, is_new_course
# check if course version is older
if course.version > meta_info['versionid']:
msg_text = _(u"A newer version of this course already exists")
messages.info(request, msg_text)
CoursePublishingLog(course=course,
new_version=meta_info['versionid'],
old_version=course.version,
user=user,
action="newer_version_exists",
data=msg_text).save()
return False, 400, is_new_course
# obtain the old sections
oldsections = list(
Section.objects.filter(course=course).values_list('pk', flat=True))
# wipe out old media
oldmedia = Media.objects.filter(course=course)
oldmedia.delete()
old_course_filename = course.filename
course.lastupdated_date = timezone.now()
except Course.DoesNotExist:
course = Course()
course.status = CourseStatus.DRAFT
is_new_course = True
old_course_version = course.version
course.shortname = meta_info['shortname']
course.title = meta_info['title']
course.description = meta_info['description']
course.version = meta_info['versionid']
course.priority = int(meta_info['priority'])
course.user = user
course.filename = f.name
course.save()
if not parse_course_contents(request, doc, course, user, is_new_course):
return False, 500, is_new_course
clean_old_course(request, user, oldsections, old_course_filename, course)
# save gamification events
if 'gamification' in meta_info:
events = parse_gamification_events(meta_info['gamification'])
for event in events:
# Only add events if the didn't exist previously
e, created = CourseGamificationEvent.objects.get_or_create(
course=course,
event=event['name'],
defaults={
'points': event['points'],
'user': user
})
if created:
msg_text = \
_(u'Gamification for "%(event)s" at course level added') \
% {'event': e.event}
messages.info(request, msg_text)
CoursePublishingLog(course=course,
new_version=meta_info['versionid'],
old_version=old_course_version,
user=user,
action="gamification_added",
data=msg_text).save()
tmp_path = replace_zip_contents(xml_path, doc, mod_name, extract_path)
# Extract the final file into the courses area for preview
zipfilepath = os.path.join(settings.COURSE_UPLOAD_DIR, f.name)
shutil.copy(tmp_path + ".zip", zipfilepath)
course_preview_path = os.path.join(settings.MEDIA_ROOT, "courses")
ZipFile(zipfilepath).extractall(path=course_preview_path)
writer = GamificationXMLWriter(course)
writer.update_gamification(request.user)
return course, 200, is_new_course
