def test_entrance_exam_milestone_removal(self): """ Unit Test: test removal of entrance exam milestone content """ parent_locator = six.text_type(self.course.location) created_block = create_xblock(parent_locator=parent_locator, user=self.user, category='chapter', display_name=('Entrance Exam'), is_entrance_exam=True) add_entrance_exam_milestone(self.course.id, created_block) content_milestones = milestones_helpers.get_course_content_milestones( six.text_type(self.course.id), six.text_type(created_block.location), self.milestone_relationship_types['FULFILLS']) self.assertEqual(len(content_milestones), 1) user = UserFactory() request = RequestFactory().request() request.user = user remove_entrance_exam_milestone_reference(request, self.course.id) content_milestones = milestones_helpers.get_course_content_milestones( six.text_type(self.course.id), six.text_type(created_block.location), self.milestone_relationship_types['FULFILLS']) self.assertEqual(len(content_milestones), 0)
def test_entrance_exam_milestone_removal(self): """ Unit Test: test removal of entrance exam milestone content """ parent_locator = unicode(self.course.location) created_block = create_xblock( parent_locator=parent_locator, user=self.user, category='chapter', display_name=('Entrance Exam'), is_entrance_exam=True ) add_entrance_exam_milestone(self.course.id, created_block) content_milestones = milestones_helpers.get_course_content_milestones( unicode(self.course.id), unicode(created_block.location), self.milestone_relationship_types['FULFILLS'] ) self.assertEqual(len(content_milestones), 1) user = UserFactory() request = RequestFactory().request() request.user = user remove_entrance_exam_milestone_reference(request, self.course.id) content_milestones = milestones_helpers.get_course_content_milestones( unicode(self.course.id), unicode(created_block.location), self.milestone_relationship_types['FULFILLS'] ) self.assertEqual(len(content_milestones), 0)
def test_entrance_exam_milestone_addition(self): """ Unit Test: test addition of entrance exam milestone content """ parent_locator = unicode(self.course.location) created_block = create_xblock(parent_locator=parent_locator, user=self.user, category='chapter', display_name=('Entrance Exam'), is_entrance_exam=True) add_entrance_exam_milestone(self.course.id, created_block) content_milestones = milestones_helpers.get_course_content_milestones( unicode(self.course.id), unicode(created_block.location), self.milestone_relationship_types['FULFILLS']) self.assertTrue(len(content_milestones)) self.assertEqual( len(milestones_helpers.get_course_milestones(self.course.id)), 1)
def test_entrance_exam_milestone_addition(self): """ Unit Test: test addition of entrance exam milestone content """ parent_locator = unicode(self.course.location) created_block = create_xblock( parent_locator=parent_locator, user=self.user, category='chapter', display_name=('Entrance Exam'), is_entrance_exam=True ) add_entrance_exam_milestone(self.course.id, created_block) content_milestones = milestones_helpers.get_course_content_milestones( unicode(self.course.id), unicode(created_block.location), self.milestone_relationship_types['FULFILLS'] ) self.assertTrue(len(content_milestones)) self.assertEqual(len(milestones_helpers.get_course_milestones(self.course.id)), 1)
def _import_handler(request, courselike_key, root_name, successful_url, context_name, courselike_module, import_func): """ Parameterized function containing the meat of import_handler. """ if not has_course_author_access(request.user, courselike_key): raise PermissionDenied() if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): if request.method == 'GET': raise NotImplementedError('coming soon') else: # Do everything in a try-except block to make sure everything is properly cleaned up. try: data_root = path(settings.GITHUB_REPO_ROOT) subdir = base64.urlsafe_b64encode(repr(courselike_key)) course_dir = data_root / subdir filename = request.FILES['course-data'].name # Use sessions to keep info about import progress session_status = request.session.setdefault("import_status", {}) courselike_string = unicode(courselike_key) + filename _save_request_status(request, courselike_string, 0) # If the course has an entrance exam then remove it and its corresponding milestone. # current course state before import. if root_name == COURSE_ROOT: if courselike_module.entrance_exam_enabled: remove_entrance_exam_milestone_reference(request, courselike_key) log.info( "entrance exam milestone content reference for course %s has been removed", courselike_module.id ) if not filename.endswith('.tar.gz'): _save_request_status(request, courselike_string, -1) return JsonResponse( { 'ErrMsg': _('We only support uploading a .tar.gz file.'), 'Stage': -1 }, status=415 ) temp_filepath = course_dir / filename if not course_dir.isdir(): os.mkdir(course_dir) logging.debug('importing course to {0}'.format(temp_filepath)) # Get upload chunks byte ranges try: matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"]) content_range = matches.groupdict() except KeyError: # Single chunk # no Content-Range header, so make one that will work content_range = {'start': 0, 'stop': 1, 'end': 2} # stream out the uploaded files in chunks to disk if int(content_range['start']) == 0: mode = "wb+" else: mode = "ab+" size = os.path.getsize(temp_filepath) # Check to make sure we haven't missed a chunk # This shouldn't happen, even if different instances are handling # the same session, but it's always better to catch errors earlier. if size < int(content_range['start']): _save_request_status(request, courselike_string, -1) log.warning( "Reported range %s does not match size downloaded so far %s", content_range['start'], size ) return JsonResponse( { 'ErrMsg': _('File upload corrupted. Please try again'), 'Stage': -1 }, status=409 ) # The last request sometimes comes twice. This happens because # nginx sends a 499 error code when the response takes too long. elif size > int(content_range['stop']) and size == int(content_range['end']): return JsonResponse({'ImportStatus': 1}) with open(temp_filepath, mode) as temp_file: for chunk in request.FILES['course-data'].chunks(): temp_file.write(chunk) size = os.path.getsize(temp_filepath) if int(content_range['stop']) != int(content_range['end']) - 1: # More chunks coming return JsonResponse({ "files": [{ "name": filename, "size": size, "deleteUrl": "", "deleteType": "", "url": reverse_course_url('import_handler', courselike_key), "thumbnailUrl": "" }] }) # Send errors to client with stage at which error occurred. except Exception as exception: # pylint: disable=broad-except _save_request_status(request, courselike_string, -1) if course_dir.isdir(): shutil.rmtree(course_dir) log.info("Course import %s: Temp data cleared", courselike_key) log.exception( "error importing course" ) return JsonResponse( { 'ErrMsg': str(exception), 'Stage': -1 }, status=400 ) # try-finally block for proper clean up after receiving last chunk. try: # This was the last chunk. log.info("Course import %s: Upload complete", courselike_key) _save_request_status(request, courselike_string, 1) tar_file = tarfile.open(temp_filepath) try: safetar_extractall(tar_file, (course_dir + '/').encode('utf-8')) except SuspiciousOperation as exc: _save_request_status(request, courselike_string, -1) return JsonResponse( { 'ErrMsg': 'Unsafe tar file. Aborting import.', 'SuspiciousFileOperationMsg': exc.args[0], 'Stage': -1 }, status=400 ) finally: tar_file.close() log.info("Course import %s: Uploaded file extracted", courselike_key) _save_request_status(request, courselike_string, 2) # find the 'course.xml' file def get_all_files(directory): """ For each file in the directory, yield a 2-tuple of (file-name, directory-path) """ for dirpath, _dirnames, filenames in os.walk(directory): for filename in filenames: yield (filename, dirpath) def get_dir_for_fname(directory, filename): """ Returns the dirpath for the first file found in the directory with the given name. If there is no file in the directory with the specified name, return None. """ for fname, dirpath in get_all_files(directory): if fname == filename: return dirpath return None dirpath = get_dir_for_fname(course_dir, root_name) if not dirpath: _save_request_status(request, courselike_string, -2) return JsonResponse( { 'ErrMsg': _('Could not find the {0} file in the package.').format(root_name), 'Stage': -2 }, status=415 ) dirpath = os.path.relpath(dirpath, data_root) logging.debug('found %s at %s', root_name, dirpath) log.info("Course import %s: Extracted file verified", courselike_key) _save_request_status(request, courselike_string, 3) with dog_stats_api.timer( 'courselike_import.time', tags=[u"courselike:{}".format(courselike_key)] ): courselike_items = import_func( modulestore(), request.user.id, settings.GITHUB_REPO_ROOT, [dirpath], load_error_modules=False, static_content_store=contentstore(), target_id=courselike_key ) new_location = courselike_items[0].location logging.debug('new course at %s', new_location) log.info("Course import %s: Course import successful", courselike_key) _save_request_status(request, courselike_string, 4) # Send errors to client with stage at which error occurred. except Exception as exception: # pylint: disable=broad-except log.exception( "error importing course" ) return JsonResponse( { 'ErrMsg': str(exception), 'Stage': -session_status[courselike_string] }, status=400 ) finally: if course_dir.isdir(): shutil.rmtree(course_dir) log.info("Course import %s: Temp data cleared", courselike_key) # set failed stage number with negative sign in case of unsuccessful import if session_status[courselike_string] != 4: _save_request_status(request, courselike_string, -abs(session_status[courselike_string])) # status == 4 represents that course has been imported successfully. if session_status[courselike_string] == 4 and root_name == COURSE_ROOT: # Reload the course so we have the latest state course = modulestore().get_course(courselike_key) if course.entrance_exam_enabled: entrance_exam_chapter = modulestore().get_items( course.id, qualifiers={'category': 'chapter'}, settings={'is_entrance_exam': True} )[0] metadata = {'entrance_exam_id': unicode(entrance_exam_chapter.location)} CourseMetadata.update_from_dict(metadata, course, request.user) add_entrance_exam_milestone(course.id, entrance_exam_chapter) log.info("Course %s Entrance exam imported", course.id) return JsonResponse({'Status': 'OK'}) elif request.method == 'GET': # assume html status_url = reverse_course_url( "import_status_handler", courselike_key, kwargs={'filename': "fillerName"} ) return render_to_response('import.html', { context_name: courselike_module, 'successful_import_redirect_url': successful_url, 'import_status_url': status_url, 'library': isinstance(courselike_key, LibraryLocator) }) else: return HttpResponseNotFound()
def import_olx(self, user_id, course_key_string, archive_path, archive_name, language): """ Import a course or library from a provided OLX .tar.gz archive. """ courselike_key = CourseKey.from_string(course_key_string) try: user = User.objects.get(pk=user_id) except User.DoesNotExist: with respect_language(language): self.status.fail(_(u'Unknown User ID: {0}').format(user_id)) return if not has_course_author_access(user, courselike_key): with respect_language(language): self.status.fail(_(u'Permission denied')) return is_library = isinstance(courselike_key, LibraryLocator) is_course = not is_library if is_library: root_name = LIBRARY_ROOT courselike_module = modulestore().get_library(courselike_key) import_func = import_library_from_xml else: root_name = COURSE_ROOT courselike_module = modulestore().get_course(courselike_key) import_func = import_course_from_xml # Locate the uploaded OLX archive (and download it from S3 if necessary) # Do everything in a try-except block to make sure everything is properly cleaned up. data_root = path(settings.GITHUB_REPO_ROOT) subdir = base64.urlsafe_b64encode(repr(courselike_key)) course_dir = data_root / subdir try: self.status.set_state(u'Unpacking') if not archive_name.endswith(u'.tar.gz'): with respect_language(language): self.status.fail( _(u'We only support uploading a .tar.gz file.')) return temp_filepath = course_dir / get_valid_filename(archive_name) if not course_dir.isdir(): # pylint: disable=no-value-for-parameter os.mkdir(course_dir) LOGGER.debug(u'importing course to {0}'.format(temp_filepath)) # Copy the OLX archive from where it was uploaded to (S3, Swift, file system, etc.) if not course_import_export_storage.exists(archive_path): LOGGER.info(u'Course import %s: Uploaded file %s not found', courselike_key, archive_path) with respect_language(language): self.status.fail(_(u'Tar file not found')) return with course_import_export_storage.open(archive_path, 'rb') as source: with open(temp_filepath, 'wb') as destination: def read_chunk(): """ Read and return a sequence of bytes from the source file. """ return source.read(FILE_READ_CHUNK) for chunk in iter(read_chunk, b''): destination.write(chunk) LOGGER.info(u'Course import %s: Download from storage complete', courselike_key) # Delete from source location course_import_export_storage.delete(archive_path) # If the course has an entrance exam then remove it and its corresponding milestone. # current course state before import. if is_course: if courselike_module.entrance_exam_enabled: fake_request = RequestFactory().get(u'/') fake_request.user = user from contentstore.views.entrance_exam import remove_entrance_exam_milestone_reference # TODO: Is this really ok? Seems dangerous for a live course remove_entrance_exam_milestone_reference( fake_request, courselike_key) LOGGER.info( u'entrance exam milestone content reference for course %s has been removed', courselike_module.id) # Send errors to client with stage at which error occurred. except Exception as exception: # pylint: disable=broad-except if course_dir.isdir(): # pylint: disable=no-value-for-parameter shutil.rmtree(course_dir) LOGGER.info(u'Course import %s: Temp data cleared', courselike_key) LOGGER.exception(u'Error importing course %s', courselike_key, exc_info=True) self.status.fail(text_type(exception)) return # try-finally block for proper clean up after receiving file. try: tar_file = tarfile.open(temp_filepath) try: safetar_extractall(tar_file, (course_dir + u'/').encode(u'utf-8')) except SuspiciousOperation as exc: LOGGER.info(u'Course import %s: Unsafe tar file - %s', courselike_key, exc.args[0]) with respect_language(language): self.status.fail(_(u'Unsafe tar file. Aborting import.')) return finally: tar_file.close() LOGGER.info(u'Course import %s: Uploaded file extracted', courselike_key) self.status.set_state(u'Verifying') self.status.increment_completed_steps() # find the 'course.xml' file def get_all_files(directory): """ For each file in the directory, yield a 2-tuple of (file-name, directory-path) """ for directory_path, _dirnames, filenames in os.walk(directory): for filename in filenames: yield (filename, directory_path) def get_dir_for_filename(directory, filename): """ Returns the directory path for the first file found in the directory with the given name. If there is no file in the directory with the specified name, return None. """ for name, directory_path in get_all_files(directory): if name == filename: return directory_path return None dirpath = get_dir_for_filename(course_dir, root_name) if not dirpath: with respect_language(language): self.status.fail( _(u'Could not find the {0} file in the package.').format( root_name)) return dirpath = os.path.relpath(dirpath, data_root) LOGGER.debug(u'found %s at %s', root_name, dirpath) LOGGER.info(u'Course import %s: Extracted file verified', courselike_key) self.status.set_state(u'Updating') self.status.increment_completed_steps() with dog_stats_api.timer( u'courselike_import.time', tags=[u"courselike:{}".format(courselike_key)]): courselike_items = import_func(modulestore(), user.id, settings.GITHUB_REPO_ROOT, [dirpath], load_error_modules=False, static_content_store=contentstore(), target_id=courselike_key) new_location = courselike_items[0].location LOGGER.debug(u'new course at %s', new_location) LOGGER.info(u'Course import %s: Course import successful', courselike_key) except Exception as exception: # pylint: disable=broad-except LOGGER.exception(u'error importing course', exc_info=True) self.status.fail(text_type(exception)) finally: if course_dir.isdir(): # pylint: disable=no-value-for-parameter shutil.rmtree(course_dir) LOGGER.info(u'Course import %s: Temp data cleared', courselike_key) if self.status.state == u'Updating' and is_course: # Reload the course so we have the latest state course = modulestore().get_course(courselike_key) if course.entrance_exam_enabled: entrance_exam_chapter = modulestore().get_items( course.id, qualifiers={u'category': u'chapter'}, settings={u'is_entrance_exam': True})[0] metadata = { u'entrance_exam_id': text_type(entrance_exam_chapter.location) } CourseMetadata.update_from_dict(metadata, course, user) from contentstore.views.entrance_exam import add_entrance_exam_milestone add_entrance_exam_milestone(course.id, entrance_exam_chapter) LOGGER.info(u'Course %s Entrance exam imported', course.id)
def _import_handler(request, courselike_key, root_name, successful_url, context_name, courselike_module, import_func): """ Parameterized function containing the meat of import_handler. """ if not has_course_author_access(request.user, courselike_key): raise PermissionDenied() if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'): if request.method == 'GET': raise NotImplementedError('coming soon') else: # Do everything in a try-except block to make sure everything is properly cleaned up. try: data_root = path(settings.GITHUB_REPO_ROOT) subdir = base64.urlsafe_b64encode(repr(courselike_key)) course_dir = data_root / subdir filename = request.FILES['course-data'].name # Use sessions to keep info about import progress session_status = request.session.setdefault( "import_status", {}) courselike_string = unicode(courselike_key) + filename _save_request_status(request, courselike_string, 0) # If the course has an entrance exam then remove it and its corresponding milestone. # current course state before import. if root_name == COURSE_ROOT: if courselike_module.entrance_exam_enabled: remove_entrance_exam_milestone_reference( request, courselike_key) log.info( "entrance exam milestone content reference for course %s has been removed", courselike_module.id) if not filename.endswith('.tar.gz'): _save_request_status(request, courselike_string, -1) return JsonResponse( { 'ErrMsg': _('We only support uploading a .tar.gz file.'), 'Stage': -1 }, status=415) temp_filepath = course_dir / filename if not course_dir.isdir(): os.mkdir(course_dir) logging.debug('importing course to {0}'.format(temp_filepath)) # Get upload chunks byte ranges try: matches = CONTENT_RE.search( request.META["HTTP_CONTENT_RANGE"]) content_range = matches.groupdict() except KeyError: # Single chunk # no Content-Range header, so make one that will work content_range = {'start': 0, 'stop': 1, 'end': 2} # stream out the uploaded files in chunks to disk if int(content_range['start']) == 0: mode = "wb+" else: mode = "ab+" size = os.path.getsize(temp_filepath) # Check to make sure we haven't missed a chunk # This shouldn't happen, even if different instances are handling # the same session, but it's always better to catch errors earlier. if size < int(content_range['start']): _save_request_status(request, courselike_string, -1) log.warning( "Reported range %s does not match size downloaded so far %s", content_range['start'], size) return JsonResponse( { 'ErrMsg': _('File upload corrupted. Please try again'), 'Stage': -1 }, status=409) # The last request sometimes comes twice. This happens because # nginx sends a 499 error code when the response takes too long. elif size > int(content_range['stop']) and size == int( content_range['end']): return JsonResponse({'ImportStatus': 1}) with open(temp_filepath, mode) as temp_file: for chunk in request.FILES['course-data'].chunks(): temp_file.write(chunk) size = os.path.getsize(temp_filepath) if int(content_range['stop']) != int(content_range['end']) - 1: # More chunks coming return JsonResponse({ "files": [{ "name": filename, "size": size, "deleteUrl": "", "deleteType": "", "url": reverse_course_url('import_handler', courselike_key), "thumbnailUrl": "" }] }) # Send errors to client with stage at which error occurred. except Exception as exception: # pylint: disable=broad-except _save_request_status(request, courselike_string, -1) if course_dir.isdir(): shutil.rmtree(course_dir) log.info("Course import %s: Temp data cleared", courselike_key) log.exception("error importing course") return JsonResponse({ 'ErrMsg': str(exception), 'Stage': -1 }, status=400) # try-finally block for proper clean up after receiving last chunk. try: # This was the last chunk. log.info("Course import %s: Upload complete", courselike_key) _save_request_status(request, courselike_string, 1) tar_file = tarfile.open(temp_filepath) try: safetar_extractall(tar_file, (course_dir + '/').encode('utf-8')) except SuspiciousOperation as exc: _save_request_status(request, courselike_string, -1) return JsonResponse( { 'ErrMsg': 'Unsafe tar file. Aborting import.', 'SuspiciousFileOperationMsg': exc.args[0], 'Stage': -1 }, status=400) finally: tar_file.close() log.info("Course import %s: Uploaded file extracted", courselike_key) _save_request_status(request, courselike_string, 2) # find the 'course.xml' file def get_all_files(directory): """ For each file in the directory, yield a 2-tuple of (file-name, directory-path) """ for dirpath, _dirnames, filenames in os.walk(directory): for filename in filenames: yield (filename, dirpath) def get_dir_for_fname(directory, filename): """ Returns the dirpath for the first file found in the directory with the given name. If there is no file in the directory with the specified name, return None. """ for fname, dirpath in get_all_files(directory): if fname == filename: return dirpath return None dirpath = get_dir_for_fname(course_dir, root_name) if not dirpath: _save_request_status(request, courselike_string, -2) return JsonResponse( { 'ErrMsg': _('Could not find the {0} file in the package.'). format(root_name), 'Stage': -2 }, status=415) dirpath = os.path.relpath(dirpath, data_root) logging.debug('found %s at %s', root_name, dirpath) log.info("Course import %s: Extracted file verified", courselike_key) _save_request_status(request, courselike_string, 3) with dog_stats_api.timer( 'courselike_import.time', tags=[u"courselike:{}".format(courselike_key)]): courselike_items = import_func( modulestore(), request.user.id, settings.GITHUB_REPO_ROOT, [dirpath], load_error_modules=False, static_content_store=contentstore(), target_id=courselike_key) new_location = courselike_items[0].location logging.debug('new course at %s', new_location) log.info("Course import %s: Course import successful", courselike_key) _save_request_status(request, courselike_string, 4) # Send errors to client with stage at which error occurred. except Exception as exception: # pylint: disable=broad-except log.exception("error importing course") return JsonResponse( { 'ErrMsg': str(exception), 'Stage': -session_status[courselike_string] }, status=400) finally: if course_dir.isdir(): shutil.rmtree(course_dir) log.info("Course import %s: Temp data cleared", courselike_key) # set failed stage number with negative sign in case of unsuccessful import if session_status[courselike_string] != 4: _save_request_status( request, courselike_string, -abs(session_status[courselike_string])) # status == 4 represents that course has been imported successfully. if session_status[ courselike_string] == 4 and root_name == COURSE_ROOT: # Reload the course so we have the latest state course = modulestore().get_course(courselike_key) if course.entrance_exam_enabled: entrance_exam_chapter = modulestore().get_items( course.id, qualifiers={'category': 'chapter'}, settings={'is_entrance_exam': True})[0] metadata = { 'entrance_exam_id': unicode(entrance_exam_chapter.location) } CourseMetadata.update_from_dict( metadata, course, request.user) add_entrance_exam_milestone(course.id, entrance_exam_chapter) log.info("Course %s Entrance exam imported", course.id) return JsonResponse({'Status': 'OK'}) elif request.method == 'GET': # assume html status_url = reverse_course_url("import_status_handler", courselike_key, kwargs={'filename': "fillerName"}) return render_to_response( 'import.html', { context_name: courselike_module, 'successful_import_redirect_url': successful_url, 'import_status_url': status_url, 'library': isinstance(courselike_key, LibraryLocator) }) else: return HttpResponseNotFound()
def import_olx(self, user_id, course_key_string, archive_path, archive_name, language): """ Import a course or library from a provided OLX .tar.gz archive. """ courselike_key = CourseKey.from_string(course_key_string) try: user = User.objects.get(pk=user_id) except User.DoesNotExist: with respect_language(language): self.status.fail(_(u'Unknown User ID: {0}').format(user_id)) return if not has_course_author_access(user, courselike_key): with respect_language(language): self.status.fail(_(u'Permission denied')) return is_library = isinstance(courselike_key, LibraryLocator) is_course = not is_library if is_library: root_name = LIBRARY_ROOT courselike_module = modulestore().get_library(courselike_key) import_func = import_library_from_xml else: root_name = COURSE_ROOT courselike_module = modulestore().get_course(courselike_key) import_func = import_course_from_xml # Locate the uploaded OLX archive (and download it from S3 if necessary) # Do everything in a try-except block to make sure everything is properly cleaned up. data_root = path(settings.GITHUB_REPO_ROOT) subdir = base64.urlsafe_b64encode(repr(courselike_key)) course_dir = data_root / subdir try: self.status.set_state(u'Unpacking') if not archive_name.endswith(u'.tar.gz'): with respect_language(language): self.status.fail(_(u'We only support uploading a .tar.gz file.')) return temp_filepath = course_dir / get_valid_filename(archive_name) if not course_dir.isdir(): # pylint: disable=no-value-for-parameter os.mkdir(course_dir) LOGGER.debug(u'importing course to {0}'.format(temp_filepath)) # Copy the OLX archive from where it was uploaded to (S3, Swift, file system, etc.) if not course_import_export_storage.exists(archive_path): LOGGER.info(u'Course import %s: Uploaded file %s not found', courselike_key, archive_path) with respect_language(language): self.status.fail(_(u'Tar file not found')) return with course_import_export_storage.open(archive_path, 'rb') as source: with open(temp_filepath, 'wb') as destination: def read_chunk(): """ Read and return a sequence of bytes from the source file. """ return source.read(FILE_READ_CHUNK) for chunk in iter(read_chunk, b''): destination.write(chunk) LOGGER.info(u'Course import %s: Download from storage complete', courselike_key) # Delete from source location course_import_export_storage.delete(archive_path) # If the course has an entrance exam then remove it and its corresponding milestone. # current course state before import. if is_course: if courselike_module.entrance_exam_enabled: fake_request = RequestFactory().get(u'/') fake_request.user = user from contentstore.views.entrance_exam import remove_entrance_exam_milestone_reference # TODO: Is this really ok? Seems dangerous for a live course remove_entrance_exam_milestone_reference(fake_request, courselike_key) LOGGER.info( u'entrance exam milestone content reference for course %s has been removed', courselike_module.id ) # Send errors to client with stage at which error occurred. except Exception as exception: # pylint: disable=broad-except if course_dir.isdir(): # pylint: disable=no-value-for-parameter shutil.rmtree(course_dir) LOGGER.info(u'Course import %s: Temp data cleared', courselike_key) LOGGER.exception(u'Error importing course %s', courselike_key, exc_info=True) self.status.fail(text_type(exception)) return # try-finally block for proper clean up after receiving file. try: tar_file = tarfile.open(temp_filepath) try: safetar_extractall(tar_file, (course_dir + u'/').encode(u'utf-8')) except SuspiciousOperation as exc: LOGGER.info(u'Course import %s: Unsafe tar file - %s', courselike_key, exc.args[0]) with respect_language(language): self.status.fail(_(u'Unsafe tar file. Aborting import.')) return finally: tar_file.close() LOGGER.info(u'Course import %s: Uploaded file extracted', courselike_key) self.status.set_state(u'Verifying') self.status.increment_completed_steps() # find the 'course.xml' file def get_all_files(directory): """ For each file in the directory, yield a 2-tuple of (file-name, directory-path) """ for directory_path, _dirnames, filenames in os.walk(directory): for filename in filenames: yield (filename, directory_path) def get_dir_for_filename(directory, filename): """ Returns the directory path for the first file found in the directory with the given name. If there is no file in the directory with the specified name, return None. """ for name, directory_path in get_all_files(directory): if name == filename: return directory_path return None dirpath = get_dir_for_filename(course_dir, root_name) if not dirpath: with respect_language(language): self.status.fail(_(u'Could not find the {0} file in the package.').format(root_name)) return dirpath = os.path.relpath(dirpath, data_root) LOGGER.debug(u'found %s at %s', root_name, dirpath) LOGGER.info(u'Course import %s: Extracted file verified', courselike_key) self.status.set_state(u'Updating') self.status.increment_completed_steps() with dog_stats_api.timer( u'courselike_import.time', tags=[u"courselike:{}".format(courselike_key)] ): courselike_items = import_func( modulestore(), user.id, settings.GITHUB_REPO_ROOT, [dirpath], load_error_modules=False, static_content_store=contentstore(), target_id=courselike_key ) new_location = courselike_items[0].location LOGGER.debug(u'new course at %s', new_location) LOGGER.info(u'Course import %s: Course import successful', courselike_key) except Exception as exception: # pylint: disable=broad-except LOGGER.exception(u'error importing course', exc_info=True) self.status.fail(text_type(exception)) finally: if course_dir.isdir(): # pylint: disable=no-value-for-parameter shutil.rmtree(course_dir) LOGGER.info(u'Course import %s: Temp data cleared', courselike_key) if self.status.state == u'Updating' and is_course: # Reload the course so we have the latest state course = modulestore().get_course(courselike_key) if course.entrance_exam_enabled: entrance_exam_chapter = modulestore().get_items( course.id, qualifiers={u'category': u'chapter'}, settings={u'is_entrance_exam': True} )[0] metadata = {u'entrance_exam_id': text_type(entrance_exam_chapter.location)} CourseMetadata.update_from_dict(metadata, course, user) from contentstore.views.entrance_exam import add_entrance_exam_milestone add_entrance_exam_milestone(course.id, entrance_exam_chapter) LOGGER.info(u'Course %s Entrance exam imported', course.id)