def monitor_import_failure(course_key,
import_step,
message=None,
exception=None):
"""
Helper method to add custom parameters to for import failures.
Arguments:
course_key: CourseKey object
import_step (str): current step in course import
message (str): any particular message to add
exception: Exception object
"""
set_custom_attribute('course_import_failure', import_step)
set_custom_attributes_for_course_key(course_key)
if message:
set_custom_attribute('course_import_failure_message', message)
if exception is not None:
exception_module = getattr(exception, '__module__', '')
separator = '.' if exception_module else ''
module_and_class = f'{exception_module}{separator}{exception.__class__.__name__}'
exc_message = str(exception)
set_custom_attribute('course_import_failure_error_class',
module_and_class)
set_custom_attribute('course_import_failure_error_message',
exc_message)
def post(self, request, course_key):
"""
Kicks off an asynchronous course import and returns an ID to be used to check
the task's status
"""
set_custom_attribute('course_import_init', True)
set_custom_attributes_for_course_key(course_key)
try:
if 'course_data' not in request.FILES:
raise self.api_error(
status_code=status.HTTP_400_BAD_REQUEST,
developer_message='Missing required parameter',
error_code='internal_error',
)
filename = request.FILES['course_data'].name
if not filename.endswith('.tar.gz'):
raise self.api_error(
status_code=status.HTTP_400_BAD_REQUEST,
developer_message='Parameter in the wrong format',
error_code='internal_error',
)
course_dir = path(
settings.GITHUB_REPO_ROOT) / base64.urlsafe_b64encode(
repr(course_key).encode('utf-8')).decode('utf-8')
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
log.debug(f'importing course to {temp_filepath}')
with open(temp_filepath, "wb+") as temp_file:
for chunk in request.FILES['course_data'].chunks():
temp_file.write(chunk)
log.info("Course import %s: Upload complete", course_key)
with open(temp_filepath, 'rb') as local_file:
django_file = File(local_file)
storage_path = course_import_export_storage.save(
'olx_import/' + filename, django_file)
async_result = import_olx.delay(request.user.id, str(course_key),
storage_path, filename,
request.LANGUAGE_CODE)
return Response({'task_id': async_result.task_id})
except Exception as e:
log.exception(
f'Course import {course_key}: Unknown error in import')
raise self.api_error(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
developer_message=str(e),
error_code='internal_error')
def _recalculate_subsection_grade(self, **kwargs):
"""
Updates a saved subsection grade.
Keyword Arguments:
user_id (int): id of applicable User object
anonymous_user_id (int, OPTIONAL): Anonymous ID of the User
course_id (string): identifying the course
usage_id (string): identifying the course block
only_if_higher (boolean): indicating whether grades should
be updated only if the new raw_earned is higher than the
previous value.
expected_modified_time (serialized timestamp): indicates when the task
was queued so that we can verify the underlying data update.
score_deleted (boolean): indicating whether the grade change is
a result of the problem's score being deleted.
event_transaction_id (string): uuid identifying the current
event transaction.
event_transaction_type (string): human-readable type of the
event at the root of the current event transaction.
score_db_table (ScoreDatabaseTableEnum): database table that houses
the changed score. Used in conjunction with expected_modified_time.
"""
try:
course_key = CourseLocator.from_string(kwargs['course_id'])
if are_grades_frozen(course_key):
log.info(
"Attempted _recalculate_subsection_grade for course '%s', but grades are frozen.",
course_key)
return
scored_block_usage_key = UsageKey.from_string(
kwargs['usage_id']).replace(course_key=course_key)
set_custom_attributes_for_course_key(course_key)
set_custom_attribute('usage_id', str(scored_block_usage_key))
# The request cache is not maintained on celery workers,
# where this code runs. So we take the values from the
# main request cache and store them in the local request
# cache. This correlates model-level grading events with
# higher-level ones.
set_event_transaction_id(kwargs.get('event_transaction_id'))
set_event_transaction_type(kwargs.get('event_transaction_type'))
# Verify the database has been updated with the scores when the task was
# created. This race condition occurs if the transaction in the task
# creator's process hasn't committed before the task initiates in the worker
# process.
has_database_updated = _has_db_updated_with_new_score(
self, scored_block_usage_key, **kwargs)
if not has_database_updated:
raise DatabaseNotReadyError
_update_subsection_grades(
course_key,
scored_block_usage_key,
kwargs['only_if_higher'],
kwargs['user_id'],
kwargs['score_deleted'],
kwargs.get('force_update_subsections', False),
)
except Exception as exc:
if not isinstance(exc, KNOWN_RETRY_ERRORS):
log.info(
"tnl-6244 grades unexpected failure: {}. task id: {}. kwargs={}"
.format(
repr(exc),
self.request.id,
kwargs,
))
raise self.retry(kwargs=kwargs, exc=exc)
def get(self,
request,
course_id,
chapter=None,
section=None,
position=None):
"""
Displays courseware accordion and associated content. If course, chapter,
and section are all specified, renders the page, or returns an error if they
are invalid.
If section is not specified, displays the accordion opened to the right
chapter.
If neither chapter or section are specified, displays the user's most
recent chapter, or the first chapter if this is the user's first visit.
Arguments:
request: HTTP request
course_id (unicode): course id
chapter (unicode): chapter url_name
section (unicode): section url_name
position (unicode): position in module, eg of <sequential> module
"""
self.course_key = CourseKey.from_string(course_id)
if not (request.user.is_authenticated
or self.enable_unenrolled_access):
return redirect_to_login(request.get_full_path())
self.original_chapter_url_name = chapter
self.original_section_url_name = section
self.chapter_url_name = chapter
self.section_url_name = section
self.position = position
self.chapter, self.section = None, None
self.course = None
self.url = request.path
try:
set_custom_attributes_for_course_key(self.course_key)
self._clean_position()
with modulestore().bulk_operations(self.course_key):
self.view = STUDENT_VIEW
self.course = get_course_with_access(
request.user,
'load',
self.course_key,
depth=CONTENT_DEPTH,
check_if_enrolled=True,
check_if_authenticated=True)
self.course_overview = CourseOverview.get_from_id(
self.course.id)
self.is_staff = has_access(request.user, 'staff', self.course)
# There's only one situation where we want to show the public view
if (not self.is_staff and self.enable_unenrolled_access
and self.course.course_visibility
== COURSE_VISIBILITY_PUBLIC
and not CourseEnrollment.is_enrolled(
request.user, self.course_key)):
self.view = PUBLIC_VIEW
self.can_masquerade = request.user.has_perm(
MASQUERADE_AS_STUDENT, self.course)
self._setup_masquerade_for_effective_user()
return self.render(request)
except Exception as exception: # pylint: disable=broad-except
return CourseTabView.handle_exceptions(request, self.course_key,
self.course, exception)
def _write_chunk(request, courselike_key):
"""
Write the OLX file data chunk from the given request to the local filesystem.
"""
# Upload .tar.gz to local filesystem for one-server installations not using S3 or Swift
data_root = path(settings.GITHUB_REPO_ROOT)
subdir = base64.urlsafe_b64encode(repr(courselike_key).encode('utf-8')).decode('utf-8')
course_dir = data_root / subdir
filename = request.FILES['course-data'].name
set_custom_attributes_for_course_key(courselike_key)
current_step = 'Uploading'
def error_response(message, status, stage):
"""Returns Json error response"""
return JsonResponse({'ErrMsg': message, 'Stage': stage}, status=status)
courselike_string = str(courselike_key) + filename
# Do everything in a try-except block to make sure everything is properly cleaned up.
try:
# Use sessions to keep info about import progress
_save_request_status(request, courselike_string, 0)
if not filename.endswith('.tar.gz'):
error_message = _('We only support uploading a .tar.gz file.')
_save_request_status(request, courselike_string, -1)
monitor_import_failure(courselike_key, current_step, message=error_message)
return error_response(error_message, 415, 0)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.info(f'Course import {courselike_key}: importing course to {temp_filepath}')
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
logging.info(f'Course import {courselike_key}: single chunk found')
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
is_initial_import_request = int(content_range['start']) == 0
if is_initial_import_request:
mode = "wb+"
set_custom_attribute('course_import_init', True)
else:
mode = "ab+"
# Appending to fail would fail if the file doesn't exist.
if not temp_filepath.exists():
error_message = _('Some chunks missed during file upload. Please try again')
_save_request_status(request, courselike_string, -1)
log.error(f'Course Import {courselike_key}: {error_message}')
monitor_import_failure(courselike_key, current_step, message=error_message)
return error_response(error_message, 409, 0)
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
error_message = _('File upload corrupted. Please try again')
_save_request_status(request, courselike_string, -1)
log.error(f'Course import {courselike_key}: A chunk has been missed')
monitor_import_failure(courselike_key, current_step, message=error_message)
return error_response(error_message, 409, 0)
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": reverse_course_url('import_handler', courselike_key),
"thumbnailUrl": ""
}]
})
log.info(f'Course import {courselike_key}: Upload complete')
with open(temp_filepath, 'rb') as local_file:
django_file = File(local_file)
storage_path = course_import_export_storage.save('olx_import/' + filename, django_file)
import_olx.delay(
request.user.id, str(courselike_key), storage_path, filename, request.LANGUAGE_CODE)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=broad-except
_save_request_status(request, courselike_string, -1)
if course_dir.isdir():
shutil.rmtree(course_dir)
log.info("Course import %s: Temp data cleared", courselike_key)
monitor_import_failure(courselike_key, current_step, exception=exception)
log.exception(f'Course import {courselike_key}: error importing course.')
return error_response(str(exception), 400, -1)
return JsonResponse({'ImportStatus': 1})
def import_olx(self, user_id, course_key_string, archive_path, archive_name,
language):
"""
Import a course or library from a provided OLX .tar.gz archive.
"""
current_step = 'Unpacking'
courselike_key = CourseKey.from_string(course_key_string)
set_code_owner_attribute_from_module(__name__)
set_custom_attributes_for_course_key(courselike_key)
log_prefix = f'Course import {courselike_key}'
self.status.set_state(current_step)
data_root = path(settings.GITHUB_REPO_ROOT)
subdir = base64.urlsafe_b64encode(
repr(courselike_key).encode('utf-8')).decode('utf-8')
course_dir = data_root / subdir
def validate_user():
"""Validate if the user exists otherwise log error. """
try:
return User.objects.get(pk=user_id)
except User.DoesNotExist as exc:
with translation_language(language):
self.status.fail(UserErrors.USER_PERMISSION_DENIED)
LOGGER.error(f'{log_prefix}: Unknown User: {user_id}')
monitor_import_failure(courselike_key, current_step, exception=exc)
return
def user_has_access(user):
"""Return True if user has studio write access to the given course."""
has_access = has_course_author_access(user, courselike_key)
if not has_access:
message = f'User permission denied: {user.username}'
with translation_language(language):
self.status.fail(UserErrors.COURSE_PERMISSION_DENIED)
LOGGER.error(f'{log_prefix}: {message}')
monitor_import_failure(courselike_key,
current_step,
message=message)
return has_access
def file_is_supported():
"""Check if it is a supported file."""
file_is_valid = archive_name.endswith('.tar.gz')
if not file_is_valid:
message = f'Unsupported file {archive_name}'
with translation_language(language):
self.status.fail(UserErrors.INVALID_FILE_TYPE)
LOGGER.error(f'{log_prefix}: {message}')
monitor_import_failure(courselike_key,
current_step,
message=message)
return file_is_valid
def file_exists_in_storage():
"""Verify archive path exists in storage."""
archive_path_exists = course_import_export_storage.exists(archive_path)
if not archive_path_exists:
message = f'Uploaded file {archive_path} not found'
with translation_language(language):
self.status.fail(UserErrors.FILE_NOT_FOUND)
LOGGER.error(f'{log_prefix}: {message}')
monitor_import_failure(courselike_key,
current_step,
message=message)
return archive_path_exists
def verify_root_name_exists(course_dir, root_name):
"""Verify root xml file exists."""
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for directory_path, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, directory_path)
def get_dir_for_filename(directory, filename):
"""
Returns the directory path for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for name, directory_path in get_all_files(directory):
if name == filename:
return directory_path
return None
dirpath = get_dir_for_filename(course_dir, root_name)
if not dirpath:
message = UserErrors.FILE_MISSING.format(root_name)
with translation_language(language):
self.status.fail(message)
LOGGER.error(f'{log_prefix}: {message}')
monitor_import_failure(courselike_key,
current_step,
message=message)
return
return dirpath
user = validate_user()
if not user:
return
if not user_has_access(user):
return
if not file_is_supported():
return
is_library = isinstance(courselike_key, LibraryLocator)
is_course = not is_library
if is_library:
root_name = LIBRARY_ROOT
courselike_module = modulestore().get_library(courselike_key)
import_func = import_library_from_xml
else:
root_name = COURSE_ROOT
courselike_module = modulestore().get_course(courselike_key)
import_func = import_course_from_xml
# Locate the uploaded OLX archive (and download it from S3 if necessary)
# Do everything in a try-except block to make sure everything is properly cleaned up.
try:
LOGGER.info(f'{log_prefix}: unpacking step started')
temp_filepath = course_dir / get_valid_filename(archive_name)
if not course_dir.isdir():
os.mkdir(course_dir)
LOGGER.info(f'{log_prefix}: importing course to {temp_filepath}')
# Copy the OLX archive from where it was uploaded to (S3, Swift, file system, etc.)
if not file_exists_in_storage():
return
with course_import_export_storage.open(archive_path, 'rb') as source:
with open(temp_filepath, 'wb') as destination:
def read_chunk():
"""
Read and return a sequence of bytes from the source file.
"""
return source.read(FILE_READ_CHUNK)
for chunk in iter(read_chunk, b''):
destination.write(chunk)
LOGGER.info(f'{log_prefix}: Download from storage complete')
# Delete from source location
course_import_export_storage.delete(archive_path)
# If the course has an entrance exam then remove it and its corresponding milestone.
# current course state before import.
if is_course:
if courselike_module.entrance_exam_enabled:
fake_request = RequestFactory().get('/')
fake_request.user = user
from .views.entrance_exam import remove_entrance_exam_milestone_reference
# TODO: Is this really ok? Seems dangerous for a live course
remove_entrance_exam_milestone_reference(
fake_request, courselike_key)
LOGGER.info(
f'{log_prefix}: entrance exam milestone content reference has been removed'
)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=broad-except
if course_dir.isdir():
shutil.rmtree(course_dir)
LOGGER.info(f'{log_prefix}: Temp data cleared')
self.status.fail(UserErrors.UNKNOWN_ERROR_IN_UNPACKING)
LOGGER.exception(f'{log_prefix}: Unknown error while unpacking',
exc_info=True)
monitor_import_failure(courselike_key,
current_step,
exception=exception)
return
# try-finally block for proper clean up after receiving file.
try:
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + '/'))
except SuspiciousOperation as exc:
with translation_language(language):
self.status.fail(UserErrors.UNSAFE_TAR_FILE)
LOGGER.error(f'{log_prefix}: Unsafe tar file')
monitor_import_failure(courselike_key, current_step, exception=exc)
return
finally:
tar_file.close()
current_step = 'Verifying'
self.status.set_state(current_step)
self.status.increment_completed_steps()
LOGGER.info(
f'{log_prefix}: Uploaded file extracted. Verification step started'
)
dirpath = verify_root_name_exists(course_dir, root_name)
if not dirpath:
return
if not validate_course_olx(courselike_key, dirpath, self.status):
return
dirpath = os.path.relpath(dirpath, data_root)
current_step = 'Updating'
self.status.set_state(current_step)
self.status.increment_completed_steps()
LOGGER.info(
f'{log_prefix}: Extracted file verified. Updating course started')
courselike_items = import_func(
modulestore(),
user.id,
settings.GITHUB_REPO_ROOT,
[dirpath],
load_error_modules=False,
static_content_store=contentstore(),
target_id=courselike_key,
verbose=True,
)
new_location = courselike_items[0].location
LOGGER.debug('new course at %s', new_location)
LOGGER.info(f'{log_prefix}: Course import successful')
set_custom_attribute('course_import_completed', True)
except (CourseImportException, InvalidProctoringProvider,
DuplicateCourseError) as known_exe:
handle_course_import_exception(courselike_key, known_exe, self.status)
except Exception as exception: # pylint: disable=broad-except
handle_course_import_exception(courselike_key,
exception,
self.status,
known=False)
finally:
if course_dir.isdir():
shutil.rmtree(course_dir)
LOGGER.info(f'{log_prefix}: Temp data cleared')
if self.status.state == 'Updating' and is_course:
# Reload the course so we have the latest state
course = modulestore().get_course(courselike_key)
if course.entrance_exam_enabled:
entrance_exam_chapter = modulestore().get_items(
course.id,
qualifiers={'category': 'chapter'},
settings={'is_entrance_exam': True})[0]
metadata = {
'entrance_exam_id': str(entrance_exam_chapter.location)
}
CourseMetadata.update_from_dict(metadata, course, user)
from .views.entrance_exam import add_entrance_exam_milestone
add_entrance_exam_milestone(course.id, entrance_exam_chapter)
LOGGER.info(
f'Course import {course.id}: Entrance exam imported')
