def _import_course_draft( xml_module_store, store, user_id, course_data_path, source_course_id, target_id, mongo_runtime ): """ This method will import all the content inside of the 'drafts' folder, if content exists. NOTE: This is not a full course import! In our current application, only verticals (and blocks beneath) can be in draft. Therefore, different call points into the import process_xml are used as the XMLModuleStore() constructor cannot simply be called (as is done for importing public content). """ draft_dir = course_data_path + "/drafts" if not os.path.exists(draft_dir): return # create a new 'System' object which will manage the importing errorlog = make_error_tracker() # The course_dir as passed to ImportSystem is expected to just be relative, not # the complete path including data_dir. ImportSystem will concatenate the two together. data_dir = xml_module_store.data_dir # Whether or not data_dir ends with a "/" differs in production vs. test. if not data_dir.endswith("/"): data_dir += "/" # Remove absolute path, leaving relative <course_name>/drafts. draft_course_dir = draft_dir.replace(data_dir, '', 1) system = ImportSystem( xmlstore=xml_module_store, course_id=source_course_id, course_dir=draft_course_dir, error_tracker=errorlog.tracker, load_error_modules=False, mixins=xml_module_store.xblock_mixins, field_data=KvsFieldData(kvs=DictKeyValueStore()), target_course_id=target_id, ) def _import_module(module): # IMPORTANT: Be sure to update the module location in the NEW namespace module_location = module.location.map_into_course(target_id) # Update the module's location to DRAFT revision # We need to call this method (instead of updating the location directly) # to ensure that pure XBlock field data is updated correctly. _update_module_location(module, module_location.replace(revision=MongoRevisionKey.draft)) parent_url = get_parent_url(module) index = index_in_children_list(module) # make sure our parent has us in its list of children # this is to make sure private only modules show up # in the list of children since they would have been # filtered out from the non-draft store export. if parent_url is not None and index is not None: course_key = descriptor.location.course_key parent_location = course_key.make_usage_key_from_deprecated_string(parent_url) # IMPORTANT: Be sure to update the parent in the NEW namespace parent_location = parent_location.map_into_course(target_id) parent = store.get_item(parent_location, depth=0) non_draft_location = module.location.map_into_course(target_id) if not any(child.block_id == module.location.block_id for child in parent.children): parent.children.insert(index, non_draft_location) store.update_item(parent, user_id) _update_and_import_module( module, store, user_id, source_course_id, target_id, runtime=mongo_runtime, ) for child in module.get_children(): _import_module(child) # Now walk the /drafts directory. # Each file in the directory will be a draft copy of the vertical. # First it is necessary to order the draft items by their desired index in the child list, # since the order in which os.walk() returns the files is not guaranteed. drafts = [] for rootdir, __, filenames in os.walk(draft_dir): for filename in filenames: if filename.startswith('._'): # Skip any OSX quarantine files, prefixed with a '._'. continue module_path = os.path.join(rootdir, filename) with open(module_path, 'r') as f: try: xml = f.read().decode('utf-8') # The process_xml() call below recursively processes all descendants. If # we call this on all verticals in a course with verticals nested below # the unit level, we try to import the same content twice, causing naming conflicts. # Therefore only process verticals at the unit level, assuming that any other # verticals must be descendants. if 'index_in_children_list' in xml: descriptor = system.process_xml(xml) # HACK: since we are doing partial imports of drafts # the vertical doesn't have the 'url-name' set in the # attributes (they are normally in the parent object, # aka sequential), so we have to replace the location.name # with the XML filename that is part of the pack filename, __ = os.path.splitext(filename) descriptor.location = descriptor.location.replace(name=filename) index = index_in_children_list(descriptor) parent_url = get_parent_url(descriptor, xml) draft_url = unicode(descriptor.location) draft = draft_node_constructor( module=descriptor, url=draft_url, parent_url=parent_url, index=index ) drafts.append(draft) except Exception: # pylint: disable=broad-except logging.exception('Error while parsing course drafts xml.') # Sort drafts by `index_in_children_list` attribute. drafts.sort(key=lambda x: x.index) for draft in get_draft_subtree_roots(drafts): try: _import_module(draft.module) except Exception: # pylint: disable=broad-except logging.exception('while importing draft descriptor %s', draft.module)
def _import_course_draft(xml_module_store, store, user_id, course_data_path, source_course_id, target_id, mongo_runtime): ''' This will import all the content inside of the 'drafts' folder, if it exists NOTE: This is not a full course import, basically in our current application only verticals (and downwards) can be in draft. Therefore, we need to use slightly different call points into the import process_xml as we can't simply call XMLModuleStore() constructor (like we do for importing public content) ''' draft_dir = course_data_path + "/drafts" if not os.path.exists(draft_dir): return # create a new 'System' object which will manage the importing errorlog = make_error_tracker() # The course_dir as passed to ImportSystem is expected to just be relative, not # the complete path including data_dir. ImportSystem will concatenate the two together. data_dir = xml_module_store.data_dir # Whether or not data_dir ends with a "/" differs in production vs. test. if not data_dir.endswith("/"): data_dir += "/" draft_course_dir = draft_dir.replace(data_dir, '', 1) system = ImportSystem( xmlstore=xml_module_store, course_id=source_course_id, course_dir=draft_course_dir, error_tracker=errorlog.tracker, load_error_modules=False, mixins=xml_module_store.xblock_mixins, field_data=KvsFieldData(kvs=DictKeyValueStore()), ) def _import_module(module): # IMPORTANT: Be sure to update the module location in the NEW namespace module_location = module.location.map_into_course(target_id) # Update the module's location to DRAFT revision # We need to call this method (instead of updating the location directly) # to ensure that pure XBlock field data is updated correctly. _update_module_location( module, module_location.replace(revision=MongoRevisionKey.draft)) parent_url = get_parent_url(module) index = index_in_children_list(module) # make sure our parent has us in its list of children # this is to make sure private only modules show up # in the list of children since they would have been # filtered out from the non-draft store export. if parent_url is not None and index is not None: course_key = descriptor.location.course_key parent_location = course_key.make_usage_key_from_deprecated_string( parent_url) # IMPORTANT: Be sure to update the parent in the NEW namespace parent_location = parent_location.map_into_course(target_id) parent = store.get_item(parent_location, depth=0) non_draft_location = module.location.map_into_course(target_id) if not any(child.block_id == module.location.block_id for child in parent.children): parent.children.insert(index, non_draft_location) store.update_item(parent, user_id) _import_module_and_update_references( module, store, user_id, source_course_id, target_id, runtime=mongo_runtime, ) for child in module.get_children(): _import_module(child) # Now walk the /vertical directory. # Each file in the directory will be a draft copy of the vertical. # First it is necessary to order the draft items by their desired index in the child list, # since the order in which os.walk() returns the files is not guaranteed. drafts = [] for dirname, _dirnames, filenames in os.walk(draft_dir): for filename in filenames: module_path = os.path.join(dirname, filename) with open(module_path, 'r') as f: try: # note, on local dev it seems like OSX will put # some extra files in the directory with "quarantine" # information. These files are binary files and will # throw exceptions when we try to parse the file # as an XML string. Let's make sure we're # dealing with a string before ingesting data = f.read() try: xml = data.decode('utf-8') except UnicodeDecodeError, err: # seems like on OSX localdev, the OS is making # quarantine files in the unzip directory # when importing courses so if we blindly try to # enumerate through the directory, we'll try # to process a bunch of binary quarantine files # (which are prefixed with a '._' character which # will dump a bunch of exceptions to the output, # although they are harmless. # # Reading online docs there doesn't seem to be # a good means to detect a 'hidden' file that works # well across all OS environments. So for now, I'm using # OSX's utilization of a leading '.' in the filename # to indicate a system hidden file. # # Better yet would be a way to figure out if this is # a binary file, but I haven't found a good way # to do this yet. if filename.startswith('._'): continue # Not a 'hidden file', then re-raise exception raise err # process_xml call below recursively processes all descendants. If # we call this on all verticals in a course with verticals nested below # the unit level, we try to import the same content twice, causing naming conflicts. # Therefore only process verticals at the unit level, assuming that any other # verticals must be descendants. if 'index_in_children_list' in xml: descriptor = system.process_xml(xml) # HACK: since we are doing partial imports of drafts # the vertical doesn't have the 'url-name' set in the # attributes (they are normally in the parent object, # aka sequential), so we have to replace the location.name # with the XML filename that is part of the pack filename, __ = os.path.splitext(filename) descriptor.location = descriptor.location.replace( name=filename) index = index_in_children_list(descriptor) parent_url = get_parent_url(descriptor, xml) draft_url = unicode(descriptor.location) draft = draft_node_constructor(module=descriptor, url=draft_url, parent_url=parent_url, index=index) drafts.append(draft) except Exception: # pylint: disable=broad-except logging.exception('Error while parsing course xml.')
def _import_course_draft( xml_module_store, store, user_id, course_data_path, source_course_id, target_id, mongo_runtime ): ''' This will import all the content inside of the 'drafts' folder, if it exists NOTE: This is not a full course import, basically in our current application only verticals (and downwards) can be in draft. Therefore, we need to use slightly different call points into the import process_xml as we can't simply call XMLModuleStore() constructor (like we do for importing public content) ''' draft_dir = course_data_path + "/drafts" if not os.path.exists(draft_dir): return # create a new 'System' object which will manage the importing errorlog = make_error_tracker() # The course_dir as passed to ImportSystem is expected to just be relative, not # the complete path including data_dir. ImportSystem will concatenate the two together. data_dir = xml_module_store.data_dir # Whether or not data_dir ends with a "/" differs in production vs. test. if not data_dir.endswith("/"): data_dir += "/" draft_course_dir = draft_dir.replace(data_dir, '', 1) system = ImportSystem( xmlstore=xml_module_store, course_id=source_course_id, course_dir=draft_course_dir, error_tracker=errorlog.tracker, load_error_modules=False, mixins=xml_module_store.xblock_mixins, field_data=KvsFieldData(kvs=DictKeyValueStore()), ) def _import_module(module): # IMPORTANT: Be sure to update the module location in the NEW namespace module_location = module.location.map_into_course(target_id) # Update the module's location to DRAFT revision # We need to call this method (instead of updating the location directly) # to ensure that pure XBlock field data is updated correctly. _update_module_location(module, module_location.replace(revision=MongoRevisionKey.draft)) parent_url = get_parent_url(module) index = index_in_children_list(module) # make sure our parent has us in its list of children # this is to make sure private only modules show up # in the list of children since they would have been # filtered out from the non-draft store export. if parent_url is not None and index is not None: course_key = descriptor.location.course_key parent_location = course_key.make_usage_key_from_deprecated_string(parent_url) # IMPORTANT: Be sure to update the parent in the NEW namespace parent_location = parent_location.map_into_course(target_id) parent = store.get_item(parent_location, depth=0) non_draft_location = module.location.map_into_course(target_id) if not any(child.block_id == module.location.block_id for child in parent.children): parent.children.insert(index, non_draft_location) store.update_item(parent, user_id) _import_module_and_update_references( module, store, user_id, source_course_id, target_id, runtime=mongo_runtime, ) for child in module.get_children(): _import_module(child) # Now walk the /vertical directory. # Each file in the directory will be a draft copy of the vertical. # First it is necessary to order the draft items by their desired index in the child list, # since the order in which os.walk() returns the files is not guaranteed. drafts = [] for dirname, _dirnames, filenames in os.walk(draft_dir): for filename in filenames: module_path = os.path.join(dirname, filename) with open(module_path, 'r') as f: try: # note, on local dev it seems like OSX will put # some extra files in the directory with "quarantine" # information. These files are binary files and will # throw exceptions when we try to parse the file # as an XML string. Let's make sure we're # dealing with a string before ingesting data = f.read() try: xml = data.decode('utf-8') except UnicodeDecodeError, err: # seems like on OSX localdev, the OS is making # quarantine files in the unzip directory # when importing courses so if we blindly try to # enumerate through the directory, we'll try # to process a bunch of binary quarantine files # (which are prefixed with a '._' character which # will dump a bunch of exceptions to the output, # although they are harmless. # # Reading online docs there doesn't seem to be # a good means to detect a 'hidden' file that works # well across all OS environments. So for now, I'm using # OSX's utilization of a leading '.' in the filename # to indicate a system hidden file. # # Better yet would be a way to figure out if this is # a binary file, but I haven't found a good way # to do this yet. if filename.startswith('._'): continue # Not a 'hidden file', then re-raise exception raise err # process_xml call below recursively processes all descendants. If # we call this on all verticals in a course with verticals nested below # the unit level, we try to import the same content twice, causing naming conflicts. # Therefore only process verticals at the unit level, assuming that any other # verticals must be descendants. if 'index_in_children_list' in xml: descriptor = system.process_xml(xml) # HACK: since we are doing partial imports of drafts # the vertical doesn't have the 'url-name' set in the # attributes (they are normally in the parent object, # aka sequential), so we have to replace the location.name # with the XML filename that is part of the pack filename, __ = os.path.splitext(filename) descriptor.location = descriptor.location.replace(name=filename) index = index_in_children_list(descriptor) parent_url = get_parent_url(descriptor, xml) draft_url = unicode(descriptor.location) draft = draft_node_constructor( module=descriptor, url=draft_url, parent_url=parent_url, index=index ) drafts.append(draft) except Exception: # pylint: disable=broad-except logging.exception('Error while parsing course xml.')