def make_object(ctx, of_type: 'CdmObjectType', name_or_ref: str, simple_name_ref: bool) -> 'TObject': """instantiates a OM class based on the object type passed as first parameter.""" # Log and ingest a message when a new manifest is created if of_type == CdmObjectType.MANIFEST_DEF: logger.debug(ctx, None, make_object.__name__, None, 'New Manifest created.', True) return switcher[of_type](ctx, name_or_ref, simple_name_ref)
def _finish_indexing(self, loaded_imports: bool) -> None: """Marks that the document was indexed.""" logger.debug(self.ctx, self._TAG, self._finish_indexing.__name__, self.at_corpus_path, 'index finish: {}'.format(self.at_corpus_path)) was_indexed_previously = self._declarations_indexed self.ctx.corpus._document_library._mark_document_as_indexed(self) self._imports_indexed = self._imports_indexed or loaded_imports self._declarations_indexed = True self._needs_indexing = not loaded_imports self._internal_objects = None # if the document declarations were indexed previously, do not log again. if not was_indexed_previously and self._is_valid: for definition in self.definitions: if definition.object_type == CdmObjectType.ENTITY_DEF: logger.debug( self.ctx, self._TAG, self._finish_indexing.__name__, definition.at_corpus_path, 'indexed entity: {}'.format(definition.at_corpus_path))
async def create_resolved_manifest_async( self, new_manifest_name: str, new_entity_document_name_format: Optional[str], directives: Optional[AttributeResolutionDirectiveSet] = None ) -> Optional['CdmManifestDefinition']: """Creates a resolved copy of the manifest. new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities. The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any sub-folders described by the pattern should exist in the corpus prior to calling this function. """ with logger._enter_scope(self._TAG, self.ctx, self.create_resolved_manifest_async.__name__): if self.entities is None: return None if not self.folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_MANIFEST_FAILED, self.manifest_name) return None if new_entity_document_name_format is None: new_entity_document_name_format = '{f}resolved/{n}.cdm.json' elif new_entity_document_name_format == '': # for back compat new_entity_document_name_format = '{n}.cdm.json' elif '{n}' not in new_entity_document_name_format: # for back compat new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json' source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.at_corpus_path, self) source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.folder.at_corpus_path, self) resolved_manifest_path_split = new_manifest_name.rfind('/') + 1 resolved_manifest_folder = None if resolved_manifest_path_split > 0: resolved_manifest_path = new_manifest_name[ 0:resolved_manifest_path_split] new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( resolved_manifest_path, self) resolved_manifest_folder = await self.ctx.corpus.fetch_object_async( new_folder_path) # type: CdmFolderDefinition if resolved_manifest_folder is None: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND, new_folder_path) return None new_manifest_name = new_manifest_name[ resolved_manifest_path_split:] else: resolved_manifest_folder = self.owner logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'resolving manifest {}'.format(source_manifest_path)) # using the references present in the resolved entities, get an entity # create an imports doc with all the necessary resolved entity references and then resolve it # sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name if new_manifest_name.lower().endswith('.manifest.cdm.json'): new_manifest_name = new_manifest_name[0:( len(new_manifest_name) - len('.manifest.cdm.json'))] resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name) # bring over any imports in this document or other bobbles resolved_manifest.schema = self.schema resolved_manifest.explanation = self.explanation resolved_manifest.document_version = self.document_version for imp in self.imports: resolved_manifest.imports.append(imp.copy()) # add the new document to the folder if resolved_manifest_folder.documents.append( resolved_manifest) is None: # when would this happen? return None for entity in self.entities: ent_def = await self._get_entity_from_reference(entity, self) if not ent_def: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_ENTITY_REF_ERROR) return None if not ent_def.in_document.folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_DOC_IS_NOT_FOLDERformat, ent_def.entity_name) return None # get the path from this manifest to the source entity. this will be the {f} replacement value source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.in_document.folder.at_corpus_path, self) f = '' if source_entity_full_path.startswith( source_manifest_folder_path): f = source_entity_full_path[len(source_manifest_folder_path ):] new_document_full_path = new_entity_document_name_format.replace( '{n}', ent_def.entity_name).replace('{f}', f) new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( new_document_full_path, self) new_document_path_split = new_document_full_path.rfind('/') + 1 new_document_path = new_document_full_path[ 0:new_document_path_split] new_document_name = new_document_full_path[ new_document_path_split:] # make sure the new folder exists folder = await self.ctx.corpus.fetch_object_async( new_document_path) # type: CdmFolderDefinition if not folder: logger.error(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND, new_document_path) return None # next create the resolved entity. with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives res_opt = ResolveOptions(ent_def.in_document, with_directives.copy()) logger.debug( self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'resolving entity {} to document {}'.format( source_entity_full_path, new_document_full_path)) resolved_entity = await ent_def.create_resolved_entity_async( ent_def.entity_name, res_opt, folder, new_document_name) if not resolved_entity: # fail all resolution, if any one entity resolution fails return None result = entity.copy(res_opt) if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path( resolved_entity.at_corpus_path, resolved_manifest) result.entity_path = relative_entity_path or result.at_corpus_path resolved_manifest.entities.append(result) logger.debug(self.ctx, self._TAG, self.create_resolved_manifest_async.__name__, self.at_corpus_path, 'calculating relationships') # Calculate the entity graph for just this manifest. await self.ctx.corpus.calculate_entity_graph_async( resolved_manifest) # Stick results into the relationships list for the manifest. await resolved_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) # needed until Matt's changes with collections where I can propigate resolved_manifest._is_dirty = True return resolved_manifest
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder.folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter( folder.namespace) # type: StorageAdapter try: if adapter.can_read(): # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'request file: {}'.format(doc_path)) json_data = await adapter.read_async(doc_path) # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'received file: {}'.format(doc_path)) else: raise Exception('Storage Adapter is not enabled to read.') except Exception as e: # log message used by navigator, do not change or remove logger.debug(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, 'fail file: {}'.format(doc_path)) # when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if res_opt and res_opt.shallow_validation: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE, doc_path, folder.Namespace, e) else: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE, doc_path, str(folder.namespace), e) return None try: fs_modified_time = await adapter.compute_last_modified_time_async( doc_path) except Exception as e: logger.warning( self._ctx, self._TAG, PersistenceLayer._load_document_from_path_async.__name__, doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED, e.Message) if not doc_name: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME) return None doc_name_lower = doc_name.lower() # If loading an model.json file, check that it is named correctly. if doc_name_lower.endswith( self.MODEL_JSON_EXTENSION ) and not doc_name.lower() == self.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None try: if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION ) or doc_name_lower.endswith( PersistenceLayer.FOLIO_EXTENSION): from cdm.persistence.cdmfolder import ManifestPersistence from cdm.persistence.cdmfolder.types import ManifestContent manifest = ManifestContent() manifest.decode(json_data) doc_content = ManifestPersistence.from_object( self._ctx, doc_name, folder.namespace, folder.folder_path, manifest) elif doc_name_lower.endswith( PersistenceLayer.MODEL_JSON_EXTENSION): if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE, doc_name, self.MODEL_JSON_EXTENSION) return None from cdm.persistence.modeljson import ManifestPersistence from cdm.persistence.modeljson.types import Model model = Model() model.decode(json_data) doc_content = await ManifestPersistence.from_object( self._ctx, model, folder) elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION): from cdm.persistence.cdmfolder import DocumentPersistence from cdm.persistence.cdmfolder.types import DocumentContent document = DocumentContent() document.decode(json_data) doc_content = DocumentPersistence.from_object( self._ctx, doc_name, folder.namespace, folder.folder_path, document) else: # Could not find a registered persistence class to handle this document type. logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING, doc_name) return None except Exception as e: logger.error(self._ctx, self._TAG, self._load_document_from_path_async.__name__, doc_path, CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE, doc_path, e) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy( ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus. default_resolution_directives), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder.folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter(folder.namespace) # type: StorageAdapter try: if adapter.can_read(): # log message used by navigator, do not change or remove logger.debug(self._TAG, self._ctx, 'request file: {}'.format(doc_path), self._load_document_from_path_async.__name__) json_data = await adapter.read_async(doc_path) # log message used by navigator, do not change or remove logger.debug(self._TAG, self._ctx, 'received file: {}'.format(doc_path), self._load_document_from_path_async.__name__) else: raise Exception('Storage Adapter is not enabled to read.') except Exception as e: # log message used by navigator, do not change or remove logger.debug(self._TAG, self._ctx, 'fail file: {}'.format(doc_path), self._load_document_from_path_async.__name__) message = 'Could not read {} from the \'{}\' namespace.\n Reason: {}'.format(doc_path, folder.namespace, e) # when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors. if res_opt and res_opt.shallow_validation: logger.warning(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__) else: logger.error(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__) return None try: fs_modified_time = await adapter.compute_last_modified_time_async(doc_path) except Exception as e: logger.warning(self._TAG, self._ctx, 'Failed to compute file last modified time. Reason {}'.format(e), self._load_document_from_path_async.__name__) if not doc_name: logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._load_document_from_path_async.__name__) return None # If loading an odi.json/model.json file, check that it is named correctly. if doc_name.lower().endswith(self.ODI_EXTENSION) and not doc_name.lower() == self.ODI_EXTENSION: logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format( doc_name, self.ODI_EXTENSION), self._load_document_from_path_async.__name__) return None if doc_name.lower().endswith(self.MODEL_JSON_EXTENSION) and not doc_name.lower() == self.MODEL_JSON_EXTENSION: logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format( doc_name, self.MODEL_JSON_EXTENSION), self._load_document_from_path_async.__name__) return None # Fetch the correct persistence class to use. persistence_class = self._fetch_registered_persistence_format(doc_name) if persistence_class: try: method = persistence_class.from_data parameters = [self._ctx, doc_name, json_data, folder] # check if from_data() is asynchronous for this persistence class. if persistence_class not in self._is_registered_persistence_async: # Cache whether this persistence class has async methods. self._is_registered_persistence_async[persistence_class] = persistence_class.is_persistence_async if self._is_registered_persistence_async[persistence_class]: doc_content = await method(*parameters) else: doc_content = method(*parameters) except Exception as e: logger.error(self._TAG, self._ctx, 'Could not convert \'{}\'. Reason \'{}\''.format(doc_name, e), self._load_document_from_path_async.__name__) return None else: # could not find a registered persistence class to handle this document type. logger.error(self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\''.format( doc_name), self._load_document_from_path_async.__name__) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy(ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus.default_resolution_directives), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content
async def _send_async_helper( self, cdm_request: 'CdmHttpRequest', callback, ctx: Optional['CdmCorpusContext']) -> 'CdmHttpResponse': """ Sends a CDM request with the retry logic helper function. :param cdm_request: The CDM Http request. :param callback: An optional parameter which specifies a callback function that gets executed after we try to execute the HTTP request. :return: The Cdm Http response. """ full_url = None # type : str if self._api_endpoint is not None: full_url = self._combine_urls(self._api_endpoint, cdm_request.requested_url) else: full_url = cdm_request.requested_url data = None # type: json # Set the content to be in the proper form and headers to denote # the content type. if cdm_request.content is not None: data = cdm_request.content cdm_request.headers['Content-Type'] = cdm_request.content_type # urllib.request.Request() expects 'data' to be in bytes, so we convert to bytes here. if data is not None: data = data.encode('utf-8') request = urllib.request.Request(full_url, method=cdm_request.method, data=data) for key in cdm_request.headers: request.add_header(key, cdm_request.headers[key]) # If the number of retries is 0, we only try once, otherwise we retry the specified # number of times. for retry_number in range(cdm_request.number_of_retries + 1): cdm_response = None # type: CdmHttpResponse has_failed = False # type: bool try: start_time = datetime.now() if ctx is not None: logger.debug( ctx, self._TAG, self._send_async_helper, None, 'Sending request: {}, request type: {}, request url: {}, retry number: {}.' .format(cdm_request.request_id, request.method, cdm_request._strip_sas_sig(), retry_number)) if cdm_request._maximum_timeout_exceeded: raise Exception('timed out') # Calculate how much longer we have before hitting the maximum timout. max_timeout = cdm_request._time_for_maximum_timeout # The request should timeout either for its own timeout or if maximum timeout is reached. timeout = min(max_timeout, cdm_request.timeout) / 1000 # Send the request and convert timeout to seconds from milliseconds. with await in_thread_urlopen(request, timeout=timeout) as response: if response is not None: end_time = datetime.now() if ctx is not None: logger.debug( ctx, self._TAG, self._send_async_helper, None, 'Response for request {} received with elapsed time: {} ms.' .format( cdm_request.request_id, (end_time - start_time).total_seconds() * 1000.0)) cdm_response = CdmHttpResponse() encoded_content = response.read() # Check whether we have appropriate attributes on the object. if hasattr(encoded_content, 'decode'): cdm_response.content = encoded_content.decode( 'utf-8') if hasattr(response, 'status'): cdm_response.reason = response.reason cdm_response.status_code = response.status # Successful requests have HTTP standard status codes in the 2xx form. cdm_response.is_successful = response.status // 100 == 2 if hasattr(response, 'getheaders'): cdm_response.response_headers = dict( response.getheaders()) except Exception as exception: end_time = datetime.now() has_failed = True if exception.args and exception.args[0].args and exception.args[ 0].args[0] == 'timed out' and ctx is not None: logger.debug( ctx, self._TAG, self._send_async_helper, None, 'Request {} timeout after {} ms.'.format( cdm_request.request_id, (end_time - start_time).total_seconds() * 1000.0)) # If the server returned an error like, 404, 500... if isinstance(exception, urllib.error.URLError): if ctx is not None: logger.debug( ctx, self._TAG, self._send_async_helper, None, 'Response for request {} received with elapsed time: {} ms.' .format(cdm_request.request_id, (end_time - start_time).total_seconds() * 1000.0)) cdm_response = CdmHttpResponse() if hasattr(exception, 'reason'): cdm_response.reason = exception.reason if hasattr(exception, 'status'): cdm_response.status_code = exception.status cdm_response.is_successful = False if callback is None or retry_number == cdm_request.number_of_retries: if retry_number != 0 and not cdm_request._maximum_timeout_exceeded: raise CdmNumberOfRetriesExceededException(exception) if exception.args and exception.args[ 0].args and exception.args[0].args[ 0] == 'timed out': raise CdmTimedOutException('Request timeout.') raise exception # Check whether we have a callback function set and whether this is not our last retry. if callback is not None and retry_number != cdm_request.number_of_retries and not cdm_request._maximum_timeout_exceeded: # Call the callback function with the retry numbers starting from 1. wait_time = callback(cdm_response, has_failed, retry_number + 1) # type: int if wait_time is None: return cdm_response # Convert from milliseconds to seconds and wait the time specified by the callback. await asyncio.sleep(wait_time / 1000) else: # CDM Http Response exists, could be successful or bad (e.g. 403/404), it is up to caller to deal with it. if cdm_response is not None: return cdm_response if retry_number < cdm_request.number_of_retries or cdm_request._maximum_timeout_exceeded: raise CdmTimedOutException('Request timeout.') # If response doesn't exist repeatedly, just throw that the number of retries has exceeded (we don't have any other information). raise CdmNumberOfRetriesExceededException() # Should never come here, but just in case throw this exception. raise CdmNumberOfRetriesExceededException()
async def create_resolved_manifest_async( self, new_manifest_name: str, new_entity_document_name_format: str ) -> Optional['CdmManifestDefinition']: """Creates a resolved copy of the manifest. new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities. The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any sub-folders described by the pattern should exist in the corpus prior to calling this function. """ if self.entities is None: return None if not self.folder: logger.error( self._TAG, self.ctx, 'Cannot resolve the manifest \'{}\' because it has not been added to a folder' .format(self.manifest_name), self.create_resolved_manifest_async.__name__) return None if new_entity_document_name_format is None: new_entity_document_name_format = '{f}resolved/{n}.cdm.json' elif new_entity_document_name_format == '': # for back compat new_entity_document_name_format = '{n}.cdm.json' elif '{n}' not in new_entity_document_name_format: # for back compat new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json' source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.at_corpus_path, self) source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.folder.at_corpus_path, self) resolved_manifest_path_split = new_manifest_name.rfind('/') + 1 resolved_manifest_folder = None if resolved_manifest_path_split > 0: resolved_manifest_path = new_manifest_name[ 0:resolved_manifest_path_split] new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path( resolved_manifest_path, self) resolved_manifest_folder = await self.ctx.corpus.fetch_object_async( new_folder_path) # type: CdmFolderDefinition if resolved_manifest_folder is None: logger.error( self._TAG, self.ctx, 'New folder for manifest not found {}'.format( new_folder_path), self.create_resolved_manifest_async.__name__) return None new_manifest_name = new_manifest_name[ resolved_manifest_path_split:] else: resolved_manifest_folder = self.owner logger.debug(self._TAG, self.ctx, 'resolving manifest {}'.format(source_manifest_path), self.create_resolved_manifest_async.__name__) # using the references present in the resolved entities, get an entity # create an imports doc with all the necessary resolved entity references and then resolve it resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name) # add the new document to the folder if resolved_manifest_folder.documents.append( resolved_manifest) is None: # when would this happen? return None for entity in self.entities: ent_def = await self._get_entity_from_reference(entity, self) if not ent_def: logger.error(self._TAG, self.ctx, 'Unable to get entity from reference', self.create_resolved_manifest_async.__name__) return None if not ent_def.in_document.folder: logger.error( self._TAG, self.ctx, 'The document containing the entity \'{}\' is not in a folder' .format(ent_def.entity_name), self.create_resolved_manifest_async.__name__) return None # get the path from this manifest to the source entity. this will be the {f} replacement value source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( ent_def.in_document.folder.at_corpus_path, self) f = '' if source_entity_full_path.startswith(source_manifest_folder_path): f = source_entity_full_path[len(source_manifest_folder_path):] new_document_full_path = new_entity_document_name_format.replace( '{n}', ent_def.entity_name).replace('{f}', f) new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path( new_document_full_path, self) new_document_path_split = new_document_full_path.rfind('/') + 1 new_document_path = new_document_full_path[ 0:new_document_path_split] new_document_name = new_document_full_path[ new_document_path_split:] # make sure the new folder exists folder = await self.ctx.corpus.fetch_object_async( new_document_path) # type: CdmFolderDefinition if not folder: logger.error( self._TAG, self.ctx, 'New folder not found {}'.format(new_document_path), self.create_resolved_manifest_async.__name__) return None # next create the resolved entity. res_opt = ResolveOptions() res_opt.wrt_doc = ent_def.in_document res_opt.directives = AttributeResolutionDirectiveSet( {'normalized', 'referenceOnly'}) logger.debug( self._TAG, self.ctx, ' resolving entity {} to document {}'.format( source_entity_full_path, new_document_full_path), self.create_resolved_manifest_async.__name__) resolved_entity = await ent_def.create_resolved_entity_async( ent_def.entity_name, res_opt, folder, new_document_name) if not resolved_entity: # fail all resolution, if any one entity resolution fails return None result = entity.copy(res_opt) if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path( resolved_entity.at_corpus_path, resolved_manifest) result.entity_path = relative_entity_path or result.at_corpus_path resolved_manifest.entities.append(result) logger.debug(self._TAG, self.ctx, ' calculating relationships', self.create_resolved_manifest_async.__name__) # Calculate the entity graph for just this manifest. await self.ctx.corpus.calculate_entity_graph_async(resolved_manifest) # Stick results into the relationships list for the manifest. await resolved_manifest.populate_manifest_relationships_async( CdmRelationshipDiscoveryStyle.EXCLUSIVE) # needed until Matt's changes with collections where I can propigate resolved_manifest._is_dirty = True return resolved_manifest