def make_object(ctx, of_type: 'CdmObjectType', name_or_ref: str, simple_name_ref: bool) -> 'TObject':
"""instantiates a OM class based on the object type passed as first parameter."""
# Log and ingest a message when a new manifest is created
if of_type == CdmObjectType.MANIFEST_DEF:
logger.debug(ctx, None, make_object.__name__, None, 'New Manifest created.', True)
return switcher[of_type](ctx, name_or_ref, simple_name_ref)
def _finish_indexing(self, loaded_imports: bool) -> None:
"""Marks that the document was indexed."""
logger.debug(self.ctx, self._TAG, self._finish_indexing.__name__,
self.at_corpus_path,
'index finish: {}'.format(self.at_corpus_path))
was_indexed_previously = self._declarations_indexed
self.ctx.corpus._document_library._mark_document_as_indexed(self)
self._imports_indexed = self._imports_indexed or loaded_imports
self._declarations_indexed = True
self._needs_indexing = not loaded_imports
self._internal_objects = None
# if the document declarations were indexed previously, do not log again.
if not was_indexed_previously and self._is_valid:
for definition in self.definitions:
if definition.object_type == CdmObjectType.ENTITY_DEF:
logger.debug(
self.ctx, self._TAG, self._finish_indexing.__name__,
definition.at_corpus_path,
'indexed entity: {}'.format(definition.at_corpus_path))
async def create_resolved_manifest_async(
self,
new_manifest_name: str,
new_entity_document_name_format: Optional[str],
directives: Optional[AttributeResolutionDirectiveSet] = None
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
with logger._enter_scope(self._TAG, self.ctx,
self.create_resolved_manifest_async.__name__):
if self.entities is None:
return None
if not self.folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_MANIFEST_FAILED,
self.manifest_name)
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND,
new_folder_path)
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
logger.debug(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
'resolving manifest {}'.format(source_manifest_path))
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
# sometimes they might send the docname, that makes sense a bit, don't include the suffix in the name
if new_manifest_name.lower().endswith('.manifest.cdm.json'):
new_manifest_name = new_manifest_name[0:(
len(new_manifest_name) - len('.manifest.cdm.json'))]
resolved_manifest = CdmManifestDefinition(self.ctx,
new_manifest_name)
# bring over any imports in this document or other bobbles
resolved_manifest.schema = self.schema
resolved_manifest.explanation = self.explanation
resolved_manifest.document_version = self.document_version
for imp in self.imports:
resolved_manifest.imports.append(imp.copy())
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_ENTITY_REF_ERROR)
return None
if not ent_def.in_document.folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_DOC_IS_NOT_FOLDERformat,
ent_def.entity_name)
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(
source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path
):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
logger.error(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_FOLDER_NOT_FOUND,
new_document_path)
return None
# next create the resolved entity.
with_directives = directives if directives is not None else self.ctx.corpus.default_resolution_directives
res_opt = ResolveOptions(ent_def.in_document,
with_directives.copy())
logger.debug(
self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path,
'resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path))
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
logger.debug(self.ctx, self._TAG,
self.create_resolved_manifest_async.__name__,
self.at_corpus_path, 'calculating relationships')
# Calculate the entity graph for just this manifest.
await self.ctx.corpus.calculate_entity_graph_async(
resolved_manifest)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str,
doc_container: 'CdmDocumentDefinition',
res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'request file: {}'.format(doc_path))
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'received file: {}'.format(doc_path))
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, 'fail file: {}'.format(doc_path))
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_READ_FAILURE,
doc_path, folder.Namespace, e)
else:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_FILE_READ_FAILURE,
doc_path, str(folder.namespace), e)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(
doc_path)
except Exception as e:
logger.warning(
self._ctx, self._TAG,
PersistenceLayer._load_document_from_path_async.__name__,
doc_path, CdmLogCode.WARN_PERSIST_FILE_MOD_COMPUTE_FAILED,
e.Message)
if not doc_name:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_NULL_DOC_NAME)
return None
doc_name_lower = doc_name.lower()
# If loading an model.json file, check that it is named correctly.
if doc_name_lower.endswith(
self.MODEL_JSON_EXTENSION
) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
try:
if doc_name_lower.endswith(PersistenceLayer.MANIFEST_EXTENSION
) or doc_name_lower.endswith(
PersistenceLayer.FOLIO_EXTENSION):
from cdm.persistence.cdmfolder import ManifestPersistence
from cdm.persistence.cdmfolder.types import ManifestContent
manifest = ManifestContent()
manifest.decode(json_data)
doc_content = ManifestPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
manifest)
elif doc_name_lower.endswith(
PersistenceLayer.MODEL_JSON_EXTENSION):
if doc_name_lower != PersistenceLayer.MODEL_JSON_EXTENSION:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_NAME_LOAD_FAILURE,
doc_name, self.MODEL_JSON_EXTENSION)
return None
from cdm.persistence.modeljson import ManifestPersistence
from cdm.persistence.modeljson.types import Model
model = Model()
model.decode(json_data)
doc_content = await ManifestPersistence.from_object(
self._ctx, model, folder)
elif doc_name_lower.endswith(PersistenceLayer.CDM_EXTENSION):
from cdm.persistence.cdmfolder import DocumentPersistence
from cdm.persistence.cdmfolder.types import DocumentContent
document = DocumentContent()
document.decode(json_data)
doc_content = DocumentPersistence.from_object(
self._ctx, doc_name, folder.namespace, folder.folder_path,
document)
else:
# Could not find a registered persistence class to handle this document type.
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path, CdmLogCode.ERR_PERSIST_CLASS_MISSING,
doc_name)
return None
except Exception as e:
logger.error(self._ctx, self._TAG,
self._load_document_from_path_async.__name__,
doc_path,
CdmLogCode.ERR_PERSIST_DOC_CONVERSION_FAILURE,
doc_path, e)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container,
directives=self._ctx.corpus.
default_resolution_directives),
doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition', res_opt: Optional[ResolveOptions] = None) \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
# log message used by navigator, do not change or remove
logger.debug(self._TAG, self._ctx, 'request file: {}'.format(doc_path), self._load_document_from_path_async.__name__)
json_data = await adapter.read_async(doc_path)
# log message used by navigator, do not change or remove
logger.debug(self._TAG, self._ctx, 'received file: {}'.format(doc_path), self._load_document_from_path_async.__name__)
else:
raise Exception('Storage Adapter is not enabled to read.')
except Exception as e:
# log message used by navigator, do not change or remove
logger.debug(self._TAG, self._ctx, 'fail file: {}'.format(doc_path), self._load_document_from_path_async.__name__)
message = 'Could not read {} from the \'{}\' namespace.\n Reason: {}'.format(doc_path, folder.namespace, e)
# when shallow validation is enabled, log messages about being unable to find referenced documents as warnings instead of errors.
if res_opt and res_opt.shallow_validation:
logger.warning(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__)
else:
logger.error(self._TAG, self._ctx, message, self._load_document_from_path_async.__name__)
return None
try:
fs_modified_time = await adapter.compute_last_modified_time_async(doc_path)
except Exception as e:
logger.warning(self._TAG, self._ctx, 'Failed to compute file last modified time. Reason {}'.format(e), self._load_document_from_path_async.__name__)
if not doc_name:
logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._load_document_from_path_async.__name__)
return None
# If loading an odi.json/model.json file, check that it is named correctly.
if doc_name.lower().endswith(self.ODI_EXTENSION) and not doc_name.lower() == self.ODI_EXTENSION:
logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format(
doc_name, self.ODI_EXTENSION), self._load_document_from_path_async.__name__)
return None
if doc_name.lower().endswith(self.MODEL_JSON_EXTENSION) and not doc_name.lower() == self.MODEL_JSON_EXTENSION:
logger.error(self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'.format(
doc_name, self.MODEL_JSON_EXTENSION), self._load_document_from_path_async.__name__)
return None
# Fetch the correct persistence class to use.
persistence_class = self._fetch_registered_persistence_format(doc_name)
if persistence_class:
try:
method = persistence_class.from_data
parameters = [self._ctx, doc_name, json_data, folder]
# check if from_data() is asynchronous for this persistence class.
if persistence_class not in self._is_registered_persistence_async:
# Cache whether this persistence class has async methods.
self._is_registered_persistence_async[persistence_class] = persistence_class.is_persistence_async
if self._is_registered_persistence_async[persistence_class]:
doc_content = await method(*parameters)
else:
doc_content = method(*parameters)
except Exception as e:
logger.error(self._TAG, self._ctx, 'Could not convert \'{}\'. Reason \'{}\''.format(doc_name, e), self._load_document_from_path_async.__name__)
return None
else:
# could not find a registered persistence class to handle this document type.
logger.error(self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\''.format(
doc_name), self._load_document_from_path_async.__name__)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(ResolveOptions(wrt_doc=doc_container, directives=self._ctx.corpus.default_resolution_directives), doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
async def _send_async_helper(
self, cdm_request: 'CdmHttpRequest', callback,
ctx: Optional['CdmCorpusContext']) -> 'CdmHttpResponse':
"""
Sends a CDM request with the retry logic helper function.
:param cdm_request: The CDM Http request.
:param callback: An optional parameter which specifies a callback
function that gets executed after we try to execute the HTTP request.
:return: The Cdm Http response.
"""
full_url = None # type : str
if self._api_endpoint is not None:
full_url = self._combine_urls(self._api_endpoint,
cdm_request.requested_url)
else:
full_url = cdm_request.requested_url
data = None # type: json
# Set the content to be in the proper form and headers to denote
# the content type.
if cdm_request.content is not None:
data = cdm_request.content
cdm_request.headers['Content-Type'] = cdm_request.content_type
# urllib.request.Request() expects 'data' to be in bytes, so we convert to bytes here.
if data is not None:
data = data.encode('utf-8')
request = urllib.request.Request(full_url,
method=cdm_request.method,
data=data)
for key in cdm_request.headers:
request.add_header(key, cdm_request.headers[key])
# If the number of retries is 0, we only try once, otherwise we retry the specified
# number of times.
for retry_number in range(cdm_request.number_of_retries + 1):
cdm_response = None # type: CdmHttpResponse
has_failed = False # type: bool
try:
start_time = datetime.now()
if ctx is not None:
logger.debug(
ctx, self._TAG, self._send_async_helper, None,
'Sending request: {}, request type: {}, request url: {}, retry number: {}.'
.format(cdm_request.request_id, request.method,
cdm_request._strip_sas_sig(), retry_number))
if cdm_request._maximum_timeout_exceeded:
raise Exception('timed out')
# Calculate how much longer we have before hitting the maximum timout.
max_timeout = cdm_request._time_for_maximum_timeout
# The request should timeout either for its own timeout or if maximum timeout is reached.
timeout = min(max_timeout, cdm_request.timeout) / 1000
# Send the request and convert timeout to seconds from milliseconds.
with await in_thread_urlopen(request,
timeout=timeout) as response:
if response is not None:
end_time = datetime.now()
if ctx is not None:
logger.debug(
ctx, self._TAG, self._send_async_helper, None,
'Response for request {} received with elapsed time: {} ms.'
.format(
cdm_request.request_id,
(end_time - start_time).total_seconds() *
1000.0))
cdm_response = CdmHttpResponse()
encoded_content = response.read()
# Check whether we have appropriate attributes on the object.
if hasattr(encoded_content, 'decode'):
cdm_response.content = encoded_content.decode(
'utf-8')
if hasattr(response, 'status'):
cdm_response.reason = response.reason
cdm_response.status_code = response.status
# Successful requests have HTTP standard status codes in the 2xx form.
cdm_response.is_successful = response.status // 100 == 2
if hasattr(response, 'getheaders'):
cdm_response.response_headers = dict(
response.getheaders())
except Exception as exception:
end_time = datetime.now()
has_failed = True
if exception.args and exception.args[0].args and exception.args[
0].args[0] == 'timed out' and ctx is not None:
logger.debug(
ctx, self._TAG, self._send_async_helper, None,
'Request {} timeout after {} ms.'.format(
cdm_request.request_id,
(end_time - start_time).total_seconds() * 1000.0))
# If the server returned an error like, 404, 500...
if isinstance(exception, urllib.error.URLError):
if ctx is not None:
logger.debug(
ctx, self._TAG, self._send_async_helper, None,
'Response for request {} received with elapsed time: {} ms.'
.format(cdm_request.request_id,
(end_time - start_time).total_seconds() *
1000.0))
cdm_response = CdmHttpResponse()
if hasattr(exception, 'reason'):
cdm_response.reason = exception.reason
if hasattr(exception, 'status'):
cdm_response.status_code = exception.status
cdm_response.is_successful = False
if callback is None or retry_number == cdm_request.number_of_retries:
if retry_number != 0 and not cdm_request._maximum_timeout_exceeded:
raise CdmNumberOfRetriesExceededException(exception)
if exception.args and exception.args[
0].args and exception.args[0].args[
0] == 'timed out':
raise CdmTimedOutException('Request timeout.')
raise exception
# Check whether we have a callback function set and whether this is not our last retry.
if callback is not None and retry_number != cdm_request.number_of_retries and not cdm_request._maximum_timeout_exceeded:
# Call the callback function with the retry numbers starting from 1.
wait_time = callback(cdm_response, has_failed,
retry_number + 1) # type: int
if wait_time is None:
return cdm_response
# Convert from milliseconds to seconds and wait the time specified by the callback.
await asyncio.sleep(wait_time / 1000)
else:
# CDM Http Response exists, could be successful or bad (e.g. 403/404), it is up to caller to deal with it.
if cdm_response is not None:
return cdm_response
if retry_number < cdm_request.number_of_retries or cdm_request._maximum_timeout_exceeded:
raise CdmTimedOutException('Request timeout.')
# If response doesn't exist repeatedly, just throw that the number of retries has exceeded (we don't have any other information).
raise CdmNumberOfRetriesExceededException()
# Should never come here, but just in case throw this exception.
raise CdmNumberOfRetriesExceededException()
async def create_resolved_manifest_async(
self, new_manifest_name: str, new_entity_document_name_format: str
) -> Optional['CdmManifestDefinition']:
"""Creates a resolved copy of the manifest.
new_entity_document_name_format specifies a pattern to use when creating documents for resolved entities.
The default is "resolved/{n}.cdm.json" to avoid a document name conflict with documents in the same folder as
the manifest. Every instance of the string {n} is replaced with the entity name from the source manifest. Any
sub-folders described by the pattern should exist in the corpus prior to calling this function.
"""
if self.entities is None:
return None
if not self.folder:
logger.error(
self._TAG, self.ctx,
'Cannot resolve the manifest \'{}\' because it has not been added to a folder'
.format(self.manifest_name),
self.create_resolved_manifest_async.__name__)
return None
if new_entity_document_name_format is None:
new_entity_document_name_format = '{f}resolved/{n}.cdm.json'
elif new_entity_document_name_format == '': # for back compat
new_entity_document_name_format = '{n}.cdm.json'
elif '{n}' not in new_entity_document_name_format: # for back compat
new_entity_document_name_format = new_entity_document_name_format + '/{n}.cdm.json'
source_manifest_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.at_corpus_path, self)
source_manifest_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.folder.at_corpus_path, self)
resolved_manifest_path_split = new_manifest_name.rfind('/') + 1
resolved_manifest_folder = None
if resolved_manifest_path_split > 0:
resolved_manifest_path = new_manifest_name[
0:resolved_manifest_path_split]
new_folder_path = self.ctx.corpus.storage.create_absolute_corpus_path(
resolved_manifest_path, self)
resolved_manifest_folder = await self.ctx.corpus.fetch_object_async(
new_folder_path) # type: CdmFolderDefinition
if resolved_manifest_folder is None:
logger.error(
self._TAG, self.ctx,
'New folder for manifest not found {}'.format(
new_folder_path),
self.create_resolved_manifest_async.__name__)
return None
new_manifest_name = new_manifest_name[
resolved_manifest_path_split:]
else:
resolved_manifest_folder = self.owner
logger.debug(self._TAG, self.ctx,
'resolving manifest {}'.format(source_manifest_path),
self.create_resolved_manifest_async.__name__)
# using the references present in the resolved entities, get an entity
# create an imports doc with all the necessary resolved entity references and then resolve it
resolved_manifest = CdmManifestDefinition(self.ctx, new_manifest_name)
# add the new document to the folder
if resolved_manifest_folder.documents.append(
resolved_manifest) is None:
# when would this happen?
return None
for entity in self.entities:
ent_def = await self._get_entity_from_reference(entity, self)
if not ent_def:
logger.error(self._TAG, self.ctx,
'Unable to get entity from reference',
self.create_resolved_manifest_async.__name__)
return None
if not ent_def.in_document.folder:
logger.error(
self._TAG, self.ctx,
'The document containing the entity \'{}\' is not in a folder'
.format(ent_def.entity_name),
self.create_resolved_manifest_async.__name__)
return None
# get the path from this manifest to the source entity. this will be the {f} replacement value
source_entity_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
ent_def.in_document.folder.at_corpus_path, self)
f = ''
if source_entity_full_path.startswith(source_manifest_folder_path):
f = source_entity_full_path[len(source_manifest_folder_path):]
new_document_full_path = new_entity_document_name_format.replace(
'{n}', ent_def.entity_name).replace('{f}', f)
new_document_full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
new_document_full_path, self)
new_document_path_split = new_document_full_path.rfind('/') + 1
new_document_path = new_document_full_path[
0:new_document_path_split]
new_document_name = new_document_full_path[
new_document_path_split:]
# make sure the new folder exists
folder = await self.ctx.corpus.fetch_object_async(
new_document_path) # type: CdmFolderDefinition
if not folder:
logger.error(
self._TAG, self.ctx,
'New folder not found {}'.format(new_document_path),
self.create_resolved_manifest_async.__name__)
return None
# next create the resolved entity.
res_opt = ResolveOptions()
res_opt.wrt_doc = ent_def.in_document
res_opt.directives = AttributeResolutionDirectiveSet(
{'normalized', 'referenceOnly'})
logger.debug(
self._TAG, self.ctx,
' resolving entity {} to document {}'.format(
source_entity_full_path, new_document_full_path),
self.create_resolved_manifest_async.__name__)
resolved_entity = await ent_def.create_resolved_entity_async(
ent_def.entity_name, res_opt, folder, new_document_name)
if not resolved_entity:
# fail all resolution, if any one entity resolution fails
return None
result = entity.copy(res_opt)
if result.object_type == CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
relative_entity_path = self.ctx.corpus.storage.create_relative_corpus_path(
resolved_entity.at_corpus_path, resolved_manifest)
result.entity_path = relative_entity_path or result.at_corpus_path
resolved_manifest.entities.append(result)
logger.debug(self._TAG, self.ctx, ' calculating relationships',
self.create_resolved_manifest_async.__name__)
# Calculate the entity graph for just this manifest.
await self.ctx.corpus.calculate_entity_graph_async(resolved_manifest)
# Stick results into the relationships list for the manifest.
await resolved_manifest.populate_manifest_relationships_async(
CdmRelationshipDiscoveryStyle.EXCLUSIVE)
# needed until Matt's changes with collections where I can propigate
resolved_manifest._is_dirty = True
return resolved_manifest
