def _declare_object_definitions(self) -> None:
"""Indexes all definitions contained by this document."""
skip_duplicate_types = {CdmObjectType.CONSTANT_ENTITY_DEF}
internal_declaration_types = {
CdmObjectType.ENTITY_DEF, CdmObjectType.PARAMETER_DEF,
CdmObjectType.TRAIT_DEF, CdmObjectType.TRAIT_GROUP_DEF,
CdmObjectType.PURPOSE_DEF, CdmObjectType.DATA_TYPE_DEF,
CdmObjectType.TYPE_ATTRIBUTE_DEF,
CdmObjectType.ENTITY_ATTRIBUTE_DEF,
CdmObjectType.ATTRIBUTE_GROUP_DEF,
CdmObjectType.CONSTANT_ENTITY_DEF,
CdmObjectType.ATTRIBUTE_CONTEXT_DEF,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF,
CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF,
CdmObjectType.ATTRIBUTE_GROUP_DEF, CdmObjectType.PROJECTION_DEF,
CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF,
CdmObjectType.OPERATION_ADD_SUPPORTING_ATTRIBUTE_DEF,
CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF,
CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF,
CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF,
CdmObjectType.OPERATION_COMBINE_ATTRIBUTES_DEF,
CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF,
CdmObjectType.OPERATION_REPLACE_AS_FOREIGN_KEY_DEF,
CdmObjectType.OPERATION_INCLUDE_ATTRIBUTES_DEF,
CdmObjectType.OPERATION_ADD_ATTRIBUTE_GROUP_DEF,
CdmObjectType.OPERATION_ALTER_TRAITS_DEF,
CdmObjectType.OPERATION_ADD_ARTIFACT_ATTRIBUTE_DEF
}
corpus_path_root = self._folder_path + self.name
for obj in self._internal_objects:
# I can't think of a better time than now to make sure any recently changed or added things have an in doc
obj.in_document = self
obj_path = obj._declared_path
if '(unspecified)' in obj_path:
continue
skip_duplicates = False
if obj.object_type in skip_duplicate_types:
# if there is a duplicate, don't complain, the path just finds the first one
skip_duplicates = True
if obj.object_type in internal_declaration_types:
corpus_path = '{}/{}'.format(corpus_path_root, obj_path)
if obj_path in self.internal_declarations and not skip_duplicates:
logger.error(self.ctx, self._TAG,
'_declare_object_definitions', corpus_path,
CdmLogCode.ERR_PATH_IS_DUPLICATE, corpus_path)
else:
self.internal_declarations[obj_path] = obj
self.ctx.corpus._register_symbol(obj_path, self)
logger.info(self.ctx, self._TAG,
self._declare_object_definitions.__name__,
corpus_path,
'declared \'{}\''.format(obj_path))
def _resolve_object_definitions(self, res_opt: 'ResolveOptions') -> None:
ctx = self.ctx
res_opt._indexing_doc = self
reference_type_set = {
CdmObjectType.ATTRIBUTE_REF, CdmObjectType.ATTRIBUTE_GROUP_REF,
CdmObjectType.ATTRIBUTE_CONTEXT_REF, CdmObjectType.DATA_TYPE_REF,
CdmObjectType.ENTITY_REF, CdmObjectType.PURPOSE_REF,
CdmObjectType.TRAIT_REF
}
for obj in self._internal_objects:
if obj.object_type in reference_type_set:
ctx._relative_path = obj._declared_path
if obj._offset_attribute_promise(obj.named_reference) < 0:
res_new = obj.fetch_object_definition(res_opt)
if not res_new:
# it's okay if references can't be resolved when shallow validation is enabled.
if res_opt.shallow_validation:
logger.warning(
self.ctx, self._TAG,
self._resolve_object_definitions.__name__,
self.at_corpus_path,
CdmLogCode.WARN_RESOLVE_REFERENCE_FAILURE,
obj.named_reference)
else:
logger.error(
self.ctx, self._TAG,
self._resolve_object_definitions.__name__,
self.at_corpus_path,
CdmLogCode.ERR_RESOLVE_REFERENCE_FAILURE,
obj.named_reference)
# don't check in this file without both of these comments. handy for debug of failed lookups
# res_test = obj.fetch_object_definition(res_opt)
else:
logger.info(
self.ctx, self._TAG,
self._resolve_object_definitions.__name__,
self.at_corpus_path,
'resolved \'{}\''.format(obj.named_reference))
elif obj.object_type == CdmObjectType.PARAMETER_DEF:
# when a parameter has a datatype that is a cdm object, validate that any default value is the
# right kind object
parameter = obj # type: CdmParameterDefinition
parameter._const_type_check(res_opt, self, None)
res_opt._indexing_doc = None
def _check_integrity(self) -> bool:
"""Validates all the objects in this document."""
error_count = 0
for obj in self._internal_objects:
if not obj.validate():
error_count += 1
else:
obj.ctx = self.ctx
logger.info(self.ctx, self._TAG, self._check_integrity.__name__,
self.at_corpus_path,
'checked \'{}\''.format(obj.at_corpus_path))
self._is_valid = error_count == 0
def register_format(self, persistence_class_name: str, assembly_name: Optional[str] = None) -> None:
try:
path_split = persistence_class_name.split('.')
class_path = '.'.join(path_split[:-1])
class_name = path_split[-1]
persistence_module = importlib.import_module(class_path)
persistence_class = getattr(persistence_module, class_name)
formats = persistence_class.formats # type: List[str]
for form in formats:
self._registered_persistence_formats[form] = persistence_class
except Exception as e:
logger.info(self._TAG, self._ctx, 'Unable to register persistence class {}. Reason: {}.'.format(persistence_class_name, e))
def enable(self) -> None:
"""
Enable the telemetry client by starting a thread for ingestion.
"""
# Check if the Kusto config and the concurrent queue has been initialized
if self._config is None or self._request_queue is None:
message = 'The telemetry client has not been initialized.'
logger.info(self._ctx, TelemetryKustoClient.__name__,
self.enable.__name__, None, message)
return
# Starts a separate thread to ingest telemetry logs into Kusto
ingestion_thread = threading.Thread(target=self._ingest_request_queue,
daemon=True)
ingestion_thread.start()
def _const_type_check(self, res_opt: 'ResolveOptions', wrt_doc: 'CdmDocumentDefinition', argument_value: Any) -> 'CdmArgumentValue':
ctx = self.ctx
replacement = argument_value
# if parameter type is entity, then the value should be an entity or ref to one
# same is true of 'dataType' dataType
ref = self.data_type_ref
if not ref:
return replacement
dt = ref.fetch_object_definition(res_opt)
if not dt:
logger.error(self.ctx, self._TAG, '_const_type_check', self.at_corpus_path, CdmLogCode.ERR_UNRECOGNIZED_DATA_TYPE, self.name)
return None
# compare with passed in value or default for parameter
p_value = argument_value
if p_value is None:
p_value = self.default_value
replacement = p_value
if p_value is not None:
if dt.is_derived_from('cdmObject', res_opt):
expected_types = []
expected = None
if dt.is_derived_from('entity', res_opt):
expected_types.append(CdmObjectType.CONSTANT_ENTITY_DEF)
expected_types.append(CdmObjectType.ENTITY_REF)
expected_types.append(CdmObjectType.ENTITY_DEF)
expected_types.append(CdmObjectType.PROJECTION_DEF)
expected = 'entity'
elif dt.is_derived_from('attribute', res_opt):
expected_types.append(CdmObjectType.ATTRIBUTE_REF)
expected_types.append(CdmObjectType.TYPE_ATTRIBUTE_DEF)
expected_types.append(CdmObjectType.ENTITY_ATTRIBUTE_DEF)
expected = 'attribute'
elif dt.is_derived_from('dataType', res_opt):
expected_types.append(CdmObjectType.DATA_TYPE_REF)
expected_types.append(CdmObjectType.DATA_TYPE_DEF)
expected = 'dataType'
elif dt.is_derived_from('purpose', res_opt):
expected_types.append(CdmObjectType.PURPOSE_REF)
expected_types.append(CdmObjectType.PURPOSE_DEF)
expected = 'purpose'
elif dt.is_derived_from('trait', res_opt):
expected_types.append(CdmObjectType.TRAIT_REF)
expected_types.append(CdmObjectType.TRAIT_DEF)
expected = 'trait'
elif dt.is_derived_from('traitGroup', res_opt):
expected_types.append(CdmObjectType.TRAIT_GROUP_REF)
expected_types.append(CdmObjectType.TRAIT_GROUP_DEF)
expected = 'traitGroup'
elif dt.is_derived_from('attributeGroup', res_opt):
expected_types.append(CdmObjectType.ATTRIBUTE_GROUP_REF)
expected_types.append(CdmObjectType.ATTRIBUTE_GROUP_DEF)
expected = 'attributeGroup'
if not expected_types:
logger.error(self.ctx, self._TAG, '_const_type_check', wrt_doc.at_corpus_path, CdmLogCode.ERR_UNEXPECTED_DATA_TYPE,
self.name)
# if a string constant, resolve to an object ref.
found_type = CdmObjectType.ERROR
if isinstance(p_value, CdmObject):
found_type = p_value.object_type
found_desc = ctx._relative_path
if isinstance(p_value, str):
if p_value == 'this.attribute' and expected == 'attribute':
# will get sorted out later when resolving traits
found_type = CdmObjectType.ATTRIBUTE_REF
else:
found_desc = p_value
from cdm.objectmodel import CdmObjectReference
seek_res_att = CdmObjectReference._offset_attribute_promise(p_value)
if seek_res_att >= 0:
# get an object there that will get resolved later after resolved attributes
replacement = CdmAttributeReference(self.ctx, p_value, True)
replacement.in_document = wrt_doc
found_type = CdmObjectType.ATTRIBUTE_REF
else:
lu = ctx.corpus._resolve_symbol_reference(res_opt, wrt_doc, p_value,
CdmObjectType.ERROR, True)
if lu:
if expected == 'attribute':
replacement = CdmAttributeReference(self.ctx, p_value, True)
replacement.in_document = wrt_doc
found_type = CdmObjectType.ATTRIBUTE_REF
else:
replacement = lu
if isinstance(replacement, CdmObject):
found_type = replacement.object_type
if expected_types.index(found_type) == -1:
logger.error(self.ctx, self._TAG, wrt_doc.at_corpus_path,
CdmLogCode.ERR_RESOLUTION_FAILURE,
self.get_name(), expected, found_desc)
else:
logger.info(self.ctx, self._TAG, self._const_type_check.__name__,
wrt_doc.at_corpus_path, 'resolved \'{}\''.format(found_desc))
return replacement
def _localize_corpus_paths(self,
new_folder: 'CdmFolderDefinition') -> bool:
all_went_well = True
logger.info(
self.ctx, self._TAG, self._localize_corpus_paths.__name__,
new_folder.at_corpus_path,
'Localizing corpus paths in document \'{}\''.format(self.name))
def import_callback(obj: 'CdmObject', path: str) -> bool:
nonlocal all_went_well
corpus_path, worked = self._localize_corpus_path(
obj.corpus_path, new_folder)
if not worked:
all_went_well = False
else:
obj.corpus_path = corpus_path
def entity_declaration_definition_callback(obj: 'CdmObject',
path: str) -> bool:
nonlocal all_went_well
corpus_path, worked = self._localize_corpus_path(
obj.entity_path, new_folder)
if not worked:
all_went_well = False
else:
obj.entity_path = corpus_path
def data_partition_callback(obj: 'CdmObject', path: str) -> bool:
nonlocal all_went_well
corpus_path, worked = self._localize_corpus_path(
obj.location, new_folder)
if not worked:
all_went_well = False
else:
obj.location = corpus_path
corpus_path, worked = self._localize_corpus_path(
obj.specialized_schema, new_folder)
if not worked:
all_went_well = False
else:
obj.specialized_schema = corpus_path
def data_partition_pattern_callback(obj: 'CdmObject',
path: str) -> bool:
nonlocal all_went_well
corpus_path, worked = self._localize_corpus_path(
obj.root_location, new_folder)
if not worked:
all_went_well = False
else:
obj.root_location = corpus_path
corpus_path, worked = self._localize_corpus_path(
obj.specialized_schema, new_folder)
if not worked:
all_went_well = False
else:
obj.specialized_schema = corpus_path
def e2e_relationship_callback(obj: 'CdmObject', path: str) -> bool:
nonlocal all_went_well
corpus_path, worked = self._localize_corpus_path(
obj.to_entity, new_folder)
if not worked:
all_went_well = False
else:
obj.to_entity = corpus_path
corpus_path, worked = self._localize_corpus_path(
obj.from_entity, new_folder)
if not worked:
all_went_well = False
else:
obj.from_entity = corpus_path
def manifest_declaration_callback(obj: 'CdmObject', path: str) -> bool:
nonlocal all_went_well
corpus_path, worked = self._localize_corpus_path(
obj.definition, new_folder)
if not worked:
all_went_well = False
else:
obj.definition = corpus_path
switcher = {
CdmObjectType.IMPORT: import_callback,
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF:
entity_declaration_definition_callback,
CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF:
entity_declaration_definition_callback,
CdmObjectType.DATA_PARTITION_DEF: data_partition_callback,
CdmObjectType.DATA_PARTITION_PATTERN_DEF:
data_partition_pattern_callback,
CdmObjectType.E2E_RELATIONSHIP_DEF: e2e_relationship_callback,
CdmObjectType.MANIFEST_DECLARATION_DEF:
manifest_declaration_callback
}
def pre_callback(obj: 'CdmObject', path: str) -> bool:
# i don't like that document needs to know a little about these objects
# in theory, we could create a virtual function on cdmObject that localizes properties
# but then every object would need to know about the documents and paths and such ...
# also, i already wrote this code.
func = switcher.get(obj.object_type)
if func:
func(obj, path)
return False
# find anything in the document that is a corpus path
self.visit('', pre_callback, None)
return all_went_well
async def _send_async_helper(
self,
cdm_request: 'CdmHttpRequest',
callback=None,
ctx: Optional['CdmCorpusContext'] = None) -> 'CdmHttpResponse':
"""
Sends a CDM request with the retry logic helper function.
:param cdm_request: The CDM Http request.
:param callback: An optional parameter which specifies a callback
function that gets executed after we try to execute the HTTP request.
:return: The Cdm Http response.
"""
full_url = None # type : str
if self._api_endpoint is not None:
full_url = self._combine_urls(self._api_endpoint,
cdm_request.requested_url)
else:
full_url = cdm_request.requested_url
data = None # type: json
# Set the content to be in the proper form and headers to denote
# the content type.
if cdm_request.content is not None:
data = cdm_request.content
cdm_request.headers['Content-Type'] = cdm_request.content_type
# urllib.request.Request() expects 'data' to be in bytes, so we convert to bytes here.
if data is not None:
data = data.encode("utf-8")
request = urllib.request.Request(full_url,
method=cdm_request.method,
data=data)
for key in cdm_request.headers:
request.add_header(key, cdm_request.headers[key])
# If the number of retries is 0, we only try once, otherwise we retry the specified
# number of times.
for retry_number in range(cdm_request.number_of_retries + 1):
cdm_response = None # type: CdmHttpResponse
has_failed = False # type: bool
try:
start_time = datetime.now()
if ctx is not None:
logger.info(
self.__class__.__name__, ctx,
'Sending request: {}, request type: {}, request url: {}, retry number: {}.'
.format(cdm_request.request_id, request.method,
cdm_request._strip_sas_sig(),
retry_number), self._send_async_helper)
# Send the request and convert timeout to seconds from milliseconds.
with urllib.request.urlopen(
request, timeout=cdm_request.timeout /
1000) as response: # type: http.client.HTTPResponse
if response is not None:
end_time = datetime.now()
if ctx is not None:
logger.info(
self.__class__.__name__, ctx,
'Reponse for request {} received with elapsed time: {} ms.'
.format(
cdm_request.request_id,
(end_time - start_time).total_seconds() *
1000.0), self._send_async_helper)
cdm_response = CdmHttpResponse()
encoded_content = response.read()
# Check whether we have appropriate attributes on the object.
if hasattr(encoded_content, 'decode'):
cdm_response.content = encoded_content.decode(
'utf-8')
if hasattr(response, 'status'):
cdm_response.reason = response.reason
cdm_response.status_code = response.status
# Successful requests have HTTP standard status codes in the 2xx form.
cdm_response.is_successful = response.status // 100 == 2
if hasattr(response, 'getheaders'):
cdm_response.response_headers = dict(
response.getheaders())
except urllib.error.URLError as exception:
has_failed = True
if callback is None or retry_number == cdm_request.number_of_retries:
if retry_number != 0:
raise CdmNumberOfRetriesExceededException(exception)
else:
if exception.args and exception.args[
0].args and exception.args[0].args[
0] == 'timed out':
if ctx is not None:
logger.info(
self.__class__.__name__, ctx,
'Request {} timeout after {} s.'.format(
cdm_request.request_id,
cdm_request.timeout / 1000),
self._send_async_helper)
raise CdmTimedOutException('Request timeout.')
else:
raise exception
except Exception as exception:
has_failed = True
raise
# Check whether we have a callback function set and whether this is not our last retry.
if callback is not None and retry_number != cdm_request.number_of_retries:
# Call the callback function with the retry numbers starting from 1.
wait_time = callback(cdm_response, has_failed,
retry_number + 1) # type: int
if wait_time is None:
return cdm_response
# Convert from milliseconds to seconds and wait the time specified by the callback.
await asyncio.sleep(wait_time / 1000)
else:
# CDM Http Response exists, could be successful or bad (e.g. 403/404), it is up to caller to deal with it.
if cdm_response is not None:
return cdm_response
if retry_number == 0:
return None
raise CdmNumberOfRetriesExceededException()
raise CdmNumberOfRetriesExceededException()
def _construct_projection_context(
self, proj_directive: 'ProjectionDirective',
attr_ctx: 'CdmAttributeContext') -> 'ProjectionContext':
"""
A function to construct projection context and populate the resolved attribute set that ExtractResolvedAttributes method can then extract
This function is the entry point for projection resolution.
This function is expected to do the following 3 things:
- Create an condition expression tree & default if appropriate
- Create and initialize Projection Context
- Process operations
"""
proj_context = None
if not self.condition:
# if no condition is provided, get default condition and persist
self.condition = ConditionExpression._get_default_condition_expression(
self.operations, self.owner)
# create an expression tree based on the condition
tree = ExpressionTree()
self._condition_expression_tree_root = tree._construct_expression_tree(
self.condition)
if not self._condition_expression_tree_root:
logger.info(
self._TAG, self.ctx,
'Optional expression missing. Implicit expression will automatically apply.',
CdmProjection._construct_projection_context.__name__)
if attr_ctx:
# Add projection to context tree
acp_proj = AttributeContextParameters()
acp_proj._under = attr_ctx
acp_proj._type = CdmAttributeContextType.PROJECTION
acp_proj._name = self.fetch_object_definition_name()
acp_proj._regarding = proj_directive._owner_ref
acp_proj._include_traits = False
ac_proj = CdmAttributeContext._create_child_under(
proj_directive._res_opt, acp_proj)
acp_source = AttributeContextParameters()
acp_source._under = ac_proj
acp_source._type = CdmAttributeContextType.SOURCE
acp_source._name = 'source'
acp_source._regarding = None
acp_source._include_traits = False
ac_source = CdmAttributeContext._create_child_under(
proj_directive._res_opt, acp_source)
if self.source.fetch_object_definition(
proj_directive._res_opt
).object_type == CdmObjectType.PROJECTION_DEF:
# A Projection
proj_context = self.source.explicit_reference._construct_projection_context(
proj_directive, ac_source)
else:
# An Entity Reference
acp_source_projection = AttributeContextParameters()
acp_source_projection._under = ac_source
acp_source_projection._type = CdmAttributeContextType.ENTITY
acp_source_projection._name = self.source.named_reference if self.source.named_reference else self.source.explicit_reference.get_name(
)
acp_source_projection._regarding = self.source
acp_source_projection._include_traits = False
ras = self.source._fetch_resolved_attributes(
proj_directive._res_opt, acp_source_projection)
# Initialize the projection context
ctx = proj_directive._owner.ctx if proj_directive._owner else None
pas_set = None
# if polymorphic keep original source as previous state
poly_source_set = None
if proj_directive._is_source_polymorphic:
poly_source_set = ProjectionResolutionCommonUtil._get_polymorphic_source_set(
proj_directive, ctx, self.source,
acp_source_projection)
# now initialize projection attribute state
pas_set = ProjectionResolutionCommonUtil._initialize_projection_attribute_state_set(
proj_directive, ctx, ras,
proj_directive._is_source_polymorphic, poly_source_set)
proj_context = ProjectionContext(proj_directive,
ras.attribute_context)
proj_context._current_attribute_state_set = pas_set
is_condition_valid = False
if self._condition_expression_tree_root:
input = InputValues()
input.no_max_depth = proj_directive._has_no_maximum_depth
input.is_array = proj_directive._is_array
input.reference_only = proj_directive._is_reference_only
input.normalized = proj_directive._is_normalized
input.structured = proj_directive._is_structured
current_depth = proj_directive._current_depth
current_depth += 1
input.next_depth = current_depth
proj_directive._current_depth = current_depth
input.max_depth = proj_directive._maximum_depth
input.min_cardinality = proj_directive._cardinality._minimum_number if proj_directive._cardinality else None
input.max_cardinality = proj_directive._cardinality._maximum_number if proj_directive._cardinality else None
is_condition_valid = ExpressionTree._evaluate_expression_tree(
self._condition_expression_tree_root, input)
if is_condition_valid and self.operations and len(
self.operations) > 0:
# Just in case operations were added programmatically, reindex operations
for i in range(len(self.operations)):
self.operations[i]._index = i + 1
# Operation
acp_gen_attr_set = AttributeContextParameters()
acp_gen_attr_set._under = attr_ctx
acp_gen_attr_set._type = CdmAttributeContextType.GENERATED_SET
acp_gen_attr_set._name = '_generatedAttributeSet'
ac_gen_attr_set = CdmAttributeContext._create_child_under(
proj_directive._res_opt, acp_gen_attr_set)
# Start with an empty list for each projection
pas_operations = ProjectionAttributeStateSet(
proj_context._current_attribute_state_set._ctx)
for operation in self.operations:
# Evaluate projections and apply to empty state
new_pas_operations = operation._append_projection_attribute_state(
proj_context, pas_operations, ac_gen_attr_set)
# If the operations fails or it is not implemented the projection cannot be evaluated so keep previous valid state.
if new_pas_operations is not None:
pas_operations = new_pas_operations
# Finally update the current state to the projection context
proj_context._current_attribute_state_set = pas_operations
return proj_context
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition') \
-> 'CdmDocumentDefinition':
# go get the doc
doc_content = None # type: Optional[CdmDocumentDefinition]
json_data = None
fs_modified_time = None
doc_path = folder.folder_path + doc_name
adapter = self._ctx.corpus.storage.fetch_adapter(
folder.namespace) # type: StorageAdapter
try:
if adapter.can_read():
json_data = await adapter.read_async(doc_path)
fs_modified_time = await adapter.compute_last_modified_time_async(
adapter.create_adapter_path(doc_path))
logger.info(self._TAG, self._ctx,
'read file: {}'.format(doc_path),
self._load_document_from_path_async.__name__)
except Exception as e:
logger.error(
self._TAG, self._ctx,
'could not read {} from the \'{}\' namespace.\n Reason: \n{}'.
format(doc_path, folder.namespace,
e), self._load_document_from_path_async.__name__)
return None
if not doc_name:
logger.error(self._TAG, self._ctx,
'Document name cannot be null or empty.',
self._load_document_from_path_async.__name__)
return None
# If loading an odi.json/model.json file, check that it is named correctly.
if doc_name.lower().endswith(
self._ODI_EXTENSION
) and not doc_name.lower() == self._ODI_EXTENSION:
logger.error(
self._TAG, self._ctx,
'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'
.format(doc_name, self._ODI_EXTENSION),
self._load_document_from_path_async.__name__)
return None
if doc_name.lower().endswith(
self._MODEL_JSON_EXTENSION
) and not doc_name.lower() == self._MODEL_JSON_EXTENSION:
logger.error(
self._TAG, self._ctx,
'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.'
.format(doc_name, self._MODEL_JSON_EXTENSION),
self._load_document_from_path_async.__name__)
return None
# Fetch the correct persistence class to use.
persistence_class = self._fetch_registered_persistence_format(doc_name)
if persistence_class:
try:
method = persistence_class.from_data
parameters = [self._ctx, doc_name, json_data, folder]
# check if from_data() is asynchronous for this persistence class.
if persistence_class not in self._is_registered_persistence_async:
# Cache whether this persistence class has async methods.
self._is_registered_persistence_async[
persistence_class] = persistence_class.is_persistence_async
if self._is_registered_persistence_async[persistence_class]:
doc_content = await method(*parameters)
else:
doc_content = method(*parameters)
except Exception as e:
logger.error(
self._TAG, self._ctx,
'Could not convert \'{}\'. Reason \'{}\''.format(
doc_name,
e), self._load_document_from_path_async.__name__)
return None
else:
# could not find a registered persistence class to handle this document type.
logger.error(
self._TAG, self._ctx,
'Could not find a persistence class to handle the file \'{}\''.
format(doc_name), self._load_document_from_path_async.__name__)
return None
# add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content
if doc_content is not None:
if doc_container:
# there are situations where a previously loaded document must be re-loaded.
# the end of that chain of work is here where the old version of the document has been removed from
# the corpus and we have created a new document and loaded it from storage and after this call we will probably
# add it to the corpus and index it, etc.
# it would be really rude to just kill that old object and replace it with this replicant, especially because
# the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that
# was just killed off but make it contain everything the new document loaded.
doc_content = doc_content.copy(
ResolveOptions(wrt_doc=doc_container), doc_container)
folder.documents.append(doc_content, doc_name)
doc_content._file_system_modified_time = fs_modified_time
doc_content._is_dirty = False
return doc_content
