def _declare_object_definitions(self) -> None: """Indexes all definitions contained by this document.""" skip_duplicate_types = {CdmObjectType.CONSTANT_ENTITY_DEF} internal_declaration_types = { CdmObjectType.ENTITY_DEF, CdmObjectType.PARAMETER_DEF, CdmObjectType.TRAIT_DEF, CdmObjectType.TRAIT_GROUP_DEF, CdmObjectType.PURPOSE_DEF, CdmObjectType.DATA_TYPE_DEF, CdmObjectType.TYPE_ATTRIBUTE_DEF, CdmObjectType.ENTITY_ATTRIBUTE_DEF, CdmObjectType.ATTRIBUTE_GROUP_DEF, CdmObjectType.CONSTANT_ENTITY_DEF, CdmObjectType.ATTRIBUTE_CONTEXT_DEF, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF, CdmObjectType.ATTRIBUTE_GROUP_DEF, CdmObjectType.PROJECTION_DEF, CdmObjectType.OPERATION_ADD_COUNT_ATTRIBUTE_DEF, CdmObjectType.OPERATION_ADD_SUPPORTING_ATTRIBUTE_DEF, CdmObjectType.OPERATION_ADD_TYPE_ATTRIBUTE_DEF, CdmObjectType.OPERATION_EXCLUDE_ATTRIBUTES_DEF, CdmObjectType.OPERATION_ARRAY_EXPANSION_DEF, CdmObjectType.OPERATION_COMBINE_ATTRIBUTES_DEF, CdmObjectType.OPERATION_RENAME_ATTRIBUTES_DEF, CdmObjectType.OPERATION_REPLACE_AS_FOREIGN_KEY_DEF, CdmObjectType.OPERATION_INCLUDE_ATTRIBUTES_DEF, CdmObjectType.OPERATION_ADD_ATTRIBUTE_GROUP_DEF, CdmObjectType.OPERATION_ALTER_TRAITS_DEF, CdmObjectType.OPERATION_ADD_ARTIFACT_ATTRIBUTE_DEF } corpus_path_root = self._folder_path + self.name for obj in self._internal_objects: # I can't think of a better time than now to make sure any recently changed or added things have an in doc obj.in_document = self obj_path = obj._declared_path if '(unspecified)' in obj_path: continue skip_duplicates = False if obj.object_type in skip_duplicate_types: # if there is a duplicate, don't complain, the path just finds the first one skip_duplicates = True if obj.object_type in internal_declaration_types: corpus_path = '{}/{}'.format(corpus_path_root, obj_path) if obj_path in self.internal_declarations and not skip_duplicates: logger.error(self.ctx, self._TAG, '_declare_object_definitions', corpus_path, CdmLogCode.ERR_PATH_IS_DUPLICATE, corpus_path) else: self.internal_declarations[obj_path] = obj self.ctx.corpus._register_symbol(obj_path, self) logger.info(self.ctx, self._TAG, self._declare_object_definitions.__name__, corpus_path, 'declared \'{}\''.format(obj_path))
def _resolve_object_definitions(self, res_opt: 'ResolveOptions') -> None: ctx = self.ctx res_opt._indexing_doc = self reference_type_set = { CdmObjectType.ATTRIBUTE_REF, CdmObjectType.ATTRIBUTE_GROUP_REF, CdmObjectType.ATTRIBUTE_CONTEXT_REF, CdmObjectType.DATA_TYPE_REF, CdmObjectType.ENTITY_REF, CdmObjectType.PURPOSE_REF, CdmObjectType.TRAIT_REF } for obj in self._internal_objects: if obj.object_type in reference_type_set: ctx._relative_path = obj._declared_path if obj._offset_attribute_promise(obj.named_reference) < 0: res_new = obj.fetch_object_definition(res_opt) if not res_new: # it's okay if references can't be resolved when shallow validation is enabled. if res_opt.shallow_validation: logger.warning( self.ctx, self._TAG, self._resolve_object_definitions.__name__, self.at_corpus_path, CdmLogCode.WARN_RESOLVE_REFERENCE_FAILURE, obj.named_reference) else: logger.error( self.ctx, self._TAG, self._resolve_object_definitions.__name__, self.at_corpus_path, CdmLogCode.ERR_RESOLVE_REFERENCE_FAILURE, obj.named_reference) # don't check in this file without both of these comments. handy for debug of failed lookups # res_test = obj.fetch_object_definition(res_opt) else: logger.info( self.ctx, self._TAG, self._resolve_object_definitions.__name__, self.at_corpus_path, 'resolved \'{}\''.format(obj.named_reference)) elif obj.object_type == CdmObjectType.PARAMETER_DEF: # when a parameter has a datatype that is a cdm object, validate that any default value is the # right kind object parameter = obj # type: CdmParameterDefinition parameter._const_type_check(res_opt, self, None) res_opt._indexing_doc = None
def _check_integrity(self) -> bool: """Validates all the objects in this document.""" error_count = 0 for obj in self._internal_objects: if not obj.validate(): error_count += 1 else: obj.ctx = self.ctx logger.info(self.ctx, self._TAG, self._check_integrity.__name__, self.at_corpus_path, 'checked \'{}\''.format(obj.at_corpus_path)) self._is_valid = error_count == 0
def register_format(self, persistence_class_name: str, assembly_name: Optional[str] = None) -> None: try: path_split = persistence_class_name.split('.') class_path = '.'.join(path_split[:-1]) class_name = path_split[-1] persistence_module = importlib.import_module(class_path) persistence_class = getattr(persistence_module, class_name) formats = persistence_class.formats # type: List[str] for form in formats: self._registered_persistence_formats[form] = persistence_class except Exception as e: logger.info(self._TAG, self._ctx, 'Unable to register persistence class {}. Reason: {}.'.format(persistence_class_name, e))
def enable(self) -> None: """ Enable the telemetry client by starting a thread for ingestion. """ # Check if the Kusto config and the concurrent queue has been initialized if self._config is None or self._request_queue is None: message = 'The telemetry client has not been initialized.' logger.info(self._ctx, TelemetryKustoClient.__name__, self.enable.__name__, None, message) return # Starts a separate thread to ingest telemetry logs into Kusto ingestion_thread = threading.Thread(target=self._ingest_request_queue, daemon=True) ingestion_thread.start()
def _const_type_check(self, res_opt: 'ResolveOptions', wrt_doc: 'CdmDocumentDefinition', argument_value: Any) -> 'CdmArgumentValue': ctx = self.ctx replacement = argument_value # if parameter type is entity, then the value should be an entity or ref to one # same is true of 'dataType' dataType ref = self.data_type_ref if not ref: return replacement dt = ref.fetch_object_definition(res_opt) if not dt: logger.error(self.ctx, self._TAG, '_const_type_check', self.at_corpus_path, CdmLogCode.ERR_UNRECOGNIZED_DATA_TYPE, self.name) return None # compare with passed in value or default for parameter p_value = argument_value if p_value is None: p_value = self.default_value replacement = p_value if p_value is not None: if dt.is_derived_from('cdmObject', res_opt): expected_types = [] expected = None if dt.is_derived_from('entity', res_opt): expected_types.append(CdmObjectType.CONSTANT_ENTITY_DEF) expected_types.append(CdmObjectType.ENTITY_REF) expected_types.append(CdmObjectType.ENTITY_DEF) expected_types.append(CdmObjectType.PROJECTION_DEF) expected = 'entity' elif dt.is_derived_from('attribute', res_opt): expected_types.append(CdmObjectType.ATTRIBUTE_REF) expected_types.append(CdmObjectType.TYPE_ATTRIBUTE_DEF) expected_types.append(CdmObjectType.ENTITY_ATTRIBUTE_DEF) expected = 'attribute' elif dt.is_derived_from('dataType', res_opt): expected_types.append(CdmObjectType.DATA_TYPE_REF) expected_types.append(CdmObjectType.DATA_TYPE_DEF) expected = 'dataType' elif dt.is_derived_from('purpose', res_opt): expected_types.append(CdmObjectType.PURPOSE_REF) expected_types.append(CdmObjectType.PURPOSE_DEF) expected = 'purpose' elif dt.is_derived_from('trait', res_opt): expected_types.append(CdmObjectType.TRAIT_REF) expected_types.append(CdmObjectType.TRAIT_DEF) expected = 'trait' elif dt.is_derived_from('traitGroup', res_opt): expected_types.append(CdmObjectType.TRAIT_GROUP_REF) expected_types.append(CdmObjectType.TRAIT_GROUP_DEF) expected = 'traitGroup' elif dt.is_derived_from('attributeGroup', res_opt): expected_types.append(CdmObjectType.ATTRIBUTE_GROUP_REF) expected_types.append(CdmObjectType.ATTRIBUTE_GROUP_DEF) expected = 'attributeGroup' if not expected_types: logger.error(self.ctx, self._TAG, '_const_type_check', wrt_doc.at_corpus_path, CdmLogCode.ERR_UNEXPECTED_DATA_TYPE, self.name) # if a string constant, resolve to an object ref. found_type = CdmObjectType.ERROR if isinstance(p_value, CdmObject): found_type = p_value.object_type found_desc = ctx._relative_path if isinstance(p_value, str): if p_value == 'this.attribute' and expected == 'attribute': # will get sorted out later when resolving traits found_type = CdmObjectType.ATTRIBUTE_REF else: found_desc = p_value from cdm.objectmodel import CdmObjectReference seek_res_att = CdmObjectReference._offset_attribute_promise(p_value) if seek_res_att >= 0: # get an object there that will get resolved later after resolved attributes replacement = CdmAttributeReference(self.ctx, p_value, True) replacement.in_document = wrt_doc found_type = CdmObjectType.ATTRIBUTE_REF else: lu = ctx.corpus._resolve_symbol_reference(res_opt, wrt_doc, p_value, CdmObjectType.ERROR, True) if lu: if expected == 'attribute': replacement = CdmAttributeReference(self.ctx, p_value, True) replacement.in_document = wrt_doc found_type = CdmObjectType.ATTRIBUTE_REF else: replacement = lu if isinstance(replacement, CdmObject): found_type = replacement.object_type if expected_types.index(found_type) == -1: logger.error(self.ctx, self._TAG, wrt_doc.at_corpus_path, CdmLogCode.ERR_RESOLUTION_FAILURE, self.get_name(), expected, found_desc) else: logger.info(self.ctx, self._TAG, self._const_type_check.__name__, wrt_doc.at_corpus_path, 'resolved \'{}\''.format(found_desc)) return replacement
def _localize_corpus_paths(self, new_folder: 'CdmFolderDefinition') -> bool: all_went_well = True logger.info( self.ctx, self._TAG, self._localize_corpus_paths.__name__, new_folder.at_corpus_path, 'Localizing corpus paths in document \'{}\''.format(self.name)) def import_callback(obj: 'CdmObject', path: str) -> bool: nonlocal all_went_well corpus_path, worked = self._localize_corpus_path( obj.corpus_path, new_folder) if not worked: all_went_well = False else: obj.corpus_path = corpus_path def entity_declaration_definition_callback(obj: 'CdmObject', path: str) -> bool: nonlocal all_went_well corpus_path, worked = self._localize_corpus_path( obj.entity_path, new_folder) if not worked: all_went_well = False else: obj.entity_path = corpus_path def data_partition_callback(obj: 'CdmObject', path: str) -> bool: nonlocal all_went_well corpus_path, worked = self._localize_corpus_path( obj.location, new_folder) if not worked: all_went_well = False else: obj.location = corpus_path corpus_path, worked = self._localize_corpus_path( obj.specialized_schema, new_folder) if not worked: all_went_well = False else: obj.specialized_schema = corpus_path def data_partition_pattern_callback(obj: 'CdmObject', path: str) -> bool: nonlocal all_went_well corpus_path, worked = self._localize_corpus_path( obj.root_location, new_folder) if not worked: all_went_well = False else: obj.root_location = corpus_path corpus_path, worked = self._localize_corpus_path( obj.specialized_schema, new_folder) if not worked: all_went_well = False else: obj.specialized_schema = corpus_path def e2e_relationship_callback(obj: 'CdmObject', path: str) -> bool: nonlocal all_went_well corpus_path, worked = self._localize_corpus_path( obj.to_entity, new_folder) if not worked: all_went_well = False else: obj.to_entity = corpus_path corpus_path, worked = self._localize_corpus_path( obj.from_entity, new_folder) if not worked: all_went_well = False else: obj.from_entity = corpus_path def manifest_declaration_callback(obj: 'CdmObject', path: str) -> bool: nonlocal all_went_well corpus_path, worked = self._localize_corpus_path( obj.definition, new_folder) if not worked: all_went_well = False else: obj.definition = corpus_path switcher = { CdmObjectType.IMPORT: import_callback, CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF: entity_declaration_definition_callback, CdmObjectType.REFERENCED_ENTITY_DECLARATION_DEF: entity_declaration_definition_callback, CdmObjectType.DATA_PARTITION_DEF: data_partition_callback, CdmObjectType.DATA_PARTITION_PATTERN_DEF: data_partition_pattern_callback, CdmObjectType.E2E_RELATIONSHIP_DEF: e2e_relationship_callback, CdmObjectType.MANIFEST_DECLARATION_DEF: manifest_declaration_callback } def pre_callback(obj: 'CdmObject', path: str) -> bool: # i don't like that document needs to know a little about these objects # in theory, we could create a virtual function on cdmObject that localizes properties # but then every object would need to know about the documents and paths and such ... # also, i already wrote this code. func = switcher.get(obj.object_type) if func: func(obj, path) return False # find anything in the document that is a corpus path self.visit('', pre_callback, None) return all_went_well
async def _send_async_helper( self, cdm_request: 'CdmHttpRequest', callback=None, ctx: Optional['CdmCorpusContext'] = None) -> 'CdmHttpResponse': """ Sends a CDM request with the retry logic helper function. :param cdm_request: The CDM Http request. :param callback: An optional parameter which specifies a callback function that gets executed after we try to execute the HTTP request. :return: The Cdm Http response. """ full_url = None # type : str if self._api_endpoint is not None: full_url = self._combine_urls(self._api_endpoint, cdm_request.requested_url) else: full_url = cdm_request.requested_url data = None # type: json # Set the content to be in the proper form and headers to denote # the content type. if cdm_request.content is not None: data = cdm_request.content cdm_request.headers['Content-Type'] = cdm_request.content_type # urllib.request.Request() expects 'data' to be in bytes, so we convert to bytes here. if data is not None: data = data.encode("utf-8") request = urllib.request.Request(full_url, method=cdm_request.method, data=data) for key in cdm_request.headers: request.add_header(key, cdm_request.headers[key]) # If the number of retries is 0, we only try once, otherwise we retry the specified # number of times. for retry_number in range(cdm_request.number_of_retries + 1): cdm_response = None # type: CdmHttpResponse has_failed = False # type: bool try: start_time = datetime.now() if ctx is not None: logger.info( self.__class__.__name__, ctx, 'Sending request: {}, request type: {}, request url: {}, retry number: {}.' .format(cdm_request.request_id, request.method, cdm_request._strip_sas_sig(), retry_number), self._send_async_helper) # Send the request and convert timeout to seconds from milliseconds. with urllib.request.urlopen( request, timeout=cdm_request.timeout / 1000) as response: # type: http.client.HTTPResponse if response is not None: end_time = datetime.now() if ctx is not None: logger.info( self.__class__.__name__, ctx, 'Reponse for request {} received with elapsed time: {} ms.' .format( cdm_request.request_id, (end_time - start_time).total_seconds() * 1000.0), self._send_async_helper) cdm_response = CdmHttpResponse() encoded_content = response.read() # Check whether we have appropriate attributes on the object. if hasattr(encoded_content, 'decode'): cdm_response.content = encoded_content.decode( 'utf-8') if hasattr(response, 'status'): cdm_response.reason = response.reason cdm_response.status_code = response.status # Successful requests have HTTP standard status codes in the 2xx form. cdm_response.is_successful = response.status // 100 == 2 if hasattr(response, 'getheaders'): cdm_response.response_headers = dict( response.getheaders()) except urllib.error.URLError as exception: has_failed = True if callback is None or retry_number == cdm_request.number_of_retries: if retry_number != 0: raise CdmNumberOfRetriesExceededException(exception) else: if exception.args and exception.args[ 0].args and exception.args[0].args[ 0] == 'timed out': if ctx is not None: logger.info( self.__class__.__name__, ctx, 'Request {} timeout after {} s.'.format( cdm_request.request_id, cdm_request.timeout / 1000), self._send_async_helper) raise CdmTimedOutException('Request timeout.') else: raise exception except Exception as exception: has_failed = True raise # Check whether we have a callback function set and whether this is not our last retry. if callback is not None and retry_number != cdm_request.number_of_retries: # Call the callback function with the retry numbers starting from 1. wait_time = callback(cdm_response, has_failed, retry_number + 1) # type: int if wait_time is None: return cdm_response # Convert from milliseconds to seconds and wait the time specified by the callback. await asyncio.sleep(wait_time / 1000) else: # CDM Http Response exists, could be successful or bad (e.g. 403/404), it is up to caller to deal with it. if cdm_response is not None: return cdm_response if retry_number == 0: return None raise CdmNumberOfRetriesExceededException() raise CdmNumberOfRetriesExceededException()
def _construct_projection_context( self, proj_directive: 'ProjectionDirective', attr_ctx: 'CdmAttributeContext') -> 'ProjectionContext': """ A function to construct projection context and populate the resolved attribute set that ExtractResolvedAttributes method can then extract This function is the entry point for projection resolution. This function is expected to do the following 3 things: - Create an condition expression tree & default if appropriate - Create and initialize Projection Context - Process operations """ proj_context = None if not self.condition: # if no condition is provided, get default condition and persist self.condition = ConditionExpression._get_default_condition_expression( self.operations, self.owner) # create an expression tree based on the condition tree = ExpressionTree() self._condition_expression_tree_root = tree._construct_expression_tree( self.condition) if not self._condition_expression_tree_root: logger.info( self._TAG, self.ctx, 'Optional expression missing. Implicit expression will automatically apply.', CdmProjection._construct_projection_context.__name__) if attr_ctx: # Add projection to context tree acp_proj = AttributeContextParameters() acp_proj._under = attr_ctx acp_proj._type = CdmAttributeContextType.PROJECTION acp_proj._name = self.fetch_object_definition_name() acp_proj._regarding = proj_directive._owner_ref acp_proj._include_traits = False ac_proj = CdmAttributeContext._create_child_under( proj_directive._res_opt, acp_proj) acp_source = AttributeContextParameters() acp_source._under = ac_proj acp_source._type = CdmAttributeContextType.SOURCE acp_source._name = 'source' acp_source._regarding = None acp_source._include_traits = False ac_source = CdmAttributeContext._create_child_under( proj_directive._res_opt, acp_source) if self.source.fetch_object_definition( proj_directive._res_opt ).object_type == CdmObjectType.PROJECTION_DEF: # A Projection proj_context = self.source.explicit_reference._construct_projection_context( proj_directive, ac_source) else: # An Entity Reference acp_source_projection = AttributeContextParameters() acp_source_projection._under = ac_source acp_source_projection._type = CdmAttributeContextType.ENTITY acp_source_projection._name = self.source.named_reference if self.source.named_reference else self.source.explicit_reference.get_name( ) acp_source_projection._regarding = self.source acp_source_projection._include_traits = False ras = self.source._fetch_resolved_attributes( proj_directive._res_opt, acp_source_projection) # Initialize the projection context ctx = proj_directive._owner.ctx if proj_directive._owner else None pas_set = None # if polymorphic keep original source as previous state poly_source_set = None if proj_directive._is_source_polymorphic: poly_source_set = ProjectionResolutionCommonUtil._get_polymorphic_source_set( proj_directive, ctx, self.source, acp_source_projection) # now initialize projection attribute state pas_set = ProjectionResolutionCommonUtil._initialize_projection_attribute_state_set( proj_directive, ctx, ras, proj_directive._is_source_polymorphic, poly_source_set) proj_context = ProjectionContext(proj_directive, ras.attribute_context) proj_context._current_attribute_state_set = pas_set is_condition_valid = False if self._condition_expression_tree_root: input = InputValues() input.no_max_depth = proj_directive._has_no_maximum_depth input.is_array = proj_directive._is_array input.reference_only = proj_directive._is_reference_only input.normalized = proj_directive._is_normalized input.structured = proj_directive._is_structured current_depth = proj_directive._current_depth current_depth += 1 input.next_depth = current_depth proj_directive._current_depth = current_depth input.max_depth = proj_directive._maximum_depth input.min_cardinality = proj_directive._cardinality._minimum_number if proj_directive._cardinality else None input.max_cardinality = proj_directive._cardinality._maximum_number if proj_directive._cardinality else None is_condition_valid = ExpressionTree._evaluate_expression_tree( self._condition_expression_tree_root, input) if is_condition_valid and self.operations and len( self.operations) > 0: # Just in case operations were added programmatically, reindex operations for i in range(len(self.operations)): self.operations[i]._index = i + 1 # Operation acp_gen_attr_set = AttributeContextParameters() acp_gen_attr_set._under = attr_ctx acp_gen_attr_set._type = CdmAttributeContextType.GENERATED_SET acp_gen_attr_set._name = '_generatedAttributeSet' ac_gen_attr_set = CdmAttributeContext._create_child_under( proj_directive._res_opt, acp_gen_attr_set) # Start with an empty list for each projection pas_operations = ProjectionAttributeStateSet( proj_context._current_attribute_state_set._ctx) for operation in self.operations: # Evaluate projections and apply to empty state new_pas_operations = operation._append_projection_attribute_state( proj_context, pas_operations, ac_gen_attr_set) # If the operations fails or it is not implemented the projection cannot be evaluated so keep previous valid state. if new_pas_operations is not None: pas_operations = new_pas_operations # Finally update the current state to the projection context proj_context._current_attribute_state_set = pas_operations return proj_context
async def _load_document_from_path_async(self, folder: 'CdmFolderDefinition', doc_name: str, doc_container: 'CdmDocumentDefinition') \ -> 'CdmDocumentDefinition': # go get the doc doc_content = None # type: Optional[CdmDocumentDefinition] json_data = None fs_modified_time = None doc_path = folder.folder_path + doc_name adapter = self._ctx.corpus.storage.fetch_adapter( folder.namespace) # type: StorageAdapter try: if adapter.can_read(): json_data = await adapter.read_async(doc_path) fs_modified_time = await adapter.compute_last_modified_time_async( adapter.create_adapter_path(doc_path)) logger.info(self._TAG, self._ctx, 'read file: {}'.format(doc_path), self._load_document_from_path_async.__name__) except Exception as e: logger.error( self._TAG, self._ctx, 'could not read {} from the \'{}\' namespace.\n Reason: \n{}'. format(doc_path, folder.namespace, e), self._load_document_from_path_async.__name__) return None if not doc_name: logger.error(self._TAG, self._ctx, 'Document name cannot be null or empty.', self._load_document_from_path_async.__name__) return None # If loading an odi.json/model.json file, check that it is named correctly. if doc_name.lower().endswith( self._ODI_EXTENSION ) and not doc_name.lower() == self._ODI_EXTENSION: logger.error( self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.' .format(doc_name, self._ODI_EXTENSION), self._load_document_from_path_async.__name__) return None if doc_name.lower().endswith( self._MODEL_JSON_EXTENSION ) and not doc_name.lower() == self._MODEL_JSON_EXTENSION: logger.error( self._TAG, self._ctx, 'Failed to load \'{}\', as it\'s not an acceptable file name. It must be {}.' .format(doc_name, self._MODEL_JSON_EXTENSION), self._load_document_from_path_async.__name__) return None # Fetch the correct persistence class to use. persistence_class = self._fetch_registered_persistence_format(doc_name) if persistence_class: try: method = persistence_class.from_data parameters = [self._ctx, doc_name, json_data, folder] # check if from_data() is asynchronous for this persistence class. if persistence_class not in self._is_registered_persistence_async: # Cache whether this persistence class has async methods. self._is_registered_persistence_async[ persistence_class] = persistence_class.is_persistence_async if self._is_registered_persistence_async[persistence_class]: doc_content = await method(*parameters) else: doc_content = method(*parameters) except Exception as e: logger.error( self._TAG, self._ctx, 'Could not convert \'{}\'. Reason \'{}\''.format( doc_name, e), self._load_document_from_path_async.__name__) return None else: # could not find a registered persistence class to handle this document type. logger.error( self._TAG, self._ctx, 'Could not find a persistence class to handle the file \'{}\''. format(doc_name), self._load_document_from_path_async.__name__) return None # add document to the folder, this sets all the folder/path things, caches name to content association and may trigger indexing on content if doc_content is not None: if doc_container: # there are situations where a previously loaded document must be re-loaded. # the end of that chain of work is here where the old version of the document has been removed from # the corpus and we have created a new document and loaded it from storage and after this call we will probably # add it to the corpus and index it, etc. # it would be really rude to just kill that old object and replace it with this replicant, especially because # the caller has no idea this happened. so... sigh ... instead of returning the new object return the one that # was just killed off but make it contain everything the new document loaded. doc_content = doc_content.copy( ResolveOptions(wrt_doc=doc_container), doc_container) folder.documents.append(doc_content, doc_name) doc_content._file_system_modified_time = fs_modified_time doc_content._is_dirty = False return doc_content