def from_data(ctx: CdmCorpusContext, data: TypeAttribute, entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition: type_attribute = ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, data.get('name')) type_attribute.purpose = PurposeReferencePersistence.from_data(ctx, data.get('purpose')) type_attribute.data_type = DataTypeReferencePersistence.from_data(ctx, data.get('dataType')) type_attribute.attribute_context = AttributeContextReferencePersistence.from_data(ctx, data.get('attributeContext')) type_attribute.resolution_guidance = AttributeResolutionGuidancePersistence.from_data(ctx, data.get('resolutionGuidance')) applied_traits = utils.create_trait_reference_array(ctx, data.get('appliedTraits')) type_attribute.applied_traits.extend(applied_traits) if data.get('isPrimaryKey') and entity_name: t2p_map = TraitToPropertyMap(type_attribute) t2p_map._update_property_value('isPrimaryKey', entity_name + '/(resolvedAttributes)/' + type_attribute.name) type_attribute.explanation = data.get('explanation') if data.get('isReadOnly') is not None: type_attribute.is_read_only = TypeAttributePersistence._property_from_data_to_bool(data.isReadOnly) if data.get('isNullable') is not None: type_attribute.is_nullable = TypeAttributePersistence._property_from_data_to_bool(data.isNullable) if data.get('sourceName'): type_attribute.source_name = TypeAttributePersistence._property_from_data_to_string(data.sourceName) if data.get('sourceOrdering') is not None: type_attribute.source_ordering = TypeAttributePersistence._property_from_data_to_int(data.sourceOrdering) if data.get('displayName'): type_attribute.display_name = TypeAttributePersistence._property_from_data_to_string(data.displayName) if data.get('description'): type_attribute.description = TypeAttributePersistence._property_from_data_to_string(data.description) if data.get('valueConstrainedToList') is not None: type_attribute.value_constrained_to_list = TypeAttributePersistence._property_from_data_to_bool(data.valueConstrainedToList) if data.get('maximumLength') is not None: type_attribute.maximum_length = TypeAttributePersistence._property_from_data_to_int(data.maximumLength) if data.get('maximumValue') is not None: type_attribute.maximum_value = TypeAttributePersistence._property_from_data_to_string(data.maximumValue) if data.get('minimumValue') is not None: type_attribute.minimum_value = TypeAttributePersistence._property_from_data_to_string(data.minimumValue) if data.get('dataFormat') is not None: try: type_attribute.data_format = CdmDataFormat(data.dataFormat) except ValueError: logger.warning(TypeAttributePersistence.__name__, ctx, 'Couldn\'t find an enum value for {}.'.format( data.dataFormat), TypeAttributePersistence.from_data.__name__) if data.get('defaultValue') is not None: type_attribute.default_value = data.defaultValue return type_attribute
def test_update_and_fetch_list_lookup(self): """Test update and fetch list lookup default value without attributeValue and displayOrder.""" corpus = CdmCorpusDefinition() cdm_attribute = CdmTypeAttributeDefinition(corpus.ctx, 'SomeAttribute') trait_to_property_map = TraitToPropertyMap(cdm_attribute) constant_values = [{'languageTag': 'en', 'displayText': 'Fax'}] trait_to_property_map._update_property_value('defaultValue', constant_values) result = trait_to_property_map._fetch_property_value('defaultValue') self.assertEqual(1, len(result)) self.assertEqual('en', result[0].get('languageTag')) self.assertEqual('Fax', result[0].get('displayText')) self.assertIsNone(result[0].get('attributeValue')) self.assertIsNone(result[0].get('displayOrder'))
def from_data(ctx: CdmCorpusContext, obj: DataColumn, entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition: type_attribute = ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, obj.name) properties = obj.origin_data_type_name.properties type_attribute.data_format = utils.syms_data_type_to_cdm_data_format(obj.origin_data_type_name) if obj.origin_data_type_name.scale != 0 or obj.origin_data_type_name.precision != 0: numeric_traits = ctx.corpus.make_ref(CdmObjectType.TRAIT_REF, 'is.data_format.numeric.shaped', True) scale_traits_arg = ctx.corpus.make_ref(CdmObjectType.ARGUMENT_DEF, 'scale', False) scale_traits_arg.Value = obj.origin_data_type_name.scale numeric_traits.arguments.append(scale_traits_arg) precision_traits_arg = ctx.corpus.make_ref(CdmObjectType.ARGUMENT_DEF, 'scale', False) precision_traits_arg.value = obj.origin_data_type_name.scale numeric_traits.arguments.append(precision_traits_arg) if properties is not None: if 'cdm:purpose' in properties: type_attribute.purpose = properties['cdm:purpose'] if 'cdm:dataType' in properties: type_attribute.data_type = properties['cdm:dataType'] if 'cdm:traits' in properties: utils.add_list_to_cdm_collection(type_attribute.applied_traits, utils.create_trait_reference_array(ctx, properties['cdm:traits'])) if 'cdm:is_primary_key' in properties and properties['cdm:is_primary_key']: t2pMap = TraitToPropertyMap(type_attribute) t2pMap._update_property_value('is_primary_key', entity_name + '/(resolvedAttributes)/' + type_attribute.name) if 'cdm:isReadOnly' in properties: type_attribute.is_read_only = properties['cdm:isReadOnly'] if 'cdm:sourceName' in properties: type_attribute.source_name = properties['cdm:source_name'] if 'cdm:sourceOrdering' in properties: type_attribute.source_ordering = properties['cdm:sourceOrdering'] if 'cdm:valueConstrainedToList' in properties: type_attribute.value_constrained_to_list = properties['cdm:valueConstrainedToList'] if 'cdm:maximumLength' in properties: type_attribute.maximum_length = properties['cdm:maximumLength'] if 'cdm:maximumValue' in properties: type_attribute.maximum_value = properties['cdm:maximumValue'] if 'cdm:minimumValue' in properties: type_attribute.minimum_value = properties['cdm:minimumValue'] if 'cdm:defaultValue' in properties: type_attribute.default_value = properties['cdm:defaultValue'] return type_attribute
def from_data(ctx: CdmCorpusContext, data: TypeAttribute, entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition: type_attribute = ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, data.get('name')) type_attribute.purpose = PurposeReferencePersistence.from_data(ctx, data.get('purpose')) type_attribute.data_type = DataTypeReferencePersistence.from_data(ctx, data.get('dataType')) cardinality = utils.cardinality_settings_from_data(data.get('cardinality'), type_attribute) if cardinality is not None: type_attribute.cardinality = cardinality type_attribute.attribute_context = AttributeContextReferencePersistence.from_data(ctx, data.get('attributeContext')) utils.add_list_to_cdm_collection(type_attribute.applied_traits, utils.create_trait_reference_array(ctx, data.get('appliedTraits'))) type_attribute.resolution_guidance = AttributeResolutionGuidancePersistence.from_data(ctx, data.get('resolutionGuidance')) if data.get('isPrimaryKey') and entity_name: t2p_map = TraitToPropertyMap(type_attribute) t2p_map._update_property_value('isPrimaryKey', entity_name + '/(resolvedAttributes)/' + type_attribute.name) type_attribute.explanation = data.explanation type_attribute.is_read_only = utils._property_from_data_to_bool(data.isReadOnly) type_attribute.is_nullable = utils._property_from_data_to_bool(data.isNullable) type_attribute.source_name = utils._property_from_data_to_string(data.sourceName) type_attribute.source_ordering = utils._property_from_data_to_int(data.sourceOrdering) type_attribute.display_name = utils._property_from_data_to_string(data.displayName) type_attribute.description = utils._property_from_data_to_string(data.description) type_attribute.value_constrained_to_list = utils._property_from_data_to_bool(data.valueConstrainedToList) type_attribute.maximum_length = utils._property_from_data_to_int(data.maximumLength) type_attribute.maximum_value = utils._property_from_data_to_string(data.maximumValue) type_attribute.minimum_value = utils._property_from_data_to_string(data.minimumValue) type_attribute.default_value = data.defaultValue type_attribute.projection = ProjectionPersistence.from_data(ctx, data.projection) if data.get('dataFormat') is not None: try: type_attribute.data_format = TypeAttributePersistence._data_type_from_data(data.dataFormat) except ValueError: logger.warning(ctx, _TAG, TypeAttributePersistence.from_data.__name__, None, CdmLogCode.WARN_PERSIST_ENUM_NOT_FOUND, data.dataFormat) return type_attribute
class CdmDataPartitionDefinition(CdmObjectDefinition, CdmFileStatus): def __init__(self, ctx: 'CdmCorpusContext', name: str) -> None: super().__init__(ctx) self._TAG = CdmDataPartitionDefinition.__name__ # The name of a data partition. self.name = name # type: str # The corpus path for the data file location. self.location = None # type: Optional[str] # Indicates whether this partition is inferred. self.inferred = False # type: bool # The list of key value pairs to give names for the replacement values from the RegEx. self.arguments = {} # type: Dict[str, List[str]] # The path of a specialized schema to use specifically for the partitions generated. self.specialized_schema = None # type: Optional[str] # The refresh time of the partition. self.refresh_time = None # type: Optional[datetime] self.last_file_modified_time = None # type: Optional[datetime] self.last_file_status_check_time = None # type: Optional[datetime] # --- internal --- self._ttpm = TraitToPropertyMap(self) @property def object_type(self) -> 'CdmObjectType': return CdmObjectType.DATA_PARTITION_DEF @property def description(self) -> str: return cast(str, self._ttpm._fetch_property_value('description')) @description.setter def description(self, val: str) -> None: self._ttpm._update_property_value('description', val) @property def last_child_file_modified_time(self) -> datetime: raise NotImplementedError() @last_child_file_modified_time.setter def last_child_file_modified_time(self, val: datetime): raise NotImplementedError() def copy( self, res_opt: Optional['ResolveOptions'] = None, host: Optional['CdmDataPartitionDefinition'] = None ) -> 'CdmDataPartitionDefinition': if not res_opt: res_opt = ResolveOptions( wrt_doc=self, directives=self.ctx.corpus.default_resolution_directives) if not host: copy = CdmDataPartitionDefinition(self.ctx, self.name) else: copy = host copy.name = self.name copy.description = self.description copy.location = self.location copy.last_file_status_check_time = self.last_file_status_check_time copy.last_file_modified_time = self.last_file_modified_time copy.inferred = self.inferred if self.arguments: # deep copy the content copy.arguments = dict() for key in self.arguments.keys(): copy.arguments[key] = list(self.arguments[key]) copy.specialized_schema = self.specialized_schema self._copy_def(res_opt, copy) return copy async def file_status_check_async(self) -> None: """Check the modified time for this object and any children.""" with logger._enter_scope(self._TAG, self.ctx, self.file_status_check_async.__name__): full_path = self.ctx.corpus.storage.create_absolute_corpus_path( self.location, self.in_document) modified_time = await self.ctx.corpus._get_last_modified_time_from_partition_path_async( full_path) # Update modified times. self.last_file_status_check_time = datetime.now(timezone.utc) self.last_file_modified_time = time_utils._max_time( modified_time, self.last_file_modified_time) await self.report_most_recent_time_async( self.last_file_modified_time) def get_name(self) -> str: return self.name def is_derived_from(self, base: str, res_opt: Optional['ResolveOptions'] = None) -> bool: return False async def report_most_recent_time_async(self, child_time: datetime) -> None: """Report most recent modified time (of current or children objects) to the parent object.""" if isinstance(self.owner, CdmFileStatus) and child_time: await cast(CdmFileStatus, self.owner).report_most_recent_time_async(child_time) def validate(self) -> bool: return True def visit(self, path_from: str, pre_children: 'VisitCallback', post_children: 'VisitCallback') -> bool: path = self._fetch_declared_path(path_from) if pre_children and pre_children(self, path): return False if self._visit_def(path, pre_children, post_children): return True if post_children and post_children(self, path): return True return False def _fetch_declared_path(self, path_from: str) -> str: return '{}{}'.format(path_from, (self.get_name() or 'UNNAMED'))
def from_data( ctx: CdmCorpusContext, data: TypeAttribute, entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition: type_attribute = ctx.corpus.make_object( CdmObjectType.TYPE_ATTRIBUTE_DEF, data.get('name')) type_attribute.purpose = PurposeReferencePersistence.from_data( ctx, data.get('purpose')) type_attribute.data_type = DataTypeReferencePersistence.from_data( ctx, data.get('dataType')) if data.get('cardinality'): min_cardinality = None if data.get('cardinality').get('minimum'): min_cardinality = data.get('cardinality').get('minimum') max_cardinality = None if data.get('cardinality').get('maximum'): max_cardinality = data.get('cardinality').get('maximum') if not min_cardinality or not max_cardinality: logger.error( _TAG, ctx, 'Both minimum and maximum are required for the Cardinality property.' ) if not CardinalitySettings._is_minimum_valid(min_cardinality): logger.error( _TAG, ctx, 'Invalid minimum cardinality {}.'.format(min_cardinality)) if not CardinalitySettings._is_maximum_valid(max_cardinality): logger.error( _TAG, ctx, 'Invalid maximum cardinality {}.'.format(max_cardinality)) if min_cardinality and max_cardinality and CardinalitySettings._is_minimum_valid( min_cardinality) and CardinalitySettings._is_maximum_valid( max_cardinality): type_attribute.cardinality = CardinalitySettings( type_attribute) type_attribute.cardinality.minimum = min_cardinality type_attribute.cardinality.maximum = max_cardinality type_attribute.attribute_context = AttributeContextReferencePersistence.from_data( ctx, data.get('attributeContext')) type_attribute.resolution_guidance = AttributeResolutionGuidancePersistence.from_data( ctx, data.get('resolutionGuidance')) applied_traits = utils.create_trait_reference_array( ctx, data.get('appliedTraits')) type_attribute.applied_traits.extend(applied_traits) if data.get('isPrimaryKey') and entity_name: t2p_map = TraitToPropertyMap(type_attribute) t2p_map._update_property_value( 'isPrimaryKey', entity_name + '/(resolvedAttributes)/' + type_attribute.name) type_attribute.explanation = data.explanation type_attribute.is_read_only = utils._property_from_data_to_bool( data.isReadOnly) type_attribute.is_nullable = utils._property_from_data_to_bool( data.isNullable) type_attribute.source_name = utils._property_from_data_to_string( data.sourceName) type_attribute.source_ordering = utils._property_from_data_to_int( data.sourceOrdering) type_attribute.display_name = utils._property_from_data_to_string( data.displayName) type_attribute.description = utils._property_from_data_to_string( data.description) type_attribute.value_constrained_to_list = utils._property_from_data_to_bool( data.valueConstrainedToList) type_attribute.maximum_length = utils._property_from_data_to_int( data.maximumLength) type_attribute.maximum_value = utils._property_from_data_to_string( data.maximumValue) type_attribute.minimum_value = utils._property_from_data_to_string( data.minimumValue) type_attribute.default_value = data.defaultValue type_attribute.projection = ProjectionPersistence.from_data( ctx, data.projection) if data.get('dataFormat') is not None: try: type_attribute.data_format = TypeAttributePersistence._data_type_from_data( data.dataFormat) except ValueError: logger.warning( TypeAttributePersistence.__name__, ctx, 'Couldn\'t find an enum value for {}.'.format( data.dataFormat), TypeAttributePersistence.from_data.__name__) return type_attribute
async def from_data( ctx: 'CdmCorpusContext', document_folder: 'CdmFolderDefinition', data: 'LocalEntity', extension_trait_def_list: List['CdmTraitDefinition'], manifest: 'CdmManifestDefinition' ) -> 'CdmLocalEntityDeclarationDefinition': local_entity_dec = ctx.corpus.make_object( CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, data.name) local_extension_trait_def_list = [] # type: List[CdmTraitDefinition] entity_doc = await DocumentPersistence.from_data( ctx, data, extension_trait_def_list, local_extension_trait_def_list) if not entity_doc: logger.error(ctx, _TAG, "from_data", None, CdmLogCode.ERR_PERSIST_DOC_FETCH_ERROR) return None document_folder.documents.append(entity_doc) # Entity schema path is the path to the doc containing the entity definition. local_entity_dec.entity_path = ctx.corpus.storage.create_relative_corpus_path( '{}/{}'.format(entity_doc.at_corpus_path, data.name), manifest) local_entity_dec.explanation = data.get('description') if data.get('lastFileStatusCheckTime'): local_entity_dec.last_file_status_check_time = dateutil.parser.parse( data.get('lastFileStatusCheckTime')) if data.get('lastFileModifiedTime'): local_entity_dec.last_file_modified_time = dateutil.parser.parse( data.get('lastFileModifiedTime')) if data.get('lastChildFileModifiedTime'): local_entity_dec.last_child_file_modified_time = dateutil.parser.parse( data.get('lastChildFileModifiedTime')) if data.get('isHidden'): is_hidden_trait = ctx.corpus.make_object(CdmObjectType.TRAIT_REF, 'is.hidden', True) local_entity_dec.exhibits_traits.append(is_hidden_trait) # Add traits for schema entity info. if data.get('schemas'): t2pm = TraitToPropertyMap(local_entity_dec) t2pm._update_property_value('cdmSchemas', data.get('schemas')) # Data partitions are part of the local entity, add them here. for element in (data.get('partitions') or []): data_partition = await DataPartitionPersistence.from_data( ctx, element, extension_trait_def_list, local_extension_trait_def_list, document_folder) if data_partition is not None: local_entity_dec.data_partitions.append(data_partition) else: logger.error(ctx, _TAG, "from_data", None, CdmLogCode.ERR_PERSIST_DOC_FETCH_ERROR) return None import_docs = await extension_helper.standard_import_detection( ctx, extension_trait_def_list, local_extension_trait_def_list) # type: List[CdmImport] extension_helper.add_import_docs_to_manifest(ctx, import_docs, entity_doc) return local_entity_dec