def from_data(ctx: CdmCorpusContext, data: TypeAttribute, entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition:
type_attribute = ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, data.get('name'))
type_attribute.purpose = PurposeReferencePersistence.from_data(ctx, data.get('purpose'))
type_attribute.data_type = DataTypeReferencePersistence.from_data(ctx, data.get('dataType'))
type_attribute.attribute_context = AttributeContextReferencePersistence.from_data(ctx, data.get('attributeContext'))
type_attribute.resolution_guidance = AttributeResolutionGuidancePersistence.from_data(ctx, data.get('resolutionGuidance'))
applied_traits = utils.create_trait_reference_array(ctx, data.get('appliedTraits'))
type_attribute.applied_traits.extend(applied_traits)
if data.get('isPrimaryKey') and entity_name:
t2p_map = TraitToPropertyMap(type_attribute)
t2p_map._update_property_value('isPrimaryKey', entity_name + '/(resolvedAttributes)/' + type_attribute.name)
type_attribute.explanation = data.get('explanation')
if data.get('isReadOnly') is not None:
type_attribute.is_read_only = TypeAttributePersistence._property_from_data_to_bool(data.isReadOnly)
if data.get('isNullable') is not None:
type_attribute.is_nullable = TypeAttributePersistence._property_from_data_to_bool(data.isNullable)
if data.get('sourceName'):
type_attribute.source_name = TypeAttributePersistence._property_from_data_to_string(data.sourceName)
if data.get('sourceOrdering') is not None:
type_attribute.source_ordering = TypeAttributePersistence._property_from_data_to_int(data.sourceOrdering)
if data.get('displayName'):
type_attribute.display_name = TypeAttributePersistence._property_from_data_to_string(data.displayName)
if data.get('description'):
type_attribute.description = TypeAttributePersistence._property_from_data_to_string(data.description)
if data.get('valueConstrainedToList') is not None:
type_attribute.value_constrained_to_list = TypeAttributePersistence._property_from_data_to_bool(data.valueConstrainedToList)
if data.get('maximumLength') is not None:
type_attribute.maximum_length = TypeAttributePersistence._property_from_data_to_int(data.maximumLength)
if data.get('maximumValue') is not None:
type_attribute.maximum_value = TypeAttributePersistence._property_from_data_to_string(data.maximumValue)
if data.get('minimumValue') is not None:
type_attribute.minimum_value = TypeAttributePersistence._property_from_data_to_string(data.minimumValue)
if data.get('dataFormat') is not None:
try:
type_attribute.data_format = CdmDataFormat(data.dataFormat)
except ValueError:
logger.warning(TypeAttributePersistence.__name__, ctx, 'Couldn\'t find an enum value for {}.'.format(
data.dataFormat), TypeAttributePersistence.from_data.__name__)
if data.get('defaultValue') is not None:
type_attribute.default_value = data.defaultValue
return type_attribute
def test_update_and_fetch_list_lookup(self):
"""Test update and fetch list lookup default value without attributeValue and displayOrder."""
corpus = CdmCorpusDefinition()
cdm_attribute = CdmTypeAttributeDefinition(corpus.ctx, 'SomeAttribute')
trait_to_property_map = TraitToPropertyMap(cdm_attribute)
constant_values = [{'languageTag': 'en', 'displayText': 'Fax'}]
trait_to_property_map._update_property_value('defaultValue',
constant_values)
result = trait_to_property_map._fetch_property_value('defaultValue')
self.assertEqual(1, len(result))
self.assertEqual('en', result[0].get('languageTag'))
self.assertEqual('Fax', result[0].get('displayText'))
self.assertIsNone(result[0].get('attributeValue'))
self.assertIsNone(result[0].get('displayOrder'))
def from_data(ctx: CdmCorpusContext, obj: DataColumn,
entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition:
type_attribute = ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, obj.name)
properties = obj.origin_data_type_name.properties
type_attribute.data_format = utils.syms_data_type_to_cdm_data_format(obj.origin_data_type_name)
if obj.origin_data_type_name.scale != 0 or obj.origin_data_type_name.precision != 0:
numeric_traits = ctx.corpus.make_ref(CdmObjectType.TRAIT_REF, 'is.data_format.numeric.shaped', True)
scale_traits_arg = ctx.corpus.make_ref(CdmObjectType.ARGUMENT_DEF, 'scale', False)
scale_traits_arg.Value = obj.origin_data_type_name.scale
numeric_traits.arguments.append(scale_traits_arg)
precision_traits_arg = ctx.corpus.make_ref(CdmObjectType.ARGUMENT_DEF, 'scale', False)
precision_traits_arg.value = obj.origin_data_type_name.scale
numeric_traits.arguments.append(precision_traits_arg)
if properties is not None:
if 'cdm:purpose' in properties:
type_attribute.purpose = properties['cdm:purpose']
if 'cdm:dataType' in properties:
type_attribute.data_type = properties['cdm:dataType']
if 'cdm:traits' in properties:
utils.add_list_to_cdm_collection(type_attribute.applied_traits, utils.create_trait_reference_array(ctx, properties['cdm:traits']))
if 'cdm:is_primary_key' in properties and properties['cdm:is_primary_key']:
t2pMap = TraitToPropertyMap(type_attribute)
t2pMap._update_property_value('is_primary_key', entity_name + '/(resolvedAttributes)/' + type_attribute.name)
if 'cdm:isReadOnly' in properties:
type_attribute.is_read_only = properties['cdm:isReadOnly']
if 'cdm:sourceName' in properties:
type_attribute.source_name = properties['cdm:source_name']
if 'cdm:sourceOrdering' in properties:
type_attribute.source_ordering = properties['cdm:sourceOrdering']
if 'cdm:valueConstrainedToList' in properties:
type_attribute.value_constrained_to_list = properties['cdm:valueConstrainedToList']
if 'cdm:maximumLength' in properties:
type_attribute.maximum_length = properties['cdm:maximumLength']
if 'cdm:maximumValue' in properties:
type_attribute.maximum_value = properties['cdm:maximumValue']
if 'cdm:minimumValue' in properties:
type_attribute.minimum_value = properties['cdm:minimumValue']
if 'cdm:defaultValue' in properties:
type_attribute.default_value = properties['cdm:defaultValue']
return type_attribute
def from_data(ctx: CdmCorpusContext, data: TypeAttribute, entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition:
type_attribute = ctx.corpus.make_object(CdmObjectType.TYPE_ATTRIBUTE_DEF, data.get('name'))
type_attribute.purpose = PurposeReferencePersistence.from_data(ctx, data.get('purpose'))
type_attribute.data_type = DataTypeReferencePersistence.from_data(ctx, data.get('dataType'))
cardinality = utils.cardinality_settings_from_data(data.get('cardinality'), type_attribute)
if cardinality is not None:
type_attribute.cardinality = cardinality
type_attribute.attribute_context = AttributeContextReferencePersistence.from_data(ctx, data.get('attributeContext'))
utils.add_list_to_cdm_collection(type_attribute.applied_traits,
utils.create_trait_reference_array(ctx, data.get('appliedTraits')))
type_attribute.resolution_guidance = AttributeResolutionGuidancePersistence.from_data(ctx, data.get('resolutionGuidance'))
if data.get('isPrimaryKey') and entity_name:
t2p_map = TraitToPropertyMap(type_attribute)
t2p_map._update_property_value('isPrimaryKey', entity_name + '/(resolvedAttributes)/' + type_attribute.name)
type_attribute.explanation = data.explanation
type_attribute.is_read_only = utils._property_from_data_to_bool(data.isReadOnly)
type_attribute.is_nullable = utils._property_from_data_to_bool(data.isNullable)
type_attribute.source_name = utils._property_from_data_to_string(data.sourceName)
type_attribute.source_ordering = utils._property_from_data_to_int(data.sourceOrdering)
type_attribute.display_name = utils._property_from_data_to_string(data.displayName)
type_attribute.description = utils._property_from_data_to_string(data.description)
type_attribute.value_constrained_to_list = utils._property_from_data_to_bool(data.valueConstrainedToList)
type_attribute.maximum_length = utils._property_from_data_to_int(data.maximumLength)
type_attribute.maximum_value = utils._property_from_data_to_string(data.maximumValue)
type_attribute.minimum_value = utils._property_from_data_to_string(data.minimumValue)
type_attribute.default_value = data.defaultValue
type_attribute.projection = ProjectionPersistence.from_data(ctx, data.projection)
if data.get('dataFormat') is not None:
try:
type_attribute.data_format = TypeAttributePersistence._data_type_from_data(data.dataFormat)
except ValueError:
logger.warning(ctx, _TAG, TypeAttributePersistence.from_data.__name__, None,
CdmLogCode.WARN_PERSIST_ENUM_NOT_FOUND, data.dataFormat)
return type_attribute
class CdmDataPartitionDefinition(CdmObjectDefinition, CdmFileStatus):
def __init__(self, ctx: 'CdmCorpusContext', name: str) -> None:
super().__init__(ctx)
self._TAG = CdmDataPartitionDefinition.__name__
# The name of a data partition.
self.name = name # type: str
# The corpus path for the data file location.
self.location = None # type: Optional[str]
# Indicates whether this partition is inferred.
self.inferred = False # type: bool
# The list of key value pairs to give names for the replacement values from the RegEx.
self.arguments = {} # type: Dict[str, List[str]]
# The path of a specialized schema to use specifically for the partitions generated.
self.specialized_schema = None # type: Optional[str]
# The refresh time of the partition.
self.refresh_time = None # type: Optional[datetime]
self.last_file_modified_time = None # type: Optional[datetime]
self.last_file_status_check_time = None # type: Optional[datetime]
# --- internal ---
self._ttpm = TraitToPropertyMap(self)
@property
def object_type(self) -> 'CdmObjectType':
return CdmObjectType.DATA_PARTITION_DEF
@property
def description(self) -> str:
return cast(str, self._ttpm._fetch_property_value('description'))
@description.setter
def description(self, val: str) -> None:
self._ttpm._update_property_value('description', val)
@property
def last_child_file_modified_time(self) -> datetime:
raise NotImplementedError()
@last_child_file_modified_time.setter
def last_child_file_modified_time(self, val: datetime):
raise NotImplementedError()
def copy(
self,
res_opt: Optional['ResolveOptions'] = None,
host: Optional['CdmDataPartitionDefinition'] = None
) -> 'CdmDataPartitionDefinition':
if not res_opt:
res_opt = ResolveOptions(
wrt_doc=self,
directives=self.ctx.corpus.default_resolution_directives)
if not host:
copy = CdmDataPartitionDefinition(self.ctx, self.name)
else:
copy = host
copy.name = self.name
copy.description = self.description
copy.location = self.location
copy.last_file_status_check_time = self.last_file_status_check_time
copy.last_file_modified_time = self.last_file_modified_time
copy.inferred = self.inferred
if self.arguments:
# deep copy the content
copy.arguments = dict()
for key in self.arguments.keys():
copy.arguments[key] = list(self.arguments[key])
copy.specialized_schema = self.specialized_schema
self._copy_def(res_opt, copy)
return copy
async def file_status_check_async(self) -> None:
"""Check the modified time for this object and any children."""
with logger._enter_scope(self._TAG, self.ctx,
self.file_status_check_async.__name__):
full_path = self.ctx.corpus.storage.create_absolute_corpus_path(
self.location, self.in_document)
modified_time = await self.ctx.corpus._get_last_modified_time_from_partition_path_async(
full_path)
# Update modified times.
self.last_file_status_check_time = datetime.now(timezone.utc)
self.last_file_modified_time = time_utils._max_time(
modified_time, self.last_file_modified_time)
await self.report_most_recent_time_async(
self.last_file_modified_time)
def get_name(self) -> str:
return self.name
def is_derived_from(self,
base: str,
res_opt: Optional['ResolveOptions'] = None) -> bool:
return False
async def report_most_recent_time_async(self,
child_time: datetime) -> None:
"""Report most recent modified time (of current or children objects) to the parent object."""
if isinstance(self.owner, CdmFileStatus) and child_time:
await cast(CdmFileStatus,
self.owner).report_most_recent_time_async(child_time)
def validate(self) -> bool:
return True
def visit(self, path_from: str, pre_children: 'VisitCallback',
post_children: 'VisitCallback') -> bool:
path = self._fetch_declared_path(path_from)
if pre_children and pre_children(self, path):
return False
if self._visit_def(path, pre_children, post_children):
return True
if post_children and post_children(self, path):
return True
return False
def _fetch_declared_path(self, path_from: str) -> str:
return '{}{}'.format(path_from, (self.get_name() or 'UNNAMED'))
def from_data(
ctx: CdmCorpusContext,
data: TypeAttribute,
entity_name: Optional[str] = None) -> CdmTypeAttributeDefinition:
type_attribute = ctx.corpus.make_object(
CdmObjectType.TYPE_ATTRIBUTE_DEF, data.get('name'))
type_attribute.purpose = PurposeReferencePersistence.from_data(
ctx, data.get('purpose'))
type_attribute.data_type = DataTypeReferencePersistence.from_data(
ctx, data.get('dataType'))
if data.get('cardinality'):
min_cardinality = None
if data.get('cardinality').get('minimum'):
min_cardinality = data.get('cardinality').get('minimum')
max_cardinality = None
if data.get('cardinality').get('maximum'):
max_cardinality = data.get('cardinality').get('maximum')
if not min_cardinality or not max_cardinality:
logger.error(
_TAG, ctx,
'Both minimum and maximum are required for the Cardinality property.'
)
if not CardinalitySettings._is_minimum_valid(min_cardinality):
logger.error(
_TAG, ctx,
'Invalid minimum cardinality {}.'.format(min_cardinality))
if not CardinalitySettings._is_maximum_valid(max_cardinality):
logger.error(
_TAG, ctx,
'Invalid maximum cardinality {}.'.format(max_cardinality))
if min_cardinality and max_cardinality and CardinalitySettings._is_minimum_valid(
min_cardinality) and CardinalitySettings._is_maximum_valid(
max_cardinality):
type_attribute.cardinality = CardinalitySettings(
type_attribute)
type_attribute.cardinality.minimum = min_cardinality
type_attribute.cardinality.maximum = max_cardinality
type_attribute.attribute_context = AttributeContextReferencePersistence.from_data(
ctx, data.get('attributeContext'))
type_attribute.resolution_guidance = AttributeResolutionGuidancePersistence.from_data(
ctx, data.get('resolutionGuidance'))
applied_traits = utils.create_trait_reference_array(
ctx, data.get('appliedTraits'))
type_attribute.applied_traits.extend(applied_traits)
if data.get('isPrimaryKey') and entity_name:
t2p_map = TraitToPropertyMap(type_attribute)
t2p_map._update_property_value(
'isPrimaryKey',
entity_name + '/(resolvedAttributes)/' + type_attribute.name)
type_attribute.explanation = data.explanation
type_attribute.is_read_only = utils._property_from_data_to_bool(
data.isReadOnly)
type_attribute.is_nullable = utils._property_from_data_to_bool(
data.isNullable)
type_attribute.source_name = utils._property_from_data_to_string(
data.sourceName)
type_attribute.source_ordering = utils._property_from_data_to_int(
data.sourceOrdering)
type_attribute.display_name = utils._property_from_data_to_string(
data.displayName)
type_attribute.description = utils._property_from_data_to_string(
data.description)
type_attribute.value_constrained_to_list = utils._property_from_data_to_bool(
data.valueConstrainedToList)
type_attribute.maximum_length = utils._property_from_data_to_int(
data.maximumLength)
type_attribute.maximum_value = utils._property_from_data_to_string(
data.maximumValue)
type_attribute.minimum_value = utils._property_from_data_to_string(
data.minimumValue)
type_attribute.default_value = data.defaultValue
type_attribute.projection = ProjectionPersistence.from_data(
ctx, data.projection)
if data.get('dataFormat') is not None:
try:
type_attribute.data_format = TypeAttributePersistence._data_type_from_data(
data.dataFormat)
except ValueError:
logger.warning(
TypeAttributePersistence.__name__, ctx,
'Couldn\'t find an enum value for {}.'.format(
data.dataFormat),
TypeAttributePersistence.from_data.__name__)
return type_attribute
async def from_data(
ctx: 'CdmCorpusContext', document_folder: 'CdmFolderDefinition',
data: 'LocalEntity',
extension_trait_def_list: List['CdmTraitDefinition'],
manifest: 'CdmManifestDefinition'
) -> 'CdmLocalEntityDeclarationDefinition':
local_entity_dec = ctx.corpus.make_object(
CdmObjectType.LOCAL_ENTITY_DECLARATION_DEF, data.name)
local_extension_trait_def_list = [] # type: List[CdmTraitDefinition]
entity_doc = await DocumentPersistence.from_data(
ctx, data, extension_trait_def_list,
local_extension_trait_def_list)
if not entity_doc:
logger.error(ctx, _TAG, "from_data", None,
CdmLogCode.ERR_PERSIST_DOC_FETCH_ERROR)
return None
document_folder.documents.append(entity_doc)
# Entity schema path is the path to the doc containing the entity definition.
local_entity_dec.entity_path = ctx.corpus.storage.create_relative_corpus_path(
'{}/{}'.format(entity_doc.at_corpus_path, data.name), manifest)
local_entity_dec.explanation = data.get('description')
if data.get('lastFileStatusCheckTime'):
local_entity_dec.last_file_status_check_time = dateutil.parser.parse(
data.get('lastFileStatusCheckTime'))
if data.get('lastFileModifiedTime'):
local_entity_dec.last_file_modified_time = dateutil.parser.parse(
data.get('lastFileModifiedTime'))
if data.get('lastChildFileModifiedTime'):
local_entity_dec.last_child_file_modified_time = dateutil.parser.parse(
data.get('lastChildFileModifiedTime'))
if data.get('isHidden'):
is_hidden_trait = ctx.corpus.make_object(CdmObjectType.TRAIT_REF,
'is.hidden', True)
local_entity_dec.exhibits_traits.append(is_hidden_trait)
# Add traits for schema entity info.
if data.get('schemas'):
t2pm = TraitToPropertyMap(local_entity_dec)
t2pm._update_property_value('cdmSchemas', data.get('schemas'))
# Data partitions are part of the local entity, add them here.
for element in (data.get('partitions') or []):
data_partition = await DataPartitionPersistence.from_data(
ctx, element, extension_trait_def_list,
local_extension_trait_def_list, document_folder)
if data_partition is not None:
local_entity_dec.data_partitions.append(data_partition)
else:
logger.error(ctx, _TAG, "from_data", None,
CdmLogCode.ERR_PERSIST_DOC_FETCH_ERROR)
return None
import_docs = await extension_helper.standard_import_detection(
ctx, extension_trait_def_list,
local_extension_trait_def_list) # type: List[CdmImport]
extension_helper.add_import_docs_to_manifest(ctx, import_docs,
entity_doc)
return local_entity_dec
