def execute(self):
"""Generate declarations for packages and save to data source."""
context = Context()
data_source = TemporalMongoDataSource()
data_source.env_type = self.env
data_source.env_group = self.group
data_source.env_name = self.name
if self.host is not None:
data_source.mongo_server = MongoServer.create_key(
mongo_server_uri=self.host)
data_source.init(context)
common = data_source.get_common_or_none()
if common is None:
common = data_source.create_common()
context.data_source = data_source
context.data_set = common
for package in self.packages:
datas = ClassInfo.get_derived_types(package, Data)
enums = ClassInfo.get_derived_types(package, IntEnum)
type_declarations = [to_type_declaration(x) for x in datas]
enum_declarations = [to_enum_declaration(x) for x in enums]
# Save declarations to db
context.data_source.save_many(TypeDecl, type_declarations,
context.data_set)
context.data_source.save_many(EnumDecl, enum_declarations,
context.data_set)
def test_inheritance_chain(self):
"""Check inheritance chain for different cases."""
self.assertEqual(['Record', 'BaseSample'],
ClassInfo.get_inheritance_chain(BaseSample))
self.assertEqual(['Record', 'BaseSample', 'DerivedSample'],
ClassInfo.get_inheritance_chain(DerivedSample))
self.assertEqual(['Data', 'ElementSample'],
ClassInfo.get_inheritance_chain(ElementSample))
def execute(self):
"""Runs specified handler."""
context = Context()
data_source = TemporalMongoDataSource()
data_source.env_type = self.env
data_source.env_group = self.group
data_source.env_name = self.name
connection_literal = 'ConnectionString'
connection = next((x for x in self.source.split(',')
if x.startswith(connection_literal)), None)
connection = connection[len(connection_literal) + 1:]
if connection is not None:
parsed_uri = uri_parser.parse_uri(connection)
server_uri = ','.join(f'{x[0]}:{x[1]}'
for x in parsed_uri['nodelist'])
# TODO: mongodb+srv://?
data_source.mongo_server = MongoServer.create_key(
mongo_server_uri=f'mongodb://{server_uri}')
data_source.init(context)
common = data_source.get_common_or_none()
if common is None:
common = data_source.create_common()
context.data_source = data_source
context.data_set = common
load_from = context.data_source.get_data_set(self.dataset,
context.data_set)
# Problem with importing types
ClassInfo.get_derived_types('datacentric', Data)
record_type: Type[TRecord] = ClassInfo.get_type(self.type)
collection_type = ClassInfo.get_ultimate_base(record_type)
record = context.data_source.load_by_key(
record_type, f'{collection_type.__name__}={self.key}', load_from)
handler_name = StringUtil.to_snake_case(self.handler)
handler = getattr(record, handler_name, None)
if handler is None:
raise Exception(
f'Type {record_type.__name__} does not have handler {handler_name}.'
)
if not handler.handler:
raise Exception(f'{handler_name} is not handler.')
handler.__call__()
def _deserialize_class(dict_: Dict[str, Any]) -> TRecord:
type_name: str = dict_.pop('_t')[-1]
type_info: type = ClassInfo.get_type(type_name)
fields = attr.fields_dict(type_info)
new_obj = type_info()
for dict_key, dict_value in dict_.items():
slot = StringUtil.to_snake_case(dict_key)
field = fields[slot]
member_type = field.type
deserialized_value: Any
if get_origin(
member_type) is not None and get_origin(member_type) is list:
deserialized_value = _deserialize_list(member_type, dict_value,
field.metadata)
elif issubclass(member_type, Data):
deserialized_value = _deserialize_class(dict_value)
elif issubclass(member_type, IntEnum):
deserialized_value = member_type[dict_value]
else:
deserialized_value = _deserialize_primitive(
member_type, dict_value, field.metadata)
setattr(new_obj, slot, deserialized_value)
return new_obj
def _get_or_create_collection(self, type_: type) -> Collection:
if type_ in self.__collection_dict:
return self.__collection_dict[type_]
root_type = ClassInfo.get_ultimate_base(type_)
collection_name = root_type.__name__
collection = self.db.get_collection(collection_name,
self.__codec_options)
self.__collection_dict[type_] = collection
return collection
def test_smoke(self):
"""Smoke test."""
with self.assertRaises(Exception):
ClassInfo.get_ultimate_base(ClassInfo)
self.assertTrue(ClassInfo.get_ultimate_base(BaseSample) == BaseSample)
self.assertTrue(
ClassInfo.get_ultimate_base(DerivedSample) == BaseSample)
self.assertTrue(
ClassInfo.get_ultimate_base(ElementSample) == ElementSample)
self.assertTrue(ClassInfo.get_ultimate_base(RootSample) == RootSample)
def serialize(obj: TRecord) -> Dict[str, Any]:
type_: type = type(obj)
dict_ = _serialize_class(obj, type_)
# Field _t contains inheritance chain of the class, starting from Record
dict_['_t'] = ClassInfo.get_inheritance_chain(type_)
# ObjectId of the dataset
dict_['_dataset'] = obj.data_set
# Remove collection name prefix from key before assigning
key_with_collection_name_prefix: str = obj.to_key()
key_without_collection_name_prefix: str = key_with_collection_name_prefix.split(
'=', 1)[1]
dict_['_key'] = key_without_collection_name_prefix
# Unique object id
dict_['_id'] = obj.id_
return dict_
def _serialize_class(obj: TRecord, expected_: type) -> Dict[str, Any]:
dict_: Dict[str, Any] = dict()
cls_type = type(obj)
# Check that object has expected type
if cls_type != expected_:
raise Exception(
f'Expected: {expected_.__name__}, actual: {cls_type.__name__}')
# Field _t contains inheritance chain of the class, starting from Record
dict_['_t'] = ClassInfo.get_inheritance_chain(cls_type)
fields = attr.fields(cls_type)
mro = cls_type.__mro__
if Record in mro:
serializable_fields = [
x for x in fields if x not in attr.fields(Record)
]
elif Data in mro:
serializable_fields = [x for x in fields if x not in attr.fields(Data)]
else:
raise Exception(
f'Cannot serialize class {cls_type.__name__} not derived from Record or Data.'
)
non_private_fields = [
x for x in serializable_fields if not x.name.startswith('_')
]
for field in non_private_fields: # type: attr.Attribute
value = getattr(obj, field.name)
expected_type = field.type
is_optional = field.metadata.get('optional', False)
is_list = get_origin(
expected_type) is not None and get_origin(expected_type) is list
if value is None:
if not is_optional and not is_list:
raise Exception(
f'Missing required field: {field.name} in type: {cls_type.__name__}'
)
continue
serialized_value: Any
if is_list:
expected_arg = get_args(expected_type)[0]
serialized_value = _serialize_list(value, expected_arg,
field.metadata)
elif issubclass(expected_type, Data):
serialized_value = _serialize_class(value, expected_type)
elif issubclass(expected_type, IntEnum):
serialized_value = _serialize_enum(value)
else:
serialized_value = _serialize_primitive(value, expected_type,
field.metadata)
dict_[StringUtil.to_pascal_case(field.name)] = serialized_value
return dict_
def _to_type_member(type_: type, metadata_: Dict[Any, Any]) -> ElementDecl:
"""Resolve attribute type hint to type related part of element declaration."""
result = ElementDecl()
# Get argument of List[...] type hint
is_list = get_origin(type_) is not None and get_origin(type_) is list
if is_list:
type_ = get_args(type_)[0]
key_ = metadata_.get('key', None)
if key_ is not None and type_ is str:
key_type = ClassInfo.get_type(key_)
result.key = _create_type_declaration_key(key_type.__module__, key_)
return result
meta_type = metadata_.get('type', None)
# Primitive types
if type_ is str:
result.value = ValueDecl(type=ValueParamType.String)
elif type_ is bool:
result.value = ValueDecl(type=ValueParamType.NullableBool)
elif type_ is float:
result.value = ValueDecl(type=ValueParamType.NullableDouble)
elif type_ is ObjectId:
result.value = ValueDecl(type=ValueParamType.NullableTemporalId)
# Date additional cases
elif type_ is dt.date:
result.value = result.value = ValueDecl(
type=ValueParamType.NullableDate)
elif type_ is dt.time:
result.value = result.value = ValueDecl(
type=ValueParamType.NullableTime)
# dt.datetime depends on metadata
elif type_ is dt.datetime:
if meta_type == 'Instant':
result.value = ValueDecl(type=ValueParamType.NullableInstant)
elif meta_type is None:
result.value = ValueDecl(type=ValueParamType.NullableDateTime)
else:
raise Exception(
f'Unexpected dt.datetime and metadata type combination: dt.datetime + {type_.__name__}'
)
# Restore int/long/Local... separation using info from metadata
elif type_ is int:
if meta_type == 'long':
result.value = ValueDecl(type=ValueParamType.NullableLong)
elif meta_type == 'LocalDate':
result.value = ValueDecl(type=ValueParamType.NullableDate)
elif meta_type == 'LocalTime':
result.value = ValueDecl(type=ValueParamType.NullableTime)
elif meta_type == 'LocalMinute':
result.value = ValueDecl(type=ValueParamType.NullableMinute)
elif meta_type == 'LocalDateTime':
result.value = ValueDecl(type=ValueParamType.NullableDateTime)
elif meta_type is None:
result.value = ValueDecl(type=ValueParamType.NullableInt)
else:
raise Exception(
f'Unexpected int and metadata type combination: int + {type_.__name__}'
)
elif issubclass(type_, Data):
result.data = _create_type_declaration_key(type_.__module__,
type_.__name__)
elif issubclass(type_, IntEnum):
result.enum = _create_enum_declaration_key(str(type_.__module__),
type_.__name__)
else:
raise Exception(f'Unexpected type {type_.__name__}')
return result
data_source = TemporalMongoDataSource()
data_source.env_type = EnvType.Test
data_source.env_group = 'Schema'
data_source.env_name = 'Default'
data_source.versioning_method = VersioningMethod.Temporal
context = Context()
context.data_source = data_source
# Delete (drop) the database to clear the existing data
context.data_source.delete_db()
# Create Common dataset and assign it to data_set property of this context
context.data_set = context.data_source.create_data_set('Common')
# Convert extracted types to declarations
type_declarations = [
to_type_declaration(x)
for x in ClassInfo.get_derived_types('datacentric', Data)
]
enum_declarations = [
to_enum_declaration(x)
for x in ClassInfo.get_derived_types('datacentric', IntEnum)
]
# Save declarations to db
context.data_source.save_many(TypeDecl, type_declarations,
context.data_set)
context.data_source.save_many(EnumDecl, enum_declarations,
context.data_set)