def reflect(self):
for field_def in self.import_def:
field_name = field_def[0]
field_name = Column.clear_name(field_name)
col = Column(field_name.lower())
self.columns.append(col)
self.is_reflected = True
def map_entity_old(self,
entity_cls: HubEntity,
bk: str = '',
join: str = '',
type: str = ''):
# is_hybrid = False
hub_name = entity_cls.cls_get_hub_name()
hub = entity_cls.cls_get_hub()
if bk:
source_field = '_id' #type: str
source_col = Column(source_field, 'integer',
tbl=hub) #type: Column
# elif is_hybrid:
# source_field = 'fk_{}{}'.format(type, hub_name)
# source_col = Column(source_field, 'integer', tbl=self.sor_table)
else:
# source_field = 'fk_' + hub_name
source_field = 'fk_{}{}'.format(type, hub_name)
source_col = Column(source_field,
'integer',
tbl=self.sor_table_name)
# target_field = '_fk_' + hub_name #type: str
target_field = 'fk_{}'.format(hub_name)
target_field = self.get_fk_name(entity_cls)
# if target_field in self.hubs:
# target_field = '_fk_parent_' + self.type + hub_name
target_col = Column(target_field, 'integer', tbl=hub)
field_mapping = LinkFieldMapping(source_col, target_col)
field_mapping.bk = bk
field_mapping.join = join
self.hubs[target_field] = hub
self.field_mappings.append(field_mapping)
def reflect(self):
self.columns = []
cursor = self.db.engine.raw_connection().cursor()
if self.db.driver == DBDrivers.ORACLE:
sql = self.sql + ' OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY'
elif self.db.driver == DBDrivers.SQLSERVER:
sql = self.sql_to_top_select(self.sql, 1)
elif self.db.driver == DBDrivers.POSTGRESS:
sql = self.sql + ' OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY'
# sql = self.sql + ' AND 1=0'
# cursor.execute(sql)
result = self.db.engine.execute(sql)
cursor = result.cursor
for sa_col in cursor.description:
if self.db.driver == DBDrivers.ORACLE:
col = Column(sa_col[0],
oracle_type_to_postgres_type(str(sa_col[1])))
elif self.db.driver == DBDrivers.SQLSERVER:
col = Column(sa_col[0],
sqlserver_type_to_postgres_type(str(sa_col[1])))
else:
col = Column(sa_col.name, sa_col.type_code)
self.columns.append(col)
self.is_reflected = True
def reflect(self):
# lees eerste regel
with open(self.file_name, 'r', **self.file_kwargs) as csvfile:
reader = csv.reader(csvfile, **self.csv_kwargs)
column_names = next(reader, None)
for name in column_names:
name = Column.clear_name(name)
col = Column(name.lower())
self.columns.append(col)
self.is_reflected = True
def add_mapping(self, *args, **kwargs):
if len(args) == 1:
hub_name = args[0]
source_fk = '_id'
target_fk = 'fk_' + hub_name
elif len(args) == 2:
source_fk = args[0]
hub_name = args[1]
target_fk = 'fk_' + hub_name
db = Dwh()
hub = Table(hub_name, db.dv)
source_field = Column('_id', tbl=hub)
target_field = Column(target_fk, 'integer')
field_mapping = LinkFieldMapping(source_field, target_field)
if 'join' in kwargs:
field_mapping.join = kwargs['join']
self.field_mappings.append(field_mapping)
def reflect_2(self) -> None:
self.columns = [] # type: List[Column]
inspector = reflection.Inspector.from_engine(self.db.engine)
columns = inspector.cls_get_columns(self.name, self.schema.name)
pks = inspector.get_primary_keys(self.name, self.schema.name)
indexes = inspector.get_indexes(self.name, self.schema.name)
self.key_names = []
for col in columns:
# col = Column(col['name'], str(col['type']), self)
col = Column(col['name'], col['type'], self)
col.is_key = (col.name in pks)
if col.is_key:
self.key_names.append(col.name)
# if self.key_names:
# col.is_key = (col.name in self.key_names)
self.columns.append(col)
for index in indexes:
self.indexes[index['name']] = index
self.is_reflected = True
def reflect(self):
self.columns = []
cursor = self.db.engine.raw_connection().cursor()
sql = self.sql + ' OFFSET 0 ROWS FETCH NEXT 1 ROWS ONLY'
result = self.db.engine.execute(sql)
cursor = result.cursor
for sa_col in cursor.description:
col = Column(sa_col.name, sa_col.type_code)
self.columns.append(col)
self.is_reflected = True
def map_sor_fk(self,
source_fk: str = '',
target_link_ref: Union[LinkReference,
DynamicLinkReference] = None,
join: str = '',
type: Union[str, 'HubEntity'] = ''):
if not (isinstance(target_link_ref, LinkReference)
or isinstance(target_link_ref, DynamicLinkReference)):
raise PyeltException(
'Link moet verwijzen naar veld van type LinkReference of DynamicLinkReference'
)
if isinstance(target_link_ref, LinkReference):
hub_name = target_link_ref.hub.__dbname__
hub = target_link_ref.hub
# if not type and target_link_ref.sub_entity_type != hub_name.replace('_hub', ''):
# type = target_link_ref.sub_entity_type
# if not type and not target_link_ref.hub.__base__ == HubEntity:
# type = target_link_ref.hub.__dbname__.replace('_hub', '')
if not source_fk or source_fk == SorToLinkMapping.auto_generated_sor_fk:
# source_fk = target_link_ref.fk
source_fk = 'fk_{}{}'.format(target_link_ref.sub_entity_type,
hub_name)
elif isinstance(target_link_ref, DynamicLinkReference):
# type verwijst hier naar de entity die dynamisch gelinkt wordt in de fk
if not type or isinstance(type, str):
raise PyeltException(
'Bij DynamicLinkReference moet type verwijzen naar een afgeleide van HubEntity'
)
hub = type.cls_get_hub()
self.type = type.__name__.lower()
source_col = Column(source_fk, 'integer', tbl=self.sor_table_name)
target_fk = target_link_ref.fk
target_col = Column(target_fk, 'integer', tbl=hub)
field_mapping = LinkFieldMapping(source_col, target_col)
field_mapping.join = join
self.hubs[target_fk] = hub
self.field_mappings.append(field_mapping)
def map_bk(self,
source_bk,
target_link_ref: Union[LinkReference, DynamicLinkReference],
join: str = '',
type: Union[str, 'HubEntity'] = ''):
if not (isinstance(target_link_ref, LinkReference)
or isinstance(target_link_ref, DynamicLinkReference)):
raise PyeltException(
'Link moet verwijzen naar veld van type LinkReference of DynamicLinkReference'
)
if isinstance(target_link_ref, LinkReference):
hub_name = target_link_ref.hub.__dbname__
hub = target_link_ref.hub
elif isinstance(target_link_ref, DynamicLinkReference):
# type verwijst hier naar de entity die dynamisch gelinkt wordt in de fk
if not type or isinstance(type, str):
raise PyeltException(
'Bij DynamicLinkReference moet type verwijzen naar een afgeleide van HubEntity'
)
hub_name = type.cls_get_hub_name()
hub = type.cls_get_hub()
self.type = type.__name__.lower()
hub_alias = target_link_ref.fk.replace('fk_', '')
source_field = '_id' #type: str
source_col = Column(source_field, 'integer', tbl=hub) #type: Column
target_fk = target_link_ref.fk
target_col = Column(target_fk, 'integer', tbl=hub)
field_mapping = LinkFieldMapping(source_col, target_col)
field_mapping.join = join
field_mapping.source_alias = hub_alias
if isinstance(source_bk, list):
source_bk = "|| '.' || ".join(source_bk)
field_mapping.bk = source_bk
self.hubs[target_fk] = hub
self.field_mappings.append(field_mapping)
def __init__(self, source: Union[str, 'Column', 'FieldTransformation'], target: Union[str, 'Column'], transform_func: 'FieldTransformation' = None, ref: str = '') -> None:
source_col = None #type: Union[FieldTransformation, Column]
target_col = None #type: Column
if isinstance(source, str):
if " " in source or ('(' in source and ')' in source):
source_col = FieldTransformation(name=source, sql=source)
else:
source_col = Column(source)
elif isinstance(source, DbFunction):
source_col = source
else:
source_col = source
if transform_func:
transform_func.field_name = source_col.name
source_col = transform_func
if isinstance(target, str):
target_col = Column(target)
else:
target_col = target
super().__init__(source_col, target_col)
self.transformation = transform_func
self.source_type = '' #type: str
self.target_type = '' #type: str
self.ref = ref #type: str