def init_source_to_sor_mappings(pipe):
mappings = []
validations = []
source_db = pipe.source_db
source_qry = SourceQuery(
source_db, """
SELECT
handelingen.id,
UPPER(handelingen.naam) as naam,
handelingen.datum,
handelingen.kosten,
organisaties.naam as org,
organisaties.adres,
(handelingen.naam || organisaties.naam) as naam2,
specialisaties.naam as spec,
specialisaties.fk_hoofd,
handelingen.fk_org + 1 as fk_plus,
handelingen.fk_spec
FROM
public.handelingen,
public.organisaties,
public.specialisaties
WHERE
organisaties.id = handelingen.fk_org AND
specialisaties.id = handelingen.fk_spec;
""", 'view_2')
source_qry.set_primary_key(['id'])
sor_mapping = SourceToSorMapping(source_qry,
'handeling_hstage',
auto_map=True)
mappings.append(sor_mapping)
source_tbl = SourceTable('handelingen', source_db.default_schema,
source_db)
source_tbl.set_primary_key(['id'])
sor_mapping = SourceToSorMapping(source_tbl,
'handeling2_hstage',
auto_map=True,
ignore_fields=['fk_org'])
mappings.append(sor_mapping)
source_tbl = SourceTable('handelingen', source_db.default_schema,
source_db)
source_tbl.set_primary_key(['id'])
sor_mapping = SourceToSorMapping(source_tbl,
'handeling_met_filter_hstage',
auto_map=True,
filter='kosten > 125')
mappings.append(sor_mapping)
return mappings
def init_source_to_sor_mappings(pipe):
mappings = []
validations = []
source_db = pipe.source_db
source_qry = SourceQuery(source_db, """
SELECT
handelingen.id,
UPPER(handelingen.naam) as naam,
handelingen.datum,
handelingen.kosten,
organisaties.naam as org,
organisaties.adres,
(handelingen.naam || organisaties.naam) as naam2,
specialisaties.naam as spec,
specialisaties.fk_hoofd,
handelingen.fk_org + 1 as fk_plus,
handelingen.fk_spec
FROM
public.handelingen,
public.organisaties,
public.specialisaties
WHERE
organisaties.id = handelingen.fk_org AND
specialisaties.id = handelingen.fk_spec;
""", 'view_1')
source_qry.set_primary_key(['naam'])
sor_mapping = SourceToSorMapping(source_qry, 'handeling_hstage', auto_map=True)
mappings.append(sor_mapping)
return mappings
def run_staging():
pipeline = Pipeline(config)
pipe = pipeline.get_or_create_pipe('test_source', source_config)
source_file = CsvFile(get_root_path() + '/sample_data/patienten1.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['patientnummer'])
mapping = SourceToSorMapping(source_file, 'persoon_hstage', auto_map=True)
pipe.mappings.append(mapping)
pipeline.run()
def init_source_to_sor_mappings():
mappings = []
source_file = CsvFile(get_root_path() +
'/PYELT/tests/data/zorgverlenersA_rob.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['zorgverlenernummer'])
sor_mapping = SourceToSorMapping(source_file,
'zorgverlener_hstage',
auto_map=True)
mappings.append(sor_mapping)
source_file = CsvFile(get_root_path() +
'/PYELT/tests/data/zorginstelling_rob.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['zorginstellings_nummer'])
sor_mapping = SourceToSorMapping(source_file,
'zorginstelling_hstage',
auto_map=True)
mappings.append(sor_mapping)
return mappings
def test_source_to_sor_mappings(self):
source_file = CsvFile(get_root_path() +
'/PYELT/tests/data/patienten1.csv',
delimiter=';')
source_file.reflect()
l = len(source_file.columns)
self.assertEqual(len(source_file.columns), 16)
self.assertEqual(len(source_file.primary_keys()), 0)
source_file.set_primary_key(['patientnummer'])
self.assertEqual(len(source_file.primary_keys()), 1)
sor_mapping = SourceToSorMapping(source_file,
'persoon_hstage',
auto_map=True)
self.assertEqual(sor_mapping.name, 'patienten1.csv -> persoon_hstage')
self.assertEqual(len(sor_mapping.field_mappings), 16)
self.pipe.mappings.append(sor_mapping)
def test_run3(self):
print('======================================================')
print('=== R U N 3a alter sor_tbl ===')
print('======================================================')
source_db = self.pipe.source_db
self.pipe.mappings = []
source_qry = SourceQuery(source_db, """
SELECT
*
FROM
public.handelingen
""", 'view_1')
source_qry.set_primary_key(['id'])
sor_mapping = SourceToSorMapping(source_qry, 'handeling_hstage', auto_map=True)
self.pipe.mappings.append(sor_mapping)
# run
self.pipeline.run()
self.assertEqual(get_row_count('sor_extra.handeling_hstage'), 3)
def test_run5(self):
print('======================================================')
print('=== R U N 5 waardes wijzen van iets naar NULL ===')
print('======================================================')
self.pipe.mappings = []
# bron uitbreiden met nieuw veld, mappen op dezelfde sor tabel
source_db = self.pipe.source_db
source_qry = SourceQuery(source_db, """
SELECT
*, NULL as extra_veld
FROM
public.handelingen
""", 'view_1')
source_qry.set_primary_key(['id'])
sor_mapping = SourceToSorMapping(source_qry, 'handeling_hstage', auto_map=True)
self.pipe.mappings.append(sor_mapping)
# run
self.pipeline.run()
self.assertEqual(get_row_count('sor_extra.handeling_hstage'), 9)
def test_run2(self):
print('======================================================')
print('=== R U N 1 test no key ===')
print('======================================================')
# maak tabellen in bron leeg
exec_sql(source_db_sql[0], test_system_config['source_connection'])
# vul bron tabellen met initiele waarden
exec_sql(source_db_sql[1], test_system_config['source_connection'])
# mappings
source_db = self.pipe.source_db
source_tbl = SourceTable('handelingen', source_db.default_schema, source_db, alias='handelingen2')
source_tbl.set_primary_key([''])
sor_mapping = SourceToSorMapping(source_tbl, 'handeling_no_key_hstage', filter='kosten > 125')
self.pipe.mappings.append(sor_mapping)
# run
self.pipeline.run()
for pipe in self.pipeline.pipes.values():
validation_msg = pipe.validate_mappings_before_ddl()
validation_msg += pipe.validate_mappings_after_ddl()
self.assertIn('Mapping <red>handelingen -> handeling_no_key_hstage</> is niet geldig. Geen geldige key opgegeven.', validation_msg)
def test_run1(self):
print('======================================================')
print('=== R U N 1 test automap ===')
print('======================================================')
# maak tabellen in bron leeg
exec_sql(source_db_sql[0], test_system_config['source_connection'])
# vul bron tabellen met initiele waarden
exec_sql(source_db_sql[1], test_system_config['source_connection'])
# mappings
source_db = self.pipe.source_db
source_tbl = SourceTable('handelingen', source_db.default_schema, source_db, alias='handelingen2')
source_tbl.set_primary_key(['id'])
sor_mapping = SourceToSorMapping(source_tbl, 'handeling_no_auto_map_hstage', auto_map=False, filter='kosten > 125')
sor_mapping.map_field('id', 'id')
sor_mapping.map_field('naam', 'naam2')
self.pipe.mappings.append(sor_mapping)
# run
self.pipeline.run()
for pipe in list(self.pipeline.pipes.values()):
validation_msg = pipe.validate_mappings_before_ddl()
validation_msg += pipe.validate_mappings_after_ddl()
self.assertIn('Mapping <red>handelingen -> handeling_no_auto_map_hstage</> is niet geldig. Auto_map moet aan staan bij SourceToSorMapping.', validation_msg)