def run_datavault():
pipeline = Pipeline(config)
pipe = pipeline.get_or_create_pipe('test_source', source_config)
from samples import customers_datavault
module = customers_datavault
pipe.register_domain(module)
pipeline.run()
def test_main(*args):
from tests.unit_tests_rob.test_configs import test_config, jsontest_config
pipeline = Pipeline(test_config)
define_test_pipe(pipeline, jsontest_config)
pipeline.run()
def test_pipeline_exists(self):
self.pipeline = Pipeline(general_config)
self.assertIsNotNone(self.pipeline.dwh)
# hj: sor is uit pipeline gehaald deze zit nu in pipe
# self.assertIsNotNone(self.pipeline.dwh.sor)
self.assertIsNotNone(self.pipeline.dwh.rdv)
self.assertIsNotNone(self.pipeline.dwh.dv)
self.assertIsNotNone(self.pipeline.dwh.sys)
def setUp(self):
print("setup:")
from tests.unit_tests_rob.test_configs import test_config, jsontest_config
self.pipeline = Pipeline(test_config)
self.pipe = self.pipeline.get_or_create_pipe('sor_test',
config=jsontest_config)
self.pipe.register_domain(test_domain)
# hj: onderstaande regel toegevoegd. Bij elke initiatie dient mappings met lege lijst te beginnen
self.pipe.mappings = []
self.pipe.mappings.extend(init_source_to_sor_mappings(self.pipe))
self.pipe.mappings.extend(init_sor_to_dv_mappings(self.pipe))
def run_staging():
pipeline = Pipeline(config)
pipe = pipeline.get_or_create_pipe('test_source', source_config)
source_file = CsvFile(get_root_path() + '/sample_data/patienten1.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['patientnummer'])
mapping = SourceToSorMapping(source_file, 'persoon_hstage', auto_map=True)
pipe.mappings.append(mapping)
pipeline.run()
class TestCase_Pipeline(unittest.TestCase):
def setUp(self):
self.pipeline = get_global_test_pipeline()
self.pipe = self.pipeline.get_or_create_pipe('test_system')
def test_pipeline_exists(self):
self.pipeline = Pipeline(general_config)
self.assertIsNotNone(self.pipeline.dwh)
# hj: sor is uit pipeline gehaald deze zit nu in pipe
# self.assertIsNotNone(self.pipeline.dwh.sor)
self.assertIsNotNone(self.pipeline.dwh.rdv)
self.assertIsNotNone(self.pipeline.dwh.dv)
self.assertIsNotNone(self.pipeline.dwh.sys)
# hj: onderstaande tests heb ik uitgezet, want proces van aanmaken is gewijzigd en geeft nu conflicten
# het zal geen reele situatie mogen zijn dat iemand schema's weg gaat gooien.
#er zou een auth-error moeten komen
# def drop_schema_dv(self):
# sql = """DROP SCHEMA IF EXISTS dv CASCADE;"""
# execute_sql(sql)
#
# def test_schema_not_existing(self):
# sql = """DROP SCHEMA IF EXISTS dv CASCADE;
# SELECT exists(select schema_name FROM information_schema.schemata WHERE schema_name = 'dv')"""
# result = execute_sql(sql)
# schema_present = result[0][0]
#
# self.assertFalse(schema_present, 'bestaat al wel!')
def test_schema_existing(self):
self.pipe.register_domain(_domain_rob)
self.pipeline.run()
sql = """SELECT exists(select schema_name FROM information_schema.schemata WHERE schema_name = 'dv')"""
result = execute_sql(sql)
schema_present = result[0][0]
self.assertTrue(schema_present, msg='bestaat nog niet!')
def test_sql_logger_exists(self):
self.pipe.register_domain(_domain_rob)
self.pipeline.run()
sql_logger = self.pipeline.sql_logger
self.assertIsNotNone(sql_logger)
def main():
pipeline = Pipeline(general_config)
pipe = pipeline.get_or_create_pipe('_test')
patienten = Patienten(pipeline.dwh)
list = patienten.load_by_hub()
patient = list[0]
a = patient.naamgegevens.initialen
patient.naamgegevens.initialen = 'T.'
patient.naamgegevens.geslachtsnaam = 'Test3'
patient.inschrijving.inschrijfnummer = '123123'
patienten.save_entity(patient)
for r in list:
print(r.__dict__)
runid = pipeline.create_new_runid()
new_patient = patienten.new(bk='12345a', runid=runid)
new_patient.personalia.voornaam = '12837612387612'
patienten.save_entity(new_patient)
def test_pipes(self):
pipeline = Pipeline({})
pipe = pipeline.get_or_create_pipe('test_system',
config=test_system_config)
self.assertIsNotNone(pipe)
self.assertIsNotNone(pipe.pipeline)
self.assertEqual(len(pipeline.pipes), 1)
self.assertEqual(len(pipeline.dwh.sors), 1)
self.assertIsNone(pipe.source_db)
self.assertIsNotNone(pipe.source_path)
self.assertIsNotNone(pipe.sor)
self.assertEqual(pipe.sor.name, 'sor_test_system')
pipe2 = pipeline.get_or_create_pipe(
'test_system2', config={'sor_schema': 'sor_test_system2'})
self.assertEqual(len(pipeline.pipes), 2)
self.assertEqual(len(pipeline.dwh.sors), 2)
pipe_met_zelfde_naam = pipeline.get_or_create_pipe(
'test_system2', config={'sor_schema': 'sor_test_system2'})
self.assertEqual(len(pipeline.pipes), 2)
self.assertEqual(len(pipeline.dwh.sors), 2)
def get_global_test_pipeline():
pipeline = Pipeline(config=general_config)
return pipeline
def get_global_test_pipeline():
#zorg dat singleton van Pipeline niet actueel is. Iedere test heeft namelijk zijn eigen pipeline en zijn eigen database nodig.
Pipeline._instance = None
pipeline = Pipeline(config=general_config)
return pipeline
class TestCase_RunProces(unittest.TestCase):
print('init_db:\n')
# uitgeschreven eigen versie voor init_db(), vanwege andere settings vergeleken met overige rob_unittests
engine = create_engine(test_config['conn_dwh'])
sql = """
DROP SCHEMA IF EXISTS sor CASCADE;
DROP SCHEMA IF EXISTS sor_test CASCADE;
DROP SCHEMA IF EXISTS rdv CASCADE;
DROP SCHEMA IF EXISTS dv CASCADE;
DROP SCHEMA IF EXISTS sys CASCADE;
"""
query = text(sql)
engine.execute(query)
def setUp(self):
print("setup:")
from tests.unit_tests_rob.test_configs import test_config, jsontest_config
self.pipeline = Pipeline(test_config)
self.pipe = self.pipeline.get_or_create_pipe('sor_test',
config=jsontest_config)
self.pipe.register_domain(test_domain)
# hj: onderstaande regel toegevoegd. Bij elke initiatie dient mappings met lege lijst te beginnen
self.pipe.mappings = []
self.pipe.mappings.extend(init_source_to_sor_mappings(self.pipe))
self.pipe.mappings.extend(init_sor_to_dv_mappings(self.pipe))
def test01_pipeline_run(self):
print("test_run1:\n")
self.pipeline.run()
test_row_count(self, 'sor_test.patient_hstage', 1)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 1)
def test01b_pipeline_run(self):
print("test_run1_her:\n")
self.pipeline.run()
self.pipeline.run()
self.pipeline.run()
self.pipeline.run()
self.pipeline.run()
""" meerdere malen de pipeline gerund, want het leek erop dat de jsonb velden "extra" en "extra" random
gevuld werden met de twee gedefineerde jsonb objecten, dat wordt opgemerkt door vaker te runnen"""
test_row_count(self, 'sor_test.patient_hstage', 1)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 1)
def test02_dv_update(self):
print("test_run2:\n")
path = jsontest_config['data_path']
# deceased boolean van True naar False en bijhorende datum verwijderd:
self.pipe.mappings[0].file_name = path + 'test2.csv'
self.pipeline.run()
test_row_count(self, 'sor_test.patient_hstage', 2)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 2)
def test02b_dv_update(self):
print("test_run2:\n")
path = jsontest_config['data_path']
# respectievelijk leegmaken + aanpassing van 2 kolommen die in DV laag aan JSONB veld "extra2" toegewezen worden
self.pipe.mappings[0].file_name = path + 'test3.csv'
self.pipeline.run()
test_row_count(self, 'sor_test.patient_hstage', 3)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 3)
result = get_field_value_from_table(
"""extra->'extra::jsonb' ->> 'contactpersoon'""",
'dwh2a.dv.patient_sat', """_runid = 0.08""")
result = result[0][0]
self.assertEqual(
result, 'Janny Jansen',
'ik verwachtte dat dit JSONB object onderdeel aangepast zou worden naar Janny Jansen'
)
def test03_order_changed(self):
""" de volgorde van de key in een bron-json-object (extra) is veranderd (nu eerst achternaam
daarna pas voornaam)"""
print("test_run3:\n")
path = jsontest_config['data_path']
self.pipe.mappings[
0].file_name = path + 'test4.csv' # JSONB keys volgorde gewijzigd
self.pipeline.run()
test_row_count(self, 'sor_test.patient_hstage', 4)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 3)
result = get_field_value_from_table(
"""jsonb_object_keys(extra->'extra::jsonb') """,
'dwh2a.dv.patient_sat', """_runid = 0.08 limit 1""")
result = result[0][0]
self.assertEqual(
result, 'voornaam',
'ik verwachtte dat de eerste key "voornaam" zou zijn')
def test04_extra_pair(self):
""" toevoegen van extra key/value pair; """
print("test_run4:\n")
path = jsontest_config['data_path']
# JSONB extra key/value pair; kennelijk wordt 'n nieuw pair in postgres voor de reeds aanwezige pairs geplaatst:
self.pipe.mappings[0].file_name = path + 'test5.csv'
self.pipeline.run()
test_row_count(self, 'sor_test.patient_hstage', 5)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 4)
result = get_field_value_from_table(
"""jsonb_object_keys(extra->'extra::jsonb') """,
'dwh2a.dv.patient_sat', """_runid = 0.10 """)
result = len(result) # het aantal keys in de dictionary
self.assertEqual(result, 4,
'ik verwachtte dat er 4 keys zouden worden gevonden')
def test05_remove_pair(self):
""" verwijderen van ('origineel') key/value pair; """
print("test_run5:\n")
path = jsontest_config['data_path']
self.pipe.mappings[0].file_name = path + 'test6.csv'
self.pipeline.run()
test_row_count(self, 'sor_test.patient_hstage', 6)
test_row_count(self, 'dv.patient_hub', 1)
test_row_count(self, 'dv.patient_sat', 5)
result = get_field_value_from_table(
"""jsonb_object_keys(extra->'extra::jsonb') """,
'dwh2a.dv.patient_sat', """_runid = 0.11 """)
result = len(result) # het aantal keys in de dictionary
print(result)
self.assertEqual(result, 3,
'ik verwachtte dat er 3 keys zouden worden gevonden')
def test_pipeline_exists(self):
pipeline = Pipeline({})
self.assertIsNotNone(pipeline)
self.assertIsNotNone(pipeline.dwh)
self.assertIsNotNone(pipeline.dwh.dv)
self.assertIsNotNone(pipeline.dwh.sys)