def __get_git_hashes(self):
git_hashes = {}
import subprocess, os
from main import get_root_path
for name in os.listdir(get_root_path()):
if os.path.isdir(name):
subdirs = os.listdir(name)
if '.git' in subdirs:
os.chdir(get_root_path() + '/' + name)
git_commit_number = subprocess.check_output(["git", "rev-parse", 'HEAD']).decode('ascii')
git_hashes[name] = git_commit_number
return str(git_hashes).replace("'", '"')
def test03_dv_update(self):
# return
print("test_run2:\n")
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/zorgverleners2_rob.csv'
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/zorgverleners2_rob.csv'
self.pipeline.run()
test_row_count(self, 'sor_test_system.zorgverlener_hstage', 8)
test_row_count(self, 'dv.zorgverlener_hub', 4)
test_row_count(self, 'dv.zorgverlener_sat', 5)
test_row_count(self, 'dv.zorgverlener_sat_personalia', 5)
test_row_count(self, 'dv.zorgverlener_adres_link', 5)
def __create_path_and_filename_by_type(logger_type: str = LoggerTypes.MAIN, runid=0.00, configs={}, filename_args = ''):
import os
from main import get_root_path
path = get_root_path() + configs['log_path']
if logger_type == LoggerTypes.SQL and 'sql_log_path' in configs:
path = get_root_path() + configs['sql_log_path']
if logger_type == LoggerTypes.DDL and 'ddl_log_path' in configs:
path = get_root_path() + configs['ddl_log_path']
if not os.path.exists(path):
os.makedirs(path)
filename = 'LOG {0:%Y-%m-%d %H.%M.%S} {1}.log'.format(datetime.now(), logger_type)
if runid:
filename = 'LOG {1:%Y-%m-%d %H.%M.%S} RUN{0:07.2f} {2}.log'.format(runid, datetime.now(), logger_type)
if filename_args:
filename = filename.replace('.log', '_' + filename_args + '.log')
return path, filename
def send_log_mail(self):
if not 'email_settings' in self.config:
return
elif 'send_log_mail_after_run' in self.config['email_settings'] and self.config['email_settings']['send_log_mail_after_run']:
params = self.config['email_settings']
params['to'] = params['to'].replace(';', ',')
from main import get_root_path
params['attachments_command'] = ' -a "' + get_root_path() + self.config['log_path'] + self.logger.filename + '"'
if self.logger.errors:
params['subject'] += ' ER IS IETS FOUT GEGAAN '
params['attachments_command'] += ' -a "' + get_root_path() + self.config['log_path'] + self.error_logger.filename + '"'
from sys import platform
if platform == "linux" or platform == "linux2":
linux_cmd = """echo -e "{msg}" | mail {attachments_command} -s "{subject}" -r "{from}" "{to}" """.format(**params)
import os
os.system(linux_cmd)
def test_run2_from_source_to_sor(self):
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/patienten2.csv'
self.pipe.mappings[0].delimiter = ';'
self.pipe.mappings[1].file_name = get_root_path(
) + '/PYELT/tests/data/zorgtrajecten4.csv'
self.pipeline.run()
test_row_count(self, 'sor_test_system.patient_hstage', 7)
test_row_count(self, 'dv.patient_hub', 5)
test_row_count(self, 'dv.patient_sat', 5)
test_row_count(self, 'dv.patient_sat_personalia', 5)
test_row_count(self, 'dv.patient_sat_adres', 7)
test_row_count(self, 'dv.patient_sat_inschrijving', 6)
test_row_count(self, 'dv.patient_sat_contactgegevens', 11)
test_row_count(self, 'dv.patient_traject_link', 5)
test_row_count(self, 'dv.traject_sat_record_status', 2,
'deleted is not null')
test_row_count(self, 'dv.patient_traject_link_sat_record_status', 2,
'deleted is not null')
def get_or_create_datatransfer_path(self):
# path = pyelt_config['datatransfer_path']
path = self.temp_datatransfer_path
if not path:
from main import get_root_path
path = get_root_path() + '/data/transfer/'
path += '/' + self.name
if not os.path.exists(path):
os.makedirs(path)
return path
def test03_dv_updates(self):
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/zorgverlenersB_rob.csv'
self.pipeline.run()
test_row_count(self, 'sor_test_system.zorgverlener_hstage', 7)
test_row_count(self, 'dv.zorgverlener_hub', 4)
test_row_count(self, 'dv.zorgverlener_sat', 4)
test_row_count(self, 'dv.zorgverlener_sat_personalia', 4)
def test08_postcode_correct(self):
print("test_run4a:\n")
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/zorgverleners4_rob.csv'
self.pipeline.run()
result = get_field_value_from_table('postcode',
'pyelt_unittests.dv.adres_sat',
"""char_length(postcode) >7""")
# detecteer dat er geen strings zijn met een te lange postcode
self.assertTrue(len(result) == 0)
def test_run2_from_source_to_sor(self):
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/patienten2.csv'
self.pipe.mappings[0].delimiter = ';'
self.pipeline.run()
get_row_count(self, 'sor_test_system.patient_hstage', 7)
get_row_count(self, 'dv.patient_hub', 5)
get_row_count(self, 'dv.patient_sat', 5)
get_row_count(self, 'dv.patient_sat_personalia', 5)
get_row_count(self, 'dv.patient_sat_adres', 7)
get_row_count(self, 'dv.patient_sat_inschrijving', 6)
get_row_count(self, 'dv.patient_sat_contactgegevens', 11)
def run_staging():
pipeline = Pipeline(config)
pipe = pipeline.get_or_create_pipe('test_source', source_config)
source_file = CsvFile(get_root_path() + '/sample_data/patienten1.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['patientnummer'])
mapping = SourceToSorMapping(source_file, 'persoon_hstage', auto_map=True)
pipe.mappings.append(mapping)
pipeline.run()
def init_source_to_sor_mappings():
mappings = []
source_file = CsvFile(get_root_path() +
'/PYELT/tests/data/zorgverlenersA_rob.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['zorgverlenernummer'])
sor_mapping = SourceToSorMapping(source_file,
'zorgverlener_hstage',
auto_map=True)
mappings.append(sor_mapping)
source_file = CsvFile(get_root_path() +
'/PYELT/tests/data/zorginstelling_rob.csv',
delimiter=';')
source_file.reflect()
source_file.set_primary_key(['zorginstellings_nummer'])
sor_mapping = SourceToSorMapping(source_file,
'zorginstelling_hstage',
auto_map=True)
mappings.append(sor_mapping)
return mappings
def test06_invalid_data_entry(self):
# huisnummer niet als getal, maar als letter ingevoerd:
print("test_run3a:\n")
expected_error = None
try:
self.pipe.mappings[0].file_name = get_root_path(
) + '/tests/data/zorgverleners3_rob.csv'
self.pipeline.run()
except Exception as err:
expected_error = err
self.assertIsNotNone(expected_error, 'ik verwachte een error')
def get_or_create_datatransfer_path(self):
path = pyelt_config['datatransfer_path']
if not path:
from main import get_root_path
path = get_root_path() + '/data/transfer/'
path += '/' + self.name
if not os.path.exists(path):
os.makedirs(path)
return path
#delete /tmp/datatranfser
if self.source_db:
path = self.source_db.get_or_create_datatransfer_path()
import shutil
shutil.rmtree(path)
def test_source_to_sor_mappings(self):
source_file = CsvFile(get_root_path() +
'/PYELT/tests/data/patienten1.csv',
delimiter=';')
source_file.reflect()
l = len(source_file.columns)
self.assertEqual(len(source_file.columns), 16)
self.assertEqual(len(source_file.primary_keys()), 0)
source_file.set_primary_key(['patientnummer'])
self.assertEqual(len(source_file.primary_keys()), 1)
sor_mapping = SourceToSorMapping(source_file,
'persoon_hstage',
auto_map=True)
self.assertEqual(sor_mapping.name, 'patienten1.csv -> persoon_hstage')
self.assertEqual(len(sor_mapping.field_mappings), 16)
self.pipe.mappings.append(sor_mapping)
def test09_null_and_hybridsat_update(self):
# deze unittest test 2 dingen: wordt een Null veld geupdate en wordt een hybride_sat geupdate
print("test_run5:\n")
self.pipe.mappings[0].file_name = get_root_path(
) + '/PYELT/tests/data/zorgverleners4_rob.csv'
self.pipeline.run()
result = get_field_value_from_dv_table(
'telnummer', 'zorgverlener', 'contactgegevens', '448',
["""type = 'mobiel2'""", """_active = True"""])
if len(result) > 0:
result = result[0][0]
else:
result = None
self.assertIsNotNone(
result,
'Ik verwacht dat een verandering in een oorspronkelijk Null veld wel in de DV laag terecht zou komen; bestaat de gebruikte bk wel?'
)
print(e)
def delete_all_ddl_logs(folder):
for the_file in os.listdir(folder):
file_path = os.path.join(folder, the_file)
if 'DDL' in the_file:
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as e:
print(e)
def delete_logs_older_than_today(folder):
today = datetime.now()
today = today.replace(hour=00, minute=00, second=00)
for the_file in os.listdir(folder):
file_path = os.path.join(folder, the_file)
datetime_string = the_file[11:30]
datetime_date = datetime.strptime(datetime_string, '%Y-%m-%d %H.%M.%S')
if datetime_date < today:
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except Exception as e:
print(e)
delete_all_logs(get_root_path() + '/logs/')
