def test_bad_directory(): config = init('de') directory = "bad_dir" # the file.verify will exit with -1 try: acc.verify(config, directory, data_type, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'parameter error: directory bad_dir does not exist') clean()
def test_bad_directory(): config = init('be') directory = "bad_dir" # the file.verify will exit with -1 try: acc.verify(config, directory, data_type, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'parameter error: directory bad_dir does not exist') clean()
def test_no_limit(): config = init('ef') find = 'limits.json' replace = 'limitsx.json' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: acc.verify(config, None, data_type, 1, True) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'configuration error: file test/schemas/limitsx.json does not exist') clean()
def test_conf_error_no_limits(): config = init('cc') find = 'limits' replace = 'limitsx' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: acc.verify(config, None, data_type, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'configuration error: limits is not configured') clean()
def test_no_limit(): config = init('bg') find = 'limits.json' replace = 'limitsx.json' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: acc.verify(config, None, data_type, 1, True) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'configuration error: file test/schemas/limitsx.json does not exist') clean()
def test_conf_error_no_limits(): config = init('bf') find = 'limits' replace = 'limitsx' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: acc.verify(config, None, data_type, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'configuration error: limits is not configured') clean()
def accumulator(conf, fname, dtype, num_files, report_by_file): """ Data Quality monitor. Parameters ---------- conf : str configuration file name, including path folder : str monitored directory data_type : str defines which data type is being evaluated num_files : int number of files that will be processed report_by_files : boolean this variable directs how to present the bad indexes in a report. If True, the indexes are related to the files, and a filename is included in the report. Otherwise, the report contains a list of bad indexes. Returns ------- bad_indexes : dict a dictionary or list containing bad indexes """ bad_indexes = acc.verify(conf, fname, dtype, int(num_files), report_by_file) print(json.dumps(bad_indexes)) return bad_indexes
def test_qualitychecks_report_for_file(): config = init('ab') data_path = os.path.join(os.getcwd(),"test/data1") new_data1 = os.path.join(data_path,"test_data1.h5") new_data2 = os.path.join(data_path,"test_data2.h5") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(data_file, new_data1, new_data2)) p.start() bad_indexes = acc.verify(config, data_path, data_type, 2, 'True') bad_indexes_type = bad_indexes[data_type] bad_data1 = bad_indexes_type[new_data1] bad_data2 = bad_indexes_type[new_data2] assert 0 in bad_data1 assert 0 in bad_data2 assert 1 in bad_data1 assert 1 in bad_data2 #assert 3 in bad_data1 #assert 3 in bad_data2 assert 4 in bad_data1 assert 4 in bad_data2 assert not res.is_text_in_file(logfile, 'no file extension specified. Monitoring for all files') clean(new_data1, new_data2)
def test_no_extentions_interrupt(): config = init('bd') data_path = os.path.join(os.getcwd(),"test/data1") new_data1 = os.path.join(data_path,"INTERRUPT") new_data2 = os.path.join(data_path,"INTERRUPT.h5") find = 'extensions' replace = 'extensionsx' mod.replace_text_in_file(config, find, replace) if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(data_file, new_data1, new_data2)) p.start() bad_indexes = acc.verify(config, data_path, data_type, 2, True) assert len(bad_indexes) is 0 assert res.is_text_in_file(logfile, 'no file extension specified. Monitoring for all files') clean()
def test_qualitychecks_one_report(): config = init('bc') data_path = os.path.join(os.getcwd(),"test/data1") new_data1 = os.path.join(data_path,"test_data3.h5") new_data2 = os.path.join(data_path,"test_data4.h5") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(data_file, new_data1, new_data2)) p.start() bad_indexes = acc.verify(config, data_path, data_type, 2, False) bad_data = bad_indexes[data_type] assert 0 in bad_data assert 4 in bad_data assert 5 in bad_data assert 9 in bad_data assert not res.is_text_in_file(logfile, 'no file extension specified. Monitoring for all files') clean(new_data1, new_data2)