def test_bad_directory(): config = init('cc') directory = "bad_dir" # the file.verify will exit with -1 try: monitor.verify(config, directory, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'parameter error: directory bad_dir does not exist') clean()
def test_bad_directory(): config = init('cc') directory = "bad_dir" # the file.verify will exit with -1 try: monitor.verify(config, directory, 1) except: pass time.sleep(1) assert res.is_text_in_file( logfile, 'parameter error: directory bad_dir does not exist') clean()
def test_conf_error_no_limits(): config = init('dd') find = 'limits' replace = 'limitsx' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: monitor.verify(config, None, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'configuration error: limits is not configured') clean()
def test_no_limit(): config = init('ee') find = 'limits.json' replace = 'limitsx.json' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: monitor.verify(config, None, 1) except: pass time.sleep(1) assert res.is_text_in_file(logfile, 'configuration error: file test/schemas/limitsx.json does not exist') clean()
def test_conf_error_no_limits(): config = init('dd') find = 'limits' replace = 'limitsx' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: monitor.verify(config, None, 1) except: pass time.sleep(1) assert res.is_text_in_file( logfile, 'configuration error: limits is not configured') clean()
def test_no_limit(): config = init('ee') find = 'limits.json' replace = 'limitsx.json' mod.replace_text_in_file(config, find, replace) # the file.verify will exit with -1 try: monitor.verify(config, None, 1) except: pass time.sleep(1) assert res.is_text_in_file( logfile, 'configuration error: file test/schemas/limitsx.json does not exist') clean()
def test_ge(): config = init('ef') data_path = os.path.join(os.getcwd(),"test/data1") data_file = os.path.join(os.getcwd(),"test/data/test_data.ge4") new_data = os.path.join(data_path,"test_data.ge4") find = 'HDF' replace = 'GE' mod.replace_text_in_file(config, find, replace) mod.delete_line_in_file(config, 'extensions') mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(data_file, new_data,)) p.start() bad_indexes_file = monitor.verify(config, data_path, 1) bad_indexes = bad_indexes_file[new_data] bad_data = bad_indexes['data'] assert 1 in bad_data assert 2 in bad_data assert 3 in bad_data assert 4 in bad_data clean()
def test_qualitychecks(): config = init('aa') data_path = os.path.join(os.getcwd(),"test/data1") new_data = os.path.join(data_path,"test_data.h5") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(data_file, new_data,)) p.start() bad_indexes_file = monitor.verify(config, data_path, 1) bad_indexes = bad_indexes_file[new_data] bad_data_white = bad_indexes['data_white'] bad_data = bad_indexes['data'] bad_data_dark = bad_indexes['data_dark'] assert 0 in bad_data_white assert 1 in bad_data_white #assert 3 in bad_data_white assert 4 in bad_data_white assert 0 in bad_data assert 3 in bad_data assert 4 in bad_data assert 0 in bad_data_dark assert 1 in bad_data_dark assert 2 in bad_data_dark #assert 3 in bad_data_dark assert 4 in bad_data_dark clean()
def test_qualitychecks(): config = init('aa') data_path = os.path.join(os.getcwd(), "test/data1") new_data = os.path.join(data_path, "test_data.h5") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=( data_file, new_data, )) p.start() bad_indexes_file = monitor.verify(config, data_path, 1) bad_indexes = bad_indexes_file[new_data] bad_data_white = bad_indexes['data_white'] bad_data = bad_indexes['data'] bad_data_dark = bad_indexes['data_dark'] assert 5 in bad_data_white assert 9 in bad_data_white assert 10 in bad_data assert 13 in bad_data assert 14 in bad_data assert 0 in bad_data_dark assert 1 in bad_data_dark assert 2 in bad_data_dark assert 3 in bad_data_dark assert 4 in bad_data_dark clean()
def test_ge(): config = init('ff') data_path = os.path.join(os.getcwd(), "test/data1") data_file = os.path.join(os.getcwd(), "test/data/test_data.ge4") new_data = os.path.join(data_path, "test_data.ge4") find = 'HDF' replace = 'GE' mod.replace_text_in_file(config, find, replace) mod.delete_line_in_file(config, 'extensions') mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=( data_file, new_data, )) p.start() bad_indexes_file = monitor.verify(config, data_path, 1) bad_indexes = bad_indexes_file[new_data] bad_data = bad_indexes['data'] assert 1 in bad_data assert 2 in bad_data assert 3 in bad_data assert 4 in bad_data clean()
def test_no_extentions_interrupt(): config = init('bb') data_path = os.path.join(os.getcwd(),"test/data1") new_data = os.path.join(data_path,"INTERRUPT") find = 'extensions' replace = 'extensionsx' mod.replace_text_in_file(config, find, replace) if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(interrupt_file, new_data,)) p.start() bad_indexes = monitor.verify(config, data_path, 2) assert len(bad_indexes) is 0 assert res.is_text_in_file(logfile, 'no file extension specified. Monitoring for all files') clean()
def test_no_extentions_interrupt(): config = init('bb') data_path = os.path.join(os.getcwd(), "test/data1") new_data = os.path.join(data_path, "INTERRUPT") find = 'extensions' replace = 'extensionsx' mod.replace_text_in_file(config, find, replace) if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=( interrupt_file, new_data, )) p.start() bad_indexes = monitor.verify(config, data_path, 2) assert len(bad_indexes) is 0 assert res.is_text_in_file( logfile, 'no file extension specified. Monitoring for all files') clean()
def test_ge_corrupted_file(): config = init('ef') data_path = os.path.join(os.getcwd(),"test/data1") new_data = os.path.join(data_path,"test_data.ge4") find = 'HDF' replace = 'GE' mod.replace_text_in_file(config, find, replace) mod.delete_line_in_file(config, 'extensions') mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=(data_file, new_data,)) p.start() bad_indexes_file = monitor.verify(config, data_path, 1) bad_indexes = bad_indexes_file[new_data] assert bad_indexes is None assert res.is_text_in_file(logfile, 'GE image size unexpected:') clean()
def monitor(conf, folder, num_files): """ Data quality monitor verifier. Parameters ---------- conf : str configuration file name including path folder : str folder name to monitor num_files : int expected number of files. This script will exit after detecting and processing given number of files. Returns ------- None """ bad_indexes = dqdmonitor.verify(conf, folder, int(num_files)) return bad_indexes
def test_ge_corrupted_file(): config = init('gg') data_path = os.path.join(os.getcwd(), "test/data1") new_data = os.path.join(data_path, "test_data.ge4") find = 'HDF' replace = 'GE' mod.replace_text_in_file(config, find, replace) mod.delete_line_in_file(config, 'extensions') mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3") if not os.path.exists(data_path): os.makedirs(data_path) p = Process(target=copy_file, args=( data_file, new_data, )) p.start() bad_indexes_file = monitor.verify(config, data_path, 1) bad_indexes = bad_indexes_file[new_data] assert bad_indexes is None assert res.is_text_in_file(logfile, 'GE image size unexpected:') clean()
# # # THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS # # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS # # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago # # Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, # # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, # # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; # # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER # # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN # # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # # POSSIBILITY OF SUCH DAMAGE. # # ######################################################################### """ Please make sure the installation :ref:`pre-requisite-reference-label` are met. This example shows how to verify file newly created in a monitored folder. This test can be done at during data collection to confirm mandatory process variables are accessible and their values are within acceptable range. """ import dquality.monitor as monitor if monitor.verify(): print ('All PVs listed in pvs.json exist and meet conditions') else: print ('Some of the PVs listed in pvs.json do not exist or do not meet conditions')