def test_bad_directory():
config = init('cc')
directory = "bad_dir"
# the file.verify will exit with -1
try:
monitor.verify(config, directory, 1)
except:
pass
time.sleep(1)
assert res.is_text_in_file(logfile, 'parameter error: directory bad_dir does not exist')
clean()
def test_bad_directory():
config = init('cc')
directory = "bad_dir"
# the file.verify will exit with -1
try:
monitor.verify(config, directory, 1)
except:
pass
time.sleep(1)
assert res.is_text_in_file(
logfile, 'parameter error: directory bad_dir does not exist')
clean()
def test_conf_error_no_limits():
config = init('dd')
find = 'limits'
replace = 'limitsx'
mod.replace_text_in_file(config, find, replace)
# the file.verify will exit with -1
try:
monitor.verify(config, None, 1)
except:
pass
time.sleep(1)
assert res.is_text_in_file(logfile, 'configuration error: limits is not configured')
clean()
def test_no_limit():
config = init('ee')
find = 'limits.json'
replace = 'limitsx.json'
mod.replace_text_in_file(config, find, replace)
# the file.verify will exit with -1
try:
monitor.verify(config, None, 1)
except:
pass
time.sleep(1)
assert res.is_text_in_file(logfile, 'configuration error: file test/schemas/limitsx.json does not exist')
clean()
def test_conf_error_no_limits():
config = init('dd')
find = 'limits'
replace = 'limitsx'
mod.replace_text_in_file(config, find, replace)
# the file.verify will exit with -1
try:
monitor.verify(config, None, 1)
except:
pass
time.sleep(1)
assert res.is_text_in_file(
logfile, 'configuration error: limits is not configured')
clean()
def test_no_limit():
config = init('ee')
find = 'limits.json'
replace = 'limitsx.json'
mod.replace_text_in_file(config, find, replace)
# the file.verify will exit with -1
try:
monitor.verify(config, None, 1)
except:
pass
time.sleep(1)
assert res.is_text_in_file(
logfile,
'configuration error: file test/schemas/limitsx.json does not exist')
clean()
def test_ge():
config = init('ef')
data_path = os.path.join(os.getcwd(),"test/data1")
data_file = os.path.join(os.getcwd(),"test/data/test_data.ge4")
new_data = os.path.join(data_path,"test_data.ge4")
find = 'HDF'
replace = 'GE'
mod.replace_text_in_file(config, find, replace)
mod.delete_line_in_file(config, 'extensions')
mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3")
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(data_file, new_data,))
p.start()
bad_indexes_file = monitor.verify(config, data_path, 1)
bad_indexes = bad_indexes_file[new_data]
bad_data = bad_indexes['data']
assert 1 in bad_data
assert 2 in bad_data
assert 3 in bad_data
assert 4 in bad_data
clean()
def test_qualitychecks():
config = init('aa')
data_path = os.path.join(os.getcwd(),"test/data1")
new_data = os.path.join(data_path,"test_data.h5")
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(data_file, new_data,))
p.start()
bad_indexes_file = monitor.verify(config, data_path, 1)
bad_indexes = bad_indexes_file[new_data]
bad_data_white = bad_indexes['data_white']
bad_data = bad_indexes['data']
bad_data_dark = bad_indexes['data_dark']
assert 0 in bad_data_white
assert 1 in bad_data_white
#assert 3 in bad_data_white
assert 4 in bad_data_white
assert 0 in bad_data
assert 3 in bad_data
assert 4 in bad_data
assert 0 in bad_data_dark
assert 1 in bad_data_dark
assert 2 in bad_data_dark
#assert 3 in bad_data_dark
assert 4 in bad_data_dark
clean()
def test_qualitychecks():
config = init('aa')
data_path = os.path.join(os.getcwd(), "test/data1")
new_data = os.path.join(data_path, "test_data.h5")
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(
data_file,
new_data,
))
p.start()
bad_indexes_file = monitor.verify(config, data_path, 1)
bad_indexes = bad_indexes_file[new_data]
bad_data_white = bad_indexes['data_white']
bad_data = bad_indexes['data']
bad_data_dark = bad_indexes['data_dark']
assert 5 in bad_data_white
assert 9 in bad_data_white
assert 10 in bad_data
assert 13 in bad_data
assert 14 in bad_data
assert 0 in bad_data_dark
assert 1 in bad_data_dark
assert 2 in bad_data_dark
assert 3 in bad_data_dark
assert 4 in bad_data_dark
clean()
def test_ge():
config = init('ff')
data_path = os.path.join(os.getcwd(), "test/data1")
data_file = os.path.join(os.getcwd(), "test/data/test_data.ge4")
new_data = os.path.join(data_path, "test_data.ge4")
find = 'HDF'
replace = 'GE'
mod.replace_text_in_file(config, find, replace)
mod.delete_line_in_file(config, 'extensions')
mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3")
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(
data_file,
new_data,
))
p.start()
bad_indexes_file = monitor.verify(config, data_path, 1)
bad_indexes = bad_indexes_file[new_data]
bad_data = bad_indexes['data']
assert 1 in bad_data
assert 2 in bad_data
assert 3 in bad_data
assert 4 in bad_data
clean()
def test_no_extentions_interrupt():
config = init('bb')
data_path = os.path.join(os.getcwd(),"test/data1")
new_data = os.path.join(data_path,"INTERRUPT")
find = 'extensions'
replace = 'extensionsx'
mod.replace_text_in_file(config, find, replace)
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(interrupt_file, new_data,))
p.start()
bad_indexes = monitor.verify(config, data_path, 2)
assert len(bad_indexes) is 0
assert res.is_text_in_file(logfile, 'no file extension specified. Monitoring for all files')
clean()
def test_no_extentions_interrupt():
config = init('bb')
data_path = os.path.join(os.getcwd(), "test/data1")
new_data = os.path.join(data_path, "INTERRUPT")
find = 'extensions'
replace = 'extensionsx'
mod.replace_text_in_file(config, find, replace)
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(
interrupt_file,
new_data,
))
p.start()
bad_indexes = monitor.verify(config, data_path, 2)
assert len(bad_indexes) is 0
assert res.is_text_in_file(
logfile, 'no file extension specified. Monitoring for all files')
clean()
def test_ge_corrupted_file():
config = init('ef')
data_path = os.path.join(os.getcwd(),"test/data1")
new_data = os.path.join(data_path,"test_data.ge4")
find = 'HDF'
replace = 'GE'
mod.replace_text_in_file(config, find, replace)
mod.delete_line_in_file(config, 'extensions')
mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3")
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(data_file, new_data,))
p.start()
bad_indexes_file = monitor.verify(config, data_path, 1)
bad_indexes = bad_indexes_file[new_data]
assert bad_indexes is None
assert res.is_text_in_file(logfile, 'GE image size unexpected:')
clean()
def monitor(conf, folder, num_files):
"""
Data quality monitor verifier.
Parameters
----------
conf : str
configuration file name including path
folder : str
folder name to monitor
num_files : int
expected number of files. This script will exit after detecting and
processing given number of files.
Returns
-------
None
"""
bad_indexes = dqdmonitor.verify(conf, folder, int(num_files))
return bad_indexes
def monitor(conf, folder, num_files):
"""
Data quality monitor verifier.
Parameters
----------
conf : str
configuration file name including path
folder : str
folder name to monitor
num_files : int
expected number of files. This script will exit after detecting and
processing given number of files.
Returns
-------
None
"""
bad_indexes = dqdmonitor.verify(conf, folder, int(num_files))
return bad_indexes
def test_ge_corrupted_file():
config = init('gg')
data_path = os.path.join(os.getcwd(), "test/data1")
new_data = os.path.join(data_path, "test_data.ge4")
find = 'HDF'
replace = 'GE'
mod.replace_text_in_file(config, find, replace)
mod.delete_line_in_file(config, 'extensions')
mod.add_line_to_file(config, "'extensions' = .ge, .ge4, .ge3")
if not os.path.exists(data_path):
os.makedirs(data_path)
p = Process(target=copy_file, args=(
data_file,
new_data,
))
p.start()
bad_indexes_file = monitor.verify(config, data_path, 1)
bad_indexes = bad_indexes_file[new_data]
assert bad_indexes is None
assert res.is_text_in_file(logfile, 'GE image size unexpected:')
clean()
# #
# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago #
# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, #
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, #
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; #
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT #
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN #
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE #
# POSSIBILITY OF SUCH DAMAGE. #
# #########################################################################
"""
Please make sure the installation :ref:`pre-requisite-reference-label` are met.
This example shows how to verify file newly created in a monitored folder.
This test can be done at during data collection to confirm mandatory process
variables are accessible and their values are within acceptable range.
"""
import dquality.monitor as monitor
if monitor.verify():
print ('All PVs listed in pvs.json exist and meet conditions')
else:
print ('Some of the PVs listed in pvs.json do not exist or do not meet conditions')
