def test_bad_file():
data_file = "data/test_datax.h5"
# the file.verify will exit with -1
try:
data.verify(config_test, data_file)
except:
pass
time.sleep(1)
assert res.is_text_in_file(logfile, 'parameter error: file data/test_datax.h5 does not exist')
clean()
def test_conf_error_no_limits():
config = init('b')
find = 'limits'
replace = 'limitsx'
mod.replace_text_in_file(config, find, replace)
# the file.verify will exit with -1
try:
data.verify(config, None)
except:
pass
time.sleep(1)
assert res.is_text_in_file(logfile, 'configuration error: limits is not configured')
clean()
def test_ge():
config = init('d')
find = 'HDF'
replace = 'GE'
mod.replace_text_in_file(config, find, replace)
data_file = os.path.join(os.getcwd(),"test/data/test_data.ge4")
bad_indexes = data.verify(config, data_file)
bad_data = bad_indexes['data']
assert 1 in bad_data
assert 2 in bad_data
assert 3 in bad_data
assert 4 in bad_data
clean()
def test_qualitychecks():
config = init('a')
bad_indexes = data.verify(config, data_file)
bad_data_white = bad_indexes['data_white']
bad_data = bad_indexes['data']
bad_data_dark = bad_indexes['data_dark']
assert 0 in bad_data_white
assert 1 in bad_data_white
#assert 3 in bad_data_white
assert 4 in bad_data_white
assert 0 in bad_data
assert 3 in bad_data
assert 4 in bad_data
assert 0 in bad_data_dark
assert 1 in bad_data_dark
assert 2 in bad_data_dark
#assert 3 in bad_data_dark
assert 4 in bad_data_dark
clean()
def data(conf, fname):
"""
Data Quality verifier.
Parameters
----------
conf : str
name of the configuration file including path. If contains only directory, 'dqconfig_test.ini' will a default
file name.
file : str
file name to do the quality checks on
Returns
-------
bad_indexes : Dict
"""
bad_indexes = dqdata.verify(conf, fname)
print(json.dumps(bad_indexes))
return bad_indexes
def data(conf, fname):
"""
Data Quality verifier.
Parameters
----------
conf : str
name of the configuration file including path. If contains only directory, 'dqconfig_test.ini' will a default
file name.
file : str
file name to do the quality checks on
Returns
-------
bad_indexes : Dict
"""
bad_indexes = dqdata.verify(conf, fname)
print(json.dumps(bad_indexes))
return bad_indexes
# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago #
# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, #
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, #
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; #
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT #
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN #
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE #
# POSSIBILITY OF SUCH DAMAGE. #
# #########################################################################
"""
Please make sure the installation :ref:`pre-requisite-reference-label` are met.
This example shows how to verify the quality of an HDF file.
"""
import os
from os.path import expanduser
from configobj import ConfigObj
from dquality.data import verify
home = expanduser("~")
config = os.path.join(home, 'dqconfig.ini')
conf = ConfigObj(config)
file = conf['file']
verify(file)
# the documentation and/or other materials provided with the #
# distribution. #
# #
# * Neither the name of UChicago Argonne, LLC, Argonne National #
# Laboratory, ANL, the U.S. Government, nor the names of its #
# contributors may be used to endorse or promote products derived #
# from this software without specific prior written permission. #
# #
# THIS SOFTWARE IS PROVIDED BY UChicago Argonne, LLC AND CONTRIBUTORS #
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT #
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS #
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UChicago #
# Argonne, LLC OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, #
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, #
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; #
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT #
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN #
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE #
# POSSIBILITY OF SUCH DAMAGE. #
# #########################################################################
"""
Please make sure the installation :ref:`pre-requisite-reference-label` are met.
This example shows how to verify the quality of an HDF file.
"""
from dquality.data import verify
verify()
