def test_load_antenna_metrics():
# load a metrics file and check some values
metrics_file = os.path.join(DATA_PATH, 'example_ant_metrics.hdf5')
metrics = ant_metrics.load_antenna_metrics(metrics_file)
assert np.isclose(metrics['final_mod_z_scores']['meanVijXPol'][(72, 'x')],
0.17529333517595402)
assert np.isclose(metrics['final_mod_z_scores']['meanVijXPol'][(72, 'y')],
0.17529333517595402)
assert np.isclose(metrics['final_mod_z_scores']['meanVijXPol'][(31, 'y')],
0.7012786080508268)
# change some values to FPE values, and write it out
metrics['final_mod_z_scores']['meanVijXPol'][(72, 'x')] = np.nan
metrics['final_mod_z_scores']['meanVijXPol'][(72, 'y')] = np.inf
metrics['final_mod_z_scores']['meanVijXPol'][(31, 'y')] = -np.inf
outpath = os.path.join(DATA_PATH, 'test_output', 'ant_metrics_output.hdf5')
metrics_io.write_metric_file(outpath, metrics, overwrite=True)
# test reading it back in, and that the values agree
metrics_new = ant_metrics.load_antenna_metrics(outpath)
assert np.isnan(metrics_new['final_mod_z_scores']['meanVijXPol'][(72,
'x')])
assert np.isinf(metrics_new['final_mod_z_scores']['meanVijXPol'][(72,
'y')])
assert np.isneginf(metrics_new['final_mod_z_scores']['meanVijXPol'][(31,
'y')])
# clean up after ourselves
os.remove(outpath)
def test_read_write_new_ant_json_files():
"""Test the new ant_metric json storage can be read and written to hdf5."""
json_infile = os.path.join(DATA_PATH, 'example_ant_metrics.json')
test_file = os.path.join(DATA_PATH, 'test_output',
'test_ant_json_to_hdf5.h5')
warn_message = [
"JSON-type files can still be read but are no longer "
"written by default.\n"
"Write to HDF5 format for future compatibility.",
]
test_metrics = uvtest.checkWarnings(metrics_io.load_metric_file,
func_args=[json_infile],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
metrics_io.write_metric_file(test_file, test_metrics, overwrite=True)
test_metrics_in = metrics_io.load_metric_file(test_file)
# The written hdf5 may have these keys that differ by design
# so ignore them.
test_metrics.pop('history', None)
test_metrics.pop('version', None)
test_metrics_in.pop('history', None)
test_metrics_in.pop('version', None)
# This function recursively walks dictionary and compares
# data types together with nt.assert_type_equal or np.allclose
assert qmtest.recursive_compare_dicts(test_metrics, test_metrics_in)
assert os.path.exists(test_file)
os.remove(test_file)
def test_write_then_recursive_load_dict_to_group_with_nested_dicts():
"""Test recursive load can gather dictionary from a nested group."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
path = '/'
good_dict = {
'filestem': test_scalar,
'history': "this is a test",
'version': hera_qm_version_str,
'1': {
'filestem': test_scalar
}
}
metrics_io.write_metric_file(test_file, good_dict)
with h5py.File(test_file, 'r') as h5file:
read_dict = metrics_io._recursively_load_dict_to_group(
h5file, '/Header/')
read_dict.update(
metrics_io._recursively_load_dict_to_group(h5file, '/Metrics/'))
metrics_io._recursively_validate_dict(read_dict)
for key in good_dict:
if isinstance(good_dict[key], dict):
assert qmtest.recursive_compare_dicts(good_dict[key],
read_dict[key])
else:
assert good_dict[key] == read_dict[key]
os.remove(test_file)
def test_write_metric_succeeds_for_existing_file_no_appellation_overwrite():
"""Test an write is successful if an existing file is given and overwrite=True."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test')
with open(test_file + '.hdf5', 'w') as f:
pass
metrics_io.write_metric_file(test_file, {}, True)
assert os.path.exists(test_file + '.hdf5')
os.remove(test_file + '.hdf5')
def test_write_metric_file_hdf5_no_appellation_exists():
"""Test hdf5 file created from write_metric_file if no appellation given."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test')
test_scalar = 'hello world'
test_array = np.arange(10)
test_dict = {'0': test_scalar, 1: {'0': test_scalar, '1': test_array}}
path = '/'
metrics_io.write_metric_file(test_file, test_dict)
test_file += '.hdf5'
nt.assert_true(os.path.exists(test_file))
os.remove(test_file)
def test_boolean_read_write_hdf5():
"""Test a Boolean type is preserved in read write loop: hdf5."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test_bool.h5')
test_bool = True
test_dict = {'good_sol': test_bool}
metrics_io.write_metric_file(test_file, test_dict, overwrite=True)
input_dict = metrics_io.load_metric_file(test_file)
assert test_dict['good_sol'], input_dict['good_sol']
assert isinstance(input_dict['good_sol'], (np.bool_, bool))
os.remove(test_file)
def test_write_metric_file_hdf5():
"""Test that correct hdf5 structure created from write_metric_file."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
test_array = np.arange(10)
test_dict = {'0': test_scalar, 1: {'0': test_scalar, '1': test_array}}
path = '/'
metrics_io.write_metric_file(test_file, test_dict)
with h5py.File(test_file, 'r') as test_h5:
nt.assert_equal(test_scalar, test_h5["/Metrics/0"].value)
nt.assert_equal(test_scalar, test_h5["/Metrics/1/0"].value)
nt.assert_true(np.allclose(test_array, test_h5["Metrics/1/1"].value))
os.remove(test_file)
def test_write_then_recursive_load_dict_to_group_no_nested_dicts():
"""Test recursive load can gather dictionary from a group."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
path = '/'
good_dict = {
'0': test_scalar,
'history': "this is a test",
'version': hera_qm_version_str
}
metrics_io.write_metric_file(test_file, good_dict)
with h5py.File(test_file, 'r') as h5file:
read_dict = metrics_io._recursively_load_dict_to_group(
h5file, '/Header/')
read_dict.update(
metrics_io._recursively_load_dict_to_group(h5file, '/Metrics/'))
metrics_io._recursively_validate_dict(read_dict)
nt.assert_dict_equal(good_dict, read_dict)
os.remove(test_file)
def test_write_metric_file_hdf5():
"""Test that correct hdf5 structure created from write_metric_file."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
test_array = np.arange(10)
test_dict = {
'filestem': test_scalar,
1: {
'filestem': test_scalar,
'1': test_array
}
}
path = '/'
metrics_io.write_metric_file(test_file, test_dict)
with h5py.File(test_file, 'r') as test_h5:
assert np.string_(test_scalar) == test_h5["/Metrics/filestem"][()]
assert np.string_(test_scalar) == test_h5["/Metrics/1/filestem"][()]
assert np.allclose(test_array, test_h5["Metrics/1/1"][()])
os.remove(test_file)
def test_write_then_load_metric_file_hdf5():
"""Test loaded in map is same as written one from hdf5."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
path = '/'
good_dict = {
'0': test_scalar,
'history': "this is a test",
'version': hera_qm_version_str,
'all_metrics': {
'0': test_scalar
}
}
metrics_io.write_metric_file(test_file, good_dict)
read_dict = metrics_io.load_metric_file(test_file)
for key in good_dict:
if isinstance(good_dict[key], dict):
nt.assert_dict_equal(good_dict[key], read_dict[key])
else:
nt.assert_equal(good_dict[key], read_dict[key])
os.remove(test_file)
