def test_load_antenna_metrics(): # load a metrics file and check some values metrics_file = os.path.join(DATA_PATH, 'example_ant_metrics.hdf5') metrics = ant_metrics.load_antenna_metrics(metrics_file) assert np.isclose(metrics['final_mod_z_scores']['meanVijXPol'][(72, 'x')], 0.17529333517595402) assert np.isclose(metrics['final_mod_z_scores']['meanVijXPol'][(72, 'y')], 0.17529333517595402) assert np.isclose(metrics['final_mod_z_scores']['meanVijXPol'][(31, 'y')], 0.7012786080508268) # change some values to FPE values, and write it out metrics['final_mod_z_scores']['meanVijXPol'][(72, 'x')] = np.nan metrics['final_mod_z_scores']['meanVijXPol'][(72, 'y')] = np.inf metrics['final_mod_z_scores']['meanVijXPol'][(31, 'y')] = -np.inf outpath = os.path.join(DATA_PATH, 'test_output', 'ant_metrics_output.hdf5') metrics_io.write_metric_file(outpath, metrics, overwrite=True) # test reading it back in, and that the values agree metrics_new = ant_metrics.load_antenna_metrics(outpath) assert np.isnan(metrics_new['final_mod_z_scores']['meanVijXPol'][(72, 'x')]) assert np.isinf(metrics_new['final_mod_z_scores']['meanVijXPol'][(72, 'y')]) assert np.isneginf(metrics_new['final_mod_z_scores']['meanVijXPol'][(31, 'y')]) # clean up after ourselves os.remove(outpath)
def test_read_write_new_ant_json_files(): """Test the new ant_metric json storage can be read and written to hdf5.""" json_infile = os.path.join(DATA_PATH, 'example_ant_metrics.json') test_file = os.path.join(DATA_PATH, 'test_output', 'test_ant_json_to_hdf5.h5') warn_message = [ "JSON-type files can still be read but are no longer " "written by default.\n" "Write to HDF5 format for future compatibility.", ] test_metrics = uvtest.checkWarnings(metrics_io.load_metric_file, func_args=[json_infile], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) metrics_io.write_metric_file(test_file, test_metrics, overwrite=True) test_metrics_in = metrics_io.load_metric_file(test_file) # The written hdf5 may have these keys that differ by design # so ignore them. test_metrics.pop('history', None) test_metrics.pop('version', None) test_metrics_in.pop('history', None) test_metrics_in.pop('version', None) # This function recursively walks dictionary and compares # data types together with nt.assert_type_equal or np.allclose assert qmtest.recursive_compare_dicts(test_metrics, test_metrics_in) assert os.path.exists(test_file) os.remove(test_file)
def test_write_then_recursive_load_dict_to_group_with_nested_dicts(): """Test recursive load can gather dictionary from a nested group.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' path = '/' good_dict = { 'filestem': test_scalar, 'history': "this is a test", 'version': hera_qm_version_str, '1': { 'filestem': test_scalar } } metrics_io.write_metric_file(test_file, good_dict) with h5py.File(test_file, 'r') as h5file: read_dict = metrics_io._recursively_load_dict_to_group( h5file, '/Header/') read_dict.update( metrics_io._recursively_load_dict_to_group(h5file, '/Metrics/')) metrics_io._recursively_validate_dict(read_dict) for key in good_dict: if isinstance(good_dict[key], dict): assert qmtest.recursive_compare_dicts(good_dict[key], read_dict[key]) else: assert good_dict[key] == read_dict[key] os.remove(test_file)
def test_write_metric_succeeds_for_existing_file_no_appellation_overwrite(): """Test an write is successful if an existing file is given and overwrite=True.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test') with open(test_file + '.hdf5', 'w') as f: pass metrics_io.write_metric_file(test_file, {}, True) assert os.path.exists(test_file + '.hdf5') os.remove(test_file + '.hdf5')
def test_write_metric_file_hdf5_no_appellation_exists(): """Test hdf5 file created from write_metric_file if no appellation given.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test') test_scalar = 'hello world' test_array = np.arange(10) test_dict = {'0': test_scalar, 1: {'0': test_scalar, '1': test_array}} path = '/' metrics_io.write_metric_file(test_file, test_dict) test_file += '.hdf5' nt.assert_true(os.path.exists(test_file)) os.remove(test_file)
def test_boolean_read_write_hdf5(): """Test a Boolean type is preserved in read write loop: hdf5.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test_bool.h5') test_bool = True test_dict = {'good_sol': test_bool} metrics_io.write_metric_file(test_file, test_dict, overwrite=True) input_dict = metrics_io.load_metric_file(test_file) assert test_dict['good_sol'], input_dict['good_sol'] assert isinstance(input_dict['good_sol'], (np.bool_, bool)) os.remove(test_file)
def test_write_metric_file_hdf5(): """Test that correct hdf5 structure created from write_metric_file.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' test_array = np.arange(10) test_dict = {'0': test_scalar, 1: {'0': test_scalar, '1': test_array}} path = '/' metrics_io.write_metric_file(test_file, test_dict) with h5py.File(test_file, 'r') as test_h5: nt.assert_equal(test_scalar, test_h5["/Metrics/0"].value) nt.assert_equal(test_scalar, test_h5["/Metrics/1/0"].value) nt.assert_true(np.allclose(test_array, test_h5["Metrics/1/1"].value)) os.remove(test_file)
def test_write_then_recursive_load_dict_to_group_no_nested_dicts(): """Test recursive load can gather dictionary from a group.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' path = '/' good_dict = { '0': test_scalar, 'history': "this is a test", 'version': hera_qm_version_str } metrics_io.write_metric_file(test_file, good_dict) with h5py.File(test_file, 'r') as h5file: read_dict = metrics_io._recursively_load_dict_to_group( h5file, '/Header/') read_dict.update( metrics_io._recursively_load_dict_to_group(h5file, '/Metrics/')) metrics_io._recursively_validate_dict(read_dict) nt.assert_dict_equal(good_dict, read_dict) os.remove(test_file)
def test_write_metric_file_hdf5(): """Test that correct hdf5 structure created from write_metric_file.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' test_array = np.arange(10) test_dict = { 'filestem': test_scalar, 1: { 'filestem': test_scalar, '1': test_array } } path = '/' metrics_io.write_metric_file(test_file, test_dict) with h5py.File(test_file, 'r') as test_h5: assert np.string_(test_scalar) == test_h5["/Metrics/filestem"][()] assert np.string_(test_scalar) == test_h5["/Metrics/1/filestem"][()] assert np.allclose(test_array, test_h5["Metrics/1/1"][()]) os.remove(test_file)
def test_write_then_load_metric_file_hdf5(): """Test loaded in map is same as written one from hdf5.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' path = '/' good_dict = { '0': test_scalar, 'history': "this is a test", 'version': hera_qm_version_str, 'all_metrics': { '0': test_scalar } } metrics_io.write_metric_file(test_file, good_dict) read_dict = metrics_io.load_metric_file(test_file) for key in good_dict: if isinstance(good_dict[key], dict): nt.assert_dict_equal(good_dict[key], read_dict[key]) else: nt.assert_equal(good_dict[key], read_dict[key]) os.remove(test_file)