def test_write_then_load_metric_warning_json():
"""Test the known warning is issued when writing then reading json."""
json_file = os.path.join(DATA_PATH, 'test_output', 'test_save.json')
test_dict = {
'history': 'Test case',
'version': '0.0.0',
'dead_ant_z_cut': 5
}
warn_message = [
"JSON-type files can still be written but are no longer "
"written by default.\n"
"Write to HDF5 format for future compatibility.",
]
uvtest.checkWarnings(metrics_io.write_metric_file, [json_file, test_dict],
{'overwrite': True},
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
assert os.path.exists(json_file)
# output_json = metrics_io.write_metric_file(json_file, test_dict)
warn_message = [
"JSON-type files can still be read but are no longer "
"written by default.\n"
]
json_dict = uvtest.checkWarnings(metrics_io.load_metric_file,
func_args=[json_file],
category=[PendingDeprecationWarning],
nwarnings=1,
message=warn_message)
# This function recursively walks dictionary and compares
# data types together with assert or np.allclose
qmtest.recursive_compare_dicts(test_dict, json_dict)
os.remove(json_file)
def test_load_ant_metrics_json(self):
json_file = os.path.join(DATA_PATH, 'example_ant_metrics.json')
hdf5_file = os.path.join(DATA_PATH, 'example_ant_metrics.hdf5')
warn_message = [
"JSON-type files can still be read but are no longer "
"written by default.\n"
"Write to HDF5 format for future compatibility."
]
json_dict = uvtest.checkWarnings(ant_metrics.load_antenna_metrics,
func_args=[json_file],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
hdf5_dict = ant_metrics.load_antenna_metrics(hdf5_file)
# The written hdf5 may have these keys that differ by design
# so ignore them.
json_dict.pop('history', None)
json_dict.pop('version', None)
hdf5_dict.pop('history', None)
hdf5_dict.pop('version', None)
# This function recursively walks dictionary and compares
# data types together with nt.assert_type_equal or np.allclose
qmtest.recursive_compare_dicts(hdf5_dict, json_dict)
def test_read_old_all_string_ant_json_files():
"""Test the new ant_metric json storage can be read and written to hdf5."""
old_json_infile = os.path.join(DATA_PATH,
'example_ant_metrics_all_string.json')
new_json_infile = os.path.join(DATA_PATH, 'example_ant_metrics.json')
warn_message = [
"JSON-type files can still be read but are no longer "
"written by default.\n"
"Write to HDF5 format for future compatibility.",
]
test_metrics_old = uvtest.checkWarnings(metrics_io.load_metric_file,
func_args=[old_json_infile],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
test_metrics_new = uvtest.checkWarnings(metrics_io.load_metric_file,
func_args=[new_json_infile],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
# The written hdf5 may have these keys that differ by design
# so ignore them.
test_metrics_old.pop('history', None)
test_metrics_old.pop('version', None)
test_metrics_new.pop('history', None)
test_metrics_new.pop('version', None)
# This function recursively walks dictionary and compares
# data types together with nt.assert_type_equal or np.allclose
qmtest.recursive_compare_dicts(test_metrics_new, test_metrics_old)
def test_read_write_old_omnical_json_files():
"""Test the old omnical json storage can be read and written to hdf5."""
json_infile = os.path.join(DATA_PATH, 'example_omnical_metrics.json')
test_file = os.path.join(DATA_PATH, 'test_output',
'test_omnical_json_to_hdf5.h5')
warn_message = [
"JSON-type files can still be read but are no longer "
"written by default.\n"
"Write to HDF5 format for future compatibility.",
]
test_metrics = uvtest.checkWarnings(metrics_io.load_metric_file,
func_args=[json_infile],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
metrics_io.write_metric_file(test_file, test_metrics, overwrite=True)
test_metrics_in = metrics_io.load_metric_file(test_file)
# The written hdf5 may have these keys that differ by design
# so ignore them.
test_metrics.pop('history', None)
test_metrics.pop('version', None)
test_metrics_in.pop('history', None)
test_metrics_in.pop('version', None)
# This function recursively walks dictionary and compares
# data types together with nt.assert_type_equal or np.allclose
qmtest.recursive_compare_dicts(test_metrics, test_metrics_in)
nt.assert_true(os.path.exists(test_file))
os.remove(test_file)
def test_save_json(self):
am = ant_metrics.AntennaMetrics(self.dataFileList,
self.reds,
fileformat='miriad')
am.iterative_antenna_metrics_and_flagging()
for stat in self.summaryStats:
self.assertTrue(hasattr(am, stat))
self.assertIn((81, 'x'), am.xants)
self.assertIn((81, 'y'), am.xants)
self.assertIn((81, 'x'), am.deadAntsRemoved)
self.assertIn((81, 'y'), am.deadAntsRemoved)
outfile = os.path.join(DATA_PATH, 'test_output',
'ant_metrics_output.json')
warn_message = [
"JSON-type files can still be written "
"but are no longer written by default.\n"
"Write to HDF5 format for future compatibility."
]
uvtest.checkWarnings(am.save_antenna_metrics,
func_args=[outfile],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
# am.save_antenna_metrics(json_file)
warn_message = [
"JSON-type files can still be read but are no longer "
"written by default.\n"
"Write to HDF5 format for future compatibility."
]
loaded = uvtest.checkWarnings(ant_metrics.load_antenna_metrics,
func_args=[outfile],
category=PendingDeprecationWarning,
nwarnings=1,
message=warn_message)
_ = loaded.pop('history', '')
jsonStats = [
'xants', 'crossed_ants', 'dead_ants', 'removal_iteration',
'final_metrics', 'all_metrics', 'final_mod_z_scores',
'all_mod_z_scores', 'cross_pol_z_cut', 'dead_ant_z_cut',
'datafile_list', 'reds', 'version'
]
for stat, jsonStat in zip(self.summaryStats, jsonStats):
file_val = loaded[jsonStat]
obj_val = getattr(am, stat)
if isinstance(file_val, dict):
qmtest.recursive_compare_dicts(file_val, obj_val)
else:
self.assertEqual(file_val, obj_val)
os.remove(outfile)
def test_write_then_recursive_load_dict_to_group_with_nested_dicts():
"""Test recursive load can gather dictionary from a nested group."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
path = '/'
good_dict = {
'filestem': test_scalar,
'history': "this is a test",
'version': hera_qm_version_str,
'1': {
'filestem': test_scalar
}
}
metrics_io.write_metric_file(test_file, good_dict)
with h5py.File(test_file, 'r') as h5file:
read_dict = metrics_io._recursively_load_dict_to_group(
h5file, '/Header/')
read_dict.update(
metrics_io._recursively_load_dict_to_group(h5file, '/Metrics/'))
metrics_io._recursively_validate_dict(read_dict)
for key in good_dict:
if isinstance(good_dict[key], dict):
assert qmtest.recursive_compare_dicts(good_dict[key],
read_dict[key])
else:
assert good_dict[key] == read_dict[key]
os.remove(test_file)
def test_write_then_load_metric_file_hdf5():
"""Test loaded in map is same as written one from hdf5."""
test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5')
test_scalar = 'hello world'
path = '/'
good_dict = {
'filestem': test_scalar,
'history': "this is a test",
'version': hera_qm_version_str,
'all_metrics': {
'filestem': test_scalar
}
}
metrics_io.write_metric_file(test_file, good_dict)
read_dict = metrics_io.load_metric_file(test_file)
for key in good_dict:
if isinstance(good_dict[key], dict):
assert qmtest.recursive_compare_dicts(good_dict[key],
read_dict[key])
else:
assert good_dict[key] == read_dict[key]
os.remove(test_file)
def test_run_metrics_two_pols(firstcal_twopol):
# These results were run with a seed of 0, the seed shouldn't matter
# but you never know.
two_pol_known_results = os.path.join(
DATA_PATH, 'example_two_polarization_firstcal_results.hdf5')
np.random.seed(0)
firstcal_twopol.FC.run_metrics(std_cut=.5)
known_output = metrics_io.load_metric_file(two_pol_known_results)
known_output.pop('history', None)
known_output.pop('version', None)
# There are some full paths of files saved in the files
# Perhaps for record keeping, but that messes up the test comparison
for key in known_output:
known_output[key].pop('fc_filename', None)
known_output[key].pop('fc_filestem', None)
known_output[key].pop('version', None)
for key in firstcal_twopol.FC.metrics:
firstcal_twopol.FC.metrics[key].pop('fc_filename', None)
firstcal_twopol.FC.metrics[key].pop('fc_filestem', None)
firstcal_twopol.FC.metrics[key].pop('version', None)
assert qmtest.recursive_compare_dicts(firstcal_twopol.FC.metrics,
known_output)
