def test_write_then_load_metric_warning_json(): """Test the known warning is issued when writing then reading json.""" json_file = os.path.join(DATA_PATH, 'test_output', 'test_save.json') test_dict = { 'history': 'Test case', 'version': '0.0.0', 'dead_ant_z_cut': 5 } warn_message = [ "JSON-type files can still be written but are no longer " "written by default.\n" "Write to HDF5 format for future compatibility.", ] uvtest.checkWarnings(metrics_io.write_metric_file, [json_file, test_dict], {'overwrite': True}, category=PendingDeprecationWarning, nwarnings=1, message=warn_message) assert os.path.exists(json_file) # output_json = metrics_io.write_metric_file(json_file, test_dict) warn_message = [ "JSON-type files can still be read but are no longer " "written by default.\n" ] json_dict = uvtest.checkWarnings(metrics_io.load_metric_file, func_args=[json_file], category=[PendingDeprecationWarning], nwarnings=1, message=warn_message) # This function recursively walks dictionary and compares # data types together with assert or np.allclose qmtest.recursive_compare_dicts(test_dict, json_dict) os.remove(json_file)
def test_load_ant_metrics_json(self): json_file = os.path.join(DATA_PATH, 'example_ant_metrics.json') hdf5_file = os.path.join(DATA_PATH, 'example_ant_metrics.hdf5') warn_message = [ "JSON-type files can still be read but are no longer " "written by default.\n" "Write to HDF5 format for future compatibility." ] json_dict = uvtest.checkWarnings(ant_metrics.load_antenna_metrics, func_args=[json_file], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) hdf5_dict = ant_metrics.load_antenna_metrics(hdf5_file) # The written hdf5 may have these keys that differ by design # so ignore them. json_dict.pop('history', None) json_dict.pop('version', None) hdf5_dict.pop('history', None) hdf5_dict.pop('version', None) # This function recursively walks dictionary and compares # data types together with nt.assert_type_equal or np.allclose qmtest.recursive_compare_dicts(hdf5_dict, json_dict)
def test_read_old_all_string_ant_json_files(): """Test the new ant_metric json storage can be read and written to hdf5.""" old_json_infile = os.path.join(DATA_PATH, 'example_ant_metrics_all_string.json') new_json_infile = os.path.join(DATA_PATH, 'example_ant_metrics.json') warn_message = [ "JSON-type files can still be read but are no longer " "written by default.\n" "Write to HDF5 format for future compatibility.", ] test_metrics_old = uvtest.checkWarnings(metrics_io.load_metric_file, func_args=[old_json_infile], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) test_metrics_new = uvtest.checkWarnings(metrics_io.load_metric_file, func_args=[new_json_infile], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) # The written hdf5 may have these keys that differ by design # so ignore them. test_metrics_old.pop('history', None) test_metrics_old.pop('version', None) test_metrics_new.pop('history', None) test_metrics_new.pop('version', None) # This function recursively walks dictionary and compares # data types together with nt.assert_type_equal or np.allclose qmtest.recursive_compare_dicts(test_metrics_new, test_metrics_old)
def test_read_write_old_omnical_json_files(): """Test the old omnical json storage can be read and written to hdf5.""" json_infile = os.path.join(DATA_PATH, 'example_omnical_metrics.json') test_file = os.path.join(DATA_PATH, 'test_output', 'test_omnical_json_to_hdf5.h5') warn_message = [ "JSON-type files can still be read but are no longer " "written by default.\n" "Write to HDF5 format for future compatibility.", ] test_metrics = uvtest.checkWarnings(metrics_io.load_metric_file, func_args=[json_infile], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) metrics_io.write_metric_file(test_file, test_metrics, overwrite=True) test_metrics_in = metrics_io.load_metric_file(test_file) # The written hdf5 may have these keys that differ by design # so ignore them. test_metrics.pop('history', None) test_metrics.pop('version', None) test_metrics_in.pop('history', None) test_metrics_in.pop('version', None) # This function recursively walks dictionary and compares # data types together with nt.assert_type_equal or np.allclose qmtest.recursive_compare_dicts(test_metrics, test_metrics_in) nt.assert_true(os.path.exists(test_file)) os.remove(test_file)
def test_save_json(self): am = ant_metrics.AntennaMetrics(self.dataFileList, self.reds, fileformat='miriad') am.iterative_antenna_metrics_and_flagging() for stat in self.summaryStats: self.assertTrue(hasattr(am, stat)) self.assertIn((81, 'x'), am.xants) self.assertIn((81, 'y'), am.xants) self.assertIn((81, 'x'), am.deadAntsRemoved) self.assertIn((81, 'y'), am.deadAntsRemoved) outfile = os.path.join(DATA_PATH, 'test_output', 'ant_metrics_output.json') warn_message = [ "JSON-type files can still be written " "but are no longer written by default.\n" "Write to HDF5 format for future compatibility." ] uvtest.checkWarnings(am.save_antenna_metrics, func_args=[outfile], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) # am.save_antenna_metrics(json_file) warn_message = [ "JSON-type files can still be read but are no longer " "written by default.\n" "Write to HDF5 format for future compatibility." ] loaded = uvtest.checkWarnings(ant_metrics.load_antenna_metrics, func_args=[outfile], category=PendingDeprecationWarning, nwarnings=1, message=warn_message) _ = loaded.pop('history', '') jsonStats = [ 'xants', 'crossed_ants', 'dead_ants', 'removal_iteration', 'final_metrics', 'all_metrics', 'final_mod_z_scores', 'all_mod_z_scores', 'cross_pol_z_cut', 'dead_ant_z_cut', 'datafile_list', 'reds', 'version' ] for stat, jsonStat in zip(self.summaryStats, jsonStats): file_val = loaded[jsonStat] obj_val = getattr(am, stat) if isinstance(file_val, dict): qmtest.recursive_compare_dicts(file_val, obj_val) else: self.assertEqual(file_val, obj_val) os.remove(outfile)
def test_write_then_recursive_load_dict_to_group_with_nested_dicts(): """Test recursive load can gather dictionary from a nested group.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' path = '/' good_dict = { 'filestem': test_scalar, 'history': "this is a test", 'version': hera_qm_version_str, '1': { 'filestem': test_scalar } } metrics_io.write_metric_file(test_file, good_dict) with h5py.File(test_file, 'r') as h5file: read_dict = metrics_io._recursively_load_dict_to_group( h5file, '/Header/') read_dict.update( metrics_io._recursively_load_dict_to_group(h5file, '/Metrics/')) metrics_io._recursively_validate_dict(read_dict) for key in good_dict: if isinstance(good_dict[key], dict): assert qmtest.recursive_compare_dicts(good_dict[key], read_dict[key]) else: assert good_dict[key] == read_dict[key] os.remove(test_file)
def test_write_then_load_metric_file_hdf5(): """Test loaded in map is same as written one from hdf5.""" test_file = os.path.join(DATA_PATH, 'test_output', 'test.h5') test_scalar = 'hello world' path = '/' good_dict = { 'filestem': test_scalar, 'history': "this is a test", 'version': hera_qm_version_str, 'all_metrics': { 'filestem': test_scalar } } metrics_io.write_metric_file(test_file, good_dict) read_dict = metrics_io.load_metric_file(test_file) for key in good_dict: if isinstance(good_dict[key], dict): assert qmtest.recursive_compare_dicts(good_dict[key], read_dict[key]) else: assert good_dict[key] == read_dict[key] os.remove(test_file)
def test_run_metrics_two_pols(firstcal_twopol): # These results were run with a seed of 0, the seed shouldn't matter # but you never know. two_pol_known_results = os.path.join( DATA_PATH, 'example_two_polarization_firstcal_results.hdf5') np.random.seed(0) firstcal_twopol.FC.run_metrics(std_cut=.5) known_output = metrics_io.load_metric_file(two_pol_known_results) known_output.pop('history', None) known_output.pop('version', None) # There are some full paths of files saved in the files # Perhaps for record keeping, but that messes up the test comparison for key in known_output: known_output[key].pop('fc_filename', None) known_output[key].pop('fc_filestem', None) known_output[key].pop('version', None) for key in firstcal_twopol.FC.metrics: firstcal_twopol.FC.metrics[key].pop('fc_filename', None) firstcal_twopol.FC.metrics[key].pop('fc_filestem', None) firstcal_twopol.FC.metrics[key].pop('version', None) assert qmtest.recursive_compare_dicts(firstcal_twopol.FC.metrics, known_output)