def write_result_to_hdf(pred_data, pred_target, cur_fp, out_dir, out_vars):
vars_dict = dict()
vars_attrs_dict = dict()
h5_reader = HDFReader(cur_fp)
gbl_attrs = h5_reader.get_global_attrs()
for key in out_vars:
data = h5_reader.get_dataset(key)
if key == "demo_flag":
pred_data_attrs = h5_reader.get_dataset_attrs(pred_target)
pred_data_attrs["long_name"] = pred_target + " from PMNet"
vars_dict["Pred_" + pred_target] = pred_data
vars_attrs_dict["Pred_" + pred_target] = pred_data_attrs
vars_dict[key] = data
vars_attrs_dict[key] = h5_reader.get_dataset_attrs(key)
os.makedirs(out_dir, exist_ok=True)
h5_writer = HDFWriter(os.path.join(out_dir, os.path.basename(cur_fp)))
h5_writer.set_global_attrs(gbl_attrs)
for key in vars_dict.keys():
data = vars_dict.get(key)
data_attrs = vars_attrs_dict.get(key)
h5_writer.create_dataset(key, data_attrs, data, data.dtype)
def test_HDFReader():
hdf_fn = r"D:\01-work_directory\03-PM2.5\PMs\data\MERSI\FY3D_MERSI_GBAL_L2_AOD_MLT_GLL_20180817_POAD_5000M_MS.HDF"
h_reader = HDFReader(hdf_fn)
data = h_reader.get_dataset("AOT_550_Mean")
slope = h_reader.get_dataset_attr("AOT_550_Mean", "Slope")
global_attrs = h_reader.get_global_attrs()
print(global_attrs)
aod_attrs = h_reader.get_dataset_attrs("AOT_550_Mean")
print(aod_attrs)
print(data.shape)
print(slope)
def dataset_format(ds_fst, scalar_f, out_fd):
os.makedirs(out_fd, exist_ok=True)
vr_dict = json_loader(scalar_f)
for f in tqdm.tqdm(ds_fst, ascii=True, desc="dataset format"):
hdf_reader = HDFReader(f)
gbl_attrs = hdf_reader.get_global_attrs()
hdf_writer = HDFWriter(os.path.join(out_fd, os.path.basename(f)))
hdf_writer.set_global_attrs(gbl_attrs)
names = hdf_reader.get_dataset_names()
for name in names:
data = hdf_reader.get_dataset(name)
attrs = hdf_reader.get_dataset_attrs(name)
if name in vr_dict.keys():
attrs["valid_range"] = vr_dict.get(name)
hdf_writer.create_dataset(name, attrs, data, data.dtype)