def link_interpolated_data(data, out_fname):
"""
Links the individual interpolated results into a
single file for easier access.
"""
for key in data:
fname = data[key]
with h5py.File(fname, 'r') as fid:
dataset_names = find(fid, dataset_class='IMAGE')
for dname in dataset_names:
create_external_link(fname, dname, out_fname, dname)
def run(self):
with self.output().temporary_path() as out_fname:
for root, _, files in os.walk(self.work_root):
# skip any private files
if basename(root)[0] == "_":
continue
for file_ in files:
if splitext(file_)[1] == ".h5":
fname = pjoin(root, file_)
grp_name = basename(dirname(fname.replace(self.work_root, "")))
with h5py.File(fname, "r") as fid:
groups = [g for g in fid]
for pth in groups:
new_path = ppjoin(self.granule, grp_name, pth)
create_external_link(fname, pth, out_fname, new_path)
with h5py.File(out_fname, "a") as fid:
fid.attrs["level1_uri"] = self.level1
def link_standard_data(input_fnames, out_fname):
# TODO: incorporate linking for multi-granule and multi-group
# datasets
"""
Links the individual reflectance and surface temperature
results into a single file for easier access.
"""
for fname in input_fnames:
with h5py.File(fname, "r") as fid:
dataset_names = find(fid, dataset_class="IMAGE")
for dname in dataset_names:
create_external_link(fname, dname, out_fname, dname)
# metadata
with h5py.File(fname, "r") as fid:
with h5py.File(out_fname, "a") as out_fid:
yaml_dname = DatasetName.NBAR_YAML.value
if yaml_dname in fid and yaml_dname not in out_fid:
fid.copy(yaml_dname, out_fid, name=yaml_dname)
yaml_dname = DatasetName.SBT_YAML.value
if yaml_dname in fid and yaml_dname not in out_fid:
fid.copy(yaml_dname, out_fid, name=yaml_dname)
def link_shadow_datasets(self_shadow_fname, cast_shadow_sun_fname,
cast_shadow_satellite_fname, out_fname):
"""
Link the self shadow mask, and the two cast shadow masks into a
single file for easier access.
"""
group_path = GroupName.SHADOW_GROUP.value
dname_fmt = DatasetName.CAST_SHADOW_FMT.value
dname = ppjoin(group_path, DatasetName.SELF_SHADOW.value)
create_external_link(self_shadow_fname, dname, out_fname, dname)
dname = ppjoin(group_path, dname_fmt.format(source='SUN'))
create_external_link(cast_shadow_sun_fname, dname, out_fname, dname)
dname = ppjoin(group_path, dname_fmt.format(source='SATELLITE'))
create_external_link(cast_shadow_satellite_fname, dname, out_fname, dname)
def link_atmospheric_results(input_targets, out_fname, npoints, workflow):
"""
Uses h5py's ExternalLink to combine the atmospheric results into
a single file.
:param input_targets:
A `list` of luigi.LocalTargets.
:param out_fname:
A `str` containing the output filename.
:param npoints:
An `int` containing the number of points (vertices) used for
evaluating the atmospheric conditions.
:param workflow:
An Enum given by wagl.constants.Workflow.
:return:
None. Results from each file in `input_targets` are linked
into the output file.
"""
base_group_name = GroupName.ATMOSPHERIC_RESULTS_GRP.value
nbar_atmospherics = False
sbt_atmospherics = False
attributes = []
for fname in input_targets:
with h5py.File(fname.path, 'r') as fid:
points = list(fid[base_group_name].keys())
# copy across several attributes on the POINT Group
# as the linking we do here links direct to a dataset
# which will create the required parent Groups
for point in points:
group = ppjoin(base_group_name, point)
attributes.append((point,
fid[group].attrs['lonlat'],
fid[group].attrs['datetime'],
fid[group].attrs['albedos']))
for point in points:
for albedo in workflow.albedos:
if albedo == Albedos.ALBEDO_TH:
datasets = [DatasetName.UPWARD_RADIATION_CHANNEL.value,
DatasetName.DOWNWARD_RADIATION_CHANNEL.value]
sbt_atmospherics = True
else:
datasets = [DatasetName.CHANNEL.value,
DatasetName.SOLAR_ZENITH_CHANNEL.value]
nbar_atmospherics = True
grp_path = ppjoin(base_group_name, point,
ALBEDO_FMT.format(a=albedo.value))
for dset in datasets:
dname = ppjoin(grp_path, dset)
create_external_link(fname.path, dname, out_fname, dname)
with h5py.File(out_fname) as fid:
group = fid[GroupName.ATMOSPHERIC_RESULTS_GRP.value]
group.attrs['npoints'] = npoints
group.attrs['nbar_atmospherics'] = nbar_atmospherics
group.attrs['sbt_atmospherics'] = sbt_atmospherics
# assign the lonlat attribute for each POINT Group
for point, lonlat, date_time, albedos in attributes:
group[point].attrs['lonlat'] = lonlat
group[point].attrs['datetime'] = date_time
group[point].attrs.create('albedos', data=albedos,
dtype=VLEN_STRING)