def _build_index(indir):
"""
Read the INDEX table for each file and build a full history
index.
The records are sorted in ascending time (earliest to most recent)
"""
df = pandas.DataFrame(columns=["filename", "band_name", "timestamp"])
for fname in Path(indir).glob("pr_wtr.eatm.[0-9]*.h5"):
with h5py.File(str(fname), "r") as fid:
tmp_df = read_h5_table(fid, "INDEX")
tmp_df["filename"] = fid.filename
df = df.append(tmp_df, sort=False)
df.sort_values("timestamp", inplace=True)
df.set_index("timestamp", inplace=True)
return df
def test_dataframe_roundtrip(self):
"""
Test that the pandas dataframe roundtrips, i.e. save to HDF5
and is read back into a dataframe seamlessly.
Float, integer, datetime and string datatypes will be
tested.
"""
df = pandas.DataFrame(self.table_data)
df["timestamps"] = pandas.date_range("1/1/2000",
periods=10,
freq="D",
tz="UTC")
df["string_data"] = ["period {}".format(i) for i in range(10)]
fname = "test_dataframe_roundtrip.h5"
with h5py.File(fname, "w", **self.memory_kwargs) as fid:
hdf5.write_dataframe(df, "dataframe", fid)
# Apply conversion to no timezone that occurs in serialisation to hdf5
# Numpy is timezone naive; pandas has timezone support
df["timestamps"] = df["timestamps"].dt.tz_convert(None)
self.assertTrue(df.equals(hdf5.read_h5_table(fid, "dataframe")))
def interpolate(acq, coefficient, ancillary_group, satellite_solar_group,
coefficients_group, out_group=None,
compression=H5CompressionFilter.LZF, filter_opts=None,
method=Method.SHEARB):
# TODO: more docstrings
"""Perform interpolation."""
if method not in Method:
msg = 'Interpolation method {} not available.'
raise Exception(msg.format(method.name))
geobox = acq.gridded_geo_box()
cols, rows = geobox.get_shape_xy()
# read the relevant tables into DataFrames
coordinator = read_h5_table(ancillary_group, DatasetName.COORDINATOR.value)
boxline = read_h5_table(satellite_solar_group, DatasetName.BOXLINE.value)
if coefficient in Workflow.NBAR.atmos_coefficients:
dataset_name = DatasetName.NBAR_COEFFICIENTS.value
elif coefficient in Workflow.SBT.atmos_coefficients:
dataset_name = DatasetName.SBT_COEFFICIENTS.value
else:
msg = "Factor name not found in available coefficients: {}"
raise ValueError(msg.format(Workflow.STANDARD.atmos_coefficients))
coefficients = read_h5_table(coefficients_group, dataset_name)
coord = np.zeros((coordinator.shape[0], 2), dtype='int')
map_x = coordinator.map_x.values
map_y = coordinator.map_y.values
coord[:, 1], coord[:, 0] = (map_x, map_y) * ~geobox.transform
centre = boxline.bisection_index.values
start = boxline.start_index.values
end = boxline.end_index.values
band_records = coefficients.band_name == acq.band_name
samples = coefficients[coefficient.value][band_records].values
func_map = {Method.BILINEAR: sheared_bilinear_interpolate,
Method.FBILINEAR: fortran_bilinear_interpolate,
Method.SHEAR: sheared_bilinear_interpolate,
Method.SHEARB: sheared_bilinear_interpolate,
Method.RBF: rbf_interpolate}
args = [cols, rows, coord, samples, start, end, centre]
if method == Method.BILINEAR:
args.extend([False, False])
elif method == Method.SHEARB:
args.extend([True, True])
else:
pass
result = func_map[method](*args)
# setup the output file/group as needed
if out_group is None:
fid = h5py.File('interpolated-coefficients.h5', driver='core',
backing_store=False)
else:
fid = out_group
if GroupName.INTERP_GROUP.value not in fid:
fid.create_group(GroupName.INTERP_GROUP.value)
if filter_opts is None:
filter_opts = {}
else:
filter_opts = filter_opts.copy()
filter_opts['chunks'] = acq.tile_size
group = fid[GroupName.INTERP_GROUP.value]
fmt = DatasetName.INTERPOLATION_FMT.value
dset_name = fmt.format(coefficient=coefficient.value, band_name=acq.band_name)
no_data = -999
attrs = {'crs_wkt': geobox.crs.ExportToWkt(),
'geotransform': geobox.transform.to_gdal(),
'no_data_value': no_data,
'interpolation_method': method.name,
'band_id': acq.band_id,
'band_name': acq.band_name,
'alias': acq.alias,
'coefficient': coefficient.value}
desc = ("Contains the interpolated result of coefficient {} "
"for band {} from sensor {}.")
attrs['description'] = desc.format(coefficient.value, acq.band_id,
acq.sensor_id)
# convert any NaN's to -999 (for float data, NaN would be more ideal ...)
result[~np.isfinite(result)] = no_data
write_h5_image(result, dset_name, group, compression, attrs, filter_opts)
if out_group is None:
return fid
def test_modtran_run(self):
"""
Tests that the interface to modtran (run_modtran)
works for known inputs.
Used to validate environment configuration/setup
"""
band_names = [
'BAND-1', 'BAND-2', 'BAND-3', 'BAND-4', 'BAND-5', 'BAND-6',
'BAND-7', 'BAND-8'
]
point = 0
albedo = Albedos.ALBEDO_0
# setup mock acquistions object
acquisitions = []
for bandn in band_names:
acq = mock.MagicMock()
acq.acquisition_datetime = datetime(2001, 1, 1)
acq.band_type = BandType.REFLECTIVE
acq.spectral_response = mock_spectral_response
acquisitions.append(acq)
# setup mock atmospherics group
attrs = {'lonlat': 'TEST'}
atmospherics = mock.MagicMock()
atmospherics.attrs = attrs
atmospherics_group = {POINT_FMT.format(p=point): atmospherics}
# Compute base path -- prefix for hdf5 file
base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value,
POINT_FMT.format(p=point))
with tempfile.TemporaryDirectory() as workdir:
run_dir = pjoin(workdir, POINT_FMT.format(p=point),
ALBEDO_FMT.format(a=albedo.value))
os.makedirs(run_dir)
# TODO replace json_input copy with json input generation
with open(INPUT_JSON, 'r') as fd:
json_data = json.load(fd)
for mod_input in json_data['MODTRAN']:
mod_input['MODTRANINPUT']['SPECTRAL'][
'FILTNM'] = SPECTRAL_RESPONSE_LS8
with open(
pjoin(run_dir,
POINT_ALBEDO_FMT.format(p=point, a=albedo.value)) +
".json", 'w') as fd:
json.dump(json_data, fd)
fid = run_modtran(
acquisitions,
atmospherics_group,
Workflow.STANDARD,
npoints=12, # number of track points
point=point,
albedos=[albedo],
modtran_exe=MODTRAN_EXE,
basedir=workdir,
out_group=None)
assert fid
# Test base attrs
assert fid[base_path].attrs['lonlat'] == 'TEST'
assert fid[base_path].attrs['datetime'] == datetime(2001, 1,
1).isoformat()
# test albedo headers?
# Summarise modtran results to surface reflectance coefficients
test_grp = fid[base_path][ALBEDO_FMT.format(a=albedo.value)]
nbar_coefficients, _ = coefficients(
read_h5_table(
fid,
pjoin(base_path, ALBEDO_FMT.format(a=albedo.value),
DatasetName.CHANNEL.value)),
read_h5_table(
fid,
pjoin(base_path, ALBEDO_FMT.format(a=albedo.value),
DatasetName.SOLAR_ZENITH_CHANNEL.value)))
expected = pd.read_csv(EXPECTED_CSV, index_col='band_name')
pd.testing.assert_frame_equal(nbar_coefficients,
expected,
check_less_precise=True)
def comparison(outdir: Union[str, Path], proc_info: bool) -> None:
"""
Test and Reference product intercomparison evaluation.
"""
outdir = Path(outdir)
if proc_info:
log_fname = outdir.joinpath(DirectoryNames.LOGS.value,
LogNames.PROC_INFO_INTERCOMPARISON.value)
else:
log_fname = outdir.joinpath(DirectoryNames.LOGS.value,
LogNames.MEASUREMENT_INTERCOMPARISON.value)
out_stream = MPIStreamIO(str(log_fname))
structlog.configure(processors=DEFAULT_PROCESSORS,
logger_factory=MPILoggerFactory(out_stream))
# processor info
rank = COMM.Get_rank()
n_processors = COMM.Get_size()
results_fname = outdir.joinpath(DirectoryNames.RESULTS.value,
FileNames.RESULTS.value)
with h5py.File(str(results_fname), "r") as fid:
dataframe = read_h5_table(fid, DatasetNames.QUERY.value)
if rank == 0:
index = dataframe.index.values.tolist()
blocks = scatter(index, n_processors)
# some basic attribute information
doc: Union[Granule, None] = load_odc_metadata(
Path(dataframe.iloc[0].yaml_pathname_reference))
attrs: Dict[str, Any] = {
"framing": doc.framing,
"thematic": False,
"proc-info": False,
}
else:
blocks = None
doc = None
attrs = dict()
COMM.Barrier()
# equally partition the work across all procesors
indices = COMM.scatter(blocks, root=0)
if proc_info:
attrs["proc-info"] = True
if rank == 0:
_LOG.info("procssing proc-info documents")
gqa_dataframe, ancillary_dataframe = _process_proc_info(
dataframe.iloc[indices], rank)
if rank == 0:
_LOG.info("saving gqa dataframe results to tables")
if not results_fname.parent.exists():
results_fname.parent.mkdir(parents=True)
with h5py.File(str(results_fname), "a") as fid:
dataset_name = PPath(DatasetGroups.INTERCOMPARISON.value,
DatasetNames.GQA_RESULTS.value)
write_dataframe(gqa_dataframe,
str(dataset_name),
fid,
attrs=attrs)
_LOG.info("saving ancillary dataframe results to tables")
if not results_fname.parent.exists():
results_fname.parent.mkdir(parents=True)
with h5py.File(str(results_fname), "a") as fid:
dataset_name = PPath(
DatasetGroups.INTERCOMPARISON.value,
DatasetNames.ANCILLARY_RESULTS.value,
)
write_dataframe(ancillary_dataframe,
str(dataset_name),
fid,
attrs=attrs)
_LOG.info("saving software versions dataframe to tables")
with h5py.File(str(results_fname), "a") as fid:
dataset_name = PPath(DatasetNames.SOFTWARE_VERSIONS.value)
software_attrs = {
"description": "ARD Pipeline software versions"
}
software_df = compare_proc_info.compare_software(dataframe)
write_dataframe(software_df,
str(dataset_name),
fid,
attrs=software_attrs)
else:
if rank == 0:
_LOG.info("processing odc-metadata documents")
results = _process_odc_doc(dataframe.iloc[indices], rank)
if rank == 0:
# save each table
_LOG.info("saving dataframes to tables")
with h5py.File(str(results_fname), "a") as fid:
attrs["thematic"] = False
write_dataframe(
results[0],
str(
PPath(
DatasetGroups.INTERCOMPARISON.value,
DatasetNames.GENERAL_RESULTS.value,
)),
fid,
attrs=attrs,
)
attrs["thematic"] = True
write_dataframe(
results[1],
str(
PPath(
DatasetGroups.INTERCOMPARISON.value,
DatasetNames.FMASK_RESULTS.value,
)),
fid,
attrs=attrs,
)
write_dataframe(
results[2],
str(
PPath(
DatasetGroups.INTERCOMPARISON.value,
DatasetNames.CONTIGUITY_RESULTS.value,
)),
fid,
attrs=attrs,
)
write_dataframe(
results[3],
str(
PPath(
DatasetGroups.INTERCOMPARISON.value,
DatasetNames.SHADOW_RESULTS.value,
)),
fid,
attrs=attrs,
)
if rank == 0:
workflow = "proc-info field" if proc_info else "product measurement"
msg = f"{workflow} comparison processing finished"
_LOG.info(msg)
def format_json(acquisitions, ancillary_group, satellite_solar_group,
lon_lat_group, workflow, out_group):
"""
Creates json files for the albedo (0) and thermal
"""
# angles data
sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value]
sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value]
longitude = lon_lat_group[DatasetName.LON.value]
latitude = lon_lat_group[DatasetName.LAT.value]
# retrieve the averaged ancillary if available
anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value)
if anc_grp is None:
anc_grp = ancillary_group
# ancillary data
coordinator = ancillary_group[DatasetName.COORDINATOR.value]
aerosol = anc_grp[DatasetName.AEROSOL.value][()]
water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()]
ozone = anc_grp[DatasetName.OZONE.value][()]
elevation = anc_grp[DatasetName.ELEVATION.value][()]
npoints = coordinator.shape[0]
view = numpy.zeros(npoints, dtype='float32')
azi = numpy.zeros(npoints, dtype='float32')
lat = numpy.zeros(npoints, dtype='float64')
lon = numpy.zeros(npoints, dtype='float64')
for i in range(npoints):
yidx = coordinator['row_index'][i]
xidx = coordinator['col_index'][i]
view[i] = sat_view[yidx, xidx]
azi[i] = sat_azi[yidx, xidx]
lat[i] = latitude[yidx, xidx]
lon[i] = longitude[yidx, xidx]
view_corrected = 180 - view
azi_corrected = azi + 180
rlon = 360 - lon
# check if in western hemisphere
idx = rlon >= 360
rlon[idx] -= 360
idx = (180 - view_corrected) < 0.1
view_corrected[idx] = 180
azi_corrected[idx] = 0
idx = azi_corrected > 360
azi_corrected[idx] -= 360
# get the modtran profiles to use based on the centre latitude
_, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat
if out_group is None:
out_group = h5py.File('atmospheric-inputs.h5', 'w')
if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group:
out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value)
group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value]
iso_time = acquisitions[0].acquisition_datetime.isoformat()
group.attrs['acquisition-datetime'] = iso_time
json_data = {}
# setup the json files required by MODTRAN
if workflow in (Workflow.STANDARD, Workflow.NBAR):
acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE]
for p in range(npoints):
for alb in Workflow.NBAR.albedos:
input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a=str(alb.value)),
'water': water_vapour,
'ozone': ozone,
'doy': acquisitions[0].julian_day(),
'visibility': -aerosol,
'lat': lat[p],
'lon': rlon[p],
'time': acquisitions[0].decimal_hour(),
'sat_azimuth': azi_corrected[p],
'sat_height': acquisitions[0].altitude / 1000.0,
'elevation': elevation,
'sat_view': view_corrected[p],
'albedo': float(alb.value),
'filter_function': acqs[0].spectral_filter_name,
'binary': False
}
if centre_lat < -23.0:
data = mpjson.midlat_summer_albedo(**input_data)
else:
data = mpjson.tropical_albedo(**input_data)
input_data['description'] = 'Input file for MODTRAN'
input_data['file_format'] = 'json'
input_data.pop('binary')
json_data[(p, alb)] = data
data = json.dumps(data, cls=JsonEncoder, indent=4)
dname = ppjoin(POINT_FMT.format(p=p),
ALBEDO_FMT.format(a=alb.value),
DatasetName.MODTRAN_INPUT.value)
write_scalar(data, dname, group, input_data)
# create json for sbt if it has been collected
if ancillary_group.attrs.get('sbt-ancillary'):
dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value)
acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL]
for p in range(npoints):
atmos_profile = read_h5_table(ancillary_group, dname.format(p=p))
n_layers = atmos_profile.shape[0] + 6
elevation = atmos_profile.iloc[0]['GeoPotential_Height']
input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a='TH'),
'ozone': ozone,
'n': n_layers,
'prof_alt': list(atmos_profile['GeoPotential_Height']),
'prof_pres': list(atmos_profile['Pressure']),
'prof_temp': list(atmos_profile['Temperature']),
'prof_water': list(atmos_profile['Relative_Humidity']),
'visibility': -aerosol,
'sat_height': acquisitions[0].altitude / 1000.0,
'gpheight': elevation,
'sat_view': view_corrected[p],
'filter_function': acqs[0].spectral_filter_name,
'binary': False
}
data = mpjson.thermal_transmittance(**input_data)
input_data['description'] = 'Input File for MODTRAN'
input_data['file_format'] = 'json'
input_data.pop('binary')
json_data[(p, Albedos.ALBEDO_TH)] = data
data = json.dumps(data, cls=JsonEncoder, indent=4)
out_dname = ppjoin(POINT_FMT.format(p=p),
ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value),
DatasetName.MODTRAN_INPUT.value)
write_scalar(data, out_dname, group, input_data)
# attach location info to each point Group
for p in range(npoints):
lonlat = (coordinator['longitude'][p], coordinator['latitude'][p])
group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat
return json_data, out_group
def calculate_coefficients(atmospheric_results_group, out_group,
compression=H5CompressionFilter.LZF,
filter_opts=None):
"""
Calculate the atmospheric coefficients from the MODTRAN output
and used in the BRDF and atmospheric correction.
Coefficients are computed for each band for each each coordinate
for each atmospheric coefficient. The atmospheric coefficients can be
found in `Workflow.STANDARD.atmos_coefficients`.
:param atmospheric_results_group:
The root HDF5 `Group` that contains the atmospheric results
from each MODTRAN run.
:param out_group:
If set to None (default) then the results will be returned
as an in-memory hdf5 file, i.e. the `core` driver. Otherwise,
a writeable HDF5 `Group` object.
The datasets will be formatted to the HDF5 TABLE specification
and the dataset names will be as follows:
* DatasetName.NBAR_COEFFICIENTS (if Workflow.STANDARD or Workflow.NBAR)
* DatasetName.SBT_COEFFICIENTS (if Workflow.STANDARD or Workflow.SBT)
:param compression:
The compression filter to use.
Default is H5CompressionFilter.LZF
:param filter_opts:
A dict of key value pairs available to the given configuration
instance of H5CompressionFilter. For example
H5CompressionFilter.LZF has the keywords *chunks* and *shuffle*
available.
Default is None, which will use the default settings for the
chosen H5CompressionFilter instance.
:return:
An opened `h5py.File` object, that is either in-memory using the
`core` driver, or on disk.
"""
nbar_coefficients = pd.DataFrame()
sbt_coefficients = pd.DataFrame()
channel_data = channel_solar_angle = upward = downward = None
# Initialise the output group/file
if out_group is None:
fid = h5py.File('atmospheric-coefficients.h5', driver='core',
backing_store=False)
else:
fid = out_group
res = atmospheric_results_group
npoints = res.attrs['npoints']
nbar_atmos = res.attrs['nbar_atmospherics']
sbt_atmos = res.attrs['sbt_atmospherics']
for point in range(npoints):
point_grp = res[POINT_FMT.format(p=point)]
lonlat = point_grp.attrs['lonlat']
timestamp = pd.to_datetime(point_grp.attrs['datetime'])
grp_path = ppjoin(POINT_FMT.format(p=point), ALBEDO_FMT)
if nbar_atmos:
channel_path = ppjoin(grp_path.format(a=Albedos.ALBEDO_0.value),
DatasetName.CHANNEL.value)
channel_data = read_h5_table(res, channel_path)
channel_solar_angle_path = ppjoin(
grp_path.format(a=Albedos.ALBEDO_0.value),
DatasetName.SOLAR_ZENITH_CHANNEL.value
)
channel_solar_angle = read_h5_table(res, channel_solar_angle_path)
if sbt_atmos:
dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value),
DatasetName.UPWARD_RADIATION_CHANNEL.value)
upward = read_h5_table(res, dname)
dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value),
DatasetName.DOWNWARD_RADIATION_CHANNEL.value)
downward = read_h5_table(res, dname)
kwargs = {'channel_data': channel_data,
'solar_zenith_angle': channel_solar_angle,
'upward_radiation': upward,
'downward_radiation': downward}
result = coefficients(**kwargs)
# insert some datetime/geospatial fields
if result[0] is not None:
result[0].insert(0, 'POINT', point)
result[0].insert(1, 'LONGITUDE', lonlat[0])
result[0].insert(2, 'LATITUDE', lonlat[1])
result[0].insert(3, 'DATETIME', timestamp)
nbar_coefficients = nbar_coefficients.append(result[0])
if result[1] is not None:
result[1].insert(0, 'POINT', point)
result[1].insert(1, 'LONGITUDE', lonlat[0])
result[1].insert(2, 'LATITUDE', lonlat[1])
result[1].insert(3, 'DATETIME', pd.to_datetime(timestamp))
sbt_coefficients = sbt_coefficients.append(result[1])
nbar_coefficients.reset_index(inplace=True)
sbt_coefficients.reset_index(inplace=True)
attrs = {'npoints': npoints}
description = "Coefficients derived from the VNIR solar irradiation."
attrs['description'] = description
dname = DatasetName.NBAR_COEFFICIENTS.value
if GroupName.COEFFICIENTS_GROUP.value not in fid:
fid.create_group(GroupName.COEFFICIENTS_GROUP.value)
group = fid[GroupName.COEFFICIENTS_GROUP.value]
if nbar_atmos:
write_dataframe(nbar_coefficients, dname, group, compression,
attrs=attrs, filter_opts=filter_opts)
description = "Coefficients derived from the THERMAL solar irradiation."
attrs['description'] = description
dname = DatasetName.SBT_COEFFICIENTS.value
if sbt_atmos:
write_dataframe(sbt_coefficients, dname, group, compression,
attrs=attrs, filter_opts=filter_opts)
if out_group is None:
return fid
def format_tp5(acquisitions, ancillary_group, satellite_solar_group,
lon_lat_group, workflow, out_group):
"""
Creates str formatted tp5 files for the albedo (0, 1) and
transmittance (t).
"""
# angles data
sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value]
sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value]
longitude = lon_lat_group[DatasetName.LON.value]
latitude = lon_lat_group[DatasetName.LAT.value]
# retrieve the averaged ancillary if available
anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value)
if anc_grp is None:
anc_grp = ancillary_group
# ancillary data
coordinator = ancillary_group[DatasetName.COORDINATOR.value]
aerosol = anc_grp[DatasetName.AEROSOL.value][()]
water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()]
ozone = anc_grp[DatasetName.OZONE.value][()]
elevation = anc_grp[DatasetName.ELEVATION.value][()]
npoints = coordinator.shape[0]
view = numpy.zeros(npoints, dtype='float32')
azi = numpy.zeros(npoints, dtype='float32')
lat = numpy.zeros(npoints, dtype='float64')
lon = numpy.zeros(npoints, dtype='float64')
for i in range(npoints):
yidx = coordinator['row_index'][i]
xidx = coordinator['col_index'][i]
view[i] = sat_view[yidx, xidx]
azi[i] = sat_azi[yidx, xidx]
lat[i] = latitude[yidx, xidx]
lon[i] = longitude[yidx, xidx]
view_corrected = 180 - view
azi_corrected = azi + 180
rlon = 360 - lon
# check if in western hemisphere
idx = rlon >= 360
rlon[idx] -= 360
idx = (180 - view_corrected) < 0.1
view_corrected[idx] = 180
azi_corrected[idx] = 0
idx = azi_corrected > 360
azi_corrected[idx] -= 360
# get the modtran profiles to use based on the centre latitude
_, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat
if centre_lat < -23.0:
albedo_profile = MIDLAT_SUMMER_ALBEDO
trans_profile = MIDLAT_SUMMER_TRANSMITTANCE
else:
albedo_profile = TROPICAL_ALBEDO
trans_profile = TROPICAL_TRANSMITTANCE
if out_group is None:
out_group = h5py.File('atmospheric-inputs.h5', 'w')
if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group:
out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value)
group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value]
iso_time = acquisitions[0].acquisition_datetime.isoformat()
group.attrs['acquisition-datetime'] = iso_time
tp5_data = {}
# setup the tp5 files required by MODTRAN
if workflow == Workflow.STANDARD or workflow == Workflow.NBAR:
acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE]
for p in range(npoints):
for alb in Workflow.NBAR.albedos:
input_data = {
'water': water_vapour,
'ozone': ozone,
'filter_function': acqs[0].spectral_filter_file,
'visibility': -aerosol,
'elevation': elevation,
'sat_height': acquisitions[0].altitude / 1000.0,
'sat_view': view_corrected[p],
'doy': acquisitions[0].julian_day(),
'binary': 'T'
}
if alb == Albedos.ALBEDO_T:
input_data['albedo'] = 0.0
input_data['sat_view_offset'] = 180.0 - view_corrected[p]
data = trans_profile.format(**input_data)
else:
input_data['albedo'] = float(alb.value)
input_data['lat'] = lat[p]
input_data['lon'] = rlon[p]
input_data['time'] = acquisitions[0].decimal_hour()
input_data['sat_azimuth'] = azi_corrected[p]
data = albedo_profile.format(**input_data)
tp5_data[(p, alb)] = data
dname = ppjoin(POINT_FMT.format(p=p),
ALBEDO_FMT.format(a=alb.value),
DatasetName.TP5.value)
write_scalar(numpy.string_(data), dname, group, input_data)
# create tp5 for sbt if it has been collected
if ancillary_group.attrs.get('sbt-ancillary'):
dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value)
acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL]
for p in range(npoints):
atmospheric_profile = []
atmos_profile = read_h5_table(ancillary_group, dname.format(p=p))
n_layers = atmos_profile.shape[0] + 6
elevation = atmos_profile.iloc[0]['GeoPotential_Height']
for i, row in atmos_profile.iterrows():
input_data = {
'gpheight': row['GeoPotential_Height'],
'pressure': row['Pressure'],
'airtemp': row['Temperature'],
'humidity': row['Relative_Humidity'],
'zero': 0.0
}
atmospheric_profile.append(SBT_FORMAT.format(**input_data))
input_data = {
'ozone': ozone,
'filter_function': acqs[0].spectral_filter_file,
'visibility': -aerosol,
'gpheight': elevation,
'n': n_layers,
'sat_height': acquisitions[0].altitude / 1000.0,
'sat_view': view_corrected[p],
'binary': 'T',
'atmospheric_profile': ''.join(atmospheric_profile)
}
data = THERMAL_TRANSMITTANCE.format(**input_data)
tp5_data[(p, Albedos.ALBEDO_TH)] = data
out_dname = ppjoin(POINT_FMT.format(p=p),
ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value),
DatasetName.TP5.value)
write_scalar(numpy.string_(data), out_dname, group, input_data)
# attach location info to each point Group
for p in range(npoints):
lonlat = (coordinator['longitude'][p], coordinator['latitude'][p])
group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat
return tp5_data, out_group
def get_water_vapour(acquisition,
water_vapour_dict,
scale_factor=0.1,
tolerance=1):
"""
Retrieve the water vapour value for an `acquisition` and the
path for the water vapour ancillary data.
"""
dt = acquisition.acquisition_datetime
geobox = acquisition.gridded_geo_box()
year = dt.strftime("%Y")
hour = dt.timetuple().tm_hour
filename = "pr_wtr.eatm.{year}.h5".format(year=year)
if "user" in water_vapour_dict:
metadata = {
"id": numpy.array([], VLEN_STRING),
"tier": WaterVapourTier.USER.name
}
return water_vapour_dict["user"], metadata
water_vapour_path = water_vapour_dict["pathname"]
datafile = pjoin(water_vapour_path, filename)
if os.path.isfile(datafile):
with h5py.File(datafile, "r") as fid:
index = read_h5_table(fid, "INDEX")
# set the tolerance in days to search back in time
max_tolerance = -datetime.timedelta(days=tolerance)
# only look for observations that have occured in the past
time_delta = index.timestamp - dt
result = time_delta[(time_delta < datetime.timedelta())
& (time_delta > max_tolerance)]
if not os.path.isfile(datafile) or result.shape[0] == 0:
if "fallback_dataset" not in water_vapour_dict:
raise AncillaryError("No actual or fallback water vapour data.")
tier = WaterVapourTier.FALLBACK_DATASET
month = dt.strftime("%B-%d").upper()
# closest previous observation
# i.e. observations are at 0000, 0600, 1200, 1800
# and an acquisition hour of 1700 will use the 1200 observation
observations = numpy.array([0, 6, 12, 18])
hr = observations[numpy.argmin(numpy.abs(hour - observations))]
dataset_name = "AVERAGE/{}/{:02d}00".format(month, hr)
datafile = water_vapour_dict["fallback_dataset"]
else:
tier = WaterVapourTier.DEFINITIVE
# get the index of the closest water vapour observation
# which would be the maximum timedelta
# as we're only dealing with negative timedelta's here
idx = result.idxmax()
record = index.iloc[idx]
dataset_name = record.dataset_name
try:
data, md_uuid = get_pixel(datafile, dataset_name, geobox.centre_lonlat)
except ValueError:
# h5py raises a ValueError not an IndexError for out of bounds
raise AncillaryError("No Water Vapour data")
# the metadata from the original file says (Kg/m^2)
# so multiply by 0.1 to get (g/cm^2)
data = data * scale_factor
metadata = {"id": numpy.array([md_uuid], VLEN_STRING), "tier": tier.name}
return data, metadata
def get_aerosol_data(acquisition, aerosol_dict):
"""
Extract the aerosol value for an acquisition.
The version 2 retrieves the data from a HDF5 file, and provides
more control over how the data is selected geo-metrically.
Better control over timedeltas.
"""
dt = acquisition.acquisition_datetime
geobox = acquisition.gridded_geo_box()
roi_poly = Polygon([
geobox.ul_lonlat, geobox.ur_lonlat, geobox.lr_lonlat, geobox.ll_lonlat
])
descr = ["AATSR_PIX", "AATSR_CMP_YEAR_MONTH", "AATSR_CMP_MONTH"]
names = [
"ATSR_LF_%Y%m", "aot_mean_%b_%Y_All_Aerosols",
"aot_mean_%b_All_Aerosols"
]
exts = ["/pix", "/cmp", "/cmp"]
pathnames = [ppjoin(ext, dt.strftime(n)) for ext, n in zip(exts, names)]
# temporary until we sort out a better default mechanism
# how do we want to support default values, whilst still support provenance
if "user" in aerosol_dict:
tier = AerosolTier.USER
metadata = {"id": numpy.array([], VLEN_STRING), "tier": tier.name}
return aerosol_dict["user"], metadata
aerosol_fname = aerosol_dict["pathname"]
data = None
delta_tolerance = datetime.timedelta(days=0.5)
with h5py.File(aerosol_fname, "r") as fid:
for pathname, description in zip(pathnames, descr):
tier = AerosolTier[description]
if pathname in fid:
df = read_h5_table(fid, pathname)
aerosol_poly = wkt.loads(fid[pathname].attrs["extents"])
if aerosol_poly.intersects(roi_poly):
if description == "AATSR_PIX":
abs_diff = (df["timestamp"] - dt).abs()
df = df[abs_diff < delta_tolerance]
df.reset_index(inplace=True, drop=True)
if df.shape[0] == 0:
continue
intersection = aerosol_poly.intersection(roi_poly)
pts = GeoSeries(
[Point(x, y) for x, y in zip(df["lon"], df["lat"])])
idx = pts.within(intersection)
data = df[idx]["aerosol"].mean()
if numpy.isfinite(data):
# ancillary metadata tracking
md = current_h5_metadata(fid, dataset_path=pathname)
metadata = {
"id": numpy.array([md["id"]], VLEN_STRING),
"tier": tier.name,
}
return data, metadata
# default aerosol value
data = 0.06
metadata = {
"id": numpy.array([], VLEN_STRING),
"tier": AerosolTier.FALLBACK_DEFAULT.name,
}
return data, metadata
def collate(outdir: Union[str, Path]) -> None:
"""
Collate the results of the product comparison.
Firstly the results are merged with the framing geometry, and second
they're summarised.
"""
outdir = Path(outdir)
log_fname = outdir.joinpath(DirectoryNames.LOGS.value,
LogNames.COLLATE.value)
if not log_fname.parent.exists():
log_fname.parent.mkdir(parents=True)
with open(log_fname, "w") as fobj:
structlog.configure(logger_factory=structlog.PrintLoggerFactory(fobj),
processors=LOG_PROCESSORS)
comparison_results_fname = outdir.joinpath(
DirectoryNames.RESULTS.value, FileNames.RESULTS.value)
_LOG.info("opening intercomparison results file",
fname=str(comparison_results_fname))
with h5py.File(str(comparison_results_fname), "a") as fid:
grp = fid[DatasetGroups.INTERCOMPARISON.value]
for dataset_name in grp:
_LOG.info("reading dataset", dataset_name=dataset_name)
dataframe = read_h5_table(grp, dataset_name)
# some important attributes
framing = grp[dataset_name].attrs["framing"]
thematic = grp[dataset_name].attrs["thematic"]
proc_info = grp[dataset_name].attrs["proc-info"]
_LOG.info(
"merging results with framing",
framing=framing,
dataset_name=dataset_name,
)
geo_dataframe = merge_framing(dataframe, framing)
out_fname = outdir.joinpath(
DirectoryNames.RESULTS.value,
FileNames[MergeLookup[DatasetNames(
dataset_name).name].value].value,
)
_LOG.info("saving as GeoJSON", out_fname=str(out_fname))
geo_dataframe.to_file(str(out_fname), driver="GeoJSONSeq")
_LOG.info("summarising")
summary_dataframe = summarise(geo_dataframe, thematic,
proc_info)
out_dname = PPath(
DatasetGroups.SUMMARY.value,
DatasetNames[SummaryLookup[DatasetNames(
dataset_name).name].value].value,
)
_LOG.info("saving summary table",
out_dataset_name=str(out_dname))
write_dataframe(summary_dataframe, str(out_dname), fid)
def get_aerosol_data(acquisition, aerosol_dict):
"""
Extract the aerosol value for an acquisition.
The version 2 retrieves the data from a HDF5 file, and provides
more control over how the data is selected geo-metrically.
Better control over timedeltas.
"""
dt = acquisition.acquisition_datetime
geobox = acquisition.gridded_geo_box()
roi_poly = Polygon([
geobox.ul_lonlat, geobox.ur_lonlat, geobox.lr_lonlat, geobox.ll_lonlat
])
descr = ['AATSR_PIX', 'AATSR_CMP_YEAR_MONTH', 'AATSR_CMP_MONTH']
names = [
'ATSR_LF_%Y%m', 'aot_mean_%b_%Y_All_Aerosols',
'aot_mean_%b_All_Aerosols'
]
exts = ['/pix', '/cmp', '/cmp']
pathnames = [ppjoin(ext, dt.strftime(n)) for ext, n in zip(exts, names)]
# temporary until we sort out a better default mechanism
# how do we want to support default values, whilst still support provenance
if 'user' in aerosol_dict:
metadata = {'data_source': 'User defined value'}
return aerosol_dict['user'], metadata
else:
aerosol_fname = aerosol_dict['pathname']
fid = h5py.File(aerosol_fname, 'r')
url = urlparse(aerosol_fname, scheme='file').geturl()
delta_tolerance = datetime.timedelta(days=0.5)
data = None
for pathname, description in zip(pathnames, descr):
if pathname in fid:
df = read_h5_table(fid, pathname)
aerosol_poly = wkt.loads(fid[pathname].attrs['extents'])
if aerosol_poly.intersects(roi_poly):
if description == 'AATSR_PIX':
abs_diff = (df['timestamp'] - dt).abs()
df = df[abs_diff < delta_tolerance]
df.reset_index(inplace=True, drop=True)
if df.shape[0] == 0:
continue
intersection = aerosol_poly.intersection(roi_poly)
pts = GeoSeries(
[Point(x, y) for x, y in zip(df['lon'], df['lat'])])
idx = pts.within(intersection)
data = df[idx]['aerosol'].mean()
if numpy.isfinite(data):
metadata = {
'data_source': description,
'dataset_pathname': pathname,
'query_date': dt,
'url': url,
'extents': wkt.dumps(intersection)
}
# ancillary metadata tracking
md = extract_ancillary_metadata(aerosol_fname)
for key in md:
metadata[key] = md[key]
fid.close()
return data, metadata
# now we officially support a default value of 0.05 which
# should make the following redundant ....
# default aerosol value
# assumes we are only processing Australia in which case it it should
# be a coastal scene
data = 0.06
metadata = {'data_source': 'Default value used; Assumed a coastal scene'}
fid.close()
return data, metadata
def table_residual(ref_fid,
test_fid,
pathname,
out_fid,
compression=H5CompressionFilter.LZF,
save_inputs=False,
filter_opts=None):
"""
Output a residual TABLE of the numerical columns, ignoring
columns with the dtype `object`.
An equivalency test using `pandas.DataFrame.equals` is also
undertaken which if False, requires further investigation to
determine the column(s) and row(s) that are different.
:param ref_fid:
A h5py file object (essentially the root Group), containing
the reference data.
:param test_fid:
A h5py file object (essentially the root Group), containing
the test data.
:param pathname:
A `str` containing the pathname to the TABLE Dataset.
:param out_fid:
A h5py file object (essentially the root Group), opened for
writing the output data.
:param compression:
The compression filter to use.
Default is H5CompressionFilter.LZF
:param save_inputs:
A `bool` indicating whether or not to save the input datasets
used for evaluating the residuals alongside the results.
Default is False.
:filter_opts:
A dict of key value pairs available to the given configuration
instance of H5CompressionFilter. For example
H5CompressionFilter.LZF has the keywords *chunks* and *shuffle*
available.
Default is None, which will use the default settings for the
chosen H5CompressionFilter instance.
:return:
None; This routine will only return None or a print statement,
this is essential for the HDF5 visit routine.
"""
class_name = 'TABLE'
ref_df = read_h5_table(ref_fid, pathname)
test_df = read_h5_table(test_fid, pathname)
# ignore any `object` dtype columns (mostly just strings)
cols = [
col for col in ref_df.columns if ref_df[col].dtype.name != 'object'
]
# difference and pandas.DataFrame.equals test
df = ref_df[cols] - test_df[cols]
equal = test_df.equals(ref_df)
# ignored cols
cols = [
col for col in ref_df.columns if ref_df[col].dtype.name == 'object'
]
# output
attrs = {
'description': 'Residuals of numerical columns only',
'columns_ignored': numpy.array(cols, VLEN_STRING),
'equivalent': equal
}
base_dname = pbasename(pathname)
group_name = ref_fid[pathname].parent.name.strip('/')
dname = ppjoin('RESULTS', class_name, 'RESIDUALS', group_name, base_dname)
write_dataframe(df,
dname,
out_fid,
compression,
attrs=attrs,
filter_opts=filter_opts)
if save_inputs:
# copy the reference data
out_grp = out_fid.require_group(ppjoin('REFERENCE-DATA', group_name))
ref_fid.copy(ref_fid[pathname], out_grp)
# copy the test data
out_grp = out_fid.require_group(ppjoin('TEST-DATA', group_name))
test_fid.copy(test_fid[pathname], out_grp)
def card4l(level1,
granule,
workflow,
vertices,
method,
pixel_quality,
landsea,
tle_path,
aerosol,
brdf,
ozone_path,
water_vapour,
dem_path,
dsm_fname,
invariant_fname,
modtran_exe,
out_fname,
ecmwf_path=None,
rori=0.52,
buffer_distance=8000,
compression=H5CompressionFilter.LZF,
filter_opts=None,
h5_driver=None,
acq_parser_hint=None,
normalized_solar_zenith=45.):
"""
CEOS Analysis Ready Data for Land.
A workflow for producing standardised products that meet the
CARD4L specification.
:param level1:
A string containing the full file pathname to the level1
dataset.
:param granule:
A string containing the granule id to process.
:param workflow:
An enum from wagl.constants.Workflow representing which
workflow workflow to run.
:param vertices:
An integer 2-tuple indicating the number of rows and columns
of sample-locations ("coordinator") to produce.
The vertex columns should be an odd number.
:param method:
An enum from wagl.constants.Method representing the
interpolation method to use during the interpolation
of the atmospheric coefficients.
:param pixel_quality:
A bool indicating whether or not to run pixel quality.
:param landsea:
A string containing the full file pathname to the directory
containing the land/sea mask datasets.
:param tle_path:
A string containing the full file pathname to the directory
containing the two line element datasets.
:param aerosol:
A string containing the full file pathname to the HDF5 file
containing the aerosol data.
:param brdf:
A dict containing either user-supplied BRDF values, or the
full file pathname to the directory containing the BRDF data
and the decadal averaged BRDF data used for acquisitions
prior to TERRA/AQUA satellite operations.
:param ozone_path:
A string containing the full file pathname to the directory
containing the ozone datasets.
:param water_vapour:
A string containing the full file pathname to the directory
containing the water vapour datasets.
:param dem_path:
A string containing the full file pathname to the directory
containing the reduced resolution DEM.
:param dsm_path:
A string containing the full file pathname to the directory
containing the Digital Surface Workflow for use in terrain
illumination correction.
:param invariant_fname:
A string containing the full file pathname to the image file
containing the invariant geo-potential data for use within
the SBT process.
:param modtran_exe:
A string containing the full file pathname to the MODTRAN
executable.
:param out_fname:
A string containing the full file pathname that will contain
the output data from the data standardisation process.
executable.
:param ecmwf_path:
A string containing the full file pathname to the directory
containing the data from the European Centre for Medium Weather
Forcast, for use within the SBT process.
:param rori:
A floating point value for surface reflectance adjustment.
TODO Fuqin to add additional documentation for this parameter.
Default is 0.52.
:param buffer_distance:
A number representing the desired distance (in the same
units as the acquisition) in which to calculate the extra
number of pixels required to buffer an image.
Default is 8000, which for an acquisition using metres would
equate to 8000 metres.
:param compression:
An enum from hdf5.compression.H5CompressionFilter representing
the desired compression filter to use for writing H5 IMAGE and
TABLE class datasets to disk.
Default is H5CompressionFilter.LZF.
:param filter_opts:
A dict containing any additional keyword arguments when
generating the configuration for the given compression Filter.
Default is None.
:param h5_driver:
The specific HDF5 file driver to use when creating the output
HDF5 file.
See http://docs.h5py.org/en/latest/high/file.html#file-drivers
for more details.
Default is None; which writes direct to disk using the
appropriate driver for the underlying OS.
:param acq_parser_hint:
A string containing any hints to provide the acquisitions
loader with.
:param normalized_solar_zenith:
Solar zenith angle to normalize for (in degrees). Default is 45 degrees.
"""
json_fmt = pjoin(POINT_FMT, ALBEDO_FMT,
''.join([POINT_ALBEDO_FMT, '.json']))
nvertices = vertices[0] * vertices[1]
container = acquisitions(level1, hint=acq_parser_hint)
# TODO: pass through an acquisitions container rather than pathname
with h5py.File(out_fname, 'w', driver=h5_driver) as fid:
fid.attrs['level1_uri'] = level1
for grp_name in container.supported_groups:
log = STATUS_LOGGER.bind(level1=container.label,
granule=granule,
granule_group=grp_name)
# root group for a given granule and resolution group
root = fid.create_group(ppjoin(granule, grp_name))
acqs = container.get_acquisitions(granule=granule, group=grp_name)
# include the resolution as a group attribute
root.attrs['resolution'] = acqs[0].resolution
# longitude and latitude
log.info('Latitude-Longitude')
create_lon_lat_grids(acqs[0], root, compression, filter_opts)
# satellite and solar angles
log.info('Satellite-Solar-Angles')
calculate_angles(acqs[0], root[GroupName.LON_LAT_GROUP.value],
root, compression, filter_opts, tle_path)
if workflow in (Workflow.STANDARD, Workflow.NBAR):
# DEM
log.info('DEM-retriveal')
get_dsm(acqs[0], dsm_fname, buffer_distance, root, compression,
filter_opts)
# slope & aspect
log.info('Slope-Aspect')
slope_aspect_arrays(acqs[0],
root[GroupName.ELEVATION_GROUP.value],
buffer_distance, root, compression,
filter_opts)
# incident angles
log.info('Incident-Angles')
incident_angles(root[GroupName.SAT_SOL_GROUP.value],
root[GroupName.SLP_ASP_GROUP.value], root,
compression, filter_opts)
# exiting angles
log.info('Exiting-Angles')
exiting_angles(root[GroupName.SAT_SOL_GROUP.value],
root[GroupName.SLP_ASP_GROUP.value], root,
compression, filter_opts)
# relative azimuth slope
log.info('Relative-Azimuth-Angles')
incident_group_name = GroupName.INCIDENT_GROUP.value
exiting_group_name = GroupName.EXITING_GROUP.value
relative_azimuth_slope(root[incident_group_name],
root[exiting_group_name], root,
compression, filter_opts)
# self shadow
log.info('Self-Shadow')
self_shadow(root[incident_group_name],
root[exiting_group_name], root, compression,
filter_opts)
# cast shadow solar source direction
log.info('Cast-Shadow-Solar-Direction')
dsm_group_name = GroupName.ELEVATION_GROUP.value
calculate_cast_shadow(acqs[0], root[dsm_group_name],
root[GroupName.SAT_SOL_GROUP.value],
buffer_distance, root, compression,
filter_opts)
# cast shadow satellite source direction
log.info('Cast-Shadow-Satellite-Direction')
calculate_cast_shadow(acqs[0], root[dsm_group_name],
root[GroupName.SAT_SOL_GROUP.value],
buffer_distance, root, compression,
filter_opts, False)
# combined shadow masks
log.info('Combined-Shadow')
combine_shadow_masks(root[GroupName.SHADOW_GROUP.value],
root[GroupName.SHADOW_GROUP.value],
root[GroupName.SHADOW_GROUP.value], root,
compression, filter_opts)
# nbar and sbt ancillary
log = STATUS_LOGGER.bind(level1=container.label,
granule=granule,
granule_group=None)
# granule root group
root = fid[granule]
# get the highest resolution group containing supported bands
acqs, grp_name = container.get_highest_resolution(granule=granule)
grn_con = container.get_granule(granule=granule, container=True)
res_group = root[grp_name]
log.info('Ancillary-Retrieval')
nbar_paths = {
'aerosol_dict': aerosol,
'water_vapour_dict': water_vapour,
'ozone_path': ozone_path,
'dem_path': dem_path,
'brdf_dict': brdf
}
collect_ancillary(grn_con, res_group[GroupName.SAT_SOL_GROUP.value],
nbar_paths, ecmwf_path, invariant_fname, vertices,
root, compression, filter_opts)
# atmospherics
log.info('Atmospherics')
ancillary_group = root[GroupName.ANCILLARY_GROUP.value]
# satellite/solar angles and lon/lat for a resolution group
sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value]
lon_lat_grp = res_group[GroupName.LON_LAT_GROUP.value]
# TODO: supported acqs in different groups pointing to different response funcs
json_data, _ = format_json(acqs, ancillary_group, sat_sol_grp,
lon_lat_grp, workflow, root)
# atmospheric inputs group
inputs_grp = root[GroupName.ATMOSPHERIC_INPUTS_GRP.value]
# radiative transfer for each point and albedo
for key in json_data:
point, albedo = key
log.info('Radiative-Transfer', point=point, albedo=albedo.value)
with tempfile.TemporaryDirectory() as tmpdir:
prepare_modtran(acqs, point, [albedo], tmpdir)
point_dir = pjoin(tmpdir, POINT_FMT.format(p=point))
workdir = pjoin(point_dir, ALBEDO_FMT.format(a=albedo.value))
json_mod_infile = pjoin(
tmpdir, json_fmt.format(p=point, a=albedo.value))
with open(json_mod_infile, 'w') as src:
json_dict = json_data[key]
if albedo == Albedos.ALBEDO_TH:
json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \
"%s/%s" % (workdir, json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"])
json_dict["MODTRAN"][1]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \
"%s/%s" % (workdir, json_dict["MODTRAN"][1]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"])
else:
json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \
"%s/%s" % (workdir, json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"])
json.dump(json_dict, src, cls=JsonEncoder, indent=4)
run_modtran(acqs, inputs_grp, workflow, nvertices, point,
[albedo], modtran_exe, tmpdir, root, compression,
filter_opts)
# atmospheric coefficients
log.info('Coefficients')
results_group = root[GroupName.ATMOSPHERIC_RESULTS_GRP.value]
calculate_coefficients(results_group, root, compression, filter_opts)
esun_values = {}
# interpolate coefficients
for grp_name in container.supported_groups:
log = STATUS_LOGGER.bind(level1=container.label,
granule=granule,
granule_group=grp_name)
log.info('Interpolation')
# acquisitions and available bands for the current group level
acqs = container.get_acquisitions(granule=granule, group=grp_name)
nbar_acqs = [
acq for acq in acqs if acq.band_type == BandType.REFLECTIVE
]
sbt_acqs = [
acq for acq in acqs if acq.band_type == BandType.THERMAL
]
res_group = root[grp_name]
sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value]
comp_grp = root[GroupName.COEFFICIENTS_GROUP.value]
for coefficient in workflow.atmos_coefficients:
if coefficient is AtmosphericCoefficients.ESUN:
continue
if coefficient in Workflow.NBAR.atmos_coefficients:
band_acqs = nbar_acqs
else:
band_acqs = sbt_acqs
for acq in band_acqs:
log.info('Interpolate',
band_id=acq.band_id,
coefficient=coefficient.value)
interpolate(acq, coefficient, ancillary_group, sat_sol_grp,
comp_grp, res_group, compression, filter_opts,
method)
# standardised products
band_acqs = []
if workflow in (Workflow.STANDARD, Workflow.NBAR):
band_acqs.extend(nbar_acqs)
if workflow in (Workflow.STANDARD, Workflow.SBT):
band_acqs.extend(sbt_acqs)
for acq in band_acqs:
interp_grp = res_group[GroupName.INTERP_GROUP.value]
if acq.band_type == BandType.THERMAL:
log.info('SBT', band_id=acq.band_id)
surface_brightness_temperature(acq, interp_grp, res_group,
compression, filter_opts)
else:
atmos_coefs = read_h5_table(
comp_grp, DatasetName.NBAR_COEFFICIENTS.value)
esun_values[acq.band_name] = (
atmos_coefs[atmos_coefs.band_name == acq.band_name][
AtmosphericCoefficients.ESUN.value]).values[0]
slp_asp_grp = res_group[GroupName.SLP_ASP_GROUP.value]
rel_slp_asp = res_group[GroupName.REL_SLP_GROUP.value]
incident_grp = res_group[GroupName.INCIDENT_GROUP.value]
exiting_grp = res_group[GroupName.EXITING_GROUP.value]
shadow_grp = res_group[GroupName.SHADOW_GROUP.value]
log.info('Surface-Reflectance', band_id=acq.band_id)
calculate_reflectance(
acq, interp_grp, sat_sol_grp, slp_asp_grp, rel_slp_asp,
incident_grp, exiting_grp, shadow_grp, ancillary_group,
rori, res_group, compression, filter_opts,
normalized_solar_zenith, esun_values[acq.band_name])
# pixel quality
sbt_only = workflow == Workflow.SBT
if pixel_quality and can_pq(level1,
acq_parser_hint) and not sbt_only:
run_pq(level1, res_group, landsea, res_group, compression,
filter_opts, AP.NBAR, acq_parser_hint)
run_pq(level1, res_group, landsea, res_group, compression,
filter_opts, AP.NBART, acq_parser_hint)
def get_band_acqs(grp_name):
acqs = container.get_acquisitions(granule=granule, group=grp_name)
nbar_acqs = [
acq for acq in acqs if acq.band_type == BandType.REFLECTIVE
]
sbt_acqs = [
acq for acq in acqs if acq.band_type == BandType.THERMAL
]
band_acqs = []
if workflow in (Workflow.STANDARD, Workflow.NBAR):
band_acqs.extend(nbar_acqs)
if workflow in (Workflow.STANDARD, Workflow.SBT):
band_acqs.extend(sbt_acqs)
return band_acqs
# wagl parameters
parameters = {
'vertices': list(vertices),
'method': method.value,
'rori': rori,
'buffer_distance': buffer_distance,
'normalized_solar_zenith': normalized_solar_zenith,
'esun': esun_values
}
# metadata yaml's
metadata = root.create_group(DatasetName.METADATA.value)
create_ard_yaml(
{
grp_name: get_band_acqs(grp_name)
for grp_name in container.supported_groups
}, ancillary_group, metadata, parameters, workflow)
