def _build_index(indir): """ Read the INDEX table for each file and build a full history index. The records are sorted in ascending time (earliest to most recent) """ df = pandas.DataFrame(columns=["filename", "band_name", "timestamp"]) for fname in Path(indir).glob("pr_wtr.eatm.[0-9]*.h5"): with h5py.File(str(fname), "r") as fid: tmp_df = read_h5_table(fid, "INDEX") tmp_df["filename"] = fid.filename df = df.append(tmp_df, sort=False) df.sort_values("timestamp", inplace=True) df.set_index("timestamp", inplace=True) return df
def test_dataframe_roundtrip(self): """ Test that the pandas dataframe roundtrips, i.e. save to HDF5 and is read back into a dataframe seamlessly. Float, integer, datetime and string datatypes will be tested. """ df = pandas.DataFrame(self.table_data) df["timestamps"] = pandas.date_range("1/1/2000", periods=10, freq="D", tz="UTC") df["string_data"] = ["period {}".format(i) for i in range(10)] fname = "test_dataframe_roundtrip.h5" with h5py.File(fname, "w", **self.memory_kwargs) as fid: hdf5.write_dataframe(df, "dataframe", fid) # Apply conversion to no timezone that occurs in serialisation to hdf5 # Numpy is timezone naive; pandas has timezone support df["timestamps"] = df["timestamps"].dt.tz_convert(None) self.assertTrue(df.equals(hdf5.read_h5_table(fid, "dataframe")))
def interpolate(acq, coefficient, ancillary_group, satellite_solar_group, coefficients_group, out_group=None, compression=H5CompressionFilter.LZF, filter_opts=None, method=Method.SHEARB): # TODO: more docstrings """Perform interpolation.""" if method not in Method: msg = 'Interpolation method {} not available.' raise Exception(msg.format(method.name)) geobox = acq.gridded_geo_box() cols, rows = geobox.get_shape_xy() # read the relevant tables into DataFrames coordinator = read_h5_table(ancillary_group, DatasetName.COORDINATOR.value) boxline = read_h5_table(satellite_solar_group, DatasetName.BOXLINE.value) if coefficient in Workflow.NBAR.atmos_coefficients: dataset_name = DatasetName.NBAR_COEFFICIENTS.value elif coefficient in Workflow.SBT.atmos_coefficients: dataset_name = DatasetName.SBT_COEFFICIENTS.value else: msg = "Factor name not found in available coefficients: {}" raise ValueError(msg.format(Workflow.STANDARD.atmos_coefficients)) coefficients = read_h5_table(coefficients_group, dataset_name) coord = np.zeros((coordinator.shape[0], 2), dtype='int') map_x = coordinator.map_x.values map_y = coordinator.map_y.values coord[:, 1], coord[:, 0] = (map_x, map_y) * ~geobox.transform centre = boxline.bisection_index.values start = boxline.start_index.values end = boxline.end_index.values band_records = coefficients.band_name == acq.band_name samples = coefficients[coefficient.value][band_records].values func_map = {Method.BILINEAR: sheared_bilinear_interpolate, Method.FBILINEAR: fortran_bilinear_interpolate, Method.SHEAR: sheared_bilinear_interpolate, Method.SHEARB: sheared_bilinear_interpolate, Method.RBF: rbf_interpolate} args = [cols, rows, coord, samples, start, end, centre] if method == Method.BILINEAR: args.extend([False, False]) elif method == Method.SHEARB: args.extend([True, True]) else: pass result = func_map[method](*args) # setup the output file/group as needed if out_group is None: fid = h5py.File('interpolated-coefficients.h5', driver='core', backing_store=False) else: fid = out_group if GroupName.INTERP_GROUP.value not in fid: fid.create_group(GroupName.INTERP_GROUP.value) if filter_opts is None: filter_opts = {} else: filter_opts = filter_opts.copy() filter_opts['chunks'] = acq.tile_size group = fid[GroupName.INTERP_GROUP.value] fmt = DatasetName.INTERPOLATION_FMT.value dset_name = fmt.format(coefficient=coefficient.value, band_name=acq.band_name) no_data = -999 attrs = {'crs_wkt': geobox.crs.ExportToWkt(), 'geotransform': geobox.transform.to_gdal(), 'no_data_value': no_data, 'interpolation_method': method.name, 'band_id': acq.band_id, 'band_name': acq.band_name, 'alias': acq.alias, 'coefficient': coefficient.value} desc = ("Contains the interpolated result of coefficient {} " "for band {} from sensor {}.") attrs['description'] = desc.format(coefficient.value, acq.band_id, acq.sensor_id) # convert any NaN's to -999 (for float data, NaN would be more ideal ...) result[~np.isfinite(result)] = no_data write_h5_image(result, dset_name, group, compression, attrs, filter_opts) if out_group is None: return fid
def test_modtran_run(self): """ Tests that the interface to modtran (run_modtran) works for known inputs. Used to validate environment configuration/setup """ band_names = [ 'BAND-1', 'BAND-2', 'BAND-3', 'BAND-4', 'BAND-5', 'BAND-6', 'BAND-7', 'BAND-8' ] point = 0 albedo = Albedos.ALBEDO_0 # setup mock acquistions object acquisitions = [] for bandn in band_names: acq = mock.MagicMock() acq.acquisition_datetime = datetime(2001, 1, 1) acq.band_type = BandType.REFLECTIVE acq.spectral_response = mock_spectral_response acquisitions.append(acq) # setup mock atmospherics group attrs = {'lonlat': 'TEST'} atmospherics = mock.MagicMock() atmospherics.attrs = attrs atmospherics_group = {POINT_FMT.format(p=point): atmospherics} # Compute base path -- prefix for hdf5 file base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value, POINT_FMT.format(p=point)) with tempfile.TemporaryDirectory() as workdir: run_dir = pjoin(workdir, POINT_FMT.format(p=point), ALBEDO_FMT.format(a=albedo.value)) os.makedirs(run_dir) # TODO replace json_input copy with json input generation with open(INPUT_JSON, 'r') as fd: json_data = json.load(fd) for mod_input in json_data['MODTRAN']: mod_input['MODTRANINPUT']['SPECTRAL'][ 'FILTNM'] = SPECTRAL_RESPONSE_LS8 with open( pjoin(run_dir, POINT_ALBEDO_FMT.format(p=point, a=albedo.value)) + ".json", 'w') as fd: json.dump(json_data, fd) fid = run_modtran( acquisitions, atmospherics_group, Workflow.STANDARD, npoints=12, # number of track points point=point, albedos=[albedo], modtran_exe=MODTRAN_EXE, basedir=workdir, out_group=None) assert fid # Test base attrs assert fid[base_path].attrs['lonlat'] == 'TEST' assert fid[base_path].attrs['datetime'] == datetime(2001, 1, 1).isoformat() # test albedo headers? # Summarise modtran results to surface reflectance coefficients test_grp = fid[base_path][ALBEDO_FMT.format(a=albedo.value)] nbar_coefficients, _ = coefficients( read_h5_table( fid, pjoin(base_path, ALBEDO_FMT.format(a=albedo.value), DatasetName.CHANNEL.value)), read_h5_table( fid, pjoin(base_path, ALBEDO_FMT.format(a=albedo.value), DatasetName.SOLAR_ZENITH_CHANNEL.value))) expected = pd.read_csv(EXPECTED_CSV, index_col='band_name') pd.testing.assert_frame_equal(nbar_coefficients, expected, check_less_precise=True)
def comparison(outdir: Union[str, Path], proc_info: bool) -> None: """ Test and Reference product intercomparison evaluation. """ outdir = Path(outdir) if proc_info: log_fname = outdir.joinpath(DirectoryNames.LOGS.value, LogNames.PROC_INFO_INTERCOMPARISON.value) else: log_fname = outdir.joinpath(DirectoryNames.LOGS.value, LogNames.MEASUREMENT_INTERCOMPARISON.value) out_stream = MPIStreamIO(str(log_fname)) structlog.configure(processors=DEFAULT_PROCESSORS, logger_factory=MPILoggerFactory(out_stream)) # processor info rank = COMM.Get_rank() n_processors = COMM.Get_size() results_fname = outdir.joinpath(DirectoryNames.RESULTS.value, FileNames.RESULTS.value) with h5py.File(str(results_fname), "r") as fid: dataframe = read_h5_table(fid, DatasetNames.QUERY.value) if rank == 0: index = dataframe.index.values.tolist() blocks = scatter(index, n_processors) # some basic attribute information doc: Union[Granule, None] = load_odc_metadata( Path(dataframe.iloc[0].yaml_pathname_reference)) attrs: Dict[str, Any] = { "framing": doc.framing, "thematic": False, "proc-info": False, } else: blocks = None doc = None attrs = dict() COMM.Barrier() # equally partition the work across all procesors indices = COMM.scatter(blocks, root=0) if proc_info: attrs["proc-info"] = True if rank == 0: _LOG.info("procssing proc-info documents") gqa_dataframe, ancillary_dataframe = _process_proc_info( dataframe.iloc[indices], rank) if rank == 0: _LOG.info("saving gqa dataframe results to tables") if not results_fname.parent.exists(): results_fname.parent.mkdir(parents=True) with h5py.File(str(results_fname), "a") as fid: dataset_name = PPath(DatasetGroups.INTERCOMPARISON.value, DatasetNames.GQA_RESULTS.value) write_dataframe(gqa_dataframe, str(dataset_name), fid, attrs=attrs) _LOG.info("saving ancillary dataframe results to tables") if not results_fname.parent.exists(): results_fname.parent.mkdir(parents=True) with h5py.File(str(results_fname), "a") as fid: dataset_name = PPath( DatasetGroups.INTERCOMPARISON.value, DatasetNames.ANCILLARY_RESULTS.value, ) write_dataframe(ancillary_dataframe, str(dataset_name), fid, attrs=attrs) _LOG.info("saving software versions dataframe to tables") with h5py.File(str(results_fname), "a") as fid: dataset_name = PPath(DatasetNames.SOFTWARE_VERSIONS.value) software_attrs = { "description": "ARD Pipeline software versions" } software_df = compare_proc_info.compare_software(dataframe) write_dataframe(software_df, str(dataset_name), fid, attrs=software_attrs) else: if rank == 0: _LOG.info("processing odc-metadata documents") results = _process_odc_doc(dataframe.iloc[indices], rank) if rank == 0: # save each table _LOG.info("saving dataframes to tables") with h5py.File(str(results_fname), "a") as fid: attrs["thematic"] = False write_dataframe( results[0], str( PPath( DatasetGroups.INTERCOMPARISON.value, DatasetNames.GENERAL_RESULTS.value, )), fid, attrs=attrs, ) attrs["thematic"] = True write_dataframe( results[1], str( PPath( DatasetGroups.INTERCOMPARISON.value, DatasetNames.FMASK_RESULTS.value, )), fid, attrs=attrs, ) write_dataframe( results[2], str( PPath( DatasetGroups.INTERCOMPARISON.value, DatasetNames.CONTIGUITY_RESULTS.value, )), fid, attrs=attrs, ) write_dataframe( results[3], str( PPath( DatasetGroups.INTERCOMPARISON.value, DatasetNames.SHADOW_RESULTS.value, )), fid, attrs=attrs, ) if rank == 0: workflow = "proc-info field" if proc_info else "product measurement" msg = f"{workflow} comparison processing finished" _LOG.info(msg)
def format_json(acquisitions, ancillary_group, satellite_solar_group, lon_lat_group, workflow, out_group): """ Creates json files for the albedo (0) and thermal """ # angles data sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value] sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value] longitude = lon_lat_group[DatasetName.LON.value] latitude = lon_lat_group[DatasetName.LAT.value] # retrieve the averaged ancillary if available anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value) if anc_grp is None: anc_grp = ancillary_group # ancillary data coordinator = ancillary_group[DatasetName.COORDINATOR.value] aerosol = anc_grp[DatasetName.AEROSOL.value][()] water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()] ozone = anc_grp[DatasetName.OZONE.value][()] elevation = anc_grp[DatasetName.ELEVATION.value][()] npoints = coordinator.shape[0] view = numpy.zeros(npoints, dtype='float32') azi = numpy.zeros(npoints, dtype='float32') lat = numpy.zeros(npoints, dtype='float64') lon = numpy.zeros(npoints, dtype='float64') for i in range(npoints): yidx = coordinator['row_index'][i] xidx = coordinator['col_index'][i] view[i] = sat_view[yidx, xidx] azi[i] = sat_azi[yidx, xidx] lat[i] = latitude[yidx, xidx] lon[i] = longitude[yidx, xidx] view_corrected = 180 - view azi_corrected = azi + 180 rlon = 360 - lon # check if in western hemisphere idx = rlon >= 360 rlon[idx] -= 360 idx = (180 - view_corrected) < 0.1 view_corrected[idx] = 180 azi_corrected[idx] = 0 idx = azi_corrected > 360 azi_corrected[idx] -= 360 # get the modtran profiles to use based on the centre latitude _, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat if out_group is None: out_group = h5py.File('atmospheric-inputs.h5', 'w') if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group: out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value) group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value] iso_time = acquisitions[0].acquisition_datetime.isoformat() group.attrs['acquisition-datetime'] = iso_time json_data = {} # setup the json files required by MODTRAN if workflow in (Workflow.STANDARD, Workflow.NBAR): acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE] for p in range(npoints): for alb in Workflow.NBAR.albedos: input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a=str(alb.value)), 'water': water_vapour, 'ozone': ozone, 'doy': acquisitions[0].julian_day(), 'visibility': -aerosol, 'lat': lat[p], 'lon': rlon[p], 'time': acquisitions[0].decimal_hour(), 'sat_azimuth': azi_corrected[p], 'sat_height': acquisitions[0].altitude / 1000.0, 'elevation': elevation, 'sat_view': view_corrected[p], 'albedo': float(alb.value), 'filter_function': acqs[0].spectral_filter_name, 'binary': False } if centre_lat < -23.0: data = mpjson.midlat_summer_albedo(**input_data) else: data = mpjson.tropical_albedo(**input_data) input_data['description'] = 'Input file for MODTRAN' input_data['file_format'] = 'json' input_data.pop('binary') json_data[(p, alb)] = data data = json.dumps(data, cls=JsonEncoder, indent=4) dname = ppjoin(POINT_FMT.format(p=p), ALBEDO_FMT.format(a=alb.value), DatasetName.MODTRAN_INPUT.value) write_scalar(data, dname, group, input_data) # create json for sbt if it has been collected if ancillary_group.attrs.get('sbt-ancillary'): dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value) acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL] for p in range(npoints): atmos_profile = read_h5_table(ancillary_group, dname.format(p=p)) n_layers = atmos_profile.shape[0] + 6 elevation = atmos_profile.iloc[0]['GeoPotential_Height'] input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a='TH'), 'ozone': ozone, 'n': n_layers, 'prof_alt': list(atmos_profile['GeoPotential_Height']), 'prof_pres': list(atmos_profile['Pressure']), 'prof_temp': list(atmos_profile['Temperature']), 'prof_water': list(atmos_profile['Relative_Humidity']), 'visibility': -aerosol, 'sat_height': acquisitions[0].altitude / 1000.0, 'gpheight': elevation, 'sat_view': view_corrected[p], 'filter_function': acqs[0].spectral_filter_name, 'binary': False } data = mpjson.thermal_transmittance(**input_data) input_data['description'] = 'Input File for MODTRAN' input_data['file_format'] = 'json' input_data.pop('binary') json_data[(p, Albedos.ALBEDO_TH)] = data data = json.dumps(data, cls=JsonEncoder, indent=4) out_dname = ppjoin(POINT_FMT.format(p=p), ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value), DatasetName.MODTRAN_INPUT.value) write_scalar(data, out_dname, group, input_data) # attach location info to each point Group for p in range(npoints): lonlat = (coordinator['longitude'][p], coordinator['latitude'][p]) group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat return json_data, out_group
def calculate_coefficients(atmospheric_results_group, out_group, compression=H5CompressionFilter.LZF, filter_opts=None): """ Calculate the atmospheric coefficients from the MODTRAN output and used in the BRDF and atmospheric correction. Coefficients are computed for each band for each each coordinate for each atmospheric coefficient. The atmospheric coefficients can be found in `Workflow.STANDARD.atmos_coefficients`. :param atmospheric_results_group: The root HDF5 `Group` that contains the atmospheric results from each MODTRAN run. :param out_group: If set to None (default) then the results will be returned as an in-memory hdf5 file, i.e. the `core` driver. Otherwise, a writeable HDF5 `Group` object. The datasets will be formatted to the HDF5 TABLE specification and the dataset names will be as follows: * DatasetName.NBAR_COEFFICIENTS (if Workflow.STANDARD or Workflow.NBAR) * DatasetName.SBT_COEFFICIENTS (if Workflow.STANDARD or Workflow.SBT) :param compression: The compression filter to use. Default is H5CompressionFilter.LZF :param filter_opts: A dict of key value pairs available to the given configuration instance of H5CompressionFilter. For example H5CompressionFilter.LZF has the keywords *chunks* and *shuffle* available. Default is None, which will use the default settings for the chosen H5CompressionFilter instance. :return: An opened `h5py.File` object, that is either in-memory using the `core` driver, or on disk. """ nbar_coefficients = pd.DataFrame() sbt_coefficients = pd.DataFrame() channel_data = channel_solar_angle = upward = downward = None # Initialise the output group/file if out_group is None: fid = h5py.File('atmospheric-coefficients.h5', driver='core', backing_store=False) else: fid = out_group res = atmospheric_results_group npoints = res.attrs['npoints'] nbar_atmos = res.attrs['nbar_atmospherics'] sbt_atmos = res.attrs['sbt_atmospherics'] for point in range(npoints): point_grp = res[POINT_FMT.format(p=point)] lonlat = point_grp.attrs['lonlat'] timestamp = pd.to_datetime(point_grp.attrs['datetime']) grp_path = ppjoin(POINT_FMT.format(p=point), ALBEDO_FMT) if nbar_atmos: channel_path = ppjoin(grp_path.format(a=Albedos.ALBEDO_0.value), DatasetName.CHANNEL.value) channel_data = read_h5_table(res, channel_path) channel_solar_angle_path = ppjoin( grp_path.format(a=Albedos.ALBEDO_0.value), DatasetName.SOLAR_ZENITH_CHANNEL.value ) channel_solar_angle = read_h5_table(res, channel_solar_angle_path) if sbt_atmos: dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value), DatasetName.UPWARD_RADIATION_CHANNEL.value) upward = read_h5_table(res, dname) dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value), DatasetName.DOWNWARD_RADIATION_CHANNEL.value) downward = read_h5_table(res, dname) kwargs = {'channel_data': channel_data, 'solar_zenith_angle': channel_solar_angle, 'upward_radiation': upward, 'downward_radiation': downward} result = coefficients(**kwargs) # insert some datetime/geospatial fields if result[0] is not None: result[0].insert(0, 'POINT', point) result[0].insert(1, 'LONGITUDE', lonlat[0]) result[0].insert(2, 'LATITUDE', lonlat[1]) result[0].insert(3, 'DATETIME', timestamp) nbar_coefficients = nbar_coefficients.append(result[0]) if result[1] is not None: result[1].insert(0, 'POINT', point) result[1].insert(1, 'LONGITUDE', lonlat[0]) result[1].insert(2, 'LATITUDE', lonlat[1]) result[1].insert(3, 'DATETIME', pd.to_datetime(timestamp)) sbt_coefficients = sbt_coefficients.append(result[1]) nbar_coefficients.reset_index(inplace=True) sbt_coefficients.reset_index(inplace=True) attrs = {'npoints': npoints} description = "Coefficients derived from the VNIR solar irradiation." attrs['description'] = description dname = DatasetName.NBAR_COEFFICIENTS.value if GroupName.COEFFICIENTS_GROUP.value not in fid: fid.create_group(GroupName.COEFFICIENTS_GROUP.value) group = fid[GroupName.COEFFICIENTS_GROUP.value] if nbar_atmos: write_dataframe(nbar_coefficients, dname, group, compression, attrs=attrs, filter_opts=filter_opts) description = "Coefficients derived from the THERMAL solar irradiation." attrs['description'] = description dname = DatasetName.SBT_COEFFICIENTS.value if sbt_atmos: write_dataframe(sbt_coefficients, dname, group, compression, attrs=attrs, filter_opts=filter_opts) if out_group is None: return fid
def format_tp5(acquisitions, ancillary_group, satellite_solar_group, lon_lat_group, workflow, out_group): """ Creates str formatted tp5 files for the albedo (0, 1) and transmittance (t). """ # angles data sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value] sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value] longitude = lon_lat_group[DatasetName.LON.value] latitude = lon_lat_group[DatasetName.LAT.value] # retrieve the averaged ancillary if available anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value) if anc_grp is None: anc_grp = ancillary_group # ancillary data coordinator = ancillary_group[DatasetName.COORDINATOR.value] aerosol = anc_grp[DatasetName.AEROSOL.value][()] water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()] ozone = anc_grp[DatasetName.OZONE.value][()] elevation = anc_grp[DatasetName.ELEVATION.value][()] npoints = coordinator.shape[0] view = numpy.zeros(npoints, dtype='float32') azi = numpy.zeros(npoints, dtype='float32') lat = numpy.zeros(npoints, dtype='float64') lon = numpy.zeros(npoints, dtype='float64') for i in range(npoints): yidx = coordinator['row_index'][i] xidx = coordinator['col_index'][i] view[i] = sat_view[yidx, xidx] azi[i] = sat_azi[yidx, xidx] lat[i] = latitude[yidx, xidx] lon[i] = longitude[yidx, xidx] view_corrected = 180 - view azi_corrected = azi + 180 rlon = 360 - lon # check if in western hemisphere idx = rlon >= 360 rlon[idx] -= 360 idx = (180 - view_corrected) < 0.1 view_corrected[idx] = 180 azi_corrected[idx] = 0 idx = azi_corrected > 360 azi_corrected[idx] -= 360 # get the modtran profiles to use based on the centre latitude _, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat if centre_lat < -23.0: albedo_profile = MIDLAT_SUMMER_ALBEDO trans_profile = MIDLAT_SUMMER_TRANSMITTANCE else: albedo_profile = TROPICAL_ALBEDO trans_profile = TROPICAL_TRANSMITTANCE if out_group is None: out_group = h5py.File('atmospheric-inputs.h5', 'w') if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group: out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value) group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value] iso_time = acquisitions[0].acquisition_datetime.isoformat() group.attrs['acquisition-datetime'] = iso_time tp5_data = {} # setup the tp5 files required by MODTRAN if workflow == Workflow.STANDARD or workflow == Workflow.NBAR: acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE] for p in range(npoints): for alb in Workflow.NBAR.albedos: input_data = { 'water': water_vapour, 'ozone': ozone, 'filter_function': acqs[0].spectral_filter_file, 'visibility': -aerosol, 'elevation': elevation, 'sat_height': acquisitions[0].altitude / 1000.0, 'sat_view': view_corrected[p], 'doy': acquisitions[0].julian_day(), 'binary': 'T' } if alb == Albedos.ALBEDO_T: input_data['albedo'] = 0.0 input_data['sat_view_offset'] = 180.0 - view_corrected[p] data = trans_profile.format(**input_data) else: input_data['albedo'] = float(alb.value) input_data['lat'] = lat[p] input_data['lon'] = rlon[p] input_data['time'] = acquisitions[0].decimal_hour() input_data['sat_azimuth'] = azi_corrected[p] data = albedo_profile.format(**input_data) tp5_data[(p, alb)] = data dname = ppjoin(POINT_FMT.format(p=p), ALBEDO_FMT.format(a=alb.value), DatasetName.TP5.value) write_scalar(numpy.string_(data), dname, group, input_data) # create tp5 for sbt if it has been collected if ancillary_group.attrs.get('sbt-ancillary'): dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value) acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL] for p in range(npoints): atmospheric_profile = [] atmos_profile = read_h5_table(ancillary_group, dname.format(p=p)) n_layers = atmos_profile.shape[0] + 6 elevation = atmos_profile.iloc[0]['GeoPotential_Height'] for i, row in atmos_profile.iterrows(): input_data = { 'gpheight': row['GeoPotential_Height'], 'pressure': row['Pressure'], 'airtemp': row['Temperature'], 'humidity': row['Relative_Humidity'], 'zero': 0.0 } atmospheric_profile.append(SBT_FORMAT.format(**input_data)) input_data = { 'ozone': ozone, 'filter_function': acqs[0].spectral_filter_file, 'visibility': -aerosol, 'gpheight': elevation, 'n': n_layers, 'sat_height': acquisitions[0].altitude / 1000.0, 'sat_view': view_corrected[p], 'binary': 'T', 'atmospheric_profile': ''.join(atmospheric_profile) } data = THERMAL_TRANSMITTANCE.format(**input_data) tp5_data[(p, Albedos.ALBEDO_TH)] = data out_dname = ppjoin(POINT_FMT.format(p=p), ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value), DatasetName.TP5.value) write_scalar(numpy.string_(data), out_dname, group, input_data) # attach location info to each point Group for p in range(npoints): lonlat = (coordinator['longitude'][p], coordinator['latitude'][p]) group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat return tp5_data, out_group
def get_water_vapour(acquisition, water_vapour_dict, scale_factor=0.1, tolerance=1): """ Retrieve the water vapour value for an `acquisition` and the path for the water vapour ancillary data. """ dt = acquisition.acquisition_datetime geobox = acquisition.gridded_geo_box() year = dt.strftime("%Y") hour = dt.timetuple().tm_hour filename = "pr_wtr.eatm.{year}.h5".format(year=year) if "user" in water_vapour_dict: metadata = { "id": numpy.array([], VLEN_STRING), "tier": WaterVapourTier.USER.name } return water_vapour_dict["user"], metadata water_vapour_path = water_vapour_dict["pathname"] datafile = pjoin(water_vapour_path, filename) if os.path.isfile(datafile): with h5py.File(datafile, "r") as fid: index = read_h5_table(fid, "INDEX") # set the tolerance in days to search back in time max_tolerance = -datetime.timedelta(days=tolerance) # only look for observations that have occured in the past time_delta = index.timestamp - dt result = time_delta[(time_delta < datetime.timedelta()) & (time_delta > max_tolerance)] if not os.path.isfile(datafile) or result.shape[0] == 0: if "fallback_dataset" not in water_vapour_dict: raise AncillaryError("No actual or fallback water vapour data.") tier = WaterVapourTier.FALLBACK_DATASET month = dt.strftime("%B-%d").upper() # closest previous observation # i.e. observations are at 0000, 0600, 1200, 1800 # and an acquisition hour of 1700 will use the 1200 observation observations = numpy.array([0, 6, 12, 18]) hr = observations[numpy.argmin(numpy.abs(hour - observations))] dataset_name = "AVERAGE/{}/{:02d}00".format(month, hr) datafile = water_vapour_dict["fallback_dataset"] else: tier = WaterVapourTier.DEFINITIVE # get the index of the closest water vapour observation # which would be the maximum timedelta # as we're only dealing with negative timedelta's here idx = result.idxmax() record = index.iloc[idx] dataset_name = record.dataset_name try: data, md_uuid = get_pixel(datafile, dataset_name, geobox.centre_lonlat) except ValueError: # h5py raises a ValueError not an IndexError for out of bounds raise AncillaryError("No Water Vapour data") # the metadata from the original file says (Kg/m^2) # so multiply by 0.1 to get (g/cm^2) data = data * scale_factor metadata = {"id": numpy.array([md_uuid], VLEN_STRING), "tier": tier.name} return data, metadata
def get_aerosol_data(acquisition, aerosol_dict): """ Extract the aerosol value for an acquisition. The version 2 retrieves the data from a HDF5 file, and provides more control over how the data is selected geo-metrically. Better control over timedeltas. """ dt = acquisition.acquisition_datetime geobox = acquisition.gridded_geo_box() roi_poly = Polygon([ geobox.ul_lonlat, geobox.ur_lonlat, geobox.lr_lonlat, geobox.ll_lonlat ]) descr = ["AATSR_PIX", "AATSR_CMP_YEAR_MONTH", "AATSR_CMP_MONTH"] names = [ "ATSR_LF_%Y%m", "aot_mean_%b_%Y_All_Aerosols", "aot_mean_%b_All_Aerosols" ] exts = ["/pix", "/cmp", "/cmp"] pathnames = [ppjoin(ext, dt.strftime(n)) for ext, n in zip(exts, names)] # temporary until we sort out a better default mechanism # how do we want to support default values, whilst still support provenance if "user" in aerosol_dict: tier = AerosolTier.USER metadata = {"id": numpy.array([], VLEN_STRING), "tier": tier.name} return aerosol_dict["user"], metadata aerosol_fname = aerosol_dict["pathname"] data = None delta_tolerance = datetime.timedelta(days=0.5) with h5py.File(aerosol_fname, "r") as fid: for pathname, description in zip(pathnames, descr): tier = AerosolTier[description] if pathname in fid: df = read_h5_table(fid, pathname) aerosol_poly = wkt.loads(fid[pathname].attrs["extents"]) if aerosol_poly.intersects(roi_poly): if description == "AATSR_PIX": abs_diff = (df["timestamp"] - dt).abs() df = df[abs_diff < delta_tolerance] df.reset_index(inplace=True, drop=True) if df.shape[0] == 0: continue intersection = aerosol_poly.intersection(roi_poly) pts = GeoSeries( [Point(x, y) for x, y in zip(df["lon"], df["lat"])]) idx = pts.within(intersection) data = df[idx]["aerosol"].mean() if numpy.isfinite(data): # ancillary metadata tracking md = current_h5_metadata(fid, dataset_path=pathname) metadata = { "id": numpy.array([md["id"]], VLEN_STRING), "tier": tier.name, } return data, metadata # default aerosol value data = 0.06 metadata = { "id": numpy.array([], VLEN_STRING), "tier": AerosolTier.FALLBACK_DEFAULT.name, } return data, metadata
def collate(outdir: Union[str, Path]) -> None: """ Collate the results of the product comparison. Firstly the results are merged with the framing geometry, and second they're summarised. """ outdir = Path(outdir) log_fname = outdir.joinpath(DirectoryNames.LOGS.value, LogNames.COLLATE.value) if not log_fname.parent.exists(): log_fname.parent.mkdir(parents=True) with open(log_fname, "w") as fobj: structlog.configure(logger_factory=structlog.PrintLoggerFactory(fobj), processors=LOG_PROCESSORS) comparison_results_fname = outdir.joinpath( DirectoryNames.RESULTS.value, FileNames.RESULTS.value) _LOG.info("opening intercomparison results file", fname=str(comparison_results_fname)) with h5py.File(str(comparison_results_fname), "a") as fid: grp = fid[DatasetGroups.INTERCOMPARISON.value] for dataset_name in grp: _LOG.info("reading dataset", dataset_name=dataset_name) dataframe = read_h5_table(grp, dataset_name) # some important attributes framing = grp[dataset_name].attrs["framing"] thematic = grp[dataset_name].attrs["thematic"] proc_info = grp[dataset_name].attrs["proc-info"] _LOG.info( "merging results with framing", framing=framing, dataset_name=dataset_name, ) geo_dataframe = merge_framing(dataframe, framing) out_fname = outdir.joinpath( DirectoryNames.RESULTS.value, FileNames[MergeLookup[DatasetNames( dataset_name).name].value].value, ) _LOG.info("saving as GeoJSON", out_fname=str(out_fname)) geo_dataframe.to_file(str(out_fname), driver="GeoJSONSeq") _LOG.info("summarising") summary_dataframe = summarise(geo_dataframe, thematic, proc_info) out_dname = PPath( DatasetGroups.SUMMARY.value, DatasetNames[SummaryLookup[DatasetNames( dataset_name).name].value].value, ) _LOG.info("saving summary table", out_dataset_name=str(out_dname)) write_dataframe(summary_dataframe, str(out_dname), fid)
def get_aerosol_data(acquisition, aerosol_dict): """ Extract the aerosol value for an acquisition. The version 2 retrieves the data from a HDF5 file, and provides more control over how the data is selected geo-metrically. Better control over timedeltas. """ dt = acquisition.acquisition_datetime geobox = acquisition.gridded_geo_box() roi_poly = Polygon([ geobox.ul_lonlat, geobox.ur_lonlat, geobox.lr_lonlat, geobox.ll_lonlat ]) descr = ['AATSR_PIX', 'AATSR_CMP_YEAR_MONTH', 'AATSR_CMP_MONTH'] names = [ 'ATSR_LF_%Y%m', 'aot_mean_%b_%Y_All_Aerosols', 'aot_mean_%b_All_Aerosols' ] exts = ['/pix', '/cmp', '/cmp'] pathnames = [ppjoin(ext, dt.strftime(n)) for ext, n in zip(exts, names)] # temporary until we sort out a better default mechanism # how do we want to support default values, whilst still support provenance if 'user' in aerosol_dict: metadata = {'data_source': 'User defined value'} return aerosol_dict['user'], metadata else: aerosol_fname = aerosol_dict['pathname'] fid = h5py.File(aerosol_fname, 'r') url = urlparse(aerosol_fname, scheme='file').geturl() delta_tolerance = datetime.timedelta(days=0.5) data = None for pathname, description in zip(pathnames, descr): if pathname in fid: df = read_h5_table(fid, pathname) aerosol_poly = wkt.loads(fid[pathname].attrs['extents']) if aerosol_poly.intersects(roi_poly): if description == 'AATSR_PIX': abs_diff = (df['timestamp'] - dt).abs() df = df[abs_diff < delta_tolerance] df.reset_index(inplace=True, drop=True) if df.shape[0] == 0: continue intersection = aerosol_poly.intersection(roi_poly) pts = GeoSeries( [Point(x, y) for x, y in zip(df['lon'], df['lat'])]) idx = pts.within(intersection) data = df[idx]['aerosol'].mean() if numpy.isfinite(data): metadata = { 'data_source': description, 'dataset_pathname': pathname, 'query_date': dt, 'url': url, 'extents': wkt.dumps(intersection) } # ancillary metadata tracking md = extract_ancillary_metadata(aerosol_fname) for key in md: metadata[key] = md[key] fid.close() return data, metadata # now we officially support a default value of 0.05 which # should make the following redundant .... # default aerosol value # assumes we are only processing Australia in which case it it should # be a coastal scene data = 0.06 metadata = {'data_source': 'Default value used; Assumed a coastal scene'} fid.close() return data, metadata
def table_residual(ref_fid, test_fid, pathname, out_fid, compression=H5CompressionFilter.LZF, save_inputs=False, filter_opts=None): """ Output a residual TABLE of the numerical columns, ignoring columns with the dtype `object`. An equivalency test using `pandas.DataFrame.equals` is also undertaken which if False, requires further investigation to determine the column(s) and row(s) that are different. :param ref_fid: A h5py file object (essentially the root Group), containing the reference data. :param test_fid: A h5py file object (essentially the root Group), containing the test data. :param pathname: A `str` containing the pathname to the TABLE Dataset. :param out_fid: A h5py file object (essentially the root Group), opened for writing the output data. :param compression: The compression filter to use. Default is H5CompressionFilter.LZF :param save_inputs: A `bool` indicating whether or not to save the input datasets used for evaluating the residuals alongside the results. Default is False. :filter_opts: A dict of key value pairs available to the given configuration instance of H5CompressionFilter. For example H5CompressionFilter.LZF has the keywords *chunks* and *shuffle* available. Default is None, which will use the default settings for the chosen H5CompressionFilter instance. :return: None; This routine will only return None or a print statement, this is essential for the HDF5 visit routine. """ class_name = 'TABLE' ref_df = read_h5_table(ref_fid, pathname) test_df = read_h5_table(test_fid, pathname) # ignore any `object` dtype columns (mostly just strings) cols = [ col for col in ref_df.columns if ref_df[col].dtype.name != 'object' ] # difference and pandas.DataFrame.equals test df = ref_df[cols] - test_df[cols] equal = test_df.equals(ref_df) # ignored cols cols = [ col for col in ref_df.columns if ref_df[col].dtype.name == 'object' ] # output attrs = { 'description': 'Residuals of numerical columns only', 'columns_ignored': numpy.array(cols, VLEN_STRING), 'equivalent': equal } base_dname = pbasename(pathname) group_name = ref_fid[pathname].parent.name.strip('/') dname = ppjoin('RESULTS', class_name, 'RESIDUALS', group_name, base_dname) write_dataframe(df, dname, out_fid, compression, attrs=attrs, filter_opts=filter_opts) if save_inputs: # copy the reference data out_grp = out_fid.require_group(ppjoin('REFERENCE-DATA', group_name)) ref_fid.copy(ref_fid[pathname], out_grp) # copy the test data out_grp = out_fid.require_group(ppjoin('TEST-DATA', group_name)) test_fid.copy(test_fid[pathname], out_grp)
def card4l(level1, granule, workflow, vertices, method, pixel_quality, landsea, tle_path, aerosol, brdf, ozone_path, water_vapour, dem_path, dsm_fname, invariant_fname, modtran_exe, out_fname, ecmwf_path=None, rori=0.52, buffer_distance=8000, compression=H5CompressionFilter.LZF, filter_opts=None, h5_driver=None, acq_parser_hint=None, normalized_solar_zenith=45.): """ CEOS Analysis Ready Data for Land. A workflow for producing standardised products that meet the CARD4L specification. :param level1: A string containing the full file pathname to the level1 dataset. :param granule: A string containing the granule id to process. :param workflow: An enum from wagl.constants.Workflow representing which workflow workflow to run. :param vertices: An integer 2-tuple indicating the number of rows and columns of sample-locations ("coordinator") to produce. The vertex columns should be an odd number. :param method: An enum from wagl.constants.Method representing the interpolation method to use during the interpolation of the atmospheric coefficients. :param pixel_quality: A bool indicating whether or not to run pixel quality. :param landsea: A string containing the full file pathname to the directory containing the land/sea mask datasets. :param tle_path: A string containing the full file pathname to the directory containing the two line element datasets. :param aerosol: A string containing the full file pathname to the HDF5 file containing the aerosol data. :param brdf: A dict containing either user-supplied BRDF values, or the full file pathname to the directory containing the BRDF data and the decadal averaged BRDF data used for acquisitions prior to TERRA/AQUA satellite operations. :param ozone_path: A string containing the full file pathname to the directory containing the ozone datasets. :param water_vapour: A string containing the full file pathname to the directory containing the water vapour datasets. :param dem_path: A string containing the full file pathname to the directory containing the reduced resolution DEM. :param dsm_path: A string containing the full file pathname to the directory containing the Digital Surface Workflow for use in terrain illumination correction. :param invariant_fname: A string containing the full file pathname to the image file containing the invariant geo-potential data for use within the SBT process. :param modtran_exe: A string containing the full file pathname to the MODTRAN executable. :param out_fname: A string containing the full file pathname that will contain the output data from the data standardisation process. executable. :param ecmwf_path: A string containing the full file pathname to the directory containing the data from the European Centre for Medium Weather Forcast, for use within the SBT process. :param rori: A floating point value for surface reflectance adjustment. TODO Fuqin to add additional documentation for this parameter. Default is 0.52. :param buffer_distance: A number representing the desired distance (in the same units as the acquisition) in which to calculate the extra number of pixels required to buffer an image. Default is 8000, which for an acquisition using metres would equate to 8000 metres. :param compression: An enum from hdf5.compression.H5CompressionFilter representing the desired compression filter to use for writing H5 IMAGE and TABLE class datasets to disk. Default is H5CompressionFilter.LZF. :param filter_opts: A dict containing any additional keyword arguments when generating the configuration for the given compression Filter. Default is None. :param h5_driver: The specific HDF5 file driver to use when creating the output HDF5 file. See http://docs.h5py.org/en/latest/high/file.html#file-drivers for more details. Default is None; which writes direct to disk using the appropriate driver for the underlying OS. :param acq_parser_hint: A string containing any hints to provide the acquisitions loader with. :param normalized_solar_zenith: Solar zenith angle to normalize for (in degrees). Default is 45 degrees. """ json_fmt = pjoin(POINT_FMT, ALBEDO_FMT, ''.join([POINT_ALBEDO_FMT, '.json'])) nvertices = vertices[0] * vertices[1] container = acquisitions(level1, hint=acq_parser_hint) # TODO: pass through an acquisitions container rather than pathname with h5py.File(out_fname, 'w', driver=h5_driver) as fid: fid.attrs['level1_uri'] = level1 for grp_name in container.supported_groups: log = STATUS_LOGGER.bind(level1=container.label, granule=granule, granule_group=grp_name) # root group for a given granule and resolution group root = fid.create_group(ppjoin(granule, grp_name)) acqs = container.get_acquisitions(granule=granule, group=grp_name) # include the resolution as a group attribute root.attrs['resolution'] = acqs[0].resolution # longitude and latitude log.info('Latitude-Longitude') create_lon_lat_grids(acqs[0], root, compression, filter_opts) # satellite and solar angles log.info('Satellite-Solar-Angles') calculate_angles(acqs[0], root[GroupName.LON_LAT_GROUP.value], root, compression, filter_opts, tle_path) if workflow in (Workflow.STANDARD, Workflow.NBAR): # DEM log.info('DEM-retriveal') get_dsm(acqs[0], dsm_fname, buffer_distance, root, compression, filter_opts) # slope & aspect log.info('Slope-Aspect') slope_aspect_arrays(acqs[0], root[GroupName.ELEVATION_GROUP.value], buffer_distance, root, compression, filter_opts) # incident angles log.info('Incident-Angles') incident_angles(root[GroupName.SAT_SOL_GROUP.value], root[GroupName.SLP_ASP_GROUP.value], root, compression, filter_opts) # exiting angles log.info('Exiting-Angles') exiting_angles(root[GroupName.SAT_SOL_GROUP.value], root[GroupName.SLP_ASP_GROUP.value], root, compression, filter_opts) # relative azimuth slope log.info('Relative-Azimuth-Angles') incident_group_name = GroupName.INCIDENT_GROUP.value exiting_group_name = GroupName.EXITING_GROUP.value relative_azimuth_slope(root[incident_group_name], root[exiting_group_name], root, compression, filter_opts) # self shadow log.info('Self-Shadow') self_shadow(root[incident_group_name], root[exiting_group_name], root, compression, filter_opts) # cast shadow solar source direction log.info('Cast-Shadow-Solar-Direction') dsm_group_name = GroupName.ELEVATION_GROUP.value calculate_cast_shadow(acqs[0], root[dsm_group_name], root[GroupName.SAT_SOL_GROUP.value], buffer_distance, root, compression, filter_opts) # cast shadow satellite source direction log.info('Cast-Shadow-Satellite-Direction') calculate_cast_shadow(acqs[0], root[dsm_group_name], root[GroupName.SAT_SOL_GROUP.value], buffer_distance, root, compression, filter_opts, False) # combined shadow masks log.info('Combined-Shadow') combine_shadow_masks(root[GroupName.SHADOW_GROUP.value], root[GroupName.SHADOW_GROUP.value], root[GroupName.SHADOW_GROUP.value], root, compression, filter_opts) # nbar and sbt ancillary log = STATUS_LOGGER.bind(level1=container.label, granule=granule, granule_group=None) # granule root group root = fid[granule] # get the highest resolution group containing supported bands acqs, grp_name = container.get_highest_resolution(granule=granule) grn_con = container.get_granule(granule=granule, container=True) res_group = root[grp_name] log.info('Ancillary-Retrieval') nbar_paths = { 'aerosol_dict': aerosol, 'water_vapour_dict': water_vapour, 'ozone_path': ozone_path, 'dem_path': dem_path, 'brdf_dict': brdf } collect_ancillary(grn_con, res_group[GroupName.SAT_SOL_GROUP.value], nbar_paths, ecmwf_path, invariant_fname, vertices, root, compression, filter_opts) # atmospherics log.info('Atmospherics') ancillary_group = root[GroupName.ANCILLARY_GROUP.value] # satellite/solar angles and lon/lat for a resolution group sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value] lon_lat_grp = res_group[GroupName.LON_LAT_GROUP.value] # TODO: supported acqs in different groups pointing to different response funcs json_data, _ = format_json(acqs, ancillary_group, sat_sol_grp, lon_lat_grp, workflow, root) # atmospheric inputs group inputs_grp = root[GroupName.ATMOSPHERIC_INPUTS_GRP.value] # radiative transfer for each point and albedo for key in json_data: point, albedo = key log.info('Radiative-Transfer', point=point, albedo=albedo.value) with tempfile.TemporaryDirectory() as tmpdir: prepare_modtran(acqs, point, [albedo], tmpdir) point_dir = pjoin(tmpdir, POINT_FMT.format(p=point)) workdir = pjoin(point_dir, ALBEDO_FMT.format(a=albedo.value)) json_mod_infile = pjoin( tmpdir, json_fmt.format(p=point, a=albedo.value)) with open(json_mod_infile, 'w') as src: json_dict = json_data[key] if albedo == Albedos.ALBEDO_TH: json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \ "%s/%s" % (workdir, json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"]) json_dict["MODTRAN"][1]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \ "%s/%s" % (workdir, json_dict["MODTRAN"][1]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"]) else: json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \ "%s/%s" % (workdir, json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"]) json.dump(json_dict, src, cls=JsonEncoder, indent=4) run_modtran(acqs, inputs_grp, workflow, nvertices, point, [albedo], modtran_exe, tmpdir, root, compression, filter_opts) # atmospheric coefficients log.info('Coefficients') results_group = root[GroupName.ATMOSPHERIC_RESULTS_GRP.value] calculate_coefficients(results_group, root, compression, filter_opts) esun_values = {} # interpolate coefficients for grp_name in container.supported_groups: log = STATUS_LOGGER.bind(level1=container.label, granule=granule, granule_group=grp_name) log.info('Interpolation') # acquisitions and available bands for the current group level acqs = container.get_acquisitions(granule=granule, group=grp_name) nbar_acqs = [ acq for acq in acqs if acq.band_type == BandType.REFLECTIVE ] sbt_acqs = [ acq for acq in acqs if acq.band_type == BandType.THERMAL ] res_group = root[grp_name] sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value] comp_grp = root[GroupName.COEFFICIENTS_GROUP.value] for coefficient in workflow.atmos_coefficients: if coefficient is AtmosphericCoefficients.ESUN: continue if coefficient in Workflow.NBAR.atmos_coefficients: band_acqs = nbar_acqs else: band_acqs = sbt_acqs for acq in band_acqs: log.info('Interpolate', band_id=acq.band_id, coefficient=coefficient.value) interpolate(acq, coefficient, ancillary_group, sat_sol_grp, comp_grp, res_group, compression, filter_opts, method) # standardised products band_acqs = [] if workflow in (Workflow.STANDARD, Workflow.NBAR): band_acqs.extend(nbar_acqs) if workflow in (Workflow.STANDARD, Workflow.SBT): band_acqs.extend(sbt_acqs) for acq in band_acqs: interp_grp = res_group[GroupName.INTERP_GROUP.value] if acq.band_type == BandType.THERMAL: log.info('SBT', band_id=acq.band_id) surface_brightness_temperature(acq, interp_grp, res_group, compression, filter_opts) else: atmos_coefs = read_h5_table( comp_grp, DatasetName.NBAR_COEFFICIENTS.value) esun_values[acq.band_name] = ( atmos_coefs[atmos_coefs.band_name == acq.band_name][ AtmosphericCoefficients.ESUN.value]).values[0] slp_asp_grp = res_group[GroupName.SLP_ASP_GROUP.value] rel_slp_asp = res_group[GroupName.REL_SLP_GROUP.value] incident_grp = res_group[GroupName.INCIDENT_GROUP.value] exiting_grp = res_group[GroupName.EXITING_GROUP.value] shadow_grp = res_group[GroupName.SHADOW_GROUP.value] log.info('Surface-Reflectance', band_id=acq.band_id) calculate_reflectance( acq, interp_grp, sat_sol_grp, slp_asp_grp, rel_slp_asp, incident_grp, exiting_grp, shadow_grp, ancillary_group, rori, res_group, compression, filter_opts, normalized_solar_zenith, esun_values[acq.band_name]) # pixel quality sbt_only = workflow == Workflow.SBT if pixel_quality and can_pq(level1, acq_parser_hint) and not sbt_only: run_pq(level1, res_group, landsea, res_group, compression, filter_opts, AP.NBAR, acq_parser_hint) run_pq(level1, res_group, landsea, res_group, compression, filter_opts, AP.NBART, acq_parser_hint) def get_band_acqs(grp_name): acqs = container.get_acquisitions(granule=granule, group=grp_name) nbar_acqs = [ acq for acq in acqs if acq.band_type == BandType.REFLECTIVE ] sbt_acqs = [ acq for acq in acqs if acq.band_type == BandType.THERMAL ] band_acqs = [] if workflow in (Workflow.STANDARD, Workflow.NBAR): band_acqs.extend(nbar_acqs) if workflow in (Workflow.STANDARD, Workflow.SBT): band_acqs.extend(sbt_acqs) return band_acqs # wagl parameters parameters = { 'vertices': list(vertices), 'method': method.value, 'rori': rori, 'buffer_distance': buffer_distance, 'normalized_solar_zenith': normalized_solar_zenith, 'esun': esun_values } # metadata yaml's metadata = root.create_group(DatasetName.METADATA.value) create_ard_yaml( { grp_name: get_band_acqs(grp_name) for grp_name in container.supported_groups }, ancillary_group, metadata, parameters, workflow)