コード例 #1
0
ファイル: metadata.py プロジェクト: truth-quark/wagl
    def load_sbt_ancillary(group):
        """
        Load the sbt ancillary data retrieved during the worlflow.
        """
        point_data = {
            DatasetName.DEWPOINT_TEMPERATURE.value: {},
            DatasetName.SURFACE_GEOPOTENTIAL.value: {},
            DatasetName.TEMPERATURE_2M.value: {},
            DatasetName.SURFACE_RELATIVE_HUMIDITY.value: {},
            DatasetName.GEOPOTENTIAL.value: {},
            DatasetName.RELATIVE_HUMIDITY.value: {},
            DatasetName.TEMPERATURE.value: {}
        }

        npoints = group[DatasetName.COORDINATOR.value].shape[0]
        for point in range(npoints):
            pnt_grp = group[POINT_FMT.format(p=point)]
            lonlat = tuple(pnt_grp.attrs['lonlat'])

            # scalars
            dname = DatasetName.DEWPOINT_TEMPERATURE.value
            point_data[dname][lonlat] = read_scalar(pnt_grp, dname)

            dname = DatasetName.SURFACE_GEOPOTENTIAL.value
            point_data[dname][lonlat] = read_scalar(pnt_grp, dname)

            dname = DatasetName.TEMPERATURE_2M.value
            point_data[dname][lonlat] = read_scalar(pnt_grp, dname)

            dname = DatasetName.SURFACE_RELATIVE_HUMIDITY.value
            point_data[dname][lonlat] = read_scalar(pnt_grp, dname)

            # tables
            dname = DatasetName.GEOPOTENTIAL.value
            dset = pnt_grp[dname]
            attrs = {k: v for k, v in dset.attrs.items()}
            df = read_h5_table(pnt_grp, dname)
            for column in df.columns:
                attrs[column] = df[column].values
            point_data[dname][lonlat] = attrs

            dname = DatasetName.RELATIVE_HUMIDITY.value
            dset = pnt_grp[dname]
            attrs = {k: v for k, v in dset.attrs.items()}
            df = read_h5_table(pnt_grp, dname)
            for column in df.columns:
                attrs[column] = df[column].values
            point_data[dname][lonlat] = attrs

            dname = DatasetName.TEMPERATURE.value
            dset = pnt_grp[dname]
            attrs = {k: v for k, v in dset.attrs.items()}
            df = read_h5_table(pnt_grp, dname)
            for column in df.columns:
                attrs[column] = df[column].values
            point_data[dname][lonlat] = attrs

        return point_data
コード例 #2
0
ファイル: multifile_workflow.py プロジェクト: sixy6e/wagl
    def run(self):
        container = acquisitions(self.level1, self.acq_parser_hint)
        acqs, group = container.get_highest_resolution(granule=self.granule)

        # output filename format
        json_fmt = pjoin(POINT_FMT, ALBEDO_FMT, "".join([POINT_ALBEDO_FMT, ".json"]))

        # input filenames
        ancillary_fname = self.input()["ancillary"].path
        sat_sol_fname = self.input()[group]["sat_sol"].path
        lon_lat_fname = self.input()[group]["lon_lat"].path

        with self.output().temporary_path() as out_fname:
            json_data = _format_json(
                acqs,
                sat_sol_fname,
                lon_lat_fname,
                ancillary_fname,
                out_fname,
                self.workflow,
            )

            # keep this as an indented block, that way the target will remain
            # atomic and be moved upon closing
            for key in json_data:
                point, albedo = key

                json_fname = json_fmt.format(p=point, a=albedo.value)

                target = pjoin(dirname(out_fname), self.base_dir, json_fname)

                workdir = pjoin(
                    dirname(out_fname),
                    self.base_dir,
                    POINT_FMT.format(p=point),
                    ALBEDO_FMT.format(a=albedo.value),
                )

                with luigi.LocalTarget(target).open("w") as src:

                    # Thermal processing has two input configurations
                    for modtran_input in json_data[key]["MODTRAN"]:
                        modtran_input["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = pjoin(
                            workdir, modtran_input["MODTRANINPUT"]["SPECTRAL"]["FILTNM"]
                        )

                    json.dump(json_data[key], src, cls=JsonEncoder, indent=4)
コード例 #3
0
def prepare_modtran(acquisitions, coordinate, albedos, basedir, modtran_exe):
    """
    Prepares the working directory for a MODTRAN execution.
    """
    data_dir = pjoin(dirname(modtran_exe), 'DATA')
    if not exists(data_dir):
        raise OSError('Cannot find MODTRAN')

    point_dir = pjoin(basedir, POINT_FMT.format(p=coordinate))
    for albedo in albedos:
        if albedo == Albedos.ALBEDO_TH:
            band_type = BandType.THERMAL
        else:
            band_type = BandType.REFLECTIVE

        acq = [acq for acq in acquisitions if acq.band_type == band_type][0]

        modtran_work = pjoin(point_dir, ALBEDO_FMT.format(a=albedo.value))

        if not exists(modtran_work):
            os.makedirs(modtran_work)

        out_fname = pjoin(modtran_work, 'mod5root.in')
        with open(out_fname, 'w') as src:
            src.write(
                POINT_ALBEDO_FMT.format(p=coordinate, a=albedo.value) + '\n')

        symlink_dir = pjoin(modtran_work, 'DATA')
        if exists(symlink_dir):
            os.unlink(symlink_dir)

        os.symlink(data_dir, symlink_dir)

        out_fname = pjoin(modtran_work, acq.spectral_filter_file)
        response = acq.spectral_response(as_list=True)
        with open(out_fname, 'wb') as src:
            src.writelines(response)
コード例 #4
0
ファイル: modtran.py プロジェクト: truth-quark/wagl
def prepare_modtran(acquisitions, coordinate, albedos, basedir):
    """
    Prepares the working directory for a MODTRAN execution.
    """

    point_dir = pjoin(basedir, POINT_FMT.format(p=coordinate))

    for albedo in albedos:
        if albedo == Albedos.ALBEDO_TH:
            band_type = BandType.THERMAL
        else:
            band_type = BandType.REFLECTIVE

        acq = [acq for acq in acquisitions if acq.band_type == band_type][0]

        modtran_work = pjoin(point_dir, ALBEDO_FMT.format(a=albedo.value))

        if not exists(modtran_work):
            os.makedirs(modtran_work)

        out_fname = pjoin(modtran_work, acq.spectral_filter_name)

        # Copy the spectral response filter file to the modtran workdir
        shutil.copy(acq.spectral_filter_filepath, out_fname)
コード例 #5
0
ファイル: test_modtran.py プロジェクト: truth-quark/wagl
    def test_modtran_run(self):
        """
        Tests that the interface to modtran (run_modtran)
        works for known inputs.
        Used to validate environment configuration/setup
        """

        band_names = [
            'BAND-1', 'BAND-2', 'BAND-3', 'BAND-4', 'BAND-5', 'BAND-6',
            'BAND-7', 'BAND-8'
        ]
        point = 0
        albedo = Albedos.ALBEDO_0

        # setup mock acquistions object
        acquisitions = []
        for bandn in band_names:
            acq = mock.MagicMock()
            acq.acquisition_datetime = datetime(2001, 1, 1)
            acq.band_type = BandType.REFLECTIVE
            acq.spectral_response = mock_spectral_response
            acquisitions.append(acq)

        # setup mock atmospherics group
        attrs = {'lonlat': 'TEST'}
        atmospherics = mock.MagicMock()
        atmospherics.attrs = attrs
        atmospherics_group = {POINT_FMT.format(p=point): atmospherics}

        # Compute base path -- prefix for hdf5 file
        base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value,
                           POINT_FMT.format(p=point))

        with tempfile.TemporaryDirectory() as workdir:
            run_dir = pjoin(workdir, POINT_FMT.format(p=point),
                            ALBEDO_FMT.format(a=albedo.value))
            os.makedirs(run_dir)

            # TODO replace json_input copy with json input generation
            with open(INPUT_JSON, 'r') as fd:
                json_data = json.load(fd)
                for mod_input in json_data['MODTRAN']:
                    mod_input['MODTRANINPUT']['SPECTRAL'][
                        'FILTNM'] = SPECTRAL_RESPONSE_LS8

            with open(
                    pjoin(run_dir,
                          POINT_ALBEDO_FMT.format(p=point, a=albedo.value)) +
                    ".json", 'w') as fd:
                json.dump(json_data, fd)

            fid = run_modtran(
                acquisitions,
                atmospherics_group,
                Workflow.STANDARD,
                npoints=12,  # number of track points
                point=point,
                albedos=[albedo],
                modtran_exe=MODTRAN_EXE,
                basedir=workdir,
                out_group=None)
            assert fid

            # Test base attrs
            assert fid[base_path].attrs['lonlat'] == 'TEST'
            assert fid[base_path].attrs['datetime'] == datetime(2001, 1,
                                                                1).isoformat()
            # test albedo headers?
            # Summarise modtran results to surface reflectance coefficients
            test_grp = fid[base_path][ALBEDO_FMT.format(a=albedo.value)]
            nbar_coefficients, _ = coefficients(
                read_h5_table(
                    fid,
                    pjoin(base_path, ALBEDO_FMT.format(a=albedo.value),
                          DatasetName.CHANNEL.value)),
                read_h5_table(
                    fid,
                    pjoin(base_path, ALBEDO_FMT.format(a=albedo.value),
                          DatasetName.SOLAR_ZENITH_CHANNEL.value)))

            expected = pd.read_csv(EXPECTED_CSV, index_col='band_name')
            pd.testing.assert_frame_equal(nbar_coefficients,
                                          expected,
                                          check_less_precise=True)
コード例 #6
0
ファイル: modtran.py プロジェクト: truth-quark/wagl
def format_json(acquisitions, ancillary_group, satellite_solar_group,
                lon_lat_group, workflow, out_group):
    """
    Creates json files for the albedo (0) and thermal
    """
    # angles data
    sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value]
    sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value]
    longitude = lon_lat_group[DatasetName.LON.value]
    latitude = lon_lat_group[DatasetName.LAT.value]

    # retrieve the averaged ancillary if available
    anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value)
    if anc_grp is None:
        anc_grp = ancillary_group

    # ancillary data
    coordinator = ancillary_group[DatasetName.COORDINATOR.value]
    aerosol = anc_grp[DatasetName.AEROSOL.value][()]
    water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()]
    ozone = anc_grp[DatasetName.OZONE.value][()]
    elevation = anc_grp[DatasetName.ELEVATION.value][()]

    npoints = coordinator.shape[0]
    view = numpy.zeros(npoints, dtype='float32')
    azi = numpy.zeros(npoints, dtype='float32')
    lat = numpy.zeros(npoints, dtype='float64')
    lon = numpy.zeros(npoints, dtype='float64')

    for i in range(npoints):
        yidx = coordinator['row_index'][i]
        xidx = coordinator['col_index'][i]
        view[i] = sat_view[yidx, xidx]
        azi[i] = sat_azi[yidx, xidx]
        lat[i] = latitude[yidx, xidx]
        lon[i] = longitude[yidx, xidx]

    view_corrected = 180 - view
    azi_corrected = azi + 180
    rlon = 360 - lon

    # check if in western hemisphere
    idx = rlon >= 360
    rlon[idx] -= 360

    idx = (180 - view_corrected) < 0.1
    view_corrected[idx] = 180
    azi_corrected[idx] = 0

    idx = azi_corrected > 360
    azi_corrected[idx] -= 360

    # get the modtran profiles to use based on the centre latitude
    _, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat

    if out_group is None:
        out_group = h5py.File('atmospheric-inputs.h5', 'w')

    if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group:
        out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value)

    group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value]
    iso_time = acquisitions[0].acquisition_datetime.isoformat()
    group.attrs['acquisition-datetime'] = iso_time

    json_data = {}
    # setup the json files required by MODTRAN
    if workflow in (Workflow.STANDARD, Workflow.NBAR):
        acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE]

        for p in range(npoints):

            for alb in Workflow.NBAR.albedos:

                input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a=str(alb.value)),
                              'water': water_vapour,
                              'ozone': ozone,
                              'doy': acquisitions[0].julian_day(),
                              'visibility': -aerosol,
                              'lat': lat[p],
                              'lon': rlon[p],
                              'time': acquisitions[0].decimal_hour(),
                              'sat_azimuth': azi_corrected[p],
                              'sat_height': acquisitions[0].altitude / 1000.0,
                              'elevation': elevation,
                              'sat_view': view_corrected[p],
                              'albedo': float(alb.value),
                              'filter_function': acqs[0].spectral_filter_name,
                              'binary': False
                              }

                if centre_lat < -23.0:
                    data = mpjson.midlat_summer_albedo(**input_data)
                else:
                    data = mpjson.tropical_albedo(**input_data)

                input_data['description'] = 'Input file for MODTRAN'
                input_data['file_format'] = 'json'
                input_data.pop('binary')

                json_data[(p, alb)] = data

                data = json.dumps(data, cls=JsonEncoder, indent=4)
                dname = ppjoin(POINT_FMT.format(p=p),
                               ALBEDO_FMT.format(a=alb.value),
                               DatasetName.MODTRAN_INPUT.value)

                write_scalar(data, dname, group, input_data)

    # create json for sbt if it has been collected
    if ancillary_group.attrs.get('sbt-ancillary'):
        dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value)
        acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL]

        for p in range(npoints):

            atmos_profile = read_h5_table(ancillary_group, dname.format(p=p))

            n_layers = atmos_profile.shape[0] + 6
            elevation = atmos_profile.iloc[0]['GeoPotential_Height']

            input_data = {'name': POINT_ALBEDO_FMT.format(p=p, a='TH'),
                          'ozone': ozone,
                          'n': n_layers,
                          'prof_alt': list(atmos_profile['GeoPotential_Height']),
                          'prof_pres': list(atmos_profile['Pressure']),
                          'prof_temp': list(atmos_profile['Temperature']),
                          'prof_water': list(atmos_profile['Relative_Humidity']),
                          'visibility': -aerosol,
                          'sat_height': acquisitions[0].altitude / 1000.0,
                          'gpheight': elevation,
                          'sat_view': view_corrected[p],
                          'filter_function': acqs[0].spectral_filter_name,
                          'binary': False
                          }

            data = mpjson.thermal_transmittance(**input_data)

            input_data['description'] = 'Input File for MODTRAN'
            input_data['file_format'] = 'json'
            input_data.pop('binary')

            json_data[(p, Albedos.ALBEDO_TH)] = data

            data = json.dumps(data, cls=JsonEncoder, indent=4)
            out_dname = ppjoin(POINT_FMT.format(p=p),
                               ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value),
                               DatasetName.MODTRAN_INPUT.value)
            write_scalar(data, out_dname, group, input_data)

    # attach location info to each point Group
    for p in range(npoints):
        lonlat = (coordinator['longitude'][p], coordinator['latitude'][p])
        group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat

    return json_data, out_group
コード例 #7
0
ファイル: modtran.py プロジェクト: truth-quark/wagl
def calculate_coefficients(atmospheric_results_group, out_group,
                           compression=H5CompressionFilter.LZF,
                           filter_opts=None):
    """
    Calculate the atmospheric coefficients from the MODTRAN output
    and used in the BRDF and atmospheric correction.
    Coefficients are computed for each band for each each coordinate
    for each atmospheric coefficient. The atmospheric coefficients can be
    found in `Workflow.STANDARD.atmos_coefficients`.

    :param atmospheric_results_group:
        The root HDF5 `Group` that contains the atmospheric results
        from each MODTRAN run.

    :param out_group:
        If set to None (default) then the results will be returned
        as an in-memory hdf5 file, i.e. the `core` driver. Otherwise,
        a writeable HDF5 `Group` object.

        The datasets will be formatted to the HDF5 TABLE specification
        and the dataset names will be as follows:

        * DatasetName.NBAR_COEFFICIENTS (if Workflow.STANDARD or Workflow.NBAR)
        * DatasetName.SBT_COEFFICIENTS (if Workflow.STANDARD or Workflow.SBT)

    :param compression:
        The compression filter to use.
        Default is H5CompressionFilter.LZF

    :param filter_opts:
        A dict of key value pairs available to the given configuration
        instance of H5CompressionFilter. For example
        H5CompressionFilter.LZF has the keywords *chunks* and *shuffle*
        available.
        Default is None, which will use the default settings for the
        chosen H5CompressionFilter instance.

    :return:
        An opened `h5py.File` object, that is either in-memory using the
        `core` driver, or on disk.
    """
    nbar_coefficients = pd.DataFrame()
    sbt_coefficients = pd.DataFrame()

    channel_data = channel_solar_angle = upward = downward = None

    # Initialise the output group/file
    if out_group is None:
        fid = h5py.File('atmospheric-coefficients.h5', driver='core',
                        backing_store=False)
    else:
        fid = out_group

    res = atmospheric_results_group
    npoints = res.attrs['npoints']
    nbar_atmos = res.attrs['nbar_atmospherics']
    sbt_atmos = res.attrs['sbt_atmospherics']

    for point in range(npoints):
        point_grp = res[POINT_FMT.format(p=point)]
        lonlat = point_grp.attrs['lonlat']
        timestamp = pd.to_datetime(point_grp.attrs['datetime'])
        grp_path = ppjoin(POINT_FMT.format(p=point), ALBEDO_FMT)

        if nbar_atmos:
            channel_path = ppjoin(grp_path.format(a=Albedos.ALBEDO_0.value),
                                  DatasetName.CHANNEL.value)
            channel_data = read_h5_table(res, channel_path)

            channel_solar_angle_path = ppjoin(
                grp_path.format(a=Albedos.ALBEDO_0.value),
                DatasetName.SOLAR_ZENITH_CHANNEL.value
            )

            channel_solar_angle = read_h5_table(res, channel_solar_angle_path)

        if sbt_atmos:
            dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value),
                           DatasetName.UPWARD_RADIATION_CHANNEL.value)
            upward = read_h5_table(res, dname)

            dname = ppjoin(grp_path.format(a=Albedos.ALBEDO_TH.value),
                           DatasetName.DOWNWARD_RADIATION_CHANNEL.value)
            downward = read_h5_table(res, dname)

        kwargs = {'channel_data': channel_data,
                  'solar_zenith_angle': channel_solar_angle,
                  'upward_radiation': upward,
                  'downward_radiation': downward}

        result = coefficients(**kwargs)

        # insert some datetime/geospatial fields
        if result[0] is not None:
            result[0].insert(0, 'POINT', point)
            result[0].insert(1, 'LONGITUDE', lonlat[0])
            result[0].insert(2, 'LATITUDE', lonlat[1])
            result[0].insert(3, 'DATETIME', timestamp)
            nbar_coefficients = nbar_coefficients.append(result[0])

        if result[1] is not None:
            result[1].insert(0, 'POINT', point)
            result[1].insert(1, 'LONGITUDE', lonlat[0])
            result[1].insert(2, 'LATITUDE', lonlat[1])
            result[1].insert(3, 'DATETIME', pd.to_datetime(timestamp))
            sbt_coefficients = sbt_coefficients.append(result[1])

    nbar_coefficients.reset_index(inplace=True)
    sbt_coefficients.reset_index(inplace=True)

    attrs = {'npoints': npoints}
    description = "Coefficients derived from the VNIR solar irradiation."
    attrs['description'] = description
    dname = DatasetName.NBAR_COEFFICIENTS.value

    if GroupName.COEFFICIENTS_GROUP.value not in fid:
        fid.create_group(GroupName.COEFFICIENTS_GROUP.value)

    group = fid[GroupName.COEFFICIENTS_GROUP.value]

    if nbar_atmos:
        write_dataframe(nbar_coefficients, dname, group, compression,
                        attrs=attrs, filter_opts=filter_opts)

    description = "Coefficients derived from the THERMAL solar irradiation."
    attrs['description'] = description
    dname = DatasetName.SBT_COEFFICIENTS.value

    if sbt_atmos:
        write_dataframe(sbt_coefficients, dname, group, compression,
                        attrs=attrs, filter_opts=filter_opts)

    if out_group is None:
        return fid
コード例 #8
0
ファイル: modtran.py プロジェクト: truth-quark/wagl
def run_modtran(acquisitions, atmospherics_group, workflow, npoints, point,
                albedos, modtran_exe, basedir, out_group,
                compression=H5CompressionFilter.LZF, filter_opts=None):
    """
    Run MODTRAN and channel results.
    """
    lonlat = atmospherics_group[POINT_FMT.format(p=point)].attrs['lonlat']

    # determine the output group/file
    if out_group is None:
        fid = h5py.File('atmospheric-results.h5', driver='core',
                        backing_store=False)
    else:
        fid = out_group

    # initial attributes
    base_attrs = {'Point': point,
                  'lonlat': lonlat,
                  'datetime': acquisitions[0].acquisition_datetime}

    base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value,
                       POINT_FMT.format(p=point))

    # what atmospheric calculations have been run and how many points
    group_name = GroupName.ATMOSPHERIC_RESULTS_GRP.value
    if group_name not in fid:
        fid.create_group(group_name)

    fid[group_name].attrs['npoints'] = npoints
    applied = workflow in (Workflow.STANDARD, Workflow.NBAR)
    fid[group_name].attrs['nbar_atmospherics'] = applied
    applied = workflow in (Workflow.STANDARD, Workflow.SBT)
    fid[group_name].attrs['sbt_atmospherics'] = applied

    acqs = acquisitions
    for albedo in albedos:
        base_attrs['Albedo'] = albedo.value
        workpath = pjoin(basedir, POINT_FMT.format(p=point),
                         ALBEDO_FMT.format(a=albedo.value))

        json_mod_infile = pjoin(workpath, ''.join(
            [POINT_ALBEDO_FMT.format(p=point, a=albedo.value), '.json']))

        group_path = ppjoin(base_path, ALBEDO_FMT.format(a=albedo.value))

        subprocess.check_call([modtran_exe, json_mod_infile], cwd=workpath)

        chn_fname = glob.glob(pjoin(workpath, '*.chn'))[0]
        tp6_fname = glob.glob(pjoin(workpath, '*.tp6'))[0]

        if albedo == Albedos.ALBEDO_TH:
            acq = [acq for acq in acqs if acq.band_type == BandType.THERMAL][0]

            channel_data = read_modtran_channel(chn_fname, tp6_fname, acq, albedo)

            attrs = base_attrs.copy()
            dataset_name = DatasetName.UPWARD_RADIATION_CHANNEL.value
            attrs['description'] = ('Upward radiation channel output from '
                                    'MODTRAN')
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data[0], dset_name, fid, compression,
                            attrs=attrs, filter_opts=filter_opts)

            # downward radiation
            attrs = base_attrs.copy()
            dataset_name = DatasetName.DOWNWARD_RADIATION_CHANNEL.value
            attrs['description'] = ('Downward radiation channel output from '
                                    'MODTRAN')
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data[1], dset_name, fid, compression,
                            attrs=attrs, filter_opts=filter_opts)
        else:
            acq = [acq for acq in acqs if
                   acq.band_type == BandType.REFLECTIVE][0]

            # Will require updating to handle JSON output from modtran
            channel_data = read_modtran_channel(chn_fname, tp6_fname, acq, albedo)

            attrs = base_attrs.copy()
            dataset_name = DatasetName.CHANNEL.value
            attrs['description'] = 'Channel output from MODTRAN'
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data[0], dset_name, fid, compression,
                            attrs=attrs, filter_opts=filter_opts)

            # solar zenith angle at surface
            attrs = base_attrs.copy()
            dataset_name = DatasetName.SOLAR_ZENITH_CHANNEL.value
            attrs['description'] = 'Solar zenith angle at different atmosphere levels'
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data[1], dset_name, fid, compression,
                            attrs=attrs, filter_opts=filter_opts)

    # metadata for a given point
    alb_vals = [alb.value for alb in workflow.albedos]
    fid[base_path].attrs['lonlat'] = lonlat
    fid[base_path].attrs['datetime'] = acqs[0].acquisition_datetime.isoformat()
    fid[base_path].attrs.create('albedos', data=alb_vals, dtype=VLEN_STRING)

    if out_group is None:
        return fid
コード例 #9
0
def format_tp5(acquisitions, ancillary_group, satellite_solar_group,
               lon_lat_group, workflow, out_group):
    """
    Creates str formatted tp5 files for the albedo (0, 1) and
    transmittance (t).
    """
    # angles data
    sat_view = satellite_solar_group[DatasetName.SATELLITE_VIEW.value]
    sat_azi = satellite_solar_group[DatasetName.SATELLITE_AZIMUTH.value]
    longitude = lon_lat_group[DatasetName.LON.value]
    latitude = lon_lat_group[DatasetName.LAT.value]

    # retrieve the averaged ancillary if available
    anc_grp = ancillary_group.get(GroupName.ANCILLARY_AVG_GROUP.value)
    if anc_grp is None:
        anc_grp = ancillary_group

    # ancillary data
    coordinator = ancillary_group[DatasetName.COORDINATOR.value]
    aerosol = anc_grp[DatasetName.AEROSOL.value][()]
    water_vapour = anc_grp[DatasetName.WATER_VAPOUR.value][()]
    ozone = anc_grp[DatasetName.OZONE.value][()]
    elevation = anc_grp[DatasetName.ELEVATION.value][()]

    npoints = coordinator.shape[0]
    view = numpy.zeros(npoints, dtype='float32')
    azi = numpy.zeros(npoints, dtype='float32')
    lat = numpy.zeros(npoints, dtype='float64')
    lon = numpy.zeros(npoints, dtype='float64')

    for i in range(npoints):
        yidx = coordinator['row_index'][i]
        xidx = coordinator['col_index'][i]
        view[i] = sat_view[yidx, xidx]
        azi[i] = sat_azi[yidx, xidx]
        lat[i] = latitude[yidx, xidx]
        lon[i] = longitude[yidx, xidx]

    view_corrected = 180 - view
    azi_corrected = azi + 180
    rlon = 360 - lon

    # check if in western hemisphere
    idx = rlon >= 360
    rlon[idx] -= 360

    idx = (180 - view_corrected) < 0.1
    view_corrected[idx] = 180
    azi_corrected[idx] = 0

    idx = azi_corrected > 360
    azi_corrected[idx] -= 360

    # get the modtran profiles to use based on the centre latitude
    _, centre_lat = acquisitions[0].gridded_geo_box().centre_lonlat
    if centre_lat < -23.0:
        albedo_profile = MIDLAT_SUMMER_ALBEDO
        trans_profile = MIDLAT_SUMMER_TRANSMITTANCE
    else:
        albedo_profile = TROPICAL_ALBEDO
        trans_profile = TROPICAL_TRANSMITTANCE

    if out_group is None:
        out_group = h5py.File('atmospheric-inputs.h5', 'w')

    if GroupName.ATMOSPHERIC_INPUTS_GRP.value not in out_group:
        out_group.create_group(GroupName.ATMOSPHERIC_INPUTS_GRP.value)

    group = out_group[GroupName.ATMOSPHERIC_INPUTS_GRP.value]
    iso_time = acquisitions[0].acquisition_datetime.isoformat()
    group.attrs['acquisition-datetime'] = iso_time

    tp5_data = {}

    # setup the tp5 files required by MODTRAN
    if workflow == Workflow.STANDARD or workflow == Workflow.NBAR:
        acqs = [a for a in acquisitions if a.band_type == BandType.REFLECTIVE]

        for p in range(npoints):
            for alb in Workflow.NBAR.albedos:
                input_data = {
                    'water': water_vapour,
                    'ozone': ozone,
                    'filter_function': acqs[0].spectral_filter_file,
                    'visibility': -aerosol,
                    'elevation': elevation,
                    'sat_height': acquisitions[0].altitude / 1000.0,
                    'sat_view': view_corrected[p],
                    'doy': acquisitions[0].julian_day(),
                    'binary': 'T'
                }
                if alb == Albedos.ALBEDO_T:
                    input_data['albedo'] = 0.0
                    input_data['sat_view_offset'] = 180.0 - view_corrected[p]
                    data = trans_profile.format(**input_data)
                else:
                    input_data['albedo'] = float(alb.value)
                    input_data['lat'] = lat[p]
                    input_data['lon'] = rlon[p]
                    input_data['time'] = acquisitions[0].decimal_hour()
                    input_data['sat_azimuth'] = azi_corrected[p]
                    data = albedo_profile.format(**input_data)

                tp5_data[(p, alb)] = data

                dname = ppjoin(POINT_FMT.format(p=p),
                               ALBEDO_FMT.format(a=alb.value),
                               DatasetName.TP5.value)
                write_scalar(numpy.string_(data), dname, group, input_data)

    # create tp5 for sbt if it has been collected
    if ancillary_group.attrs.get('sbt-ancillary'):
        dname = ppjoin(POINT_FMT, DatasetName.ATMOSPHERIC_PROFILE.value)
        acqs = [a for a in acquisitions if a.band_type == BandType.THERMAL]

        for p in range(npoints):
            atmospheric_profile = []
            atmos_profile = read_h5_table(ancillary_group, dname.format(p=p))
            n_layers = atmos_profile.shape[0] + 6
            elevation = atmos_profile.iloc[0]['GeoPotential_Height']

            for i, row in atmos_profile.iterrows():
                input_data = {
                    'gpheight': row['GeoPotential_Height'],
                    'pressure': row['Pressure'],
                    'airtemp': row['Temperature'],
                    'humidity': row['Relative_Humidity'],
                    'zero': 0.0
                }
                atmospheric_profile.append(SBT_FORMAT.format(**input_data))

            input_data = {
                'ozone': ozone,
                'filter_function': acqs[0].spectral_filter_file,
                'visibility': -aerosol,
                'gpheight': elevation,
                'n': n_layers,
                'sat_height': acquisitions[0].altitude / 1000.0,
                'sat_view': view_corrected[p],
                'binary': 'T',
                'atmospheric_profile': ''.join(atmospheric_profile)
            }

            data = THERMAL_TRANSMITTANCE.format(**input_data)
            tp5_data[(p, Albedos.ALBEDO_TH)] = data
            out_dname = ppjoin(POINT_FMT.format(p=p),
                               ALBEDO_FMT.format(a=Albedos.ALBEDO_TH.value),
                               DatasetName.TP5.value)
            write_scalar(numpy.string_(data), out_dname, group, input_data)

    # attach location info to each point Group
    for p in range(npoints):
        lonlat = (coordinator['longitude'][p], coordinator['latitude'][p])
        group[POINT_FMT.format(p=p)].attrs['lonlat'] = lonlat

    return tp5_data, out_group
コード例 #10
0
def run_modtran(acquisitions,
                atmospherics_group,
                workflow,
                npoints,
                point,
                albedos,
                modtran_exe,
                basedir,
                out_group,
                compression=H5CompressionFilter.LZF,
                filter_opts=None):
    """
    Run MODTRAN and return the flux and channel results.
    """
    lonlat = atmospherics_group[POINT_FMT.format(p=point)].attrs['lonlat']

    # determine the output group/file
    if out_group is None:
        fid = h5py.File('atmospheric-results.h5',
                        driver='core',
                        backing_store=False)
    else:
        fid = out_group

    # initial attributes
    base_attrs = {
        'Point': point,
        'lonlat': lonlat,
        'datetime': acquisitions[0].acquisition_datetime
    }

    base_path = ppjoin(GroupName.ATMOSPHERIC_RESULTS_GRP.value,
                       POINT_FMT.format(p=point))

    # what atmospheric calculations have been run and how many points
    group_name = GroupName.ATMOSPHERIC_RESULTS_GRP.value
    if group_name not in fid:
        fid.create_group(group_name)

    fid[group_name].attrs['npoints'] = npoints
    applied = workflow == Workflow.STANDARD or workflow == Workflow.NBAR
    fid[group_name].attrs['nbar_atmospherics'] = applied
    applied = workflow == Workflow.STANDARD or workflow == Workflow.SBT
    fid[group_name].attrs['sbt_atmospherics'] = applied

    acqs = acquisitions
    for albedo in albedos:
        base_attrs['Albedo'] = albedo.value
        workpath = pjoin(basedir, POINT_FMT.format(p=point),
                         ALBEDO_FMT.format(a=albedo.value))
        group_path = ppjoin(base_path, ALBEDO_FMT.format(a=albedo.value))

        subprocess.check_call([modtran_exe], cwd=workpath)
        chn_fname = glob.glob(pjoin(workpath, '*.chn'))[0]

        if albedo == Albedos.ALBEDO_TH:
            acq = [acq for acq in acqs if acq.band_type == BandType.THERMAL][0]
            channel_data = read_modtran_channel(chn_fname, acq, albedo)

            # upward radiation
            attrs = base_attrs.copy()
            dataset_name = DatasetName.UPWARD_RADIATION_CHANNEL.value
            attrs['description'] = ('Upward radiation channel output from '
                                    'MODTRAN')
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data[0],
                            dset_name,
                            fid,
                            compression,
                            attrs=attrs,
                            filter_opts=filter_opts)

            # downward radiation
            attrs = base_attrs.copy()
            dataset_name = DatasetName.DOWNWARD_RADIATION_CHANNEL.value
            attrs['description'] = ('Downward radiation channel output from '
                                    'MODTRAN')
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data[1],
                            dset_name,
                            fid,
                            compression,
                            attrs=attrs,
                            filter_opts=filter_opts)
        else:
            acq = [
                acq for acq in acqs if acq.band_type == BandType.REFLECTIVE
            ][0]
            flux_fname = glob.glob(pjoin(workpath, '*_b.flx'))[0]
            flux_data, altitudes = read_modtran_flux(flux_fname)
            channel_data = read_modtran_channel(chn_fname, acq, albedo)

            # ouput the flux data
            attrs = base_attrs.copy()
            dset_name = ppjoin(group_path, DatasetName.FLUX.value)
            attrs['description'] = 'Flux output from MODTRAN'
            write_dataframe(flux_data,
                            dset_name,
                            fid,
                            compression,
                            attrs=attrs,
                            filter_opts=filter_opts)

            # output the altitude data
            attrs = base_attrs.copy()
            attrs['description'] = 'Altitudes output from MODTRAN'
            attrs['altitude_levels'] = altitudes.shape[0]
            attrs['units'] = 'km'
            dset_name = ppjoin(group_path, DatasetName.ALTITUDES.value)
            write_dataframe(altitudes,
                            dset_name,
                            fid,
                            compression,
                            attrs=attrs,
                            filter_opts=filter_opts)

            # accumulate the solar irradiance
            transmittance = True if albedo == Albedos.ALBEDO_T else False
            response = acq.spectral_response()
            accumulated = calculate_solar_radiation(flux_data, response,
                                                    altitudes.shape[0],
                                                    transmittance)

            attrs = base_attrs.copy()
            dset_name = ppjoin(group_path, DatasetName.SOLAR_IRRADIANCE.value)
            description = ("Accumulated solar irradiation for point {} "
                           "and albedo {}.")
            attrs['description'] = description.format(point, albedo.value)
            write_dataframe(accumulated,
                            dset_name,
                            fid,
                            compression,
                            attrs=attrs,
                            filter_opts=filter_opts)

            attrs = base_attrs.copy()
            dataset_name = DatasetName.CHANNEL.value
            attrs['description'] = 'Channel output from MODTRAN'
            dset_name = ppjoin(group_path, dataset_name)
            write_dataframe(channel_data,
                            dset_name,
                            fid,
                            compression,
                            attrs=attrs,
                            filter_opts=filter_opts)

    # metadata for a given point
    alb_vals = [alb.value for alb in workflow.albedos]
    fid[base_path].attrs['lonlat'] = lonlat
    fid[base_path].attrs['datetime'] = acqs[0].acquisition_datetime.isoformat()
    fid[base_path].attrs.create('albedos', data=alb_vals, dtype=VLEN_STRING)

    if out_group is None:
        return fid
コード例 #11
0
def collect_sbt_ancillary(
    acquisition,
    lonlats,
    ancillary_path,
    invariant_fname=None,
    out_group=None,
    compression=H5CompressionFilter.LZF,
    filter_opts=None,
):
    """
    Collects the ancillary data required for surface brightness
    temperature.

    :param acquisition:
        An instance of an `Acquisition` object.

    :param lonlats:
        A `list` of tuples containing (longitude, latitude) coordinates.

    :param ancillary_path:
        A `str` containing the directory pathname to the ECMWF
        ancillary data.

    :param invariant_fname:
        A `str` containing the file pathname to the invariant geopotential
        data.

    :param out_group:
        If set to None (default) then the results will be returned
        as an in-memory hdf5 file, i.e. the `core` driver. Otherwise,
        a writeable HDF5 `Group` object.

    :param compression:
        The compression filter to use.
        Default is H5CompressionFilter.LZF

    :filter_opts:
        A dict of key value pairs available to the given configuration
        instance of H5CompressionFilter. For example
        H5CompressionFilter.LZF has the keywords *chunks* and *shuffle*
        available.
        Default is None, which will use the default settings for the
        chosen H5CompressionFilter instance.

    :return:
        An opened `h5py.File` object, that is either in-memory using the
        `core` driver, or on disk.
    """
    # Initialise the output files
    if out_group is None:
        fid = h5py.File("sbt-ancillary.h5",
                        "w",
                        driver="core",
                        backing_store=False)
    else:
        fid = out_group

    fid.attrs["sbt-ancillary"] = True

    dt = acquisition.acquisition_datetime

    description = ("Combined Surface and Pressure Layer data retrieved from "
                   "the ECWMF catalogue.")
    attrs = {"description": description, "Date used for querying ECWMF": dt}

    for i, lonlat in enumerate(lonlats):
        pnt = POINT_FMT.format(p=i)
        # get data located at the surface
        dew = ecwmf_dewpoint_temperature(ancillary_path, lonlat, dt)
        t2m = ecwmf_temperature_2metre(ancillary_path, lonlat, dt)
        sfc_prs = ecwmf_surface_pressure(ancillary_path, lonlat, dt)
        sfc_hgt = ecwmf_elevation(invariant_fname, lonlat)
        sfc_rh = relative_humdity(t2m[0], dew[0])

        # output the scalar data along with the attrs
        dname = ppjoin(pnt, DatasetName.DEWPOINT_TEMPERATURE.value)
        write_scalar(dew[0], dname, fid, dew[1])

        dname = ppjoin(pnt, DatasetName.TEMPERATURE_2M.value)
        write_scalar(t2m[0], dname, fid, t2m[1])

        dname = ppjoin(pnt, DatasetName.SURFACE_PRESSURE.value)
        write_scalar(sfc_prs[0], dname, fid, sfc_prs[1])

        dname = ppjoin(pnt, DatasetName.SURFACE_GEOPOTENTIAL.value)
        write_scalar(sfc_hgt[0], dname, fid, sfc_hgt[1])

        dname = ppjoin(pnt, DatasetName.SURFACE_RELATIVE_HUMIDITY.value)
        attrs = {"description": "Relative Humidity calculated at the surface"}
        write_scalar(sfc_rh, dname, fid, attrs)

        # get the data from each of the pressure levels (1 -> 1000 ISBL)
        gph = ecwmf_geo_potential(ancillary_path, lonlat, dt)
        tmp = ecwmf_temperature(ancillary_path, lonlat, dt)
        rh = ecwmf_relative_humidity(ancillary_path, lonlat, dt)

        dname = ppjoin(pnt, DatasetName.GEOPOTENTIAL.value)
        write_dataframe(gph[0],
                        dname,
                        fid,
                        compression,
                        attrs=gph[1],
                        filter_opts=filter_opts)

        dname = ppjoin(pnt, DatasetName.TEMPERATURE.value)
        write_dataframe(tmp[0],
                        dname,
                        fid,
                        compression,
                        attrs=tmp[1],
                        filter_opts=filter_opts)

        dname = ppjoin(pnt, DatasetName.RELATIVE_HUMIDITY.value)
        write_dataframe(rh[0],
                        dname,
                        fid,
                        compression,
                        attrs=rh[1],
                        filter_opts=filter_opts)

        # combine the surface and higher pressure layers into a single array
        cols = [
            "GeoPotential_Height", "Pressure", "Temperature",
            "Relative_Humidity"
        ]
        layers = pandas.DataFrame(columns=cols,
                                  index=range(rh[0].shape[0]),
                                  dtype="float64")

        layers["GeoPotential_Height"] = gph[0]["GeoPotential_Height"].values
        layers["Pressure"] = ECWMF_LEVELS[::-1]
        layers["Temperature"] = tmp[0]["Temperature"].values
        layers["Relative_Humidity"] = rh[0]["Relative_Humidity"].values

        # define the surface level
        df = pandas.DataFrame(
            {
                "GeoPotential_Height": sfc_hgt[0],
                "Pressure": sfc_prs[0],
                "Temperature": kelvin_2_celcius(t2m[0]),
                "Relative_Humidity": sfc_rh,
            },
            index=[0],
        )

        # MODTRAN requires the height to be ascending
        # and the pressure to be descending
        wh = (layers["GeoPotential_Height"] >
              sfc_hgt[0]) & (layers["Pressure"] < sfc_prs[0].round())
        df = df.append(layers[wh])
        df.reset_index(drop=True, inplace=True)

        dname = ppjoin(pnt, DatasetName.ATMOSPHERIC_PROFILE.value)
        write_dataframe(df,
                        dname,
                        fid,
                        compression,
                        attrs=attrs,
                        filter_opts=filter_opts)

        fid[pnt].attrs["lonlat"] = lonlat

    if out_group is None:
        return fid
コード例 #12
0
ファイル: standardise.py プロジェクト: truth-quark/wagl
def card4l(level1,
           granule,
           workflow,
           vertices,
           method,
           pixel_quality,
           landsea,
           tle_path,
           aerosol,
           brdf,
           ozone_path,
           water_vapour,
           dem_path,
           dsm_fname,
           invariant_fname,
           modtran_exe,
           out_fname,
           ecmwf_path=None,
           rori=0.52,
           buffer_distance=8000,
           compression=H5CompressionFilter.LZF,
           filter_opts=None,
           h5_driver=None,
           acq_parser_hint=None,
           normalized_solar_zenith=45.):
    """
    CEOS Analysis Ready Data for Land.
    A workflow for producing standardised products that meet the
    CARD4L specification.

    :param level1:
        A string containing the full file pathname to the level1
        dataset.

    :param granule:
        A string containing the granule id to process.

    :param workflow:
        An enum from wagl.constants.Workflow representing which
        workflow workflow to run.

    :param vertices:
        An integer 2-tuple indicating the number of rows and columns
        of sample-locations ("coordinator") to produce.
        The vertex columns should be an odd number.

    :param method:
        An enum from wagl.constants.Method representing the
        interpolation method to use during the interpolation
        of the atmospheric coefficients.

    :param pixel_quality:
        A bool indicating whether or not to run pixel quality.

    :param landsea:
        A string containing the full file pathname to the directory
        containing the land/sea mask datasets.

    :param tle_path:
        A string containing the full file pathname to the directory
        containing the two line element datasets.

    :param aerosol:
        A string containing the full file pathname to the HDF5 file
        containing the aerosol data.

    :param brdf:
        A dict containing either user-supplied BRDF values, or the
        full file pathname to the directory containing the BRDF data
        and the decadal averaged BRDF data used for acquisitions
        prior to TERRA/AQUA satellite operations.

    :param ozone_path:
        A string containing the full file pathname to the directory
        containing the ozone datasets.

    :param water_vapour:
        A string containing the full file pathname to the directory
        containing the water vapour datasets.

    :param dem_path:
        A string containing the full file pathname to the directory
        containing the reduced resolution DEM.

    :param dsm_path:
        A string containing the full file pathname to the directory
        containing the Digital Surface Workflow for use in terrain
        illumination correction.

    :param invariant_fname:
        A string containing the full file pathname to the image file
        containing the invariant geo-potential data for use within
        the SBT process.

    :param modtran_exe:
        A string containing the full file pathname to the MODTRAN
        executable.

    :param out_fname:
        A string containing the full file pathname that will contain
        the output data from the data standardisation process.
        executable.

    :param ecmwf_path:
        A string containing the full file pathname to the directory
        containing the data from the European Centre for Medium Weather
        Forcast, for use within the SBT process.

    :param rori:
        A floating point value for surface reflectance adjustment.
        TODO Fuqin to add additional documentation for this parameter.
        Default is 0.52.

    :param buffer_distance:
        A number representing the desired distance (in the same
        units as the acquisition) in which to calculate the extra
        number of pixels required to buffer an image.
        Default is 8000, which for an acquisition using metres would
        equate to 8000 metres.

    :param compression:
        An enum from hdf5.compression.H5CompressionFilter representing
        the desired compression filter to use for writing H5 IMAGE and
        TABLE class datasets to disk.
        Default is H5CompressionFilter.LZF.

    :param filter_opts:
        A dict containing any additional keyword arguments when
        generating the configuration for the given compression Filter.
        Default is None.

    :param h5_driver:
        The specific HDF5 file driver to use when creating the output
        HDF5 file.
        See http://docs.h5py.org/en/latest/high/file.html#file-drivers
        for more details.
        Default is None; which writes direct to disk using the
        appropriate driver for the underlying OS.

    :param acq_parser_hint:
        A string containing any hints to provide the acquisitions
        loader with.

    :param normalized_solar_zenith:
        Solar zenith angle to normalize for (in degrees). Default is 45 degrees.
    """
    json_fmt = pjoin(POINT_FMT, ALBEDO_FMT,
                     ''.join([POINT_ALBEDO_FMT, '.json']))
    nvertices = vertices[0] * vertices[1]

    container = acquisitions(level1, hint=acq_parser_hint)

    # TODO: pass through an acquisitions container rather than pathname
    with h5py.File(out_fname, 'w', driver=h5_driver) as fid:
        fid.attrs['level1_uri'] = level1

        for grp_name in container.supported_groups:
            log = STATUS_LOGGER.bind(level1=container.label,
                                     granule=granule,
                                     granule_group=grp_name)

            # root group for a given granule and resolution group
            root = fid.create_group(ppjoin(granule, grp_name))
            acqs = container.get_acquisitions(granule=granule, group=grp_name)

            # include the resolution as a group attribute
            root.attrs['resolution'] = acqs[0].resolution

            # longitude and latitude
            log.info('Latitude-Longitude')
            create_lon_lat_grids(acqs[0], root, compression, filter_opts)

            # satellite and solar angles
            log.info('Satellite-Solar-Angles')
            calculate_angles(acqs[0], root[GroupName.LON_LAT_GROUP.value],
                             root, compression, filter_opts, tle_path)

            if workflow in (Workflow.STANDARD, Workflow.NBAR):

                # DEM
                log.info('DEM-retriveal')
                get_dsm(acqs[0], dsm_fname, buffer_distance, root, compression,
                        filter_opts)

                # slope & aspect
                log.info('Slope-Aspect')
                slope_aspect_arrays(acqs[0],
                                    root[GroupName.ELEVATION_GROUP.value],
                                    buffer_distance, root, compression,
                                    filter_opts)

                # incident angles
                log.info('Incident-Angles')
                incident_angles(root[GroupName.SAT_SOL_GROUP.value],
                                root[GroupName.SLP_ASP_GROUP.value], root,
                                compression, filter_opts)

                # exiting angles
                log.info('Exiting-Angles')
                exiting_angles(root[GroupName.SAT_SOL_GROUP.value],
                               root[GroupName.SLP_ASP_GROUP.value], root,
                               compression, filter_opts)

                # relative azimuth slope
                log.info('Relative-Azimuth-Angles')
                incident_group_name = GroupName.INCIDENT_GROUP.value
                exiting_group_name = GroupName.EXITING_GROUP.value
                relative_azimuth_slope(root[incident_group_name],
                                       root[exiting_group_name], root,
                                       compression, filter_opts)

                # self shadow
                log.info('Self-Shadow')
                self_shadow(root[incident_group_name],
                            root[exiting_group_name], root, compression,
                            filter_opts)

                # cast shadow solar source direction
                log.info('Cast-Shadow-Solar-Direction')
                dsm_group_name = GroupName.ELEVATION_GROUP.value
                calculate_cast_shadow(acqs[0], root[dsm_group_name],
                                      root[GroupName.SAT_SOL_GROUP.value],
                                      buffer_distance, root, compression,
                                      filter_opts)

                # cast shadow satellite source direction
                log.info('Cast-Shadow-Satellite-Direction')
                calculate_cast_shadow(acqs[0], root[dsm_group_name],
                                      root[GroupName.SAT_SOL_GROUP.value],
                                      buffer_distance, root, compression,
                                      filter_opts, False)

                # combined shadow masks
                log.info('Combined-Shadow')
                combine_shadow_masks(root[GroupName.SHADOW_GROUP.value],
                                     root[GroupName.SHADOW_GROUP.value],
                                     root[GroupName.SHADOW_GROUP.value], root,
                                     compression, filter_opts)

        # nbar and sbt ancillary
        log = STATUS_LOGGER.bind(level1=container.label,
                                 granule=granule,
                                 granule_group=None)

        # granule root group
        root = fid[granule]

        # get the highest resolution group containing supported bands
        acqs, grp_name = container.get_highest_resolution(granule=granule)

        grn_con = container.get_granule(granule=granule, container=True)
        res_group = root[grp_name]

        log.info('Ancillary-Retrieval')
        nbar_paths = {
            'aerosol_dict': aerosol,
            'water_vapour_dict': water_vapour,
            'ozone_path': ozone_path,
            'dem_path': dem_path,
            'brdf_dict': brdf
        }
        collect_ancillary(grn_con, res_group[GroupName.SAT_SOL_GROUP.value],
                          nbar_paths, ecmwf_path, invariant_fname, vertices,
                          root, compression, filter_opts)

        # atmospherics
        log.info('Atmospherics')

        ancillary_group = root[GroupName.ANCILLARY_GROUP.value]

        # satellite/solar angles and lon/lat for a resolution group
        sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value]
        lon_lat_grp = res_group[GroupName.LON_LAT_GROUP.value]

        # TODO: supported acqs in different groups pointing to different response funcs
        json_data, _ = format_json(acqs, ancillary_group, sat_sol_grp,
                                   lon_lat_grp, workflow, root)

        # atmospheric inputs group
        inputs_grp = root[GroupName.ATMOSPHERIC_INPUTS_GRP.value]

        # radiative transfer for each point and albedo
        for key in json_data:
            point, albedo = key

            log.info('Radiative-Transfer', point=point, albedo=albedo.value)

            with tempfile.TemporaryDirectory() as tmpdir:

                prepare_modtran(acqs, point, [albedo], tmpdir)

                point_dir = pjoin(tmpdir, POINT_FMT.format(p=point))
                workdir = pjoin(point_dir, ALBEDO_FMT.format(a=albedo.value))

                json_mod_infile = pjoin(
                    tmpdir, json_fmt.format(p=point, a=albedo.value))

                with open(json_mod_infile, 'w') as src:
                    json_dict = json_data[key]

                    if albedo == Albedos.ALBEDO_TH:

                        json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \
                            "%s/%s" % (workdir, json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"])
                        json_dict["MODTRAN"][1]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \
                            "%s/%s" % (workdir, json_dict["MODTRAN"][1]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"])

                    else:

                        json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"] = \
                            "%s/%s" % (workdir, json_dict["MODTRAN"][0]["MODTRANINPUT"]["SPECTRAL"]["FILTNM"])

                    json.dump(json_dict, src, cls=JsonEncoder, indent=4)

                run_modtran(acqs, inputs_grp, workflow, nvertices, point,
                            [albedo], modtran_exe, tmpdir, root, compression,
                            filter_opts)

        # atmospheric coefficients
        log.info('Coefficients')
        results_group = root[GroupName.ATMOSPHERIC_RESULTS_GRP.value]
        calculate_coefficients(results_group, root, compression, filter_opts)
        esun_values = {}
        # interpolate coefficients
        for grp_name in container.supported_groups:
            log = STATUS_LOGGER.bind(level1=container.label,
                                     granule=granule,
                                     granule_group=grp_name)
            log.info('Interpolation')

            # acquisitions and available bands for the current group level
            acqs = container.get_acquisitions(granule=granule, group=grp_name)
            nbar_acqs = [
                acq for acq in acqs if acq.band_type == BandType.REFLECTIVE
            ]
            sbt_acqs = [
                acq for acq in acqs if acq.band_type == BandType.THERMAL
            ]

            res_group = root[grp_name]
            sat_sol_grp = res_group[GroupName.SAT_SOL_GROUP.value]
            comp_grp = root[GroupName.COEFFICIENTS_GROUP.value]

            for coefficient in workflow.atmos_coefficients:
                if coefficient is AtmosphericCoefficients.ESUN:
                    continue
                if coefficient in Workflow.NBAR.atmos_coefficients:
                    band_acqs = nbar_acqs
                else:
                    band_acqs = sbt_acqs

                for acq in band_acqs:
                    log.info('Interpolate',
                             band_id=acq.band_id,
                             coefficient=coefficient.value)
                    interpolate(acq, coefficient, ancillary_group, sat_sol_grp,
                                comp_grp, res_group, compression, filter_opts,
                                method)

            # standardised products
            band_acqs = []

            if workflow in (Workflow.STANDARD, Workflow.NBAR):
                band_acqs.extend(nbar_acqs)

            if workflow in (Workflow.STANDARD, Workflow.SBT):
                band_acqs.extend(sbt_acqs)

            for acq in band_acqs:
                interp_grp = res_group[GroupName.INTERP_GROUP.value]

                if acq.band_type == BandType.THERMAL:
                    log.info('SBT', band_id=acq.band_id)
                    surface_brightness_temperature(acq, interp_grp, res_group,
                                                   compression, filter_opts)
                else:
                    atmos_coefs = read_h5_table(
                        comp_grp, DatasetName.NBAR_COEFFICIENTS.value)
                    esun_values[acq.band_name] = (
                        atmos_coefs[atmos_coefs.band_name == acq.band_name][
                            AtmosphericCoefficients.ESUN.value]).values[0]

                    slp_asp_grp = res_group[GroupName.SLP_ASP_GROUP.value]
                    rel_slp_asp = res_group[GroupName.REL_SLP_GROUP.value]
                    incident_grp = res_group[GroupName.INCIDENT_GROUP.value]
                    exiting_grp = res_group[GroupName.EXITING_GROUP.value]
                    shadow_grp = res_group[GroupName.SHADOW_GROUP.value]

                    log.info('Surface-Reflectance', band_id=acq.band_id)
                    calculate_reflectance(
                        acq, interp_grp, sat_sol_grp, slp_asp_grp, rel_slp_asp,
                        incident_grp, exiting_grp, shadow_grp, ancillary_group,
                        rori, res_group, compression, filter_opts,
                        normalized_solar_zenith, esun_values[acq.band_name])

            # pixel quality
            sbt_only = workflow == Workflow.SBT
            if pixel_quality and can_pq(level1,
                                        acq_parser_hint) and not sbt_only:
                run_pq(level1, res_group, landsea, res_group, compression,
                       filter_opts, AP.NBAR, acq_parser_hint)
                run_pq(level1, res_group, landsea, res_group, compression,
                       filter_opts, AP.NBART, acq_parser_hint)

        def get_band_acqs(grp_name):
            acqs = container.get_acquisitions(granule=granule, group=grp_name)
            nbar_acqs = [
                acq for acq in acqs if acq.band_type == BandType.REFLECTIVE
            ]
            sbt_acqs = [
                acq for acq in acqs if acq.band_type == BandType.THERMAL
            ]

            band_acqs = []
            if workflow in (Workflow.STANDARD, Workflow.NBAR):
                band_acqs.extend(nbar_acqs)

            if workflow in (Workflow.STANDARD, Workflow.SBT):
                band_acqs.extend(sbt_acqs)

            return band_acqs

        # wagl parameters
        parameters = {
            'vertices': list(vertices),
            'method': method.value,
            'rori': rori,
            'buffer_distance': buffer_distance,
            'normalized_solar_zenith': normalized_solar_zenith,
            'esun': esun_values
        }

        # metadata yaml's
        metadata = root.create_group(DatasetName.METADATA.value)
        create_ard_yaml(
            {
                grp_name: get_band_acqs(grp_name)
                for grp_name in container.supported_groups
            }, ancillary_group, metadata, parameters, workflow)