def requires(self): container = acquisitions(self.level1, self.acq_parser_hint) acqs = container.get_acquisitions(group=self.group, granule=self.granule) # NBAR & SBT acquisitions nbar_acqs = [a for a in acqs if a.band_type == BandType.REFLECTIVE] sbt_acqs = [a for a in acqs if a.band_type == BandType.THERMAL] tasks = {} for coefficient in self.workflow.atmos_coefficients: if coefficient in Workflow.NBAR.atmos_coefficients: band_acqs = nbar_acqs else: band_acqs = sbt_acqs for acq in band_acqs: key = (acq.band_name, coefficient) kwargs = { 'level1': self.level1, 'work_root': self.work_root, 'granule': self.granule, 'group': self.group, 'band_name': acq.band_name, 'coefficient': coefficient, 'workflow': self.workflow, 'vertices': self.vertices, 'method': self.method } tasks[key] = InterpolateCoefficient(**kwargs) return tasks
def run(self): container = acquisitions(self.level1, self.acq_parser_hint) acqs, group = container.get_highest_resolution(granule=self.granule) # output filename format output_fmt = pjoin(POINT_FMT, ALBEDO_FMT, ''.join([POINT_ALBEDO_FMT, '.tp5'])) # input filenames ancillary_fname = self.input()['ancillary'].path sat_sol_fname = self.input()[group]['sat_sol'].path lon_lat_fname = self.input()[group]['lon_lat'].path with self.output().temporary_path() as out_fname: tp5_data = _format_tp5(acqs, sat_sol_fname, lon_lat_fname, ancillary_fname, out_fname, self.workflow) # keep this as an indented block, that way the target will remain # atomic and be moved upon closing for key in tp5_data: point, albedo = key tp5_fname = output_fmt.format(p=point, a=albedo.value) target = pjoin(dirname(out_fname), self.base_dir, tp5_fname) with luigi.LocalTarget(target).open('w') as src: src.writelines(tp5_data[key])
def test_missing_utm_file(self): """ Test assertion is raised on missing utm file """ geobox = acquisitions(LS7_SCENE1).get_all_acquisitions()[0].gridded_geo_box() with self.assertRaises(AssertionError): calc_land_sea_mask(geobox, ancillary_path=LAND_SEA_RASTERS)
def requires(self): with open(self.level1_list) as src: level1_list = [level1.strip() for level1 in src.readlines()] for level1 in level1_list: container = acquisitions(level1) outdir = pjoin(self.outdir, "{}.wagl".format(container.label)) for granule in container.granules: kwargs = { "level1": level1, "granule": granule, "workflow": self.workflow, "vertices": self.vertices, "pixel_quality": self.pixel_quality, "method": self.method, "modtran_exe": self.modtran_exe, "outdir": outdir, "land_sea_path": self.land_sea_path, "aerosol": self.aerosol, "brdf": self.brdf, "ozone_path": self.ozone_path, "water_vapour": self.water_vapour, "dem_path": self.dem_path, "ecmwf_path": self.ecmwf_path, "invariant_height_fname": self.invariant_height_fname, "dsm_fname": self.dsm_fname, "tle_path": self.tle_path, "rori": self.rori, "compression": self.compression, "filter_opts": self.filter_opts, "buffer_distance": self.buffer_distance, "h5_driver": self.h5_driver, } yield DataStandardisation(**kwargs)
def _can_process(l1t_path, granule): _LOG.debug('Checking L1T: %r', l1t_path) acqs = acquisitions(l1t_path).get_all_acquisitions(granule) landsat_path = int(acqs[0].path) landsat_row = int(acqs[0].row) # TODO # the path/row exclusion logic is not long-term viable and the prototype # for S2 will follow a similar exclusion logic, but use MGRS tiles instead. # A geometry exclusion is probably better suited in going forward with # multi-sensor/platform support # Is it an Australian scene? That's all we support at the moment. # (numbers specified by Lan-Wei.) msg = 'Not an Australian {} ({}): {}' if not (87 <= landsat_path <= 116): msg = msg.format('path', landsat_path, basename(l1t_path)) _LOG.info(msg) return False, msg if not (67 <= landsat_row <= 91): msg = msg.format('row', landsat_row, basename(l1t_path)) _LOG.info(msg) return False, msg # Do we have a reference dir available to compute GQA? ref_dir, msg = get_acq_reference_directory(acqs[0]) if not ref_dir: return ref_dir, msg return True, None
def filter_granule_worker(out_stream): count = 0 for level1_dataset in _level1_dataset_path_iter(Path(level1_root), *find_options): try: container = acquisitions(str(level1_dataset)) except Exception as e: logging.warning('encountered unexpected error for %s: %s', str(level1_dataset), e) logging.exception(e) continue granule_md = get_archive_metadata(level1_dataset) for granule, sensing_date in granule_md.items(): tile_id = granule.split('_')[-2] if tile_id not in tile_ids: logging.debug('granule %s with MGRS tile ID %s outside AOI', granule, tile_id) continue ymd = sensing_date.strftime('%Y-%m-%d') package = Package( level1=str(level1_dataset), workdir='', granule=granule, pkgdir=join(pkgdir, ymd) ) if package.output().exists(): logging.debug('granule %s already processed', granule) continue logging.debug('level1 dataset %s needs to be processed', level1_dataset) print(level1_dataset, file=out_stream) count += len(granule_md.keys()) # To handle multigranule files break return out_stream, count
def get_acquisition(l1t_path, granule): # Get the acquisitions, metadata, and filter by wavelength acqs = acquisitions(l1t_path).get_all_acquisitions(granule=granule) # TODO include MGRS id logic # TODO improve path/row or MGRS id decision logic # check if the path/row is identified as a reef scene path = acqs[0].path row = acqs[0].row df = pandas.read_csv(REEF_PR) reef_scene = ((df.Path == path) & (df.Row == row)).any() # Get the wavelengths to filter the acquisitions # TODO parse min/max as args not config if reef_scene: min_lambda = CONFIG.getfloat('work', 'reef_min_lambda') max_lambda = CONFIG.getfloat('work', 'reef_max_lambda') else: min_lambda = CONFIG.getfloat('work', 'min_lambda') max_lambda = CONFIG.getfloat('work', 'max_lambda') # only accept a single wavelength (for now...) acq = [ acq for acq in acqs if (acq.band_type == BandType.REFLECTIVE and min_lambda < acq.wavelength[1] <= max_lambda) ] return acq[0]
def run(self): container = acquisitions(self.level1, self.acq_parser_hint) # out_path = container.get_root(self.work_root, granule=self.granule) acqs = container.get_all_acquisitions(self.granule) atmospheric_inputs_fname = self.input().path base_dir = pjoin(self.work_root, self.base_dir) albedos = [Albedos(a) for a in self.albedos] prepare_modtran(acqs, self.point, albedos, base_dir) with self.output().temporary_path() as out_fname: nvertices = self.vertices[0] * self.vertices[1] _run_modtran( acqs, self.modtran_exe, base_dir, self.point, albedos, self.workflow, nvertices, atmospheric_inputs_fname, out_fname, self.compression, self.filter_opts, )
def run(self): container = acquisitions(self.level1, self.acq_parser_hint) grn = container.get_granule(granule=self.granule, container=True) sbt_path = None nbar_paths = { "aerosol_dict": self.aerosol, "water_vapour_dict": self.water_vapour, "ozone_path": self.ozone_path, "dem_path": self.dem_path, "brdf_dict": self.brdf, } if self.workflow == Workflow.STANDARD or self.workflow == Workflow.SBT: sbt_path = self.ecmwf_path with self.output().temporary_path() as out_fname: _collect_ancillary( grn, self.input().path, nbar_paths, sbt_path, self.invariant_height_fname, self.vertices, out_fname, self.compression, self.filter_opts, )
def test_granules_s2a_scene1(self): container = acquisitions(S2A_SCENE1) self.assertEqual(len(container.granules), 1) self.assertEqual( container.granules[0], "S2A_OPER_MSI_L1C_TL_SGS__20171207T032513_A012840_T55JEJ_N02.06", )
def test_granules_s2b_scene1(self): container = acquisitions(S2B_SCENE1) self.assertEqual(len(container.granules), 1) self.assertEqual( container.granules[0], "S2B_OPER_MSI_L1C_TL_SGS__20170719T012130_A001915_T56JKT_N02.05", )
def requires(self): container = acquisitions(self.level1, self.acq_parser_hint) acqs = container.get_acquisitions(group=self.group, granule=self.granule) # NBAR & SBT acquisitions nbar_acqs = [a for a in acqs if a.band_type == BandType.REFLECTIVE] sbt_acqs = [a for a in acqs if a.band_type == BandType.THERMAL] tasks = {} for coefficient in self.workflow.atmos_coefficients: if coefficient in Workflow.NBAR.atmos_coefficients: band_acqs = nbar_acqs else: band_acqs = sbt_acqs for acq in band_acqs: key = (acq.band_name, coefficient) kwargs = { "level1": self.level1, "work_root": self.work_root, "granule": self.granule, "group": self.group, "band_name": acq.band_name, "coefficient": coefficient, "workflow": self.workflow, "vertices": self.vertices, "method": self.method, } tasks[key] = InterpolateCoefficient(**kwargs) return tasks
def requires(self): with open(self.level1_list) as src: level1_list = [level1.strip() for level1 in src.readlines()] for level1 in level1_list: container = acquisitions(level1) outdir = pjoin(self.outdir, '{}.wagl'.format(container.label)) for granule in container.granules: kwargs = {'level1': level1, 'granule': granule, 'workflow': self.workflow, 'vertices': self.vertices, 'pixel_quality': self.pixel_quality, 'method': self.method, 'modtran_exe': self.modtran_exe, 'outdir': outdir, 'land_sea_path': self.land_sea_path, 'aerosol': self.aerosol, 'brdf': self.brdf, 'ozone_path': self.ozone_path, 'water_vapour': self.water_vapour, 'dem_path': self.dem_path, 'ecmwf_path': self.ecmwf_path, 'invariant_height_fname': self.invariant_height_fname, 'dsm_fname': self.dsm_fname, 'tle_path': self.tle_path, 'rori': self.rori, 'compression': self.compression, 'filter_opts': self.filter_opts, 'buffer_distance': self.buffer_distance, 'h5_driver': self.h5_driver} yield DataStandardisation(**kwargs)
def run(self): container = acquisitions(self.level1, self.acq_parser_hint) acqs = container.get_acquisitions(self.group, self.granule) # inputs inputs = self.input() interpolation_fname = inputs['interpolation'].path slp_asp_fname = inputs['slp_asp'].path incident_fname = inputs['incident'].path exiting_fname = inputs['exiting'].path relative_slope_fname = inputs['rel_slope'].path shadow_fname = inputs['shadow'].path sat_sol_fname = inputs['sat_sol'].path ancillary_fname = inputs['ancillary'].path # get the acquisition we wish to process acq = [acq for acq in acqs if acq.band_name == self.band_name][0] with self.output().temporary_path() as out_fname: _calculate_reflectance(acq, acqs, interpolation_fname, sat_sol_fname, slp_asp_fname, relative_slope_fname, incident_fname, exiting_fname, shadow_fname, ancillary_fname, self.rori, out_fname, self.compression, self.filter_opts)
def requires(self): band_acqs = [] container = acquisitions(self.level1, self.acq_parser_hint) acqs = container.get_acquisitions(group=self.group, granule=self.granule) # NBAR acquisitions if self.workflow == Workflow.STANDARD or self.workflow == Workflow.NBAR: band_acqs.extend([a for a in acqs if a.band_type == BandType.REFLECTIVE]) # SBT acquisitions if self.workflow == Workflow.STANDARD or self.workflow == Workflow.SBT: band_acqs.extend([a for a in acqs if a.band_type == BandType.THERMAL]) tasks = [] for acq in band_acqs: kwargs = { "level1": self.level1, "work_root": self.work_root, "granule": self.granule, "group": self.group, "band_name": acq.band_name, "workflow": self.workflow, "vertices": self.vertices, "method": self.method, } if acq.band_type == BandType.THERMAL: tasks.append(SurfaceTemperature(**kwargs)) else: kwargs["dsm_fname"] = self.dsm_fname kwargs["buffer_distance"] = self.buffer_distance tasks.append(SurfaceReflectance(**kwargs)) return tasks
def prepare_dataset(path, acq_parser_hint=None, granule=None): """ Returns a dictionary of image paths, granule id and metadata file location for the granules contained within the input file """ acq_container = acquisitions(path, acq_parser_hint) tasks = [] if granule is None: granules = acq_container.granules else: granules = [granule] for granule_id in granules: image_dict = OrderedDict([ ('B01', {}), ('B02', {}), ('B03', {}), ('B04', {}), ('B05', {}), ('B06', {}), ('B07', {}), ('B08', {}), ('B8A', {}), ('B09', {}), ('B10', {}), ('B11', {}), ('B12', {}) ]) for group_id in acq_container.groups: acqs = acq_container.get_acquisitions(granule=granule_id, group=group_id, only_supported_bands=False) for acq in acqs: key = Path(acq.uri).stem[-3:] if key in image_dict: image_dict[key] = {'path': acq.uri, 'layer': '1'} tasks.append(tuple([image_dict, granule_id, acq.granule_xml])) return tasks
def test_land_sea(self): """ Check calc_land_sea_mask for similar result """ precomputed_mean = 0.8835457063711911 geobox = acquisitions(LS8_SCENE1).get_all_acquisitions()[0].gridded_geo_box() land_sea_mask = get_land_sea_mask(geobox, ancillary_path=LAND_SEA_RASTERS) self.assertAlmostEqual(land_sea_mask.mean(), precomputed_mean, places=2)
def run(self): acqs = (acquisitions(self.level1, self.acq_parser_hint).get_acquisitions( self.group, self.granule)) with self.output().temporary_path() as out_fname: _get_dsm(acqs[0], self.dsm_fname, self.buffer_distance, out_fname, self.compression, self.filter_opts)
def run(self): acq = (acquisitions(self.level1, self.acq_parser_hint).get_acquisitions( self.group, self.granule))[0] with self.output().temporary_path() as out_fname: _create_lon_lat_grids(acq, out_fname, self.compression, self.filter_opts)
def output(self): out_dirs = [self.reflectance_dir, self.shadow_dir, self.interpolation_dir] container = acquisitions(self.level1, self.acq_parser_hint) for granule in container.granules: for group in container.supported_groups: pth = container.get_root(self.work_root, group, granule) for out_dir in out_dirs: yield luigi.LocalTarget(pjoin(pth, out_dir))
def test_load_acquisitions_s2a_scene1(self): container = acquisitions(S2A_SCENE1) self.assertEqual(len(container.get_all_acquisitions()), 11) self.assertEqual(len(container.get_acquisitions(group='RES-GROUP-0')), 4) self.assertEqual(len(container.get_acquisitions(group='RES-GROUP-1')), 6) self.assertEqual(len(container.get_acquisitions(group='RES-GROUP-2')), 1)
def run(self): acqs = (acquisitions(self.level1, self.acq_parser_hint).get_acquisitions( self.group, self.granule)) dsm_fname = self.input().path with self.output().temporary_path() as out_fname: _slope_aspect_arrays(acqs[0], dsm_fname, self.buffer_distance, out_fname, self.compression, self.filter_opts)
def run(self): acqs = (acquisitions(self.level1, self.acq_parser_hint).get_acquisitions( self.group, self.granule)) with self.output().temporary_path() as out_fname: _calculate_angles(acqs[0], self.input().path, out_fname, self.compression, self.filter_opts, self.tle_path)
def run(self): container = acquisitions(self.level1, self.acq_parser_hint) acqs = container.get_acquisitions(self.group, self.granule) acq = [acq for acq in acqs if acq.band_name == self.band_name][0] with self.output().temporary_path() as out_fname: interpolation_fname = self.input()['interpolation'].path ancillary_fname = self.input()['ancillary'].path _surface_brightness_temperature(acq, acqs, interpolation_fname, ancillary_fname, out_fname, self.compression, self.filter_opts)
def requires(self): with open(self.level1_list) as src: level1_list = [level1.strip() for level1 in src.readlines()] for level1 in level1_list: work_root = pjoin(self.workdir, '{}.ARD'.format(basename(level1))) container = acquisitions(level1, self.acq_parser_hint) for granule in container.granules: work_dir = container.get_root(work_root, granule=granule) # TODO; pkgdir for landsat data pkgdir = pjoin(self.pkgdir, basename(dirname(level1))) yield Package(level1, work_dir, granule, pkgdir)
def requires(self): with open(self.level1_list) as src: level1_list = [level1.strip() for level1 in src.readlines()] for level1 in level1_list: work_root = pjoin(self.workdir, '{}.ARD'.format(basename(level1))) container = acquisitions(level1, self.acq_parser_hint) for granule in container.granules: work_dir = container.get_root(work_root, granule=granule) acq = container.get_acquisitions(None, granule, False)[0] ymd = acq.acquisition_datetime.strftime('%Y-%m-%d') pkgdir = pjoin(self.pkgdir, ymd) yield Package(level1, work_dir, granule, pkgdir)
def run(self): acqs = (acquisitions(self.level1, self.acq_parser_hint).get_acquisitions( self.group, self.granule)) # input filenames dsm_fname = self.input()['dsm'].path sat_sol_fname = self.input()['sat_sol'].path with self.output().temporary_path() as out_fname: _calculate_cast_shadow(acqs[0], dsm_fname, self.buffer_distance, sat_sol_fname, out_fname, self.compression, self.filter_opts, False)
def tasks(level1_list): # TODO check with Lan-Wei regarding multi-granule vs single-granule # gqa operation. # Below demo's submit all granules as single granule gqa operation # (same as wagl) for level1 in level1_list: container = acquisitions(level1, self.acq_parser_hint) for granule in container.granules: # TODO enable updating dataset in-place # if update_source: # yield UpdateSource(level1, work_root) # else: yield GQATask(level1, granule, self.workdir)
def can_pq(level1, acq_parser_hint=None): """ A simple test to check if we can process a scene through the pq pipeline. :param level1: An `str` containing the file path name to the directory containing the level-1 data. :return: True if the scene can be processed through PQ, else False. """ supported = ["LANDSAT_5", "LANDSAT_7", "LANDSAT_8"] acq = acquisitions(level1, acq_parser_hint).get_acquisitions()[0] return acq.platform_id in supported
def requires(self): with open(self.level1_list) as src: level1_list = [level1.strip() for level1 in src.readlines()] for level1 in level1_list: work_name = "{}-wagl".format(basename(level1)) container = acquisitions(level1, self.acq_parser_hint) for granule in container.granules: # as each granule is independent, include the granule as the work root work_root = pjoin(self.outdir, work_name, granule) if "group" in self.task.get_param_names(): for group in container.supported_groups: yield self.task(level1, work_root, granule, group) else: yield self.task(level1, work_root, granule)