def process(self, pd, image, band): """ Write final product in the archive directory 'archive_dir' is defined in config.ini file Naming convention from Design Document :param pd: instance of S2L_Product class :param image: input instance of S2L_ImageFile class :param band: band being processed :return: output instance of instance of S2L_ImageFile class """ # TODO : add production date? # /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR res = image.xRes product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L( pd) sensor = pd.sensor_name relative_orbit = config.get('relative_orbit') native = band in ['B08', 'B10', 'B11'] if sensor == 'LS8' else \ band in ['B05', 'B06', 'B07', 'B08'] s2_band = pd.get_s2like_band(band) if not native: band = s2_band band_rootName = "_".join([ 'L2F', 'T' + tilecode, datatake_sensing_start, sensor, 'R{:0>3}'.format(relative_orbit) ]) metadata.mtd['band_rootName_F'] = band_rootName outfile = "_".join([band_rootName, band, '{}m'.format(int(res)) ]) + '.TIF' # Naming convention from Sentinel-2-Products-Specification-Document (p294) tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series newpath = self.band_path(tsdir, product_name, granule_compact_name, outfile, native=native) COG = config.getboolean('COG') log.debug('New: ' + newpath) image.write(creation_options=['COMPRESS=LZW'], filepath=newpath, COG=COG, band=band) metadata.mtd.get('bands_path_F').append(newpath) # declare output internally self.images[s2_band] = image.filepath # declare output in config file config.set('imageout_dir', image.dirpath) config.set('imageout_' + band, image.filename) return image
def smac_correction_grid(obs_datetime, extent, hcs_code, resolution=120): output_filename = 'output_file.tif' ecmwf_data = EMWF_Product( config.get('cams_dir'), cams_hourly_directory=config.get('cams_hourly_dir'), cams_climatology_directory=config.get('cams_climatology_dir'), observation_datetime=obs_datetime) new_SRS = gdal.osr.SpatialReference() new_SRS.ImportFromEPSG(int(4326)) if ecmwf_data.is_valid: # Write cams file cams_file = 'cams_file.tif' etype = gdal.GDT_Float32 driver = gdal.GetDriverByName('GTiff') dst_ds = driver.Create(cams_file, xsize=ecmwf_data.longitude.size, ysize=ecmwf_data.latitude.size, bands=4, eType=etype, options=[]) dst_ds.SetProjection(new_SRS.ExportToWkt()) x_res = (ecmwf_data.longitude.max() - ecmwf_data.longitude.min()) / ecmwf_data.longitude.size y_res = (ecmwf_data.latitude.max() - ecmwf_data.latitude.min()) / ecmwf_data.latitude.size geotranform = (ecmwf_data.longitude.min(), x_res, 0, ecmwf_data.latitude.max(), 0, -y_res) dst_ds.SetGeoTransform(geotranform) dst_ds.GetRasterBand(1).WriteArray(ecmwf_data.aod550.astype(np.float)) dst_ds.GetRasterBand(2).WriteArray(ecmwf_data.tcwv.astype(np.float)) dst_ds.GetRasterBand(3).WriteArray(ecmwf_data.gtco3.astype(np.float)) dst_ds.GetRasterBand(4).WriteArray(ecmwf_data.msl.astype(np.float)) dst_ds.FlushCache() # Warp cams data on input spatial extent options = gdal.WarpOptions(srcSRS=dst_ds.GetProjection(), dstSRS=hcs_code, xRes=resolution, yRes=resolution, resampleAlg='cubicspline', outputBounds=extent) gdal.Warp(output_filename, cams_file, options=options) dst_ds = None return output_filename
def base_path(product): relative_orbit = config.get('relative_orbit') acqdate = dt.datetime.strftime(product.acqdate, '%Y%m%d') tilecode = product.mtl.mgrs if tilecode.startswith('T'): tilecode = tilecode[1:] return "_".join(['L2F', tilecode, acqdate, product.sensor_name, 'R{:0>3}'.format(relative_orbit)]), tilecode
def preprocess(self, product): product_name, granule_compact_name, tilecode, _ = self.base_path_S2L( product) metadata.mtd['product_H_name'] = product_name metadata.mtd['granule_H_name'] = granule_compact_name metadata.mtd['product_creation_date'] = metadata.mtd.get( 'product_creation_date', dt.datetime.now()) outdir = os.path.join(config.get('archive_dir'), tilecode) """ # Creation of S2 folder tree structure tree = core.QI_MTD.S2_structure.generate_S2_structure_XML(out_xml='', product_name=product_name, tile_name=granule_compact_name, save_xml=False) core.QI_MTD.S2_structure.create_architecture(outdir, tree, create_empty_files=True) """ log.debug('Create folder : ' + os.path.join(outdir, product_name)) change_nodes = { 'PRODUCT_NAME': product_name, 'TILE_NAME': granule_compact_name, } core.QI_MTD.S2_structure.create_architecture( outdir, metadata.hardcoded_values.get('s2_struct_xml'), change_nodes=change_nodes, create_empty_files=False)
def update_configuration(args, tile=None): # init S2L_config and save to wd if not config.initialize(args.S2L_configfile): return if args.confParams is not None: config.overload(args.confParams) config.set('wd', os.path.join(args.wd, str(os.getpid()))) references_map_file = config.get('references_map') if args.refImage: config.set('refImage', args.refImage) elif references_map_file and tile: # load dataset with open(references_map_file) as j: references_map = json.load(j) config.set('refImage', references_map.get(tile)) else: config.set('refImage', None) config.set( 'hlsplus', config.getboolean('doPackager') or config.getboolean('doPackagerL2F')) config.set('debug', args.debug) config.set('generate_intermediate_products', args.generate_intermediate_products) if hasattr(args, 'l2a'): config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def update_configuration(args, tile=None): # init S2L_config and save to wd if not config.initialize(args.S2L_configfile): return if args.confParams is not None: config.overload(args.confParams) use_pid = False if use_pid: output_folder = str(os.getpid()) else: date_now = datetime.datetime.now().strftime('%Y%m%dT_%H%M%S') output_folder = f'{"" if args.no_log_date else f"{date_now}_"}{compute_config_hash(args, config)}' config.set('wd', os.path.join(args.wd, output_folder)) references_map_file = config.get('references_map') if args.refImage: config.set('refImage', args.refImage) elif references_map_file and tile: # load dataset with open(references_map_file) as j: references_map = json.load(j) config.set('refImage', references_map.get(tile)) else: config.set('refImage', None) config.set( 'hlsplus', config.getboolean('doPackager') or config.getboolean('doPackagerL2F')) config.set('debug', args.debug) config.set('generate_intermediate_products', args.generate_intermediate_products) if hasattr(args, 'l2a'): config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def main(with_multiprocess_support=False): parser = configure_arguments() args = parser.parse_args() log.configure_loggers(log_path=args.wd, is_debug=args.debug, without_date=args.no_log_date) if args.operational_mode is None: parser.print_help() return 1 # convert list of bands if provided if args.bands is not None: args.bands = args.bands.split(',') products, start_date, end_date = configure_sen2like(args) if products is None: return 1 if args.operational_mode == 'multi-tile-mode' and with_multiprocess_support and not args.no_run: number_of_process = args.jobs if number_of_process is None: number_of_process = config.get('number_of_process', 1) params = [(tile, _products, args, start_date, end_date) for tile, _products in products.items()] with Pool(int(number_of_process)) as pool: pool.starmap(start_process, params) else: if args.no_run: logger.info("No-run mode: Products will only be listed") for tile, _products in products.items(): start_process(tile, _products, args, start_date, end_date) return 0
def stitch_multi(product, product_file, new_product_file): ds_product_src = gdal.Open(product_file) ds_new_product_src = gdal.Open(new_product_file) filepath_out = os.path.join(config.get('wd'), product.name, 'tie_points_STITCHED.TIF') for i in range(1, ds_product_src.RasterCount + 1): array_product = ds_product_src.GetRasterBand(i).ReadAsArray() array_new_product = ds_new_product_src.GetRasterBand(i).ReadAsArray() np.copyto(array_product, array_new_product, where=array_product == 0) if i == 1: # write with gdal driver = gdal.GetDriverByName('GTiff') ds_dst = driver.Create(filepath_out, bands=ds_product_src.RasterCount, xsize=ds_product_src.RasterXSize, ysize=ds_product_src.RasterYSize, eType=gdal.GDT_Int16) ds_dst.SetProjection(ds_product_src.GetProjection()) ds_dst.SetGeoTransform(ds_product_src.GetGeoTransform()) # write band ds_dst.GetRasterBand(i).WriteArray(array_product) ds_dst.FlushCache() ds_product_src = None ds_new_product_src = None ds_dst = None return filepath_out
def base_path_S2L(product): """ See https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi/naming-convention More information https://sentinel.esa.int/documents/247904/685211/Sentinel-2-Products-Specification-Document at p74, p438 Needed parameters : datastrip sensing start datatake sensing start absolute orbit relative orbit product generation time Product baseline number """ relative_orbit = config.get('relative_orbit') file_date = dt.datetime.strftime(product.file_date, '%Y%m%dT%H%M%S') # generation time if product.sensor == 'S2': datatake_sensing_start = dt.datetime.strftime( product.dt_sensing_start, '%Y%m%dT%H%M%S') datastrip_sensing_start = dt.datetime.strftime( product.ds_sensing_start, '%Y%m%dT%H%M%S') absolute_orbit = config.get('absolute_orbit') else: datatake_sensing_start = dt.datetime.strftime( product.acqdate, '%Y%m%dT%H%M%S') datastrip_sensing_start = file_date absolute_orbit = metadata.hardcoded_values.get('L8_absolute_orbit') PDGS = metadata.hardcoded_values.get('PDGS') tilecode = product.mtl.mgrs if tilecode.startswith('T'): tilecode = tilecode[1:] sensor = product.mtl.sensor[0:3] # OLI / MSI / OLI_TIRS product_name = "_".join([ product.sensor_name, '{}L2H'.format(sensor), datatake_sensing_start, 'N' + PDGS, 'R{:0>3}'.format(relative_orbit), 'T' + tilecode, file_date ]) + '.SAFE' granule_compact_name = "_".join([ 'L2H', 'T' + tilecode, 'A' + absolute_orbit, datastrip_sensing_start, product.sensor_name, 'R{:0>3}'.format(relative_orbit) ]) return product_name, granule_compact_name, tilecode, datatake_sensing_start
def _save_as_image_file(self, image_template, array, product, band, extension): path = os.path.join(config.get('wd'), product.name, product.get_band_file(band).rootname + extension) image_file = image_template.duplicate(path, array=array) if config.getboolean('generate_intermediate_products'): image_file.write(creation_options=['COMPRESS=LZW']) return image_file
def generate_S2_tile_id(product, H_F, AC): tilecode = product.mtl.mgrs pdgs = metadata.hardcoded_values.get('PDGS', '9999') PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]]) acqdate = dt.datetime.strftime(product.acqdate, '%Y%m%dT%H%M%S') if AC.endswith('_'): AC = AC[:-1] tile_id = '_'.join([product.sensor_name, 'OPER', 'MSI', 'L2{}'.format(H_F), AC, acqdate, 'A{}'.format(config.get('absolute_orbit')), tilecode, 'N{}'.format(PDGS)]) return tile_id
def reframe(self, image, product, band=None, dtype=None): # Add margin margin = int(config.get('reframe_margin', self.margin)) log.debug(f"Using {margin} as margin.") product_image = mgrs_framing.reframe(image, self.tile, filepath_out=self.output_file(product, band, image, "_PREREFRAMED"), order=0, margin=margin, dtype=dtype, compute_offsets=True) if config.getboolean('generate_intermediate_products'): product_image.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count return product_image
def preprocess(self, product): self.get_new_product(product) if self.new_product is None: return # Stitch validity mask and angles product_validity_masks = [] product_nodata_masks = [] product_angles = [] for _product in [product, self.new_product.reader(self.new_product.path)]: is_mask_valid = True # Validity mask if _product.mtl.mask_filename is None: is_mask_valid = _product.mtl.get_valid_pixel_mask( os.path.join(config.get("wd"), _product.name, 'valid_pixel_mask.tif')) if is_mask_valid: product_validity_masks.append(self.reframe(S2L_ImageFile(_product.mtl.mask_filename), _product)) product_nodata_masks.append(self.reframe(S2L_ImageFile(_product.mtl.nodata_mask_filename), _product)) # Angles if _product.mtl.angles_file is None: _product.mtl.get_angle_images(os.path.join(config.get("wd"), _product.name, 'tie_points.tif')) filepath_out = os.path.join(config.get('wd'), _product.name, 'tie_points_PREREFRAMED.TIF') mgrs_framing.reframeMulti(_product.mtl.angles_file, self.tile, filepath_out=filepath_out, order=0) product_angles.append(filepath_out) if None not in product_validity_masks: stitched_mask = self.stitch(product, product_validity_masks[0], product_validity_masks[1]) stitched_mask.write(creation_options=['COMPRESS=LZW']) product.mtl.mask_filename = stitched_mask.filepath if None not in product_nodata_masks: stitched_mask = self.stitch(product, product_nodata_masks[0], product_nodata_masks[1]) stitched_mask.write(creation_options=['COMPRESS=LZW']) product.mtl.nodata_mask_filename = stitched_mask.filepath stitched_angles = self.stitch_multi(product, product_angles[0], product_angles[1]) product.mtl.angles_file = stitched_angles # Stitch reference band (needed by geometry module) band = config.get('reference_band', 'B04') if config.getboolean('doMatchingCorrection') and config.get('refImage'): image = product.get_band_file(band) self.process(product, image, band)
def preprocess(self, product): # reinit dx/dy config.set('dx', 0) config.set('dy', 0) if product.sensor != 'S2': # Reframe angles and masks filepath_out = os.path.join(config.get('wd'), product.name, 'tie_points_REFRAMED.TIF') mgrs_framing.reframeMulti(product.mtl.angles_file, product.mtl.mgrs, filepath_out, config.getfloat('dx'), config.getfloat('dy'), order=0) product.mtl.angles_file = filepath_out # Reframe mask if product.mtl.mask_filename: filepath_out = os.path.join(config.get('wd'), product.name, 'valid_pixel_mask_REFRAMED.TIF') image = S2L_ImageFile(product.mtl.mask_filename) imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'), config.getfloat('dy'), order=0) imageout.write(creation_options=['COMPRESS=LZW']) product.mtl.mask_filename = filepath_out # Reframe nodata mask if product.mtl.nodata_mask_filename: filepath_out = os.path.join(config.get('wd'), product.name, 'nodata_pixel_mask_REFRAMED.TIF') image = S2L_ImageFile(product.mtl.nodata_mask_filename) imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'), config.getfloat('dy'), order=0) imageout.write(creation_options=['COMPRESS=LZW']) product.mtl.nodata_mask_filename = filepath_out # Matching for dx/dy correction? band = config.get('reference_band', 'B04') if config.getboolean('doMatchingCorrection') and config.get('refImage'): config.set('freeze_dx_dy', False) image = product.get_band_file(band) self.process(product, image, band) # goal is to feed dx, dy in config config.set('freeze_dx_dy', True) metadata.qi.update({'COREGISTRATION_BEFORE_CORRECTION': self._tmp_stats.get('MEAN'.format(band))})
def convert_to_reflectance_from_reflectance_cal_product(mtl, data_in, band): """Applied conversion to TOA reflectance, assuming product is calibrated in reflectance as S2 MSI and LS8 OLI Required mtl.processing_dic, a dictionary including the list of band to be processing""" log.debug("Conversion to TOA") reflectance_data = None if mtl.sensor == 'OLI' or mtl.sensor == 'OLI_TIRS': # LANDSAT 8 log.info("Sun Zenith angle : {} deg".format(mtl.sun_zenith_angle)) sun_elevation_angle = 90. - mtl.sun_zenith_angle log.info("Sun Elevation angle : {} deg".format(sun_elevation_angle)) gain = offset = None for k, x in list(mtl.rho_radio_coefficient_dic.items()): if 'B' + x['Band_id'] == band: gain = str(x['Gain']) offset = str(x['Offset']) log.info('Band Id : {} Gain : {} / Offset : {}'.format( x['Band_id'], gain, offset)) if gain is not None and offset is not None: reflectance_data = (np.float32(data_in) * np.float32(gain) + np.float32(offset)) / np.sin( sun_elevation_angle * np.pi / 180.) mask = (data_in <= 0) reflectance_data[mask] = 0 elif band in ('B10', 'B11'): offset = float(config.get('offset')) gain = float(config.get('gain')) reflectance_data = np.float32(data_in) / gain - offset mask = (data_in <= 0) reflectance_data[mask] = 0 elif mtl.sensor == 'MSI': # apply quantification value reflectance_data = np.float32(data_in) / float( mtl.quantification_value) return reflectance_data
def process(self, pd, image, band): """ Write final product in the archive directory 'archive_dir' is defined in config.ini file Naming convention from Design Document :param pd: instance of S2L_Product class :param image: input instance of S2L_ImageFile class :param band: band being processed :return: outpu t instance of instance of S2L_ImageFile class """ # TODO : add production date? # TODO : change L8 band numbers to S2 numbers convention? # /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR res = image.xRes outdir, tilecode = self.base_path(pd) outfile = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF' tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series newpath = os.path.join(tsdir, outdir, outfile) log.debug('New: ' + newpath) image.write(creation_options=['COMPRESS=LZW'], filepath=newpath) # declare output internally self.images[band] = image.filepath # declare output in config file config.set('imageout_dir', image.dirpath) config.set('imageout_' + band, image.filename) if config.getboolean('hlsplus'): res = 30 outfile_30m = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF' newpath_30m = os.path.join(tsdir, outdir, outfile_30m) if pd.sensor == 'S2': # create 30m band as well # resampling log.info('Resampling to 30m: Start...') image_30m = mgrs_framing.resample(S2L_ImageFile(newpath), res, newpath_30m) image_30m.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count log.info('Resampling to 30m: End') if pd.sensor == 'L8' and band in pd.image30m: # copy 30m band as well # write pd.image30m[band].write(creation_options=['COMPRESS=LZW'], filepath=newpath_30m) del pd.image30m[band] return image
def preprocess(self, pd): # check most recent HLS S2 products available archive_dir = config.get('archive_dir') tsdir = join(archive_dir, pd.mtl.mgrs) # list products with dates pdlist = [] for pdpath in sorted(glob.glob(tsdir + '/L2F_*_S2*')): pdname = basename(pdpath) date = dt.datetime.strptime(pdname.split('_')[2], '%Y%m%d').date() if date <= pd.acqdate.date(): pdlist.append([date, pdpath]) # Handle new format aswell for pdpath in sorted(glob.glob(tsdir + '/S2*L2F_*')): pdname = basename(pdpath) date = dt.datetime.strptime( os.path.splitext(pdname.split('_')[2])[0], '%Y%m%dT%H%M%S').date() if date <= pd.acqdate.date(): pdlist.append([date, pdpath]) # sort by date pdlist.sort() # reset ref list and keep 2 last ones self.reference_products = [] nb_products = int(config.get('predict_nb_products', 2)) for date, pdname in pdlist[-nb_products:]: product = S2L_HLS_Product(pdname) if product.product is not None: self.reference_products.append(product) for product in self.reference_products: log.info('Selected product: {}'.format(product.name))
def read_metadata(self, granule_folder='GRANULE'): # extract metadata self.mtl = readers.get_reader(self.path) if self.mtl is None: return self.mtl = self.mtl(self.path) try: self.update_site_info(config.get('tile', None)) except AttributeError: # Some products not need to update their site information pass # retrieve acquisition date in a datetime format scene_center_time = self.mtl.scene_center_time n = len( self.mtl.scene_center_time.split('.')[-1]) - 1 # do not count Z if n < 6: # fill with zeros scene_center_time = self.mtl.scene_center_time.replace( 'Z', (6 - n) * '0' + 'Z') self.acqdate = dt.datetime.strptime( self.mtl.observation_date + ' ' + scene_center_time, "%Y-%m-%d %H:%M:%S.%fZ") if 'S2' in self.sensor or self.mtl.data_type in [ 'Level-2F', 'Level-2H' ]: # Sentinel 2 self.dt_sensing_start = dt.datetime.strptime( self.mtl.dt_sensing_start, "%Y-%m-%dT%H:%M:%S.%fZ") self.ds_sensing_start = dt.datetime.strptime( self.mtl.ds_sensing_start, "%Y-%m-%dT%H:%M:%S.%fZ") self.file_date = dt.datetime.strptime(self.mtl.file_date, "%Y-%m-%dT%H:%M:%S.%fZ") logger.debug("Datatake sensing start: {}".format( self.dt_sensing_start)) logger.debug("Datastrip sensing start: {}".format( self.ds_sensing_start)) else: self.file_date = dt.datetime.strptime(self.mtl.file_date, "%Y-%m-%dT%H:%M:%SZ") logger.debug("Product generation time: {}".format(self.file_date)) logger.debug("Acquisition Date: {}".format(self.acqdate))
def output_file(self, product, band, extension=None): if extension is None: extension = self.ext return os.path.join(config.get('wd'), product.name, product.get_band_file(band).rootname + extension)
def process(self, product, image, band): wd = os.path.join(config.get('wd'), product.name) self._output_file = self.output_file(product, band) self._tmp_stats = {} log.info('Start') # MGRS reframing for Landsat8 if product.sensor == 'L8': log.debug('{} {}'.format(config.getfloat('dx'), config.getfloat('dy'))) image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy')) # Resampling to 30m for S2 (HLS) elif product.sensor == 'S2': if not config.getboolean('hlsplus'): image = self._resample(image) else: # refine geometry # if config.getfloat('dx') > 0.3 or config.getfloat('dy') > 0.3: log.debug("{} {}".format(config.getfloat('dx'), config.getfloat('dy'))) image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy')) # matching for having QA stats if config.get('refImage'): # try to adapt resolution, changing end of reference filename refImage_path = config.get('refImage') if not os.path.exists(refImage_path): return image # open image ref imageref = S2L_ImageFile(refImage_path) # if refImage resolution does not fit if imageref.xRes != image.xRes: # new refImage filepath refImage_noext = os.path.splitext(refImage_path)[0] if refImage_noext.endswith(f"_{int(imageref.xRes)}m"): refImage_noext = refImage_noext[:-len(f"_{int(imageref.xRes)}m")] refImage_path = refImage_noext + f"_{int(image.xRes)}m.TIF" # compute (resample), or load if exists if not os.path.exists(refImage_path): log.info("Resampling of the reference image") # compute imageref = mgrs_framing.resample(imageref, image.xRes, refImage_path) # write for reuse imageref.write(DCmode=True, creation_options=['COMPRESS=LZW']) else: # or load if exists log.info("Change reference image to:" + refImage_path) imageref = S2L_ImageFile(refImage_path) # open mask mask = S2L_ImageFile(product.mtl.mask_filename) if config.getboolean('freeze_dx_dy'): # do Geometry Assessment only if required assess_geometry_bands = config.get('doAssessGeometry', default='').split(',') if product.sensor != 'S2': assess_geometry_bands = [product.reverse_bands_mapping.get(band) for band in assess_geometry_bands] if assess_geometry_bands and band in assess_geometry_bands: log.info("Geometry assessment for band %s" % band) # Coarse resolution of correlation grid (only for stats) self._matching(imageref, image, wd, mask) else: # Fine resolution of correlation grid (for accurate dx dy computation) dx, dy = self._matching(imageref, image, wd, mask) # save values for correction on bands config.set('dx', dx) config.set('dy', dy) log.info("Geometrical Offsets (DX/DY): {}m {}m".format(config.getfloat('dx'), config.getfloat('dy'))) # Append bands name to keys for key, item in self._tmp_stats.items(): if config.get('reference_band') != band: self._tmp_stats[key+'_{}'.format(band)] = self._tmp_stats.pop(key) metadata.qi.update(self._tmp_stats) log.info('End') return image
def process(self, product, image, band): log.info('Start') if not config.getboolean('hlsplus'): log.warning( 'Skipping Data Fusion because doPackager and doPackagerL2F options are not activated' ) log.info('End') return image # save into file before processing (packager will need it) product.image30m[band] = image if band == 'B01': log.warning('Skipping Data Fusion for B01.') log.info('End') return image if len(self.reference_products) == 0: log.warning( 'Skipping Data Fusion. Reason: no S2 products available in the past' ) log.info('End') return image if not product.get_s2like_band(band): log.warning( 'Skipping Data Fusion. Reason: no S2 matching band for {}'. format(band)) log.info('End') return image # method selection predict_method = config.get('predict_method', 'predict') if len(self.reference_products) == 1: log.warning( 'Not enough Sentinel2 products for the predict (only one product). Using last S2 product as ref.' ) predict_method = 'composite' # general info band_s2 = product.get_s2like_band(band) image_file_L2F = self.reference_products[0].get_band_file(band_s2, plus=True) output_shape = (image_file_L2F.ySize, image_file_L2F.xSize) # method: prediction (from the 2 most recent S2 products) if predict_method == 'predict': # Use QA (product selection) to apply Composting : qa_mask = self._get_qa_band(output_shape) # predict array_L2H_predict, array_L2F_predict = self._predict( product, band_s2, qa_mask, output_shape) # save if config.getboolean('generate_intermediate_products'): self._save_as_image_file(image_file_L2F, qa_mask, product, band, '_FUSION_QA.TIF') self._save_as_image_file(image_file_L2F, array_L2H_predict, product, band, '_FUSION_L2H_PREDICT.TIF') self._save_as_image_file(image_file_L2F, array_L2F_predict, product, band, '_FUSION_L2F_PREDICT.TIF') # method: composite (most recent valid pixels from N products) elif predict_method == 'composite': # composite array_L2H_predict, array_L2F_predict = self._composite( product, band_s2, output_shape) # save if config.getboolean('generate_intermediate_products'): self._save_as_image_file(image_file_L2F, array_L2H_predict, product, band, '_FUSION_L2H_COMPO.TIF') self._save_as_image_file(image_file_L2F, array_L2F_predict, product, band, '_FUSION_L2F_COMPO.TIF') # method: unknown else: log.error( f'Unknown predict method: {predict_method}. Please check your configuration.' ) return None # fusion L8/S2 mask_filename = product.mtl.nodata_mask_filename array_out = self._fusion(image, array_L2H_predict, array_L2F_predict, mask_filename).astype(np.float32) image = self._save_as_image_file(image_file_L2F, array_out, product, band, '_FUSION_L2H_PREDICT.TIF') log.info('End') return image
def start_process(tile, products, args, start_date, end_date): update_configuration(args, tile) config.set('tile', tile) logger.debug("Processing tile {}".format(tile)) downloader = InputProductArchive(config) _products = downloader.get_products_from_urls( products, start_date, end_date, product_mode=args.operational_mode == 'product-mode') if args.no_run: logger.info("Tile: %s" % tile) if not _products: logger.info("No products found.") for product in _products: tile_message = f'[ Tile coverage = {product.tile_coverage:6.0f}% ]' \ if product.tile_coverage is not None else '' cloud_message = f'[ Cloud coverage = {product.cloud_cover:6.0f}% ]' \ if product.cloud_cover is not None else '' logger.info("%s %s %s" % (tile_message, cloud_message, product.path)) return for product in _products: _product = None if config.getboolean('use_sen2cor'): # Disable Atmospheric correction config.overload('doAtmcor=False') # Run sen2core logger.debug("<<< RUNNING SEN2CORE... >>>") sen2cor_command = os.path.abspath(config.get('sen2cor_path')) sen2cor_output_dir = os.path.join(config.get('wd'), 'sen2cor', os.path.basename(product.path)) if not os.path.exists(sen2cor_output_dir): os.makedirs(sen2cor_output_dir) try: subprocess.run([ 'python', sen2cor_command, product.path, "--output_dir", sen2cor_output_dir, "--work_dir", sen2cor_output_dir, "--sc_only" ], check=True) except subprocess.CalledProcessError as run_error: logger.error("An error occurred during the run of sen2cor") logger.error(run_error) continue # Read output product generated_product = next(os.walk(sen2cor_output_dir))[1] if len(generated_product) != 1: logger.error("Sen2Cor error: Cannot get output product") continue _product = product.reader( os.path.join(sen2cor_output_dir, generated_product[0])) if _product is None: _product = product.reader(product.path) # Update processing configuration config.set('productName', _product.name) config.set('sensor', _product.sensor) config.set('observation_date', _product.mtl.observation_date) config.set('relative_orbit', _product.mtl.relative_orbit) config.set('absolute_orbit', _product.mtl.absolute_orbit) config.set('mission', _product.mtl.mission) config.set('none_S2_product_for_fusion', False) # Disable Atmospheric correction for Level-2A products atmcor = config.get('doAtmcor') if _product.mtl.data_type in ('Level-2A', 'L2TP', 'L2A'): config.overload('s2_processing_level=LEVEL2A') logger.info( "Processing Level-2A product: Atmospheric correction is disabled." ) config.overload('doAtmcor=False') else: config.overload('s2_processing_level=LEVEL1C') process(_product, args) # Restore atmcor status config.overload(f'doAtmcor={atmcor}') del _product if len(_products) == 0: logger.error('No product for tile %s' % tile)
def process(product, args): """Launch process on product.""" bands = args.bands # create working directory and save conf (traceability) if not os.path.exists(os.path.join(config.get("wd"), product.name)): os.makedirs(os.path.join(config.get("wd"), product.name)) # displays logger.debug("{} {}".format(product.sensor, product.path)) # list of the blocks that are available list_of_blocks = tuple(S2L_config.PROC_BLOCKS.keys()) # copy MTL files in wd wd = os.path.join(config.get("wd"), product.name) # copy MTL files in final product shutil.copyfile( product.mtl.mtl_file_name, os.path.join(wd, os.path.basename(product.mtl.mtl_file_name))) if product.mtl.tile_metadata: shutil.copyfile( product.mtl.tile_metadata, os.path.join(wd, os.path.basename(product.mtl.tile_metadata))) # Angles extraction product.mtl.get_angle_images( os.path.join(config.get("wd"), product.name, 'tie_points.tif')) product.mtl.get_valid_pixel_mask( os.path.join(config.get("wd"), product.name, 'valid_pixel_mask.tif')) # !! Initialization of each block for block_name in list_of_blocks: get_module(block_name).initialize() # !! Pre processing !! # Run the preprocessing method of each block for block_name in list_of_blocks: generic_process_step(block_name, product, "preprocess") # !! Processing !! # save S2L_config file in wd config.savetofile( os.path.join(config.get('wd'), product.name, 'processing_start.cfg')) # For each band or a selection of bands: if bands is None: # get all bands bands = product.bands elif product.sensor != 'S2': bands = [ product.reverse_bands_mapping.get(band, band) for band in bands ] if args.parallelize_bands: # Multi processus params = [(product, band, list_of_blocks, config, mtd.metadata, PROCESS_INSTANCES) for band in bands] with Pool() as pool: results = pool.starmap(process_band, params) bands_filenames, packager_files, configs, updated_metadatas = zip( *results) if configs and configs[0].parser is not None: S2L_config.config = configs[0] if updated_metadatas: for updated_metadata in updated_metadatas: mtd.metadata.update(updated_metadata) for packager_file in packager_files: for process_instance in packager_file: PROCESS_INSTANCES[process_instance].images.update( packager_file[process_instance]) for band, filename in PROCESS_INSTANCES[ process_instance].images.items(): S2L_config.config.set('imageout_dir', os.path.dirname(filename)) S2L_config.config.set('imageout_' + band, os.path.basename(filename)) else: # Single processus bands_filenames = [] for band in bands: # process the band through each block bands_filenames.append( process_band(product, band, list_of_blocks, config, mtd.metadata)) # Save image path if bands_filenames == [None] * len(bands_filenames): logger.error("No valid band provided for input product.") logger.error("Valids band for products are: %s" % str(list(product.bands))) return # !! Post processing !! # Run the postprocessing method of each block for block_name in list_of_blocks: generic_process_step(block_name, product, "postprocess") # Clear metadata mtd.metadata.clear() # save S2L_config file in wd S2L_config.config.savetofile( os.path.join(S2L_config.config.get('wd'), product.name, 'processing_end.cfg'))
def process(product, bands): """Launch process on product.""" # create working directory and save conf (traceability) if not os.path.exists(os.path.join(config.get("wd"), product.name)): os.makedirs(os.path.join(config.get("wd"), product.name)) # displays logger.debug("{} {}".format(product.sensor, product.path)) # list of the blocks that are available list_of_blocks = S2L_config.PROC_BLOCKS.keys() # copy MTL files in wd wd = os.path.join(config.get("wd"), product.name) # copy MTL files in final product shutil.copyfile( product.mtl.mtl_file_name, os.path.join(wd, os.path.basename(product.mtl.mtl_file_name))) if product.mtl.tile_metadata: shutil.copyfile( product.mtl.tile_metadata, os.path.join(wd, os.path.basename(product.mtl.tile_metadata))) # Angles extraction product.mtl.get_angle_images( os.path.join(config.get("wd"), product.name, 'tie_points.tif')) product.mtl.get_valid_pixel_mask( os.path.join(config.get("wd"), product.name, 'valid_pixel_mask.tif')) # !! Initialization of each block for block_name in list_of_blocks: get_module(block_name).initialize() # !! Pre processing !! # Run the preprocessing method of each block for block_name in list_of_blocks: generic_process_step(block_name, product, "preprocess") # !! Processing !! # save S2L_config file in wd config.savetofile( os.path.join(config.get('wd'), product.name, 'processing_start.cfg')) # For each band or a selection of bands: if bands is None: # get all bands bands = product.bands elif product.sensor != 'S2': bands = [product.reverse_bands_mapping.get(band) for band in bands] bands_filenames = [] for band in bands: # process the band through each block bands_filenames.append(process_band(product, band, list_of_blocks)) # save image path if bands_filenames == [None] * len(bands_filenames): logger.error("No valid band provided for input product.") logger.error("Valids band for products are: %s" % str(list(product.bands))) return # !! Post processing !! # Run the postprocessing method of each block for block_name in list_of_blocks: generic_process_step(block_name, product, "postprocess") # Clear metadata metadata.clear() # save S2L_config file in wd config.savetofile( os.path.join(config.get('wd'), product.name, 'processing_end.cfg'))
def _manual_replaces(self, product): # GENERAL_INFO # ------------ copy_elements([ './General_Info/TILE_ID', './General_Info/DATASTRIP_ID', './General_Info/DOWNLINK_PRIORITY', './General_Info/SENSING_TIME', './General_Info/Archiving_Info' ], self.root_in, self.root_out, self.root_bb) if product.mtl.data_type == 'Level-1C' or 'L1' in product.mtl.data_type: l1c_tile_id = find_element_by_path( self.root_in, './General_Info/TILE_ID')[0].text l2a_tile_id = "NONE" else: l1c_tile_id = find_element_by_path( self.root_in, './General_Info/L1C_TILE_ID')[0].text l2a_tile_id = find_element_by_path( self.root_in, './General_Info/TILE_ID')[0].text tilecode = product.mtl.mgrs pdgs = metadata.hardcoded_values.get('PDGS', '9999') PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]]) AC = self.root_in.findall('.//ARCHIVING_CENTRE') AC = AC[0].text if AC else 'ZZZ' acqdate = dt.datetime.strftime(product.acqdate, '%Y%m%dT%H%M%S') tile_id = '_'.join([ product.sensor_name, 'OPER', 'MSI', 'L2{}'.format(self.H_F), AC, acqdate, 'A{}'.format(config.get('absolute_orbit')), tilecode, 'N{}'.format(PDGS) ]) change_elm(self.root_out, './General_Info/L1_TILE_ID', new_value=l1c_tile_id) change_elm(self.root_out, './General_Info/L2A_TILE_ID', new_value=l2a_tile_id) change_elm(self.root_out, './General_Info/TILE_ID', new_value=tile_id) # Geometric_info # --------------- copy_elements([ './Geometric_Info/Tile_Geocoding/HORIZONTAL_CS_NAME', './Geometric_Info/Tile_Geocoding/HORIZONTAL_CS_CODE' ], self.root_in, self.root_out, self.root_bb) g = loads(search_db(tilecode, search='UTM_WKT')) xMin = int(g.bounds[0]) yMin = int(g.bounds[1]) change_elm(self.root_out, './Geometric_Info/Tile_Geocoding/Geoposition/ULX', new_value=str(xMin)) change_elm(self.root_out, './Geometric_Info/Tile_Geocoding/Geoposition/ULY', new_value=str(yMin)) self._remove_children('./Geometric_Info/Tile_Angles', tag='Viewing_Incidence_Angles_Grids') angles_path = os.path.join( 'GRANULE', metadata.mtd.get('granule_{}_name'.format(self.H_F)), 'QI_DATA', metadata.mtd.get('ang_filename')) change_elm(self.root_out, './Geometric_Info/Tile_Angles/Acquisition_Angles_Filename', new_value=angles_path) rm_elm_with_tag(self.root_out, tag='Sun_Angles_Grid') rm_elm_with_tag(self.root_out, tag='Viewing_Incidence_Angle_Grid') copy_elements(['./Geometric_Info/Tile_Angles/Mean_Sun_Angle'], self.root_in, self.root_out, self.root_bb) copy_elements( ['./Geometric_Info/Tile_Angles/Mean_Viewing_Incidence_Angle_List'], self.root_in, self.root_out, self.root_bb) # Quality indicators info # ----------------------- self._remove_children('./Quality_Indicators_Info/Image_Content_QI') copy_children(self.root_in, './Quality_Indicators_Info/Image_Content_QI', self.root_out, './Quality_Indicators_Info/Image_Content_QI') # Replace masks with all existing self._remove_children('./Quality_Indicators_Info/Pixel_Level_QI', tag='MASK_FILENAME') for mask in metadata.mtd.get('masks_{}'.format(self.H_F)): create_child(self.root_out, './Quality_Indicators_Info/Pixel_Level_QI', tag=mask.get('tag'), text=mask.get('text'), attribs=mask.get('attribs')) msk_text = find_element_by_path( self.root_in, './Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME')[0].text ini_grn_name = re.search(r'GRANULE/(.*?)/QI_DATA', msk_text).group(1) elems = find_element_by_path( self.root_out, './Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME') for elem in elems: elem.text = elem.text.replace( ini_grn_name, metadata.mtd.get('granule_{}_name'.format(self.H_F))) rm_elm_with_tag(self.root_out, tag='PVI_FILENAME') rm_elm_with_tag(self.root_out, tag='QL_B12118A_FILENAME') rm_elm_with_tag(self.root_out, tag='QL_B432_FILENAME') # Get all created quicklooks (including PVI) for ql in metadata.mtd.get('quicklooks_{}'.format(self.H_F)): ql_path = re.search(r'GRANULE(.*)', ql).group() band_rootName = metadata.mtd.get(f'band_rootName_{self.H_F}') ql_name = re.search(r'{}_(.*)'.format(band_rootName), ql_path).group(1) create_child(self.root_out, './Quality_Indicators_Info', tag="{}_FILENAME".format( os.path.splitext(ql_name)[0]), text=ql_path)
def manual_replaces(self, product): # GENERAL_INFO # ------------ acqdate = dt.datetime.strftime(product.acqdate, '%Y-%m-%dT%H:%M:%S.%fZ') change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_START_TIME', new_value=acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_STOP_TIME', new_value=acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI', new_value=metadata.mtd.get('product_{}_name'.format(self.H_F))) change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL', new_value='Level-2{}'.format(self.H_F)) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE', new_value=f'{product.sensor}OLI2{self.H_F}') pdgs = config.get('PDGS', '9999') PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]]) change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS) generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[ :-3] + 'Z' # -3 to keep only 3 decimals change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time) change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SPACECRAFT_NAME', new_value=product.mtl.mission) change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/DATATAKE_SENSING_START', new_value=acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SENSING_ORBIT_NUMBER', new_value=config.get('relative_orbit')) self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule') for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))): adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0] create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', tag='IMAGE_FILE', text=adjusted_path) tile_id = generate_LS8_tile_id(product, self.H_F) change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=tile_id, attr_to_change='granuleIdentifier') change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat') if not config.getboolean('doSbaf'): # FIXME : get product image characteristics from origin sensor (LS8 here), # copying from another template fro example pass U = distance_variation_corr(product.acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U', new_value=str(U)) # Geometric_info # --------------- tilecode = product.mtl.mgrs if tilecode.startswith('T'): tilecode = tilecode[1:] footprint = search_db(tilecode, search='MGRS_REF') # adding back first element, to get a complete polygon fp = footprint.split(' ') footprint = ' '.join(fp + [fp[0], fp[1]]) chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint) # Auxiliary_Data_Info # ------------------- self.remove_children('./Auxiliary_Data_Info/GIPP_List', exceptions=['Input_Product_Info']) config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0] create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn, attribs={"version": pdgs, "type": "GIP_S2LIKE"}) # Quality_Indicators_Info # ----------------------- self.remove_children('./Quality_Indicators_Info', exceptions=['Input_Product_Info']) change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info', attr_to_change='type', new_value=product.mtl.mission) change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info', new_value=product.mtl.landsat_scene_id)
def postprocess(self, pd): """ Copy auxiliary files in the final output like mask, angle files Input product metadata file is also copied. :param pd: instance of S2L_Product class """ # output directory product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L( pd) tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series outdir = product_name product_path = os.path.join(tsdir, outdir) qi_dir = os.path.join(product_path, 'GRANULE', granule_compact_name, 'QI_DATA') # copy angles file outfile = "_".join([metadata.mtd.get('band_rootName_H'), 'ANG' ]) + '.TIF' metadata.mtd['ang_filename'] = outfile shutil.copyfile(pd.mtl.angles_file, os.path.join(qi_dir, outfile)) # copy mask files if "S2" in pd.sensor: tree_in = ElementTree.parse( pd.mtl.tile_metadata) # Tree of the input mtd (S2 MTD.xml) root_in = tree_in.getroot() mask_elements = find_element_by_path( root_in, './Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME') for element in mask_elements: mask_file = os.path.join(pd.path, element.text) if os.path.exists(mask_file): shutil.copyfile( mask_file, os.path.join(qi_dir, os.path.basename(mask_file))) metadata.mtd.get('masks_H').append({ "tag": "MASK_FILENAME", "attribs": element.attrib, "text": element.text }) # copy valid pixel mask outfile = "_".join( [metadata.mtd.get('band_rootName_H'), pd.sensor, 'MSK']) + '.TIF' fpath = os.path.join(qi_dir, outfile) metadata.mtd.get('masks_H').append({ "tag": "MASK_FILENAME", "attribs": { "type": "MSK_VALPIX" }, "text": os.path.relpath(fpath, product_path) }) if config.getboolean('COG'): img_object = S2L_ImageFile(pd.mtl.mask_filename, mode='r') img_object.write(filepath=fpath, COG=True, band='MASK') else: shutil.copyfile(pd.mtl.mask_filename, fpath) # QI directory qipath = os.path.join(tsdir, 'QI') if not os.path.exists(qipath): os.makedirs(qipath) # save config file in QI cfgname = "_".join([outdir, 'INFO']) + '.cfg' cfgpath = os.path.join(tsdir, 'QI', cfgname) config.savetofile(os.path.join(config.get('wd'), pd.name, cfgpath)) # save correl file in QI if os.path.exists( os.path.join(config.get('wd'), pd.name, 'correl_res.txt')): corrname = "_".join([outdir, 'CORREL']) + '.csv' corrpath = os.path.join(tsdir, 'QI', corrname) shutil.copy( os.path.join(config.get('wd'), pd.name, 'correl_res.txt'), corrpath) if len(self.images.keys()) > 1: # true color QL band_list = ["B04", "B03", "B02"] qlname = "_".join( [metadata.mtd.get('band_rootName_H'), 'QL', 'B432']) + '.jpg' qlpath = os.path.join(qi_dir, qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) metadata.mtd.get('quicklooks_H').append(qlpath) # false color QL band_list = ["B12", "B11", "B8A"] qlname = "_".join([ metadata.mtd.get('band_rootName_H'), 'QL', 'B12118A' ]) + '.jpg' qlpath = os.path.join(qi_dir, qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) metadata.mtd.get('quicklooks_H').append(qlpath) else: # grayscale QL band_list = list(self.images.keys()) qlname = "_".join([ metadata.mtd.get('band_rootName_H'), 'QL', band_list[0] ]) + '.jpg' qlpath = os.path.join(qi_dir, qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) metadata.mtd.get('quicklooks_H').append(qlpath) # PVI band_list = ["B04", "B03", "B02"] pvi_filename = "_".join([metadata.mtd.get('band_rootName_H'), 'PVI' ]) + '.TIF' qlpath = os.path.join(qi_dir, pvi_filename) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95), xRes=320, yRes=320, creationOptions=['COMPRESS=LZW'], format='GTIFF') metadata.mtd.get('quicklooks_H').append(qlpath) # Clear images as packager is the last process self.images.clear() # Write QI report as XML bb_QI_path = metadata.hardcoded_values.get('bb_QIH_path') out_QI_path = os.path.join(qi_dir, 'L2H_QI_Report.xml') in_QI_path = glob.glob( os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA', 'L2A_QI_Report.xml')) log.info( 'QI report for input product found : {} (searched at {})'.format( len(in_QI_path) != 0, os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA', 'L2A_QI_Report.xml'))) in_QI_path = in_QI_path[0] if len(in_QI_path) != 0 else None Qi_Writer = QiWriter(bb_QI_path, outfile=out_QI_path, init_QI_path=in_QI_path, H_F='H') Qi_Writer._manual_replaces(pd) Qi_Writer.write(pretty_print=True) # TODO UNCOMMENT BELOW FOR XSD CHECK product_QI_xsd = metadata.hardcoded_values.get('product_QIH_xsd') log.info('QI Report is valid : {}'.format( Qi_Writer.validate_schema(product_QI_xsd, out_QI_path))) # Write product MTD bb_S2_product = metadata.hardcoded_values.get('bb_S2H_product') bb_L8_product = metadata.hardcoded_values.get('bb_L8H_product') product_mtd_path = 'MTD_{}L2H.xml'.format( pd.mtl.sensor[0:3]) # MSI / OLI/ OLI_TIRS product_MTD_outpath = os.path.join(tsdir, product_name, product_mtd_path) mtd_pd_writer = MTD_writer_S2(bb_S2_product, pd.mtl.mtl_file_name, H_F='H') if pd.sensor == 'S2' \ else MTD_writer_LS8(bb_L8_product, H_F='H') mtd_pd_writer._manual_replaces(pd) mtd_pd_writer.write(product_MTD_outpath, pretty_print=True) # TODO UNCOMMENT BELOW FOR XSD CHECK # product_mtd_xsd = metadata.hardcoded_values.get('product_mtd_xsd') # log.info('Product MTD is valid : {}'.format(mtd_pd_writer.validate_schema(product_mtd_xsd, # product_MTD_outpath))) # Write tile MTD bb_S2_tile = metadata.hardcoded_values.get('bb_S2H_tile') bb_L8_tile = metadata.hardcoded_values.get('bb_L8H_tile') tile_mtd_path = 'MTD_TL_L2H.xml' tile_MTD_outpath = os.path.join(product_path, 'GRANULE', granule_compact_name, tile_mtd_path) mtd_tl_writer = MTD_tile_writer_S2(bb_S2_tile, pd.mtl.tile_metadata, H_F='H') if pd.sensor == 'S2' \ else MTD_tile_writer_LS8(bb_L8_tile, H_F='H') mtd_tl_writer._manual_replaces(pd) mtd_tl_writer.write(tile_MTD_outpath, pretty_print=True)
def manual_replaces(self, product): # GENERAL_INFO # ------------ elements_to_copy = ['./General_Info/Product_Info/Datatake', './General_Info/Product_Info/PRODUCT_START_TIME', './General_Info/Product_Info/PRODUCT_STOP_TIME', './General_Info/Product_Info/Query_Options', ] copy_elements(elements_to_copy, self.root_in, self.root_out, self.root_bb) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI', new_value=metadata.mtd.get('product_{}_name'.format(self.H_F))) change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL', new_value='Level-2{}'.format(self.H_F)) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE', new_value='S2MSI2{}'.format(self.H_F)) pdgs = config.get('PDGS', '9999') PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]]) AC = self.root_in.findall('.//ARCHIVING_CENTRE') if AC: metadata.mtd['S2_AC'] = AC[0].text change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS) generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[ :-3] + 'Z' # -3 to keep only 3 decimals change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time) self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule') for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))): adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0] create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', tag='IMAGE_FILE', text=adjusted_path) grnl_id = \ find_element_by_path(self.root_in, './General_Info/Product_Info/Product_Organisation/Granule_List/Granule') if grnl_id: change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=generate_S2_tile_id(product, self.H_F, metadata.mtd['S2_AC']), attr_to_change='granuleIdentifier') change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat') else: pass # Fixme # If Sbaf is done, we keep the values inside the backbone (S2A values) if not config.getboolean('doSbaf'): # copy_elements(['./General_Info/Product_Image_Characteristics/Special_Values', # './General_Info/Product_Image_Characteristics/Image_Display_Order', # './General_Info/Product_Image_Characteristics/Reflectance_Conversion', # './General_Info/Product_Image_Characteristics/Spectral_Information_List'], # self.root_in, self.root_out, self.root_bb) pass # FIXME : get product image characteristics from origin sensor (S2 here), # copying from another template for example (see commented lines above) copy_elements(['./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U'], self.root_in, self.root_out) # Geometric_info # --------------- tilecode = product.mtl.mgrs if tilecode.startswith('T'): tilecode = tilecode[1:] footprint = search_db(tilecode, search='MGRS_REF') # adding back first element, to get a complete polygon fp = footprint.split(' ') footprint = ' '.join(fp + [fp[0], fp[1]]) chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint) copy_elements(['./Geometric_Info/Coordinate_Reference_System'], self.root_in, self.root_out, self.root_bb) # Auxiliary_Data_Info # ------------------- self.remove_children('./Auxiliary_Data_Info/GIPP_List') copy_children(self.root_in, './Auxiliary_Data_Info/GIPP_List', self.root_out, './Auxiliary_Data_Info/GIPP_List') config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0] create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn, attribs={"version": pdgs, "type": "GIP_S2LIKE"}) for tag in ['PRODUCTION_DEM_TYPE', 'IERS_BULLETIN_FILENAME', 'ECMWF_DATA_REF', 'SNOW_CLIMATOLOGY_MAP', 'ESACCI_WaterBodies_Map', 'ESACCI_LandCover_Map', 'ESACCI_SnowCondition_Map_Dir']: elem = find_element_by_path(self.root_in, './Auxiliary_Data_Info/' + tag) if len(elem) != 0: new_value = elem[0].text else: new_value = "NONE" change_elm(self.root_out, rpath='./Auxiliary_Data_Info/' + tag, new_value=new_value) # Fill GRI_List gri_elems = self.root_in.findall('.//GRI_FILENAME') for gri_elm in gri_elems: create_child(self.root_out, './Auxiliary_Data_Info/GRI_List', tag="GRI_FILENAME", text=gri_elm.text) # Quality_Indicators_Info # ----------------------- copy_elements(['./Quality_Indicators_Info'], self.root_in, self.root_out, self.root_bb)
def write(self, creation_options=None, DCmode=False, filepath=None, nodata_value=0, COG:bool=False, band:str=None): """ write to file :param creation_options: gdal create options :param DCmode: if true, the type is kept. Otherwise float are converted to int16 using offset and gain from config :param filepath: :param COG : whether to create COG output format or not :param band : provide information about the band, to set the overviews downsampling algorithm. band can be 'MASK', 'QA', or None for all others """ if creation_options is None: creation_options = [] # if filepath is override if filepath is None: filepath = self.filepath # Ensure that file extension is tiff if not filepath.endswith('.tif'): filepath = os.path.splitext(filepath)[0] + ".TIF" # check if directory to create if not os.path.exists(self.dirpath): os.makedirs(self.dirpath) # write with gdal etype = gdal.GetDataTypeByName(self.array.dtype.name) if self.array.dtype.name.endswith('int8'): # work around to GDT_Unknown etype = 1 elif 'float' in self.array.dtype.name and not DCmode: # float to UInt16 etype = gdal.GDT_UInt16 # Update image attributes self.setFilePath(filepath) # Create folders hierarchy if needed: if not os.path.exists(self.dirpath): os.makedirs(self.dirpath) if not COG: driver = gdal.GetDriverByName('GTiff') dst_ds = driver.Create(self.filepath, xsize=self.xSize, ysize=self.ySize, bands=1, eType=etype, options=creation_options) else: driver = gdal.GetDriverByName('MEM') dst_ds = driver.Create('', xsize=self.xSize, ysize=self.ySize, bands=1, eType=etype) dst_ds.SetProjection(self.projection) geotranform = (self.xMin, self.xRes, 0, self.yMax, 0, self.yRes) log.debug(geotranform) dst_ds.SetGeoTransform(geotranform) if 'float' in self.array.dtype.name and not DCmode: # float to UInt16 with scaling factor of 10000 offset = float(config.get('offset')) gain = float(config.get('gain')) dst_ds.GetRasterBand(1).WriteArray(((offset + self.array).clip(min=0) * gain).astype(np.uint16)) # set GTiff metadata dst_ds.GetRasterBand(1).SetScale(1 / gain) dst_ds.GetRasterBand(1).SetOffset(offset) else: dst_ds.GetRasterBand(1).WriteArray(self.array) if nodata_value: dst_ds.GetRasterBand(1).SetNoDataValue(nodata_value) if COG: resampling_algo = config.get('resampling_algo_MASK') if band in ['QA', 'MASK'] else config.get('resampling_algo') downsampling_levels = config.get('downsampling_levels_{}'.format(int(self.xRes)), config.get('downsampling_levels_10')) # If the res isn't [10, 15, 20, 30, 60], consider it as 30 downsampling_levels = [int(x) for x in downsampling_levels.split(" ")] # Overloading creation options creation_options = [opt for opt in creation_options if opt.split("=")[0] not in ['TILED', 'COMPRESS', 'INTERLEAVE', 'BLOCKYSIZE', 'BLOCKXSIZE', 'PREDICTOR']] + \ \ ['TILED=YES', "COMPRESS=" + config.get('compression'), "INTERLEAVE=" + config.get('interleave'), "BLOCKXSIZE=" + str(config.get('internal_tiling')), "BLOCKYSIZE=" + str(config.get('internal_tiling')), "PREDICTOR=" + str(config.get('predictor'))] # FIXME : in this gdal version, driver GTiff does not support creation option GDAL_TIFF_OVR_BLOCKSIZE to set the # FIXME : internal overview blocksize ; however it is set at 128 as default, as requested here # Source : https://gdal.org/drivers/raster/gtiff.html#raster-gtiff # add in options : "GDAL_TIFF_OVR_BLOCKSIZE=" + str(config.get('internal_overviews')) dst_ds.BuildOverviews(resampling_algo, downsampling_levels) driver_Gtiff = gdal.GetDriverByName('GTiff') data_set2 = driver_Gtiff.CreateCopy(self.filepath, dst_ds, options=creation_options + ['COPY_SRC_OVERVIEWS=YES']) data_set2 = None dst_ds.FlushCache() dst_ds = None log.info('Written: {}'.format(self.filepath))
def postprocess(self, pd): """ Copy auxiliary files in the final output like mask, angle files Input product metadata file is also copied. :param pd: instance of S2L_Product class """ # output directory outdir, tilecode = self.base_path(pd) tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series # copy MTL files in final product outfile = os.path.basename(pd.mtl.mtl_file_name) shutil.copyfile(pd.mtl.mtl_file_name, os.path.join(tsdir, outdir, outfile)) if pd.mtl.tile_metadata: outfile = os.path.basename(pd.mtl.tile_metadata) shutil.copyfile(pd.mtl.tile_metadata, os.path.join(tsdir, outdir, outfile)) # copy angles file outfile = "_".join([outdir, 'ANG']) + '.TIF' shutil.copyfile(pd.mtl.angles_file, os.path.join(tsdir, outdir, outfile)) # copy valid pixel mask outfile = "_".join([outdir, 'MSK']) + '.TIF' shutil.copyfile(pd.mtl.mask_filename, os.path.join(tsdir, outdir, outfile)) # QI directory qipath = os.path.join(tsdir, 'QI') if not os.path.exists(qipath): os.makedirs(qipath) # save config file in QI cfgname = "_".join([outdir, 'INFO']) + '.cfg' cfgpath = os.path.join(tsdir, 'QI', cfgname) config.savetofile(os.path.join(config.get('wd'), pd.name, cfgpath)) # save correl file in QI if os.path.exists(os.path.join(config.get('wd'), pd.name, 'correl_res.txt')): corrname = "_".join([outdir, 'CORREL']) + '.csv' corrpath = os.path.join(tsdir, 'QI', corrname) shutil.copy(os.path.join(config.get('wd'), pd.name, 'correl_res.txt'), corrpath) if len(self.images.keys()) > 1: # true color QL band_list = ["B04", "B03", "B02"] qlname = "_".join([outdir, 'QL', 'B432']) + '.jpg' qlpath = os.path.join(tsdir, 'QI', 'QL_B432', qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) # false color QL band_list = ["B12", "B11", "B8A"] qlname = "_".join([outdir, 'QL', 'B12118A']) + '.jpg' qlpath = os.path.join(tsdir, 'QI', 'QL_B12118A', qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) else: # grayscale QL band_list = list(self.images.keys()) qlname = "_".join([outdir, 'QL', band_list[0]]) + '.jpg' qlpath = os.path.join(tsdir, 'QI', f'QL_{band_list[0]}', qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) # Clear images as packager is the last process self.images.clear()