def update_configuration(args, tile=None): # init S2L_config and save to wd if not config.initialize(args.S2L_configfile): return if args.confParams is not None: config.overload(args.confParams) use_pid = False if use_pid: output_folder = str(os.getpid()) else: date_now = datetime.datetime.now().strftime('%Y%m%dT_%H%M%S') output_folder = f'{"" if args.no_log_date else f"{date_now}_"}{compute_config_hash(args, config)}' config.set('wd', os.path.join(args.wd, output_folder)) references_map_file = config.get('references_map') if args.refImage: config.set('refImage', args.refImage) elif references_map_file and tile: # load dataset with open(references_map_file) as j: references_map = json.load(j) config.set('refImage', references_map.get(tile)) else: config.set('refImage', None) config.set( 'hlsplus', config.getboolean('doPackager') or config.getboolean('doPackagerL2F')) config.set('debug', args.debug) config.set('generate_intermediate_products', args.generate_intermediate_products) if hasattr(args, 'l2a'): config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def update_configuration(args, tile=None): # init S2L_config and save to wd if not config.initialize(args.S2L_configfile): return if args.confParams is not None: config.overload(args.confParams) config.set('wd', os.path.join(args.wd, str(os.getpid()))) references_map_file = config.get('references_map') if args.refImage: config.set('refImage', args.refImage) elif references_map_file and tile: # load dataset with open(references_map_file) as j: references_map = json.load(j) config.set('refImage', references_map.get(tile)) else: config.set('refImage', None) config.set( 'hlsplus', config.getboolean('doPackager') or config.getboolean('doPackagerL2F')) config.set('debug', args.debug) config.set('generate_intermediate_products', args.generate_intermediate_products) if hasattr(args, 'l2a'): config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def generic_process_step(blockname, pd, process_step): """From the name of the block, import the module, get the class, create object from class, run the process step method of object. This supposes that there all the names are the same (e.g. S2L_GeometryKLT) :param blockname: The block to process :param pd: :param process_step: The step to process :return: """ # check if block is switch ON if not config.getboolean('do' + blockname.split('_')[-1]): return # check if block is applicable to the sensor (L8 or S2) if pd.sensor not in S2L_config.PROC_BLOCKS[blockname]['applicability']: return class_instance = get_module(blockname) # create object and run preprocess if method exists! processus = getattr(class_instance, process_step, None) if processus is not None: return processus(pd)
def process(self, product, image, band): log.info('Start') # init to None offset = None slope = None if product.mtl.mission == "Sentinel-2A": # skip for S2A metadata.qi['SBAF_COEFFICIENT_{}'.format(band)] = 1 metadata.qi['SBAF_OFFSET_{}'.format(band)] = 0 return image elif product.mtl.mission == "Sentinel-2B": # S2B => L8 + L8 => S2A adj_coef1 = self.getOLILikeCoef("Sentinel-2B") adj_coef2 = self.getSen2likeCoef("LANDSAT_8") band_sbaf1 = band band_sbaf2 = Landsat8Product.get_band_from_s2(band) if band_sbaf1 in adj_coef1 and band_sbaf2 in adj_coef2: slope1, offset1 = adj_coef1[band_sbaf1]['coef'] slope2, offset2 = adj_coef2[band_sbaf2]['coef'] # merging coefficients slope = slope2 * slope1 offset = slope2 * offset1 + offset2 else: log.error("No Sbaf coefficient defined for {}".format(band)) elif product.mtl.mission == "LANDSAT_8": # L8 => S2A band_sbaf = band adj_coef = self.getSen2likeCoef("LANDSAT_8") if band_sbaf in adj_coef: slope, offset = adj_coef[band_sbaf]['coef'] else: metadata.qi['SBAF_COEFFICIENT_{}'.format(band)] = 1 metadata.qi['SBAF_OFFSET_{}'.format(band)] = 0 log.error("No Sbaf coefficient defined for {}".format(band)) return image metadata.qi['SBAF_COEFFICIENT_{}'.format(band)] = slope metadata.qi['SBAF_OFFSET_{}'.format(band)] = offset # Apply SBAF if offset is not None and slope is not None: log.debug("Applying SBAF") new = image.array new = new * slope + offset # transfer no data new[image.array == 0] = 0 # Format Output : duplicate, link to product as parameter image = image.duplicate(self.output_file(product, band), array=new.astype(np.float32)) if config.getboolean('generate_intermediate_products'): image.write(creation_options=['COMPRESS=LZW']) log.info('End') return image
def _save_as_image_file(self, image_template, array, product, band, extension): path = os.path.join(config.get('wd'), product.name, product.get_band_file(band).rootname + extension) image_file = image_template.duplicate(path, array=array) if config.getboolean('generate_intermediate_products'): image_file.write(creation_options=['COMPRESS=LZW']) return image_file
def process(self, pd, image, band): """ Write final product in the archive directory 'archive_dir' is defined in config.ini file Naming convention from Design Document :param pd: instance of S2L_Product class :param image: input instance of S2L_ImageFile class :param band: band being processed :return: output instance of instance of S2L_ImageFile class """ # TODO : add production date? # /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR res = image.xRes product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L( pd) sensor = pd.sensor_name relative_orbit = config.get('relative_orbit') native = band in ['B08', 'B10', 'B11'] if sensor == 'LS8' else \ band in ['B05', 'B06', 'B07', 'B08'] s2_band = pd.get_s2like_band(band) if not native: band = s2_band band_rootName = "_".join([ 'L2F', 'T' + tilecode, datatake_sensing_start, sensor, 'R{:0>3}'.format(relative_orbit) ]) metadata.mtd['band_rootName_F'] = band_rootName outfile = "_".join([band_rootName, band, '{}m'.format(int(res)) ]) + '.TIF' # Naming convention from Sentinel-2-Products-Specification-Document (p294) tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series newpath = self.band_path(tsdir, product_name, granule_compact_name, outfile, native=native) COG = config.getboolean('COG') log.debug('New: ' + newpath) image.write(creation_options=['COMPRESS=LZW'], filepath=newpath, COG=COG, band=band) metadata.mtd.get('bands_path_F').append(newpath) # declare output internally self.images[s2_band] = image.filepath # declare output in config file config.set('imageout_dir', image.dirpath) config.set('imageout_' + band, image.filename) return image
def reframe(self, image, product, band=None, dtype=None): # Add margin margin = int(config.get('reframe_margin', self.margin)) log.debug(f"Using {margin} as margin.") product_image = mgrs_framing.reframe(image, self.tile, filepath_out=self.output_file(product, band, image, "_PREREFRAMED"), order=0, margin=margin, dtype=dtype, compute_offsets=True) if config.getboolean('generate_intermediate_products'): product_image.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count return product_image
def generic_process_band(blockname, pd, image, band): """ from the name of the block, import the module, get the class, create object from class, run the main method of object. This supposes that there all the names are the same (e.g. S2L_GeometryKLT) """ # check if block is switch ON logger.debug(config.getboolean('do' + blockname.split('_')[-1])) if not config.getboolean('do' + blockname.split('_')[-1]): return image, None # check if block is applicable to the sensor (L8, L9 or S2) if pd.sensor not in S2L_config.PROC_BLOCKS[blockname]['applicability']: return image, None class_instance = get_module(blockname) # create object and run it! return class_instance.process(pd, image, band), class_instance
def _reframe(self, product, imagein, dx=0., dy=0.): log.info('MGRS Framing: Start...') # reframe on MGRS imageout = mgrs_framing.reframe(imagein, product.mtl.mgrs, self._output_file, dx, dy, dtype=np.float32) # display if config.getboolean('generate_intermediate_products'): imageout.write(DCmode=True) # digital count log.info('MGRS Framing: End') return imageout
def process(self, product, image, band): log.info('Start') # SMAC correction extent = image.getCorners(outEPSG=4326) array_in = image.array array_out = smac_correction(product, array_in, extent, band) image = image.duplicate(self.output_file(product, band), array_out) if config.getboolean('generate_intermediate_products'): image.write(creation_options=['COMPRESS=LZW']) log.info('End') return image
def _resample(self, imagein): # display log.info('Resampling to 30m: Start...') # resampling imageout = mgrs_framing.resample(imagein, 30, self._output_file) # display if config.getboolean('generate_intermediate_products'): imageout.write(DCmode=True) # digital count log.info('Resampling to 30m: End') return imageout
def process(self, pd, image, band): """ Write final product in the archive directory 'archive_dir' is defined in config.ini file Naming convention from Design Document :param pd: instance of S2L_Product class :param image: input instance of S2L_ImageFile class :param band: band being processed :return: outpu t instance of instance of S2L_ImageFile class """ # TODO : add production date? # TODO : change L8 band numbers to S2 numbers convention? # /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR res = image.xRes outdir, tilecode = self.base_path(pd) outfile = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF' tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series newpath = os.path.join(tsdir, outdir, outfile) log.debug('New: ' + newpath) image.write(creation_options=['COMPRESS=LZW'], filepath=newpath) # declare output internally self.images[band] = image.filepath # declare output in config file config.set('imageout_dir', image.dirpath) config.set('imageout_' + band, image.filename) if config.getboolean('hlsplus'): res = 30 outfile_30m = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF' newpath_30m = os.path.join(tsdir, outdir, outfile_30m) if pd.sensor == 'S2': # create 30m band as well # resampling log.info('Resampling to 30m: Start...') image_30m = mgrs_framing.resample(S2L_ImageFile(newpath), res, newpath_30m) image_30m.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count log.info('Resampling to 30m: End') if pd.sensor == 'L8' and band in pd.image30m: # copy 30m band as well # write pd.image30m[band].write(creation_options=['COMPRESS=LZW'], filepath=newpath_30m) del pd.image30m[band] return image
def _nbar(self, product, image, band): # Get BRDF coefficients brdf_coef_set = get_brdf_coefficient(product, band) log.debug('BRDF Coefficient Set :{}'.format(brdf_coef_set)) CMATRIX = normalized_brdf(self.KVOL_NORM, self.KGEO_NORM, self.KVOL_INPUT, self.KGEO_INPUT, brdf_coef_set) IM1 = image.array U = IM1 >= 0 if config.getboolean('debug'): out_stat(IM1[U]) CMATRIX_full = resize(CMATRIX, IM1.shape) IM = CMATRIX_full U = IM >= 0 if config.getboolean('debug'): out_stat(IM[U]) OUT = CMATRIX_full * IM1 OUT[IM1 <= 0] = 0 return OUT
def process(self, product, image, band): log.info('Start') # convert to TOA (gain + offset) array_in = image.array array_out = convert_to_reflectance_from_reflectance_cal_product( product.mtl, array_in, band) image = image.duplicate(self.output_file(product, band), array=array_out) if config.getboolean('generate_intermediate_products'): image.write(creation_options=['COMPRESS=LZW']) log.info('End') return image
def preprocess(self, product): self.get_new_product(product) if self.new_product is None: return # Stitch validity mask and angles product_validity_masks = [] product_nodata_masks = [] product_angles = [] for _product in [product, self.new_product.reader(self.new_product.path)]: is_mask_valid = True # Validity mask if _product.mtl.mask_filename is None: is_mask_valid = _product.mtl.get_valid_pixel_mask( os.path.join(config.get("wd"), _product.name, 'valid_pixel_mask.tif')) if is_mask_valid: product_validity_masks.append(self.reframe(S2L_ImageFile(_product.mtl.mask_filename), _product)) product_nodata_masks.append(self.reframe(S2L_ImageFile(_product.mtl.nodata_mask_filename), _product)) # Angles if _product.mtl.angles_file is None: _product.mtl.get_angle_images(os.path.join(config.get("wd"), _product.name, 'tie_points.tif')) filepath_out = os.path.join(config.get('wd'), _product.name, 'tie_points_PREREFRAMED.TIF') mgrs_framing.reframeMulti(_product.mtl.angles_file, self.tile, filepath_out=filepath_out, order=0) product_angles.append(filepath_out) if None not in product_validity_masks: stitched_mask = self.stitch(product, product_validity_masks[0], product_validity_masks[1]) stitched_mask.write(creation_options=['COMPRESS=LZW']) product.mtl.mask_filename = stitched_mask.filepath if None not in product_nodata_masks: stitched_mask = self.stitch(product, product_nodata_masks[0], product_nodata_masks[1]) stitched_mask.write(creation_options=['COMPRESS=LZW']) product.mtl.nodata_mask_filename = stitched_mask.filepath stitched_angles = self.stitch_multi(product, product_angles[0], product_angles[1]) product.mtl.angles_file = stitched_angles # Stitch reference band (needed by geometry module) band = config.get('reference_band', 'B04') if config.getboolean('doMatchingCorrection') and config.get('refImage'): image = product.get_band_file(band) self.process(product, image, band)
def preprocess(self, product): # reinit dx/dy config.set('dx', 0) config.set('dy', 0) if product.sensor != 'S2': # Reframe angles and masks filepath_out = os.path.join(config.get('wd'), product.name, 'tie_points_REFRAMED.TIF') mgrs_framing.reframeMulti(product.mtl.angles_file, product.mtl.mgrs, filepath_out, config.getfloat('dx'), config.getfloat('dy'), order=0) product.mtl.angles_file = filepath_out # Reframe mask if product.mtl.mask_filename: filepath_out = os.path.join(config.get('wd'), product.name, 'valid_pixel_mask_REFRAMED.TIF') image = S2L_ImageFile(product.mtl.mask_filename) imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'), config.getfloat('dy'), order=0) imageout.write(creation_options=['COMPRESS=LZW']) product.mtl.mask_filename = filepath_out # Reframe nodata mask if product.mtl.nodata_mask_filename: filepath_out = os.path.join(config.get('wd'), product.name, 'nodata_pixel_mask_REFRAMED.TIF') image = S2L_ImageFile(product.mtl.nodata_mask_filename) imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'), config.getfloat('dy'), order=0) imageout.write(creation_options=['COMPRESS=LZW']) product.mtl.nodata_mask_filename = filepath_out # Matching for dx/dy correction? band = config.get('reference_band', 'B04') if config.getboolean('doMatchingCorrection') and config.get('refImage'): config.set('freeze_dx_dy', False) image = product.get_band_file(band) self.process(product, image, band) # goal is to feed dx, dy in config config.set('freeze_dx_dy', True) metadata.qi.update({'COREGISTRATION_BEFORE_CORRECTION': self._tmp_stats.get('MEAN'.format(band))})
def process(self, product, image, band): log.info('Start') # coeff for this band? if get_brdf_coefficient(product, band) is None: log.warning('No BRDF coefficient for {}'.format(band)) image_out = image else: # Compute Kernels self._computeKernels(product, band) # NBAR correction OUT = self._nbar(product, image, band) # Format Output : duplicate, link to product as parameter image_out = image.duplicate(self.output_file(product, band), array=OUT.astype(np.float32)) if config.getboolean('generate_intermediate_products'): image_out.write(creation_options=['COMPRESS=LZW']) log.info('End') return image_out
def manual_replaces(self, product): # GENERAL_INFO # ------------ acqdate = dt.datetime.strftime(product.acqdate, '%Y-%m-%dT%H:%M:%S.%fZ') change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_START_TIME', new_value=acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_STOP_TIME', new_value=acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI', new_value=metadata.mtd.get('product_{}_name'.format(self.H_F))) change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL', new_value='Level-2{}'.format(self.H_F)) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE', new_value=f'{product.sensor}OLI2{self.H_F}') pdgs = config.get('PDGS', '9999') PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]]) change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS) generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[ :-3] + 'Z' # -3 to keep only 3 decimals change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time) change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SPACECRAFT_NAME', new_value=product.mtl.mission) change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/DATATAKE_SENSING_START', new_value=acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SENSING_ORBIT_NUMBER', new_value=config.get('relative_orbit')) self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule') for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))): adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0] create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', tag='IMAGE_FILE', text=adjusted_path) tile_id = generate_LS8_tile_id(product, self.H_F) change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=tile_id, attr_to_change='granuleIdentifier') change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat') if not config.getboolean('doSbaf'): # FIXME : get product image characteristics from origin sensor (LS8 here), # copying from another template fro example pass U = distance_variation_corr(product.acqdate) change_elm(self.root_out, rpath='./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U', new_value=str(U)) # Geometric_info # --------------- tilecode = product.mtl.mgrs if tilecode.startswith('T'): tilecode = tilecode[1:] footprint = search_db(tilecode, search='MGRS_REF') # adding back first element, to get a complete polygon fp = footprint.split(' ') footprint = ' '.join(fp + [fp[0], fp[1]]) chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint) # Auxiliary_Data_Info # ------------------- self.remove_children('./Auxiliary_Data_Info/GIPP_List', exceptions=['Input_Product_Info']) config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0] create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn, attribs={"version": pdgs, "type": "GIP_S2LIKE"}) # Quality_Indicators_Info # ----------------------- self.remove_children('./Quality_Indicators_Info', exceptions=['Input_Product_Info']) change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info', attr_to_change='type', new_value=product.mtl.mission) change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info', new_value=product.mtl.landsat_scene_id)
def _computeKernels(self, product, band=None): lat = product.mtl.get_scene_center_coordinates()[1] scene_center_latitude = lat theta_s = get_mean_sun_angle(scene_center_latitude) metadata.qi['CONSTANT_SOLAR_ZENITH_ANGLE'] = theta_s log.debug('theta_s: {}'.format(theta_s)) # Read TP , unit = degree, scale=100 src_ds = gdal.Open(product.mtl.angles_file) nBands = src_ds.RasterCount if nBands == 4: # VAA, VZA, SAA, SZA VAA = src_ds.GetRasterBand(1).ReadAsArray().astype( np.float32) / 100.0 VZA = src_ds.GetRasterBand(2).ReadAsArray().astype( np.float32) / 100.0 SAA = src_ds.GetRasterBand(3).ReadAsArray().astype( np.float32) / 100.0 SZA = src_ds.GetRasterBand(4).ReadAsArray().astype( np.float32) / 100.0 else: # VAA for each band, VZA for each band, SAA, SZZ angle_band_index = product.get_angles_band_index(band) VAA = src_ds.GetRasterBand(angle_band_index + 1).ReadAsArray().astype( np.float32) / 100.0 VZA = src_ds.GetRasterBand(13 + angle_band_index + 1).ReadAsArray().astype( np.float32) / 100.0 SAA = src_ds.GetRasterBand(nBands - 1).ReadAsArray().astype( np.float32) / 100.0 SZA = src_ds.GetRasterBand(nBands).ReadAsArray().astype( np.float32) / 100.0 # close src_ds = None metadata.qi['MEAN_DELTA_AZIMUTH'] = np.mean(SAA - VAA) % 360 if config.getboolean('debug'): out_stat(VAA, 'VAA') out_stat(VZA, 'VZA') out_stat(SAA, 'SAA') out_stat(SZA, 'SZA') # Prepare KGEO Input log.debug( '-------------------- INPUT --------------------------------------' ) log.debug( '---- ---------KGEO INPUT COMPUTATION ------------------------------' ) self.KGEO_INPUT = li_sparse_kernel(SZA, VZA, SAA - VAA) # Prepare KVOL Input : log.debug( '------------- KVOL INPUT COMPUTATION ------------------------------' ) self.KVOL_INPUT = ross_thick(SZA, VZA, SAA - VAA) # Prepare KGEO Norm : SZA_NORM = np.ones(VAA.shape) * theta_s VZA_NORM = np.zeros(VAA.shape) DPHI_NORM = np.zeros(VAA.shape) log.debug( '-------------------NORM-------------------------------------------' ) log.debug( '------------- KGEO NORM COMPUTATION ------------------------------' ) self.KGEO_NORM = li_sparse_kernel(SZA_NORM, VZA_NORM, DPHI_NORM) log.debug('---- KVOL NORM COMPUTATION ---') self.KVOL_NORM = ross_thick(SZA_NORM, VZA_NORM, DPHI_NORM) log.debug( '------------------------------------------------------------------' ) log.debug( '--------------- KGEO INPUT STAT-----------------------------------' ) log.debug('---- KGEO INPUT ---') if config.getboolean('debug'): out_stat(self.KGEO_INPUT) log.debug('---- KVOL INPUT ---') if config.getboolean('debug'): out_stat(self.KVOL_INPUT) log.debug('---- KGEO NORM ---') if config.getboolean('debug'): out_stat(self.KGEO_NORM) log.debug('---- KVOL NORM') if config.getboolean('debug'): out_stat(self.KVOL_NORM)
def process(self, product, image, band): wd = os.path.join(config.get('wd'), product.name) self._output_file = self.output_file(product, band) self._tmp_stats = {} log.info('Start') # MGRS reframing for Landsat8 if product.sensor == 'L8': log.debug('{} {}'.format(config.getfloat('dx'), config.getfloat('dy'))) image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy')) # Resampling to 30m for S2 (HLS) elif product.sensor == 'S2': if not config.getboolean('hlsplus'): image = self._resample(image) else: # refine geometry # if config.getfloat('dx') > 0.3 or config.getfloat('dy') > 0.3: log.debug("{} {}".format(config.getfloat('dx'), config.getfloat('dy'))) image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy')) # matching for having QA stats if config.get('refImage'): # try to adapt resolution, changing end of reference filename refImage_path = config.get('refImage') if not os.path.exists(refImage_path): return image # open image ref imageref = S2L_ImageFile(refImage_path) # if refImage resolution does not fit if imageref.xRes != image.xRes: # new refImage filepath refImage_noext = os.path.splitext(refImage_path)[0] if refImage_noext.endswith(f"_{int(imageref.xRes)}m"): refImage_noext = refImage_noext[:-len(f"_{int(imageref.xRes)}m")] refImage_path = refImage_noext + f"_{int(image.xRes)}m.TIF" # compute (resample), or load if exists if not os.path.exists(refImage_path): log.info("Resampling of the reference image") # compute imageref = mgrs_framing.resample(imageref, image.xRes, refImage_path) # write for reuse imageref.write(DCmode=True, creation_options=['COMPRESS=LZW']) else: # or load if exists log.info("Change reference image to:" + refImage_path) imageref = S2L_ImageFile(refImage_path) # open mask mask = S2L_ImageFile(product.mtl.mask_filename) if config.getboolean('freeze_dx_dy'): # do Geometry Assessment only if required assess_geometry_bands = config.get('doAssessGeometry', default='').split(',') if product.sensor != 'S2': assess_geometry_bands = [product.reverse_bands_mapping.get(band) for band in assess_geometry_bands] if assess_geometry_bands and band in assess_geometry_bands: log.info("Geometry assessment for band %s" % band) # Coarse resolution of correlation grid (only for stats) self._matching(imageref, image, wd, mask) else: # Fine resolution of correlation grid (for accurate dx dy computation) dx, dy = self._matching(imageref, image, wd, mask) # save values for correction on bands config.set('dx', dx) config.set('dy', dy) log.info("Geometrical Offsets (DX/DY): {}m {}m".format(config.getfloat('dx'), config.getfloat('dy'))) # Append bands name to keys for key, item in self._tmp_stats.items(): if config.get('reference_band') != band: self._tmp_stats[key+'_{}'.format(band)] = self._tmp_stats.pop(key) metadata.qi.update(self._tmp_stats) log.info('End') return image
def start_process(tile, products, args, start_date, end_date): update_configuration(args, tile) config.set('tile', tile) logger.debug("Processing tile {}".format(tile)) downloader = InputProductArchive(config) _products = downloader.get_products_from_urls( products, start_date, end_date, product_mode=args.operational_mode == 'product-mode') if args.no_run: logger.info("Tile: %s" % tile) if not _products: logger.info("No products found.") for product in _products: tile_message = f'[ Tile coverage = {product.tile_coverage:6.0f}% ]' \ if product.tile_coverage is not None else '' cloud_message = f'[ Cloud coverage = {product.cloud_cover:6.0f}% ]' \ if product.cloud_cover is not None else '' logger.info("%s %s %s" % (tile_message, cloud_message, product.path)) return for product in _products: _product = None if config.getboolean('use_sen2cor'): # Disable Atmospheric correction config.overload('doAtmcor=False') # Run sen2core logger.debug("<<< RUNNING SEN2CORE... >>>") sen2cor_command = os.path.abspath(config.get('sen2cor_path')) sen2cor_output_dir = os.path.join(config.get('wd'), 'sen2cor', os.path.basename(product.path)) if not os.path.exists(sen2cor_output_dir): os.makedirs(sen2cor_output_dir) try: subprocess.run([ 'python', sen2cor_command, product.path, "--output_dir", sen2cor_output_dir, "--work_dir", sen2cor_output_dir, "--sc_only" ], check=True) except subprocess.CalledProcessError as run_error: logger.error("An error occurred during the run of sen2cor") logger.error(run_error) continue # Read output product generated_product = next(os.walk(sen2cor_output_dir))[1] if len(generated_product) != 1: logger.error("Sen2Cor error: Cannot get output product") continue _product = product.reader( os.path.join(sen2cor_output_dir, generated_product[0])) if _product is None: _product = product.reader(product.path) # Update processing configuration config.set('productName', _product.name) config.set('sensor', _product.sensor) config.set('observation_date', _product.mtl.observation_date) config.set('relative_orbit', _product.mtl.relative_orbit) config.set('absolute_orbit', _product.mtl.absolute_orbit) config.set('mission', _product.mtl.mission) config.set('none_S2_product_for_fusion', False) # Disable Atmospheric correction for Level-2A products atmcor = config.get('doAtmcor') if _product.mtl.data_type in ('Level-2A', 'L2TP', 'L2A'): config.overload('s2_processing_level=LEVEL2A') logger.info( "Processing Level-2A product: Atmospheric correction is disabled." ) config.overload('doAtmcor=False') else: config.overload('s2_processing_level=LEVEL1C') process(_product, args) # Restore atmcor status config.overload(f'doAtmcor={atmcor}') del _product if len(_products) == 0: logger.error('No product for tile %s' % tile)
def postprocess(self, pd): """ Copy auxiliary files in the final output like mask, angle files Input product metadata file is also copied. :param pd: instance of S2L_Product class """ # output directory product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L( pd) tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series outdir = product_name product_path = os.path.join(tsdir, outdir) qi_dir = os.path.join(product_path, 'GRANULE', granule_compact_name, 'QI_DATA') # copy angles file outfile = "_".join([metadata.mtd.get('band_rootName_H'), 'ANG' ]) + '.TIF' metadata.mtd['ang_filename'] = outfile shutil.copyfile(pd.mtl.angles_file, os.path.join(qi_dir, outfile)) # copy mask files if "S2" in pd.sensor: tree_in = ElementTree.parse( pd.mtl.tile_metadata) # Tree of the input mtd (S2 MTD.xml) root_in = tree_in.getroot() mask_elements = find_element_by_path( root_in, './Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME') for element in mask_elements: mask_file = os.path.join(pd.path, element.text) if os.path.exists(mask_file): shutil.copyfile( mask_file, os.path.join(qi_dir, os.path.basename(mask_file))) metadata.mtd.get('masks_H').append({ "tag": "MASK_FILENAME", "attribs": element.attrib, "text": element.text }) # copy valid pixel mask outfile = "_".join( [metadata.mtd.get('band_rootName_H'), pd.sensor, 'MSK']) + '.TIF' fpath = os.path.join(qi_dir, outfile) metadata.mtd.get('masks_H').append({ "tag": "MASK_FILENAME", "attribs": { "type": "MSK_VALPIX" }, "text": os.path.relpath(fpath, product_path) }) if config.getboolean('COG'): img_object = S2L_ImageFile(pd.mtl.mask_filename, mode='r') img_object.write(filepath=fpath, COG=True, band='MASK') else: shutil.copyfile(pd.mtl.mask_filename, fpath) # QI directory qipath = os.path.join(tsdir, 'QI') if not os.path.exists(qipath): os.makedirs(qipath) # save config file in QI cfgname = "_".join([outdir, 'INFO']) + '.cfg' cfgpath = os.path.join(tsdir, 'QI', cfgname) config.savetofile(os.path.join(config.get('wd'), pd.name, cfgpath)) # save correl file in QI if os.path.exists( os.path.join(config.get('wd'), pd.name, 'correl_res.txt')): corrname = "_".join([outdir, 'CORREL']) + '.csv' corrpath = os.path.join(tsdir, 'QI', corrname) shutil.copy( os.path.join(config.get('wd'), pd.name, 'correl_res.txt'), corrpath) if len(self.images.keys()) > 1: # true color QL band_list = ["B04", "B03", "B02"] qlname = "_".join( [metadata.mtd.get('band_rootName_H'), 'QL', 'B432']) + '.jpg' qlpath = os.path.join(qi_dir, qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) metadata.mtd.get('quicklooks_H').append(qlpath) # false color QL band_list = ["B12", "B11", "B8A"] qlname = "_".join([ metadata.mtd.get('band_rootName_H'), 'QL', 'B12118A' ]) + '.jpg' qlpath = os.path.join(qi_dir, qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) metadata.mtd.get('quicklooks_H').append(qlpath) else: # grayscale QL band_list = list(self.images.keys()) qlname = "_".join([ metadata.mtd.get('band_rootName_H'), 'QL', band_list[0] ]) + '.jpg' qlpath = os.path.join(qi_dir, qlname) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95)) metadata.mtd.get('quicklooks_H').append(qlpath) # PVI band_list = ["B04", "B03", "B02"] pvi_filename = "_".join([metadata.mtd.get('band_rootName_H'), 'PVI' ]) + '.TIF' qlpath = os.path.join(qi_dir, pvi_filename) quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95), xRes=320, yRes=320, creationOptions=['COMPRESS=LZW'], format='GTIFF') metadata.mtd.get('quicklooks_H').append(qlpath) # Clear images as packager is the last process self.images.clear() # Write QI report as XML bb_QI_path = metadata.hardcoded_values.get('bb_QIH_path') out_QI_path = os.path.join(qi_dir, 'L2H_QI_Report.xml') in_QI_path = glob.glob( os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA', 'L2A_QI_Report.xml')) log.info( 'QI report for input product found : {} (searched at {})'.format( len(in_QI_path) != 0, os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA', 'L2A_QI_Report.xml'))) in_QI_path = in_QI_path[0] if len(in_QI_path) != 0 else None Qi_Writer = QiWriter(bb_QI_path, outfile=out_QI_path, init_QI_path=in_QI_path, H_F='H') Qi_Writer._manual_replaces(pd) Qi_Writer.write(pretty_print=True) # TODO UNCOMMENT BELOW FOR XSD CHECK product_QI_xsd = metadata.hardcoded_values.get('product_QIH_xsd') log.info('QI Report is valid : {}'.format( Qi_Writer.validate_schema(product_QI_xsd, out_QI_path))) # Write product MTD bb_S2_product = metadata.hardcoded_values.get('bb_S2H_product') bb_L8_product = metadata.hardcoded_values.get('bb_L8H_product') product_mtd_path = 'MTD_{}L2H.xml'.format( pd.mtl.sensor[0:3]) # MSI / OLI/ OLI_TIRS product_MTD_outpath = os.path.join(tsdir, product_name, product_mtd_path) mtd_pd_writer = MTD_writer_S2(bb_S2_product, pd.mtl.mtl_file_name, H_F='H') if pd.sensor == 'S2' \ else MTD_writer_LS8(bb_L8_product, H_F='H') mtd_pd_writer._manual_replaces(pd) mtd_pd_writer.write(product_MTD_outpath, pretty_print=True) # TODO UNCOMMENT BELOW FOR XSD CHECK # product_mtd_xsd = metadata.hardcoded_values.get('product_mtd_xsd') # log.info('Product MTD is valid : {}'.format(mtd_pd_writer.validate_schema(product_mtd_xsd, # product_MTD_outpath))) # Write tile MTD bb_S2_tile = metadata.hardcoded_values.get('bb_S2H_tile') bb_L8_tile = metadata.hardcoded_values.get('bb_L8H_tile') tile_mtd_path = 'MTD_TL_L2H.xml' tile_MTD_outpath = os.path.join(product_path, 'GRANULE', granule_compact_name, tile_mtd_path) mtd_tl_writer = MTD_tile_writer_S2(bb_S2_tile, pd.mtl.tile_metadata, H_F='H') if pd.sensor == 'S2' \ else MTD_tile_writer_LS8(bb_L8_tile, H_F='H') mtd_tl_writer._manual_replaces(pd) mtd_tl_writer.write(tile_MTD_outpath, pretty_print=True)
def manual_replaces(self, product): # GENERAL_INFO # ------------ elements_to_copy = ['./General_Info/Product_Info/Datatake', './General_Info/Product_Info/PRODUCT_START_TIME', './General_Info/Product_Info/PRODUCT_STOP_TIME', './General_Info/Product_Info/Query_Options', ] copy_elements(elements_to_copy, self.root_in, self.root_out, self.root_bb) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI', new_value=metadata.mtd.get('product_{}_name'.format(self.H_F))) change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL', new_value='Level-2{}'.format(self.H_F)) change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE', new_value='S2MSI2{}'.format(self.H_F)) pdgs = config.get('PDGS', '9999') PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]]) AC = self.root_in.findall('.//ARCHIVING_CENTRE') if AC: metadata.mtd['S2_AC'] = AC[0].text change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS) generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[ :-3] + 'Z' # -3 to keep only 3 decimals change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time) self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule') for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))): adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0] create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', tag='IMAGE_FILE', text=adjusted_path) grnl_id = \ find_element_by_path(self.root_in, './General_Info/Product_Info/Product_Organisation/Granule_List/Granule') if grnl_id: change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=generate_S2_tile_id(product, self.H_F, metadata.mtd['S2_AC']), attr_to_change='granuleIdentifier') change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule', new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat') else: pass # Fixme # If Sbaf is done, we keep the values inside the backbone (S2A values) if not config.getboolean('doSbaf'): # copy_elements(['./General_Info/Product_Image_Characteristics/Special_Values', # './General_Info/Product_Image_Characteristics/Image_Display_Order', # './General_Info/Product_Image_Characteristics/Reflectance_Conversion', # './General_Info/Product_Image_Characteristics/Spectral_Information_List'], # self.root_in, self.root_out, self.root_bb) pass # FIXME : get product image characteristics from origin sensor (S2 here), # copying from another template for example (see commented lines above) copy_elements(['./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U'], self.root_in, self.root_out) # Geometric_info # --------------- tilecode = product.mtl.mgrs if tilecode.startswith('T'): tilecode = tilecode[1:] footprint = search_db(tilecode, search='MGRS_REF') # adding back first element, to get a complete polygon fp = footprint.split(' ') footprint = ' '.join(fp + [fp[0], fp[1]]) chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint) copy_elements(['./Geometric_Info/Coordinate_Reference_System'], self.root_in, self.root_out, self.root_bb) # Auxiliary_Data_Info # ------------------- self.remove_children('./Auxiliary_Data_Info/GIPP_List') copy_children(self.root_in, './Auxiliary_Data_Info/GIPP_List', self.root_out, './Auxiliary_Data_Info/GIPP_List') config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0] create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn, attribs={"version": pdgs, "type": "GIP_S2LIKE"}) for tag in ['PRODUCTION_DEM_TYPE', 'IERS_BULLETIN_FILENAME', 'ECMWF_DATA_REF', 'SNOW_CLIMATOLOGY_MAP', 'ESACCI_WaterBodies_Map', 'ESACCI_LandCover_Map', 'ESACCI_SnowCondition_Map_Dir']: elem = find_element_by_path(self.root_in, './Auxiliary_Data_Info/' + tag) if len(elem) != 0: new_value = elem[0].text else: new_value = "NONE" change_elm(self.root_out, rpath='./Auxiliary_Data_Info/' + tag, new_value=new_value) # Fill GRI_List gri_elems = self.root_in.findall('.//GRI_FILENAME') for gri_elm in gri_elems: create_child(self.root_out, './Auxiliary_Data_Info/GRI_List', tag="GRI_FILENAME", text=gri_elm.text) # Quality_Indicators_Info # ----------------------- copy_elements(['./Quality_Indicators_Info'], self.root_in, self.root_out, self.root_bb)
def process(self, product, image, band): log.info('Start') if not config.getboolean('hlsplus'): log.warning( 'Skipping Data Fusion because doPackager and doPackagerL2F options are not activated' ) log.info('End') return image # save into file before processing (packager will need it) product.image30m[band] = image if band == 'B01': log.warning('Skipping Data Fusion for B01.') log.info('End') return image if len(self.reference_products) == 0: log.warning( 'Skipping Data Fusion. Reason: no S2 products available in the past' ) log.info('End') return image if not product.get_s2like_band(band): log.warning( 'Skipping Data Fusion. Reason: no S2 matching band for {}'. format(band)) log.info('End') return image # method selection predict_method = config.get('predict_method', 'predict') if len(self.reference_products) == 1: log.warning( 'Not enough Sentinel2 products for the predict (only one product). Using last S2 product as ref.' ) predict_method = 'composite' # general info band_s2 = product.get_s2like_band(band) image_file_L2F = self.reference_products[0].get_band_file(band_s2, plus=True) output_shape = (image_file_L2F.ySize, image_file_L2F.xSize) # method: prediction (from the 2 most recent S2 products) if predict_method == 'predict': # Use QA (product selection) to apply Composting : qa_mask = self._get_qa_band(output_shape) # predict array_L2H_predict, array_L2F_predict = self._predict( product, band_s2, qa_mask, output_shape) # save if config.getboolean('generate_intermediate_products'): self._save_as_image_file(image_file_L2F, qa_mask, product, band, '_FUSION_QA.TIF') self._save_as_image_file(image_file_L2F, array_L2H_predict, product, band, '_FUSION_L2H_PREDICT.TIF') self._save_as_image_file(image_file_L2F, array_L2F_predict, product, band, '_FUSION_L2F_PREDICT.TIF') # method: composite (most recent valid pixels from N products) elif predict_method == 'composite': # composite array_L2H_predict, array_L2F_predict = self._composite( product, band_s2, output_shape) # save if config.getboolean('generate_intermediate_products'): self._save_as_image_file(image_file_L2F, array_L2H_predict, product, band, '_FUSION_L2H_COMPO.TIF') self._save_as_image_file(image_file_L2F, array_L2F_predict, product, band, '_FUSION_L2F_COMPO.TIF') # method: unknown else: log.error( f'Unknown predict method: {predict_method}. Please check your configuration.' ) return None # fusion L8/S2 mask_filename = product.mtl.nodata_mask_filename array_out = self._fusion(image, array_L2H_predict, array_L2F_predict, mask_filename).astype(np.float32) image = self._save_as_image_file(image_file_L2F, array_out, product, band, '_FUSION_L2H_PREDICT.TIF') log.info('End') return image