def update_configuration(args, tile=None):
# init S2L_config and save to wd
if not config.initialize(args.S2L_configfile):
return
if args.confParams is not None:
config.overload(args.confParams)
use_pid = False
if use_pid:
output_folder = str(os.getpid())
else:
date_now = datetime.datetime.now().strftime('%Y%m%dT_%H%M%S')
output_folder = f'{"" if args.no_log_date else f"{date_now}_"}{compute_config_hash(args, config)}'
config.set('wd', os.path.join(args.wd, output_folder))
references_map_file = config.get('references_map')
if args.refImage:
config.set('refImage', args.refImage)
elif references_map_file and tile:
# load dataset
with open(references_map_file) as j:
references_map = json.load(j)
config.set('refImage', references_map.get(tile))
else:
config.set('refImage', None)
config.set(
'hlsplus',
config.getboolean('doPackager') or config.getboolean('doPackagerL2F'))
config.set('debug', args.debug)
config.set('generate_intermediate_products',
args.generate_intermediate_products)
if hasattr(args, 'l2a'):
config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def update_configuration(args, tile=None):
# init S2L_config and save to wd
if not config.initialize(args.S2L_configfile):
return
if args.confParams is not None:
config.overload(args.confParams)
config.set('wd', os.path.join(args.wd, str(os.getpid())))
references_map_file = config.get('references_map')
if args.refImage:
config.set('refImage', args.refImage)
elif references_map_file and tile:
# load dataset
with open(references_map_file) as j:
references_map = json.load(j)
config.set('refImage', references_map.get(tile))
else:
config.set('refImage', None)
config.set(
'hlsplus',
config.getboolean('doPackager') or config.getboolean('doPackagerL2F'))
config.set('debug', args.debug)
config.set('generate_intermediate_products',
args.generate_intermediate_products)
if hasattr(args, 'l2a'):
config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def generic_process_step(blockname, pd, process_step):
"""From the name of the block, import the module, get the class,
create object from class, run the process step method of object.
This supposes that there all the names are the same (e.g. S2L_GeometryKLT)
:param blockname: The block to process
:param pd:
:param process_step: The step to process
:return:
"""
# check if block is switch ON
if not config.getboolean('do' + blockname.split('_')[-1]):
return
# check if block is applicable to the sensor (L8 or S2)
if pd.sensor not in S2L_config.PROC_BLOCKS[blockname]['applicability']:
return
class_instance = get_module(blockname)
# create object and run preprocess if method exists!
processus = getattr(class_instance, process_step, None)
if processus is not None:
return processus(pd)
def process(self, product, image, band):
log.info('Start')
# init to None
offset = None
slope = None
if product.mtl.mission == "Sentinel-2A":
# skip for S2A
metadata.qi['SBAF_COEFFICIENT_{}'.format(band)] = 1
metadata.qi['SBAF_OFFSET_{}'.format(band)] = 0
return image
elif product.mtl.mission == "Sentinel-2B":
# S2B => L8 + L8 => S2A
adj_coef1 = self.getOLILikeCoef("Sentinel-2B")
adj_coef2 = self.getSen2likeCoef("LANDSAT_8")
band_sbaf1 = band
band_sbaf2 = Landsat8Product.get_band_from_s2(band)
if band_sbaf1 in adj_coef1 and band_sbaf2 in adj_coef2:
slope1, offset1 = adj_coef1[band_sbaf1]['coef']
slope2, offset2 = adj_coef2[band_sbaf2]['coef']
# merging coefficients
slope = slope2 * slope1
offset = slope2 * offset1 + offset2
else:
log.error("No Sbaf coefficient defined for {}".format(band))
elif product.mtl.mission == "LANDSAT_8":
# L8 => S2A
band_sbaf = band
adj_coef = self.getSen2likeCoef("LANDSAT_8")
if band_sbaf in adj_coef:
slope, offset = adj_coef[band_sbaf]['coef']
else:
metadata.qi['SBAF_COEFFICIENT_{}'.format(band)] = 1
metadata.qi['SBAF_OFFSET_{}'.format(band)] = 0
log.error("No Sbaf coefficient defined for {}".format(band))
return image
metadata.qi['SBAF_COEFFICIENT_{}'.format(band)] = slope
metadata.qi['SBAF_OFFSET_{}'.format(band)] = offset
# Apply SBAF
if offset is not None and slope is not None:
log.debug("Applying SBAF")
new = image.array
new = new * slope + offset
# transfer no data
new[image.array == 0] = 0
# Format Output : duplicate, link to product as parameter
image = image.duplicate(self.output_file(product, band), array=new.astype(np.float32))
if config.getboolean('generate_intermediate_products'):
image.write(creation_options=['COMPRESS=LZW'])
log.info('End')
return image
def _save_as_image_file(self, image_template, array, product, band,
extension):
path = os.path.join(config.get('wd'), product.name,
product.get_band_file(band).rootname + extension)
image_file = image_template.duplicate(path, array=array)
if config.getboolean('generate_intermediate_products'):
image_file.write(creation_options=['COMPRESS=LZW'])
return image_file
def process(self, pd, image, band):
"""
Write final product in the archive directory
'archive_dir' is defined in config.ini file
Naming convention from Design Document
:param pd: instance of S2L_Product class
:param image: input instance of S2L_ImageFile class
:param band: band being processed
:return: output instance of instance of S2L_ImageFile class
"""
# TODO : add production date?
# /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR
res = image.xRes
product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L(
pd)
sensor = pd.sensor_name
relative_orbit = config.get('relative_orbit')
native = band in ['B08', 'B10', 'B11'] if sensor == 'LS8' else \
band in ['B05', 'B06', 'B07', 'B08']
s2_band = pd.get_s2like_band(band)
if not native:
band = s2_band
band_rootName = "_".join([
'L2F', 'T' + tilecode, datatake_sensing_start, sensor,
'R{:0>3}'.format(relative_orbit)
])
metadata.mtd['band_rootName_F'] = band_rootName
outfile = "_".join([band_rootName, band, '{}m'.format(int(res))
]) + '.TIF'
# Naming convention from Sentinel-2-Products-Specification-Document (p294)
tsdir = os.path.join(config.get('archive_dir'),
tilecode) # ts = temporal series
newpath = self.band_path(tsdir,
product_name,
granule_compact_name,
outfile,
native=native)
COG = config.getboolean('COG')
log.debug('New: ' + newpath)
image.write(creation_options=['COMPRESS=LZW'],
filepath=newpath,
COG=COG,
band=band)
metadata.mtd.get('bands_path_F').append(newpath)
# declare output internally
self.images[s2_band] = image.filepath
# declare output in config file
config.set('imageout_dir', image.dirpath)
config.set('imageout_' + band, image.filename)
return image
def reframe(self, image, product, band=None, dtype=None):
# Add margin
margin = int(config.get('reframe_margin', self.margin))
log.debug(f"Using {margin} as margin.")
product_image = mgrs_framing.reframe(image, self.tile,
filepath_out=self.output_file(product, band, image, "_PREREFRAMED"),
order=0, margin=margin, dtype=dtype, compute_offsets=True)
if config.getboolean('generate_intermediate_products'):
product_image.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count
return product_image
def generic_process_band(blockname, pd, image, band):
"""
from the name of the block, import the module, get the class,
create object from class, run the main method of object.
This supposes that there all the names are the same (e.g. S2L_GeometryKLT)
"""
# check if block is switch ON
logger.debug(config.getboolean('do' + blockname.split('_')[-1]))
if not config.getboolean('do' + blockname.split('_')[-1]):
return image, None
# check if block is applicable to the sensor (L8, L9 or S2)
if pd.sensor not in S2L_config.PROC_BLOCKS[blockname]['applicability']:
return image, None
class_instance = get_module(blockname)
# create object and run it!
return class_instance.process(pd, image, band), class_instance
def _reframe(self, product, imagein, dx=0., dy=0.):
log.info('MGRS Framing: Start...')
# reframe on MGRS
imageout = mgrs_framing.reframe(imagein, product.mtl.mgrs, self._output_file, dx, dy, dtype=np.float32)
# display
if config.getboolean('generate_intermediate_products'):
imageout.write(DCmode=True) # digital count
log.info('MGRS Framing: End')
return imageout
def process(self, product, image, band):
log.info('Start')
# SMAC correction
extent = image.getCorners(outEPSG=4326)
array_in = image.array
array_out = smac_correction(product, array_in, extent, band)
image = image.duplicate(self.output_file(product, band), array_out)
if config.getboolean('generate_intermediate_products'):
image.write(creation_options=['COMPRESS=LZW'])
log.info('End')
return image
def _resample(self, imagein):
# display
log.info('Resampling to 30m: Start...')
# resampling
imageout = mgrs_framing.resample(imagein, 30, self._output_file)
# display
if config.getboolean('generate_intermediate_products'):
imageout.write(DCmode=True) # digital count
log.info('Resampling to 30m: End')
return imageout
def process(self, pd, image, band):
"""
Write final product in the archive directory
'archive_dir' is defined in config.ini file
Naming convention from Design Document
:param pd: instance of S2L_Product class
:param image: input instance of S2L_ImageFile class
:param band: band being processed
:return: outpu t instance of instance of S2L_ImageFile class
"""
# TODO : add production date?
# TODO : change L8 band numbers to S2 numbers convention?
# /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR
res = image.xRes
outdir, tilecode = self.base_path(pd)
outfile = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF'
tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series
newpath = os.path.join(tsdir, outdir, outfile)
log.debug('New: ' + newpath)
image.write(creation_options=['COMPRESS=LZW'], filepath=newpath)
# declare output internally
self.images[band] = image.filepath
# declare output in config file
config.set('imageout_dir', image.dirpath)
config.set('imageout_' + band, image.filename)
if config.getboolean('hlsplus'):
res = 30
outfile_30m = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF'
newpath_30m = os.path.join(tsdir, outdir, outfile_30m)
if pd.sensor == 'S2':
# create 30m band as well
# resampling
log.info('Resampling to 30m: Start...')
image_30m = mgrs_framing.resample(S2L_ImageFile(newpath), res, newpath_30m)
image_30m.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count
log.info('Resampling to 30m: End')
if pd.sensor == 'L8' and band in pd.image30m:
# copy 30m band as well
# write
pd.image30m[band].write(creation_options=['COMPRESS=LZW'], filepath=newpath_30m)
del pd.image30m[band]
return image
def _nbar(self, product, image, band):
# Get BRDF coefficients
brdf_coef_set = get_brdf_coefficient(product, band)
log.debug('BRDF Coefficient Set :{}'.format(brdf_coef_set))
CMATRIX = normalized_brdf(self.KVOL_NORM, self.KGEO_NORM,
self.KVOL_INPUT, self.KGEO_INPUT,
brdf_coef_set)
IM1 = image.array
U = IM1 >= 0
if config.getboolean('debug'):
out_stat(IM1[U])
CMATRIX_full = resize(CMATRIX, IM1.shape)
IM = CMATRIX_full
U = IM >= 0
if config.getboolean('debug'):
out_stat(IM[U])
OUT = CMATRIX_full * IM1
OUT[IM1 <= 0] = 0
return OUT
def process(self, product, image, band):
log.info('Start')
# convert to TOA (gain + offset)
array_in = image.array
array_out = convert_to_reflectance_from_reflectance_cal_product(
product.mtl, array_in, band)
image = image.duplicate(self.output_file(product, band),
array=array_out)
if config.getboolean('generate_intermediate_products'):
image.write(creation_options=['COMPRESS=LZW'])
log.info('End')
return image
def preprocess(self, product):
self.get_new_product(product)
if self.new_product is None:
return
# Stitch validity mask and angles
product_validity_masks = []
product_nodata_masks = []
product_angles = []
for _product in [product, self.new_product.reader(self.new_product.path)]:
is_mask_valid = True
# Validity mask
if _product.mtl.mask_filename is None:
is_mask_valid = _product.mtl.get_valid_pixel_mask(
os.path.join(config.get("wd"), _product.name, 'valid_pixel_mask.tif'))
if is_mask_valid:
product_validity_masks.append(self.reframe(S2L_ImageFile(_product.mtl.mask_filename), _product))
product_nodata_masks.append(self.reframe(S2L_ImageFile(_product.mtl.nodata_mask_filename), _product))
# Angles
if _product.mtl.angles_file is None:
_product.mtl.get_angle_images(os.path.join(config.get("wd"), _product.name, 'tie_points.tif'))
filepath_out = os.path.join(config.get('wd'), _product.name, 'tie_points_PREREFRAMED.TIF')
mgrs_framing.reframeMulti(_product.mtl.angles_file, self.tile, filepath_out=filepath_out, order=0)
product_angles.append(filepath_out)
if None not in product_validity_masks:
stitched_mask = self.stitch(product, product_validity_masks[0], product_validity_masks[1])
stitched_mask.write(creation_options=['COMPRESS=LZW'])
product.mtl.mask_filename = stitched_mask.filepath
if None not in product_nodata_masks:
stitched_mask = self.stitch(product, product_nodata_masks[0], product_nodata_masks[1])
stitched_mask.write(creation_options=['COMPRESS=LZW'])
product.mtl.nodata_mask_filename = stitched_mask.filepath
stitched_angles = self.stitch_multi(product, product_angles[0], product_angles[1])
product.mtl.angles_file = stitched_angles
# Stitch reference band (needed by geometry module)
band = config.get('reference_band', 'B04')
if config.getboolean('doMatchingCorrection') and config.get('refImage'):
image = product.get_band_file(band)
self.process(product, image, band)
def preprocess(self, product):
# reinit dx/dy
config.set('dx', 0)
config.set('dy', 0)
if product.sensor != 'S2':
# Reframe angles and masks
filepath_out = os.path.join(config.get('wd'), product.name, 'tie_points_REFRAMED.TIF')
mgrs_framing.reframeMulti(product.mtl.angles_file, product.mtl.mgrs, filepath_out, config.getfloat('dx'),
config.getfloat('dy'), order=0)
product.mtl.angles_file = filepath_out
# Reframe mask
if product.mtl.mask_filename:
filepath_out = os.path.join(config.get('wd'), product.name, 'valid_pixel_mask_REFRAMED.TIF')
image = S2L_ImageFile(product.mtl.mask_filename)
imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'),
config.getfloat('dy'), order=0)
imageout.write(creation_options=['COMPRESS=LZW'])
product.mtl.mask_filename = filepath_out
# Reframe nodata mask
if product.mtl.nodata_mask_filename:
filepath_out = os.path.join(config.get('wd'), product.name, 'nodata_pixel_mask_REFRAMED.TIF')
image = S2L_ImageFile(product.mtl.nodata_mask_filename)
imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'),
config.getfloat('dy'), order=0)
imageout.write(creation_options=['COMPRESS=LZW'])
product.mtl.nodata_mask_filename = filepath_out
# Matching for dx/dy correction?
band = config.get('reference_band', 'B04')
if config.getboolean('doMatchingCorrection') and config.get('refImage'):
config.set('freeze_dx_dy', False)
image = product.get_band_file(band)
self.process(product, image, band)
# goal is to feed dx, dy in config
config.set('freeze_dx_dy', True)
metadata.qi.update({'COREGISTRATION_BEFORE_CORRECTION': self._tmp_stats.get('MEAN'.format(band))})
def process(self, product, image, band):
log.info('Start')
# coeff for this band?
if get_brdf_coefficient(product, band) is None:
log.warning('No BRDF coefficient for {}'.format(band))
image_out = image
else:
# Compute Kernels
self._computeKernels(product, band)
# NBAR correction
OUT = self._nbar(product, image, band)
# Format Output : duplicate, link to product as parameter
image_out = image.duplicate(self.output_file(product, band),
array=OUT.astype(np.float32))
if config.getboolean('generate_intermediate_products'):
image_out.write(creation_options=['COMPRESS=LZW'])
log.info('End')
return image_out
def manual_replaces(self, product):
# GENERAL_INFO
# ------------
acqdate = dt.datetime.strftime(product.acqdate, '%Y-%m-%dT%H:%M:%S.%fZ')
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_START_TIME', new_value=acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_STOP_TIME', new_value=acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI',
new_value=metadata.mtd.get('product_{}_name'.format(self.H_F)))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL',
new_value='Level-2{}'.format(self.H_F))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE',
new_value=f'{product.sensor}OLI2{self.H_F}')
pdgs = config.get('PDGS', '9999')
PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]])
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS)
generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[
:-3] + 'Z' # -3 to keep only 3 decimals
change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SPACECRAFT_NAME',
new_value=product.mtl.mission)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/DATATAKE_SENSING_START',
new_value=acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SENSING_ORBIT_NUMBER',
new_value=config.get('relative_orbit'))
self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule')
for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))):
adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0]
create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
tag='IMAGE_FILE', text=adjusted_path)
tile_id = generate_LS8_tile_id(product, self.H_F)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=tile_id, attr_to_change='granuleIdentifier')
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat')
if not config.getboolean('doSbaf'):
# FIXME : get product image characteristics from origin sensor (LS8 here),
# copying from another template fro example
pass
U = distance_variation_corr(product.acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U',
new_value=str(U))
# Geometric_info
# ---------------
tilecode = product.mtl.mgrs
if tilecode.startswith('T'):
tilecode = tilecode[1:]
footprint = search_db(tilecode, search='MGRS_REF')
# adding back first element, to get a complete polygon
fp = footprint.split(' ')
footprint = ' '.join(fp + [fp[0], fp[1]])
chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint)
# Auxiliary_Data_Info
# -------------------
self.remove_children('./Auxiliary_Data_Info/GIPP_List', exceptions=['Input_Product_Info'])
config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0]
create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn,
attribs={"version": pdgs, "type": "GIP_S2LIKE"})
# Quality_Indicators_Info
# -----------------------
self.remove_children('./Quality_Indicators_Info', exceptions=['Input_Product_Info'])
change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info', attr_to_change='type',
new_value=product.mtl.mission)
change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info',
new_value=product.mtl.landsat_scene_id)
def _computeKernels(self, product, band=None):
lat = product.mtl.get_scene_center_coordinates()[1]
scene_center_latitude = lat
theta_s = get_mean_sun_angle(scene_center_latitude)
metadata.qi['CONSTANT_SOLAR_ZENITH_ANGLE'] = theta_s
log.debug('theta_s: {}'.format(theta_s))
# Read TP , unit = degree, scale=100
src_ds = gdal.Open(product.mtl.angles_file)
nBands = src_ds.RasterCount
if nBands == 4:
# VAA, VZA, SAA, SZA
VAA = src_ds.GetRasterBand(1).ReadAsArray().astype(
np.float32) / 100.0
VZA = src_ds.GetRasterBand(2).ReadAsArray().astype(
np.float32) / 100.0
SAA = src_ds.GetRasterBand(3).ReadAsArray().astype(
np.float32) / 100.0
SZA = src_ds.GetRasterBand(4).ReadAsArray().astype(
np.float32) / 100.0
else:
# VAA for each band, VZA for each band, SAA, SZZ
angle_band_index = product.get_angles_band_index(band)
VAA = src_ds.GetRasterBand(angle_band_index +
1).ReadAsArray().astype(
np.float32) / 100.0
VZA = src_ds.GetRasterBand(13 + angle_band_index +
1).ReadAsArray().astype(
np.float32) / 100.0
SAA = src_ds.GetRasterBand(nBands - 1).ReadAsArray().astype(
np.float32) / 100.0
SZA = src_ds.GetRasterBand(nBands).ReadAsArray().astype(
np.float32) / 100.0
# close
src_ds = None
metadata.qi['MEAN_DELTA_AZIMUTH'] = np.mean(SAA - VAA) % 360
if config.getboolean('debug'):
out_stat(VAA, 'VAA')
out_stat(VZA, 'VZA')
out_stat(SAA, 'SAA')
out_stat(SZA, 'SZA')
# Prepare KGEO Input
log.debug(
'-------------------- INPUT --------------------------------------'
)
log.debug(
'---- ---------KGEO INPUT COMPUTATION ------------------------------'
)
self.KGEO_INPUT = li_sparse_kernel(SZA, VZA, SAA - VAA)
# Prepare KVOL Input :
log.debug(
'------------- KVOL INPUT COMPUTATION ------------------------------'
)
self.KVOL_INPUT = ross_thick(SZA, VZA, SAA - VAA)
# Prepare KGEO Norm :
SZA_NORM = np.ones(VAA.shape) * theta_s
VZA_NORM = np.zeros(VAA.shape)
DPHI_NORM = np.zeros(VAA.shape)
log.debug(
'-------------------NORM-------------------------------------------'
)
log.debug(
'------------- KGEO NORM COMPUTATION ------------------------------'
)
self.KGEO_NORM = li_sparse_kernel(SZA_NORM, VZA_NORM, DPHI_NORM)
log.debug('---- KVOL NORM COMPUTATION ---')
self.KVOL_NORM = ross_thick(SZA_NORM, VZA_NORM, DPHI_NORM)
log.debug(
'------------------------------------------------------------------'
)
log.debug(
'--------------- KGEO INPUT STAT-----------------------------------'
)
log.debug('---- KGEO INPUT ---')
if config.getboolean('debug'):
out_stat(self.KGEO_INPUT)
log.debug('---- KVOL INPUT ---')
if config.getboolean('debug'):
out_stat(self.KVOL_INPUT)
log.debug('---- KGEO NORM ---')
if config.getboolean('debug'):
out_stat(self.KGEO_NORM)
log.debug('---- KVOL NORM')
if config.getboolean('debug'):
out_stat(self.KVOL_NORM)
def process(self, product, image, band):
wd = os.path.join(config.get('wd'), product.name)
self._output_file = self.output_file(product, band)
self._tmp_stats = {}
log.info('Start')
# MGRS reframing for Landsat8
if product.sensor == 'L8':
log.debug('{} {}'.format(config.getfloat('dx'), config.getfloat('dy')))
image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy'))
# Resampling to 30m for S2 (HLS)
elif product.sensor == 'S2':
if not config.getboolean('hlsplus'):
image = self._resample(image)
else:
# refine geometry
# if config.getfloat('dx') > 0.3 or config.getfloat('dy') > 0.3:
log.debug("{} {}".format(config.getfloat('dx'), config.getfloat('dy')))
image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy'))
# matching for having QA stats
if config.get('refImage'):
# try to adapt resolution, changing end of reference filename
refImage_path = config.get('refImage')
if not os.path.exists(refImage_path):
return image
# open image ref
imageref = S2L_ImageFile(refImage_path)
# if refImage resolution does not fit
if imageref.xRes != image.xRes:
# new refImage filepath
refImage_noext = os.path.splitext(refImage_path)[0]
if refImage_noext.endswith(f"_{int(imageref.xRes)}m"):
refImage_noext = refImage_noext[:-len(f"_{int(imageref.xRes)}m")]
refImage_path = refImage_noext + f"_{int(image.xRes)}m.TIF"
# compute (resample), or load if exists
if not os.path.exists(refImage_path):
log.info("Resampling of the reference image")
# compute
imageref = mgrs_framing.resample(imageref, image.xRes, refImage_path)
# write for reuse
imageref.write(DCmode=True, creation_options=['COMPRESS=LZW'])
else:
# or load if exists
log.info("Change reference image to:" + refImage_path)
imageref = S2L_ImageFile(refImage_path)
# open mask
mask = S2L_ImageFile(product.mtl.mask_filename)
if config.getboolean('freeze_dx_dy'):
# do Geometry Assessment only if required
assess_geometry_bands = config.get('doAssessGeometry', default='').split(',')
if product.sensor != 'S2':
assess_geometry_bands = [product.reverse_bands_mapping.get(band) for band in assess_geometry_bands]
if assess_geometry_bands and band in assess_geometry_bands:
log.info("Geometry assessment for band %s" % band)
# Coarse resolution of correlation grid (only for stats)
self._matching(imageref, image, wd, mask)
else:
# Fine resolution of correlation grid (for accurate dx dy computation)
dx, dy = self._matching(imageref, image, wd, mask)
# save values for correction on bands
config.set('dx', dx)
config.set('dy', dy)
log.info("Geometrical Offsets (DX/DY): {}m {}m".format(config.getfloat('dx'), config.getfloat('dy')))
# Append bands name to keys
for key, item in self._tmp_stats.items():
if config.get('reference_band') != band:
self._tmp_stats[key+'_{}'.format(band)] = self._tmp_stats.pop(key)
metadata.qi.update(self._tmp_stats)
log.info('End')
return image
def start_process(tile, products, args, start_date, end_date):
update_configuration(args, tile)
config.set('tile', tile)
logger.debug("Processing tile {}".format(tile))
downloader = InputProductArchive(config)
_products = downloader.get_products_from_urls(
products,
start_date,
end_date,
product_mode=args.operational_mode == 'product-mode')
if args.no_run:
logger.info("Tile: %s" % tile)
if not _products:
logger.info("No products found.")
for product in _products:
tile_message = f'[ Tile coverage = {product.tile_coverage:6.0f}% ]' \
if product.tile_coverage is not None else ''
cloud_message = f'[ Cloud coverage = {product.cloud_cover:6.0f}% ]' \
if product.cloud_cover is not None else ''
logger.info("%s %s %s" %
(tile_message, cloud_message, product.path))
return
for product in _products:
_product = None
if config.getboolean('use_sen2cor'):
# Disable Atmospheric correction
config.overload('doAtmcor=False')
# Run sen2core
logger.debug("<<< RUNNING SEN2CORE... >>>")
sen2cor_command = os.path.abspath(config.get('sen2cor_path'))
sen2cor_output_dir = os.path.join(config.get('wd'), 'sen2cor',
os.path.basename(product.path))
if not os.path.exists(sen2cor_output_dir):
os.makedirs(sen2cor_output_dir)
try:
subprocess.run([
'python', sen2cor_command, product.path, "--output_dir",
sen2cor_output_dir, "--work_dir", sen2cor_output_dir,
"--sc_only"
],
check=True)
except subprocess.CalledProcessError as run_error:
logger.error("An error occurred during the run of sen2cor")
logger.error(run_error)
continue
# Read output product
generated_product = next(os.walk(sen2cor_output_dir))[1]
if len(generated_product) != 1:
logger.error("Sen2Cor error: Cannot get output product")
continue
_product = product.reader(
os.path.join(sen2cor_output_dir, generated_product[0]))
if _product is None:
_product = product.reader(product.path)
# Update processing configuration
config.set('productName', _product.name)
config.set('sensor', _product.sensor)
config.set('observation_date', _product.mtl.observation_date)
config.set('relative_orbit', _product.mtl.relative_orbit)
config.set('absolute_orbit', _product.mtl.absolute_orbit)
config.set('mission', _product.mtl.mission)
config.set('none_S2_product_for_fusion', False)
# Disable Atmospheric correction for Level-2A products
atmcor = config.get('doAtmcor')
if _product.mtl.data_type in ('Level-2A', 'L2TP', 'L2A'):
config.overload('s2_processing_level=LEVEL2A')
logger.info(
"Processing Level-2A product: Atmospheric correction is disabled."
)
config.overload('doAtmcor=False')
else:
config.overload('s2_processing_level=LEVEL1C')
process(_product, args)
# Restore atmcor status
config.overload(f'doAtmcor={atmcor}')
del _product
if len(_products) == 0:
logger.error('No product for tile %s' % tile)
def postprocess(self, pd):
"""
Copy auxiliary files in the final output like mask, angle files
Input product metadata file is also copied.
:param pd: instance of S2L_Product class
"""
# output directory
product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L(
pd)
tsdir = os.path.join(config.get('archive_dir'),
tilecode) # ts = temporal series
outdir = product_name
product_path = os.path.join(tsdir, outdir)
qi_dir = os.path.join(product_path, 'GRANULE', granule_compact_name,
'QI_DATA')
# copy angles file
outfile = "_".join([metadata.mtd.get('band_rootName_H'), 'ANG'
]) + '.TIF'
metadata.mtd['ang_filename'] = outfile
shutil.copyfile(pd.mtl.angles_file, os.path.join(qi_dir, outfile))
# copy mask files
if "S2" in pd.sensor:
tree_in = ElementTree.parse(
pd.mtl.tile_metadata) # Tree of the input mtd (S2 MTD.xml)
root_in = tree_in.getroot()
mask_elements = find_element_by_path(
root_in,
'./Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME')
for element in mask_elements:
mask_file = os.path.join(pd.path, element.text)
if os.path.exists(mask_file):
shutil.copyfile(
mask_file,
os.path.join(qi_dir, os.path.basename(mask_file)))
metadata.mtd.get('masks_H').append({
"tag": "MASK_FILENAME",
"attribs": element.attrib,
"text": element.text
})
# copy valid pixel mask
outfile = "_".join(
[metadata.mtd.get('band_rootName_H'), pd.sensor, 'MSK']) + '.TIF'
fpath = os.path.join(qi_dir, outfile)
metadata.mtd.get('masks_H').append({
"tag":
"MASK_FILENAME",
"attribs": {
"type": "MSK_VALPIX"
},
"text":
os.path.relpath(fpath, product_path)
})
if config.getboolean('COG'):
img_object = S2L_ImageFile(pd.mtl.mask_filename, mode='r')
img_object.write(filepath=fpath, COG=True, band='MASK')
else:
shutil.copyfile(pd.mtl.mask_filename, fpath)
# QI directory
qipath = os.path.join(tsdir, 'QI')
if not os.path.exists(qipath):
os.makedirs(qipath)
# save config file in QI
cfgname = "_".join([outdir, 'INFO']) + '.cfg'
cfgpath = os.path.join(tsdir, 'QI', cfgname)
config.savetofile(os.path.join(config.get('wd'), pd.name, cfgpath))
# save correl file in QI
if os.path.exists(
os.path.join(config.get('wd'), pd.name, 'correl_res.txt')):
corrname = "_".join([outdir, 'CORREL']) + '.csv'
corrpath = os.path.join(tsdir, 'QI', corrname)
shutil.copy(
os.path.join(config.get('wd'), pd.name, 'correl_res.txt'),
corrpath)
if len(self.images.keys()) > 1:
# true color QL
band_list = ["B04", "B03", "B02"]
qlname = "_".join(
[metadata.mtd.get('band_rootName_H'), 'QL', 'B432']) + '.jpg'
qlpath = os.path.join(qi_dir, qlname)
quicklook(pd, self.images, band_list, qlpath,
config.get("quicklook_jpeg_quality", 95))
metadata.mtd.get('quicklooks_H').append(qlpath)
# false color QL
band_list = ["B12", "B11", "B8A"]
qlname = "_".join([
metadata.mtd.get('band_rootName_H'), 'QL', 'B12118A'
]) + '.jpg'
qlpath = os.path.join(qi_dir, qlname)
quicklook(pd, self.images, band_list, qlpath,
config.get("quicklook_jpeg_quality", 95))
metadata.mtd.get('quicklooks_H').append(qlpath)
else:
# grayscale QL
band_list = list(self.images.keys())
qlname = "_".join([
metadata.mtd.get('band_rootName_H'), 'QL', band_list[0]
]) + '.jpg'
qlpath = os.path.join(qi_dir, qlname)
quicklook(pd, self.images, band_list, qlpath,
config.get("quicklook_jpeg_quality", 95))
metadata.mtd.get('quicklooks_H').append(qlpath)
# PVI
band_list = ["B04", "B03", "B02"]
pvi_filename = "_".join([metadata.mtd.get('band_rootName_H'), 'PVI'
]) + '.TIF'
qlpath = os.path.join(qi_dir, pvi_filename)
quicklook(pd,
self.images,
band_list,
qlpath,
config.get("quicklook_jpeg_quality", 95),
xRes=320,
yRes=320,
creationOptions=['COMPRESS=LZW'],
format='GTIFF')
metadata.mtd.get('quicklooks_H').append(qlpath)
# Clear images as packager is the last process
self.images.clear()
# Write QI report as XML
bb_QI_path = metadata.hardcoded_values.get('bb_QIH_path')
out_QI_path = os.path.join(qi_dir, 'L2H_QI_Report.xml')
in_QI_path = glob.glob(
os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA',
'L2A_QI_Report.xml'))
log.info(
'QI report for input product found : {} (searched at {})'.format(
len(in_QI_path) != 0,
os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA',
'L2A_QI_Report.xml')))
in_QI_path = in_QI_path[0] if len(in_QI_path) != 0 else None
Qi_Writer = QiWriter(bb_QI_path,
outfile=out_QI_path,
init_QI_path=in_QI_path,
H_F='H')
Qi_Writer._manual_replaces(pd)
Qi_Writer.write(pretty_print=True)
# TODO UNCOMMENT BELOW FOR XSD CHECK
product_QI_xsd = metadata.hardcoded_values.get('product_QIH_xsd')
log.info('QI Report is valid : {}'.format(
Qi_Writer.validate_schema(product_QI_xsd, out_QI_path)))
# Write product MTD
bb_S2_product = metadata.hardcoded_values.get('bb_S2H_product')
bb_L8_product = metadata.hardcoded_values.get('bb_L8H_product')
product_mtd_path = 'MTD_{}L2H.xml'.format(
pd.mtl.sensor[0:3]) # MSI / OLI/ OLI_TIRS
product_MTD_outpath = os.path.join(tsdir, product_name,
product_mtd_path)
mtd_pd_writer = MTD_writer_S2(bb_S2_product, pd.mtl.mtl_file_name, H_F='H') if pd.sensor == 'S2' \
else MTD_writer_LS8(bb_L8_product, H_F='H')
mtd_pd_writer._manual_replaces(pd)
mtd_pd_writer.write(product_MTD_outpath, pretty_print=True)
# TODO UNCOMMENT BELOW FOR XSD CHECK
# product_mtd_xsd = metadata.hardcoded_values.get('product_mtd_xsd')
# log.info('Product MTD is valid : {}'.format(mtd_pd_writer.validate_schema(product_mtd_xsd,
# product_MTD_outpath)))
# Write tile MTD
bb_S2_tile = metadata.hardcoded_values.get('bb_S2H_tile')
bb_L8_tile = metadata.hardcoded_values.get('bb_L8H_tile')
tile_mtd_path = 'MTD_TL_L2H.xml'
tile_MTD_outpath = os.path.join(product_path, 'GRANULE',
granule_compact_name, tile_mtd_path)
mtd_tl_writer = MTD_tile_writer_S2(bb_S2_tile, pd.mtl.tile_metadata, H_F='H') if pd.sensor == 'S2' \
else MTD_tile_writer_LS8(bb_L8_tile, H_F='H')
mtd_tl_writer._manual_replaces(pd)
mtd_tl_writer.write(tile_MTD_outpath, pretty_print=True)
def manual_replaces(self, product):
# GENERAL_INFO
# ------------
elements_to_copy = ['./General_Info/Product_Info/Datatake',
'./General_Info/Product_Info/PRODUCT_START_TIME',
'./General_Info/Product_Info/PRODUCT_STOP_TIME',
'./General_Info/Product_Info/Query_Options',
]
copy_elements(elements_to_copy, self.root_in, self.root_out, self.root_bb)
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI',
new_value=metadata.mtd.get('product_{}_name'.format(self.H_F)))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL',
new_value='Level-2{}'.format(self.H_F))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE',
new_value='S2MSI2{}'.format(self.H_F))
pdgs = config.get('PDGS', '9999')
PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]])
AC = self.root_in.findall('.//ARCHIVING_CENTRE')
if AC:
metadata.mtd['S2_AC'] = AC[0].text
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS)
generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[
:-3] + 'Z' # -3 to keep only 3 decimals
change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time)
self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule')
for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))):
adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0]
create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
tag='IMAGE_FILE', text=adjusted_path)
grnl_id = \
find_element_by_path(self.root_in, './General_Info/Product_Info/Product_Organisation/Granule_List/Granule')
if grnl_id:
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=generate_S2_tile_id(product, self.H_F, metadata.mtd['S2_AC']),
attr_to_change='granuleIdentifier')
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat')
else:
pass # Fixme
# If Sbaf is done, we keep the values inside the backbone (S2A values)
if not config.getboolean('doSbaf'):
# copy_elements(['./General_Info/Product_Image_Characteristics/Special_Values',
# './General_Info/Product_Image_Characteristics/Image_Display_Order',
# './General_Info/Product_Image_Characteristics/Reflectance_Conversion',
# './General_Info/Product_Image_Characteristics/Spectral_Information_List'],
# self.root_in, self.root_out, self.root_bb)
pass
# FIXME : get product image characteristics from origin sensor (S2 here),
# copying from another template for example (see commented lines above)
copy_elements(['./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U'], self.root_in,
self.root_out)
# Geometric_info
# ---------------
tilecode = product.mtl.mgrs
if tilecode.startswith('T'):
tilecode = tilecode[1:]
footprint = search_db(tilecode, search='MGRS_REF')
# adding back first element, to get a complete polygon
fp = footprint.split(' ')
footprint = ' '.join(fp + [fp[0], fp[1]])
chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint)
copy_elements(['./Geometric_Info/Coordinate_Reference_System'], self.root_in, self.root_out, self.root_bb)
# Auxiliary_Data_Info
# -------------------
self.remove_children('./Auxiliary_Data_Info/GIPP_List')
copy_children(self.root_in, './Auxiliary_Data_Info/GIPP_List',
self.root_out, './Auxiliary_Data_Info/GIPP_List')
config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0]
create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn,
attribs={"version": pdgs, "type": "GIP_S2LIKE"})
for tag in ['PRODUCTION_DEM_TYPE',
'IERS_BULLETIN_FILENAME',
'ECMWF_DATA_REF',
'SNOW_CLIMATOLOGY_MAP',
'ESACCI_WaterBodies_Map',
'ESACCI_LandCover_Map',
'ESACCI_SnowCondition_Map_Dir']:
elem = find_element_by_path(self.root_in, './Auxiliary_Data_Info/' + tag)
if len(elem) != 0:
new_value = elem[0].text
else:
new_value = "NONE"
change_elm(self.root_out, rpath='./Auxiliary_Data_Info/' + tag, new_value=new_value)
# Fill GRI_List
gri_elems = self.root_in.findall('.//GRI_FILENAME')
for gri_elm in gri_elems:
create_child(self.root_out, './Auxiliary_Data_Info/GRI_List', tag="GRI_FILENAME", text=gri_elm.text)
# Quality_Indicators_Info
# -----------------------
copy_elements(['./Quality_Indicators_Info'], self.root_in, self.root_out, self.root_bb)
def process(self, product, image, band):
log.info('Start')
if not config.getboolean('hlsplus'):
log.warning(
'Skipping Data Fusion because doPackager and doPackagerL2F options are not activated'
)
log.info('End')
return image
# save into file before processing (packager will need it)
product.image30m[band] = image
if band == 'B01':
log.warning('Skipping Data Fusion for B01.')
log.info('End')
return image
if len(self.reference_products) == 0:
log.warning(
'Skipping Data Fusion. Reason: no S2 products available in the past'
)
log.info('End')
return image
if not product.get_s2like_band(band):
log.warning(
'Skipping Data Fusion. Reason: no S2 matching band for {}'.
format(band))
log.info('End')
return image
# method selection
predict_method = config.get('predict_method', 'predict')
if len(self.reference_products) == 1:
log.warning(
'Not enough Sentinel2 products for the predict (only one product). Using last S2 product as ref.'
)
predict_method = 'composite'
# general info
band_s2 = product.get_s2like_band(band)
image_file_L2F = self.reference_products[0].get_band_file(band_s2,
plus=True)
output_shape = (image_file_L2F.ySize, image_file_L2F.xSize)
# method: prediction (from the 2 most recent S2 products)
if predict_method == 'predict':
# Use QA (product selection) to apply Composting :
qa_mask = self._get_qa_band(output_shape)
# predict
array_L2H_predict, array_L2F_predict = self._predict(
product, band_s2, qa_mask, output_shape)
# save
if config.getboolean('generate_intermediate_products'):
self._save_as_image_file(image_file_L2F, qa_mask, product,
band, '_FUSION_QA.TIF')
self._save_as_image_file(image_file_L2F, array_L2H_predict,
product, band,
'_FUSION_L2H_PREDICT.TIF')
self._save_as_image_file(image_file_L2F, array_L2F_predict,
product, band,
'_FUSION_L2F_PREDICT.TIF')
# method: composite (most recent valid pixels from N products)
elif predict_method == 'composite':
# composite
array_L2H_predict, array_L2F_predict = self._composite(
product, band_s2, output_shape)
# save
if config.getboolean('generate_intermediate_products'):
self._save_as_image_file(image_file_L2F, array_L2H_predict,
product, band,
'_FUSION_L2H_COMPO.TIF')
self._save_as_image_file(image_file_L2F, array_L2F_predict,
product, band,
'_FUSION_L2F_COMPO.TIF')
# method: unknown
else:
log.error(
f'Unknown predict method: {predict_method}. Please check your configuration.'
)
return None
# fusion L8/S2
mask_filename = product.mtl.nodata_mask_filename
array_out = self._fusion(image, array_L2H_predict, array_L2F_predict,
mask_filename).astype(np.float32)
image = self._save_as_image_file(image_file_L2F, array_out, product,
band, '_FUSION_L2H_PREDICT.TIF')
log.info('End')
return image
