def process(self, pd, image, band):
"""
Write final product in the archive directory
'archive_dir' is defined in config.ini file
Naming convention from Design Document
:param pd: instance of S2L_Product class
:param image: input instance of S2L_ImageFile class
:param band: band being processed
:return: output instance of instance of S2L_ImageFile class
"""
# TODO : add production date?
# /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR
res = image.xRes
product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L(
pd)
sensor = pd.sensor_name
relative_orbit = config.get('relative_orbit')
native = band in ['B08', 'B10', 'B11'] if sensor == 'LS8' else \
band in ['B05', 'B06', 'B07', 'B08']
s2_band = pd.get_s2like_band(band)
if not native:
band = s2_band
band_rootName = "_".join([
'L2F', 'T' + tilecode, datatake_sensing_start, sensor,
'R{:0>3}'.format(relative_orbit)
])
metadata.mtd['band_rootName_F'] = band_rootName
outfile = "_".join([band_rootName, band, '{}m'.format(int(res))
]) + '.TIF'
# Naming convention from Sentinel-2-Products-Specification-Document (p294)
tsdir = os.path.join(config.get('archive_dir'),
tilecode) # ts = temporal series
newpath = self.band_path(tsdir,
product_name,
granule_compact_name,
outfile,
native=native)
COG = config.getboolean('COG')
log.debug('New: ' + newpath)
image.write(creation_options=['COMPRESS=LZW'],
filepath=newpath,
COG=COG,
band=band)
metadata.mtd.get('bands_path_F').append(newpath)
# declare output internally
self.images[s2_band] = image.filepath
# declare output in config file
config.set('imageout_dir', image.dirpath)
config.set('imageout_' + band, image.filename)
return image
def smac_correction_grid(obs_datetime, extent, hcs_code, resolution=120):
output_filename = 'output_file.tif'
ecmwf_data = EMWF_Product(
config.get('cams_dir'),
cams_hourly_directory=config.get('cams_hourly_dir'),
cams_climatology_directory=config.get('cams_climatology_dir'),
observation_datetime=obs_datetime)
new_SRS = gdal.osr.SpatialReference()
new_SRS.ImportFromEPSG(int(4326))
if ecmwf_data.is_valid:
# Write cams file
cams_file = 'cams_file.tif'
etype = gdal.GDT_Float32
driver = gdal.GetDriverByName('GTiff')
dst_ds = driver.Create(cams_file,
xsize=ecmwf_data.longitude.size,
ysize=ecmwf_data.latitude.size,
bands=4,
eType=etype,
options=[])
dst_ds.SetProjection(new_SRS.ExportToWkt())
x_res = (ecmwf_data.longitude.max() -
ecmwf_data.longitude.min()) / ecmwf_data.longitude.size
y_res = (ecmwf_data.latitude.max() -
ecmwf_data.latitude.min()) / ecmwf_data.latitude.size
geotranform = (ecmwf_data.longitude.min(), x_res, 0,
ecmwf_data.latitude.max(), 0, -y_res)
dst_ds.SetGeoTransform(geotranform)
dst_ds.GetRasterBand(1).WriteArray(ecmwf_data.aod550.astype(np.float))
dst_ds.GetRasterBand(2).WriteArray(ecmwf_data.tcwv.astype(np.float))
dst_ds.GetRasterBand(3).WriteArray(ecmwf_data.gtco3.astype(np.float))
dst_ds.GetRasterBand(4).WriteArray(ecmwf_data.msl.astype(np.float))
dst_ds.FlushCache()
# Warp cams data on input spatial extent
options = gdal.WarpOptions(srcSRS=dst_ds.GetProjection(),
dstSRS=hcs_code,
xRes=resolution,
yRes=resolution,
resampleAlg='cubicspline',
outputBounds=extent)
gdal.Warp(output_filename, cams_file, options=options)
dst_ds = None
return output_filename
def base_path(product):
relative_orbit = config.get('relative_orbit')
acqdate = dt.datetime.strftime(product.acqdate, '%Y%m%d')
tilecode = product.mtl.mgrs
if tilecode.startswith('T'):
tilecode = tilecode[1:]
return "_".join(['L2F', tilecode, acqdate, product.sensor_name, 'R{:0>3}'.format(relative_orbit)]), tilecode
def preprocess(self, product):
product_name, granule_compact_name, tilecode, _ = self.base_path_S2L(
product)
metadata.mtd['product_H_name'] = product_name
metadata.mtd['granule_H_name'] = granule_compact_name
metadata.mtd['product_creation_date'] = metadata.mtd.get(
'product_creation_date', dt.datetime.now())
outdir = os.path.join(config.get('archive_dir'), tilecode)
"""
# Creation of S2 folder tree structure
tree = core.QI_MTD.S2_structure.generate_S2_structure_XML(out_xml='', product_name=product_name,
tile_name=granule_compact_name, save_xml=False)
core.QI_MTD.S2_structure.create_architecture(outdir, tree, create_empty_files=True)
"""
log.debug('Create folder : ' + os.path.join(outdir, product_name))
change_nodes = {
'PRODUCT_NAME': product_name,
'TILE_NAME': granule_compact_name,
}
core.QI_MTD.S2_structure.create_architecture(
outdir,
metadata.hardcoded_values.get('s2_struct_xml'),
change_nodes=change_nodes,
create_empty_files=False)
def update_configuration(args, tile=None):
# init S2L_config and save to wd
if not config.initialize(args.S2L_configfile):
return
if args.confParams is not None:
config.overload(args.confParams)
config.set('wd', os.path.join(args.wd, str(os.getpid())))
references_map_file = config.get('references_map')
if args.refImage:
config.set('refImage', args.refImage)
elif references_map_file and tile:
# load dataset
with open(references_map_file) as j:
references_map = json.load(j)
config.set('refImage', references_map.get(tile))
else:
config.set('refImage', None)
config.set(
'hlsplus',
config.getboolean('doPackager') or config.getboolean('doPackagerL2F'))
config.set('debug', args.debug)
config.set('generate_intermediate_products',
args.generate_intermediate_products)
if hasattr(args, 'l2a'):
config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def update_configuration(args, tile=None):
# init S2L_config and save to wd
if not config.initialize(args.S2L_configfile):
return
if args.confParams is not None:
config.overload(args.confParams)
use_pid = False
if use_pid:
output_folder = str(os.getpid())
else:
date_now = datetime.datetime.now().strftime('%Y%m%dT_%H%M%S')
output_folder = f'{"" if args.no_log_date else f"{date_now}_"}{compute_config_hash(args, config)}'
config.set('wd', os.path.join(args.wd, output_folder))
references_map_file = config.get('references_map')
if args.refImage:
config.set('refImage', args.refImage)
elif references_map_file and tile:
# load dataset
with open(references_map_file) as j:
references_map = json.load(j)
config.set('refImage', references_map.get(tile))
else:
config.set('refImage', None)
config.set(
'hlsplus',
config.getboolean('doPackager') or config.getboolean('doPackagerL2F'))
config.set('debug', args.debug)
config.set('generate_intermediate_products',
args.generate_intermediate_products)
if hasattr(args, 'l2a'):
config.set('s2_processing_level', 'LEVEL2A' if args.l2a else "LEVEL1C")
def main(with_multiprocess_support=False):
parser = configure_arguments()
args = parser.parse_args()
log.configure_loggers(log_path=args.wd,
is_debug=args.debug,
without_date=args.no_log_date)
if args.operational_mode is None:
parser.print_help()
return 1
# convert list of bands if provided
if args.bands is not None:
args.bands = args.bands.split(',')
products, start_date, end_date = configure_sen2like(args)
if products is None:
return 1
if args.operational_mode == 'multi-tile-mode' and with_multiprocess_support and not args.no_run:
number_of_process = args.jobs
if number_of_process is None:
number_of_process = config.get('number_of_process', 1)
params = [(tile, _products, args, start_date, end_date)
for tile, _products in products.items()]
with Pool(int(number_of_process)) as pool:
pool.starmap(start_process, params)
else:
if args.no_run:
logger.info("No-run mode: Products will only be listed")
for tile, _products in products.items():
start_process(tile, _products, args, start_date, end_date)
return 0
def stitch_multi(product, product_file, new_product_file):
ds_product_src = gdal.Open(product_file)
ds_new_product_src = gdal.Open(new_product_file)
filepath_out = os.path.join(config.get('wd'), product.name, 'tie_points_STITCHED.TIF')
for i in range(1, ds_product_src.RasterCount + 1):
array_product = ds_product_src.GetRasterBand(i).ReadAsArray()
array_new_product = ds_new_product_src.GetRasterBand(i).ReadAsArray()
np.copyto(array_product, array_new_product, where=array_product == 0)
if i == 1:
# write with gdal
driver = gdal.GetDriverByName('GTiff')
ds_dst = driver.Create(filepath_out, bands=ds_product_src.RasterCount,
xsize=ds_product_src.RasterXSize, ysize=ds_product_src.RasterYSize,
eType=gdal.GDT_Int16)
ds_dst.SetProjection(ds_product_src.GetProjection())
ds_dst.SetGeoTransform(ds_product_src.GetGeoTransform())
# write band
ds_dst.GetRasterBand(i).WriteArray(array_product)
ds_dst.FlushCache()
ds_product_src = None
ds_new_product_src = None
ds_dst = None
return filepath_out
def base_path_S2L(product):
"""
See https://sentinel.esa.int/web/sentinel/user-guides/sentinel-2-msi/naming-convention
More information https://sentinel.esa.int/documents/247904/685211/Sentinel-2-Products-Specification-Document
at p74, p438
Needed parameters : datastrip sensing start
datatake sensing start
absolute orbit
relative orbit
product generation time
Product baseline number
"""
relative_orbit = config.get('relative_orbit')
file_date = dt.datetime.strftime(product.file_date,
'%Y%m%dT%H%M%S') # generation time
if product.sensor == 'S2':
datatake_sensing_start = dt.datetime.strftime(
product.dt_sensing_start, '%Y%m%dT%H%M%S')
datastrip_sensing_start = dt.datetime.strftime(
product.ds_sensing_start, '%Y%m%dT%H%M%S')
absolute_orbit = config.get('absolute_orbit')
else:
datatake_sensing_start = dt.datetime.strftime(
product.acqdate, '%Y%m%dT%H%M%S')
datastrip_sensing_start = file_date
absolute_orbit = metadata.hardcoded_values.get('L8_absolute_orbit')
PDGS = metadata.hardcoded_values.get('PDGS')
tilecode = product.mtl.mgrs
if tilecode.startswith('T'):
tilecode = tilecode[1:]
sensor = product.mtl.sensor[0:3] # OLI / MSI / OLI_TIRS
product_name = "_".join([
product.sensor_name, '{}L2H'.format(sensor),
datatake_sensing_start, 'N' + PDGS,
'R{:0>3}'.format(relative_orbit), 'T' + tilecode, file_date
]) + '.SAFE'
granule_compact_name = "_".join([
'L2H', 'T' + tilecode, 'A' + absolute_orbit,
datastrip_sensing_start, product.sensor_name,
'R{:0>3}'.format(relative_orbit)
])
return product_name, granule_compact_name, tilecode, datatake_sensing_start
def _save_as_image_file(self, image_template, array, product, band,
extension):
path = os.path.join(config.get('wd'), product.name,
product.get_band_file(band).rootname + extension)
image_file = image_template.duplicate(path, array=array)
if config.getboolean('generate_intermediate_products'):
image_file.write(creation_options=['COMPRESS=LZW'])
return image_file
def generate_S2_tile_id(product, H_F, AC):
tilecode = product.mtl.mgrs
pdgs = metadata.hardcoded_values.get('PDGS', '9999')
PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]])
acqdate = dt.datetime.strftime(product.acqdate, '%Y%m%dT%H%M%S')
if AC.endswith('_'):
AC = AC[:-1]
tile_id = '_'.join([product.sensor_name, 'OPER', 'MSI', 'L2{}'.format(H_F), AC, acqdate,
'A{}'.format(config.get('absolute_orbit')), tilecode, 'N{}'.format(PDGS)])
return tile_id
def reframe(self, image, product, band=None, dtype=None):
# Add margin
margin = int(config.get('reframe_margin', self.margin))
log.debug(f"Using {margin} as margin.")
product_image = mgrs_framing.reframe(image, self.tile,
filepath_out=self.output_file(product, band, image, "_PREREFRAMED"),
order=0, margin=margin, dtype=dtype, compute_offsets=True)
if config.getboolean('generate_intermediate_products'):
product_image.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count
return product_image
def preprocess(self, product):
self.get_new_product(product)
if self.new_product is None:
return
# Stitch validity mask and angles
product_validity_masks = []
product_nodata_masks = []
product_angles = []
for _product in [product, self.new_product.reader(self.new_product.path)]:
is_mask_valid = True
# Validity mask
if _product.mtl.mask_filename is None:
is_mask_valid = _product.mtl.get_valid_pixel_mask(
os.path.join(config.get("wd"), _product.name, 'valid_pixel_mask.tif'))
if is_mask_valid:
product_validity_masks.append(self.reframe(S2L_ImageFile(_product.mtl.mask_filename), _product))
product_nodata_masks.append(self.reframe(S2L_ImageFile(_product.mtl.nodata_mask_filename), _product))
# Angles
if _product.mtl.angles_file is None:
_product.mtl.get_angle_images(os.path.join(config.get("wd"), _product.name, 'tie_points.tif'))
filepath_out = os.path.join(config.get('wd'), _product.name, 'tie_points_PREREFRAMED.TIF')
mgrs_framing.reframeMulti(_product.mtl.angles_file, self.tile, filepath_out=filepath_out, order=0)
product_angles.append(filepath_out)
if None not in product_validity_masks:
stitched_mask = self.stitch(product, product_validity_masks[0], product_validity_masks[1])
stitched_mask.write(creation_options=['COMPRESS=LZW'])
product.mtl.mask_filename = stitched_mask.filepath
if None not in product_nodata_masks:
stitched_mask = self.stitch(product, product_nodata_masks[0], product_nodata_masks[1])
stitched_mask.write(creation_options=['COMPRESS=LZW'])
product.mtl.nodata_mask_filename = stitched_mask.filepath
stitched_angles = self.stitch_multi(product, product_angles[0], product_angles[1])
product.mtl.angles_file = stitched_angles
# Stitch reference band (needed by geometry module)
band = config.get('reference_band', 'B04')
if config.getboolean('doMatchingCorrection') and config.get('refImage'):
image = product.get_band_file(band)
self.process(product, image, band)
def preprocess(self, product):
# reinit dx/dy
config.set('dx', 0)
config.set('dy', 0)
if product.sensor != 'S2':
# Reframe angles and masks
filepath_out = os.path.join(config.get('wd'), product.name, 'tie_points_REFRAMED.TIF')
mgrs_framing.reframeMulti(product.mtl.angles_file, product.mtl.mgrs, filepath_out, config.getfloat('dx'),
config.getfloat('dy'), order=0)
product.mtl.angles_file = filepath_out
# Reframe mask
if product.mtl.mask_filename:
filepath_out = os.path.join(config.get('wd'), product.name, 'valid_pixel_mask_REFRAMED.TIF')
image = S2L_ImageFile(product.mtl.mask_filename)
imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'),
config.getfloat('dy'), order=0)
imageout.write(creation_options=['COMPRESS=LZW'])
product.mtl.mask_filename = filepath_out
# Reframe nodata mask
if product.mtl.nodata_mask_filename:
filepath_out = os.path.join(config.get('wd'), product.name, 'nodata_pixel_mask_REFRAMED.TIF')
image = S2L_ImageFile(product.mtl.nodata_mask_filename)
imageout = mgrs_framing.reframe(image, product.mtl.mgrs, filepath_out, config.getfloat('dx'),
config.getfloat('dy'), order=0)
imageout.write(creation_options=['COMPRESS=LZW'])
product.mtl.nodata_mask_filename = filepath_out
# Matching for dx/dy correction?
band = config.get('reference_band', 'B04')
if config.getboolean('doMatchingCorrection') and config.get('refImage'):
config.set('freeze_dx_dy', False)
image = product.get_band_file(band)
self.process(product, image, band)
# goal is to feed dx, dy in config
config.set('freeze_dx_dy', True)
metadata.qi.update({'COREGISTRATION_BEFORE_CORRECTION': self._tmp_stats.get('MEAN'.format(band))})
def convert_to_reflectance_from_reflectance_cal_product(mtl, data_in, band):
"""Applied conversion to TOA reflectance, assuming product is calibrated
in reflectance as S2 MSI and LS8 OLI
Required mtl.processing_dic, a dictionary including the list of band
to be processing"""
log.debug("Conversion to TOA")
reflectance_data = None
if mtl.sensor == 'OLI' or mtl.sensor == 'OLI_TIRS':
# LANDSAT 8
log.info("Sun Zenith angle : {} deg".format(mtl.sun_zenith_angle))
sun_elevation_angle = 90. - mtl.sun_zenith_angle
log.info("Sun Elevation angle : {} deg".format(sun_elevation_angle))
gain = offset = None
for k, x in list(mtl.rho_radio_coefficient_dic.items()):
if 'B' + x['Band_id'] == band:
gain = str(x['Gain'])
offset = str(x['Offset'])
log.info('Band Id : {} Gain : {} / Offset : {}'.format(
x['Band_id'], gain, offset))
if gain is not None and offset is not None:
reflectance_data = (np.float32(data_in) * np.float32(gain) +
np.float32(offset)) / np.sin(
sun_elevation_angle * np.pi / 180.)
mask = (data_in <= 0)
reflectance_data[mask] = 0
elif band in ('B10', 'B11'):
offset = float(config.get('offset'))
gain = float(config.get('gain'))
reflectance_data = np.float32(data_in) / gain - offset
mask = (data_in <= 0)
reflectance_data[mask] = 0
elif mtl.sensor == 'MSI':
# apply quantification value
reflectance_data = np.float32(data_in) / float(
mtl.quantification_value)
return reflectance_data
def process(self, pd, image, band):
"""
Write final product in the archive directory
'archive_dir' is defined in config.ini file
Naming convention from Design Document
:param pd: instance of S2L_Product class
:param image: input instance of S2L_ImageFile class
:param band: band being processed
:return: outpu t instance of instance of S2L_ImageFile class
"""
# TODO : add production date?
# TODO : change L8 band numbers to S2 numbers convention?
# /data/HLS_DATA/Archive/Site_Name/TILE_ID/S2L_DATEACQ_DATEPROD_SENSOR/S2L_DATEACQ_DATEPROD_SENSOR
res = image.xRes
outdir, tilecode = self.base_path(pd)
outfile = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF'
tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series
newpath = os.path.join(tsdir, outdir, outfile)
log.debug('New: ' + newpath)
image.write(creation_options=['COMPRESS=LZW'], filepath=newpath)
# declare output internally
self.images[band] = image.filepath
# declare output in config file
config.set('imageout_dir', image.dirpath)
config.set('imageout_' + band, image.filename)
if config.getboolean('hlsplus'):
res = 30
outfile_30m = "_".join([outdir, band, '{}m'.format(int(res))]) + '.TIF'
newpath_30m = os.path.join(tsdir, outdir, outfile_30m)
if pd.sensor == 'S2':
# create 30m band as well
# resampling
log.info('Resampling to 30m: Start...')
image_30m = mgrs_framing.resample(S2L_ImageFile(newpath), res, newpath_30m)
image_30m.write(creation_options=['COMPRESS=LZW'], DCmode=True) # digital count
log.info('Resampling to 30m: End')
if pd.sensor == 'L8' and band in pd.image30m:
# copy 30m band as well
# write
pd.image30m[band].write(creation_options=['COMPRESS=LZW'], filepath=newpath_30m)
del pd.image30m[band]
return image
def preprocess(self, pd):
# check most recent HLS S2 products available
archive_dir = config.get('archive_dir')
tsdir = join(archive_dir, pd.mtl.mgrs)
# list products with dates
pdlist = []
for pdpath in sorted(glob.glob(tsdir + '/L2F_*_S2*')):
pdname = basename(pdpath)
date = dt.datetime.strptime(pdname.split('_')[2], '%Y%m%d').date()
if date <= pd.acqdate.date():
pdlist.append([date, pdpath])
# Handle new format aswell
for pdpath in sorted(glob.glob(tsdir + '/S2*L2F_*')):
pdname = basename(pdpath)
date = dt.datetime.strptime(
os.path.splitext(pdname.split('_')[2])[0],
'%Y%m%dT%H%M%S').date()
if date <= pd.acqdate.date():
pdlist.append([date, pdpath])
# sort by date
pdlist.sort()
# reset ref list and keep 2 last ones
self.reference_products = []
nb_products = int(config.get('predict_nb_products', 2))
for date, pdname in pdlist[-nb_products:]:
product = S2L_HLS_Product(pdname)
if product.product is not None:
self.reference_products.append(product)
for product in self.reference_products:
log.info('Selected product: {}'.format(product.name))
def read_metadata(self, granule_folder='GRANULE'):
# extract metadata
self.mtl = readers.get_reader(self.path)
if self.mtl is None:
return
self.mtl = self.mtl(self.path)
try:
self.update_site_info(config.get('tile', None))
except AttributeError:
# Some products not need to update their site information
pass
# retrieve acquisition date in a datetime format
scene_center_time = self.mtl.scene_center_time
n = len(
self.mtl.scene_center_time.split('.')[-1]) - 1 # do not count Z
if n < 6:
# fill with zeros
scene_center_time = self.mtl.scene_center_time.replace(
'Z', (6 - n) * '0' + 'Z')
self.acqdate = dt.datetime.strptime(
self.mtl.observation_date + ' ' + scene_center_time,
"%Y-%m-%d %H:%M:%S.%fZ")
if 'S2' in self.sensor or self.mtl.data_type in [
'Level-2F', 'Level-2H'
]: # Sentinel 2
self.dt_sensing_start = dt.datetime.strptime(
self.mtl.dt_sensing_start, "%Y-%m-%dT%H:%M:%S.%fZ")
self.ds_sensing_start = dt.datetime.strptime(
self.mtl.ds_sensing_start, "%Y-%m-%dT%H:%M:%S.%fZ")
self.file_date = dt.datetime.strptime(self.mtl.file_date,
"%Y-%m-%dT%H:%M:%S.%fZ")
logger.debug("Datatake sensing start: {}".format(
self.dt_sensing_start))
logger.debug("Datastrip sensing start: {}".format(
self.ds_sensing_start))
else:
self.file_date = dt.datetime.strptime(self.mtl.file_date,
"%Y-%m-%dT%H:%M:%SZ")
logger.debug("Product generation time: {}".format(self.file_date))
logger.debug("Acquisition Date: {}".format(self.acqdate))
def output_file(self, product, band, extension=None):
if extension is None:
extension = self.ext
return os.path.join(config.get('wd'), product.name, product.get_band_file(band).rootname + extension)
def process(self, product, image, band):
wd = os.path.join(config.get('wd'), product.name)
self._output_file = self.output_file(product, band)
self._tmp_stats = {}
log.info('Start')
# MGRS reframing for Landsat8
if product.sensor == 'L8':
log.debug('{} {}'.format(config.getfloat('dx'), config.getfloat('dy')))
image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy'))
# Resampling to 30m for S2 (HLS)
elif product.sensor == 'S2':
if not config.getboolean('hlsplus'):
image = self._resample(image)
else:
# refine geometry
# if config.getfloat('dx') > 0.3 or config.getfloat('dy') > 0.3:
log.debug("{} {}".format(config.getfloat('dx'), config.getfloat('dy')))
image = self._reframe(product, image, config.getfloat('dx'), config.getfloat('dy'))
# matching for having QA stats
if config.get('refImage'):
# try to adapt resolution, changing end of reference filename
refImage_path = config.get('refImage')
if not os.path.exists(refImage_path):
return image
# open image ref
imageref = S2L_ImageFile(refImage_path)
# if refImage resolution does not fit
if imageref.xRes != image.xRes:
# new refImage filepath
refImage_noext = os.path.splitext(refImage_path)[0]
if refImage_noext.endswith(f"_{int(imageref.xRes)}m"):
refImage_noext = refImage_noext[:-len(f"_{int(imageref.xRes)}m")]
refImage_path = refImage_noext + f"_{int(image.xRes)}m.TIF"
# compute (resample), or load if exists
if not os.path.exists(refImage_path):
log.info("Resampling of the reference image")
# compute
imageref = mgrs_framing.resample(imageref, image.xRes, refImage_path)
# write for reuse
imageref.write(DCmode=True, creation_options=['COMPRESS=LZW'])
else:
# or load if exists
log.info("Change reference image to:" + refImage_path)
imageref = S2L_ImageFile(refImage_path)
# open mask
mask = S2L_ImageFile(product.mtl.mask_filename)
if config.getboolean('freeze_dx_dy'):
# do Geometry Assessment only if required
assess_geometry_bands = config.get('doAssessGeometry', default='').split(',')
if product.sensor != 'S2':
assess_geometry_bands = [product.reverse_bands_mapping.get(band) for band in assess_geometry_bands]
if assess_geometry_bands and band in assess_geometry_bands:
log.info("Geometry assessment for band %s" % band)
# Coarse resolution of correlation grid (only for stats)
self._matching(imageref, image, wd, mask)
else:
# Fine resolution of correlation grid (for accurate dx dy computation)
dx, dy = self._matching(imageref, image, wd, mask)
# save values for correction on bands
config.set('dx', dx)
config.set('dy', dy)
log.info("Geometrical Offsets (DX/DY): {}m {}m".format(config.getfloat('dx'), config.getfloat('dy')))
# Append bands name to keys
for key, item in self._tmp_stats.items():
if config.get('reference_band') != band:
self._tmp_stats[key+'_{}'.format(band)] = self._tmp_stats.pop(key)
metadata.qi.update(self._tmp_stats)
log.info('End')
return image
def process(self, product, image, band):
log.info('Start')
if not config.getboolean('hlsplus'):
log.warning(
'Skipping Data Fusion because doPackager and doPackagerL2F options are not activated'
)
log.info('End')
return image
# save into file before processing (packager will need it)
product.image30m[band] = image
if band == 'B01':
log.warning('Skipping Data Fusion for B01.')
log.info('End')
return image
if len(self.reference_products) == 0:
log.warning(
'Skipping Data Fusion. Reason: no S2 products available in the past'
)
log.info('End')
return image
if not product.get_s2like_band(band):
log.warning(
'Skipping Data Fusion. Reason: no S2 matching band for {}'.
format(band))
log.info('End')
return image
# method selection
predict_method = config.get('predict_method', 'predict')
if len(self.reference_products) == 1:
log.warning(
'Not enough Sentinel2 products for the predict (only one product). Using last S2 product as ref.'
)
predict_method = 'composite'
# general info
band_s2 = product.get_s2like_band(band)
image_file_L2F = self.reference_products[0].get_band_file(band_s2,
plus=True)
output_shape = (image_file_L2F.ySize, image_file_L2F.xSize)
# method: prediction (from the 2 most recent S2 products)
if predict_method == 'predict':
# Use QA (product selection) to apply Composting :
qa_mask = self._get_qa_band(output_shape)
# predict
array_L2H_predict, array_L2F_predict = self._predict(
product, band_s2, qa_mask, output_shape)
# save
if config.getboolean('generate_intermediate_products'):
self._save_as_image_file(image_file_L2F, qa_mask, product,
band, '_FUSION_QA.TIF')
self._save_as_image_file(image_file_L2F, array_L2H_predict,
product, band,
'_FUSION_L2H_PREDICT.TIF')
self._save_as_image_file(image_file_L2F, array_L2F_predict,
product, band,
'_FUSION_L2F_PREDICT.TIF')
# method: composite (most recent valid pixels from N products)
elif predict_method == 'composite':
# composite
array_L2H_predict, array_L2F_predict = self._composite(
product, band_s2, output_shape)
# save
if config.getboolean('generate_intermediate_products'):
self._save_as_image_file(image_file_L2F, array_L2H_predict,
product, band,
'_FUSION_L2H_COMPO.TIF')
self._save_as_image_file(image_file_L2F, array_L2F_predict,
product, band,
'_FUSION_L2F_COMPO.TIF')
# method: unknown
else:
log.error(
f'Unknown predict method: {predict_method}. Please check your configuration.'
)
return None
# fusion L8/S2
mask_filename = product.mtl.nodata_mask_filename
array_out = self._fusion(image, array_L2H_predict, array_L2F_predict,
mask_filename).astype(np.float32)
image = self._save_as_image_file(image_file_L2F, array_out, product,
band, '_FUSION_L2H_PREDICT.TIF')
log.info('End')
return image
def start_process(tile, products, args, start_date, end_date):
update_configuration(args, tile)
config.set('tile', tile)
logger.debug("Processing tile {}".format(tile))
downloader = InputProductArchive(config)
_products = downloader.get_products_from_urls(
products,
start_date,
end_date,
product_mode=args.operational_mode == 'product-mode')
if args.no_run:
logger.info("Tile: %s" % tile)
if not _products:
logger.info("No products found.")
for product in _products:
tile_message = f'[ Tile coverage = {product.tile_coverage:6.0f}% ]' \
if product.tile_coverage is not None else ''
cloud_message = f'[ Cloud coverage = {product.cloud_cover:6.0f}% ]' \
if product.cloud_cover is not None else ''
logger.info("%s %s %s" %
(tile_message, cloud_message, product.path))
return
for product in _products:
_product = None
if config.getboolean('use_sen2cor'):
# Disable Atmospheric correction
config.overload('doAtmcor=False')
# Run sen2core
logger.debug("<<< RUNNING SEN2CORE... >>>")
sen2cor_command = os.path.abspath(config.get('sen2cor_path'))
sen2cor_output_dir = os.path.join(config.get('wd'), 'sen2cor',
os.path.basename(product.path))
if not os.path.exists(sen2cor_output_dir):
os.makedirs(sen2cor_output_dir)
try:
subprocess.run([
'python', sen2cor_command, product.path, "--output_dir",
sen2cor_output_dir, "--work_dir", sen2cor_output_dir,
"--sc_only"
],
check=True)
except subprocess.CalledProcessError as run_error:
logger.error("An error occurred during the run of sen2cor")
logger.error(run_error)
continue
# Read output product
generated_product = next(os.walk(sen2cor_output_dir))[1]
if len(generated_product) != 1:
logger.error("Sen2Cor error: Cannot get output product")
continue
_product = product.reader(
os.path.join(sen2cor_output_dir, generated_product[0]))
if _product is None:
_product = product.reader(product.path)
# Update processing configuration
config.set('productName', _product.name)
config.set('sensor', _product.sensor)
config.set('observation_date', _product.mtl.observation_date)
config.set('relative_orbit', _product.mtl.relative_orbit)
config.set('absolute_orbit', _product.mtl.absolute_orbit)
config.set('mission', _product.mtl.mission)
config.set('none_S2_product_for_fusion', False)
# Disable Atmospheric correction for Level-2A products
atmcor = config.get('doAtmcor')
if _product.mtl.data_type in ('Level-2A', 'L2TP', 'L2A'):
config.overload('s2_processing_level=LEVEL2A')
logger.info(
"Processing Level-2A product: Atmospheric correction is disabled."
)
config.overload('doAtmcor=False')
else:
config.overload('s2_processing_level=LEVEL1C')
process(_product, args)
# Restore atmcor status
config.overload(f'doAtmcor={atmcor}')
del _product
if len(_products) == 0:
logger.error('No product for tile %s' % tile)
def process(product, args):
"""Launch process on product."""
bands = args.bands
# create working directory and save conf (traceability)
if not os.path.exists(os.path.join(config.get("wd"), product.name)):
os.makedirs(os.path.join(config.get("wd"), product.name))
# displays
logger.debug("{} {}".format(product.sensor, product.path))
# list of the blocks that are available
list_of_blocks = tuple(S2L_config.PROC_BLOCKS.keys())
# copy MTL files in wd
wd = os.path.join(config.get("wd"), product.name)
# copy MTL files in final product
shutil.copyfile(
product.mtl.mtl_file_name,
os.path.join(wd, os.path.basename(product.mtl.mtl_file_name)))
if product.mtl.tile_metadata:
shutil.copyfile(
product.mtl.tile_metadata,
os.path.join(wd, os.path.basename(product.mtl.tile_metadata)))
# Angles extraction
product.mtl.get_angle_images(
os.path.join(config.get("wd"), product.name, 'tie_points.tif'))
product.mtl.get_valid_pixel_mask(
os.path.join(config.get("wd"), product.name, 'valid_pixel_mask.tif'))
# !! Initialization of each block
for block_name in list_of_blocks:
get_module(block_name).initialize()
# !! Pre processing !!
# Run the preprocessing method of each block
for block_name in list_of_blocks:
generic_process_step(block_name, product, "preprocess")
# !! Processing !!
# save S2L_config file in wd
config.savetofile(
os.path.join(config.get('wd'), product.name, 'processing_start.cfg'))
# For each band or a selection of bands:
if bands is None:
# get all bands
bands = product.bands
elif product.sensor != 'S2':
bands = [
product.reverse_bands_mapping.get(band, band) for band in bands
]
if args.parallelize_bands:
# Multi processus
params = [(product, band, list_of_blocks, config, mtd.metadata,
PROCESS_INSTANCES) for band in bands]
with Pool() as pool:
results = pool.starmap(process_band, params)
bands_filenames, packager_files, configs, updated_metadatas = zip(
*results)
if configs and configs[0].parser is not None:
S2L_config.config = configs[0]
if updated_metadatas:
for updated_metadata in updated_metadatas:
mtd.metadata.update(updated_metadata)
for packager_file in packager_files:
for process_instance in packager_file:
PROCESS_INSTANCES[process_instance].images.update(
packager_file[process_instance])
for band, filename in PROCESS_INSTANCES[
process_instance].images.items():
S2L_config.config.set('imageout_dir',
os.path.dirname(filename))
S2L_config.config.set('imageout_' + band,
os.path.basename(filename))
else:
# Single processus
bands_filenames = []
for band in bands:
# process the band through each block
bands_filenames.append(
process_band(product, band, list_of_blocks, config,
mtd.metadata)) # Save image path
if bands_filenames == [None] * len(bands_filenames):
logger.error("No valid band provided for input product.")
logger.error("Valids band for products are: %s" %
str(list(product.bands)))
return
# !! Post processing !!
# Run the postprocessing method of each block
for block_name in list_of_blocks:
generic_process_step(block_name, product, "postprocess")
# Clear metadata
mtd.metadata.clear()
# save S2L_config file in wd
S2L_config.config.savetofile(
os.path.join(S2L_config.config.get('wd'), product.name,
'processing_end.cfg'))
def process(product, bands):
"""Launch process on product."""
# create working directory and save conf (traceability)
if not os.path.exists(os.path.join(config.get("wd"), product.name)):
os.makedirs(os.path.join(config.get("wd"), product.name))
# displays
logger.debug("{} {}".format(product.sensor, product.path))
# list of the blocks that are available
list_of_blocks = S2L_config.PROC_BLOCKS.keys()
# copy MTL files in wd
wd = os.path.join(config.get("wd"), product.name)
# copy MTL files in final product
shutil.copyfile(
product.mtl.mtl_file_name,
os.path.join(wd, os.path.basename(product.mtl.mtl_file_name)))
if product.mtl.tile_metadata:
shutil.copyfile(
product.mtl.tile_metadata,
os.path.join(wd, os.path.basename(product.mtl.tile_metadata)))
# Angles extraction
product.mtl.get_angle_images(
os.path.join(config.get("wd"), product.name, 'tie_points.tif'))
product.mtl.get_valid_pixel_mask(
os.path.join(config.get("wd"), product.name, 'valid_pixel_mask.tif'))
# !! Initialization of each block
for block_name in list_of_blocks:
get_module(block_name).initialize()
# !! Pre processing !!
# Run the preprocessing method of each block
for block_name in list_of_blocks:
generic_process_step(block_name, product, "preprocess")
# !! Processing !!
# save S2L_config file in wd
config.savetofile(
os.path.join(config.get('wd'), product.name, 'processing_start.cfg'))
# For each band or a selection of bands:
if bands is None:
# get all bands
bands = product.bands
elif product.sensor != 'S2':
bands = [product.reverse_bands_mapping.get(band) for band in bands]
bands_filenames = []
for band in bands:
# process the band through each block
bands_filenames.append(process_band(product, band,
list_of_blocks)) # save image path
if bands_filenames == [None] * len(bands_filenames):
logger.error("No valid band provided for input product.")
logger.error("Valids band for products are: %s" %
str(list(product.bands)))
return
# !! Post processing !!
# Run the postprocessing method of each block
for block_name in list_of_blocks:
generic_process_step(block_name, product, "postprocess")
# Clear metadata
metadata.clear()
# save S2L_config file in wd
config.savetofile(
os.path.join(config.get('wd'), product.name, 'processing_end.cfg'))
def _manual_replaces(self, product):
# GENERAL_INFO
# ------------
copy_elements([
'./General_Info/TILE_ID', './General_Info/DATASTRIP_ID',
'./General_Info/DOWNLINK_PRIORITY', './General_Info/SENSING_TIME',
'./General_Info/Archiving_Info'
], self.root_in, self.root_out, self.root_bb)
if product.mtl.data_type == 'Level-1C' or 'L1' in product.mtl.data_type:
l1c_tile_id = find_element_by_path(
self.root_in, './General_Info/TILE_ID')[0].text
l2a_tile_id = "NONE"
else:
l1c_tile_id = find_element_by_path(
self.root_in, './General_Info/L1C_TILE_ID')[0].text
l2a_tile_id = find_element_by_path(
self.root_in, './General_Info/TILE_ID')[0].text
tilecode = product.mtl.mgrs
pdgs = metadata.hardcoded_values.get('PDGS', '9999')
PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]])
AC = self.root_in.findall('.//ARCHIVING_CENTRE')
AC = AC[0].text if AC else 'ZZZ'
acqdate = dt.datetime.strftime(product.acqdate, '%Y%m%dT%H%M%S')
tile_id = '_'.join([
product.sensor_name, 'OPER', 'MSI', 'L2{}'.format(self.H_F), AC,
acqdate, 'A{}'.format(config.get('absolute_orbit')), tilecode,
'N{}'.format(PDGS)
])
change_elm(self.root_out,
'./General_Info/L1_TILE_ID',
new_value=l1c_tile_id)
change_elm(self.root_out,
'./General_Info/L2A_TILE_ID',
new_value=l2a_tile_id)
change_elm(self.root_out, './General_Info/TILE_ID', new_value=tile_id)
# Geometric_info
# ---------------
copy_elements([
'./Geometric_Info/Tile_Geocoding/HORIZONTAL_CS_NAME',
'./Geometric_Info/Tile_Geocoding/HORIZONTAL_CS_CODE'
], self.root_in, self.root_out, self.root_bb)
g = loads(search_db(tilecode, search='UTM_WKT'))
xMin = int(g.bounds[0])
yMin = int(g.bounds[1])
change_elm(self.root_out,
'./Geometric_Info/Tile_Geocoding/Geoposition/ULX',
new_value=str(xMin))
change_elm(self.root_out,
'./Geometric_Info/Tile_Geocoding/Geoposition/ULY',
new_value=str(yMin))
self._remove_children('./Geometric_Info/Tile_Angles',
tag='Viewing_Incidence_Angles_Grids')
angles_path = os.path.join(
'GRANULE', metadata.mtd.get('granule_{}_name'.format(self.H_F)),
'QI_DATA', metadata.mtd.get('ang_filename'))
change_elm(self.root_out,
'./Geometric_Info/Tile_Angles/Acquisition_Angles_Filename',
new_value=angles_path)
rm_elm_with_tag(self.root_out, tag='Sun_Angles_Grid')
rm_elm_with_tag(self.root_out, tag='Viewing_Incidence_Angle_Grid')
copy_elements(['./Geometric_Info/Tile_Angles/Mean_Sun_Angle'],
self.root_in, self.root_out, self.root_bb)
copy_elements(
['./Geometric_Info/Tile_Angles/Mean_Viewing_Incidence_Angle_List'],
self.root_in, self.root_out, self.root_bb)
# Quality indicators info
# -----------------------
self._remove_children('./Quality_Indicators_Info/Image_Content_QI')
copy_children(self.root_in,
'./Quality_Indicators_Info/Image_Content_QI',
self.root_out,
'./Quality_Indicators_Info/Image_Content_QI')
# Replace masks with all existing
self._remove_children('./Quality_Indicators_Info/Pixel_Level_QI',
tag='MASK_FILENAME')
for mask in metadata.mtd.get('masks_{}'.format(self.H_F)):
create_child(self.root_out,
'./Quality_Indicators_Info/Pixel_Level_QI',
tag=mask.get('tag'),
text=mask.get('text'),
attribs=mask.get('attribs'))
msk_text = find_element_by_path(
self.root_in,
'./Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME')[0].text
ini_grn_name = re.search(r'GRANULE/(.*?)/QI_DATA', msk_text).group(1)
elems = find_element_by_path(
self.root_out,
'./Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME')
for elem in elems:
elem.text = elem.text.replace(
ini_grn_name,
metadata.mtd.get('granule_{}_name'.format(self.H_F)))
rm_elm_with_tag(self.root_out, tag='PVI_FILENAME')
rm_elm_with_tag(self.root_out, tag='QL_B12118A_FILENAME')
rm_elm_with_tag(self.root_out, tag='QL_B432_FILENAME')
# Get all created quicklooks (including PVI)
for ql in metadata.mtd.get('quicklooks_{}'.format(self.H_F)):
ql_path = re.search(r'GRANULE(.*)', ql).group()
band_rootName = metadata.mtd.get(f'band_rootName_{self.H_F}')
ql_name = re.search(r'{}_(.*)'.format(band_rootName),
ql_path).group(1)
create_child(self.root_out,
'./Quality_Indicators_Info',
tag="{}_FILENAME".format(
os.path.splitext(ql_name)[0]),
text=ql_path)
def manual_replaces(self, product):
# GENERAL_INFO
# ------------
acqdate = dt.datetime.strftime(product.acqdate, '%Y-%m-%dT%H:%M:%S.%fZ')
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_START_TIME', new_value=acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_STOP_TIME', new_value=acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI',
new_value=metadata.mtd.get('product_{}_name'.format(self.H_F)))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL',
new_value='Level-2{}'.format(self.H_F))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE',
new_value=f'{product.sensor}OLI2{self.H_F}')
pdgs = config.get('PDGS', '9999')
PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]])
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS)
generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[
:-3] + 'Z' # -3 to keep only 3 decimals
change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SPACECRAFT_NAME',
new_value=product.mtl.mission)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/DATATAKE_SENSING_START',
new_value=acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Datatake/SENSING_ORBIT_NUMBER',
new_value=config.get('relative_orbit'))
self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule')
for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))):
adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0]
create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
tag='IMAGE_FILE', text=adjusted_path)
tile_id = generate_LS8_tile_id(product, self.H_F)
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=tile_id, attr_to_change='granuleIdentifier')
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat')
if not config.getboolean('doSbaf'):
# FIXME : get product image characteristics from origin sensor (LS8 here),
# copying from another template fro example
pass
U = distance_variation_corr(product.acqdate)
change_elm(self.root_out, rpath='./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U',
new_value=str(U))
# Geometric_info
# ---------------
tilecode = product.mtl.mgrs
if tilecode.startswith('T'):
tilecode = tilecode[1:]
footprint = search_db(tilecode, search='MGRS_REF')
# adding back first element, to get a complete polygon
fp = footprint.split(' ')
footprint = ' '.join(fp + [fp[0], fp[1]])
chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint)
# Auxiliary_Data_Info
# -------------------
self.remove_children('./Auxiliary_Data_Info/GIPP_List', exceptions=['Input_Product_Info'])
config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0]
create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn,
attribs={"version": pdgs, "type": "GIP_S2LIKE"})
# Quality_Indicators_Info
# -----------------------
self.remove_children('./Quality_Indicators_Info', exceptions=['Input_Product_Info'])
change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info', attr_to_change='type',
new_value=product.mtl.mission)
change_elm(self.root_out, './Quality_Indicators_Info/Input_Product_Info',
new_value=product.mtl.landsat_scene_id)
def postprocess(self, pd):
"""
Copy auxiliary files in the final output like mask, angle files
Input product metadata file is also copied.
:param pd: instance of S2L_Product class
"""
# output directory
product_name, granule_compact_name, tilecode, datatake_sensing_start = self.base_path_S2L(
pd)
tsdir = os.path.join(config.get('archive_dir'),
tilecode) # ts = temporal series
outdir = product_name
product_path = os.path.join(tsdir, outdir)
qi_dir = os.path.join(product_path, 'GRANULE', granule_compact_name,
'QI_DATA')
# copy angles file
outfile = "_".join([metadata.mtd.get('band_rootName_H'), 'ANG'
]) + '.TIF'
metadata.mtd['ang_filename'] = outfile
shutil.copyfile(pd.mtl.angles_file, os.path.join(qi_dir, outfile))
# copy mask files
if "S2" in pd.sensor:
tree_in = ElementTree.parse(
pd.mtl.tile_metadata) # Tree of the input mtd (S2 MTD.xml)
root_in = tree_in.getroot()
mask_elements = find_element_by_path(
root_in,
'./Quality_Indicators_Info/Pixel_Level_QI/MASK_FILENAME')
for element in mask_elements:
mask_file = os.path.join(pd.path, element.text)
if os.path.exists(mask_file):
shutil.copyfile(
mask_file,
os.path.join(qi_dir, os.path.basename(mask_file)))
metadata.mtd.get('masks_H').append({
"tag": "MASK_FILENAME",
"attribs": element.attrib,
"text": element.text
})
# copy valid pixel mask
outfile = "_".join(
[metadata.mtd.get('band_rootName_H'), pd.sensor, 'MSK']) + '.TIF'
fpath = os.path.join(qi_dir, outfile)
metadata.mtd.get('masks_H').append({
"tag":
"MASK_FILENAME",
"attribs": {
"type": "MSK_VALPIX"
},
"text":
os.path.relpath(fpath, product_path)
})
if config.getboolean('COG'):
img_object = S2L_ImageFile(pd.mtl.mask_filename, mode='r')
img_object.write(filepath=fpath, COG=True, band='MASK')
else:
shutil.copyfile(pd.mtl.mask_filename, fpath)
# QI directory
qipath = os.path.join(tsdir, 'QI')
if not os.path.exists(qipath):
os.makedirs(qipath)
# save config file in QI
cfgname = "_".join([outdir, 'INFO']) + '.cfg'
cfgpath = os.path.join(tsdir, 'QI', cfgname)
config.savetofile(os.path.join(config.get('wd'), pd.name, cfgpath))
# save correl file in QI
if os.path.exists(
os.path.join(config.get('wd'), pd.name, 'correl_res.txt')):
corrname = "_".join([outdir, 'CORREL']) + '.csv'
corrpath = os.path.join(tsdir, 'QI', corrname)
shutil.copy(
os.path.join(config.get('wd'), pd.name, 'correl_res.txt'),
corrpath)
if len(self.images.keys()) > 1:
# true color QL
band_list = ["B04", "B03", "B02"]
qlname = "_".join(
[metadata.mtd.get('band_rootName_H'), 'QL', 'B432']) + '.jpg'
qlpath = os.path.join(qi_dir, qlname)
quicklook(pd, self.images, band_list, qlpath,
config.get("quicklook_jpeg_quality", 95))
metadata.mtd.get('quicklooks_H').append(qlpath)
# false color QL
band_list = ["B12", "B11", "B8A"]
qlname = "_".join([
metadata.mtd.get('band_rootName_H'), 'QL', 'B12118A'
]) + '.jpg'
qlpath = os.path.join(qi_dir, qlname)
quicklook(pd, self.images, band_list, qlpath,
config.get("quicklook_jpeg_quality", 95))
metadata.mtd.get('quicklooks_H').append(qlpath)
else:
# grayscale QL
band_list = list(self.images.keys())
qlname = "_".join([
metadata.mtd.get('band_rootName_H'), 'QL', band_list[0]
]) + '.jpg'
qlpath = os.path.join(qi_dir, qlname)
quicklook(pd, self.images, band_list, qlpath,
config.get("quicklook_jpeg_quality", 95))
metadata.mtd.get('quicklooks_H').append(qlpath)
# PVI
band_list = ["B04", "B03", "B02"]
pvi_filename = "_".join([metadata.mtd.get('band_rootName_H'), 'PVI'
]) + '.TIF'
qlpath = os.path.join(qi_dir, pvi_filename)
quicklook(pd,
self.images,
band_list,
qlpath,
config.get("quicklook_jpeg_quality", 95),
xRes=320,
yRes=320,
creationOptions=['COMPRESS=LZW'],
format='GTIFF')
metadata.mtd.get('quicklooks_H').append(qlpath)
# Clear images as packager is the last process
self.images.clear()
# Write QI report as XML
bb_QI_path = metadata.hardcoded_values.get('bb_QIH_path')
out_QI_path = os.path.join(qi_dir, 'L2H_QI_Report.xml')
in_QI_path = glob.glob(
os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA',
'L2A_QI_Report.xml'))
log.info(
'QI report for input product found : {} (searched at {})'.format(
len(in_QI_path) != 0,
os.path.join(pd.path, 'GRANULE', '*', 'QI_DATA',
'L2A_QI_Report.xml')))
in_QI_path = in_QI_path[0] if len(in_QI_path) != 0 else None
Qi_Writer = QiWriter(bb_QI_path,
outfile=out_QI_path,
init_QI_path=in_QI_path,
H_F='H')
Qi_Writer._manual_replaces(pd)
Qi_Writer.write(pretty_print=True)
# TODO UNCOMMENT BELOW FOR XSD CHECK
product_QI_xsd = metadata.hardcoded_values.get('product_QIH_xsd')
log.info('QI Report is valid : {}'.format(
Qi_Writer.validate_schema(product_QI_xsd, out_QI_path)))
# Write product MTD
bb_S2_product = metadata.hardcoded_values.get('bb_S2H_product')
bb_L8_product = metadata.hardcoded_values.get('bb_L8H_product')
product_mtd_path = 'MTD_{}L2H.xml'.format(
pd.mtl.sensor[0:3]) # MSI / OLI/ OLI_TIRS
product_MTD_outpath = os.path.join(tsdir, product_name,
product_mtd_path)
mtd_pd_writer = MTD_writer_S2(bb_S2_product, pd.mtl.mtl_file_name, H_F='H') if pd.sensor == 'S2' \
else MTD_writer_LS8(bb_L8_product, H_F='H')
mtd_pd_writer._manual_replaces(pd)
mtd_pd_writer.write(product_MTD_outpath, pretty_print=True)
# TODO UNCOMMENT BELOW FOR XSD CHECK
# product_mtd_xsd = metadata.hardcoded_values.get('product_mtd_xsd')
# log.info('Product MTD is valid : {}'.format(mtd_pd_writer.validate_schema(product_mtd_xsd,
# product_MTD_outpath)))
# Write tile MTD
bb_S2_tile = metadata.hardcoded_values.get('bb_S2H_tile')
bb_L8_tile = metadata.hardcoded_values.get('bb_L8H_tile')
tile_mtd_path = 'MTD_TL_L2H.xml'
tile_MTD_outpath = os.path.join(product_path, 'GRANULE',
granule_compact_name, tile_mtd_path)
mtd_tl_writer = MTD_tile_writer_S2(bb_S2_tile, pd.mtl.tile_metadata, H_F='H') if pd.sensor == 'S2' \
else MTD_tile_writer_LS8(bb_L8_tile, H_F='H')
mtd_tl_writer._manual_replaces(pd)
mtd_tl_writer.write(tile_MTD_outpath, pretty_print=True)
def manual_replaces(self, product):
# GENERAL_INFO
# ------------
elements_to_copy = ['./General_Info/Product_Info/Datatake',
'./General_Info/Product_Info/PRODUCT_START_TIME',
'./General_Info/Product_Info/PRODUCT_STOP_TIME',
'./General_Info/Product_Info/Query_Options',
]
copy_elements(elements_to_copy, self.root_in, self.root_out, self.root_bb)
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_URI',
new_value=metadata.mtd.get('product_{}_name'.format(self.H_F)))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_LEVEL',
new_value='Level-2{}'.format(self.H_F))
change_elm(self.root_out, rpath='./General_Info/Product_Info/PRODUCT_TYPE',
new_value='S2MSI2{}'.format(self.H_F))
pdgs = config.get('PDGS', '9999')
PDGS = '.'.join([pdgs[:len(pdgs) // 2], pdgs[len(pdgs) // 2:]])
AC = self.root_in.findall('.//ARCHIVING_CENTRE')
if AC:
metadata.mtd['S2_AC'] = AC[0].text
change_elm(self.root_out, rpath='./General_Info/Product_Info/PROCESSING_BASELINE', new_value=PDGS)
generation_time = dt.datetime.strftime(metadata.mtd.get('product_creation_date'), '%Y-%m-%dT%H:%M:%S.%f')[
:-3] + 'Z' # -3 to keep only 3 decimals
change_elm(self.root_out, rpath='./General_Info/Product_Info/GENERATION_TIME', new_value=generation_time)
self.remove_children('./General_Info/Product_Info/Product_Organisation/Granule_List/Granule')
for band_path in sorted(set(metadata.mtd.get('bands_path_{}'.format(self.H_F)))):
adjusted_path = os.path.splitext(re.sub(r'^.*?GRANULE', 'GRANULE', band_path))[0]
create_child(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
tag='IMAGE_FILE', text=adjusted_path)
grnl_id = \
find_element_by_path(self.root_in, './General_Info/Product_Info/Product_Organisation/Granule_List/Granule')
if grnl_id:
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=generate_S2_tile_id(product, self.H_F, metadata.mtd['S2_AC']),
attr_to_change='granuleIdentifier')
change_elm(self.root_out, rpath='./General_Info/Product_Info/Product_Organisation/Granule_List/Granule',
new_value=self.IMAGE_FORMAT[config.get('output_format')], attr_to_change='imageFormat')
else:
pass # Fixme
# If Sbaf is done, we keep the values inside the backbone (S2A values)
if not config.getboolean('doSbaf'):
# copy_elements(['./General_Info/Product_Image_Characteristics/Special_Values',
# './General_Info/Product_Image_Characteristics/Image_Display_Order',
# './General_Info/Product_Image_Characteristics/Reflectance_Conversion',
# './General_Info/Product_Image_Characteristics/Spectral_Information_List'],
# self.root_in, self.root_out, self.root_bb)
pass
# FIXME : get product image characteristics from origin sensor (S2 here),
# copying from another template for example (see commented lines above)
copy_elements(['./General_Info/Product_Image_Characteristics/Reflectance_Conversion/U'], self.root_in,
self.root_out)
# Geometric_info
# ---------------
tilecode = product.mtl.mgrs
if tilecode.startswith('T'):
tilecode = tilecode[1:]
footprint = search_db(tilecode, search='MGRS_REF')
# adding back first element, to get a complete polygon
fp = footprint.split(' ')
footprint = ' '.join(fp + [fp[0], fp[1]])
chg_elm_with_tag(self.root_out, tag='EXT_POS_LIST', new_value=footprint)
copy_elements(['./Geometric_Info/Coordinate_Reference_System'], self.root_in, self.root_out, self.root_bb)
# Auxiliary_Data_Info
# -------------------
self.remove_children('./Auxiliary_Data_Info/GIPP_List')
copy_children(self.root_in, './Auxiliary_Data_Info/GIPP_List',
self.root_out, './Auxiliary_Data_Info/GIPP_List')
config_fn = os.path.splitext(os.path.basename(config.parser.config_file))[0]
create_child(self.root_out, './Auxiliary_Data_Info/GIPP_List', tag="GIPP_FILENAME", text=config_fn,
attribs={"version": pdgs, "type": "GIP_S2LIKE"})
for tag in ['PRODUCTION_DEM_TYPE',
'IERS_BULLETIN_FILENAME',
'ECMWF_DATA_REF',
'SNOW_CLIMATOLOGY_MAP',
'ESACCI_WaterBodies_Map',
'ESACCI_LandCover_Map',
'ESACCI_SnowCondition_Map_Dir']:
elem = find_element_by_path(self.root_in, './Auxiliary_Data_Info/' + tag)
if len(elem) != 0:
new_value = elem[0].text
else:
new_value = "NONE"
change_elm(self.root_out, rpath='./Auxiliary_Data_Info/' + tag, new_value=new_value)
# Fill GRI_List
gri_elems = self.root_in.findall('.//GRI_FILENAME')
for gri_elm in gri_elems:
create_child(self.root_out, './Auxiliary_Data_Info/GRI_List', tag="GRI_FILENAME", text=gri_elm.text)
# Quality_Indicators_Info
# -----------------------
copy_elements(['./Quality_Indicators_Info'], self.root_in, self.root_out, self.root_bb)
def write(self, creation_options=None, DCmode=False, filepath=None, nodata_value=0, COG:bool=False, band:str=None):
"""
write to file
:param creation_options: gdal create options
:param DCmode: if true, the type is kept. Otherwise float are converted to int16 using
offset and gain from config
:param filepath:
:param COG : whether to create COG output format or not
:param band : provide information about the band, to set the overviews downsampling algorithm.
band can be 'MASK', 'QA', or None for all others
"""
if creation_options is None:
creation_options = []
# if filepath is override
if filepath is None:
filepath = self.filepath
# Ensure that file extension is tiff
if not filepath.endswith('.tif'):
filepath = os.path.splitext(filepath)[0] + ".TIF"
# check if directory to create
if not os.path.exists(self.dirpath):
os.makedirs(self.dirpath)
# write with gdal
etype = gdal.GetDataTypeByName(self.array.dtype.name)
if self.array.dtype.name.endswith('int8'):
# work around to GDT_Unknown
etype = 1
elif 'float' in self.array.dtype.name and not DCmode:
# float to UInt16
etype = gdal.GDT_UInt16
# Update image attributes
self.setFilePath(filepath)
# Create folders hierarchy if needed:
if not os.path.exists(self.dirpath):
os.makedirs(self.dirpath)
if not COG:
driver = gdal.GetDriverByName('GTiff')
dst_ds = driver.Create(self.filepath, xsize=self.xSize,
ysize=self.ySize, bands=1, eType=etype, options=creation_options)
else:
driver = gdal.GetDriverByName('MEM')
dst_ds = driver.Create('', xsize=self.xSize,
ysize=self.ySize, bands=1, eType=etype)
dst_ds.SetProjection(self.projection)
geotranform = (self.xMin, self.xRes, 0, self.yMax, 0, self.yRes)
log.debug(geotranform)
dst_ds.SetGeoTransform(geotranform)
if 'float' in self.array.dtype.name and not DCmode:
# float to UInt16 with scaling factor of 10000
offset = float(config.get('offset'))
gain = float(config.get('gain'))
dst_ds.GetRasterBand(1).WriteArray(((offset + self.array).clip(min=0) * gain).astype(np.uint16))
# set GTiff metadata
dst_ds.GetRasterBand(1).SetScale(1 / gain)
dst_ds.GetRasterBand(1).SetOffset(offset)
else:
dst_ds.GetRasterBand(1).WriteArray(self.array)
if nodata_value:
dst_ds.GetRasterBand(1).SetNoDataValue(nodata_value)
if COG:
resampling_algo = config.get('resampling_algo_MASK') if band in ['QA', 'MASK'] else config.get('resampling_algo')
downsampling_levels = config.get('downsampling_levels_{}'.format(int(self.xRes)), config.get('downsampling_levels_10')) # If the res isn't [10, 15, 20, 30, 60], consider it as 30
downsampling_levels = [int(x) for x in downsampling_levels.split(" ")]
# Overloading creation options
creation_options = [opt for opt in creation_options if opt.split("=")[0] not in
['TILED', 'COMPRESS', 'INTERLEAVE', 'BLOCKYSIZE', 'BLOCKXSIZE', 'PREDICTOR']] + \
\
['TILED=YES',
"COMPRESS=" + config.get('compression'),
"INTERLEAVE=" + config.get('interleave'),
"BLOCKXSIZE=" + str(config.get('internal_tiling')),
"BLOCKYSIZE=" + str(config.get('internal_tiling')),
"PREDICTOR=" + str(config.get('predictor'))]
# FIXME : in this gdal version, driver GTiff does not support creation option GDAL_TIFF_OVR_BLOCKSIZE to set the
# FIXME : internal overview blocksize ; however it is set at 128 as default, as requested here
# Source : https://gdal.org/drivers/raster/gtiff.html#raster-gtiff
# add in options : "GDAL_TIFF_OVR_BLOCKSIZE=" + str(config.get('internal_overviews'))
dst_ds.BuildOverviews(resampling_algo, downsampling_levels)
driver_Gtiff = gdal.GetDriverByName('GTiff')
data_set2 = driver_Gtiff.CreateCopy(self.filepath, dst_ds, options=creation_options + ['COPY_SRC_OVERVIEWS=YES'])
data_set2 = None
dst_ds.FlushCache()
dst_ds = None
log.info('Written: {}'.format(self.filepath))
def postprocess(self, pd):
"""
Copy auxiliary files in the final output like mask, angle files
Input product metadata file is also copied.
:param pd: instance of S2L_Product class
"""
# output directory
outdir, tilecode = self.base_path(pd)
tsdir = os.path.join(config.get('archive_dir'), tilecode) # ts = temporal series
# copy MTL files in final product
outfile = os.path.basename(pd.mtl.mtl_file_name)
shutil.copyfile(pd.mtl.mtl_file_name, os.path.join(tsdir, outdir, outfile))
if pd.mtl.tile_metadata:
outfile = os.path.basename(pd.mtl.tile_metadata)
shutil.copyfile(pd.mtl.tile_metadata, os.path.join(tsdir, outdir, outfile))
# copy angles file
outfile = "_".join([outdir, 'ANG']) + '.TIF'
shutil.copyfile(pd.mtl.angles_file, os.path.join(tsdir, outdir, outfile))
# copy valid pixel mask
outfile = "_".join([outdir, 'MSK']) + '.TIF'
shutil.copyfile(pd.mtl.mask_filename, os.path.join(tsdir, outdir, outfile))
# QI directory
qipath = os.path.join(tsdir, 'QI')
if not os.path.exists(qipath):
os.makedirs(qipath)
# save config file in QI
cfgname = "_".join([outdir, 'INFO']) + '.cfg'
cfgpath = os.path.join(tsdir, 'QI', cfgname)
config.savetofile(os.path.join(config.get('wd'), pd.name, cfgpath))
# save correl file in QI
if os.path.exists(os.path.join(config.get('wd'), pd.name, 'correl_res.txt')):
corrname = "_".join([outdir, 'CORREL']) + '.csv'
corrpath = os.path.join(tsdir, 'QI', corrname)
shutil.copy(os.path.join(config.get('wd'), pd.name, 'correl_res.txt'), corrpath)
if len(self.images.keys()) > 1:
# true color QL
band_list = ["B04", "B03", "B02"]
qlname = "_".join([outdir, 'QL', 'B432']) + '.jpg'
qlpath = os.path.join(tsdir, 'QI', 'QL_B432', qlname)
quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95))
# false color QL
band_list = ["B12", "B11", "B8A"]
qlname = "_".join([outdir, 'QL', 'B12118A']) + '.jpg'
qlpath = os.path.join(tsdir, 'QI', 'QL_B12118A', qlname)
quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95))
else:
# grayscale QL
band_list = list(self.images.keys())
qlname = "_".join([outdir, 'QL', band_list[0]]) + '.jpg'
qlpath = os.path.join(tsdir, 'QI', f'QL_{band_list[0]}', qlname)
quicklook(pd, self.images, band_list, qlpath, config.get("quicklook_jpeg_quality", 95))
# Clear images as packager is the last process
self.images.clear()
