def _run(self, meta_data, data_dir, output_dir):
"""Perform processor tasks.
:param meta_data: IP metadata
:param str data_dir: path to data directory
:param str output_dir: path to output processor directory
:return dict: QI metadata
"""
response_data = {
'isMeasurementOf':
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.now(),
'value':
False
}
# process primary product type
response_data.update(
self._ordinary_control(self._get_file_path(meta_data['title'])))
# process level2 product type if defined
level2_product = self.get_processing_level2(meta_data)
if level2_product:
response_data.update(
self._ordinary_control(self._get_file_path(
level2_product['title']),
level=2))
else:
Logger.error("Level2 product not found for {}".format(
meta_data['title']))
response_data['value'] = False
return response_data
def __init__(self, username, password, archive, backup_archive=None):
"""Connect API.
Raise ProcessorFailedError on failure
"""
from sentinelsat.sentinel import SentinelAPI, SentinelAPIError
# remember settings for query()
self.archive = archive
self.backup_archive = backup_archive
# connect API
try:
self.api = SentinelAPI(username, password, archive)
except (SentinelAPIError, ConnectionError) as e:
self.api = None
if backup_archive:
# re-try with backup archive
Logger.error(
"Unable to connect {} ({}). Re-trying with {}...".format(
archive, e, backup_archive))
try:
self.api = SentinelAPI(username, password, backup_archive)
except (SentinelAPIError, ConnectionError) as e:
self.api = None
if self.api is None:
raise ProcessorFailedError(self,
"Unable to connect: {}".format(e),
set_status=False)
Logger.debug("Sentinel API connected")
def _run(self, meta_data, data_dir, output_dir):
"""Perform processor tasks.
:param meta_data: IP metadata
:param str data_dir: path to data directory
:param str output_dir: path to output processor directory
:return dict: QI metadata
"""
response_data = {}
# reference image must be defined
try:
im_reference = self.config.abs_path(
self.config['geometry']['reference_image'])
except KeyError:
Logger.error("Reference image not defined")
self.set_response_status(DbIpOperationStatus.failed)
return response_data
if not os.path.exists(im_reference):
Logger.error("Reference image '{}' not found".format(im_reference))
self.set_response_status(DbIpOperationStatus.failed)
return response_data
# check if stack is already available
output_dir = self._get_ip_output_path(meta_data['title'])
if os.path.exists(output_dir):
files = self.filter_files(output_dir,
extension='.tif',
pattern='stack_*')
if len(files) > 0:
if all([os.stat(f).st_size > 0 for f in files]):
Logger.debug('Stack ({}) already available, no operation '
'done'.format(output_dir))
response_data.update(
self._run_stack_ordinary_control(
meta_data, data_dir, output_dir, response_data))
return response_data
Logger.debug('Stack ({}) already available, but has size 0 B. '
'A new one will be created.'.format(output_dir))
# compute stack
if not os.path.exists(output_dir):
os.mkdir(output_dir)
try:
lh_title = os.path.split(output_dir)[-1]
stack_name = self.get_stack_name(lh_title)
self.create_stack(data_dir, output_dir, stack_name)
response_data.update(
self._run_stack_ordinary_control(meta_data, data_dir,
output_dir, response_data))
except ProcessorRejectedError:
self.set_response_status(DbIpOperationStatus.rejected)
except ProcessorFailedError:
self.set_response_status(DbIpOperationStatus.failed)
return response_data
def __del__(self):
if not hasattr(self, "api"):
return
from landsatxplore.exceptions import EarthExplorerError
try:
self.api.logout()
except EarthExplorerError as e:
Logger.error("Landsat server is down. {}".format(e))
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
Logger.info('Running thematic validation control')
response_data = {
'isMeasurementOf':
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.now(),
"value":
False
}
reference_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['reference_layer'])
if not os.path.isfile(reference_fn):
Logger.error("File {} not found".format(reference_fn))
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
themes = self.config['land_product']['product_type']
for theme in themes:
if theme == 'classification':
classification_qi_ = self._lp_classification_validation(
self.config)
response_data.update({"classification": classification_qi_})
if float(classification_qi_['overallAccuracy']) >= \
float(self.config['land_product']['thematic_accuracy']):
response_data['value'] = True
else:
response_data['value'] = False
elif theme == 'regression':
regression_qi = self._lp_regression_validation(self.config)
regression_prod_name = self.config['land_product'][
'regression_name']
response_data.update({regression_prod_name: regression_qi})
print(float(regression_qi['rmse']))
print(float(self.config['land_product']['rmse_accuracy']))
if float(regression_qi['rmse']) <= \
float(self.config['land_product']['rmse_accuracy']):
response_data['value'] = True
else:
response_data['value'] = False
if response_data['value'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def _run(self, meta_data, data_dir, output_dir):
"""Perform processor tasks.
:param meta_data: IP metadata
:param str data_dir: path to data directory
:param str output_dir: path to output processor directory
:return dict: QI metadata
"""
self.connector = self.connect()
response_data = {
'isMeasurementOf': '{}/#{}'.format(
self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime": datetime.now(),
'status': 'IN_PROGRESS'
}
try:
self._download_file(
meta_data['id'], meta_data['title'],
output_dir
)
downfile = os.path.join(
output_dir,
meta_data['title'] + self.extension
)
response_data['status'] = 'FINISHED'
response_data['complete'] = True
response_data['date'] = datetime_format(
self.file_timestamp(downfile)
)
response_data['value'] = True
# download level2 product
level2_product = self.get_processing_level2(meta_data)
if level2_product:
self._download_file(
level2_product['id'],
level2_product['title'],
output_dir
)
except QCProcessorDownloadError as e:
Logger.error("Unable to download {}: {}".format(
meta_data['id'], e
))
response_data['status'] = 'NOT_AVAILABLE'
response_data['complete'] = False
response_data['value'] = False
self.set_response_status(DbIpOperationStatus.failed)
return response_data
def download_file(self, uuid, output_dir):
from sentinelsat.sentinel import SentinelAPIError, InvalidChecksumError
try:
self.api.download(uuid, output_dir, checksum=True)
except (SentinelAPIError, ConnectionError, InvalidChecksumError) as e:
# re-try with backup archive
Logger.error(
"Unable to access {} ({}). Re-trying with {}...".format(
self.archive, e, self.backup_archive))
self.api.api_url = self.backup_archive
try:
self.api.download(uuid, output_dir, checksum=True)
except (SentinelAPIError, ConnectionError,
InvalidChecksumError) as e:
raise QCProcessorDownloadError(e)
def is_valid(self, response_file):
"""Validate response QI metadata file.
:param str response_file: response file to be validated
:return bool:
"""
data = self.load(response_file)
if data is None:
return False
try:
validate(data, schema=self._schema)
Logger.debug("JSON response {} is valid".format(response_file))
except ValidationError as e:
Logger.error("File {} validation against schema failed: {}".format(
response_file, e))
return False
return True
def compute(self, data_dir, output_dir):
"""Compute cloud coverage
:param str data_dir: input data directory
:param str output_dir: output directory
:return str: output file
"""
import fmask
# compute mask in 20m resolution
try:
self.run_fmask(data_dir, self._result['qi.files']['fmask'],
self.comp_res)
except subprocess.CalledProcessError as e:
Logger.error("fmask failed: {}".format(e))
self.set_response_status(DbIpOperationStatus.failed)
return None
return self._result['qi.files']['fmask']
def query(self, footprint, kwargs):
"""Query API.
Raise ProcessorFailedError on failure
:return: result
"""
from sentinelsat.sentinel import SentinelAPIError
result = None
try:
result = self.api.query(footprint, **kwargs)
except (SentinelAPIError, ConnectionError) as e:
if self.backup_archive:
# re-try with backup archive
Logger.error(
"Unable to access {}. Re-trying with {}...".format(
self.archive, self.backup_archive))
self.api.api_url = self.backup_archive
try:
result = self.api.query(footprint, **kwargs)
except (SentinelAPIError, ConnectionError) as e:
pass # exception will be raised anyway
raise ProcessorFailedError(self,
"Unable to query API: {}".format(e),
set_status=False)
if kwargs['producttype'] == 'S2MSI1C' and self.filter_by_tiles:
result_filtered = OrderedDict()
for ip, items in result.items():
for tile in self.filter_by_tiles:
if tile == items['tileid']:
result_filtered[ip] = items
break
if ip not in result_filtered.keys():
Logger.info("IP {} skipped by tile filter".format(ip))
return result_filtered
return result
def calibrate(self, l1_data_dir, l2_data_dir):
"""Create L2 products."""
manager_dir = os.getcwd()
l1_product_dir = os.path.join(manager_dir, l1_data_dir)
l2_product_dir = os.path.join(manager_dir, l2_data_dir)
os.mkdir(l2_product_dir)
try:
self.run_calibration(l1_product_dir)
except subprocess.CalledProcessError as e:
Logger.error("Calibration failed: {}".format(e))
self.set_response_status(DbIpOperationStatus.failed)
return {}
self.transform_l2_to_tif(l1_product_dir, l2_product_dir)
self.copy_kept_products(l1_product_dir, l2_product_dir)
self.clean_l1_directory(l1_product_dir)
return {}
def load(self, response_file):
"""Load QI metadata content from file.
:param str response_file: input filename (JSON file)
:return dict: file content or None on error
"""
try:
with open(response_file) as fd:
response_content = fd.read()
except FileNotFoundError:
Logger.critical("No response file found")
return None
try:
return json.loads(response_content)
except json.decoder.JSONDecodeError as e:
Logger.error("File {} is not valid JSON file: {}".format(
response_file, e))
return response_content
def _ordinary_control(self, filepath, level=1):
"""Perform ordinary control.
:param str filepath: path to zip file
:param int level: level number for response
"""
from osgeo import gdal, osr
if level not in (1, 2):
raise ProcessorCriticalError(self,
"Unsupported level: {}".format(level))
response_data = {'value': False}
# 1. unarchive product
dirname = os.path.join(
os.path.dirname(filepath),
os.path.basename(filepath).rstrip(self.extension) +
self.data_dir_suf)
try:
if not os.path.exists(filepath) and \
not os.path.exists(dirname):
raise ProcessorRejectedError(
self,
"No input data: {}/{} not found".format(filepath, dirname))
if not os.path.exists(dirname):
Logger.info("Unarchiving {}...".format(filepath))
dirname = self.unarchive(filepath, dirname)
except ProcessorRejectedError:
return response_data
# 2. check if all bands are available
try:
img_files, band_res = self.check_bands(dirname, level)
except ProcessorRejectedError:
return response_data
level_key = 'level{}'.format(level)
response_data[level_key] = {}
response_data[level_key]['rastersComplete'] = True
response_data[level_key]['channels'] = len(img_files)
response_data[level_key]['lineage'] = self.get_lineage_level(filepath)
# 3. check if raster file is readable
try:
self.check_bands_read(img_files)
except ProcessorRejectedError:
return response_data
Logger.info("All imagery files found and readable")
response_data[level_key]['rastersRead'.format(level)] = True
# 4. read raster characteristics
try:
epsg_res, format_res = self.check_bands_properties(
img_files, band_res)
self.check_epsg_tile_name(epsg_res, filepath)
except ProcessorRejectedError:
return response_data
# update response (epgs, format, bands)
response_data[level_key]['epsg'] = int(epsg_res)
if format_res == 'JP2OpenJPEG':
data_format = 'JPEG'
elif format_res == 'GTiff':
data_format = 'TIFF'
else:
data_format = '?'
response_data[level_key]['format'] = data_format
response_data[level_key]['bands'] = band_res
# 5. check metadata
metadata_read = self.check_metadata(dirname, level)
response_data[level_key]['metadataRead'] = metadata_read[0]
if self.has_calibration_metadata:
response_data[level_key]['calibrationMetadata'] = metadata_read[1]
else:
response_data[level_key]['calibrationMetadata'] = True
# 6. selected for next control
qi_failed = []
for attr in ('rastersComplete', 'rastersRead', 'metadataRead',
'calibrationMetadata'):
if not response_data[level_key][attr]:
qi_failed.append(attr)
response_data['value'] = len(qi_failed) < 1
if qi_failed:
Logger.error("Rejected because of {}".format(','.join(qi_failed)))
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
from lxml import etree
Logger.info('Running LP metadata control')
response_data = {
"isMeasurementOf":
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.datetime.now(),
"value":
False,
}
try:
lp_meta_fn = glob.glob(
os.path.join(
self.config['map_product']['path'],
self.config['land_product']['product_metadata'] +
'*.xml'))[0]
except IndexError:
self.set_response_status(DbIpOperationStatus.rejected)
if 'product_metadata' in self.config['land_product']:
response_data['metadataSpecification'] = self.config[
'land_product']['product_metadata']
else:
response_data['metadataSpecification'] = ''
try:
Parser = etree.HTMLParser()
XMLDoc = etree.parse(open(lp_meta_fn, 'r'), Parser)
Logger.info('Land Product metadata available')
response_data['metadataAvailable'] = True
# validate xml
Elements = XMLDoc.xpath('//characterstring')
i = 0
for Element in Elements:
if (Element.text) is not None:
i += 1
if i > 5:
response_data[
'metadataCompliancy'] = self.validate_xml_metadata(
lp_meta_fn)
except:
Logger.error('Land Prodcut metadata not available')
response_data['metadataAvailable'] = False
response_data['metadataCompliancy'] = False
if response_data['metadataAvailable'] == True and \
response_data['metadataCompliancy'] == True:
response_data['value'] = True
if response_data['value'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def calibrate(self, l1_data_dir, l2_data_dir):
"""Create L2 products."""
# temp_dir needed because use_cases contain resolution in their names
# and sen2cor is not capable of working with output_dir with
# a resolution in the name
if self.config.has_section('sen2cor'):
sen2cor_2_8 = '02.08' in self.config['sen2cor']['path']
sen2cor_path = self.config['sen2cor']['path']
if sen2cor_path[-1] != os.sep:
sen2cor_path += os.sep
else:
sen2cor_2_8 = True
sen2cor_path = ''
output_path = os.path.join(self.temp_dir,
os.path.basename(l1_data_dir))
if sen2cor_2_8:
# Sen2Cor 2.8
# https://forum.step.esa.int/t/sen2cor-2-8-fails-on-product-from-early-2016-bool-object-has-no-attribute-spacecraft-name/16046
sen2cor_string = '{}L2A_Process --resolution {} --cr_only ' \
'--output_dir {} {}'.format(
sen2cor_path,
self.config['land_product']['geometric_resolution'],
output_path, os.path.abspath(l1_data_dir)
)
else:
# Sen2Cor 2.5.5
sen2cor_string = '{}{}L2A_Process' \
' --resolution {}' \
' --cr_only {}'.format(
self.config['sen2cor']['path'], os.path.sep,
self.config['land_product']['geometric_resolution'],
os.path.abspath(l1_data_dir)
)
Logger.debug("Running {}".format(sen2cor_string))
try:
subprocess.run(sen2cor_string, shell=True, check=True)
except subprocess.CalledProcessError as e:
Logger.error("Calibration failed: {}".format(e))
self.set_response_status(DbIpOperationStatus.failed)
return {}
if sen2cor_2_8:
# now move the L2 product from the temp dir to the right one
sen2cor_dir = os.listdir(output_path)[0]
source_path = os.path.join(output_path, sen2cor_dir)
# do not use move() to avoid cross-link device error
shutil.copytree(source_path, l2_data_dir)
Logger.debug("Directory {} copied to {}".format(
source_path, l2_data_dir))
Logger.debug("Image file test (source): {}".format(
self.filter_files(source_path,
extension=self.img_extension)[0]))
Logger.debug("Image file test (target): {}".format(
self.filter_files(l2_data_dir,
extension=self.img_extension)[0]))
# remove tmp output directory
shutil.rmtree(output_path)
Logger.debug("Directory {} removed".format(output_path))
return {}
def _run(self, meta_data, data_dir, output_dir):
"""Perform processor tasks.
:param meta_data: IP metadata
:param str data_dir: Path to data directory
:param str output_dir: path to output processor directory
:return dict: QI metadata
"""
import arosics
response_data = {
self.isMeasurementOfSection: [{
"id": "http://qcmms.esa.int/detailed_control#GEOMETRY",
"generatedAtTime": datetime.now(),
"requirement": False
}]
}
# reference image must be defined
try:
im_reference = self.config.abs_path(
self.config['geometry']['reference_image'])
except KeyError:
Logger.error("Reference image not defined")
self.set_response_status(DbIpOperationStatus.failed)
return
if not os.path.exists(im_reference):
Logger.error("Reference image '{}' not found".format(im_reference))
self.set_response_status(DbIpOperationStatus.failed)
return
lh_output_dir = self.get_lh_dir(data_dir)
bands2stack = self.config['geometry'].get('band4match', 3)
resolution = self.config['land_product'].get('geometric_resolution')
correct_shifts = self.config['geometry'].get('correct_shifts', False)
has_stack = 'harmonization_stack' in self.config['processors']
try:
stack_on = self.config['geometry']['stack_on']
if stack_on is True and has_stack is False:
Logger.warning("Harmonization stack not available")
stack_on = False
except KeyError:
stack_on = has_stack
try:
if stack_on:
im_target = self._get_target_stack(lh_output_dir)
else:
filepattern_suff = '_{}m{}'.format(
resolution,
self.img_extension) \
if 'L2A' in data_dir else '{}'.format(self.img_extension)
filepattern = '{}{}'.format(
self.bands2stack(bands2stack, data_dir), filepattern_suff)
try:
im_target = self.filter_files(data_dir, filepattern)[0]
except IndexError:
raise ProcessorCriticalError(
self, "Pattern '{}' not found in {}".format(
filepattern, data_dir))
Logger.debug("Target image: {}".format(im_target))
except ProcessorFailedError:
self.set_response_status(DbIpOperationStatus.failed)
return
# check EPSG codes
if self.getEpsgCode(im_target) != self.getEpsgCode(im_reference):
Logger.error("Inconsistent reference and image EPSG codes")
self.set_response_status(DbIpOperationStatus.rejected)
return
try:
mask = tempfile.mktemp(prefix='mask_')
self.create_mask(data_dir, mask, resolution)
except ProcessorFailedError:
self.set_response_status(DbIpOperationStatus.failed)
return
# arocics settings
coreg_file_prefix = ''
if correct_shifts is True:
coreg_file_prefix += 'coreg_'
if stack_on:
coreg_file_prefix += 'stack_'
aro_raster_path = os.path.join(
lh_output_dir, '{}{}.tif'.format(coreg_file_prefix,
os.path.split(lh_output_dir)[-1]))
kwargs = {
'fmt_out': 'GTIFF',
'grid_res': 30,
'max_points': 1000,
'mask_baddata_tgt': mask,
'path_out': aro_raster_path,
'projectDir': lh_output_dir,
'nodata': self.get_nodata_values(),
'window_size': (128, 128)
}
if stack_on:
kwargs['s_b4match'] = bands2stack
# run arocics
Logger.debug(
"Running coreg for reference: {}; image: {} with args {}".format(
im_reference, im_target, kwargs))
try:
CRL = arosics.COREG_LOCAL(im_reference, im_target, **kwargs)
except AssertionError as e:
Logger.error("Coreg failed: {}".format(e))
self.set_response_status(DbIpOperationStatus.failed)
return response_data
try:
CRL_points_table = CRL.tiepoint_grid.get_CoRegPoints_table()
except ValueError:
CRL_points_table = None
try:
self.to_PointFormat(
CRL_points_table,
self._result['qi.files']['gml_before_correction'],
CRL.outFillVal,
epsg=self.getEpsgCode(im_target))
except ProcessorRejectedError:
return response_data
if correct_shifts is True:
CRL.correct_shifts()
CRL_after_corr = arosics.COREG_LOCAL(im_reference, CRL.path_out,
**kwargs)
try:
CRL_points_table_after_corr = \
CRL_after_corr.tiepoint_grid.get_CoRegPoints_table()
except ValueError:
CRL_points_table_after_corr = None
try:
self.to_PointFormat(
CRL_points_table_after_corr,
self._result['qi.files']['gml_after_correction'],
CRL_after_corr.outFillVal,
epsg=self.getEpsgCode(im_target))
except ProcessorRejectedError:
Logger.debug(
'No GCP points found in the GML after correction. Copying '
'the one before the correction as the one after.')
from shutil import copy2
copy2(self._result['qi.files']['gml_before_correction'],
self._result['qi.files']['gml_after_correction'])
else:
CRL_after_corr = None
if self.visualize is True or self.save_visualization is True:
self._visualize(CRL, CRL_after_corr, self.visualize,
self.save_visualization, correct_shifts,
output_dir, resolution, aro_raster_path, stack_on)
# update response attributes
pixel_metadata_coding_conf = self.config['pixel_metadata_coding']
response_data[self.isMeasurementOfSection][0].update({
'mask':
self.file_basename(
self._result['qi.files']['gml_before_correction']),
'rasterCoding':
pixel_metadata_coding_conf[self.identifier],
'lineage':
'http://qcmms.esa.int/Arosics_v{}'.format(arosics.__version__)
})
try:
stats = self.compute_stats(CRL_points_table, resolution,
CRL.outFillVal)
response_data[self.isMeasurementOfSection][0].update(stats)
except ProcessorFailedError:
return response_data
os.remove(mask)
return response_data
def __init__(self, processor, msg, set_status=True):
Logger.error("{} processor rejected: {}".format(
processor.__class__.__name__, msg))
if set_status:
# set processor response status
processor.set_response_status(DbIpOperationStatus.rejected)
