class QCProcessorSearch(QCProcessorMultiBase):
"""Multi-mission search processor [feasibility control]."""
identifier = identifier_from_file(__file__)
def __init__(self, config, response):
super(QCProcessorSearch, self).__init__(
config, response
)
self.metapath = os.path.join(
self.config['project']['path'], self.config['project']['metapath']
)
# remove csv file (platform-dependent processors append new lines)
csv_file = os.path.join(
self.metapath,
'{}_fullmetadata.csv'.format(
self.config['land_product']['product_abbrev']
)
)
if os.path.exists(csv_file):
os.remove(csv_file)
def processor_sentinel2(self):
from processors.search.sentinel import QCProcessorSearchSentinel
return QCProcessorSearchSentinel
def processor_landsat8(self):
from processors.search.landsat import QCProcessorSearchLandsat
return QCProcessorSearchLandsat
class QCProcessorGeometryQuality(QCProcessorMultiBase):
"""Geometry quality control processor [detailed control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.geometry_quality.sentinel import QCProcessorGeometryQualitySentinel
return QCProcessorGeometryQualitySentinel
def processor_landsat8(self):
from processors.geometry_quality.landsat import QCProcessorGeometryQualityLandsat
return QCProcessorGeometryQualityLandsat
class QCProcessorHarmonizationStack(QCProcessorMultiBase):
"""Processor creating a resampled stack from individual band files [multi-sensor control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.harmonization_stack.sentinel import QCProcessorHarmonizationStackSentinel
return QCProcessorHarmonizationStackSentinel
def processor_landsat8(self):
from processors.harmonization_stack.landsat import QCProcessorHarmonizationStackLandsat
return QCProcessorHarmonizationStackLandsat
class QCProcessorDownload(QCProcessorMultiBase):
"""Download processor [delivery control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.download.sentinel import QCProcessorDownloadSentinel
return QCProcessorDownloadSentinel
def processor_landsat8(self):
from processors.download.landsat import QCProcessorDownloadLandsat
return QCProcessorDownloadLandsat
class QCProcessorHarmonizationControl(QCProcessorMultiBase):
"""Harmonization control processor [multi-sensor control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.harmonization_control.sentinel import QCProcessorHarmonizationControlSentinel
return QCProcessorHarmonizationControlSentinel
def processor_landsat8(self):
from processors.harmonization_control.landsat import QCProcessorHarmonizationControlLandsat
return QCProcessorHarmonizationControlLandsat
class QCProcessorValidPixels(QCProcessorMultiBase):
"""Validity pixel control processor [detailed control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.valid_pixels.sentinel import QCProcessorValidPixelsSentinel
return QCProcessorValidPixelsSentinel
def processor_landsat8(self):
from processors.valid_pixels.landsat import QCProcessorValidPixelsLandsat
return QCProcessorValidPixelsLandsat
class QCProcessorRadiometryControl(QCProcessorMultiBase):
"""Radiometry control processor [detailed control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.radiometry_control.sentinel import QCProcessorRadiometryControlSentinel
return QCProcessorRadiometryControlSentinel
def processor_landsat8(self):
from processors.radiometry_control.landsat import QCProcessorRadiometryControlLandsat
return QCProcessorRadiometryControlLandsat
class QCProcessorCloudCoverage(QCProcessorMultiBase):
"""Cloud coverage control processor [defailed control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.cloud_coverage.sentinel import QCProcessorCloudCoverageSentinel
return QCProcessorCloudCoverageSentinel
def processor_landsat8(self):
from processors.cloud_coverage.landsat import QCProcessorCloudCoverageLandsat
return QCProcessorCloudCoverageLandsat
class QCProcessorL2Calibration(QCProcessorMultiBase):
"""Processor to create L2 products [delivery control]."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
from processors.l2_calibration.sentinel import \
QCProcessorL2CalibrationSentinel
return QCProcessorL2CalibrationSentinel
def processor_landsat8(self):
from processors.l2_calibration.landsat import \
QCProcessorL2CalibrationLandsat
return QCProcessorL2CalibrationLandsat
class QCProcessorTemplateIP(QCProcessorMultiBase):
"""Template image product multi-sensor processor."""
identifier = identifier_from_file(__file__)
def processor_sentinel2(self):
"""Sentinel-2 specific implementation.
:return QCProcessorTemplateIPSentinel:
"""
from processors.template_ip.sentinel import QCProcessorTemplateIPSentinel
return QCProcessorTemplateIPSentinel
def processor_landsat8(self):
"""Landsat-8 specific implementation.
:return QCProcessorTemplateIPLandsat:
"""
from processors.template_ip.landsat import QCProcessorTemplateIPLandsat
return QCProcessorTemplateIPLandsat
class QCProcessorLPMetadataControl(QCProcessorLPBase):
"""Land Product metadata control processor [validation control].
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "lpMetadataControlMetric"
isMeasurementOfSection = "qualityIndicators"
def check_dependency(self):
"""Check processor's software dependencies.
"""
from lxml import etree
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
from lxml import etree
Logger.info('Running LP metadata control')
response_data = {
"isMeasurementOf":
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.datetime.now(),
"value":
False,
}
try:
lp_meta_fn = glob.glob(
os.path.join(
self.config['map_product']['path'],
self.config['land_product']['product_metadata'] +
'*.xml'))[0]
except IndexError:
self.set_response_status(DbIpOperationStatus.rejected)
if 'product_metadata' in self.config['land_product']:
response_data['metadataSpecification'] = self.config[
'land_product']['product_metadata']
else:
response_data['metadataSpecification'] = ''
try:
Parser = etree.HTMLParser()
XMLDoc = etree.parse(open(lp_meta_fn, 'r'), Parser)
Logger.info('Land Product metadata available')
response_data['metadataAvailable'] = True
# validate xml
Elements = XMLDoc.xpath('//characterstring')
i = 0
for Element in Elements:
if (Element.text) is not None:
i += 1
if i > 5:
response_data[
'metadataCompliancy'] = self.validate_xml_metadata(
lp_meta_fn)
except:
Logger.error('Land Prodcut metadata not available')
response_data['metadataAvailable'] = False
response_data['metadataCompliancy'] = False
if response_data['metadataAvailable'] == True and \
response_data['metadataCompliancy'] == True:
response_data['value'] = True
if response_data['value'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def validate_xml_metadata(self, lp_meta_fn):
"""Check validity of XML LP metdata
:param str lp_meta_fn: LP mentadata path and filename
:return bool: true if metadata valid
"""
xsd_path = './processors/lp_metadata_control/lp_schema.xsd'
xmlschema_doc = etree.parse(xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_doc = etree.parse(lp_meta_fn)
result = xmlschema.validate(xml_doc)
if result:
Logger.info('Land Product INSPIRE metadata is valid')
return result
class QCProcessorLPInit(QCProcessorLPBase):
"""Processor performing LP initialization.
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "ipForLpInformationMetric"
def check_dependency(self):
"""Check processor's software dependecies.
"""
pass
def get_sensors(self):
"""Get acquisition sensors information.
:return dict: acquisition information
"""
acquisition_information = []
if self.config['image_products']['primary_platform'] == 'Sentinel-2':
acquisition_information.append(
{
"platform": {
"id": "https://earth.esa.int/concept/sentinel-2",
"platformShortName": "Sentinel-2"
},
"instrument": {
"id": "https://earth.esa.int/concept/s2-msi",
"instrumentShortName": "MSI"
}
}
)
if 'supplementary_platform' in self.config['image_products'].keys() and \
self.config['image_products']['supplementary_platform'] == 'Landsat-8':
acquisition_information.append(
{
"instrument": {
"instrumentShortName": "OLI",
"id": "https://earth.esa.int/concept/oli"
},
"platform": {
"platformShortName": "Landsat-8",
"id": "https://earth.esa.int/concept/landsat-8"
}
}
)
return acquisition_information
def get_raster_coding(self):
"""Get raster coding from configuration.
:return list: list of raster coding
"""
rc = []
for value, label in self.config['land_product']['raster_coding'].items():
if value == 'classification':
for k in label:
rc.append({
"name": 'classification_' + k,
"min": label[k],
"max": label[k]
})
elif value == 'regression':
rc.append({
"name": "regression",
"min": label['min'],
"max": label['max']
})
elif value == 'unclassifiable' or value == 'out_of_aoi':
rc.append({
"name": value,
"min": label,
"max": label
})
return rc
def run(self):
"""Run processor.
"""
# log start computation
self._run_start()
self.add_response(
self._run()
)
# log computation finished
self._run_done()
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
from sentinelsat.sentinel import geojson_to_wkt, read_geojson
return {
"type": "Feature",
"id": "http://qcmms-cat.spacebel.be/eo-catalog/series/{}/datasets/{}".format(
self.config['catalog']['lp_parent_identifier'],
self.config['land_product']['product_abbrev']),
"geometry": json.loads(wkt2json(self.read_aoi(self.config['land_product']['aoi']))),
"properties": {
"title": self.config['land_product']['product_abbrev'],
"identifier": self.config['land_product']['product_abbrev'],
"status": "PLANNED",
"kind": "http://purl.org/dc/dcmitype/Dataset",
"parentIdentifier": self.config['catalog']['lp_parent_identifier'],
"abstract": self.config['land_product']['product_abstract'],
# "date": "2020-05-12T00:00:00Z/2020-05-12T23:59:59Z"
"date": datetime.datetime.now(),
"categories": [
{
"term": "https://earth.esa.int/concept/" + self.config['land_product']['product_term'],
"label": self.config['land_product']['product_term']
},
{
"term": "http://www.eionet.europa.eu/gemet/concept/4599",
"label": "land"
},
{
"term": "https://earth.esa.int/concept/sentinel-2",
"label": "Sentinel-2"
}
],
"updated": datetime.datetime.now(),
"qualifiedAttribution": [
{
"type": "Attribution",
"agent": [
{
"type": "Organization",
"email": "*****@*****.**",
"name": "ESA/ESRIN",
"phone": "tel:+39 06 94180777",
"uri": "http://www.earth.esa.int",
"hasAddress": {
"country-name": "Italy",
"postal-code": "00044",
"locality": "Frascati",
"street-address": "Via Galileo Galilei CP. 64"
}
}
],
"role": "originator"
}
],
"acquisitionInformation": self.get_sensors(),
"productInformation": {
"productType": "classification",
"availabilityTime": "2019-06-20T15:23:57Z",
"format": "geoTIFF",
"referenceSystemIdentifier": "http://www.opengis.net/def/crs/EPSG/0/3035",
"qualityInformation": {
"qualityIndicators": []
}
},
"additionalAttributes": {
"product_focus": self.config['land_product']['product_focus'],
"lpReference": "EN-EEA.IDM.R0.18.009_Annex_8 Table 11",
"temporal_coverage": self.config['land_product']['temporal_focus'],
"geometric_resolution": self.config['land_product']['geometric_resolution'],
"grid": self.config['land_product']['grid'],
"crs": self.config['land_product']['crs'],
"geometric_accuracy": self.config['land_product']['geometric_accuracy'],
"thematic_accuracy": self.config['land_product']['thematic_accuracy'],
"data_type": self.config['land_product']['data_type'],
"mmu_pixels": self.config['land_product']['mmu_pixels'],
"necessary_attributes": self.config['land_product']['necessary_attributes'].split(','),
"rasterCoding": self.get_raster_coding(),
"seasonal_window": self.config['image_products']['seasonal_window'],
},
"links": {
"via": [
{
"href": "http://qcmms-cat.spacebel.be/eo-catalog/series/{}/datasets".format(
self.config['catalog']['collection']
),
"type": "application/geo+json",
"title": "Input data"
}
]
}
}
}
class QCProcessorVpxCoverage(QCProcessorLPBase):
"""Valid pixels coverage control processor [coverage control].
:param config: processor-related config file
:param response: processor QI metadata response managed by the manager
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "ipForLpInformationMetric"
level2_data = True
def __init__(self, config, response):
super(QCProcessorVpxCoverage, self).__init__(config, response)
self.platform_type = None
self.data_dir_suf = ''
def check_dependency(self):
"""Check processor's software dependencies.
"""
import numpy as np
from osgeo import gdal, gdalconst, gdal_array
def get_output_file(self, year):
"""Get output filename.
:param int year: year
:return str: target filename
"""
return os.path.join(self.config['project']['path'],
self.config['project']['downpath'],
self.identifier + '_10m_' + str(year) + '.tif')
def get_years(self):
"""Get years from configuration.
:return range: start-end year
"""
return range(self.config['image_products']['datefrom'].year,
self.config['image_products']['dateto'].year + 1)
def compute_coverage(self):
"""Compute vpx coverage from input valid pixel masks.
:return: path to output file
"""
# collect years
years = {}
for yr in self.get_years():
years[yr] = []
# collect input files from last IP processor
processed_ips = Logger.db_handler().processed_ips_last('valid_pixels')
ip_idx = 1
ip_count = len(processed_ips)
if ip_count == 0:
# create empty vpx_coverage file
from osgeo import gdal, gdalconst
im_reference = self.config.abs_path(
self.config['geometry']['reference_image'])
ids = gdal.Open(im_reference, gdalconst.GA_ReadOnly)
iproj = ids.GetProjection()
itrans = ids.GetGeoTransform()
vpx_band = ids.GetRasterBand(1)
for yr in years.keys():
out_file = self.get_output_file(yr)
driver = gdal.GetDriverByName('GTiff')
ods = driver.Create(out_file,
vpx_band.XSize,
vpx_band.YSize,
eType=vpx_band.DataType)
ods.SetGeoTransform(itrans)
ods.SetProjection(iproj)
ods = None
self.tif2jpg(out_file)
ids = None
raise ProcessorFailedError(self, "No input valid layers found")
for ip, platform_type, status in processed_ips:
Logger.info("Processing {}... ({}/{})".format(
ip, ip_idx, ip_count))
ip_idx += 1
# set current platform type
self.platform_type = QCPlatformType(platform_type)
if self.config['image_products'].get('{}_processing_level2'.format(
self.get_platform_type())) == 'S2MSI2A':
self.data_dir_suf = '.SAFE'
else:
self.data_dir_suf = ''
# delete previous results if needed
if status not in (DbIpOperationStatus.unchanged,
DbIpOperationStatus.rejected):
do_run = True
if self.get_last_ip_status(ip,
status) == DbIpOperationStatus.rejected:
Logger.info("{} skipped - rejected".format(ip))
continue
yr = self.get_ip_year(ip)
data_dir = self.get_data_dir(ip)
try:
years[yr] += self.filter_files(
data_dir, 'valid_pixels_{}m.tif'.format(
self.config['land_product']['geometric_resolution']))
except KeyError:
raise ProcessorFailedError(
self,
'Inconsistency between years in metadata and years in the '
'config file. Years from the config file are {}, but you '
'are querying year {}'.format(years, yr))
vpx_files = {}
for yr, input_files in years.items():
if len(input_files) < 1:
Logger.warning(
"No Vpx layers to be processed for {}".format(yr))
continue
# define output file
output_file = self.get_output_file(yr)
vpx_files[yr] = output_file
if os.path.exists(output_file):
# run processor if output file does not exist
continue
status = DbIpOperationStatus.updated if os.path.exists(output_file) \
else DbIpOperationStatus.added
Logger.info("Running countVpx for {}: {} layers".format(
yr, len(input_files)))
# run processor
try:
self.count_vpx(input_files, output_file)
except ProcessorFailedError:
pass
# log processor IP operation
if os.path.exists(output_file):
timestamp = self.file_timestamp(output_file)
else:
timestamp = None
# TBD
### self.lp_operation(status, timestamp=timestamp)
return vpx_files
def get_ip_year(self, ip):
"""Get image product filename.
:param str ip: image product title
:return dict: metadata
"""
meta_data = JsonIO.read(
os.path.join(self.config['project']['path'],
self.config['project']['metapath'], ip + ".geojson"))
return meta_data['Sensing start'].year
def count_vpx(self, input_files, output_file):
"""
Perform valid pixels coverage.
0: "noData", 1: "valid", 2: "clouds", 3: "shadows", 4: "snow", 5: "water"
Raise ProcessorFailedError on failure.
:param list input_files: list of input files
:param str output_file: output filename
"""
import numpy as np
from osgeo import gdal, gdalconst, gdal_array
try:
# open input data
ids = gdal.Open(input_files[0], gdalconst.GA_ReadOnly)
iproj = ids.GetProjection()
itrans = ids.GetGeoTransform()
vpx_band = ids.GetRasterBand(1)
# open output data
driver = gdal.GetDriverByName('GTiff')
ods = driver.Create(output_file,
vpx_band.XSize,
vpx_band.YSize,
eType=vpx_band.DataType)
ods.SetGeoTransform(itrans)
ods.SetProjection(iproj)
# create countVpx array
dtype = gdal_array.GDALTypeCodeToNumericTypeCode(
ids.GetRasterBand(1).DataType)
# dtype vs. no of images < 255 ?
vpx_count = np.zeros((vpx_band.YSize, vpx_band.XSize), dtype=dtype)
ids = None
# count valid pixels
for i in range(len(input_files)):
ids = gdal.Open(input_files[i], gdalconst.GA_ReadOnly)
vpx_band = ids.GetRasterBand(1)
vpx_arr = vpx_band.ReadAsArray()
vpx_count += vpx_arr
# save count
ods.GetRasterBand(1).WriteArray(vpx_count)
# ods.GetRasterBand(1).SetNoDataValue(vpx_band.GetNoDataValue())
# set color table
StyleReader(self.identifier).set_band_colors(ods)
# close data sources & write out
ids = None
ods = None
except RuntimeError as e:
raise ProcessorFailedError(
self, "Count Vpx processor failed: {}".format(e))
def compute_vpx_stats(self, vpx_file):
"""Compute stats for valid pixels coverage.
:param str vpx_file: vpx file path
:return dict: QI metadata
"""
from osgeo import gdal
# compute min/max
value, count, ncells = self.compute_value_count(vpx_file)
vpx_pct = 0.0
for idx in range(len(value)):
if value[idx] == 0:
vpx_pct = count[idx] / ncells * 100
data = {
"min": int(min(value)),
"max": int(max(value)),
"gapPct": round(vpx_pct, 4),
"mask": self.file_basename(self.tif2jpg(vpx_file))
}
ds = None
return data
def get_quality_indicators(self):
"""Get quality indicators.
:return dict: QI metadata
"""
vpx_timestamp = datetime.datetime.now()
years = {}
vpx_timestamp = None
value = False
fitness = 'FULL'
for yr, vpx_file in self.compute_coverage().items():
# take the last file
if not vpx_timestamp:
vpx_timestamp = self.file_timestamp(vpx_file)
years[yr] = self.compute_vpx_stats(vpx_file)
if value is False and years[yr]['max'] > 0:
# at least one non-zero pixel
value = True
if fitness == 'FULL' and years[yr]['min'] == 0:
# FULL -> PARTIAL: at least one zero pixel
fitness = 'PARTIAL'
if value is False:
# no coverage
fitness = 'NO'
return {
"value": value,
"generatedAtTime": vpx_timestamp,
"vpxCoverage": years,
"fitnessForPurpose": fitness
}
def _run(self):
"""Run computation.
:return dict: QI metadata
"""
response_data = {
'isMeasurementOf':
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
'value':
False
}
response_data.update(self.get_quality_indicators())
return response_data
class QCProcessorLPOrdinaryControl(QCProcessorLPBase):
"""Land Product ordinary control processor [validation control].
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "lpOrdinaryControlMetric"
isMeasurementOfSection = "qualityIndicators"
def check_dependency(self):
"""Check processor's dependencies."""
import numpy
from osgeo import gdal
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
Logger.info('Running ordinary control')
response_data = {
'isMeasurementOf':
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
'value':
False
}
prod_raster = self.config['land_product']['product_type'][
-1] + '_raster'
lp_fn = os.path.join(self.config['map_product']['path'],
self.config['map_product'][prod_raster])
lp_characteristics = self._lp_characteristics(lp_fn)
response_data.update(lp_characteristics)
if lp_characteristics['read'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
res = self.config['land_product']['geometric_resolution']
if lp_characteristics['xRes'] == res and lp_characteristics[
'xRes'] == res:
response_data['value'] = True
else:
response_data['value'] = False
if str(lp_characteristics['epsg']) == str(
self.config['land_product']['epsg']):
response_data['value'] = True
else:
response_data['value'] = False
if str(lp_characteristics['dataType']) == str(
self.config['land_product']['data_type']):
response_data['value'] = True
else:
response_data['value'] = False
if str(lp_characteristics['rasterFormat']) == str(
self.config['land_product']['delivery_format']):
response_data['value'] = True
else:
response_data['value'] = False
if response_data['value'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def _calc_aoiCoveragePct(self, input_zone_polygon, input_value_raster,
lp_min, lp_max, unclassifiable, out_of_aoi):
"""Calculate Land Product coverage statistics.
:param str input_zone_polygon: input vector file with zones
:param str input_value_raster: input raster value file
:param int lp_min: LP min value
:param float lp_max: LP max value
:param int unclassifiable: unclassifiable value
:param out_of_aoi: output AOI vector file
"""
raster = gdal.Open(input_value_raster)
shp = ogr.Open(input_zone_polygon)
lyr = shp.GetLayer()
# Get raster georeference info
transform = raster.GetGeoTransform()
xOrigin = transform[0]
yOrigin = transform[3]
pixelWidth = transform[1]
pixelHeight = transform[5]
sourceSR = lyr.GetSpatialRef()
# gdal 2.4.2 vs. gdal 3 in docker
if int(osgeo.__version__[0]) >= 3:
sourceSR.SetAxisMappingStrategy(
osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER)
feat = lyr.GetNextFeature()
geom = feat.GetGeometryRef()
if (geom.GetGeometryName() == 'MULTIPOLYGON'):
count = 0
pointsX = []
pointsY = []
for polygon in geom:
geomInner = geom.GetGeometryRef(count)
ring = geomInner.GetGeometryRef(0)
numpoints = ring.GetPointCount()
for p in range(numpoints):
lon, lat, z = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
count += 1
elif (geom.GetGeometryName() == 'POLYGON'):
ring = geom.GetGeometryRef(0)
numpoints = ring.GetPointCount()
pointsX = []
pointsY = []
for p in range(numpoints):
lon, lat, z = ring.GetPoint(p)
pointsX.append(lon)
pointsY.append(lat)
else:
sys.exit(
"ERROR: Geometry needs to be either Polygon or Multipolygon")
xmin = min(pointsX)
xmax = max(pointsX)
ymin = min(pointsY)
ymax = max(pointsY)
xoff = int((xmin - xOrigin) / pixelWidth)
yoff = int((yOrigin - ymax) / pixelWidth)
xcount = int((xmax - xmin) / pixelWidth) + 1
ycount = int((ymax - ymin) / pixelWidth) + 1
# Memory target raster
target_ds = gdal.GetDriverByName('MEM').Create('', xcount, ycount, 1,
gdal.GDT_Byte)
target_ds.SetGeoTransform((
xmin,
pixelWidth,
0,
ymax,
0,
pixelHeight,
))
raster_srs = osr.SpatialReference()
raster_srs.ImportFromWkt(raster.GetProjectionRef())
target_ds.SetProjection(raster_srs.ExportToWkt())
# Rasterize zone polygon to raster
gdal.RasterizeLayer(target_ds, [1], lyr, burn_values=[1])
banddataraster = raster.GetRasterBand(1)
dataraster = banddataraster.ReadAsArray(xoff, yoff, xcount,
ycount).astype(np.float)
bandmask = target_ds.GetRasterBand(1)
datamask = bandmask.ReadAsArray(0, 0, xcount, ycount).astype(np.float)
zoneraster = np.ma.masked_array(dataraster, np.logical_not(datamask))
# Calculate overlay statistics
lp_maped_px = np.sum(
((zoneraster >= lp_min) & (zoneraster <= lp_max)) * 1)
lp_out_px = np.sum(
((zoneraster == unclassifiable) & (zoneraster == out_of_aoi)) * 1)
_aoiCoveragePct = lp_maped_px / (lp_maped_px + lp_out_px) * 100.0
return _aoiCoveragePct
def _lp_characteristics(self, lp_fn):
"""Get LP characteristics to check.
:param lp_fn: input lp raster file
:return dict: lp characteristics
"""
import numpy as np
from osgeo import gdal, gdal_array, osr
from osgeo import gdalconst
gdal.UseExceptions()
lp_characteristics = {}
try:
ids = gdal.Open(lp_fn, gdalconst.GA_ReadOnly)
lp_characteristics['read'] = True
except RuntimeError:
lp_characteristics['read'] = False
return lp_characteristics
# Spatial resolution
ids = gdal.Open(lp_fn, gdalconst.GA_ReadOnly)
img_array = ids.ReadAsArray()
geotransform = list(ids.GetGeoTransform())
lp_characteristics['xRes'] = abs(geotransform[1])
lp_characteristics['yRes'] = abs(geotransform[5])
# Projection
proj = osr.SpatialReference(wkt=ids.GetProjection())
map_epsg = (proj.GetAttrValue('AUTHORITY', 1))
lp_characteristics['epsg'] = map_epsg
# Coding data type
map_dtype = gdal_array.GDALTypeCodeToNumericTypeCode(
ids.GetRasterBand(1).DataType)
if (map_dtype == np.dtype('uint8')):
lp_characteristics['dataType'] = 'u8'
else:
lp_characteristics['dataType'] = str(ids.GetRasterBand(1).DataType)
# Map extent in the 'map_epsg'
ulx, xres, xskew, uly, yskew, yres = ids.GetGeoTransform()
lrx = ulx + (ids.RasterXSize * xres)
lry = uly + (ids.RasterYSize * yres)
lp_characteristics['extentUlLr'] = [ulx, uly, lrx, lry]
# Do spatial overlay
aoi_polygon = os.path.join(self.config['map_product']['path'],
self.config['map_product']['map_aoi'])
coding_val = []
for prod in self.config['land_product']['product_type']:
for val in self.config['land_product']['raster_coding'][prod]:
coding_val.append(
self.config['land_product']['raster_coding'][prod][val])
lp_min = min(coding_val)
lp_max = max(coding_val)
unclassifiable = self.config['land_product']['raster_coding'][
'unclassifiable']
out_of_aoi = self.config['land_product']['raster_coding']['out_of_aoi']
lp_characteristics['aoiCoveragePct'] = self._calc_aoiCoveragePct(
aoi_polygon, lp_fn, lp_min, lp_max, unclassifiable, out_of_aoi)
# Map format
raster_format = lp_fn.split('.')[-1]
if raster_format == 'tif':
raster_format = "GeoTIFF"
lp_characteristics['rasterFormat'] = raster_format
return lp_characteristics
class QCProcessorLPThematicValidationControl(QCProcessorLPBase):
"""Land Product thematic validation control processor [validation control].
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "lpThematicValidationMetric"
isMeasurementOfSection = "qualityIndicators"
def check_dependency(self):
"""Check processor's software dependecies.
"""
import sklearn
import scipy
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
Logger.info('Running thematic validation control')
response_data = {
'isMeasurementOf':
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.now(),
"value":
False
}
reference_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['reference_layer'])
if not os.path.isfile(reference_fn):
Logger.error("File {} not found".format(reference_fn))
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
themes = self.config['land_product']['product_type']
for theme in themes:
if theme == 'classification':
classification_qi_ = self._lp_classification_validation(
self.config)
response_data.update({"classification": classification_qi_})
if float(classification_qi_['overallAccuracy']) >= \
float(self.config['land_product']['thematic_accuracy']):
response_data['value'] = True
else:
response_data['value'] = False
elif theme == 'regression':
regression_qi = self._lp_regression_validation(self.config)
regression_prod_name = self.config['land_product'][
'regression_name']
response_data.update({regression_prod_name: regression_qi})
print(float(regression_qi['rmse']))
print(float(self.config['land_product']['rmse_accuracy']))
if float(regression_qi['rmse']) <= \
float(self.config['land_product']['rmse_accuracy']):
response_data['value'] = True
else:
response_data['value'] = False
if response_data['value'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def _read_point_data(self, ras_fn, vec_fn, attrib, no_data):
"""Raster map ~ vector reference spatial overlay.
:param str ras_fn: input raster file
:param str vec_fn: input vector file
:param str attrib: attribute
:param int no_data: no data value
"""
from osgeo import gdal, ogr
src_ds = gdal.Open(ras_fn)
gt = src_ds.GetGeoTransform()
rb = src_ds.GetRasterBand(1)
ds = ogr.Open(vec_fn)
lyr = ds.GetLayer()
ref_val = []
map_val = []
for feat in lyr:
geom = feat.GetGeometryRef()
if geom.GetGeometryName() == 'POLYGON':
mx, my = geom.Centroid().GetX(), geom.Centroid().GetY()
elif geom.GetGeometryName() == 'POINT':
mx, my = geom.GetX(), geom.GetY()
px = int((mx - gt[0]) / gt[1])
py = int((my - gt[3]) / gt[5])
intval = rb.ReadAsArray(px, py, 1, 1)
if not ((intval[0][0] == no_data) or
(feat.GetField(attrib) == no_data)):
map_val.append(intval[0][0])
ref_val.append(feat.GetField(attrib))
return ref_val, map_val
def _lp_classification_validation(self, config):
"""Extract classification thematic quality indicators.
:param dict config: configuration
:return dict: QI metadata
"""
import numpy as np
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import cohen_kappa_score
classification_qi = {}
lp_map_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['classification_raster'])
reference_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['reference_layer'])
reference_attrib = self.config['map_product'][
'classification_attribute']
no_data = self.config['land_product']['raster_coding']['out_of_aoi']
ref_val_, map_val_ = self._read_point_data(lp_map_fn, reference_fn,
reference_attrib, no_data)
c_report = classification_report(ref_val_, map_val_, output_dict=True)
c_matrix = confusion_matrix(ref_val_, map_val_)
ref_lineage_ = reference_fn.split('/')[-1]
overall_accuracy_ = (np.sum(c_matrix.diagonal()) /
np.sum(c_matrix)) * 100
producers_accuracy_ = (c_report['weighted avg']['precision']) * 100
users_accuracy_ = (c_report['weighted avg']['recall']) * 100
kappa_ = (cohen_kappa_score(
ref_val_, map_val_, labels=None, weights=None)) * 100
classes = config['land_product']['raster_coding']['classification']
classes_names = [
k for k, v in sorted(classes.items(), key=lambda item: item[1])
]
return {
"codingClasses": classes_names,
"lineage": "http://qcmms.esa.int/{}".format(ref_lineage_),
"overallAccuracy": round(overall_accuracy_, 2),
"producersAccuracy": round(producers_accuracy_, 2),
"usersAccuracy": round(users_accuracy_, 2),
"kappa": round(kappa_, 2),
"confusionMatrix": c_matrix.tolist()
}
def _lp_regression_validation(self, config):
"""Extract regression thematic quality indicators.
:param dict config: configuration
:return dict: QI metadata
"""
import numpy as np
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from scipy.stats import pearsonr
regression_qi = {}
lp_map_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['regression_raster'])
reference_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['reference_layer'])
reference_attrib = self.config['map_product']['regression_attribute']
no_data = self.config['land_product']['raster_coding']['out_of_aoi']
ref_val_, map_val_ = self._read_point_data(lp_map_fn, reference_fn,
reference_attrib, no_data)
MAE_ = mean_absolute_error(ref_val_, map_val_)
MSE_ = mean_squared_error(ref_val_, map_val_)
RMSE_ = np.sqrt(mean_squared_error(ref_val_, map_val_))
pearson_r_, p_val_ = pearsonr(ref_val_, map_val_)
ref_lineage_ = reference_fn.split('/')[-1]
regression_values = config['land_product']['raster_coding'][
'regression']
return {
"lineage": "http://qcmms.esa.int/{}".format(ref_lineage_),
"codingValues": regression_values,
"mae": round(MAE_, 2),
"mse": round(MSE_, 2),
"rmse": round(RMSE_, 2),
"pearsonR": round(pearson_r_, 2)
}
class QCProcessorLPInterpretationControl(QCProcessorLPBase):
"""Land Product interpretation control processor [interpretation control].
Be aware it is an optional processor!
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "lpInterpretationMetric"
isMeasurementOfSection = "qualityIndicators"
def check_dependency(self):
"""Check processor's software dependencies.
"""
pass
def _run(self):
"""Perform processor's tasks.
:return dict: QI metadata
"""
Logger.info('Running interpretation control')
response_data = {
"isMeasurementOf":
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.now(),
"value":
False
}
lp_interpretation = self._read_interpretation_qi()
if lp_interpretation is None:
# no LP interpretation metadata available
return {}
for ltype in self.config['land_product']['product_type']:
if ltype == 'classification':
if lp_interpretation[ltype]['overallAccuracy'] >= \
self.config['land_product']['thematic_accuracy']:
response_data['value'] = True
else:
response_data['value'] = False
response_data[ltype] = lp_interpretation[ltype]
elif ltype == 'regression':
reg_name = self.config['land_product']['regression_name']
if 'rmse_accuracy' in self.config['land_product']:
if lp_interpretation[reg_name]['rmse'] <= \
self.config['land_product']['rmse_accuracy']:
response_data['value'] = True
else:
response_data['value'] = False
response_data[reg_name] = lp_interpretation[reg_name]
if response_data['value'] is False:
self.set_response_status(DbIpOperationStatus.rejected)
return response_data
def _read_interpretation_qi(self):
"""Read LP interpretation quality indicators.
:return dict: quality indicators
"""
try:
lp_interpretation_fn = os.path.join(
self.config['map_product']['path'],
self.config['map_product']['map_interpretation_qi'])
except KeyError:
Logger.info("{} is not defined".format('map_interpretation_qi'))
return None
if not os.path.isfile(lp_interpretation_fn):
Logger.info("File {} not found".format(lp_interpretation_fn))
return None
return JsonIO.read(lp_interpretation_fn)
class QCProcessorTemplateLP(QCProcessorLPBase):
"""Template land product processor.
"""
identifier = identifier_from_file(__file__)
isMeasurementOf = "lpInterpretationMetric"
def run(self):
"""Run processor.
Define this functions only if your processor is the first in a queue.
Check processors.lp_init for a real example.
"""
self.add_response(self._run())
def _run(self):
"""Perform processor's tasks.
Check processors.lp_ordinary_control for a real example.
:param meta_file: path to JSON metafile
:param str data_dir: path to data directory
:param str output_dir: path to output processor directory
:return dict: QI metadata
"""
response_data = {
"type": "Feature",
"id":
"http://qcmms-cat.spacebel.be/eo-catalog/series/EOP:MAPRADIX:LP_TUC1/datasets/IMD_2018_010m",
"geometry": {
"type":
"Polygon",
"coordinates": [[[13.58808613662857, 50.54373233188457],
[13.616762732856103, 49.55648784343155],
[15.134970502622016, 49.56467629785398],
[15.137769755767613, 50.552210622261306],
[13.58808613662857, 50.54373233188457]]]
},
"properties": {
"title":
"IMD_2018_010m",
"identifier":
"IMD_2018_010m",
"status":
"PLANNED",
"kind":
"http://purl.org/dc/dcmitype/Dataset",
"parentIdentifier":
"EOP:ESA:LP:TUC1",
"collection":
"EOP:ESA:GR1:UC1",
"abstract":
"The high-resolution imperviousness product capture the percentage of soil sealing. Built-up areas are characterized by the substitution of the original (semi-) natural land cover or water surface with an artificial, often impervious cover. This product of imperviousness layer constitutes the main status layer. There is per-pixel estimates of impermeable cover of soil (soil sealing) and are mapped as the degree of imperviousness (0-100%). Imperviousness 2018 is the continuation of the existing HRL imperviousness status product for the 2018 reference year, but with an increase in spatial resolution from 20m to (now) 10m.",
"date":
"2020-05-12T00:00:00Z/2020-05-12T23:59:59Z",
"categories": [{
"term": "https://earth.esa.int/concept/urban",
"label": "Forestry"
}, {
"term": "http://www.eionet.europa.eu/gemet/concept/4599",
"label": "land"
}, {
"term": "https://earth.esa.int/concept/sentinel-2",
"label": "Sentinel-2"
}],
"updated":
"2020-05-12T15:23:57Z",
"qualifiedAttribution": [{
"type":
"Attribution",
"agent": [{
"type": "Organization",
"email": "*****@*****.**",
"name": "ESA/ESRIN",
"phone": "tel:+39 06 94180777",
"uri": "http://www.earth.esa.int",
"hasAddress": {
"country-name": "Italy",
"postal-code": "00044",
"locality": "Frascati",
"street-address": "Via Galileo Galilei CP. 64"
}
}],
"role":
"originator"
}],
"acquisitionInformation": [{
"platform": {
"id": "https://earth.esa.int/concept/sentinel-2",
"platformShortName": "Sentinel-2"
},
"instrument": {
"id": "https://earth.esa.int/concept/s2-msi",
"instrumentShortName": "MSI"
}
}],
"productInformation": {
"productType": "classification",
"availabilityTime": "2019-06-20T15:23:57Z",
"format": "geoTIFF",
"referenceSystemIdentifier":
"http://www.opengis.net/def/crs/EPSG/0/3035",
"qualityInformation": {
"qualityIndicators": [{
"isMeasurementOf":
"http://qcmms.esa.int/quality-indicators/#lpInterpretationMetric",
"generatedAtTime": "2019-10-17T17:20:32.30Z",
"value": True,
"classification": {
"codingClasses": ["non-urban", "urban"],
"overallAccuracy":
91.0,
"producersAccuracy":
99.0,
"usersAccuracy":
92.0,
"kappa":
88.0,
"confusionMatrix": [[92, 8], [1, 99]],
"lineage":
"http://qcmms.esa.int/Mx_sealing_simple_v0.9"
},
"densityCover": {
"mae":
10.25,
"mse":
174.86,
"rmse":
13.22,
"pearsonR":
0.69,
"lineage":
"http://qcmms.esa.int/Mx_sealing_simple_v0.9"
}
}, {
"isMeasurementOf":
"http://qcmms.esa.int/quality-indicators/#ipForLpInformationMetric",
"value":
True,
"generatedAtTime":
"2019-10-17T17:20:32.30Z",
"vpxCoverage": {
"2018": {
"min":
3,
"max":
22,
"gapPct":
0.0,
"mask":
"http://93.91.57.111/UC1/image_products/vpx_coverage_10m.jpg"
}
},
"fitnessForPurpose":
"PARTIAL",
"lineage":
"http://qcmms.esa.int/Mx_vpx_v0.9",
"generatedAtTime":
"2020-05-12T17:20:32.30Z"
}, {
"isMeasurementOf":
"http://qcmms.esa.int/quality-indicators/#lpMetadataControlMetric",
"value": True,
"generatedAtTime": "2020-05-12T17:20:32.30Z",
"metadataAvailable": True,
"metadataSpecification": "INSPIRE",
"metadataCompliancy": True
}, {
"isMeasurementOf":
"http://qcmms.esa.int/quality-indicators/#lpOrdinaryControlMetric",
"value":
True,
"generatedAtTime":
"2019-10-17T17:20:32.30Z",
"read":
True,
"xRes":
10.0,
"yRes":
10.0,
"epsg":
"3035",
"dataType":
"u8",
"rasterFormat":
"GeoTIFF",
"extentUlLr":
[4621500.0, 3019160.0, 4659740.0, 2988580.0],
"aoiCoveragePct":
100.0
}, {
"isMeasurementOf":
"http://qcmms.esa.int/quality-indicators/#lpThematicValidationMetric",
"value": True,
"generatedAtTime": "2020-05-12T17:20:32.30Z",
"classification": {
"codingClasses": ["non-urban", "urban"],
"lineage":
"http://qcmms.esa.int/prague_sealing_references_330.shp",
"overallAccuracy": 91.2,
"producersAccuracy": 90.4,
"usersAccuracy": 93.0,
"kappa": 88.3,
"confusionMatrix": [[92, 8], [1, 99]]
},
"densityCover": {
"mae": 10.2,
"mse": 120.3,
"rmse": 11.0,
"pearsonR": 0.71,
"lineage":
"http://qcmms.esa.int/prague_sealing_references_330.shp",
"codingValues": {
"non-sealing": 0,
"sealingMin": 1,
"sealingMax": 100,
"unclassified": 254,
"outsideAoi": 254
}
}
}]
}
},
"additionalAttributes": {
"product_focus":
"classification",
"lpReference":
"EN-EEA.IDM.R0.18.009_Annex_8 Table 11",
"temporal_coverage":
"status",
"geometric_resolution":
10,
"grid":
"EEA Reference Grid",
"crs":
"European ETRS89 LAEA",
"geometric_accuracy":
0.5,
"thematic_accuracy":
90,
"data_type":
"u8",
"mmu_pixels":
1,
"necessary_attributes":
["raster value", "count", "class names"],
"raster_coding": [{
"name": "non-impervious",
"min": 0,
"max": 0
}, {
"name": "impervious",
"min": 1,
"max": 100
}, {
"name": "unclassified",
"min": 254,
"max": 254
}, {
"name": "outside area",
"min": 255,
"max": 255
}],
"seasonal_window": [5, 6, 7, 8]
},
"links": {
"previews": [{
"href":
"https://qcmms-cat.spacebel.be/archive/lp/land_tuc1.png",
"type": "image/png",
"title": "Quicklook"
}],
"via": [{
"href":
"http://qcmms-cat.spacebel.be/eo-catalog/series/EOP:ESA:GR1:UC1/datasets",
"type": "application/geo+json",
"title": "Input data"
}]
}
}
}
return response_data
