class QCProcessorSearch(QCProcessorMultiBase): """Multi-mission search processor [feasibility control].""" identifier = identifier_from_file(__file__) def __init__(self, config, response): super(QCProcessorSearch, self).__init__( config, response ) self.metapath = os.path.join( self.config['project']['path'], self.config['project']['metapath'] ) # remove csv file (platform-dependent processors append new lines) csv_file = os.path.join( self.metapath, '{}_fullmetadata.csv'.format( self.config['land_product']['product_abbrev'] ) ) if os.path.exists(csv_file): os.remove(csv_file) def processor_sentinel2(self): from processors.search.sentinel import QCProcessorSearchSentinel return QCProcessorSearchSentinel def processor_landsat8(self): from processors.search.landsat import QCProcessorSearchLandsat return QCProcessorSearchLandsat
class QCProcessorGeometryQuality(QCProcessorMultiBase): """Geometry quality control processor [detailed control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.geometry_quality.sentinel import QCProcessorGeometryQualitySentinel return QCProcessorGeometryQualitySentinel def processor_landsat8(self): from processors.geometry_quality.landsat import QCProcessorGeometryQualityLandsat return QCProcessorGeometryQualityLandsat
class QCProcessorHarmonizationStack(QCProcessorMultiBase): """Processor creating a resampled stack from individual band files [multi-sensor control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.harmonization_stack.sentinel import QCProcessorHarmonizationStackSentinel return QCProcessorHarmonizationStackSentinel def processor_landsat8(self): from processors.harmonization_stack.landsat import QCProcessorHarmonizationStackLandsat return QCProcessorHarmonizationStackLandsat
class QCProcessorDownload(QCProcessorMultiBase): """Download processor [delivery control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.download.sentinel import QCProcessorDownloadSentinel return QCProcessorDownloadSentinel def processor_landsat8(self): from processors.download.landsat import QCProcessorDownloadLandsat return QCProcessorDownloadLandsat
class QCProcessorHarmonizationControl(QCProcessorMultiBase): """Harmonization control processor [multi-sensor control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.harmonization_control.sentinel import QCProcessorHarmonizationControlSentinel return QCProcessorHarmonizationControlSentinel def processor_landsat8(self): from processors.harmonization_control.landsat import QCProcessorHarmonizationControlLandsat return QCProcessorHarmonizationControlLandsat
class QCProcessorValidPixels(QCProcessorMultiBase): """Validity pixel control processor [detailed control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.valid_pixels.sentinel import QCProcessorValidPixelsSentinel return QCProcessorValidPixelsSentinel def processor_landsat8(self): from processors.valid_pixels.landsat import QCProcessorValidPixelsLandsat return QCProcessorValidPixelsLandsat
class QCProcessorRadiometryControl(QCProcessorMultiBase): """Radiometry control processor [detailed control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.radiometry_control.sentinel import QCProcessorRadiometryControlSentinel return QCProcessorRadiometryControlSentinel def processor_landsat8(self): from processors.radiometry_control.landsat import QCProcessorRadiometryControlLandsat return QCProcessorRadiometryControlLandsat
class QCProcessorCloudCoverage(QCProcessorMultiBase): """Cloud coverage control processor [defailed control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.cloud_coverage.sentinel import QCProcessorCloudCoverageSentinel return QCProcessorCloudCoverageSentinel def processor_landsat8(self): from processors.cloud_coverage.landsat import QCProcessorCloudCoverageLandsat return QCProcessorCloudCoverageLandsat
class QCProcessorL2Calibration(QCProcessorMultiBase): """Processor to create L2 products [delivery control].""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): from processors.l2_calibration.sentinel import \ QCProcessorL2CalibrationSentinel return QCProcessorL2CalibrationSentinel def processor_landsat8(self): from processors.l2_calibration.landsat import \ QCProcessorL2CalibrationLandsat return QCProcessorL2CalibrationLandsat
class QCProcessorTemplateIP(QCProcessorMultiBase): """Template image product multi-sensor processor.""" identifier = identifier_from_file(__file__) def processor_sentinel2(self): """Sentinel-2 specific implementation. :return QCProcessorTemplateIPSentinel: """ from processors.template_ip.sentinel import QCProcessorTemplateIPSentinel return QCProcessorTemplateIPSentinel def processor_landsat8(self): """Landsat-8 specific implementation. :return QCProcessorTemplateIPLandsat: """ from processors.template_ip.landsat import QCProcessorTemplateIPLandsat return QCProcessorTemplateIPLandsat
class QCProcessorLPMetadataControl(QCProcessorLPBase): """Land Product metadata control processor [validation control]. """ identifier = identifier_from_file(__file__) isMeasurementOf = "lpMetadataControlMetric" isMeasurementOfSection = "qualityIndicators" def check_dependency(self): """Check processor's software dependencies. """ from lxml import etree def _run(self): """Perform processor's tasks. :return dict: QI metadata """ from lxml import etree Logger.info('Running LP metadata control') response_data = { "isMeasurementOf": '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.datetime.now(), "value": False, } try: lp_meta_fn = glob.glob( os.path.join( self.config['map_product']['path'], self.config['land_product']['product_metadata'] + '*.xml'))[0] except IndexError: self.set_response_status(DbIpOperationStatus.rejected) if 'product_metadata' in self.config['land_product']: response_data['metadataSpecification'] = self.config[ 'land_product']['product_metadata'] else: response_data['metadataSpecification'] = '' try: Parser = etree.HTMLParser() XMLDoc = etree.parse(open(lp_meta_fn, 'r'), Parser) Logger.info('Land Product metadata available') response_data['metadataAvailable'] = True # validate xml Elements = XMLDoc.xpath('//characterstring') i = 0 for Element in Elements: if (Element.text) is not None: i += 1 if i > 5: response_data[ 'metadataCompliancy'] = self.validate_xml_metadata( lp_meta_fn) except: Logger.error('Land Prodcut metadata not available') response_data['metadataAvailable'] = False response_data['metadataCompliancy'] = False if response_data['metadataAvailable'] == True and \ response_data['metadataCompliancy'] == True: response_data['value'] = True if response_data['value'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data def validate_xml_metadata(self, lp_meta_fn): """Check validity of XML LP metdata :param str lp_meta_fn: LP mentadata path and filename :return bool: true if metadata valid """ xsd_path = './processors/lp_metadata_control/lp_schema.xsd' xmlschema_doc = etree.parse(xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_doc = etree.parse(lp_meta_fn) result = xmlschema.validate(xml_doc) if result: Logger.info('Land Product INSPIRE metadata is valid') return result
class QCProcessorLPInit(QCProcessorLPBase): """Processor performing LP initialization. """ identifier = identifier_from_file(__file__) isMeasurementOf = "ipForLpInformationMetric" def check_dependency(self): """Check processor's software dependecies. """ pass def get_sensors(self): """Get acquisition sensors information. :return dict: acquisition information """ acquisition_information = [] if self.config['image_products']['primary_platform'] == 'Sentinel-2': acquisition_information.append( { "platform": { "id": "https://earth.esa.int/concept/sentinel-2", "platformShortName": "Sentinel-2" }, "instrument": { "id": "https://earth.esa.int/concept/s2-msi", "instrumentShortName": "MSI" } } ) if 'supplementary_platform' in self.config['image_products'].keys() and \ self.config['image_products']['supplementary_platform'] == 'Landsat-8': acquisition_information.append( { "instrument": { "instrumentShortName": "OLI", "id": "https://earth.esa.int/concept/oli" }, "platform": { "platformShortName": "Landsat-8", "id": "https://earth.esa.int/concept/landsat-8" } } ) return acquisition_information def get_raster_coding(self): """Get raster coding from configuration. :return list: list of raster coding """ rc = [] for value, label in self.config['land_product']['raster_coding'].items(): if value == 'classification': for k in label: rc.append({ "name": 'classification_' + k, "min": label[k], "max": label[k] }) elif value == 'regression': rc.append({ "name": "regression", "min": label['min'], "max": label['max'] }) elif value == 'unclassifiable' or value == 'out_of_aoi': rc.append({ "name": value, "min": label, "max": label }) return rc def run(self): """Run processor. """ # log start computation self._run_start() self.add_response( self._run() ) # log computation finished self._run_done() def _run(self): """Perform processor's tasks. :return dict: QI metadata """ from sentinelsat.sentinel import geojson_to_wkt, read_geojson return { "type": "Feature", "id": "http://qcmms-cat.spacebel.be/eo-catalog/series/{}/datasets/{}".format( self.config['catalog']['lp_parent_identifier'], self.config['land_product']['product_abbrev']), "geometry": json.loads(wkt2json(self.read_aoi(self.config['land_product']['aoi']))), "properties": { "title": self.config['land_product']['product_abbrev'], "identifier": self.config['land_product']['product_abbrev'], "status": "PLANNED", "kind": "http://purl.org/dc/dcmitype/Dataset", "parentIdentifier": self.config['catalog']['lp_parent_identifier'], "abstract": self.config['land_product']['product_abstract'], # "date": "2020-05-12T00:00:00Z/2020-05-12T23:59:59Z" "date": datetime.datetime.now(), "categories": [ { "term": "https://earth.esa.int/concept/" + self.config['land_product']['product_term'], "label": self.config['land_product']['product_term'] }, { "term": "http://www.eionet.europa.eu/gemet/concept/4599", "label": "land" }, { "term": "https://earth.esa.int/concept/sentinel-2", "label": "Sentinel-2" } ], "updated": datetime.datetime.now(), "qualifiedAttribution": [ { "type": "Attribution", "agent": [ { "type": "Organization", "email": "*****@*****.**", "name": "ESA/ESRIN", "phone": "tel:+39 06 94180777", "uri": "http://www.earth.esa.int", "hasAddress": { "country-name": "Italy", "postal-code": "00044", "locality": "Frascati", "street-address": "Via Galileo Galilei CP. 64" } } ], "role": "originator" } ], "acquisitionInformation": self.get_sensors(), "productInformation": { "productType": "classification", "availabilityTime": "2019-06-20T15:23:57Z", "format": "geoTIFF", "referenceSystemIdentifier": "http://www.opengis.net/def/crs/EPSG/0/3035", "qualityInformation": { "qualityIndicators": [] } }, "additionalAttributes": { "product_focus": self.config['land_product']['product_focus'], "lpReference": "EN-EEA.IDM.R0.18.009_Annex_8 Table 11", "temporal_coverage": self.config['land_product']['temporal_focus'], "geometric_resolution": self.config['land_product']['geometric_resolution'], "grid": self.config['land_product']['grid'], "crs": self.config['land_product']['crs'], "geometric_accuracy": self.config['land_product']['geometric_accuracy'], "thematic_accuracy": self.config['land_product']['thematic_accuracy'], "data_type": self.config['land_product']['data_type'], "mmu_pixels": self.config['land_product']['mmu_pixels'], "necessary_attributes": self.config['land_product']['necessary_attributes'].split(','), "rasterCoding": self.get_raster_coding(), "seasonal_window": self.config['image_products']['seasonal_window'], }, "links": { "via": [ { "href": "http://qcmms-cat.spacebel.be/eo-catalog/series/{}/datasets".format( self.config['catalog']['collection'] ), "type": "application/geo+json", "title": "Input data" } ] } } }
class QCProcessorVpxCoverage(QCProcessorLPBase): """Valid pixels coverage control processor [coverage control]. :param config: processor-related config file :param response: processor QI metadata response managed by the manager """ identifier = identifier_from_file(__file__) isMeasurementOf = "ipForLpInformationMetric" level2_data = True def __init__(self, config, response): super(QCProcessorVpxCoverage, self).__init__(config, response) self.platform_type = None self.data_dir_suf = '' def check_dependency(self): """Check processor's software dependencies. """ import numpy as np from osgeo import gdal, gdalconst, gdal_array def get_output_file(self, year): """Get output filename. :param int year: year :return str: target filename """ return os.path.join(self.config['project']['path'], self.config['project']['downpath'], self.identifier + '_10m_' + str(year) + '.tif') def get_years(self): """Get years from configuration. :return range: start-end year """ return range(self.config['image_products']['datefrom'].year, self.config['image_products']['dateto'].year + 1) def compute_coverage(self): """Compute vpx coverage from input valid pixel masks. :return: path to output file """ # collect years years = {} for yr in self.get_years(): years[yr] = [] # collect input files from last IP processor processed_ips = Logger.db_handler().processed_ips_last('valid_pixels') ip_idx = 1 ip_count = len(processed_ips) if ip_count == 0: # create empty vpx_coverage file from osgeo import gdal, gdalconst im_reference = self.config.abs_path( self.config['geometry']['reference_image']) ids = gdal.Open(im_reference, gdalconst.GA_ReadOnly) iproj = ids.GetProjection() itrans = ids.GetGeoTransform() vpx_band = ids.GetRasterBand(1) for yr in years.keys(): out_file = self.get_output_file(yr) driver = gdal.GetDriverByName('GTiff') ods = driver.Create(out_file, vpx_band.XSize, vpx_band.YSize, eType=vpx_band.DataType) ods.SetGeoTransform(itrans) ods.SetProjection(iproj) ods = None self.tif2jpg(out_file) ids = None raise ProcessorFailedError(self, "No input valid layers found") for ip, platform_type, status in processed_ips: Logger.info("Processing {}... ({}/{})".format( ip, ip_idx, ip_count)) ip_idx += 1 # set current platform type self.platform_type = QCPlatformType(platform_type) if self.config['image_products'].get('{}_processing_level2'.format( self.get_platform_type())) == 'S2MSI2A': self.data_dir_suf = '.SAFE' else: self.data_dir_suf = '' # delete previous results if needed if status not in (DbIpOperationStatus.unchanged, DbIpOperationStatus.rejected): do_run = True if self.get_last_ip_status(ip, status) == DbIpOperationStatus.rejected: Logger.info("{} skipped - rejected".format(ip)) continue yr = self.get_ip_year(ip) data_dir = self.get_data_dir(ip) try: years[yr] += self.filter_files( data_dir, 'valid_pixels_{}m.tif'.format( self.config['land_product']['geometric_resolution'])) except KeyError: raise ProcessorFailedError( self, 'Inconsistency between years in metadata and years in the ' 'config file. Years from the config file are {}, but you ' 'are querying year {}'.format(years, yr)) vpx_files = {} for yr, input_files in years.items(): if len(input_files) < 1: Logger.warning( "No Vpx layers to be processed for {}".format(yr)) continue # define output file output_file = self.get_output_file(yr) vpx_files[yr] = output_file if os.path.exists(output_file): # run processor if output file does not exist continue status = DbIpOperationStatus.updated if os.path.exists(output_file) \ else DbIpOperationStatus.added Logger.info("Running countVpx for {}: {} layers".format( yr, len(input_files))) # run processor try: self.count_vpx(input_files, output_file) except ProcessorFailedError: pass # log processor IP operation if os.path.exists(output_file): timestamp = self.file_timestamp(output_file) else: timestamp = None # TBD ### self.lp_operation(status, timestamp=timestamp) return vpx_files def get_ip_year(self, ip): """Get image product filename. :param str ip: image product title :return dict: metadata """ meta_data = JsonIO.read( os.path.join(self.config['project']['path'], self.config['project']['metapath'], ip + ".geojson")) return meta_data['Sensing start'].year def count_vpx(self, input_files, output_file): """ Perform valid pixels coverage. 0: "noData", 1: "valid", 2: "clouds", 3: "shadows", 4: "snow", 5: "water" Raise ProcessorFailedError on failure. :param list input_files: list of input files :param str output_file: output filename """ import numpy as np from osgeo import gdal, gdalconst, gdal_array try: # open input data ids = gdal.Open(input_files[0], gdalconst.GA_ReadOnly) iproj = ids.GetProjection() itrans = ids.GetGeoTransform() vpx_band = ids.GetRasterBand(1) # open output data driver = gdal.GetDriverByName('GTiff') ods = driver.Create(output_file, vpx_band.XSize, vpx_band.YSize, eType=vpx_band.DataType) ods.SetGeoTransform(itrans) ods.SetProjection(iproj) # create countVpx array dtype = gdal_array.GDALTypeCodeToNumericTypeCode( ids.GetRasterBand(1).DataType) # dtype vs. no of images < 255 ? vpx_count = np.zeros((vpx_band.YSize, vpx_band.XSize), dtype=dtype) ids = None # count valid pixels for i in range(len(input_files)): ids = gdal.Open(input_files[i], gdalconst.GA_ReadOnly) vpx_band = ids.GetRasterBand(1) vpx_arr = vpx_band.ReadAsArray() vpx_count += vpx_arr # save count ods.GetRasterBand(1).WriteArray(vpx_count) # ods.GetRasterBand(1).SetNoDataValue(vpx_band.GetNoDataValue()) # set color table StyleReader(self.identifier).set_band_colors(ods) # close data sources & write out ids = None ods = None except RuntimeError as e: raise ProcessorFailedError( self, "Count Vpx processor failed: {}".format(e)) def compute_vpx_stats(self, vpx_file): """Compute stats for valid pixels coverage. :param str vpx_file: vpx file path :return dict: QI metadata """ from osgeo import gdal # compute min/max value, count, ncells = self.compute_value_count(vpx_file) vpx_pct = 0.0 for idx in range(len(value)): if value[idx] == 0: vpx_pct = count[idx] / ncells * 100 data = { "min": int(min(value)), "max": int(max(value)), "gapPct": round(vpx_pct, 4), "mask": self.file_basename(self.tif2jpg(vpx_file)) } ds = None return data def get_quality_indicators(self): """Get quality indicators. :return dict: QI metadata """ vpx_timestamp = datetime.datetime.now() years = {} vpx_timestamp = None value = False fitness = 'FULL' for yr, vpx_file in self.compute_coverage().items(): # take the last file if not vpx_timestamp: vpx_timestamp = self.file_timestamp(vpx_file) years[yr] = self.compute_vpx_stats(vpx_file) if value is False and years[yr]['max'] > 0: # at least one non-zero pixel value = True if fitness == 'FULL' and years[yr]['min'] == 0: # FULL -> PARTIAL: at least one zero pixel fitness = 'PARTIAL' if value is False: # no coverage fitness = 'NO' return { "value": value, "generatedAtTime": vpx_timestamp, "vpxCoverage": years, "fitnessForPurpose": fitness } def _run(self): """Run computation. :return dict: QI metadata """ response_data = { 'isMeasurementOf': '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), 'value': False } response_data.update(self.get_quality_indicators()) return response_data
class QCProcessorLPOrdinaryControl(QCProcessorLPBase): """Land Product ordinary control processor [validation control]. """ identifier = identifier_from_file(__file__) isMeasurementOf = "lpOrdinaryControlMetric" isMeasurementOfSection = "qualityIndicators" def check_dependency(self): """Check processor's dependencies.""" import numpy from osgeo import gdal def _run(self): """Perform processor's tasks. :return dict: QI metadata """ Logger.info('Running ordinary control') response_data = { 'isMeasurementOf': '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), 'value': False } prod_raster = self.config['land_product']['product_type'][ -1] + '_raster' lp_fn = os.path.join(self.config['map_product']['path'], self.config['map_product'][prod_raster]) lp_characteristics = self._lp_characteristics(lp_fn) response_data.update(lp_characteristics) if lp_characteristics['read'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data res = self.config['land_product']['geometric_resolution'] if lp_characteristics['xRes'] == res and lp_characteristics[ 'xRes'] == res: response_data['value'] = True else: response_data['value'] = False if str(lp_characteristics['epsg']) == str( self.config['land_product']['epsg']): response_data['value'] = True else: response_data['value'] = False if str(lp_characteristics['dataType']) == str( self.config['land_product']['data_type']): response_data['value'] = True else: response_data['value'] = False if str(lp_characteristics['rasterFormat']) == str( self.config['land_product']['delivery_format']): response_data['value'] = True else: response_data['value'] = False if response_data['value'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data def _calc_aoiCoveragePct(self, input_zone_polygon, input_value_raster, lp_min, lp_max, unclassifiable, out_of_aoi): """Calculate Land Product coverage statistics. :param str input_zone_polygon: input vector file with zones :param str input_value_raster: input raster value file :param int lp_min: LP min value :param float lp_max: LP max value :param int unclassifiable: unclassifiable value :param out_of_aoi: output AOI vector file """ raster = gdal.Open(input_value_raster) shp = ogr.Open(input_zone_polygon) lyr = shp.GetLayer() # Get raster georeference info transform = raster.GetGeoTransform() xOrigin = transform[0] yOrigin = transform[3] pixelWidth = transform[1] pixelHeight = transform[5] sourceSR = lyr.GetSpatialRef() # gdal 2.4.2 vs. gdal 3 in docker if int(osgeo.__version__[0]) >= 3: sourceSR.SetAxisMappingStrategy( osgeo.osr.OAMS_TRADITIONAL_GIS_ORDER) feat = lyr.GetNextFeature() geom = feat.GetGeometryRef() if (geom.GetGeometryName() == 'MULTIPOLYGON'): count = 0 pointsX = [] pointsY = [] for polygon in geom: geomInner = geom.GetGeometryRef(count) ring = geomInner.GetGeometryRef(0) numpoints = ring.GetPointCount() for p in range(numpoints): lon, lat, z = ring.GetPoint(p) pointsX.append(lon) pointsY.append(lat) count += 1 elif (geom.GetGeometryName() == 'POLYGON'): ring = geom.GetGeometryRef(0) numpoints = ring.GetPointCount() pointsX = [] pointsY = [] for p in range(numpoints): lon, lat, z = ring.GetPoint(p) pointsX.append(lon) pointsY.append(lat) else: sys.exit( "ERROR: Geometry needs to be either Polygon or Multipolygon") xmin = min(pointsX) xmax = max(pointsX) ymin = min(pointsY) ymax = max(pointsY) xoff = int((xmin - xOrigin) / pixelWidth) yoff = int((yOrigin - ymax) / pixelWidth) xcount = int((xmax - xmin) / pixelWidth) + 1 ycount = int((ymax - ymin) / pixelWidth) + 1 # Memory target raster target_ds = gdal.GetDriverByName('MEM').Create('', xcount, ycount, 1, gdal.GDT_Byte) target_ds.SetGeoTransform(( xmin, pixelWidth, 0, ymax, 0, pixelHeight, )) raster_srs = osr.SpatialReference() raster_srs.ImportFromWkt(raster.GetProjectionRef()) target_ds.SetProjection(raster_srs.ExportToWkt()) # Rasterize zone polygon to raster gdal.RasterizeLayer(target_ds, [1], lyr, burn_values=[1]) banddataraster = raster.GetRasterBand(1) dataraster = banddataraster.ReadAsArray(xoff, yoff, xcount, ycount).astype(np.float) bandmask = target_ds.GetRasterBand(1) datamask = bandmask.ReadAsArray(0, 0, xcount, ycount).astype(np.float) zoneraster = np.ma.masked_array(dataraster, np.logical_not(datamask)) # Calculate overlay statistics lp_maped_px = np.sum( ((zoneraster >= lp_min) & (zoneraster <= lp_max)) * 1) lp_out_px = np.sum( ((zoneraster == unclassifiable) & (zoneraster == out_of_aoi)) * 1) _aoiCoveragePct = lp_maped_px / (lp_maped_px + lp_out_px) * 100.0 return _aoiCoveragePct def _lp_characteristics(self, lp_fn): """Get LP characteristics to check. :param lp_fn: input lp raster file :return dict: lp characteristics """ import numpy as np from osgeo import gdal, gdal_array, osr from osgeo import gdalconst gdal.UseExceptions() lp_characteristics = {} try: ids = gdal.Open(lp_fn, gdalconst.GA_ReadOnly) lp_characteristics['read'] = True except RuntimeError: lp_characteristics['read'] = False return lp_characteristics # Spatial resolution ids = gdal.Open(lp_fn, gdalconst.GA_ReadOnly) img_array = ids.ReadAsArray() geotransform = list(ids.GetGeoTransform()) lp_characteristics['xRes'] = abs(geotransform[1]) lp_characteristics['yRes'] = abs(geotransform[5]) # Projection proj = osr.SpatialReference(wkt=ids.GetProjection()) map_epsg = (proj.GetAttrValue('AUTHORITY', 1)) lp_characteristics['epsg'] = map_epsg # Coding data type map_dtype = gdal_array.GDALTypeCodeToNumericTypeCode( ids.GetRasterBand(1).DataType) if (map_dtype == np.dtype('uint8')): lp_characteristics['dataType'] = 'u8' else: lp_characteristics['dataType'] = str(ids.GetRasterBand(1).DataType) # Map extent in the 'map_epsg' ulx, xres, xskew, uly, yskew, yres = ids.GetGeoTransform() lrx = ulx + (ids.RasterXSize * xres) lry = uly + (ids.RasterYSize * yres) lp_characteristics['extentUlLr'] = [ulx, uly, lrx, lry] # Do spatial overlay aoi_polygon = os.path.join(self.config['map_product']['path'], self.config['map_product']['map_aoi']) coding_val = [] for prod in self.config['land_product']['product_type']: for val in self.config['land_product']['raster_coding'][prod]: coding_val.append( self.config['land_product']['raster_coding'][prod][val]) lp_min = min(coding_val) lp_max = max(coding_val) unclassifiable = self.config['land_product']['raster_coding'][ 'unclassifiable'] out_of_aoi = self.config['land_product']['raster_coding']['out_of_aoi'] lp_characteristics['aoiCoveragePct'] = self._calc_aoiCoveragePct( aoi_polygon, lp_fn, lp_min, lp_max, unclassifiable, out_of_aoi) # Map format raster_format = lp_fn.split('.')[-1] if raster_format == 'tif': raster_format = "GeoTIFF" lp_characteristics['rasterFormat'] = raster_format return lp_characteristics
class QCProcessorLPThematicValidationControl(QCProcessorLPBase): """Land Product thematic validation control processor [validation control]. """ identifier = identifier_from_file(__file__) isMeasurementOf = "lpThematicValidationMetric" isMeasurementOfSection = "qualityIndicators" def check_dependency(self): """Check processor's software dependecies. """ import sklearn import scipy def _run(self): """Perform processor's tasks. :return dict: QI metadata """ Logger.info('Running thematic validation control') response_data = { 'isMeasurementOf': '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.now(), "value": False } reference_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['reference_layer']) if not os.path.isfile(reference_fn): Logger.error("File {} not found".format(reference_fn)) self.set_response_status(DbIpOperationStatus.rejected) return response_data themes = self.config['land_product']['product_type'] for theme in themes: if theme == 'classification': classification_qi_ = self._lp_classification_validation( self.config) response_data.update({"classification": classification_qi_}) if float(classification_qi_['overallAccuracy']) >= \ float(self.config['land_product']['thematic_accuracy']): response_data['value'] = True else: response_data['value'] = False elif theme == 'regression': regression_qi = self._lp_regression_validation(self.config) regression_prod_name = self.config['land_product'][ 'regression_name'] response_data.update({regression_prod_name: regression_qi}) print(float(regression_qi['rmse'])) print(float(self.config['land_product']['rmse_accuracy'])) if float(regression_qi['rmse']) <= \ float(self.config['land_product']['rmse_accuracy']): response_data['value'] = True else: response_data['value'] = False if response_data['value'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data def _read_point_data(self, ras_fn, vec_fn, attrib, no_data): """Raster map ~ vector reference spatial overlay. :param str ras_fn: input raster file :param str vec_fn: input vector file :param str attrib: attribute :param int no_data: no data value """ from osgeo import gdal, ogr src_ds = gdal.Open(ras_fn) gt = src_ds.GetGeoTransform() rb = src_ds.GetRasterBand(1) ds = ogr.Open(vec_fn) lyr = ds.GetLayer() ref_val = [] map_val = [] for feat in lyr: geom = feat.GetGeometryRef() if geom.GetGeometryName() == 'POLYGON': mx, my = geom.Centroid().GetX(), geom.Centroid().GetY() elif geom.GetGeometryName() == 'POINT': mx, my = geom.GetX(), geom.GetY() px = int((mx - gt[0]) / gt[1]) py = int((my - gt[3]) / gt[5]) intval = rb.ReadAsArray(px, py, 1, 1) if not ((intval[0][0] == no_data) or (feat.GetField(attrib) == no_data)): map_val.append(intval[0][0]) ref_val.append(feat.GetField(attrib)) return ref_val, map_val def _lp_classification_validation(self, config): """Extract classification thematic quality indicators. :param dict config: configuration :return dict: QI metadata """ import numpy as np from sklearn.metrics import classification_report from sklearn.metrics import confusion_matrix from sklearn.metrics import cohen_kappa_score classification_qi = {} lp_map_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['classification_raster']) reference_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['reference_layer']) reference_attrib = self.config['map_product'][ 'classification_attribute'] no_data = self.config['land_product']['raster_coding']['out_of_aoi'] ref_val_, map_val_ = self._read_point_data(lp_map_fn, reference_fn, reference_attrib, no_data) c_report = classification_report(ref_val_, map_val_, output_dict=True) c_matrix = confusion_matrix(ref_val_, map_val_) ref_lineage_ = reference_fn.split('/')[-1] overall_accuracy_ = (np.sum(c_matrix.diagonal()) / np.sum(c_matrix)) * 100 producers_accuracy_ = (c_report['weighted avg']['precision']) * 100 users_accuracy_ = (c_report['weighted avg']['recall']) * 100 kappa_ = (cohen_kappa_score( ref_val_, map_val_, labels=None, weights=None)) * 100 classes = config['land_product']['raster_coding']['classification'] classes_names = [ k for k, v in sorted(classes.items(), key=lambda item: item[1]) ] return { "codingClasses": classes_names, "lineage": "http://qcmms.esa.int/{}".format(ref_lineage_), "overallAccuracy": round(overall_accuracy_, 2), "producersAccuracy": round(producers_accuracy_, 2), "usersAccuracy": round(users_accuracy_, 2), "kappa": round(kappa_, 2), "confusionMatrix": c_matrix.tolist() } def _lp_regression_validation(self, config): """Extract regression thematic quality indicators. :param dict config: configuration :return dict: QI metadata """ import numpy as np from sklearn.metrics import mean_absolute_error from sklearn.metrics import mean_squared_error from scipy.stats import pearsonr regression_qi = {} lp_map_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['regression_raster']) reference_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['reference_layer']) reference_attrib = self.config['map_product']['regression_attribute'] no_data = self.config['land_product']['raster_coding']['out_of_aoi'] ref_val_, map_val_ = self._read_point_data(lp_map_fn, reference_fn, reference_attrib, no_data) MAE_ = mean_absolute_error(ref_val_, map_val_) MSE_ = mean_squared_error(ref_val_, map_val_) RMSE_ = np.sqrt(mean_squared_error(ref_val_, map_val_)) pearson_r_, p_val_ = pearsonr(ref_val_, map_val_) ref_lineage_ = reference_fn.split('/')[-1] regression_values = config['land_product']['raster_coding'][ 'regression'] return { "lineage": "http://qcmms.esa.int/{}".format(ref_lineage_), "codingValues": regression_values, "mae": round(MAE_, 2), "mse": round(MSE_, 2), "rmse": round(RMSE_, 2), "pearsonR": round(pearson_r_, 2) }
class QCProcessorLPInterpretationControl(QCProcessorLPBase): """Land Product interpretation control processor [interpretation control]. Be aware it is an optional processor! """ identifier = identifier_from_file(__file__) isMeasurementOf = "lpInterpretationMetric" isMeasurementOfSection = "qualityIndicators" def check_dependency(self): """Check processor's software dependencies. """ pass def _run(self): """Perform processor's tasks. :return dict: QI metadata """ Logger.info('Running interpretation control') response_data = { "isMeasurementOf": '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.now(), "value": False } lp_interpretation = self._read_interpretation_qi() if lp_interpretation is None: # no LP interpretation metadata available return {} for ltype in self.config['land_product']['product_type']: if ltype == 'classification': if lp_interpretation[ltype]['overallAccuracy'] >= \ self.config['land_product']['thematic_accuracy']: response_data['value'] = True else: response_data['value'] = False response_data[ltype] = lp_interpretation[ltype] elif ltype == 'regression': reg_name = self.config['land_product']['regression_name'] if 'rmse_accuracy' in self.config['land_product']: if lp_interpretation[reg_name]['rmse'] <= \ self.config['land_product']['rmse_accuracy']: response_data['value'] = True else: response_data['value'] = False response_data[reg_name] = lp_interpretation[reg_name] if response_data['value'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data def _read_interpretation_qi(self): """Read LP interpretation quality indicators. :return dict: quality indicators """ try: lp_interpretation_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['map_interpretation_qi']) except KeyError: Logger.info("{} is not defined".format('map_interpretation_qi')) return None if not os.path.isfile(lp_interpretation_fn): Logger.info("File {} not found".format(lp_interpretation_fn)) return None return JsonIO.read(lp_interpretation_fn)
class QCProcessorTemplateLP(QCProcessorLPBase): """Template land product processor. """ identifier = identifier_from_file(__file__) isMeasurementOf = "lpInterpretationMetric" def run(self): """Run processor. Define this functions only if your processor is the first in a queue. Check processors.lp_init for a real example. """ self.add_response(self._run()) def _run(self): """Perform processor's tasks. Check processors.lp_ordinary_control for a real example. :param meta_file: path to JSON metafile :param str data_dir: path to data directory :param str output_dir: path to output processor directory :return dict: QI metadata """ response_data = { "type": "Feature", "id": "http://qcmms-cat.spacebel.be/eo-catalog/series/EOP:MAPRADIX:LP_TUC1/datasets/IMD_2018_010m", "geometry": { "type": "Polygon", "coordinates": [[[13.58808613662857, 50.54373233188457], [13.616762732856103, 49.55648784343155], [15.134970502622016, 49.56467629785398], [15.137769755767613, 50.552210622261306], [13.58808613662857, 50.54373233188457]]] }, "properties": { "title": "IMD_2018_010m", "identifier": "IMD_2018_010m", "status": "PLANNED", "kind": "http://purl.org/dc/dcmitype/Dataset", "parentIdentifier": "EOP:ESA:LP:TUC1", "collection": "EOP:ESA:GR1:UC1", "abstract": "The high-resolution imperviousness product capture the percentage of soil sealing. Built-up areas are characterized by the substitution of the original (semi-) natural land cover or water surface with an artificial, often impervious cover. This product of imperviousness layer constitutes the main status layer. There is per-pixel estimates of impermeable cover of soil (soil sealing) and are mapped as the degree of imperviousness (0-100%). Imperviousness 2018 is the continuation of the existing HRL imperviousness status product for the 2018 reference year, but with an increase in spatial resolution from 20m to (now) 10m.", "date": "2020-05-12T00:00:00Z/2020-05-12T23:59:59Z", "categories": [{ "term": "https://earth.esa.int/concept/urban", "label": "Forestry" }, { "term": "http://www.eionet.europa.eu/gemet/concept/4599", "label": "land" }, { "term": "https://earth.esa.int/concept/sentinel-2", "label": "Sentinel-2" }], "updated": "2020-05-12T15:23:57Z", "qualifiedAttribution": [{ "type": "Attribution", "agent": [{ "type": "Organization", "email": "*****@*****.**", "name": "ESA/ESRIN", "phone": "tel:+39 06 94180777", "uri": "http://www.earth.esa.int", "hasAddress": { "country-name": "Italy", "postal-code": "00044", "locality": "Frascati", "street-address": "Via Galileo Galilei CP. 64" } }], "role": "originator" }], "acquisitionInformation": [{ "platform": { "id": "https://earth.esa.int/concept/sentinel-2", "platformShortName": "Sentinel-2" }, "instrument": { "id": "https://earth.esa.int/concept/s2-msi", "instrumentShortName": "MSI" } }], "productInformation": { "productType": "classification", "availabilityTime": "2019-06-20T15:23:57Z", "format": "geoTIFF", "referenceSystemIdentifier": "http://www.opengis.net/def/crs/EPSG/0/3035", "qualityInformation": { "qualityIndicators": [{ "isMeasurementOf": "http://qcmms.esa.int/quality-indicators/#lpInterpretationMetric", "generatedAtTime": "2019-10-17T17:20:32.30Z", "value": True, "classification": { "codingClasses": ["non-urban", "urban"], "overallAccuracy": 91.0, "producersAccuracy": 99.0, "usersAccuracy": 92.0, "kappa": 88.0, "confusionMatrix": [[92, 8], [1, 99]], "lineage": "http://qcmms.esa.int/Mx_sealing_simple_v0.9" }, "densityCover": { "mae": 10.25, "mse": 174.86, "rmse": 13.22, "pearsonR": 0.69, "lineage": "http://qcmms.esa.int/Mx_sealing_simple_v0.9" } }, { "isMeasurementOf": "http://qcmms.esa.int/quality-indicators/#ipForLpInformationMetric", "value": True, "generatedAtTime": "2019-10-17T17:20:32.30Z", "vpxCoverage": { "2018": { "min": 3, "max": 22, "gapPct": 0.0, "mask": "http://93.91.57.111/UC1/image_products/vpx_coverage_10m.jpg" } }, "fitnessForPurpose": "PARTIAL", "lineage": "http://qcmms.esa.int/Mx_vpx_v0.9", "generatedAtTime": "2020-05-12T17:20:32.30Z" }, { "isMeasurementOf": "http://qcmms.esa.int/quality-indicators/#lpMetadataControlMetric", "value": True, "generatedAtTime": "2020-05-12T17:20:32.30Z", "metadataAvailable": True, "metadataSpecification": "INSPIRE", "metadataCompliancy": True }, { "isMeasurementOf": "http://qcmms.esa.int/quality-indicators/#lpOrdinaryControlMetric", "value": True, "generatedAtTime": "2019-10-17T17:20:32.30Z", "read": True, "xRes": 10.0, "yRes": 10.0, "epsg": "3035", "dataType": "u8", "rasterFormat": "GeoTIFF", "extentUlLr": [4621500.0, 3019160.0, 4659740.0, 2988580.0], "aoiCoveragePct": 100.0 }, { "isMeasurementOf": "http://qcmms.esa.int/quality-indicators/#lpThematicValidationMetric", "value": True, "generatedAtTime": "2020-05-12T17:20:32.30Z", "classification": { "codingClasses": ["non-urban", "urban"], "lineage": "http://qcmms.esa.int/prague_sealing_references_330.shp", "overallAccuracy": 91.2, "producersAccuracy": 90.4, "usersAccuracy": 93.0, "kappa": 88.3, "confusionMatrix": [[92, 8], [1, 99]] }, "densityCover": { "mae": 10.2, "mse": 120.3, "rmse": 11.0, "pearsonR": 0.71, "lineage": "http://qcmms.esa.int/prague_sealing_references_330.shp", "codingValues": { "non-sealing": 0, "sealingMin": 1, "sealingMax": 100, "unclassified": 254, "outsideAoi": 254 } } }] } }, "additionalAttributes": { "product_focus": "classification", "lpReference": "EN-EEA.IDM.R0.18.009_Annex_8 Table 11", "temporal_coverage": "status", "geometric_resolution": 10, "grid": "EEA Reference Grid", "crs": "European ETRS89 LAEA", "geometric_accuracy": 0.5, "thematic_accuracy": 90, "data_type": "u8", "mmu_pixels": 1, "necessary_attributes": ["raster value", "count", "class names"], "raster_coding": [{ "name": "non-impervious", "min": 0, "max": 0 }, { "name": "impervious", "min": 1, "max": 100 }, { "name": "unclassified", "min": 254, "max": 254 }, { "name": "outside area", "min": 255, "max": 255 }], "seasonal_window": [5, 6, 7, 8] }, "links": { "previews": [{ "href": "https://qcmms-cat.spacebel.be/archive/lp/land_tuc1.png", "type": "image/png", "title": "Quicklook" }], "via": [{ "href": "http://qcmms-cat.spacebel.be/eo-catalog/series/EOP:ESA:GR1:UC1/datasets", "type": "application/geo+json", "title": "Input data" }] } } } return response_data