def _run(self, meta_data, data_dir, output_dir): """Perform processor tasks. :param meta_data: IP metadata :param str data_dir: path to data directory :param str output_dir: path to output processor directory :return dict: QI metadata """ response_data = { 'isMeasurementOf': '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.now(), 'value': False } # process primary product type response_data.update( self._ordinary_control(self._get_file_path(meta_data['title']))) # process level2 product type if defined level2_product = self.get_processing_level2(meta_data) if level2_product: response_data.update( self._ordinary_control(self._get_file_path( level2_product['title']), level=2)) else: Logger.error("Level2 product not found for {}".format( meta_data['title'])) response_data['value'] = False return response_data
def __init__(self, username, password, archive, backup_archive=None): """Connect API. Raise ProcessorFailedError on failure """ from sentinelsat.sentinel import SentinelAPI, SentinelAPIError # remember settings for query() self.archive = archive self.backup_archive = backup_archive # connect API try: self.api = SentinelAPI(username, password, archive) except (SentinelAPIError, ConnectionError) as e: self.api = None if backup_archive: # re-try with backup archive Logger.error( "Unable to connect {} ({}). Re-trying with {}...".format( archive, e, backup_archive)) try: self.api = SentinelAPI(username, password, backup_archive) except (SentinelAPIError, ConnectionError) as e: self.api = None if self.api is None: raise ProcessorFailedError(self, "Unable to connect: {}".format(e), set_status=False) Logger.debug("Sentinel API connected")
def _run(self, meta_data, data_dir, output_dir): """Perform processor tasks. :param meta_data: IP metadata :param str data_dir: path to data directory :param str output_dir: path to output processor directory :return dict: QI metadata """ response_data = {} # reference image must be defined try: im_reference = self.config.abs_path( self.config['geometry']['reference_image']) except KeyError: Logger.error("Reference image not defined") self.set_response_status(DbIpOperationStatus.failed) return response_data if not os.path.exists(im_reference): Logger.error("Reference image '{}' not found".format(im_reference)) self.set_response_status(DbIpOperationStatus.failed) return response_data # check if stack is already available output_dir = self._get_ip_output_path(meta_data['title']) if os.path.exists(output_dir): files = self.filter_files(output_dir, extension='.tif', pattern='stack_*') if len(files) > 0: if all([os.stat(f).st_size > 0 for f in files]): Logger.debug('Stack ({}) already available, no operation ' 'done'.format(output_dir)) response_data.update( self._run_stack_ordinary_control( meta_data, data_dir, output_dir, response_data)) return response_data Logger.debug('Stack ({}) already available, but has size 0 B. ' 'A new one will be created.'.format(output_dir)) # compute stack if not os.path.exists(output_dir): os.mkdir(output_dir) try: lh_title = os.path.split(output_dir)[-1] stack_name = self.get_stack_name(lh_title) self.create_stack(data_dir, output_dir, stack_name) response_data.update( self._run_stack_ordinary_control(meta_data, data_dir, output_dir, response_data)) except ProcessorRejectedError: self.set_response_status(DbIpOperationStatus.rejected) except ProcessorFailedError: self.set_response_status(DbIpOperationStatus.failed) return response_data
def __del__(self): if not hasattr(self, "api"): return from landsatxplore.exceptions import EarthExplorerError try: self.api.logout() except EarthExplorerError as e: Logger.error("Landsat server is down. {}".format(e))
def _run(self): """Perform processor's tasks. :return dict: QI metadata """ Logger.info('Running thematic validation control') response_data = { 'isMeasurementOf': '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.now(), "value": False } reference_fn = os.path.join( self.config['map_product']['path'], self.config['map_product']['reference_layer']) if not os.path.isfile(reference_fn): Logger.error("File {} not found".format(reference_fn)) self.set_response_status(DbIpOperationStatus.rejected) return response_data themes = self.config['land_product']['product_type'] for theme in themes: if theme == 'classification': classification_qi_ = self._lp_classification_validation( self.config) response_data.update({"classification": classification_qi_}) if float(classification_qi_['overallAccuracy']) >= \ float(self.config['land_product']['thematic_accuracy']): response_data['value'] = True else: response_data['value'] = False elif theme == 'regression': regression_qi = self._lp_regression_validation(self.config) regression_prod_name = self.config['land_product'][ 'regression_name'] response_data.update({regression_prod_name: regression_qi}) print(float(regression_qi['rmse'])) print(float(self.config['land_product']['rmse_accuracy'])) if float(regression_qi['rmse']) <= \ float(self.config['land_product']['rmse_accuracy']): response_data['value'] = True else: response_data['value'] = False if response_data['value'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data
def _run(self, meta_data, data_dir, output_dir): """Perform processor tasks. :param meta_data: IP metadata :param str data_dir: path to data directory :param str output_dir: path to output processor directory :return dict: QI metadata """ self.connector = self.connect() response_data = { 'isMeasurementOf': '{}/#{}'.format( self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.now(), 'status': 'IN_PROGRESS' } try: self._download_file( meta_data['id'], meta_data['title'], output_dir ) downfile = os.path.join( output_dir, meta_data['title'] + self.extension ) response_data['status'] = 'FINISHED' response_data['complete'] = True response_data['date'] = datetime_format( self.file_timestamp(downfile) ) response_data['value'] = True # download level2 product level2_product = self.get_processing_level2(meta_data) if level2_product: self._download_file( level2_product['id'], level2_product['title'], output_dir ) except QCProcessorDownloadError as e: Logger.error("Unable to download {}: {}".format( meta_data['id'], e )) response_data['status'] = 'NOT_AVAILABLE' response_data['complete'] = False response_data['value'] = False self.set_response_status(DbIpOperationStatus.failed) return response_data
def download_file(self, uuid, output_dir): from sentinelsat.sentinel import SentinelAPIError, InvalidChecksumError try: self.api.download(uuid, output_dir, checksum=True) except (SentinelAPIError, ConnectionError, InvalidChecksumError) as e: # re-try with backup archive Logger.error( "Unable to access {} ({}). Re-trying with {}...".format( self.archive, e, self.backup_archive)) self.api.api_url = self.backup_archive try: self.api.download(uuid, output_dir, checksum=True) except (SentinelAPIError, ConnectionError, InvalidChecksumError) as e: raise QCProcessorDownloadError(e)
def is_valid(self, response_file): """Validate response QI metadata file. :param str response_file: response file to be validated :return bool: """ data = self.load(response_file) if data is None: return False try: validate(data, schema=self._schema) Logger.debug("JSON response {} is valid".format(response_file)) except ValidationError as e: Logger.error("File {} validation against schema failed: {}".format( response_file, e)) return False return True
def compute(self, data_dir, output_dir): """Compute cloud coverage :param str data_dir: input data directory :param str output_dir: output directory :return str: output file """ import fmask # compute mask in 20m resolution try: self.run_fmask(data_dir, self._result['qi.files']['fmask'], self.comp_res) except subprocess.CalledProcessError as e: Logger.error("fmask failed: {}".format(e)) self.set_response_status(DbIpOperationStatus.failed) return None return self._result['qi.files']['fmask']
def query(self, footprint, kwargs): """Query API. Raise ProcessorFailedError on failure :return: result """ from sentinelsat.sentinel import SentinelAPIError result = None try: result = self.api.query(footprint, **kwargs) except (SentinelAPIError, ConnectionError) as e: if self.backup_archive: # re-try with backup archive Logger.error( "Unable to access {}. Re-trying with {}...".format( self.archive, self.backup_archive)) self.api.api_url = self.backup_archive try: result = self.api.query(footprint, **kwargs) except (SentinelAPIError, ConnectionError) as e: pass # exception will be raised anyway raise ProcessorFailedError(self, "Unable to query API: {}".format(e), set_status=False) if kwargs['producttype'] == 'S2MSI1C' and self.filter_by_tiles: result_filtered = OrderedDict() for ip, items in result.items(): for tile in self.filter_by_tiles: if tile == items['tileid']: result_filtered[ip] = items break if ip not in result_filtered.keys(): Logger.info("IP {} skipped by tile filter".format(ip)) return result_filtered return result
def calibrate(self, l1_data_dir, l2_data_dir): """Create L2 products.""" manager_dir = os.getcwd() l1_product_dir = os.path.join(manager_dir, l1_data_dir) l2_product_dir = os.path.join(manager_dir, l2_data_dir) os.mkdir(l2_product_dir) try: self.run_calibration(l1_product_dir) except subprocess.CalledProcessError as e: Logger.error("Calibration failed: {}".format(e)) self.set_response_status(DbIpOperationStatus.failed) return {} self.transform_l2_to_tif(l1_product_dir, l2_product_dir) self.copy_kept_products(l1_product_dir, l2_product_dir) self.clean_l1_directory(l1_product_dir) return {}
def load(self, response_file): """Load QI metadata content from file. :param str response_file: input filename (JSON file) :return dict: file content or None on error """ try: with open(response_file) as fd: response_content = fd.read() except FileNotFoundError: Logger.critical("No response file found") return None try: return json.loads(response_content) except json.decoder.JSONDecodeError as e: Logger.error("File {} is not valid JSON file: {}".format( response_file, e)) return response_content
def _ordinary_control(self, filepath, level=1): """Perform ordinary control. :param str filepath: path to zip file :param int level: level number for response """ from osgeo import gdal, osr if level not in (1, 2): raise ProcessorCriticalError(self, "Unsupported level: {}".format(level)) response_data = {'value': False} # 1. unarchive product dirname = os.path.join( os.path.dirname(filepath), os.path.basename(filepath).rstrip(self.extension) + self.data_dir_suf) try: if not os.path.exists(filepath) and \ not os.path.exists(dirname): raise ProcessorRejectedError( self, "No input data: {}/{} not found".format(filepath, dirname)) if not os.path.exists(dirname): Logger.info("Unarchiving {}...".format(filepath)) dirname = self.unarchive(filepath, dirname) except ProcessorRejectedError: return response_data # 2. check if all bands are available try: img_files, band_res = self.check_bands(dirname, level) except ProcessorRejectedError: return response_data level_key = 'level{}'.format(level) response_data[level_key] = {} response_data[level_key]['rastersComplete'] = True response_data[level_key]['channels'] = len(img_files) response_data[level_key]['lineage'] = self.get_lineage_level(filepath) # 3. check if raster file is readable try: self.check_bands_read(img_files) except ProcessorRejectedError: return response_data Logger.info("All imagery files found and readable") response_data[level_key]['rastersRead'.format(level)] = True # 4. read raster characteristics try: epsg_res, format_res = self.check_bands_properties( img_files, band_res) self.check_epsg_tile_name(epsg_res, filepath) except ProcessorRejectedError: return response_data # update response (epgs, format, bands) response_data[level_key]['epsg'] = int(epsg_res) if format_res == 'JP2OpenJPEG': data_format = 'JPEG' elif format_res == 'GTiff': data_format = 'TIFF' else: data_format = '?' response_data[level_key]['format'] = data_format response_data[level_key]['bands'] = band_res # 5. check metadata metadata_read = self.check_metadata(dirname, level) response_data[level_key]['metadataRead'] = metadata_read[0] if self.has_calibration_metadata: response_data[level_key]['calibrationMetadata'] = metadata_read[1] else: response_data[level_key]['calibrationMetadata'] = True # 6. selected for next control qi_failed = [] for attr in ('rastersComplete', 'rastersRead', 'metadataRead', 'calibrationMetadata'): if not response_data[level_key][attr]: qi_failed.append(attr) response_data['value'] = len(qi_failed) < 1 if qi_failed: Logger.error("Rejected because of {}".format(','.join(qi_failed))) self.set_response_status(DbIpOperationStatus.rejected) return response_data
def _run(self): """Perform processor's tasks. :return dict: QI metadata """ from lxml import etree Logger.info('Running LP metadata control') response_data = { "isMeasurementOf": '{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf), "generatedAtTime": datetime.datetime.now(), "value": False, } try: lp_meta_fn = glob.glob( os.path.join( self.config['map_product']['path'], self.config['land_product']['product_metadata'] + '*.xml'))[0] except IndexError: self.set_response_status(DbIpOperationStatus.rejected) if 'product_metadata' in self.config['land_product']: response_data['metadataSpecification'] = self.config[ 'land_product']['product_metadata'] else: response_data['metadataSpecification'] = '' try: Parser = etree.HTMLParser() XMLDoc = etree.parse(open(lp_meta_fn, 'r'), Parser) Logger.info('Land Product metadata available') response_data['metadataAvailable'] = True # validate xml Elements = XMLDoc.xpath('//characterstring') i = 0 for Element in Elements: if (Element.text) is not None: i += 1 if i > 5: response_data[ 'metadataCompliancy'] = self.validate_xml_metadata( lp_meta_fn) except: Logger.error('Land Prodcut metadata not available') response_data['metadataAvailable'] = False response_data['metadataCompliancy'] = False if response_data['metadataAvailable'] == True and \ response_data['metadataCompliancy'] == True: response_data['value'] = True if response_data['value'] is False: self.set_response_status(DbIpOperationStatus.rejected) return response_data
def calibrate(self, l1_data_dir, l2_data_dir): """Create L2 products.""" # temp_dir needed because use_cases contain resolution in their names # and sen2cor is not capable of working with output_dir with # a resolution in the name if self.config.has_section('sen2cor'): sen2cor_2_8 = '02.08' in self.config['sen2cor']['path'] sen2cor_path = self.config['sen2cor']['path'] if sen2cor_path[-1] != os.sep: sen2cor_path += os.sep else: sen2cor_2_8 = True sen2cor_path = '' output_path = os.path.join(self.temp_dir, os.path.basename(l1_data_dir)) if sen2cor_2_8: # Sen2Cor 2.8 # https://forum.step.esa.int/t/sen2cor-2-8-fails-on-product-from-early-2016-bool-object-has-no-attribute-spacecraft-name/16046 sen2cor_string = '{}L2A_Process --resolution {} --cr_only ' \ '--output_dir {} {}'.format( sen2cor_path, self.config['land_product']['geometric_resolution'], output_path, os.path.abspath(l1_data_dir) ) else: # Sen2Cor 2.5.5 sen2cor_string = '{}{}L2A_Process' \ ' --resolution {}' \ ' --cr_only {}'.format( self.config['sen2cor']['path'], os.path.sep, self.config['land_product']['geometric_resolution'], os.path.abspath(l1_data_dir) ) Logger.debug("Running {}".format(sen2cor_string)) try: subprocess.run(sen2cor_string, shell=True, check=True) except subprocess.CalledProcessError as e: Logger.error("Calibration failed: {}".format(e)) self.set_response_status(DbIpOperationStatus.failed) return {} if sen2cor_2_8: # now move the L2 product from the temp dir to the right one sen2cor_dir = os.listdir(output_path)[0] source_path = os.path.join(output_path, sen2cor_dir) # do not use move() to avoid cross-link device error shutil.copytree(source_path, l2_data_dir) Logger.debug("Directory {} copied to {}".format( source_path, l2_data_dir)) Logger.debug("Image file test (source): {}".format( self.filter_files(source_path, extension=self.img_extension)[0])) Logger.debug("Image file test (target): {}".format( self.filter_files(l2_data_dir, extension=self.img_extension)[0])) # remove tmp output directory shutil.rmtree(output_path) Logger.debug("Directory {} removed".format(output_path)) return {}
def _run(self, meta_data, data_dir, output_dir): """Perform processor tasks. :param meta_data: IP metadata :param str data_dir: Path to data directory :param str output_dir: path to output processor directory :return dict: QI metadata """ import arosics response_data = { self.isMeasurementOfSection: [{ "id": "http://qcmms.esa.int/detailed_control#GEOMETRY", "generatedAtTime": datetime.now(), "requirement": False }] } # reference image must be defined try: im_reference = self.config.abs_path( self.config['geometry']['reference_image']) except KeyError: Logger.error("Reference image not defined") self.set_response_status(DbIpOperationStatus.failed) return if not os.path.exists(im_reference): Logger.error("Reference image '{}' not found".format(im_reference)) self.set_response_status(DbIpOperationStatus.failed) return lh_output_dir = self.get_lh_dir(data_dir) bands2stack = self.config['geometry'].get('band4match', 3) resolution = self.config['land_product'].get('geometric_resolution') correct_shifts = self.config['geometry'].get('correct_shifts', False) has_stack = 'harmonization_stack' in self.config['processors'] try: stack_on = self.config['geometry']['stack_on'] if stack_on is True and has_stack is False: Logger.warning("Harmonization stack not available") stack_on = False except KeyError: stack_on = has_stack try: if stack_on: im_target = self._get_target_stack(lh_output_dir) else: filepattern_suff = '_{}m{}'.format( resolution, self.img_extension) \ if 'L2A' in data_dir else '{}'.format(self.img_extension) filepattern = '{}{}'.format( self.bands2stack(bands2stack, data_dir), filepattern_suff) try: im_target = self.filter_files(data_dir, filepattern)[0] except IndexError: raise ProcessorCriticalError( self, "Pattern '{}' not found in {}".format( filepattern, data_dir)) Logger.debug("Target image: {}".format(im_target)) except ProcessorFailedError: self.set_response_status(DbIpOperationStatus.failed) return # check EPSG codes if self.getEpsgCode(im_target) != self.getEpsgCode(im_reference): Logger.error("Inconsistent reference and image EPSG codes") self.set_response_status(DbIpOperationStatus.rejected) return try: mask = tempfile.mktemp(prefix='mask_') self.create_mask(data_dir, mask, resolution) except ProcessorFailedError: self.set_response_status(DbIpOperationStatus.failed) return # arocics settings coreg_file_prefix = '' if correct_shifts is True: coreg_file_prefix += 'coreg_' if stack_on: coreg_file_prefix += 'stack_' aro_raster_path = os.path.join( lh_output_dir, '{}{}.tif'.format(coreg_file_prefix, os.path.split(lh_output_dir)[-1])) kwargs = { 'fmt_out': 'GTIFF', 'grid_res': 30, 'max_points': 1000, 'mask_baddata_tgt': mask, 'path_out': aro_raster_path, 'projectDir': lh_output_dir, 'nodata': self.get_nodata_values(), 'window_size': (128, 128) } if stack_on: kwargs['s_b4match'] = bands2stack # run arocics Logger.debug( "Running coreg for reference: {}; image: {} with args {}".format( im_reference, im_target, kwargs)) try: CRL = arosics.COREG_LOCAL(im_reference, im_target, **kwargs) except AssertionError as e: Logger.error("Coreg failed: {}".format(e)) self.set_response_status(DbIpOperationStatus.failed) return response_data try: CRL_points_table = CRL.tiepoint_grid.get_CoRegPoints_table() except ValueError: CRL_points_table = None try: self.to_PointFormat( CRL_points_table, self._result['qi.files']['gml_before_correction'], CRL.outFillVal, epsg=self.getEpsgCode(im_target)) except ProcessorRejectedError: return response_data if correct_shifts is True: CRL.correct_shifts() CRL_after_corr = arosics.COREG_LOCAL(im_reference, CRL.path_out, **kwargs) try: CRL_points_table_after_corr = \ CRL_after_corr.tiepoint_grid.get_CoRegPoints_table() except ValueError: CRL_points_table_after_corr = None try: self.to_PointFormat( CRL_points_table_after_corr, self._result['qi.files']['gml_after_correction'], CRL_after_corr.outFillVal, epsg=self.getEpsgCode(im_target)) except ProcessorRejectedError: Logger.debug( 'No GCP points found in the GML after correction. Copying ' 'the one before the correction as the one after.') from shutil import copy2 copy2(self._result['qi.files']['gml_before_correction'], self._result['qi.files']['gml_after_correction']) else: CRL_after_corr = None if self.visualize is True or self.save_visualization is True: self._visualize(CRL, CRL_after_corr, self.visualize, self.save_visualization, correct_shifts, output_dir, resolution, aro_raster_path, stack_on) # update response attributes pixel_metadata_coding_conf = self.config['pixel_metadata_coding'] response_data[self.isMeasurementOfSection][0].update({ 'mask': self.file_basename( self._result['qi.files']['gml_before_correction']), 'rasterCoding': pixel_metadata_coding_conf[self.identifier], 'lineage': 'http://qcmms.esa.int/Arosics_v{}'.format(arosics.__version__) }) try: stats = self.compute_stats(CRL_points_table, resolution, CRL.outFillVal) response_data[self.isMeasurementOfSection][0].update(stats) except ProcessorFailedError: return response_data os.remove(mask) return response_data
def __init__(self, processor, msg, set_status=True): Logger.error("{} processor rejected: {}".format( processor.__class__.__name__, msg)) if set_status: # set processor response status processor.set_response_status(DbIpOperationStatus.rejected)