def save_response(self, processor=None):
"""Produce manager response. Save QI metadata on disk.
:param QCProcesor processor: processor name for incremental response
:return str: target path
"""
response_content = None
for response in self.response:
# render IP response into string
response_content = self._response_composer.render(
response
)
# save response content to JSON file (incremental)
if processor:
self._response_composer.save(
response_content,
self._response_composer.get_filename(response, processor.identifier)
)
# save response content to JSON file (final)
self._response_composer.save(
response_content,
self._response_composer.get_filename(response)
)
if not response_content:
Logger.warning("No response content")
return self._response_composer.target_dir
def _delivery_control(self, filename, expected_filesize=None):
"""Performs delivery control.
* check if download file exists
* check file size when expected value given
Raise QCProcessorDownloadError on failure
:param str filename: filepath to check
:param int expected_filesize: expected file size in bytes or None
"""
# check if file exists
if not os.path.exists(filename):
raise QCProcessorDownloadError(
"File {} doesn't exist".format(filename
))
if expected_filesize:
# check expected filesize if given
filesize = os.path.getsize(filename)
if filesize != expected_filesize:
raise QCProcessorDownloadError(
"File {} size ({}) differs from expected value ({})".format(
filename, filesize, expected_filesize
))
Logger.debug("File {} expected filesize check passed".format(
filename
))
def set_processors(self, processors=[]):
"""Set list of processors to be performed.
:param list processors: list of processors to be registered
(if none than configuration will be used)
"""
if not processors:
try:
processors = self.config['processors']
except KeyError:
raise ConfigError(self._config_files,
"list of processors not defined"
)
else:
# override configuration
self.config['processors'] = processors
if not processors:
return
for identifier in processors:
found = False
for processor in processors_list:
if processor.identifier == identifier:
Logger.debug("'{}' processor registered".format(
processor.identifier
))
self._processors.append(
processor(self.config, self.response)
)
found = True
break
if not found:
self.config.processor_not_found(identifier)
def test_tc_034a(self):
"""Check availability of pixel-level multi-sensor metadata.
This test case consists to check that the QC Manager checks
availability of pixel level multi-sensor metadata.
"""
from processors.cloud_coverage import QCProcessorCloudCoverage
processor_cc = QCProcessorCloudCoverage(self._manager.config,
self._manager.response)
processor_cc.run()
assert processor_cc.get_response_status() != DbIpOperationStatus.failed
from processors.geometry_quality import QCProcessorGeometryQuality
processor_cc = QCProcessorGeometryQuality(self._manager.config,
self._manager.response)
processor_cc.run()
assert processor_cc.get_response_status() != DbIpOperationStatus.failed
# run harmonization control
from processors.harmonization_control import QCProcessorHarmonizationControl
processor = QCProcessorHarmonizationControl(self._manager.config,
self._manager.response)
processor.run()
self.do_009_034a()
Logger.info("Checking availability of pixel-level "
"multi-sensor metadata")
def test_tc_026a(pytestconfig):
"""Run one individual QC Manager processor
This test case consists to check that the QC Manager runs individual
QC Manager processor.
"""
from bin import run_manager
from manager import QCManager
QCManager(
config_file_all,
cleanup=-1
)
with open(config_files[3]) as config_yaml:
parsed_config = yaml.load(config_yaml, Loader=yaml.FullLoader)
log_dir_rel = parsed_config['logging']['directory']
log_dir = os.path.join(pytestconfig.rootdir, log_dir_rel)
for test_id in range(1, 7):
assert not os.path.isdir(log_dir), \
'Logs not cleaned up - no way to check if the next ' \
'processor works or not'
ip_config_file = os.path.join(
os.path.dirname(__file__), '..', 'tests',
'manager_tests_configs', 'test_{}.yaml'.format(test_id))
test_config_files = config_files + [ip_config_file]
run_manager.main(test_config_files)
assert len(os.listdir(log_dir)) > 1, \
'No logs created for config test_{}.yaml'.format(test_id)
QCManager(
config_file_all,
cleanup=-1
)
Logger.info("Running individual QC Manager processor")
def file_basename(self, filepath):
"""Return file basename.
:param str filepath: path to the file
:return str: file basename
"""
base_path = os.path.abspath(
filepath)[len(os.path.abspath(self.config['project']['path'])) +
1:]
tuc_name = self.config['catalog']['ip_parent_identifier'].split(
':')[-1]
# determine URL for catalog
url = base_path
if self.config.has_section('catalog') and \
self.config['catalog'].get('response_url'):
url = '{}/{}/{}'.format(
self.config['catalog']['response_url'].rstrip('/'), tuc_name,
base_path)
# copy file to www directory if defined
www_dir = self.config['catalog'].get('www_dir')
if www_dir:
target = os.path.join(www_dir, tuc_name, base_path.lstrip('/'))
target_dir = os.path.dirname(target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
shutil.copyfile(
filepath,
target,
)
Logger.debug("File {} copied to {}".format(filepath, target))
return url
def test_tc_010a(self):
"""Read detailed metadata.
This test case consists to check that the QC Manager can read
the detailed metadata.
"""
self.check_responses('detailedControlMetric',
('geometry', 0, 'rmseX'),
self.check_value_type(float))
self.check_responses('detailedControlMetric',
('geometry', 0, 'rmseY'),
self.check_value_type(float))
self.check_responses('detailedControlMetric',
('geometry', 0, 'diffXmax'),
self.check_value_type(float))
self.check_responses('detailedControlMetric',
('geometry', 0, 'diffYmax'),
self.check_value_type(float))
self.check_responses('detailedControlMetric',
('geometry', 0, 'medianAbsShift'),
self.check_value_type(float))
self.check_responses('detailedControlMetric',
('geometry', 0, 'validGCPs'),
self.check_value_type(int))
Logger.info("Reading detailed metadata")
def check_bands(self, dirname, level):
"""Check raster bands.
:param str dirname: image product directory.
:param int level: level to be checked (1, 2)
:return tuple: image filenames, bands
"""
img_files_all = self.filter_files(dirname,
extension=self.img_extension)
band_res = []
img_files = []
bands = self.get_bands(level)
for band in bands:
pattern = r'.*_{}.*{}'.format(band, self.img_extension)
found = False
for item in img_files_all:
if not re.search(pattern, item):
continue
found = True
Logger.debug("File {} found: {}".format(pattern, item))
# B10 or B10_10m, ...
band_res.append({'id': item[item.find(band):].split('.')[0]})
img_files.append(item)
if not found:
raise ProcessorRejectedError(
self, "{} not found in {}".format(pattern, dirname))
return img_files, band_res
def _run(self, meta_data, data_dir, output_dir):
"""Perform processor tasks.
:param meta_data: IP metadata
:param str data_dir: path to data directory
:param str output_dir: path to output processor directory
:return dict: QI metadata
"""
response_data = {
'isMeasurementOf':
'{}/#{}'.format(self._measurement_prefix, self.isMeasurementOf),
"generatedAtTime":
datetime.now(),
'value':
False
}
# process primary product type
response_data.update(
self._ordinary_control(self._get_file_path(meta_data['title'])))
# process level2 product type if defined
level2_product = self.get_processing_level2(meta_data)
if level2_product:
response_data.update(
self._ordinary_control(self._get_file_path(
level2_product['title']),
level=2))
else:
Logger.error("Level2 product not found for {}".format(
meta_data['title']))
response_data['value'] = False
return response_data
def get_last_response(self, identifier):
"""Get response from previous job.
:param str identifier: IP identifier
"""
data = super(QCProcessorSearchBase, self).get_last_response(identifier,
full=True)
try:
qi = data['properties']['productInformation']\
['qualityInformation']['qualityIndicators']
except TypeError:
Logger.debug("Broken previous job. Creating new response.")
return None
# search for feasibilityControlMetric
idx = 0
for item in qi:
if item["isMeasurementOf"].endswith('feasibilityControlMetric'):
break
idx += 1
# remove deliveryControlMetric, ...
data['properties']['productInformation']\
['qualityInformation']['qualityIndicators'] = qi[:idx+1]
return data
def create_stack(self, data_dir, output_dir, stack_name):
"""Create stack of all bands.
:param data_dir: directory with the Sentinel scene
:param output_dir: path to a directory where the stack will be saved
:param stack_name: stack filename
"""
import rasterio
paths_resampled = self._resample_bands(data_dir)
with rasterio.open(paths_resampled[0]) as band1:
meta = band1.meta
if meta['driver'] != 'GTiff':
meta['driver'] = 'GTiff'
stack_length = self.get_stack_length()
meta.update(count=stack_length)
stack_path = os.path.join(output_dir, stack_name)
Logger.debug("Creating stack {} from {} bands...".format(
stack_path, len(paths_resampled)))
with rasterio.open(stack_path, 'w', **meta) as stack:
stack.write(band1.read(1), 1)
for band_id, band in enumerate(paths_resampled[1:], start=2):
with rasterio.open(band) as b:
x = b.read(1)
stack.write(x, band_id)
# resampled single band not needed anymore
os.remove(band)
# delete also the first band
os.remove(paths_resampled[0])
def __init__(self, username, password, archive, backup_archive=None):
"""Connect API.
Raise ProcessorFailedError on failure
"""
from sentinelsat.sentinel import SentinelAPI, SentinelAPIError
# remember settings for query()
self.archive = archive
self.backup_archive = backup_archive
# connect API
try:
self.api = SentinelAPI(username, password, archive)
except (SentinelAPIError, ConnectionError) as e:
self.api = None
if backup_archive:
# re-try with backup archive
Logger.error(
"Unable to connect {} ({}). Re-trying with {}...".format(
archive, e, backup_archive))
try:
self.api = SentinelAPI(username, password, backup_archive)
except (SentinelAPIError, ConnectionError) as e:
self.api = None
if self.api is None:
raise ProcessorFailedError(self,
"Unable to connect: {}".format(e),
set_status=False)
Logger.debug("Sentinel API connected")
def get_response_data(self, data, extra_data={}):
# select for delivery control?
qi_failed = []
for attr in ('Format correctness', 'General quality',
'Geometric quality', 'Radiometric quality',
'Sensor quality'):
if data[attr] != 'PASSED':
qi_failed.append(attr)
selected_for_delivery_control = len(qi_failed) < 1
if qi_failed:
# log reason why it's failing
Logger.info("Rejected because of {}".format(','.join(qi_failed)))
extra_data['bbox'] = wkt2bbox(data['footprint'])
extra_data['geometry'] = json.loads(wkt2json(data['footprint']))
extra_data['qualityDegradation'] = max(
float(data['Degraded MSI data percentage']),
float(data['Degraded ancillary data percentage']))
extra_data['processingLevel'] = data['Processing level'].split('-')[1]
extra_data['size'] = int(float(data['Size'].split(' ')[0]) * 1000)
extra_data['formatCorrectnessMetric'] = data[
'Format correctness'] == 'PASSED'
extra_data['generalQualityMetric'] = data[
'General quality'] == 'PASSED'
extra_data['geometricQualityMetric'] = data[
'Geometric quality'] == 'PASSED'
extra_data['radiometricQualityMetric'] = data[
'Radiometric quality'] == 'PASSED'
extra_data['sensorQualityMetric'] = data['Sensor quality'] == 'PASSED'
return selected_for_delivery_control, \
super(QCProcessorSearchSentinel, self).get_response_data(
data, extra_data
)
def _get_qi_results_path(self, ip):
"""Get IP specific QI results path.
:param str ip: image product
:return str: output path
"""
# no output path defined, assuming QI results
output_path = os.path.join(
self.config['project']['path'],
self.config['project']['downpath'],
ip + self.data_dir_suf,
)
if not os.path.exists(output_path):
# no output directory defined
Logger.debug("Output path {} does not exist".format(output_path))
return None
dirs = os.listdir(output_path)
if 'GRANULE' in dirs:
# only one directory is expected here (sentinel-2)
dirs = os.listdir(os.path.join(output_path, 'GRANULE'))
if len(dirs) != 1:
raise ProcessorCriticalError(
"Unexpected number of data sub-directories")
return os.path.join(output_path, 'GRANULE', dirs[0], 'QI_DATA',
'QCMMS')
return os.path.join(output_path, 'QI_DATA', 'QCMMS')
def set_identifier(self, identifier):
"""Set processor identifier.
:param str: processor identifier
"""
self.identifier = identifier
Logger.info("QCProcessor{} config started".format(
self.identifier.capitalize()))
def test_tc_004a(self):
"""Identify delivered IP
This test case consists to check that the QC Manager identifies all
delivered (locally downloaded) image products for further processing.
"""
self.do_004a_033a()
Logger.info("Identifying delivered IP")
def test_tc_011(self):
"""Read pixel metadata lineage
This test case consists to check that the QC Manager can read the
metadata lineage from image products.
"""
self.do_011_034b()
Logger.info("Reading pixel metadata lineage")
def test_tc_004b(self):
"""Compare delivery IP with expected
This test case consists to check that the downloaded IP validated the MD5 checksum.
"""
self.do_004b_033a()
Logger.info("Comparing delivery IP with expected")
def test_tc_037a(self):
"""Compare temporal coverage with specification.
This test case consists to check that the QC Manager compares the
temporal coverage with defined requirements in LPST.
"""
self.do_013a_037a()
Logger.info("Comparing temporal coverage with specification")
def test_tc_037b(self):
"""Returning temporal coverage statistics.
This test case consists to check that the QC Manager creates temporal
coverage comparison statistics.
"""
self.do_013b_037b()
Logger.info("Returning temporal coverage statistics")
def test_tc_034b(self):
"""Check lineage of pixel-level multi-sensor metadata.
This test case consists to check that the QC Manager checks lineage
of the pixel-level multi-sensor metadata.
"""
self.do_011_034b()
Logger.info("Checking lineage of pixel-level multi-sensor metadata")
def __del__(self):
if not hasattr(self, "api"):
return
from landsatxplore.exceptions import EarthExplorerError
try:
self.api.logout()
except EarthExplorerError as e:
Logger.error("Landsat server is down. {}".format(e))
def test_tc_031(self):
"""Get metadata of all multi-sensor acquired IP.
This test case consists to check that the QC Manager is getting
metadata of the defined multi-sensor IP.
"""
self.do_002a_031()
Logger.info("Getting metadata of all multi-sensor acquired IP")
def test_tc_041b(self):
"""Check consistency of the time-series LP metadata.
This test case consists to check that the QC Manager checks
consistency of the time-series LP metadata.
"""
self.do_022b_041b()
Logger.info('Checking consistency of the time-series LP metadata')
def test_036(self):
"""Create temporal coverage.
This test case consists to check that the QC Manager creates temporal
coverage layers based on identified set of input IP and defined time
step criteria.
"""
self.do_012_036()
Logger.info("Creating temporal coverage")
def test_tc_026b():
"""Run the full processors stack.
This test case consists to check that the QC Manager runs the set of
QC Manager processors in correct order.
"""
from bin import run_manager
run_manager.main(config_file_all)
Logger.info("Running set of QC Manager processors")
def test_tc_001(self):
"""Read LPST parameters
This test case consists to check that the QC Manager reads the LPST
correctly.
"""
self.do_001_030()
Logger.info("Reading LPST parameters")
def test_tc_024b(self):
"""Compare LP thematic accuracy with reference.
This test case consists to check that the QC Manager compares
the resulting Land Product with the reference data set.
"""
self.do_024b_042()
Logger.info("Comparing LP thematic accuracy with reference")
def test_tc_013a(self):
"""Test if the fitnessForPurpose is specified.
This test case consists to check that the QC Manager compares
the created spatial coverage layer with requirements defined in
the LPST.
"""
self.do_013a_037a()
Logger.info("Comparing spatial coverage with specification")
def test_tc_012(self):
"""Create raster spatial layer
This test case consists to check that the QC Manager creates spatial
coverage layer based on selected set of quality raster metadata and map
algebra definition.
"""
self.do_012_036()
Logger.info("Creating raster spatial layer")