def resample(job):
"""Resample the scene to some areas."""
defaults = {
"radius_of_influence": None,
"resampler": "nearest",
"reduce_data": True,
"cache_dir": None,
"mask_area": False,
"epsilon": 0.0
}
product_list = job['product_list']
conf = _get_plugin_conf(product_list, '/product_list', defaults)
job['resampled_scenes'] = {}
scn = job['scene']
for area in product_list['product_list']['areas']:
area_conf = _get_plugin_conf(product_list,
'/product_list/areas/' + str(area), conf)
LOG.info('Resampling to %s', str(area))
if area is None:
minarea = get_config_value(product_list,
'/product_list/areas/' + str(area),
'use_min_area')
maxarea = get_config_value(product_list,
'/product_list/areas/' + str(area),
'use_max_area')
if minarea is True:
job['resampled_scenes'][area] = scn.resample(
scn.min_area(), **area_conf)
elif maxarea is True:
job['resampled_scenes'][area] = scn.resample(
scn.max_area(), **area_conf)
else:
job['resampled_scenes'][area] = scn
else:
job['resampled_scenes'][area] = scn.resample(area, **area_conf)
def sza_check(job):
"""Remove products which are not valid for the current Sun zenith angle."""
scn_mda = _get_scene_metadata(job)
scn_mda.update(job['input_mda'])
start_time = scn_mda['start_time']
product_list = job['product_list']
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
products = list(
product_list['product_list']['areas'][area]['products'].keys())
for product in products:
prod_path = "/product_list/areas/%s/products/%s" % (area, product)
lon = get_config_value(product_list, prod_path, "sunzen_check_lon")
lat = get_config_value(product_list, prod_path, "sunzen_check_lat")
if lon is None or lat is None:
LOG.debug(
"No 'sunzen_check_lon' or 'sunzen_check_lat' configured, "
"can\'t check Sun elevation for %s / %s", area, product)
continue
sunzen = sun_zenith_angle(start_time, lon, lat)
LOG.debug("Sun zenith angle is %.2f degrees", sunzen)
# Check nighttime limit
limit = get_config_value(product_list, prod_path,
"sunzen_minimum_angle")
if limit is not None:
if sunzen < limit:
LOG.info(
"Sun zenith angle to small for nighttime "
"product '%s', product removed.", product)
dpath.util.delete(product_list, prod_path)
continue
# Check daytime limit
limit = get_config_value(product_list, prod_path,
"sunzen_maximum_angle")
if limit is not None:
if sunzen > limit:
LOG.info(
"Sun zenith angle too large for daytime "
"product '%s', product removed.", product)
dpath.util.delete(product_list, prod_path)
continue
if len(product_list['product_list']['areas'][area]['products']) == 0:
LOG.info("Removing empty area: %s", area)
dpath.util.delete(product_list, '/product_list/areas/%s' % area)
def resample(job):
"""Resample the scene to some areas."""
defaults = {
"radius_of_influence": None,
"resampler": "nearest",
"reduce_data": True,
"cache_dir": None,
"mask_area": False,
"epsilon": 0.0
}
product_list = job['product_list']
conf = _get_plugin_conf(product_list, '/product_list', defaults)
job['resampled_scenes'] = {}
scn = job['scene']
for area in product_list['product_list']['areas']:
area_conf = _get_plugin_conf(product_list,
'/product_list/areas/' + str(area), conf)
LOG.debug('Resampling to %s', str(area))
if area == 'None':
minarea = get_config_value(product_list,
'/product_list/areas/' + str(area),
'use_min_area')
maxarea = get_config_value(product_list,
'/product_list/areas/' + str(area),
'use_max_area')
native = conf.get('resampler') == 'native'
if minarea is True:
job['resampled_scenes'][area] = scn.resample(
scn.min_area(), **area_conf)
elif maxarea is True:
job['resampled_scenes'][area] = scn.resample(
scn.max_area(), **area_conf)
elif native:
job['resampled_scenes'][area] = scn.resample(
resampler='native')
else:
# The composites need to be created for the saving to work
if not set(scn.keys()).issuperset(scn.wishlist):
LOG.debug(
"Generating composites for 'null' area (satellite projection)."
)
scn.load(scn.wishlist, generate=True)
job['resampled_scenes'][area] = scn
else:
LOG.debug("area: %s, area_conf: %s", area, str(area_conf))
job['resampled_scenes'][area] = scn.resample(area, **area_conf)
def _get_plugin_conf(product_list, path, defaults):
conf = {}
for key in defaults:
conf[key] = get_config_value(product_list,
path,
key,
default=defaults.get(key))
return conf
def check_sunlight_coverage(job):
"""Remove products with too low daytime coverage.
This plugins looks for a parameter called `min_sunlight_coverage` in the
product list, expressed in % (so between 0 and 100). If the sunlit fraction
is less than configured, the affected products will be discarded.
"""
if get_twilight_poly is None:
LOG.error("Trollsched import failed, sunlight coverage calculation not possible")
LOG.info("Keeping all products")
return
scn_mda = job['scene'].attrs.copy()
scn_mda.update(job['input_mda'])
platform_name = scn_mda['platform_name']
start_time = scn_mda['start_time']
end_time = scn_mda['end_time']
sensor = scn_mda['sensor']
if isinstance(sensor, (list, tuple, set)):
sensor = list(sensor)[0]
LOG.warning("Possibly many sensors given, taking only one for "
"coverage calculations: %s", sensor)
product_list = job['product_list']
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
products = list(product_list['product_list']['areas'][area]['products'].keys())
for product in products:
try:
if isinstance(product, tuple):
prod = job['resampled_scenes'][area][product[0]]
else:
prod = job['resampled_scenes'][area][product]
except KeyError:
LOG.warning("No dataset %s for this scene and area %s", product, area)
continue
else:
area_def = prod.attrs['area']
prod_path = "/product_list/areas/%s/products/%s" % (area, product)
config = get_config_value(product_list, prod_path, "sunlight_coverage")
if config is None:
continue
min_day = config.get('min')
use_pass = config.get('check_pass', False)
if use_pass:
overpass = Pass(platform_name, start_time, end_time, instrument=sensor)
else:
overpass = None
if min_day is None:
continue
coverage = _get_sunlight_coverage(area_def, start_time, overpass)
product_list['product_list']['areas'][area]['area_sunlight_coverage_percent'] = coverage * 100
if coverage < (min_day / 100.0):
LOG.info("Not enough sunlight coverage in "
"product '%s', removed.", product)
dpath.util.delete(product_list, prod_path)
def covers(job):
"""Check area coverage.
Remove areas with too low coverage from the worklist.
"""
if Pass is None:
LOG.error("Trollsched import failed, coverage calculation not possible")
LOG.info("Keeping all areas")
return
col_area = job['product_list']['product_list'].get('coverage_by_collection_area', False)
if col_area and 'collection_area_id' in job['input_mda']:
if job['input_mda']['collection_area_id'] not in job['product_list']['product_list']['areas']:
raise AbortProcessing(
"Area collection ID '%s' does not match "
"production area(s) %s" % (job['input_mda']['collection_area_id'],
str(list(job['product_list']['product_list']))))
product_list = job['product_list'].copy()
scn_mda = job['scene'].attrs.copy()
scn_mda.update(job['input_mda'])
platform_name = scn_mda['platform_name']
start_time = scn_mda['start_time']
end_time = scn_mda['end_time']
sensor = scn_mda['sensor']
if isinstance(sensor, (list, tuple, set)):
sensor = list(sensor)[0]
LOG.warning("Possibly many sensors given, taking only one for "
"coverage calculations: %s", sensor)
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
area_path = "/product_list/areas/%s" % area
min_coverage = get_config_value(product_list,
area_path,
"min_coverage")
if not min_coverage:
LOG.debug("Minimum area coverage not given or set to zero "
"for area %s", area)
continue
cov = get_scene_coverage(platform_name, start_time, end_time,
sensor, area)
product_list['product_list']['areas'][area]['area_coverage_percent'] = cov
if cov < min_coverage:
LOG.info(
"Area coverage %.2f %% below threshold %.2f %%",
cov, min_coverage)
LOG.info("Removing area %s from the worklist", area)
dpath.util.delete(product_list, area_path)
else:
LOG.debug("Area coverage %.2f %% above threshold %.2f %% - Carry on",
cov, min_coverage)
job['product_list'] = product_list
def check_platform(job):
"""Check if the platform is valid. If not, discard the scene."""
mda = job['input_mda']
product_list = job['product_list']
conf = get_config_value(product_list, '/product_list', 'processed_platforms')
if conf is None:
return
platform = mda['platform_name']
if platform not in conf:
raise AbortProcessing(
"'%s' not in list of allowed platforms" % platform)
def metadata_alias(job):
"""Replace input metadata values with aliases."""
mda_out = job['input_mda'].copy()
product_list = job['product_list']
aliases = get_config_value(product_list, '/product_list', 'metadata_aliases')
if aliases is None:
return
for key in aliases:
if key in mda_out:
val = mda_out[key]
if isinstance(val, (list, tuple, set)):
typ = type(val)
new_vals = typ([aliases[key].get(itm, itm) for itm in val])
mda_out[key] = new_vals
else:
mda_out[key] = aliases[key].get(mda_out[key], mda_out[key])
job['input_mda'] = mda_out.copy()
def _check_coverage_for_area(area, product_list, platform_name, start_time,
end_time, sensor, scene):
"""Check area coverage for single area.
Helper for covers(). Changes product_list in-place.
"""
area_path = "/product_list/areas/%s" % area
min_coverage = get_config_value(product_list, area_path, "min_coverage")
if not min_coverage:
LOG.debug(
"Minimum area coverage not given or set to zero "
"for area %s", area)
return
_check_overall_coverage_for_area(area, product_list, platform_name,
start_time, end_time, sensor,
min_coverage)
def check_metadata(job):
"""Check the message metadata.
If the metadata does not match the configured values, the scene
will be discarded.
"""
mda = job['input_mda']
product_list = job['product_list']
conf = get_config_value(product_list, '/product_list', 'check_metadata')
if conf is None:
return
for key, val in conf.items():
if key not in mda:
LOG.warning("Metadata item '%s' not in the input message.", key)
continue
if mda[key] not in val:
raise AbortProcessing("Metadata '%s' item '%s' not in '%s'" %
(key, mda[key], str(val)))
def __call__(self, job):
"""Call the publisher."""
mda = job['input_mda'].copy()
mda.pop('dataset', None)
mda.pop('collection', None)
for fmat, _fmat_config in plist_iter(
job['product_list']['product_list']):
prod_path = "/product_list/areas/%s/products/%s" % (
fmat['area'], fmat['product'])
topic_pattern = get_config_value(job['product_list'], prod_path,
"publish_topic")
file_mda = mda.copy()
try:
file_mda['uri'] = fmat['filename']
except KeyError:
continue
file_mda['uid'] = os.path.basename(fmat['filename'])
topic = compose(topic_pattern, fmat)
msg = Message(topic, 'file', file_mda)
LOG.debug('Publishing %s', str(msg))
self.pub.send(str(msg))
self.pub.stop()
def test_null_area(self):
from trollflow2.dict_tools import get_config_value
path = "/product_list/areas/None/products/cloudtype"
expected = "/tmp/satdmz/pps/www/latest_2018/"
res = get_config_value(self.prodlist, path, "output_dir")
self.assertEqual(res, expected)
def check_sunlight_coverage(job):
"""Remove products with too low/high sunlight coverage.
This plugins looks for a dictionary called `sunlight_coverage` in
the product list, with members `min` and/or `max` that define the
minimum and/or maximum allowed sunlight coverage within the scene.
The limits are expressed in % (so between 0 and 100). If the
sunlit fraction is outside the set limits, the affected products
will be discarded. It is also possible to define `check_pass:
True` in this dictionary to check the sunlit fraction within the
overpass of an polar-orbiting satellite.
"""
if get_twilight_poly is None:
LOG.error(
"Trollsched import failed, sunlight coverage calculation not possible"
)
LOG.info("Keeping all products")
return
scn_mda = _get_scene_metadata(job)
scn_mda.update(job['input_mda'])
platform_name = scn_mda['platform_name']
start_time = scn_mda['start_time']
end_time = scn_mda['end_time']
sensor = scn_mda['sensor']
if isinstance(sensor, (list, tuple, set)):
sensor = list(sensor)
if len(sensor) > 1:
LOG.warning(
"Multiple sensors given, taking only one for "
"coverage calculations: %s", sensor[0])
sensor = sensor[0]
product_list = job['product_list']
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
products = list(
product_list['product_list']['areas'][area]['products'].keys())
try:
area_def = get_area_def(area)
except AreaNotFound:
area_def = None
coverage = {True: None, False: None}
overpass = None
for product in products:
prod_path = "/product_list/areas/%s/products/%s" % (area, product)
config = get_config_value(product_list, prod_path,
"sunlight_coverage")
if config is None:
continue
min_day = config.get('min')
max_day = config.get('max')
check_pass = config.get('check_pass', False)
if min_day is None and max_day is None:
LOG.debug("Sunlight coverage not configured for %s / %s",
product, area)
continue
if area_def is None:
area_def = _get_product_area_def(job, area, product)
if area_def is None:
continue
if check_pass and overpass is None:
overpass = Pass(platform_name,
start_time,
end_time,
instrument=sensor)
if coverage[check_pass] is None:
coverage[check_pass] = _get_sunlight_coverage(
area_def, start_time, overpass)
area_conf = product_list['product_list']['areas'][area]
area_conf[
'area_sunlight_coverage_percent'] = coverage[check_pass] * 100
if min_day is not None and coverage[check_pass] < (min_day /
100.0):
LOG.info("Not enough sunlight coverage for "
f"product '{product!s}', removed. Needs at least "
f"{min_day:.1f}%, got {coverage[check_pass]:.1%}.")
dpath.util.delete(product_list, prod_path)
if max_day is not None and coverage[check_pass] > (max_day /
100.0):
LOG.info("Too much sunlight coverage for "
f"product '{product!s}', removed. Needs at most "
f"{max_day:.1f}%, got {coverage[check_pass]:.1%}.")
dpath.util.delete(product_list, prod_path)
def test_config_value_same_level(self):
"""Test the config value at the same level."""
from trollflow2.dict_tools import get_config_value
expected = "/tmp/satdmz/pps/www/latest_2018/"
res = get_config_value(self.prodlist, self.path, "output_dir")
self.assertEqual(res, expected)
def test_config_value_parent_level(self):
"""Test getting a config value from the parent level."""
from trollflow2.dict_tools import get_config_value
expected = "{start_time:%Y%m%d_%H%M}_{areaname:s}_{productname}.{format}"
res = get_config_value(self.prodlist, self.path, "fname_pattern")
self.assertEqual(res, expected)
def test_config_value_missing_own_default(self):
"""Test getting a missing value with default."""
from trollflow2.dict_tools import get_config_value
res = get_config_value(self.prodlist, self.path, "nothing", default=42)
self.assertEqual(res, 42)
def test_config_value_missing(self):
"""Test getting a missing config value."""
from trollflow2.dict_tools import get_config_value
res = get_config_value(self.prodlist, self.path, "nothing")
self.assertIsNone(res)
def test_config_value_common(self):
"""Test getting a common config value."""
from trollflow2.dict_tools import get_config_value
expected = "foo"
res = get_config_value(self.prodlist, self.path, "something")
self.assertEqual(res, expected)
