def resample(job): """Resample the scene to some areas.""" defaults = { "radius_of_influence": None, "resampler": "nearest", "reduce_data": True, "cache_dir": None, "mask_area": False, "epsilon": 0.0 } product_list = job['product_list'] conf = _get_plugin_conf(product_list, '/product_list', defaults) job['resampled_scenes'] = {} scn = job['scene'] for area in product_list['product_list']['areas']: area_conf = _get_plugin_conf(product_list, '/product_list/areas/' + str(area), conf) LOG.info('Resampling to %s', str(area)) if area is None: minarea = get_config_value(product_list, '/product_list/areas/' + str(area), 'use_min_area') maxarea = get_config_value(product_list, '/product_list/areas/' + str(area), 'use_max_area') if minarea is True: job['resampled_scenes'][area] = scn.resample( scn.min_area(), **area_conf) elif maxarea is True: job['resampled_scenes'][area] = scn.resample( scn.max_area(), **area_conf) else: job['resampled_scenes'][area] = scn else: job['resampled_scenes'][area] = scn.resample(area, **area_conf)
def sza_check(job): """Remove products which are not valid for the current Sun zenith angle.""" scn_mda = _get_scene_metadata(job) scn_mda.update(job['input_mda']) start_time = scn_mda['start_time'] product_list = job['product_list'] areas = list(product_list['product_list']['areas'].keys()) for area in areas: products = list( product_list['product_list']['areas'][area]['products'].keys()) for product in products: prod_path = "/product_list/areas/%s/products/%s" % (area, product) lon = get_config_value(product_list, prod_path, "sunzen_check_lon") lat = get_config_value(product_list, prod_path, "sunzen_check_lat") if lon is None or lat is None: LOG.debug( "No 'sunzen_check_lon' or 'sunzen_check_lat' configured, " "can\'t check Sun elevation for %s / %s", area, product) continue sunzen = sun_zenith_angle(start_time, lon, lat) LOG.debug("Sun zenith angle is %.2f degrees", sunzen) # Check nighttime limit limit = get_config_value(product_list, prod_path, "sunzen_minimum_angle") if limit is not None: if sunzen < limit: LOG.info( "Sun zenith angle to small for nighttime " "product '%s', product removed.", product) dpath.util.delete(product_list, prod_path) continue # Check daytime limit limit = get_config_value(product_list, prod_path, "sunzen_maximum_angle") if limit is not None: if sunzen > limit: LOG.info( "Sun zenith angle too large for daytime " "product '%s', product removed.", product) dpath.util.delete(product_list, prod_path) continue if len(product_list['product_list']['areas'][area]['products']) == 0: LOG.info("Removing empty area: %s", area) dpath.util.delete(product_list, '/product_list/areas/%s' % area)
def resample(job): """Resample the scene to some areas.""" defaults = { "radius_of_influence": None, "resampler": "nearest", "reduce_data": True, "cache_dir": None, "mask_area": False, "epsilon": 0.0 } product_list = job['product_list'] conf = _get_plugin_conf(product_list, '/product_list', defaults) job['resampled_scenes'] = {} scn = job['scene'] for area in product_list['product_list']['areas']: area_conf = _get_plugin_conf(product_list, '/product_list/areas/' + str(area), conf) LOG.debug('Resampling to %s', str(area)) if area == 'None': minarea = get_config_value(product_list, '/product_list/areas/' + str(area), 'use_min_area') maxarea = get_config_value(product_list, '/product_list/areas/' + str(area), 'use_max_area') native = conf.get('resampler') == 'native' if minarea is True: job['resampled_scenes'][area] = scn.resample( scn.min_area(), **area_conf) elif maxarea is True: job['resampled_scenes'][area] = scn.resample( scn.max_area(), **area_conf) elif native: job['resampled_scenes'][area] = scn.resample( resampler='native') else: # The composites need to be created for the saving to work if not set(scn.keys()).issuperset(scn.wishlist): LOG.debug( "Generating composites for 'null' area (satellite projection)." ) scn.load(scn.wishlist, generate=True) job['resampled_scenes'][area] = scn else: LOG.debug("area: %s, area_conf: %s", area, str(area_conf)) job['resampled_scenes'][area] = scn.resample(area, **area_conf)
def _get_plugin_conf(product_list, path, defaults): conf = {} for key in defaults: conf[key] = get_config_value(product_list, path, key, default=defaults.get(key)) return conf
def check_sunlight_coverage(job): """Remove products with too low daytime coverage. This plugins looks for a parameter called `min_sunlight_coverage` in the product list, expressed in % (so between 0 and 100). If the sunlit fraction is less than configured, the affected products will be discarded. """ if get_twilight_poly is None: LOG.error("Trollsched import failed, sunlight coverage calculation not possible") LOG.info("Keeping all products") return scn_mda = job['scene'].attrs.copy() scn_mda.update(job['input_mda']) platform_name = scn_mda['platform_name'] start_time = scn_mda['start_time'] end_time = scn_mda['end_time'] sensor = scn_mda['sensor'] if isinstance(sensor, (list, tuple, set)): sensor = list(sensor)[0] LOG.warning("Possibly many sensors given, taking only one for " "coverage calculations: %s", sensor) product_list = job['product_list'] areas = list(product_list['product_list']['areas'].keys()) for area in areas: products = list(product_list['product_list']['areas'][area]['products'].keys()) for product in products: try: if isinstance(product, tuple): prod = job['resampled_scenes'][area][product[0]] else: prod = job['resampled_scenes'][area][product] except KeyError: LOG.warning("No dataset %s for this scene and area %s", product, area) continue else: area_def = prod.attrs['area'] prod_path = "/product_list/areas/%s/products/%s" % (area, product) config = get_config_value(product_list, prod_path, "sunlight_coverage") if config is None: continue min_day = config.get('min') use_pass = config.get('check_pass', False) if use_pass: overpass = Pass(platform_name, start_time, end_time, instrument=sensor) else: overpass = None if min_day is None: continue coverage = _get_sunlight_coverage(area_def, start_time, overpass) product_list['product_list']['areas'][area]['area_sunlight_coverage_percent'] = coverage * 100 if coverage < (min_day / 100.0): LOG.info("Not enough sunlight coverage in " "product '%s', removed.", product) dpath.util.delete(product_list, prod_path)
def covers(job): """Check area coverage. Remove areas with too low coverage from the worklist. """ if Pass is None: LOG.error("Trollsched import failed, coverage calculation not possible") LOG.info("Keeping all areas") return col_area = job['product_list']['product_list'].get('coverage_by_collection_area', False) if col_area and 'collection_area_id' in job['input_mda']: if job['input_mda']['collection_area_id'] not in job['product_list']['product_list']['areas']: raise AbortProcessing( "Area collection ID '%s' does not match " "production area(s) %s" % (job['input_mda']['collection_area_id'], str(list(job['product_list']['product_list'])))) product_list = job['product_list'].copy() scn_mda = job['scene'].attrs.copy() scn_mda.update(job['input_mda']) platform_name = scn_mda['platform_name'] start_time = scn_mda['start_time'] end_time = scn_mda['end_time'] sensor = scn_mda['sensor'] if isinstance(sensor, (list, tuple, set)): sensor = list(sensor)[0] LOG.warning("Possibly many sensors given, taking only one for " "coverage calculations: %s", sensor) areas = list(product_list['product_list']['areas'].keys()) for area in areas: area_path = "/product_list/areas/%s" % area min_coverage = get_config_value(product_list, area_path, "min_coverage") if not min_coverage: LOG.debug("Minimum area coverage not given or set to zero " "for area %s", area) continue cov = get_scene_coverage(platform_name, start_time, end_time, sensor, area) product_list['product_list']['areas'][area]['area_coverage_percent'] = cov if cov < min_coverage: LOG.info( "Area coverage %.2f %% below threshold %.2f %%", cov, min_coverage) LOG.info("Removing area %s from the worklist", area) dpath.util.delete(product_list, area_path) else: LOG.debug("Area coverage %.2f %% above threshold %.2f %% - Carry on", cov, min_coverage) job['product_list'] = product_list
def check_platform(job): """Check if the platform is valid. If not, discard the scene.""" mda = job['input_mda'] product_list = job['product_list'] conf = get_config_value(product_list, '/product_list', 'processed_platforms') if conf is None: return platform = mda['platform_name'] if platform not in conf: raise AbortProcessing( "'%s' not in list of allowed platforms" % platform)
def metadata_alias(job): """Replace input metadata values with aliases.""" mda_out = job['input_mda'].copy() product_list = job['product_list'] aliases = get_config_value(product_list, '/product_list', 'metadata_aliases') if aliases is None: return for key in aliases: if key in mda_out: val = mda_out[key] if isinstance(val, (list, tuple, set)): typ = type(val) new_vals = typ([aliases[key].get(itm, itm) for itm in val]) mda_out[key] = new_vals else: mda_out[key] = aliases[key].get(mda_out[key], mda_out[key]) job['input_mda'] = mda_out.copy()
def _check_coverage_for_area(area, product_list, platform_name, start_time, end_time, sensor, scene): """Check area coverage for single area. Helper for covers(). Changes product_list in-place. """ area_path = "/product_list/areas/%s" % area min_coverage = get_config_value(product_list, area_path, "min_coverage") if not min_coverage: LOG.debug( "Minimum area coverage not given or set to zero " "for area %s", area) return _check_overall_coverage_for_area(area, product_list, platform_name, start_time, end_time, sensor, min_coverage)
def check_metadata(job): """Check the message metadata. If the metadata does not match the configured values, the scene will be discarded. """ mda = job['input_mda'] product_list = job['product_list'] conf = get_config_value(product_list, '/product_list', 'check_metadata') if conf is None: return for key, val in conf.items(): if key not in mda: LOG.warning("Metadata item '%s' not in the input message.", key) continue if mda[key] not in val: raise AbortProcessing("Metadata '%s' item '%s' not in '%s'" % (key, mda[key], str(val)))
def __call__(self, job): """Call the publisher.""" mda = job['input_mda'].copy() mda.pop('dataset', None) mda.pop('collection', None) for fmat, _fmat_config in plist_iter( job['product_list']['product_list']): prod_path = "/product_list/areas/%s/products/%s" % ( fmat['area'], fmat['product']) topic_pattern = get_config_value(job['product_list'], prod_path, "publish_topic") file_mda = mda.copy() try: file_mda['uri'] = fmat['filename'] except KeyError: continue file_mda['uid'] = os.path.basename(fmat['filename']) topic = compose(topic_pattern, fmat) msg = Message(topic, 'file', file_mda) LOG.debug('Publishing %s', str(msg)) self.pub.send(str(msg)) self.pub.stop()
def test_null_area(self): from trollflow2.dict_tools import get_config_value path = "/product_list/areas/None/products/cloudtype" expected = "/tmp/satdmz/pps/www/latest_2018/" res = get_config_value(self.prodlist, path, "output_dir") self.assertEqual(res, expected)
def check_sunlight_coverage(job): """Remove products with too low/high sunlight coverage. This plugins looks for a dictionary called `sunlight_coverage` in the product list, with members `min` and/or `max` that define the minimum and/or maximum allowed sunlight coverage within the scene. The limits are expressed in % (so between 0 and 100). If the sunlit fraction is outside the set limits, the affected products will be discarded. It is also possible to define `check_pass: True` in this dictionary to check the sunlit fraction within the overpass of an polar-orbiting satellite. """ if get_twilight_poly is None: LOG.error( "Trollsched import failed, sunlight coverage calculation not possible" ) LOG.info("Keeping all products") return scn_mda = _get_scene_metadata(job) scn_mda.update(job['input_mda']) platform_name = scn_mda['platform_name'] start_time = scn_mda['start_time'] end_time = scn_mda['end_time'] sensor = scn_mda['sensor'] if isinstance(sensor, (list, tuple, set)): sensor = list(sensor) if len(sensor) > 1: LOG.warning( "Multiple sensors given, taking only one for " "coverage calculations: %s", sensor[0]) sensor = sensor[0] product_list = job['product_list'] areas = list(product_list['product_list']['areas'].keys()) for area in areas: products = list( product_list['product_list']['areas'][area]['products'].keys()) try: area_def = get_area_def(area) except AreaNotFound: area_def = None coverage = {True: None, False: None} overpass = None for product in products: prod_path = "/product_list/areas/%s/products/%s" % (area, product) config = get_config_value(product_list, prod_path, "sunlight_coverage") if config is None: continue min_day = config.get('min') max_day = config.get('max') check_pass = config.get('check_pass', False) if min_day is None and max_day is None: LOG.debug("Sunlight coverage not configured for %s / %s", product, area) continue if area_def is None: area_def = _get_product_area_def(job, area, product) if area_def is None: continue if check_pass and overpass is None: overpass = Pass(platform_name, start_time, end_time, instrument=sensor) if coverage[check_pass] is None: coverage[check_pass] = _get_sunlight_coverage( area_def, start_time, overpass) area_conf = product_list['product_list']['areas'][area] area_conf[ 'area_sunlight_coverage_percent'] = coverage[check_pass] * 100 if min_day is not None and coverage[check_pass] < (min_day / 100.0): LOG.info("Not enough sunlight coverage for " f"product '{product!s}', removed. Needs at least " f"{min_day:.1f}%, got {coverage[check_pass]:.1%}.") dpath.util.delete(product_list, prod_path) if max_day is not None and coverage[check_pass] > (max_day / 100.0): LOG.info("Too much sunlight coverage for " f"product '{product!s}', removed. Needs at most " f"{max_day:.1f}%, got {coverage[check_pass]:.1%}.") dpath.util.delete(product_list, prod_path)
def test_config_value_same_level(self): """Test the config value at the same level.""" from trollflow2.dict_tools import get_config_value expected = "/tmp/satdmz/pps/www/latest_2018/" res = get_config_value(self.prodlist, self.path, "output_dir") self.assertEqual(res, expected)
def test_config_value_parent_level(self): """Test getting a config value from the parent level.""" from trollflow2.dict_tools import get_config_value expected = "{start_time:%Y%m%d_%H%M}_{areaname:s}_{productname}.{format}" res = get_config_value(self.prodlist, self.path, "fname_pattern") self.assertEqual(res, expected)
def test_config_value_missing_own_default(self): """Test getting a missing value with default.""" from trollflow2.dict_tools import get_config_value res = get_config_value(self.prodlist, self.path, "nothing", default=42) self.assertEqual(res, 42)
def test_config_value_missing(self): """Test getting a missing config value.""" from trollflow2.dict_tools import get_config_value res = get_config_value(self.prodlist, self.path, "nothing") self.assertIsNone(res)
def test_config_value_common(self): """Test getting a common config value.""" from trollflow2.dict_tools import get_config_value expected = "foo" res = get_config_value(self.prodlist, self.path, "something") self.assertEqual(res, expected)