def test_pass_instrument_interface(self):
tstart = datetime(2018, 10, 16, 2, 48, 29)
tend = datetime(2018, 10, 16, 3, 2, 38)
instruments = set(('viirs', 'avhrr', 'modis'))
overp = Pass('NOAA-20',
tstart,
tend,
orb=self.n20orb,
instrument=instruments)
self.assertEqual(overp.instrument, 'avhrr')
instruments = set(('viirs', 'modis'))
overp = Pass('NOAA-20',
tstart,
tend,
orb=self.n20orb,
instrument=instruments)
self.assertEqual(overp.instrument, 'viirs')
instruments = set(('amsu-a', 'mhs'))
self.assertRaises(TypeError,
Pass,
self,
'NOAA-20',
tstart,
tend,
orb=self.n20orb,
instrument=instruments)
def test_generate_metno_xml(self):
import xml.etree.ElementTree as ET
root = ET.Element("acquisition-schedule")
orig = (
'<acquisition-schedule><pass satellite="FENGYUN 3D" aos="20190105010145" los="20190105011715" '
'orbit="5907" max-elevation="52.943" asimuth-at-max-elevation="107.385" asimuth-at-aos="18.555" '
'pass-direction="D" satellite-lon-at-aos="76.204" satellite-lat-at-aos="80.739" '
'tle-epoch="20181229125844.110848" /></acquisition-schedule>')
tstart = datetime.strptime("2019-01-05T01:01:45", "%Y-%m-%dT%H:%M:%S")
tend = tstart + timedelta(seconds=60 * 15.5)
tle1 = '1 43010U 17072A 18363.54078832 -.00000045 00000-0 -79715-6 0 9999'
tle2 = '2 43010 98.6971 300.6571 0001567 143.5989 216.5282 14.19710974 58158'
mypass = Pass('FENGYUN 3D',
tstart,
tend,
instrument='mersi2',
tle1=tle1,
tle2=tle2)
coords = (10.72, 59.942, 0.1)
mypass.generate_metno_xml(coords, root)
self.assertEqual(ET.tostring(root).decode("utf-8"), orig)
def find_sectors(self):
"""Identify sectors with at least some coverage by the provided scene.
Returns
-------
list
area_id of each sector with some coverage.
"""
data = self.message.data
overpass = Pass(
data["platform_name"],
self.scene.start_time,
self.scene.end_time,
instrument="viirs",
)
logger.debug(f"Created overpass {overpass}")
logger.debug(f"args: {data['platform_name']} :: "
"{self.scene.start_time} :: {self.scene.end_time}")
sectors = []
coverage_threashold = float(
tutil.get_env_var("COVERAGE_THRESHOLD", 0.1))
for sector_def in parse_area_file(AREA_DEF):
logger.debug("Checking coverage for %s", sector_def.area_id)
coverage = overpass.area_coverage(sector_def)
logger.debug("{} coverage: {}".format(sector_def.area_id,
coverage))
if coverage > coverage_threashold:
sectors.append(sector_def)
return sectors
def _granule_covers_region(granule_metadata, region):
granule_pass = Pass(_get_platform_name(granule_metadata),
granule_metadata["start_time"],
granule_metadata["end_time"],
instrument=_get_sensor(granule_metadata))
coverage = granule_pass.area_coverage(region)
if coverage > 0:
coverage_str = f"is overlapping region {region.description:s} by fraction {coverage:.5f}"
_log_overlap_message(granule_metadata, coverage_str)
return True
return False
def _predict(self, granule_metadata, step):
gr_time = granule_metadata["start_time"]
while True:
gr_time += step
gr_pass = Pass(_get_platform_name(granule_metadata),
gr_time,
gr_time + self.granule_duration,
instrument=_get_sensor(granule_metadata))
if not gr_pass.area_coverage(self.region) > 0:
break
self.planned_granule_times.add(gr_time)
def test_arctic_is_not_antarctic(self):
tstart = datetime(2021, 2, 3, 16, 28, 3)
tend = datetime(2021, 2, 3, 16, 31, 3)
overp = Pass('Metop-B',
tstart,
tend,
orb=self.mborb,
instrument='avhrr')
cov_south = overp.area_coverage(self.antarctica)
cov_north = overp.area_coverage(self.arctica)
assert cov_north == 0
assert cov_south != 0
def check_sunlight_coverage(job):
"""Remove products with too low daytime coverage.
This plugins looks for a parameter called `min_sunlight_coverage` in the
product list, expressed in % (so between 0 and 100). If the sunlit fraction
is less than configured, the affected products will be discarded.
"""
if get_twilight_poly is None:
LOG.error("Trollsched import failed, sunlight coverage calculation not possible")
LOG.info("Keeping all products")
return
scn_mda = job['scene'].attrs.copy()
scn_mda.update(job['input_mda'])
platform_name = scn_mda['platform_name']
start_time = scn_mda['start_time']
end_time = scn_mda['end_time']
sensor = scn_mda['sensor']
if isinstance(sensor, (list, tuple, set)):
sensor = list(sensor)[0]
LOG.warning("Possibly many sensors given, taking only one for "
"coverage calculations: %s", sensor)
product_list = job['product_list']
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
products = list(product_list['product_list']['areas'][area]['products'].keys())
for product in products:
try:
if isinstance(product, tuple):
prod = job['resampled_scenes'][area][product[0]]
else:
prod = job['resampled_scenes'][area][product]
except KeyError:
LOG.warning("No dataset %s for this scene and area %s", product, area)
continue
else:
area_def = prod.attrs['area']
prod_path = "/product_list/areas/%s/products/%s" % (area, product)
config = get_config_value(product_list, prod_path, "sunlight_coverage")
if config is None:
continue
min_day = config.get('min')
use_pass = config.get('check_pass', False)
if use_pass:
overpass = Pass(platform_name, start_time, end_time, instrument=sensor)
else:
overpass = None
if min_day is None:
continue
coverage = _get_sunlight_coverage(area_def, start_time, overpass)
product_list['product_list']['areas'][area]['area_sunlight_coverage_percent'] = coverage * 100
if coverage < (min_day / 100.0):
LOG.info("Not enough sunlight coverage in "
"product '%s', removed.", product)
dpath.util.delete(product_list, prod_path)
def invoke(self, context):
"""Invoke"""
self.use_lock = context.get("use_lock", False)
self.logger.debug("Locking is used in coverage checker: %s",
str(self.use_lock))
if self.use_lock:
self.logger.debug(
"Scene loader acquires lock of previous "
"worker: %s", str(context["prev_lock"]))
utils.acquire_lock(context["prev_lock"])
scene = context["content"]
overpass = Pass(scene.info["platform_name"],
scene.info['start_time'],
scene.info['end_time'],
instrument=scene.info["sensor"][0])
areas = []
for area_name in scene.info["areas"]:
self.logger.info("Checking coverage of %s", area_name)
try:
min_coverage = context["min_coverage"][area_name]
except KeyError:
self.logger.warning("No minimum coverage given, "
"assuming 0 % coverage needed")
areas.append(area_name)
continue
if utils.covers(overpass, area_name, min_coverage, self.logger):
areas.append(area_name)
else:
self.logger.info("Area coverage too low, skipping %s",
area_name)
continue
if len(areas) > 0:
scene.info["areas"] = areas
context["output_queue"].put(scene)
else:
self.logger.info("No areas with enough coverage")
if utils.release_locks([context["lock"]]):
self.logger.debug("Scene loader releases own lock %s",
str(context["lock"]))
time.sleep(1)
# Wait until the lock has been released downstream
if self.use_lock:
utils.acquire_lock(context["lock"])
utils.release_locks([context["lock"]])
# After all the items have been processed, release the lock for
# the previous step
utils.release_locks([context["prev_lock"]],
log=self.logger.debug,
log_msg="Scene loader releases lock of " +
"previous worker")
def granule_inside_area(start_time,
end_time,
platform_name,
instrument,
area_def,
thr_area_coverage,
tle_file=None):
"""Check if a satellite data granule is over area interest, using the start and
end times from the filename
"""
try:
metop = Orbital(platform_name, tle_file)
except KeyError:
LOG.exception(
'Failed getting orbital data for {0}'.format(platform_name))
LOG.critical(
'Cannot determine orbit! Probably TLE file problems...\n' +
'Granule will be set to be inside area of interest disregarding')
return True
tle1 = metop.tle.line1
tle2 = metop.tle.line2
mypass = Pass(platform_name,
start_time,
end_time,
instrument=instrument,
tle1=tle1,
tle2=tle2)
acov = mypass.area_coverage(area_def)
LOG.debug("Granule coverage of area %s: %f", area_def.area_id, acov)
is_inside = (acov > thr_area_coverage)
if is_inside:
from pyresample.boundary import AreaDefBoundary
from trollsched.drawing import save_fig
area_boundary = AreaDefBoundary(area_def, frequency=100)
area_boundary = area_boundary.contour_poly
save_fig(mypass, poly=area_boundary, directory='/tmp')
return
def test_swath_boundary(self):
tstart = datetime(2018, 10, 16, 2, 48, 29)
tend = datetime(2018, 10, 16, 3, 2, 38)
overp = Pass('NOAA-20', tstart, tend, orb=self.n20orb, instrument='viirs')
overp_boundary = SwathBoundary(overp)
cont = overp_boundary.contour()
assertNumpyArraysEqual(cont[0], LONS1)
assertNumpyArraysEqual(cont[1], LATS1)
tstart = datetime(2018, 10, 16, 4, 29, 4)
tend = datetime(2018, 10, 16, 4, 30, 29, 400000)
overp = Pass('NOAA-20', tstart, tend, orb=self.n20orb, instrument='viirs')
overp_boundary = SwathBoundary(overp, frequency=200)
cont = overp_boundary.contour()
assertNumpyArraysEqual(cont[0], LONS2)
assertNumpyArraysEqual(cont[1], LATS2)
# NOAA-19 AVHRR:
tstart = datetime.strptime('20181016 04:00:00', '%Y%m%d %H:%M:%S')
tend = datetime.strptime('20181016 04:01:00', '%Y%m%d %H:%M:%S')
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr')
overp_boundary = SwathBoundary(overp, frequency=500)
cont = overp_boundary.contour()
assertNumpyArraysEqual(cont[0], LONS3)
assertNumpyArraysEqual(cont[1], LATS3)
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr/3')
overp_boundary = SwathBoundary(overp, frequency=500)
cont = overp_boundary.contour()
assertNumpyArraysEqual(cont[0], LONS3)
assertNumpyArraysEqual(cont[1], LATS3)
def test_meos_pass_list(self):
orig = (
" 1 20190105 FENGYUN 3D 5907 52.943 01:01:45 n/a 01:17:15 15:30 18.6 107.4 -- "
"Undefined(Scheduling not done 1546650105 ) a3d0df0cd289244e2f39f613f229a5cc D"
)
tstart = datetime.strptime("2019-01-05T01:01:45", "%Y-%m-%dT%H:%M:%S")
tend = tstart + timedelta(seconds=60 * 15.5)
tle1 = '1 43010U 17072A 18363.54078832 -.00000045 00000-0 -79715-6 0 9999'
tle2 = '2 43010 98.6971 300.6571 0001567 143.5989 216.5282 14.19710974 58158'
mypass = Pass('FENGYUN 3D',
tstart,
tend,
instrument='mersi2',
tle1=tle1,
tle2=tle2)
coords = (10.72, 59.942, 0.1)
meos_format_str = mypass.print_meos(coords, line_no=1)
self.assertEqual(meos_format_str, orig)
mypass = Pass('FENGYUN 3D',
tstart,
tend,
instrument='mersi-2',
tle1=tle1,
tle2=tle2)
coords = (10.72, 59.942, 0.1)
meos_format_str = mypass.print_meos(coords, line_no=1)
self.assertEqual(meos_format_str, orig)
def process_xmlrequest(filename, plotdir, output_file, excluded_satellites):
tree = ET.parse(filename)
root = tree.getroot()
for child in root:
if child.tag == 'pass':
LOG.debug("Pass: %s", str(child.attrib))
platform_name = SATELLITE_NAMES.get(child.attrib['satellite'],
child.attrib['satellite'])
instrument = INSTRUMENT.get(platform_name)
if not instrument:
LOG.error('Instrument unknown! Platform = %s', platform_name)
continue
if platform_name in excluded_satellites:
LOG.debug('Platform name excluded: %s', platform_name)
continue
try:
overpass = Pass(platform_name,
datetime.strptime(child.attrib['start-time'],
'%Y-%m-%d-%H:%M:%S'),
datetime.strptime(child.attrib['end-time'],
'%Y-%m-%d-%H:%M:%S'),
instrument=instrument)
except KeyError as err:
LOG.warning('Failed on satellite %s: %s', platform_name,
str(err))
continue
save_fig(overpass, directory=plotdir)
child.set('img', overpass.fig)
child.set('rec', 'True')
LOG.debug("Plot saved - plotdir = %s, platform_name = %s", plotdir,
platform_name)
tree.write(output_file, encoding='utf-8', xml_declaration=True)
with open(output_file) as fpt:
lines = fpt.readlines()
lines.insert(
1, "<?xml-stylesheet type='text/xsl' href='reqreader.xsl'?>")
with open(output_file, 'w') as fpt:
fpt.writelines(lines)
def create_scene_from_mda(self, mda):
"""Read the metadata *mda* and return a corresponding MPOP scene.
"""
time_slot = (mda.get('start_time') or mda.get('nominal_time')
or mda.get('end_time'))
# orbit is not given for GEO satellites, use None
if 'orbit_number' not in mda:
mda['orbit_number'] = None
platform = mda["platform_name"]
LOGGER.info("platform %s time %s", str(platform), str(time_slot))
if isinstance(mda['sensor'], (list, tuple, set)):
sensor = mda['sensor'][0]
else:
sensor = mda['sensor']
# Create satellite scene
global_data = GF.create_scene(satname=str(platform),
satnumber='',
instrument=str(sensor),
time_slot=time_slot,
orbit=mda['orbit_number'],
variant=mda.get('variant', ''))
LOGGER.debug("Creating scene for satellite %s and time %s",
str(platform), str(time_slot))
if mda['orbit_number'] is not None or mda.get('orbit_type') == "polar":
global_data.overpass = Pass(platform,
mda['start_time'],
mda['end_time'],
instrument=sensor)
# Update missing information to global_data.info{}
# TODO: this should be fixed in mpop.
global_data.info.update(mda)
global_data.info['time'] = time_slot
return global_data
def test_swath_coverage(self):
# NOAA-19 AVHRR:
tstart = datetime.strptime('20181016 03:54:13', '%Y%m%d %H:%M:%S')
tend = datetime.strptime('20181016 03:55:13', '%Y%m%d %H:%M:%S')
overp = Pass('NOAA-19',
tstart,
tend,
orb=self.n19orb,
instrument='avhrr')
cov = overp.area_coverage(self.euron1)
self.assertEqual(cov, 0)
overp = Pass('NOAA-19',
tstart,
tend,
orb=self.n19orb,
instrument='avhrr',
frequency=80)
cov = overp.area_coverage(self.euron1)
self.assertEqual(cov, 0)
tstart = datetime.strptime('20181016 04:00:00', '%Y%m%d %H:%M:%S')
tend = datetime.strptime('20181016 04:01:00', '%Y%m%d %H:%M:%S')
overp = Pass('NOAA-19',
tstart,
tend,
orb=self.n19orb,
instrument='avhrr')
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
overp = Pass('NOAA-19',
tstart,
tend,
orb=self.n19orb,
instrument='avhrr',
frequency=100)
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
overp = Pass('NOAA-19',
tstart,
tend,
orb=self.n19orb,
instrument='avhrr/3',
frequency=133)
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
overp = Pass('NOAA-19',
tstart,
tend,
orb=self.n19orb,
instrument='avhrr',
frequency=300)
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
# ASCAT and AVHRR on Metop-B:
tstart = datetime.strptime("2019-01-02T10:19:39", "%Y-%m-%dT%H:%M:%S")
tend = tstart + timedelta(seconds=180)
tle1 = '1 38771U 12049A 19002.35527803 .00000000 00000+0 21253-4 0 00017'
tle2 = '2 38771 98.7284 63.8171 0002025 96.0390 346.4075 14.21477776326431'
mypass = Pass('Metop-B',
tstart,
tend,
instrument='ascat',
tle1=tle1,
tle2=tle2)
cov = mypass.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.322812, 5)
mypass = Pass('Metop-B',
tstart,
tend,
instrument='avhrr',
tle1=tle1,
tle2=tle2)
cov = mypass.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.357324, 5)
tstart = datetime.strptime("2019-01-05T01:01:45", "%Y-%m-%dT%H:%M:%S")
tend = tstart + timedelta(seconds=60 * 15.5)
tle1 = '1 43010U 17072A 18363.54078832 -.00000045 00000-0 -79715-6 0 9999'
tle2 = '2 43010 98.6971 300.6571 0001567 143.5989 216.5282 14.19710974 58158'
mypass = Pass('FENGYUN 3D',
tstart,
tend,
instrument='mersi2',
tle1=tle1,
tle2=tle2)
cov = mypass.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.786836, 5)
def collect(self, granule_metadata):
"""
Parameters:
granule_metadata : metadata
"""
# Check if input data is being waited for
platform = granule_metadata['platform_name']
start_time = granule_metadata['start_time']
end_time = granule_metadata['end_time']
for ptime in self.planned_granule_times:
if abs(start_time - ptime) < timedelta(seconds=3):
self.granule_times.add(ptime)
self.granules.append(granule_metadata)
LOG.info("Added %s (%s) granule to area %s",
platform,
str(start_time),
self.region.area_id)
# If last granule return swath and cleanup
if self.granule_times == self.planned_granule_times:
LOG.info("Collection finished for area: " +
str(self.region.area_id))
return self.finish()
else:
return
# Get corners from input data
if self.granule_duration is None:
self.granule_duration = end_time - start_time
LOG.debug("Estimated granule duration to " +
str(self.granule_duration))
granule_pass = Pass(platform, start_time, end_time,
instrument=granule_metadata["sensor"])
# If file is within region, make pass prediction to know what to wait
# for
if granule_pass.area_coverage(self.region) > 0:
self.granule_times.add(start_time)
self.granules.append(granule_metadata)
# Computation of the predicted granules within the region
if not self.planned_granule_times:
self.planned_granule_times.add(start_time)
LOG.info("Added %s (%s) granule to area %s",
platform,
str(start_time),
self.region.area_id)
LOG.debug(
"Predicting granules covering " + self.region.area_id)
gr_time = start_time
while True:
gr_time += self.granule_duration
gr_pass = Pass(platform, gr_time,
gr_time + self.granule_duration,
instrument=granule_metadata["sensor"])
if not gr_pass.area_coverage(self.region) > 0:
break
self.planned_granule_times.add(gr_time)
gr_time = start_time
while True:
gr_time -= self.granule_duration
gr_pass = Pass(platform, gr_time,
gr_time + self.granule_duration,
instrument=granule_metadata["sensor"])
if not gr_pass.area_coverage(self.region) > 0:
break
self.planned_granule_times.add(gr_time)
LOG.info(
"Planned granules: " + str(sorted(self.planned_granule_times)))
self.timeout = (max(self.planned_granule_times)
+ self.granule_duration
+ self.timeliness)
LOG.info("Planned timeout: " + self.timeout.isoformat())
else:
try:
LOG.debug("Granule %s is not overlapping %s",
granule_metadata["uri"], self.region.name)
except KeyError:
try:
LOG.debug("Granule with start and end times = " +
str(granule_metadata["start_time"]) + " " +
str(granule_metadata["end_time"]) +
"is not overlapping " + str(self.region.name))
except KeyError:
LOG.debug("Failed printing debug info...")
LOG.debug("Keys in granule_metadata = " +
str(granule_metadata.keys()))
# If last granule return swath and cleanup
if (self.granule_times and
(self.granule_times == self.planned_granule_times)):
LOG.debug("Collection finished for area: " +
str(self.region.area_id))
return self.finish()
def _process(self, context):
"""Process a context."""
glbl = context["content"]["scene"]
extra_metadata = context["content"]["extra_metadata"]
with open(context["product_list"], "r") as fid:
product_config = ordered_load(fid)
# Handle config options
kwargs = {}
kwargs['mask_area'] = context.get('mask_area', True)
self.logger.debug("Setting area masking to %s",
str(kwargs['mask_area']))
kwargs['nprocs'] = context.get('nprocs', 1)
self.logger.debug("Using %d CPUs for resampling.", kwargs['nprocs'])
kwargs['resampler'] = context.get('resampler', "nearest")
self.logger.debug("Using resampling method: '%s'.",
kwargs['resampler'])
try:
kwargs['cache_dir'] = context['cache_dir']
self.logger.debug("Setting projection cache dir to %s",
kwargs['cache_dir'])
except (AttributeError, KeyError):
pass
prod_list = product_config["product_list"]
# Overpass for coverage calculations
scn_metadata = glbl.attrs
if product_config['common'].get('coverage_check', True) and Pass:
overpass = Pass(scn_metadata['platform_name'],
scn_metadata['start_time'],
scn_metadata['end_time'],
instrument=scn_metadata['sensor'][0])
else:
overpass = None
# Get the area ID from metadata dict
area_id = extra_metadata['area_id']
# Check for area coverage
if overpass is not None:
min_coverage = prod_list[area_id].get("min_coverage", 0.0)
if not utils.covers(overpass, area_id, min_coverage, self.logger):
return
kwargs['radius_of_influence'] = None
try:
area_config = product_config["product_list"][area_id]
kwargs['radius_of_influence'] = \
area_config.get("srch_radius", context["radius"])
except (AttributeError, KeyError):
kwargs['radius_of_influence'] = 10000.
if kwargs['radius_of_influence'] is None:
self.logger.debug("Using default search radius.")
else:
self.logger.debug("Using search radius %d meters.",
int(kwargs['radius_of_influence']))
# Set lock if locking is used
if self.use_lock:
self.logger.debug("Resampler acquires own lock %s",
str(context["lock"]))
utils.acquire_lock(context["lock"])
if area_id == "satproj":
self.logger.info("Using satellite projection")
lcl = glbl
else:
metadata = glbl.attrs
self.logger.info("Resampling time slot %s to area %s",
metadata["start_time"], area_id)
lcl = glbl.resample(area_id, **kwargs)
# Add area ID to the scene attributes so everything needed
# in filename composing is in the same dictionary
lcl.attrs["area_id"] = area_id
metadata = extra_metadata.copy()
metadata["product_config"] = product_config
metadata["products"] = prod_list[area_id]['products']
self.logger.debug(
"Inserting lcl (area: %s, start_time: %s) "
"to writer's queue", area_id, str(scn_metadata["start_time"]))
context["output_queue"].put({'scene': lcl, 'extra_metadata': metadata})
if utils.release_locks([context["lock"]]):
self.logger.debug("Resampler releases own lock %s",
str(context["lock"]))
# Wait 1 second to ensure next worker has time to acquire the
# lock
time.sleep(1)
# Wait until the lock has been released downstream
if self.use_lock:
utils.acquire_lock(context["lock"])
utils.release_locks([context["lock"]])
del lcl
lcl = None
#orb = Orbital("Metop-"+satname)
#dtobj = datetime(int(sat_pos_time.strftime('%Y')),
# int(sat_pos_time.strftime('%m')),
# int(sat_pos_time.strftime('%d')),
# int(sat_pos_time.strftime('%H')),
# int(sat_pos_time.strftime('%M')),
# 0)
#print("---")
#print(orb.get_lonlatalt(dtobj))
#print("---")
#lonlat = orb.get_lonlatalt(dtobj)
#if lonlat[0] >= -10. and lonlat[0] <= 20. and lonlat[1] >= 40 and lonlat[1] <= 60:
granule_pass = Pass("Metop-"+satname, glbl.start_time, glbl.end_time, instrument=glbl['natural_color'].sensor)
if granule_pass.area_coverage(europe) > 0 :
print("Region over Switzerland, making CCS4 domain...")
#local_data = glbl.resample("ccs4large")
local_data = glbl.resample(europe)
local_data.save_dataset('night_fog', outDir+"METOP-"+satname+"_fog-europe_"+st+".jpg")
local_data.save_dataset('natural_color', outDir+"METOP-"+satname+"_overview-europe_"+st+".jpg")
cw.add_coastlines_to_file(outDir+"METOP-"+satname+"_fog-europe_"+st+".jpg", europe, resolution='l', level=1, outline=(255, 255, 255))
cw.add_coastlines_to_file(outDir+"METOP-"+satname+"_overview-europe_"+st+".jpg", europe, resolution='l', level=1, outline=(255, 255, 255))
cw.add_borders_to_file(outDir+"METOP-"+satname+"_fog-europe_"+st+".jpg", europe, outline=(255, 255, 255),resolution='i')
cw.add_borders_to_file(outDir+"METOP-"+satname+"_overview-europe_"+st+".jpg", europe, outline=(255, 255, 255),resolution='i')
if os.path.getsize(outDir+"METOP-"+satname+"_fog-europe_"+st+".jpg") < 170000:
os.remove(outDir+"METOP-"+satname+"_fog-europe_"+st+".jpg")
if os.path.getsize(outDir+"METOP-"+satname+"_overview-europe_"+st+".jpg") < 170000:
os.remove(outDir+"METOP-"+satname+"_overview-europe_"+st+".jpg")
def test_swath_coverage(self):
# NOAA-19 AVHRR:
tstart = datetime.strptime('20181016 03:54:13', '%Y%m%d %H:%M:%S')
tend = datetime.strptime('20181016 03:55:13', '%Y%m%d %H:%M:%S')
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr')
cov = overp.area_coverage(self.euron1)
self.assertEqual(cov, 0)
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr', frequency=80)
cov = overp.area_coverage(self.euron1)
self.assertEqual(cov, 0)
tstart = datetime.strptime('20181016 04:00:00', '%Y%m%d %H:%M:%S')
tend = datetime.strptime('20181016 04:01:00', '%Y%m%d %H:%M:%S')
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr')
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr', frequency=100)
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr/3', frequency=133)
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
overp = Pass('NOAA-19', tstart, tend, orb=self.n19orb, instrument='avhrr', frequency=300)
cov = overp.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.103526, 5)
# ASCAT and AVHRR on Metop-B:
tstart = datetime.strptime("2019-01-02T10:19:39", "%Y-%m-%dT%H:%M:%S")
tend = tstart + timedelta(seconds=180)
tle1 = '1 38771U 12049A 19002.35527803 .00000000 00000+0 21253-4 0 00017'
tle2 = '2 38771 98.7284 63.8171 0002025 96.0390 346.4075 14.21477776326431'
mypass = Pass('Metop-B', tstart, tend, instrument='ascat', tle1=tle1, tle2=tle2)
cov = mypass.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.322812, 5)
mypass = Pass('Metop-B', tstart, tend, instrument='avhrr', tle1=tle1, tle2=tle2)
cov = mypass.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.357324, 5)
tstart = datetime.strptime("2019-01-05T01:01:45", "%Y-%m-%dT%H:%M:%S")
tend = tstart + timedelta(seconds=60*15.5)
tle1 = '1 43010U 17072A 18363.54078832 -.00000045 00000-0 -79715-6 0 9999'
tle2 = '2 43010 98.6971 300.6571 0001567 143.5989 216.5282 14.19710974 58158'
mypass = Pass('FENGYUN 3D', tstart, tend, instrument='mersi2', tle1=tle1, tle2=tle2)
cov = mypass.area_coverage(self.euron1)
self.assertAlmostEqual(cov, 0.786836, 5)
def check_sunlight_coverage(job):
"""Remove products with too low/high sunlight coverage.
This plugins looks for a dictionary called `sunlight_coverage` in
the product list, with members `min` and/or `max` that define the
minimum and/or maximum allowed sunlight coverage within the scene.
The limits are expressed in % (so between 0 and 100). If the
sunlit fraction is outside the set limits, the affected products
will be discarded. It is also possible to define `check_pass:
True` in this dictionary to check the sunlit fraction within the
overpass of an polar-orbiting satellite.
"""
if get_twilight_poly is None:
LOG.error(
"Trollsched import failed, sunlight coverage calculation not possible"
)
LOG.info("Keeping all products")
return
scn_mda = _get_scene_metadata(job)
scn_mda.update(job['input_mda'])
platform_name = scn_mda['platform_name']
start_time = scn_mda['start_time']
end_time = scn_mda['end_time']
sensor = scn_mda['sensor']
if isinstance(sensor, (list, tuple, set)):
sensor = list(sensor)
if len(sensor) > 1:
LOG.warning(
"Multiple sensors given, taking only one for "
"coverage calculations: %s", sensor[0])
sensor = sensor[0]
product_list = job['product_list']
areas = list(product_list['product_list']['areas'].keys())
for area in areas:
products = list(
product_list['product_list']['areas'][area]['products'].keys())
try:
area_def = get_area_def(area)
except AreaNotFound:
area_def = None
coverage = {True: None, False: None}
overpass = None
for product in products:
prod_path = "/product_list/areas/%s/products/%s" % (area, product)
config = get_config_value(product_list, prod_path,
"sunlight_coverage")
if config is None:
continue
min_day = config.get('min')
max_day = config.get('max')
check_pass = config.get('check_pass', False)
if min_day is None and max_day is None:
LOG.debug("Sunlight coverage not configured for %s / %s",
product, area)
continue
if area_def is None:
area_def = _get_product_area_def(job, area, product)
if area_def is None:
continue
if check_pass and overpass is None:
overpass = Pass(platform_name,
start_time,
end_time,
instrument=sensor)
if coverage[check_pass] is None:
coverage[check_pass] = _get_sunlight_coverage(
area_def, start_time, overpass)
area_conf = product_list['product_list']['areas'][area]
area_conf[
'area_sunlight_coverage_percent'] = coverage[check_pass] * 100
if min_day is not None and coverage[check_pass] < (min_day /
100.0):
LOG.info("Not enough sunlight coverage for "
f"product '{product!s}', removed. Needs at least "
f"{min_day:.1f}%, got {coverage[check_pass]:.1%}.")
dpath.util.delete(product_list, prod_path)
if max_day is not None and coverage[check_pass] > (max_day /
100.0):
LOG.info("Too much sunlight coverage for "
f"product '{product!s}', removed. Needs at most "
f"{max_day:.1f}%, got {coverage[check_pass]:.1%}.")
dpath.util.delete(product_list, prod_path)
def get_scene_coverage(platform_name, start_time, end_time, sensor, area_id):
"""Get scene area coverage in percentages."""
overpass = Pass(platform_name, start_time, end_time, instrument=sensor)
area_def = get_area_def(area_id)
return 100 * overpass.area_coverage(area_def)
def collect(self, granule_metadata):
"""
Parameters:
granule_metadata : metadata
"""
# Check if input data is being waited for
if "tle_platform_name" in granule_metadata:
platform = granule_metadata['tle_platform_name']
else:
platform = granule_metadata['platform_name']
start_time = granule_metadata['start_time']
if ("end_time" not in granule_metadata
and self.granule_duration is not None):
granule_metadata["end_time"] = (granule_metadata["start_time"] +
self.granule_duration)
end_time = granule_metadata['end_time']
if start_time > end_time:
old_end_time = end_time
end_date = start_time.date()
if end_time.time() < start_time.time():
end_date += timedelta(days=1)
end_time = datetime.combine(end_date, end_time.time())
LOG.debug('Adjusted end time from %s to %s.', old_end_time,
end_time)
granule_metadata['end_time'] = end_time
LOG.debug("Adding area ID to metadata: %s", str(self.region.area_id))
granule_metadata['collection_area_id'] = self.region.area_id
self.last_file_added = False
for ptime in self.planned_granule_times:
if abs(start_time - ptime) < timedelta(seconds=3) and \
ptime not in self.granule_times:
self.granule_times.add(ptime)
self.granules.append(granule_metadata)
self.last_file_added = True
LOG.info("Added %s (%s) granule to area %s", platform,
str(start_time), self.region.area_id)
# If last granule return swath and cleanup
# if self.granule_times == self.planned_granule_times:
if self.is_swath_complete():
LOG.info("Collection finished for area: %s",
str(self.region.area_id))
return self.finish()
else:
try:
new_timeout = (max(self.planned_granule_times -
self.granule_times) +
self.granule_duration + self.timeliness)
except ValueError:
LOG.error("Calculation of new timeout failed, "
"keeping previous timeout.")
LOG.error("Planned: %s", self.planned_granule_times)
LOG.error("Received: %s", self.granule_times)
return
if new_timeout < self.timeout:
self.timeout = new_timeout
LOG.info("Adjusted timeout: %s",
self.timeout.isoformat())
return
# Get corners from input data
if self.granule_duration is None:
self.granule_duration = end_time - start_time
LOG.debug("Estimated granule duration to %s",
str(self.granule_duration))
LOG.info("Platform name %s and sensor %s: Start and end times = %s %s",
str(platform), str(granule_metadata["sensor"]),
start_time.strftime('%Y%m%d %H:%M:%S'),
end_time.strftime('%Y%m%d %H:%M:%S'))
self.sensor = granule_metadata["sensor"]
if isinstance(self.sensor, list):
self.sensor = self.sensor[0]
granule_pass = Pass(platform,
start_time,
end_time,
instrument=self.sensor)
# If file is within region, make pass prediction to know what to wait
# for
if granule_pass.area_coverage(self.region) > 0:
self.granule_times.add(start_time)
self.granules.append(granule_metadata)
self.last_file_added = True
# Computation of the predicted granules within the region
if not self.planned_granule_times:
self.planned_granule_times.add(start_time)
LOG.info("Added %s (%s) granule to area %s", platform,
str(start_time), self.region.area_id)
LOG.debug("Predicting granules covering %s",
self.region.area_id)
gr_time = start_time
while True:
gr_time += self.granule_duration
gr_pass = Pass(platform,
gr_time,
gr_time + self.granule_duration,
instrument=self.sensor)
if not gr_pass.area_coverage(self.region) > 0:
break
self.planned_granule_times.add(gr_time)
gr_time = start_time
while True:
gr_time -= self.granule_duration
gr_pass = Pass(platform,
gr_time,
gr_time + self.granule_duration,
instrument=self.sensor)
if not gr_pass.area_coverage(self.region) > 0:
break
self.planned_granule_times.add(gr_time)
LOG.info("Planned granules for %s: %s", self.region.name,
str(sorted(self.planned_granule_times)))
self.timeout = (max(self.planned_granule_times) +
self.granule_duration + self.timeliness)
LOG.info("Planned timeout for %s: %s", self.region.name,
self.timeout.isoformat())
else:
try:
LOG.debug("Granule %s is not overlapping %s",
granule_metadata["uri"], self.region.name)
except KeyError:
try:
LOG.debug(
"Granule with start and end times = %s %s "
"is not overlapping %s",
str(granule_metadata["start_time"]),
str(granule_metadata["end_time"]),
str(self.region.name))
except KeyError:
LOG.debug("Failed printing debug info...")
LOG.debug("Keys in granule_metadata = %s",
str(granule_metadata.keys()))
# If last granule return swath and cleanup
if self.is_swath_complete():
LOG.debug("Collection finished for area: %s",
str(self.region.area_id))
return self.finish()
def invoke(self, context):
"""Invoke"""
# Set locking status, default to False
self.use_lock = context.get("use_lock", False)
self.logger.debug("Locking is used in resampler: %s",
str(self.use_lock))
if self.use_lock:
self.logger.debug(
"Compositor acquires lock of previous "
"worker: %s", str(context["prev_lock"]))
utils.acquire_lock(context["prev_lock"])
glbl = context["content"]
with open(context["product_list"], "r") as fid:
product_config = yaml.load(fid)
# Handle config options
kwargs = {}
kwargs['precompute'] = context.get('precompute', False)
kwargs['mask_area'] = context.get('mask_area', True)
self.logger.debug("Setting precompute to %s and masking to %s",
str(kwargs['precompute']), str(kwargs['mask_area']))
kwargs['nprocs'] = context.get('nprocs', 1)
self.logger.debug("Using %d CPUs for resampling.", kwargs['nprocs'])
kwargs['resampler'] = context.get('proj_method', "nearest")
self.logger.debug("Using resampling method: '%s'.",
kwargs['resampler'])
try:
kwargs['cache_dir'] = context['cache_dir']
self.logger.debug("Setting projection cache dir to %s",
kwargs['cache_dir'])
except (AttributeError, KeyError):
pass
prod_list = product_config["product_list"]
# Overpass for coverage calculations
try:
metadata = glbl.attrs
except AttributeError:
metadata = glbl.info
if product_config['common'].get('coverage_check', True):
overpass = Pass(metadata['platform_name'],
metadata['start_time'],
metadata['end_time'],
instrument=metadata['sensor'][0])
else:
overpass = None
for area_id in prod_list:
# Check for area coverage
if overpass is not None:
min_coverage = prod_list[area_id].get("min_coverage", 0.0)
if not utils.covers(overpass, area_id, min_coverage,
self.logger):
continue
kwargs['radius_of_influence'] = None
try:
area_config = product_config["product_list"][area_id]
kwargs['radius_of_influence'] = \
area_config.get("srch_radius", context["radius"])
except (AttributeError, KeyError):
kwargs['radius_of_influence'] = 10000.
if kwargs['radius_of_influence'] is None:
self.logger.debug("Using default search radius.")
else:
self.logger.debug("Using search radius %d meters.",
int(kwargs['radius_of_influence']))
# Set lock if locking is used
if self.use_lock:
self.logger.debug("Resampler acquires own lock %s",
str(context["lock"]))
utils.acquire_lock(context["lock"])
# if area_id not in glbl.info["areas"]:
# utils.release_locks([context["lock"]])
# continue
if area_id == "satproj":
self.logger.info("Using satellite projection")
lcl = glbl
else:
try:
metadata = glbl.attrs
except AttributeError:
metadata = glbl.info
self.logger.info("Resampling time slot %s to area %s",
metadata["start_time"], area_id)
lcl = glbl.resample(area_id, **kwargs)
try:
metadata = lcl.attrs
except AttributeError:
metadata = lcl.info
metadata["product_config"] = product_config
metadata["area_id"] = area_id
metadata["products"] = prod_list[area_id]['products']
self.logger.debug(
"Inserting lcl (area: %s, start_time: %s) "
"to writer's queue", area_id, str(metadata["start_time"]))
context["output_queue"].put(lcl)
del lcl
lcl = None
if utils.release_locks([context["lock"]]):
self.logger.debug("Resampler releases own lock %s",
str(context["lock"]))
# Wait 1 second to ensure next worker has time to acquire the
# lock
time.sleep(1)
# Wait until the lock has been released downstream
if self.use_lock:
utils.acquire_lock(context["lock"])
utils.release_locks([context["lock"]])
# After all the items have been processed, release the lock for
# the previous step
utils.release_locks([context["prev_lock"]],
log=self.logger.debug,
log_msg="Resampler releses lock of previous " +
"worker: %s" % str(context["prev_lock"]))
