def layout(self, img, area):
"""Sets the layout in the given image for the specified area.
"""
if not self.shapes_dir:
raise ValueError("Missing path to shape files.")
layout_area = area
if layout_area is None:
raise ValueError("Area of image is None, can't add layout.")
if isinstance(layout_area, str):
layout_area = get_area_def(area)
pil_image = img.pil_image()
resolution = self._get_resolution(area)
base_layouter = Layouter(
pil_image, layout_area, self.shapes_dir, resolution)
layouter = LayouterFactory.create_layouter(
self.product_config, base_layouter)
layouter.layout()
arr = np.array(pil_image)
if len(img.channels) == 1:
img.channels[0] = np.ma.array(arr[:, :] / 255.0)
else:
for idx in range(len(img.channels)):
img.channels[idx] = np.ma.array(arr[:, :, idx] / 255.0)
def get_maximum_extent_ll(area_def_names):
'''Get maximum extend needed to produce all the defined areas
given in *area_def_names*.
'''
maximum_area_extent = [None, None, None, None]
for area in area_def_names:
area_def = get_area_def(area['definition'])
lons, lats = get_area_boundaries(area_def)
left_lon, down_lat, right_lon, up_lat = \
np.min(lons.side4), \
np.min(lats.side3), \
np.max(lons.side2), \
np.max(lats.side1)
if maximum_area_extent[0] is None:
maximum_area_extent = [left_lon, down_lat, right_lon, up_lat]
else:
if maximum_area_extent[0] > left_lon:
maximum_area_extent[0] = left_lon
if maximum_area_extent[1] > down_lat:
maximum_area_extent[1] = down_lat
if maximum_area_extent[2] < right_lon:
maximum_area_extent[2] = right_lon
if maximum_area_extent[3] < up_lat:
maximum_area_extent[3] = up_lat
return maximum_area_extent
def layout(self, img, area):
"""Sets the layout in the given image for the specified area.
"""
if not self.shapes_dir:
raise ValueError("Missing path to shape files.")
layout_area = area
if layout_area is None:
raise ValueError("Area of image is None, can't add layout.")
if isinstance(layout_area, str):
layout_area = get_area_def(area)
pil_image = img.pil_image()
resolution = self._get_resolution(area)
base_layouter = Layouter(pil_image, layout_area, self.shapes_dir,
resolution)
layouter = LayouterFactory.create_layouter(self.product_config,
base_layouter)
layouter.layout()
arr = np.array(pil_image)
if len(img.channels) == 1:
img.channels[0] = np.ma.array(arr[:, :] / 255.0)
else:
for idx in range(len(img.channels)):
img.channels[idx] = np.ma.array(arr[:, :, idx] / 255.0)
def get_geostationary_view_zen_angles(sublon, area_def_name):
"""Calculate the satellite zenith angles for geostationary satellites
providing the earth location (*sublon*)
and *area_def_name*.
Stores the result in the given *cache* parameter.
"""
area_def = get_area_def(area_def_name)
lons, lats = area_def.get_lonlats()
TWOPI = 6.28318
R = 6371.
H = 35680.
DEGRAD = 360. / TWOPI
zlon = np.ma.masked_array(lons)
zlon[zlon < 0] += 360.
zsublon = sublon
if zsublon < 0:
zsublon += 360.
diflon = abs(zlon - zsublon) / DEGRAD
diflat = abs(lats) / DEGRAD
cospsi = np.cos(diflon) * np.cos(diflat)
psi = np.arccos(cospsi)
ssqr = R * R + (R + H) * (R + H) - 2 * R * (R + H) * cospsi
s = np.sqrt(ssqr)
sinndr = (R / s) * np.sin(psi)
sinzen = ((R + H) / R) * sinndr
angzen = np.arcsin(sinzen) * DEGRAD
# store the result in the cache
return angzen
class SegmentedSwath(Satellite):
def __init__(self, area, (satname, number, variant)):
Satellite.__init__(self, (satname, number, variant))
self.area = get_area_def(area)
self.granules = []
self.planned_granules = []
self.timeout = None
def check_cosmo_area(nc_cosmo, nx, ny, area):
x = nc_cosmo.variables['y_1'][:]
y = nc_cosmo.variables['x_1'][:]
x_min_cosmo = x.min() - 500
x_max_cosmo = x.max() + 500
y_min_cosmo = y.min() - 500
y_max_cosmo = y.max() + 500
area_wanted = get_area_def(area)
x_min_wanted = area_wanted.area_extent[1]
x_max_wanted = area_wanted.area_extent[3]
y_min_wanted = area_wanted.area_extent[0]
y_max_wanted = area_wanted.area_extent[2]
x = np.sort(x)
dx_cosmo = x[1] - x[0]
y = np.sort(y)
dy_cosmo = y[1] - y[0]
if dy_cosmo != area_wanted.pixel_size_y or dx_cosmo != area_wanted.pixel_size_x:
print(
"Error: the pixel size of the wind data doesn't match with the chosen area definition"
)
quit()
if x_min_cosmo <= x_min_wanted and x_max_cosmo >= x_max_wanted and y_min_cosmo <= y_min_wanted and y_max_cosmo >= y_max_wanted:
x_min_cut = abs(x_min_cosmo - x_min_wanted) / 1000
x_max_cut = abs(x_max_cosmo - x_max_wanted) / 1000
y_min_cut = abs(y_min_cosmo - y_min_wanted) / 1000
y_max_cut = abs(y_max_cosmo - y_max_wanted) / 1000
else:
print("Error: the area chosen (" + area +
") is larger than the wind data (cosmo) area available")
return int(x_min_cut), int(x_max_cut), int(y_min_cut), int(y_max_cut)
def get_maximum_extent_ll(area_def_names):
'''Get maximum extend needed to produce all the defined areas
given in *area_def_names*.
'''
maximum_area_extent = [None, None, None, None]
for area in area_def_names:
area_def = get_area_def(area['definition'])
lons, lats = get_area_boundaries(area_def)
left_lon, down_lat, right_lon, up_lat = \
np.min(lons.side4), \
np.min(lats.side3), \
np.max(lons.side2), \
np.max(lats.side1)
if maximum_area_extent[0] is None:
maximum_area_extent = [left_lon, down_lat, right_lon, up_lat]
else:
if maximum_area_extent[0] > left_lon:
maximum_area_extent[0] = left_lon
if maximum_area_extent[1] > down_lat:
maximum_area_extent[1] = down_lat
if maximum_area_extent[2] < right_lon:
maximum_area_extent[2] = right_lon
if maximum_area_extent[3] < up_lat:
maximum_area_extent[3] = up_lat
return maximum_area_extent
def get_geostationary_view_zen_angles(sublon, area_def_name):
"""Calculate the satellite zenith angles for geostationary satellites
providing the earth location (*sublon*)
and *area_def_name*.
Stores the result in the given *cache* parameter.
"""
area_def = get_area_def(area_def_name)
lons, lats = area_def.get_lonlats()
TWOPI = 6.28318
R = 6371.
H = 35680.
DEGRAD = 360. / TWOPI
zlon = np.ma.masked_array(lons)
zlon[zlon < 0] += 360.
zsublon = sublon
if zsublon < 0:
zsublon += 360.
diflon = abs(zlon - zsublon) / DEGRAD
diflat = abs(lats) / DEGRAD
cospsi = np.cos(diflon) * np.cos(diflat)
psi = np.arccos(cospsi)
ssqr = R * R + (R + H) * (R + H) - 2 * R * (R + H) * cospsi
s = np.sqrt(ssqr)
sinndr = (R / s) * np.sin(psi)
sinzen = ((R + H) / R) * sinndr
angzen = np.arcsin(sinzen) * DEGRAD
# store the result in the cache
return angzen
def area_def_names_to_extent(area_def_names, proj4_str,
default_extent=(-5567248.07, -5570248.48,
5570248.48, 5567248.07)):
'''Convert a list of *area_def_names* to maximal area extent in destination
projection defined by *proj4_str*. *default_extent* gives the extreme
values. Default value is MSG3 extents at lat0=0.0. If a boundary of one of
the area_defs is entirely invalid, the *default_extent* is taken.
'''
if type(area_def_names) is not list:
area_def_names = [area_def_names]
maximum_extent = None
for name in area_def_names:
try:
adef = get_area_def(name)
if "proj=geos" in adef.proj4_string:
maximum_extent = update_max_extent(maximum_extent,
adef.area_extent)
continue
boundaries = adef.get_boundary_lonlats()
except pyresample.utils.AreaNotFound:
LOGGER.warning('Area definition not found ' + name)
continue
except AttributeError:
boundaries = name.get_boundary_lonlats()
if (any(boundaries[0].side1 > 1e20) or
any(boundaries[0].side2 > 1e20) or
any(boundaries[0].side3 > 1e20) or
any(boundaries[0].side4 > 1e20)):
if default_extent:
maximum_extent = list(default_extent)
continue
else:
return None
lon_sides = (boundaries[0].side1, boundaries[0].side2,
boundaries[0].side3, boundaries[0].side4)
lat_sides = (boundaries[1].side1, boundaries[1].side2,
boundaries[1].side3, boundaries[1].side4)
maximum_extent = boundaries_to_extent(proj4_str, maximum_extent,
default_extent,
lon_sides, lat_sides)
if not maximum_extent:
return None
maximum_extent = list(maximum_extent)
maximum_extent[0] -= 10000
maximum_extent[1] -= 10000
maximum_extent[2] += 10000
maximum_extent[3] += 10000
return maximum_extent
def setup(decoder):
"""Setup the granule triggerer.
"""
granule_triggers = []
for section in CONFIG.sections():
regions = [
get_area_def(region)
for region in CONFIG.get(section, "regions").split()
]
timeliness = timedelta(minutes=CONFIG.getint(section, "timeliness"))
try:
duration = timedelta(seconds=CONFIG.getfloat(section, "duration"))
except NoOptionError:
duration = None
collectors = [
region_collector.RegionCollector(region, timeliness, duration)
for region in regions
]
try:
observer_class = CONFIG.get(section, "watcher")
pattern = CONFIG.get(section, "pattern")
parser = Parser(pattern)
glob = parser.globify()
except NoOptionError:
observer_class = None
try:
publish_topic = CONFIG.get(section, "publish_topic")
except NoOptionError:
publish_topic = None
if observer_class in ["PollingObserver", "Observer"]:
LOGGER.debug("Using %s for %s", observer_class, section)
granule_trigger = \
trigger.WatchDogTrigger(collectors,
terminator,
decoder,
[glob],
observer_class,
publish_topic=publish_topic)
else:
LOGGER.debug("Using posttroll for %s", section)
granule_trigger = trigger.PostTrollTrigger(
collectors,
terminator,
CONFIG.get(section, 'service').split(','),
CONFIG.get(section, 'topics').split(','),
publish_topic=publish_topic)
granule_triggers.append(granule_trigger)
return granule_triggers
def generic_covers(scene, area_item):
area_def = get_area_def(area_item.attrib['id'])
min_coverage = float(area_item.attrib.get('min_coverage', 0))
min_coverage /= 100.0
cov = coverage(scene, area_def)
if cov <= min_coverage:
LOGGER.info("Coverage too small %.1f%% (out of %.1f%%) with %s",
cov * 100, min_coverage * 100, area_item.attrib['name'])
return False
else:
LOGGER.info("Coverage %.1f%% with %s", cov * 100,
area_item.attrib['name'])
return True
def add_overlay_config(self, config_file):
"""Add overlay to image parsing a configuration file.
"""
import ConfigParser
conf = ConfigParser.ConfigParser()
conf.read(os.path.join(CONFIG_PATH, "mpop.cfg"))
coast_dir = conf.get('shapes', 'dir')
logger.debug("Getting area for overlay: " + str(self.area.area_id))
try:
import aggdraw
from pycoast import ContourWriterAGG
cw_ = ContourWriterAGG(coast_dir)
except ImportError:
logger.warning("AGGdraw lib not installed...width and opacity properties are not available for overlays.")
from pycoast import ContourWriter
cw_ = ContourWriter(coast_dir)
logger.debug("Getting area for overlay: " + str(self.area))
if self.area is None:
raise ValueError("Area of image is None, can't add overlay.")
if self.mode != "RGB":
self.convert("RGB")
img = self.pil_image()
from mpop.projector import get_area_def
if isinstance(self.area, str):
self.area = get_area_def(self.area)
logger.info("Add overlays to image.")
logger.debug("Area = " + str(self.area.area_id))
foreground=cw_.add_overlay_from_config(config_file, self.area)
img.paste(foreground,mask=foreground.split()[-1])
arr = np.array(img)
if len(self.channels) == 1:
self.channels[0] = np.ma.array(arr[:, :] / 255.0)
else:
for idx in range(len(self.channels)):
self.channels[idx] = np.ma.array(arr[:, :, idx] / 255.0)
def setup(decoder):
"""Setup the granule triggerer.
"""
granule_triggers = []
for section in CONFIG.sections():
regions = [get_area_def(region)
for region in CONFIG.get(section, "regions").split()]
timeliness = timedelta(minutes=CONFIG.getint(section, "timeliness"))
try:
duration = timedelta(seconds=CONFIG.getfloat(section, "duration"))
except NoOptionError:
duration = None
collectors = [region_collector.RegionCollector(region, timeliness, duration)
for region in regions]
try:
observer_class = CONFIG.get(section, "watcher")
pattern = CONFIG.get(section, "pattern")
parser = Parser(pattern)
glob = parser.globify()
except NoOptionError:
observer_class = None
try:
publish_topic = CONFIG.get(section, "publish_topic")
except NoOptionError:
publish_topic = None
if observer_class in ["PollingObserver", "Observer"]:
LOGGER.debug("Using %s for %s", observer_class, section)
granule_trigger = \
trigger.WatchDogTrigger(collectors,
terminator,
decoder,
[glob],
observer_class,
publish_topic=publish_topic)
else:
LOGGER.debug("Using posttroll for %s", section)
granule_trigger = trigger.PostTrollTrigger(
collectors, terminator,
CONFIG.get(section, 'service').split(','),
CONFIG.get(section, 'topics').split(','),
publish_topic=publish_topic)
granule_triggers.append(granule_trigger)
return granule_triggers
def add_overlay_config(self, config_file):
"""Add overlay to image parsing a configuration file.
"""
import ConfigParser
conf = ConfigParser.ConfigParser()
conf.read(os.path.join(CONFIG_PATH, "mpop.cfg"))
coast_dir = conf.get('shapes', 'dir')
logger.debug("Getting area for overlay: " + str(self.area.area_id))
try:
import aggdraw
from pycoast import ContourWriterAGG
cw_ = ContourWriterAGG(coast_dir)
except ImportError:
logger.warning(
"AGGdraw lib not installed...width and opacity properties are not available for overlays."
)
from pycoast import ContourWriter
cw_ = ContourWriter(coast_dir)
logger.debug("Getting area for overlay: " + str(self.area))
if self.area is None:
raise ValueError("Area of image is None, can't add overlay.")
if self.mode != "RGB":
self.convert("RGB")
img = self.pil_image()
from mpop.projector import get_area_def
if isinstance(self.area, str):
self.area = get_area_def(self.area)
logger.info("Add overlays to image.")
logger.debug("Area = " + str(self.area.area_id))
foreground = cw_.add_overlay_from_config(config_file, self.area)
img.paste(foreground, mask=foreground.split()[-1])
arr = np.array(img)
if len(self.channels) == 1:
self.channels[0] = np.ma.array(arr[:, :] / 255.0)
else:
for idx in range(len(self.channels)):
self.channels[idx] = np.ma.array(arr[:, :, idx] / 255.0)
def generic_covers(scene, area_item):
area_def = get_area_def(area_item.attrib['id'])
min_coverage = float(
area_item.attrib.get('min_coverage', 0))
min_coverage /= 100.0
cov = coverage(scene, area_def)
if cov <= min_coverage:
LOGGER.info("Coverage too small %.1f%% (out of %.1f%%) with %s",
cov * 100, min_coverage * 100,
area_item.attrib['name'])
return False
else:
LOGGER.info("Coverage %.1f%% with %s",
cov * 100, area_item.attrib['name'])
return True
def create_world_composite(msg, proc_func_params):
"""
Creates a world composite images out of an dataset message
"""
items = []
for elem in msg.data['dataset']:
url = urlparse(elem['uri'])
if url.netloc != '':
LOGGER.error('uri not supported: %s',
format(elem['uri']))
return None
area = get_area_def(msg.data['area']['name'])
t_gatherer = msg.data['gatherer_time']
if not isinstance(t_gatherer, datetime):
try:
t_gatherer = datetime.strptime(
t_gatherer, '%Y%m%d%H%M%S')
except:
t_gatherer = None
items.append((url.path, area, t_gatherer))
lon_limits = {}
erosion_size = None
smooth_width = None
if proc_func_params:
# order images
if 'order' in proc_func_params:
order_list = proc_func_params['order'].split('|')
sort_key = partial(_match_order_index, order_list)
items = sorted(items, key=sort_key, reverse=True)
if 'lon_limits' in proc_func_params:
sat_lon_list = proc_func_params['lon_limits'].split('|')
for sat_lon in sat_lon_list:
sat, min_lon, max_lon = sat_lon.split(',')
lon_limits[sat] = (float(min_lon), float(max_lon))
if 'erosion_size' in proc_func_params:
erosion_size = float(proc_func_params['erosion_size'])
if 'smooth_width' in proc_func_params:
smooth_width = float(proc_func_params['smooth_width'])
return _create_world_composite(items, lon_limits=lon_limits,
erosion_size=erosion_size,
smooth_width=smooth_width)
def read_dem():
"""Read digital elevation model (DEM) in netCDF format.
"""
filename = "/data/COALITION2/database/topography/GTOPO30/dtm_acquire_ccs4_float_v3.nc"
print("... read from file: ", filename)
# Load data from netCDF file
ds = Dataset(filename, 'r')
data = ds.variables[chn_name][:, :]
print(type(data))
print(type(ma.asarray(data)))
area_def = get_area_def("ccs4")
def create_world_composite(msg, proc_func_params):
"""
Creates a world composite images out of an dataset message
"""
items = []
for elem in msg.data['dataset']:
url = urlparse(elem['uri'])
if url.netloc != '':
LOGGER.error('uri not supported: %s', format(elem['uri']))
return None
area = get_area_def(msg.data['area']['name'])
t_gatherer = msg.data['gatherer_time']
if not isinstance(t_gatherer, datetime):
try:
t_gatherer = datetime.strptime(t_gatherer, '%Y%m%d%H%M%S')
except:
t_gatherer = None
items.append((url.path, area, t_gatherer))
lon_limits = {}
erosion_size = None
smooth_width = None
if proc_func_params:
# order images
if 'order' in proc_func_params:
order_list = proc_func_params['order'].split('|')
sort_key = partial(_match_order_index, order_list)
items = sorted(items, key=sort_key, reverse=True)
if 'lon_limits' in proc_func_params:
sat_lon_list = proc_func_params['lon_limits'].split('|')
for sat_lon in sat_lon_list:
sat, min_lon, max_lon = sat_lon.split(',')
lon_limits[sat] = (float(min_lon), float(max_lon))
if 'erosion_size' in proc_func_params:
erosion_size = float(proc_func_params['erosion_size'])
if 'smooth_width' in proc_func_params:
smooth_width = float(proc_func_params['smooth_width'])
return _create_world_composite(items,
lon_limits=lon_limits,
erosion_size=erosion_size,
smooth_width=smooth_width)
def get_view_zen_angles(sat_name, tle_filename, area_def_name, time_slot):
"""Calculate the satellite zenith angles for the given satellite
(*sat_name*, *tle_filename*), *area_def_name* and *time slot*.
Stores the result in the given *cache* parameter.
"""
try:
from pyorbital.orbital import Orbital
except ImportError:
LOGGER.warning("Could not load pyorbital modules")
return
area_def = get_area_def(area_def_name)
lons, lats = area_def.get_lonlats()
orbital_obj = Orbital(sat_name, tle_filename)
elevation = orbital_obj.get_observer_look(time_slot, lons, lats, 0)[1]
view_zen_data = np.subtract(90, np.ma.masked_outside(elevation, 0, 90))
return view_zen_data
def get_view_zen_angles(sat_name, tle_filename, area_def_name,
time_slot):
"""Calculate the satellite zenith angles for the given satellite
(*sat_name*, *tle_filename*), *area_def_name* and *time slot*.
Stores the result in the given *cache* parameter.
"""
try:
from pyorbital.orbital import Orbital
except ImportError:
LOGGER.warning("Could not load pyorbital modules")
return
area_def = get_area_def(area_def_name)
lons, lats = area_def.get_lonlats()
orbital_obj = Orbital(sat_name, tle_filename)
elevation = orbital_obj.get_observer_look(time_slot, lons, lats, 0)[1]
view_zen_data = np.subtract(90, np.ma.masked_outside(elevation, 0, 90))
return view_zen_data
def get_maximum_extent(self):
'''Get maximum extend needed to produce all defined areas.
'''
self.maximum_area_extent = [None, None, None, None]
for area in self.area_def_names:
extent = get_area_def(area)
if self.maximum_area_extent[0] is None:
self.maximum_area_extent = list(extent.area_extent)
else:
if self.maximum_area_extent[0] > extent.area_extent[0]:
self.maximum_area_extent[0] = extent.area_extent[0]
if self.maximum_area_extent[1] > extent.area_extent[1]:
self.maximum_area_extent[1] = extent.area_extent[1]
if self.maximum_area_extent[2] < extent.area_extent[2]:
self.maximum_area_extent[2] = extent.area_extent[2]
if self.maximum_area_extent[3] < extent.area_extent[3]:
self.maximum_area_extent[3] = extent.area_extent[3]
def get_maximum_extent(self):
'''Get maximum extend needed to produce all defined areas.
'''
self.maximum_area_extent = [None, None, None, None]
for area in self.area_def_names:
extent = get_area_def(area)
if self.maximum_area_extent[0] is None:
self.maximum_area_extent = list(extent.area_extent)
else:
if self.maximum_area_extent[0] > extent.area_extent[0]:
self.maximum_area_extent[0] = extent.area_extent[0]
if self.maximum_area_extent[1] > extent.area_extent[1]:
self.maximum_area_extent[1] = extent.area_extent[1]
if self.maximum_area_extent[2] < extent.area_extent[2]:
self.maximum_area_extent[2] = extent.area_extent[2]
if self.maximum_area_extent[3] < extent.area_extent[3]:
self.maximum_area_extent[3] = extent.area_extent[3]
def covers(overpass, area_item):
try:
area_def = get_area_def(area_item.attrib['id'])
min_coverage = float(area_item.attrib.get('min_coverage', 0))
min_coverage /= 100.0
coverage = overpass.area_coverage(area_def)
if coverage <= min_coverage:
LOGGER.info("Coverage too small %.1f%% (out of %.1f%%) with %s",
coverage * 100, min_coverage * 100,
area_item.attrib['name'])
return False
else:
LOGGER.info("Coverage %.1f%% with %s", coverage * 100,
area_item.attrib['name'])
except AttributeError:
LOGGER.warning("Can't compute area coverage with %s!",
area_item.attrib['name'])
return True
def get_maximum_extent(area_def_names):
"""Get maximum extend needed to produce all defined areas.
"""
maximum_area_extent = [None, None, None, None]
for area in area_def_names:
extent = get_area_def(area)
if maximum_area_extent[0] is None:
maximum_area_extent = list(extent.area_extent)
else:
if maximum_area_extent[0] > extent.area_extent[0]:
maximum_area_extent[0] = extent.area_extent[0]
if maximum_area_extent[1] > extent.area_extent[1]:
maximum_area_extent[1] = extent.area_extent[1]
if maximum_area_extent[2] < extent.area_extent[2]:
maximum_area_extent[2] = extent.area_extent[2]
if maximum_area_extent[3] < extent.area_extent[3]:
maximum_area_extent[3] = extent.area_extent[3]
return maximum_area_extent
def covers(overpass, area_item):
try:
area_def = get_area_def(area_item.attrib['id'])
min_coverage = float(
area_item.attrib.get('min_coverage', 0))
min_coverage /= 100.0
coverage = overpass.area_coverage(area_def)
if coverage <= min_coverage:
LOGGER.info("Coverage too small %.1f%% (out of %.1f%%) with %s",
coverage * 100, min_coverage * 100,
area_item.attrib['name'])
return False
else:
LOGGER.info("Coverage %.1f%% with %s",
coverage * 100, area_item.attrib['name'])
except AttributeError:
LOGGER.warning("Can't compute area coverage with %s!",
area_item.attrib['name'])
return True
def load_rgb(satellite, satellite_nr, satellites_name, time_slot, rgb, area,
in_msg, data_CTP):
if rgb != 'CTP':
# read the data we would like to forecast
global_data_RGBforecast = GeostationaryFactory.create_scene(
satellite, satellite_nr, satellites_name, time_slot)
#global_data_RGBforecast = GeostationaryFactory.create_scene(in_msg.sat, str(10), "seviri", time_slot)
# area we would like to read
area_loaded = get_area_def(
"EuropeCanary95") #(in_windshift.areaExtraction)
# load product, global_data is changed in this step!
area_loaded = load_products(global_data_RGBforecast, [rgb], in_msg,
area_loaded)
print '... project data to desired area ', area
fns = global_data_RGBforecast.project(area)
else:
fns = deepcopy(data_CTP["CTP"].data)
return fns[rgb].data
def area_def_names_to_extent(area_def_names, proj4_str,
default_extent=(-5567248.07, -5570248.48,
5570248.48, 5567248.07)):
'''Convert a list of *area_def_names* to maximal area extent in
destination projection defined by *proj4_str*. *default_extent*
gives the extreme values. Default value is MSG3 extents at
lat0=0.0.
'''
if type(area_def_names) is not list:
area_def_names = [area_def_names]
maximum_extent = None
for name in area_def_names:
try:
boundaries = get_area_def(name).get_boundary_lonlats()
except pyresample.utils.AreaNotFound:
LOGGER.warning('Area definition not found ' + name)
continue
except AttributeError:
boundaries = name.get_boundary_lonlats()
lon_sides = (boundaries[0].side1, boundaries[0].side2,
boundaries[0].side3, boundaries[0].side4)
lat_sides = (boundaries[1].side1, boundaries[1].side2,
boundaries[1].side3, boundaries[1].side4)
maximum_extent = boundaries_to_extent(proj4_str, maximum_extent,
default_extent,
lon_sides, lat_sides)
maximum_extent[0] -= 10000
maximum_extent[1] -= 10000
maximum_extent[2] += 10000
maximum_extent[3] += 10000
return maximum_extent
def figure_labels(labels, outputFile, timeObs, dt, area_plot="ccs4", add_name = None, verbose=True):
if verbose:
print("*** produce label figures")
yearS, monthS, dayS, hourS, minS = string_date(timeObs)
data_time = timeObs + timedelta(minutes = dt)
yearSf, monthSf, daySf, hourSf, minSf = string_date(data_time)
labels = np.flipud(labels)
obj_area = get_area_def(area_plot)
fig, ax = prepare_figure(obj_area)
plt.contour(labels,[0.5],colors='y')
#plt.imshow(labels, origin="lower")
PIL_image = fig2img ( fig )
if add_name is not None:
PIL_image.save(create_dir(outputFile)+"Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+"_Forc"+hourSf+minSf+add_name+".png")
path = (outputFile)+"Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+"_Forc"+hourSf+minSf+add_name+".png"
else:
PIL_image.save(create_dir(outputFile)+"Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+"_Forc"+hourSf+minSf+".png")
path = (outputFile)+"Forecast"+yearS+monthS+dayS+"_Obs"+hourS+minS+"_Forc"+hourSf+minSf+".png"
print("... display ",path," &")
plt.close( fig)
month = time_slot.month
day = time_slot.day
hour = time_slot.hour
minute = time_slot.minute
yearS = str(year)
#yearS = yearS[2:]
monthS = "%02d" % month
dayS = "%02d" % day
hourS = "%02d" % hour
minS = "%02d" % minute
dateS = yearS + '-' + monthS + '-' + dayS
timeS = hourS + ':' + minS + " UTC"
# define area object
obj_area = get_area_def(area) #(in_windshift.ObjArea)
size_x = obj_area.pixel_size_x
size_y = obj_area.pixel_size_y
# define area
proj4_string = obj_area.proj4_string
# e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
area_extent = obj_area.area_extent
# e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
area_tuple = (proj4_string, area_extent)
# read CTP to distinguish high, medium and low clouds
global_data_CTP = GeostationaryFactory.create_scene(
in_msg.sat,
str(in_msg.sat_nr).zfill(2), "seviri", time_slot)
#global_data_CTP = GeostationaryFactory.create_scene(in_msg.sat, str(10), "seviri", time_slot)
def _create_world_composite(items,
lon_limits=None,
erosion_size=20,
smooth_width=20):
# smooth_sigma = 4
img = None
for (path, area, timeslot) in items:
if not isinstance(area, AreaDefinition):
area = get_area_def(area)
next_img = read_image(path, area, timeslot)
if img is None:
img = next_img
else:
# scaled_smooth_sigma = smooth_sigma * (float(img.width) / 1000.0)
img_mask = reduce(np.ma.mask_or,
[chn.mask for chn in img.channels])
next_img_mask = reduce(np.ma.mask_or,
[chn.mask for chn in next_img.channels])
# Mask overlapping areas away
if lon_limits:
for sat in lon_limits:
if sat in path:
mask_limits = calc_pixel_mask_limits(
area, lon_limits[sat])
for lim in mask_limits:
next_img_mask[:, lim[0]:lim[1]] = 1
break
alpha = np.ones(next_img_mask.shape, dtype='float')
alpha[next_img_mask] = 0.0
if erosion_size is not None and smooth_width is not None:
scaled_erosion_size = erosion_size * (float(img.width) /
1000.0)
scaled_smooth_width = smooth_width * (float(img.width) /
1000.0)
# smooth_alpha = ndi.gaussian_filter(
# ndi.grey_erosion(alpha, size=(scaled_erosion_size,
# scaled_erosion_size)),
# scaled_smooth_sigma)
smooth_alpha = ndi.uniform_filter(
ndi.grey_erosion(alpha,
size=(scaled_erosion_size,
scaled_erosion_size)),
scaled_smooth_width)
smooth_alpha[img_mask] = alpha[img_mask]
else:
smooth_alpha = alpha
for i in range(0, min(len(img.channels), len(next_img.channels))):
chdata = next_img.channels[i].data * smooth_alpha + \
img.channels[i].data * (1 - smooth_alpha)
chmask = np.logical_and(img_mask, next_img_mask)
img.channels[i] = \
np.ma.masked_where(chmask, chdata)
return img
import sys, string, os
from subprocess import call
from PIL import Image, ImageFont, ImageDraw
import cll_composites
year = int(sys.argv[1])
month = int(sys.argv[2])
day = int(sys.argv[3])
hour = int(sys.argv[4])
min = int(sys.argv[5])
outDir = sys.argv[6] + '/'
time_slot = datetime.datetime(year, month, day, hour, min)
#global_data = GeostationaryFactory.create_scene("meteosat", "11", "seviri", time_slot)
global_data = GeostationaryFactory.create_scene("meteosat", "11", "seviri", time_slot)
europe = get_area_def("EuropeCanary")
global_data.load([0.7,6.2,10.8], area_extent=europe.area_extent)
local_data = global_data.project("pifn")
local_data_hrv = global_data.project("pifh")
img_webproduct = local_data.image.webproduct()
img_pifir = local_data.image.pifir()
img_pifwv = local_data_hrv.image.pifwv()
img_pifhrv = local_data_hrv.image.pifhrv()
yearS = str(year)
yearS = yearS[2:]
monthS = "%02d" % month
dayS = "%02d" % day
hourS = "%02d" % hour
minS = "%02d" % min
def _create_world_composite(items, lon_limits=None,
erosion_size=20,
smooth_width=20):
# smooth_sigma = 4
img = None
for (path, area, timeslot) in items:
if not isinstance(area, AreaDefinition):
area = get_area_def(area)
next_img = read_image(path, area, timeslot)
if img is None:
img = next_img
else:
# scaled_smooth_sigma = smooth_sigma * (float(img.width) / 1000.0)
img_mask = reduce(np.ma.mask_or,
[chn.mask for chn in img.channels])
next_img_mask = reduce(np.ma.mask_or,
[chn.mask for chn in next_img.channels])
# Mask overlapping areas away
if lon_limits:
for sat in lon_limits:
if sat in path:
mask_limits = calc_pixel_mask_limits(area,
lon_limits[sat])
for lim in mask_limits:
next_img_mask[:, lim[0]:lim[1]] = 1
break
alpha = np.ones(next_img_mask.shape, dtype='float')
alpha[next_img_mask] = 0.0
if erosion_size is not None and smooth_width is not None:
scaled_erosion_size = erosion_size * (float(img.width) /
1000.0)
scaled_smooth_width = smooth_width * (float(img.width) /
1000.0)
# smooth_alpha = ndi.gaussian_filter(
# ndi.grey_erosion(alpha, size=(scaled_erosion_size,
# scaled_erosion_size)),
# scaled_smooth_sigma)
smooth_alpha = ndi.uniform_filter(
ndi.grey_erosion(alpha, size=(scaled_erosion_size,
scaled_erosion_size)),
scaled_smooth_width)
smooth_alpha[img_mask] = alpha[img_mask]
else:
smooth_alpha = alpha
for i in range(0, min(len(img.channels), len(next_img.channels))):
chdata = next_img.channels[i].data * smooth_alpha + \
img.channels[i].data * (1 - smooth_alpha)
chmask = np.logical_and(img_mask, next_img_mask)
img.channels[i] = \
np.ma.masked_where(chmask, chdata)
return img
loglevel = logging.INFO
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
logger = logging.getLogger("gatherer")
decoder = get_metadata
granule_triggers = []
pub = publisher.NoisyPublisher("gatherer")
# TODO: get this from the product config files
regions = [
get_area_def(region)
for region in config.get("default", "regions").split()
]
for section in config.sections():
if section == "default":
continue
timeliness = timedelta(minutes=config.getint(section, "timeliness"))
try:
duration = timedelta(seconds=config.getfloat(section, "duration"))
except NoOptionError:
duration = None
collectors = [
region_collector.RegionCollector(region, timeliness, duration)
for region in regions
#print "global_data[prop_str].product_name=",global_data[prop_str].product_name
#area='odyssey'
area = 'odysseyS25'
reproject = True
if reproject:
print('-------------------')
print("start projection")
# PROJECT data to new area
data = global_data.project(area, precompute=True)
#data[prop_str].product_name = global_data[prop_str].product_name
#data[prop_str].units = global_data[prop_str].units
global_data = data
obj_area = get_area_def(area)
outputFile = outputDir + 'ODY_' + prop_str + '-' + area + '_' + yearS[
2:] + monthS + dayS + hourS + minS + '.png'
# define area
print('-------------------')
print('obj_area ', obj_area)
proj4_string = obj_area.proj4_string
# e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
print('proj4_string ', proj4_string)
area_extent = obj_area.area_extent
# e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
area_def = (proj4_string, area_extent)
print('-------------------')
print('area_def ', area_def)
def load_generic(satscene, options, calibrate=True, area_extent=None,
area_def_names=None):
"""Read imager data from file and load it into *satscene*.
"""
del options
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOGGER.debug("Channels to load from %s: %s" % (satscene.instrument_name,
satscene.channels_to_load))
# Compulsory global attributes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
from_area = False
if area_extent is None and satscene.area is not None:
if not satscene.area_def:
satscene.area = get_area_def(satscene.area_id)
area_extent = satscene.area.area_extent
from_area = True
area_converted_to_extent = False
for chn in satscene.channels_to_load:
if from_area:
try:
metadata = xrit.sat.load(satscene.fullname, satscene.time_slot,
chn, only_metadata=True)
if(satscene.area_def.proj_dict["proj"] != "geos" or
float(satscene.area_def.proj_dict["lon_0"]) !=
metadata.sublon):
raise ValueError("Slicing area must be in "
"geos projection, and lon_0 should match "
"the satellite's position.")
except ReaderError, err:
# if channel can't be found, go on with next channel
LOGGER.error(str(err))
continue
# Convert area definitions to maximal area_extent
if not area_converted_to_extent and area_def_names is not None:
try:
metadata = xrit.sat.load(satscene.fullname, satscene.time_slot,
chn, only_metadata=True)
except ReaderError as err:
LOGGER.warning(str(err))
continue
# if area_extent is given, assume it gives the maximum
# extent of the satellite view
if area_extent is not None:
area_extent = area_def_names_to_extent(area_def_names,
metadata.proj4_params,
area_extent)
# otherwise use the default value (MSG3 extent at
# lon0=0.0), that is, do not pass default_extent=area_extent
else:
area_extent = area_def_names_to_extent(area_def_names,
metadata.proj4_params)
area_converted_to_extent = True
try:
image = xrit.sat.load(satscene.fullname,
satscene.time_slot,
chn,
mask=True,
calibrate=calibrate)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOGGER.warning(
"Loading non calibrated data since calibration failed.")
image = xrit.sat.load(satscene.fullname,
satscene.time_slot,
chn,
mask=True,
calibrate=False)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except ReaderError as err:
# if channel can't be found, go on with next channel
LOGGER.warning(str(err))
continue
satscene[chn] = data
satscene[chn].info['units'] = metadata.calibration_unit
satscene[chn].info['satname'] = satscene.satname
satscene[chn].info['satnumber'] = satscene.number
satscene[chn].info['instrument_name'] = satscene.instrument_name
satscene[chn].info['time'] = satscene.time_slot
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split(" ")
proj_dict = {}
for param in proj_params:
key, val = param.split("=")
proj_dict[key] = val
if IS_PYRESAMPLE_LOADED:
# Build area_def on-the-fly
satscene[chn].area = geometry.AreaDefinition(
satscene.satname + satscene.instrument_name +
str(metadata.area_extent) +
str(data.shape),
"On-the-fly area",
proj_dict["proj"],
proj_dict,
data.shape[1],
data.shape[0],
metadata.area_extent)
else:
LOGGER.info("Could not build area, pyresample missing...")
def geotiff_save(self, filename, compression=6,
tags=None, gdal_options=None,
blocksize=0, geotransform=None,
spatialref=None, floating_point=False,
writer_options=None):
"""Save the image to the given *filename* in geotiff_ format, with the
*compression* level in [0, 9]. 0 means not compressed. The *tags*
argument is a dict of tags to include in the image (as metadata). By
default it uses the 'area' instance to generate geotransform and
spatialref information, this can be overwritten by the arguments
*geotransform* and *spatialref*. *floating_point* allows the saving of
'L' mode images in floating point format if set to True.
When argument *writer_options* is not none and entry 'fill_value_subst'
is included, its numeric value will be used to substitute image data
that would be equal to the fill_value (used to replace masked data).
.. _geotiff: http://trac.osgeo.org/geotiff/
"""
from osgeo import gdal, osr
raster = gdal.GetDriverByName("GTiff")
tags = tags or {}
writer_options = writer_options or {}
if floating_point:
if self.mode != "L":
raise ValueError("Image must be in 'L' mode for floating point"
" geotif saving")
if self.fill_value is None:
logger.warning("Image with floats cannot be transparent, "
"so setting fill_value to 0")
self.fill_value = 0
channels = [self.channels[0].astype(np.float64)]
fill_value = self.fill_value or [0]
gformat = gdal.GDT_Float64
opacity = 0
else:
nbits = int(tags.get("NBITS", "8"))
if nbits > 16:
dtype = np.uint32
gformat = gdal.GDT_UInt32
elif nbits > 8:
dtype = np.uint16
gformat = gdal.GDT_UInt16
else:
dtype = np.uint8
gformat = gdal.GDT_Byte
opacity = np.iinfo(dtype).max
channels, fill_value = self._finalize(dtype)
fill_value_subst = writer_options.get(
write_opts.WR_OPT_FILL_VALUE_SUBST, None)
if fill_value is not None and fill_value_subst is not None:
for i, chan in enumerate(channels):
np.place(chan, chan == fill_value[i], int(fill_value_subst))
logger.debug("Saving to GeoTiff.")
if tags is not None:
self.tags.update(tags)
if gdal_options is not None:
self.gdal_options.update(gdal_options)
g_opts = ["=".join(i) for i in self.gdal_options.items()]
if compression != 0:
g_opts.append("COMPRESS=DEFLATE")
g_opts.append("ZLEVEL=" + str(compression))
if blocksize != 0:
g_opts.append("TILED=YES")
g_opts.append("BLOCKXSIZE=" + str(blocksize))
g_opts.append("BLOCKYSIZE=" + str(blocksize))
if(self.mode == "L"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
1,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels,
opacity, fill_value)
elif(self.mode == "LA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds,
channels[:-1], channels[1],
fill_value)
elif(self.mode == "RGB"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
3,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels,
opacity, fill_value)
elif(self.mode == "RGBA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds,
channels[:-1], channels[3],
fill_value)
else:
raise NotImplementedError("Saving to GeoTIFF using image mode"
" %s is not implemented." % self.mode)
# Create raster GeoTransform based on upper left corner and pixel
# resolution ... if not overwritten by argument geotransform.
if geotransform:
dst_ds.SetGeoTransform(geotransform)
if spatialref:
if not isinstance(spatialref, str):
spatialref = spatialref.ExportToWkt()
dst_ds.SetProjection(spatialref)
else:
from pyresample import utils
from mpop.projector import get_area_def
try:
area = get_area_def(self.area)
except (utils.AreaNotFound, AttributeError):
area = self.area
try:
adfgeotransform = [area.area_extent[0], area.pixel_size_x, 0,
area.area_extent[3], 0, -area.pixel_size_y]
dst_ds.SetGeoTransform(adfgeotransform)
srs = osr.SpatialReference()
srs.ImportFromProj4(area.proj4_string)
srs.SetProjCS(area.proj_id)
try:
srs.SetWellKnownGeogCS(area.proj_dict['ellps'])
except KeyError:
pass
try:
# Check for epsg code.
srs.ImportFromEPSG(int(area.proj_dict['init'].
lower().split('epsg:')[1]))
except (KeyError, IndexError):
pass
srs = srs.ExportToWkt()
dst_ds.SetProjection(srs)
except AttributeError:
logger.exception("Could not load geographic data, invalid area")
self.tags.update({'TIFFTAG_DATETIME':
self.time_slot.strftime("%Y:%m:%d %H:%M:%S")})
dst_ds.SetMetadata(self.tags, '')
# Close the dataset
dst_ds = None
def read(self, filename):
"""Reader for the NWCSAF/MSG cloudtype. Use *filename* to read data.
"""
import tables
self.cloudtype = MsgCloudTypeData()
self.processing_flags = MsgCloudTypeData()
self.cloudphase = MsgCloudTypeData()
h5f = tables.openFile(filename)
# pylint: disable-msg=W0212
self.package = h5f.root._v_attrs["PACKAGE"]
self.saf = h5f.root._v_attrs["SAF"]
self.product_name = h5f.root._v_attrs["PRODUCT_NAME"]
self.num_of_columns = h5f.root._v_attrs["NC"]
self.num_of_lines = h5f.root._v_attrs["NL"]
self.projection_name = h5f.root._v_attrs["PROJECTION_NAME"]
self.region_name = h5f.root._v_attrs["REGION_NAME"]
self.cfac = h5f.root._v_attrs["CFAC"]
self.lfac = h5f.root._v_attrs["LFAC"]
self.coff = h5f.root._v_attrs["COFF"]
self.loff = h5f.root._v_attrs["LOFF"]
self.nb_param = h5f.root._v_attrs["NB_PARAMETERS"]
self.gp_sc_id = h5f.root._v_attrs["GP_SC_ID"]
self.image_acquisition_time = h5f.root._v_attrs["IMAGE_ACQUISITION_TIME"]
self.spectral_channel_id = h5f.root._v_attrs["SPECTRAL_CHANNEL_ID"]
self.nominal_product_time = h5f.root._v_attrs["NOMINAL_PRODUCT_TIME"]
self.sgs_product_quality = h5f.root._v_attrs["SGS_PRODUCT_QUALITY"]
self.sgs_product_completeness = h5f.root._v_attrs["SGS_PRODUCT_COMPLETENESS"]
self.product_algorithm_version = h5f.root._v_attrs["PRODUCT_ALGORITHM_VERSION"]
# pylint: enable-msg=W0212
# ------------------------
# The cloudtype data
self.cloudtype.data = h5f.root.CT[:, :]
self.cloudtype.scaling_factor = h5f.root.CT.attrs["SCALING_FACTOR"]
self.cloudtype.offset = h5f.root.CT.attrs["OFFSET"]
self.cloudtype.num_of_lines = h5f.root.CT.attrs["N_LINES"]
self.cloudtype.num_of_columns = h5f.root.CT.attrs["N_COLS"]
self.shape = (self.cloudtype.num_of_lines,
self.cloudtype.num_of_columns)
self.cloudtype.product = h5f.root.CT.attrs["PRODUCT"]
self.cloudtype.id = h5f.root.CT.attrs["ID"]
# ------------------------
# The cloud phase data
self.cloudphase.data = h5f.root.CT_PHASE[:, :]
self.cloudphase.scaling_factor = h5f.root.CT_PHASE.attrs["SCALING_FACTOR"]
self.cloudphase.offset = h5f.root.CT_PHASE.attrs["OFFSET"]
self.cloudphase.num_of_lines = h5f.root.CT_PHASE.attrs["N_LINES"]
self.cloudphase.num_of_columns = h5f.root.CT_PHASE.attrs["N_COLS"]
self.cloudphase.product = h5f.root.CT_PHASE.attrs["PRODUCT"]
self.cloudphase.id = h5f.root.CT_PHASE.attrs["ID"]
# ------------------------
# The cloudtype processing/quality flags
self.processing_flags.data = h5f.root.CT_QUALITY[:, :]
self.processing_flags.scaling_factor = \
h5f.root.CT_QUALITY.attrs["SCALING_FACTOR"]
self.processing_flags.offset = h5f.root.CT_QUALITY.attrs["OFFSET"]
self.processing_flags.num_of_lines = h5f.root.CT_QUALITY.attrs["N_LINES"]
self.processing_flags.num_of_columns = h5f.root.CT_QUALITY.attrs["N_COLS"]
self.processing_flags.product = h5f.root.CT_QUALITY.attrs["PRODUCT"]
self.processing_flags.id = h5f.root.CT_QUALITY.attrs["ID"]
# ------------------------
h5f.close()
self.cloudtype = (self.cloudtype.data * self.cloudtype.scaling_factor
+ self.cloudtype.offset)
self.cloudphase = (self.cloudphase.data * self.cloudphase.scaling_factor
+ self.cloudphase.offset)
self.processing_flags = self.processing_flags.data
self.area = get_area_def(self.region_name)
self.filled = True
regex = r"-[ABC]"
subst = ""
bname = re.sub(regex, subst, bname, 0)
regex = r"(world)"
subst = "global"
bname = re.sub(regex, subst, bname, 0)
outFile="/tmp/"+os.path.basename(bname)
finalFile=outputDirectory+"/"+os.path.basename(bname)
masterImage.save(outFile)
regex = r"\d{10}"
datetime = re.findall(regex, bname)[0]
cw = ContourWriterAGG('/opt/pytroll/shapes')
world = get_area_def('world_plat_1350_675')
cw.add_coastlines_to_file(outFile, world, resolution='l', level=1, outline=(255, 255, 255))
cw.add_coastlines_to_file(outFile, world, resolution='l', level=1, outline=(255, 255, 255))
cw.add_borders_to_file(outFile, world, outline=(255, 255, 255),resolution='i')
cw.add_borders_to_file(outFile, world, outline=(255, 255, 255),resolution='i')
img = Image.open(outFile)
img = img.convert("RGB")
draw = ImageDraw.Draw(img)
print(img.size)
draw.rectangle([(0, 0), (img.size[0], 33)], fill=(255,165,0,200))
font = ImageFont.truetype("/usr/openv/java/jre/lib/fonts/LucidaTypewriterBold.ttf", 28)
textSizeName = draw.textsize("Meteop A+B", font=font)
textSizeDate = draw.textsize(datetime + " - 12h", font=font)
textSizeWho = draw.textsize("EUMETSAT/MeteoSwiss/PyTROLL", font=font)
draw.text((5, 3),"Meteop A+B",(25,25,25),font=font)
def load_generic(satscene, options, calibrate=True, area_extent=None,
area_def_names=None, filenames=None):
"""Read imager data from file and load it into *satscene*.
"""
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOGGER.debug("Channels to load from %s: %s" % (satscene.instrument_name,
satscene.channels_to_load))
# Compulsory global attributes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
from_area = False
if satscene.end_time is not None:
time_slot = satscene.time_slot, satscene.end_time
else:
time_slot = satscene.time_slot
if area_extent is None and satscene.area is not None:
if not satscene.area_def:
satscene.area = get_area_def(satscene.area_id)
area_extent = satscene.area.area_extent
from_area = True
area_converted_to_extent = False
for chn in satscene.channels_to_load:
use_filenames = False
# Sort out filenames
if filenames is not None:
for section in options.keys():
if section.endswith('-level1'):
break
try:
pattern_pro = eval(options[section].get('filename_pro'))
except TypeError:
pattern_pro = None
try:
pattern_epi = eval(options[section].get('filename_epi'))
except TypeError:
pattern_epi = None
pattern = eval(options[section].get('filename'))
epilogue = None
prologue = None
image_files = []
if pattern_epi is not None:
glob_epi = satscene.time_slot.strftime(
pattern_epi) % ({'segment': "EPI".ljust(9, '_'),
'channel': chn + '*'})
else:
glob_epi = 'eggs_and_spam'
if pattern_pro is not None:
glob_pro = satscene.time_slot.strftime(
pattern_pro) % ({'segment': "PRO".ljust(9, '_'),
'channel': chn + '*'})
else:
glob_pro = 'eggs_and_spam'
glob_img = satscene.time_slot.strftime(
pattern) % ({'segment': "*", 'channel': chn + '*'})
for filename in filenames:
if fnmatch.fnmatch(os.path.basename(filename), glob_img):
image_files.append(filename)
elif pattern_pro is not None and fnmatch.fnmatch(
os.path.basename(filename),
glob_pro):
prologue = filename
elif pattern_epi is not None and fnmatch.fnmatch(
os.path.basename(filename),
glob_epi):
epilogue = filename
if len(image_files) == 0 and prologue is None and epilogue is None:
use_filenames = False
else:
use_filenames = True
if from_area:
try:
if use_filenames:
metadata = xrit.sat.load_files(prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
only_metadata=True)
else:
metadata = xrit.sat.load(satscene.fullname,
time_slot,
chn,
only_metadata=True)
if(satscene.area_def.proj_dict["proj"] != "geos" or
float(satscene.area_def.proj_dict["lon_0"]) !=
metadata.sublon):
raise ValueError("Slicing area must be in "
"geos projection, and lon_0 should match "
"the satellite's position.")
except ReaderError, err:
# if channel can't be found, go on with next channel
LOGGER.error(str(err))
continue
# Convert area definitions to maximal area_extent
if not area_converted_to_extent and area_def_names is not None:
try:
if use_filenames:
metadata = xrit.sat.load_files(prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
only_metadata=True)
else:
metadata = xrit.sat.load(satscene.fullname,
time_slot,
chn,
only_metadata=True)
except ReaderError as err:
LOGGER.warning(str(err))
continue
# if area_extent is given, assume it gives the maximum
# extent of the satellite view
if area_extent is not None:
area_extent = area_def_names_to_extent(area_def_names,
metadata.proj4_params,
area_extent)
# otherwise use the default value (MSG3 extent at
# lon0=0.0), that is, do not pass default_extent=area_extent
else:
area_extent = area_def_names_to_extent(area_def_names,
metadata.proj4_params,
default_extent=None)
if area_extent is None:
LOGGER.info('Could not derive area_extent from area_def_names')
area_converted_to_extent = True
try:
if use_filenames:
image = xrit.sat.load_files(prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
mask=True,
calibrate=calibrate)
else:
image = xrit.sat.load(satscene.fullname,
time_slot,
chn,
mask=True,
calibrate=calibrate)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOGGER.warning(
"Loading non calibrated data since calibration failed.")
if use_filenames:
image = xrit.sat.load_files(prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
mask=True,
calibrate=False)
else:
image = xrit.sat.load(satscene.fullname,
time_slot,
chn,
mask=True,
calibrate=False)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except ReaderError as err:
# if channel can't be found, go on with next channel
LOGGER.warning(str(err))
continue
satscene[chn] = data
satscene[chn].info['units'] = metadata.calibration_unit
satscene[chn].info['sublon'] = metadata.sublon
satscene[chn].info['satname'] = satscene.satname
satscene[chn].info['satnumber'] = satscene.number
satscene[chn].info['instrument_name'] = satscene.instrument_name
satscene[chn].info['time'] = satscene.time_slot
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split(" ")
proj_dict = {}
for param in proj_params:
key, val = param.split("=")
proj_dict[key] = val
if IS_PYRESAMPLE_LOADED:
# Build area_def on-the-fly
satscene[chn].area = geometry.AreaDefinition(
satscene.satname + satscene.instrument_name +
str(metadata.area_extent) +
str(data.shape),
"On-the-fly area",
proj_dict["proj"],
proj_dict,
data.shape[1],
data.shape[0],
metadata.area_extent)
else:
LOGGER.info("Could not build area, pyresample missing...")
t = datetime.datetime(2016, 4, 29, 12, 00)
lat = 0.00
lon = 25.00
alt = 0.
h = 8000.
get_parallaxed_coor('meteosat 10', msgtle, t, lat, lon, alt, h)
# Example time slots
time_slot = datetime.datetime(2016, 4, 29, 12, 00)
time_slot2 = datetime.datetime(2016, 4, 29, 18, 00)
time_slot3 = datetime.datetime(2016, 4, 29, 00, 00)
# Import test data into pytroll
global_data = GeostationaryFactory.create_scene("meteosat", "10", "seviri",
time_slot)
europe = get_area_def("EuropeCanary")
global_data.load([0.6, 3.9, 10.8], area_extent=europe.area_extent)
lonlats = global_data[10.8].area.get_lonlats()
# Loop for plot dijurnal changes of azimute and elevation
#hr = range(24)
#time = [datetime.datetime(2016, 4, 29, i, 00) for i in hr]
#obslist = [msgorb.get_observer_look(t, oflon, oflat, ofalt) for t in time]
#poslist = [msgorb.get_position(t) for t in time]
#azi = [obs[0] for obs in obslist]
#ele = [obs[1] for obs in obslist]
#fig = plt.subplot()
#plt.plot(hr, azi)
#plt.plot(hr, ele)
time1 = []
for i in range(5, 65, 5):
leadS = "%02d" % i
#diff["t"+leadS] = {}
diff = []
diff1 = []
yearS, monthS, dayS, hourS, minS = string_date(time_slot0 +
timedelta(minutes=i))
#print ("*** read data for ", in_msg.sat_str(),in_msg.sat_nr_str(), "seviri", time_slot0+timedelta(minutes = i))
global_data = GeostationaryFactory.create_scene(
in_msg.sat_str(), in_msg.sat_nr_str(), "seviri",
time_slot0 + timedelta(minutes=i))
area_loaded = get_area_def(
"EuropeCanary95") #(in_windshift.areaExtraction)
area_loaded = load_products(global_data, ['CTT'], in_msg, area_loaded)
data = global_data.project("ccs4")
img_obs = deepcopy(data['CTT'].data)
img_obs.mask[:, :] = False
if True:
print("pickles/" + year0S + month0S + day0S + "_" + hour0S +
min0S + "_CTT_t" + leadS + "_1layer.p")
tmp = pickle.load(
open(
"pickles/" + year0S + month0S + day0S + "_" + hour0S +
min0S + "_CTT_t" + leadS + "_1layer.p", "rb"))
tmp = (tmp[0] - img_obs)
diff1.append(tmp)
else:
loglevel = logging.INFO
handler.setLevel(loglevel)
logging.getLogger('').setLevel(loglevel)
logging.getLogger('').addHandler(handler)
logging.getLogger("posttroll").setLevel(logging.INFO)
logger = logging.getLogger("gatherer")
decoder = get_metadata
granule_triggers = []
pub = publisher.NoisyPublisher("gatherer")
# TODO: get this from the product config files
regions = [get_area_def(region)
for region in config.get("default", "regions").split()]
for section in config.sections():
if section == "default":
continue
timeliness = timedelta(minutes=config.getint(section, "timeliness"))
try:
duration = timedelta(seconds=config.getfloat(section, "duration"))
except NoOptionError:
duration = None
collectors = [region_collector.RegionCollector(
region, timeliness, duration) for region in regions]
try:
from __future__ import division
from __future__ import print_function
from mpop.satellites import GeostationaryFactory
from mpop.projector import get_area_def
import datetime
time_slot = datetime.datetime(2014, 0o7, 16, 13, 30)
time_slot = datetime.datetime(2014, 0o7, 23, 00, 0o5)
global_data = GeostationaryFactory.create_scene("meteosat", "00", "seviri",
time_slot)
europe = get_area_def("ccs4")
global_data.load(
[0.6, 0.8, 1.6, 3.9, 6.2, 7.3, 8.7, 9.7, 10.8, 12.0, 13.4,
'HRV']) #, area_extent=europe.area_extent
print(global_data)
rgb = 'IR_108'
min_data = global_data[rgb].data.min()
max_data = global_data[rgb].data.max()
img = global_data.image.channel_image(rgb)
#img = global_data.image.hr_overview()
print(type(img))
print(dir(img))
print("min/max", min_data, max_data)
#img.show()
PIL_image = img.pil_image()
outputFile = 'test_IR_108.png'
img.save(outputFile, optimize=True)
print("save file as ", outputFile)
parser.add_argument('--sat_id', dest='sat_id', action="store", help="Satellite ID", default="8888")
parser.add_argument('--data_cat', dest='data_cat', action="store", help="Category of data (one of GORN, GPRN, P**N)", default="GORN")
parser.add_argument('--area', dest='areadef', action="store", help="Area name, the definition must exist in your areas configuration file", default="nrEURO1km_NPOL_COALeqc")
parser.add_argument('--ph_unit', dest='ph_unit', action="store", help="Physical unit", default="CELSIUS")
parser.add_argument('--data_src', dest='data_src', action="store", help="Data source", default="EUMETCAST")
args = parser.parse_args()
if (args.input_dir != None):
os.chdir(args.input_dir)
cfg = vars(args)
if (args.cfg != None):
with open(args.cfg, 'r') as ymlfile:
cfg = yaml.load(ymlfile)
narea = get_area_def(args.areadef)
global_data = Scene(sensor="images", reader="generic_image", area=narea)
global_data.load(['image'])
global_data['image'].info['area'] = narea
fname = global_data['image'].info['filename']
ofname = fname[:-3] + "tif"
#global_data.save_dataset('image', filename="out.png", writer="simple_image")
global_data.save_dataset('image', filename=ofname, writer="ninjotiff",
sat_id=cfg['sat_id'],
chan_id=cfg['chan_id'],
data_cat=cfg['data_cat'],
data_source=cfg['data_src'],
physic_unit=cfg['ph_unit'])
def geotiff_save(self, filename, compression=6,
tags=None, gdal_options=None,
blocksize=0, geotransform=None,
spatialref=None, floating_point=False):
"""Save the image to the given *filename* in geotiff_ format, with the
*compression* level in [0, 9]. 0 means not compressed. The *tags*
argument is a dict of tags to include in the image (as metadata). By
default it uses the 'area' instance to generate geotransform and
spatialref information, this can be overwritten by the arguments
*geotransform* and *spatialref*. *floating_point* allows the saving of
'L' mode images in floating point format if set to True.
.. _geotiff: http://trac.osgeo.org/geotiff/
"""
from osgeo import gdal, osr
raster = gdal.GetDriverByName("GTiff")
if floating_point:
if self.mode != "L":
raise ValueError("Image must be in 'L' mode for floating point"
" geotif saving")
channels = [self.channels[0].astype(np.float64)]
fill_value = self.fill_value or 0
gformat = gdal.GDT_Float64
else:
channels, fill_value = self._finalize()
gformat = gdal.GDT_Byte
LOG.debug("Saving to GeoTiff.")
if tags is not None:
self.tags.update(tags)
if gdal_options is not None:
self.gdal_options.update(gdal_options)
g_opts = ["=".join(i) for i in self.gdal_options.items()]
if compression != 0:
g_opts.append("COMPRESS=DEFLATE")
g_opts.append("ZLEVEL=" + str(compression))
if blocksize != 0:
g_opts.append("TILED=YES")
g_opts.append("BLOCKXSIZE=" + str(blocksize))
g_opts.append("BLOCKYSIZE=" + str(blocksize))
if(self.mode == "L"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
1,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif(self.mode == "LA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds,
channels[:-1], channels[1],
fill_value)
elif(self.mode == "RGB"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
3,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif(self.mode == "RGBA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels[:-1], channels[3], fill_value)
else:
raise NotImplementedError("Saving to GeoTIFF using image mode"
" %s is not implemented."%self.mode)
# Create raster GeoTransform based on upper left corner and pixel
# resolution ... if not overwritten by argument geotranform.
if geotransform:
dst_ds.SetGeoTransform(geotransform)
if spatialref:
if not isinstance(spatialref, str):
spatialref = spatialref.ExportToWkt()
dst_ds.SetProjection(spatialref)
else:
try:
from pyresample import utils
from mpop.projector import get_area_def
area = get_area_def(self.area)
except (utils.AreaNotFound, AttributeError):
area = self.area
try:
adfgeotransform = [area.area_extent[0], area.pixel_size_x, 0,
area.area_extent[3], 0, -area.pixel_size_y]
dst_ds.SetGeoTransform(adfgeotransform)
srs = osr.SpatialReference()
srs.ImportFromProj4(area.proj4_string)
srs.SetProjCS(area.proj_id)
try:
srs.SetWellKnownGeogCS(area.proj_dict['ellps'])
except KeyError:
pass
try:
# Check for epsg code.
srs.SetAuthority('PROJCS', 'EPSG',
int(area.proj_dict['init'].
split('epsg:')[1]))
except (KeyError, IndexError):
pass
srs = srs.ExportToWkt()
dst_ds.SetProjection(srs)
except AttributeError:
LOG.exception("Could not load geographic data, invalid area")
self.tags.update({'TIFFTAG_DATETIME':
self.time_slot.strftime("%Y:%m:%d %H:%M:%S")})
dst_ds.SetMetadata(self.tags, '')
# Close the dataset
dst_ds = None
def load_generic(satscene, options, calibrate=True, area_extent=None):
"""Read seviri data from file and load it into *satscene*.
"""
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOG.debug("Channels to load from %s: %s" % (satscene.instrument_name, satscene.channels_to_load))
# Compulsory global attribudes
satscene.info["title"] = (
satscene.satname.capitalize()
+ satscene.number
+ " satellite, "
+ satscene.instrument_name.capitalize()
+ " instrument."
)
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
from_area = False
if area_extent is None and satscene.area is not None:
if not satscene.area_def:
satscene.area = get_area_def(satscene.area_id)
area_extent = satscene.area.area_extent
from_area = True
for chn in satscene.channels_to_load:
if from_area:
try:
metadata = xrit.sat.load(satscene.fullname, satscene.time_slot, chn, only_metadata=True)
if (
satscene.area_def.proj_dict["proj"] != "geos"
or float(satscene.area_def.proj_dict["lon_0"]) != metadata.sublon
):
raise ValueError(
"Slicing area must be in "
"geos projection, and lon_0 should match the"
" satellite's position."
)
except SatReaderError:
# if channel can't be found, go on with next channel
continue
try:
image = xrit.sat.load(satscene.fullname, satscene.time_slot, chn, mask=True, calibrate=calibrate)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOG.warning("Loading non calibrated data since calibration failed.")
image = xrit.sat.load(satscene.fullname, satscene.time_slot, chn, mask=True, calibrate=False)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except SatReaderError:
# if channel can't be found, go on with next channel
continue
satscene[chn] = data
satscene[chn].info["units"] = metadata.calibration_unit
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split(" ")
proj_dict = {}
for param in proj_params:
key, val = param.split("=")
proj_dict[key] = val
if is_pyresample_loaded:
# Build area_def on-the-fly
satscene[chn].area = geometry.AreaDefinition(
satscene.satname + satscene.instrument_name + str(metadata.area_extent) + str(data.shape),
"On-the-fly area",
proj_dict["proj"],
proj_dict,
data.shape[1],
data.shape[0],
metadata.area_extent,
)
else:
LOG.info("Could not build area, pyresample missing...")
def draw_images(self, area):
'''Generate images from local data using given area name and
product definitions.
'''
params = self.get_parameters(area)
# Create images for each color composite
for product in area:
params.update(self.get_parameters(product))
if product.tag == "dump":
try:
self.save_to_netcdf(self.local_data,
product,
params)
except IOError:
LOGGER.error("Saving projected data to NetCDF failed!")
continue
elif product.tag != "product":
continue
# TODO
# Check if satellite is one that should be processed
# if not self.check_satellite(product):
# Skip this product, if the return value is True
# continue
# Check if Sun zenith angle limits match this product
if 'sunzen_night_minimum' in product.attrib or \
'sunzen_day_maximum' in product.attrib:
if 'sunzen_xy_loc' in product.attrib:
xy_loc = [int(x) for x in
product.attrib['sunzen_xy_loc'].split(',')]
lonlat = None
else:
xy_loc = None
if 'sunzen_lonlat' in product.attrib:
lonlat = [float(x) for x in
product.attrib['sunzen_lonlat'].split(',')]
else:
lonlat = None
if not self.check_sunzen(product.attrib, area_def=get_area_def(area.attrib['id']),
xy_loc=xy_loc, lonlat=lonlat):
# If the return value is False, skip this product
continue
try:
# Check if this combination is defined
func = getattr(self.local_data.image, product.attrib['id'])
LOGGER.debug("Generating %s", product.attrib['id'])
img = func()
img.info.update(self.global_data.info)
img.info["product_name"] = product.attrib.get("name",
product.attrib["id"])
except AttributeError:
# Log incorrect product funcion name
LOGGER.error('Incorrect product id: %s for area %s',
product.attrib['id'], area.attrib['name'])
except KeyError as err:
# log missing channel
LOGGER.warning('Missing channel on product %s for area %s: %s',
product.attrib['name'], area.attrib['name'], str(err))
except Exception:
# log other errors
LOGGER.exception('Error on product %s for area %s',
product.attrib['name'],
area.attrib['name'])
else:
self.writer.write(img, product, params)
# log and publish completion of this area def
LOGGER.info('Area %s completed', area.attrib['name'])
def plot_msg_minus_cosmo(in_msg):
# do statistics for the last full hour (minutes=0, seconds=0)
in_msg.datetime = datetime(in_msg.datetime.year, in_msg.datetime.month,
in_msg.datetime.day, in_msg.datetime.hour, 0, 0)
area_loaded = choose_area_loaded_msg(in_msg.sat, in_msg.sat_nr,
in_msg.datetime)
# define contour write for coasts, borders, rivers
cw = ContourWriterAGG(in_msg.mapDir)
# check if input data is complete
if in_msg.verbose:
print("*** check input data for ", in_msg.sat_str())
RGBs = check_input(in_msg,
in_msg.sat_str(layout="%(sat)s") + in_msg.sat_nr_str(),
in_msg.datetime)
# in_msg.sat_nr might be changed to backup satellite
if in_msg.verbose:
print('*** Create plots for ')
print(' Satellite/Sensor: ' + in_msg.sat_str())
print(' Satellite number: ' + in_msg.sat_nr_str() + ' // ' +
str(in_msg.sat_nr))
print(' Satellite instrument: ' + in_msg.instrument)
print(' Date/Time: ' + str(in_msg.datetime))
print(' RGBs: ', in_msg.RGBs)
print(' Area: ', in_msg.areas)
print(' reader level: ', in_msg.reader_level)
# define satellite data object
#global_data = GeostationaryFactory.create_scene(in_msg.sat, in_msg.sat_nr_str(), "seviri", in_msg.datetime)
global_data = GeostationaryFactory.create_scene(in_msg.sat_str(),
in_msg.sat_nr_str(),
in_msg.instrument,
in_msg.datetime)
# global_data = GeostationaryFactory.create_scene("msg-ot", "", "Overshooting_Tops", in_msg.datetime)
if len(RGBs) == 0 and len(in_msg.postprocessing_areas) == 0:
return RGBs
if in_msg.verbose:
print(
"*** load satellite channels for " + in_msg.sat_str() +
in_msg.sat_nr_str() + " ", global_data.fullname)
# initialize processed RGBs
RGBs_done = []
# -------------------------------------------------------------------
# load reflectivities, brightness temperatures, NWC-SAF products ...
# -------------------------------------------------------------------
area_loaded = load_products(global_data, RGBs, in_msg, area_loaded)
cosmo_input_file = "input_cosmo_cronjob.py"
print("... read COSMO input file: ", cosmo_input_file)
in_cosmo = parse_commandline_and_read_inputfile(
input_file=cosmo_input_file)
# add composite
in_msg.scpOutput = True
in_msg.resize_montage = 70
in_msg.postprocessing_montage = [[
"MSG_IR-108cpc", "COSMO_SYNMSG-BT-CL-IR10.8",
"MSG_IR-108-COSMO-minus-MSGpc"
]]
in_msg.scpProducts = [[
"MSG_IR-108cpc", "COSMO_SYNMSG-BT-CL-IR10.8",
"MSG_IR-108-COSMO-minus-MSGpc"
]]
#in_msg.scpProducts = ["all"]
# define satellite data object
cosmo_data = GeostationaryFactory.create_scene(in_cosmo.sat_str(),
in_cosmo.sat_nr_str(),
in_cosmo.instrument,
in_cosmo.datetime)
area_loaded_cosmo = load_products(cosmo_data, ['SYNMSG_BT_CL_IR10.8'],
in_cosmo, area_loaded)
# preprojecting the data to another area
# --------------------------------------
if len(RGBs) > 0:
for area in in_msg.areas:
print("")
obj_area = get_area_def(area)
if area != 'ccs4':
print("*** WARNING, diff MSG-COSMO only implemented for ccs4")
continue
# reproject data to new area
print(area_loaded)
if obj_area == area_loaded:
if in_msg.verbose:
print("*** Use data for the area loaded: ", area)
#obj_area = area_loaded
data = global_data
resolution = 'l'
else:
if in_msg.verbose:
print("*** Reproject data to area: ", area,
"(org projection: ", area_loaded.name, ")")
obj_area = get_area_def(area)
# PROJECT data to new area
data = global_data.project(area, precompute=True)
resolution = 'i'
if in_msg.parallax_correction:
loaded_products = [chn.name for chn in data.loaded_channels()]
if 'CTH' not in loaded_products:
print("*** Error in plot_msg (" +
inspect.getfile(inspect.currentframe()) + ")")
print(
" Cloud Top Height is needed for parallax correction "
)
print(
" either load CTH or specify the estimation of the CTH in the input file (load 10.8 in this case)"
)
quit()
if in_msg.verbose:
print(
" perform parallax correction for loaded channels: ",
loaded_products)
data = data.parallax_corr(fill=in_msg.parallax_gapfilling,
estimate_cth=in_msg.estimate_cth,
replace=True)
# save reprojected data
if area in in_msg.save_reprojected_data:
save_reprojected_data(data, area, in_msg)
# apply a mask to the data (switched off at the moment)
if False:
mask_data(data, area)
# save average values
if in_msg.save_statistics:
mean_array = zeros(len(RGBs))
#statisticFile = '/data/COALITION2/database/meteosat/ccs4/'+yearS+'/'+monthS+'/'+dayS+'/MSG_'+area+'_'+yearS[2:]+monthS+dayS+'.txt'
statisticFile = './' + yearS + '-' + monthS + '-' + dayS + '/MSG_' + area + '_' + yearS[
2:] + monthS + dayS + '.txt'
if in_msg.verbose:
print("*** write statistics (average values) to " +
statisticFile)
f1 = open(statisticFile, 'a') # mode append
i_rgb = 0
for rgb in RGBs:
if rgb in products.MSG_color:
mean_array[i_rgb] = data[rgb.replace("c",
"")].data.mean()
i_rgb = i_rgb + 1
# create string to write
str2write = dateS + ' ' + hourS + ' : ' + minS + ' UTC '
for mm in mean_array:
str2write = str2write + ' ' + "%7.2f" % mm
str2write = str2write + "\n"
f1.write(str2write)
f1.close()
# creating plots/images
if in_msg.make_plots:
# choose map resolution
in_msg.resolution = choose_map_resolution(
area, in_msg.mapResolution)
# define area
proj4_string = obj_area.proj4_string
# e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
area_extent = obj_area.area_extent
# e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
area_tuple = (proj4_string, area_extent)
RGBs = ['IR_108-COSMO-minus-MSG']
print(data['IR_108'].data.shape)
print(cosmo_data['SYNMSG_BT_CL_IR10.8'].data.shape)
diff_MSG_COSMO = cosmo_data['SYNMSG_BT_CL_IR10.8'].data - data[
'IR_108'].data
HRV_enhance_str = ''
# add IR difference as "channel object" to satellite regional "data" object
data.channels.append(
Channel(name=RGBs[0],
wavelength_range=[0., 0., 0.],
resolution=data['IR_108'].resolution,
data=diff_MSG_COSMO))
for rgb in RGBs:
if not check_loaded_channels(rgb, data):
continue
PIL_image = create_PIL_image(rgb,
data,
in_msg,
obj_area=obj_area)
# !!! in_msg.colorbar[rgb] is initialized inside (give attention to rgbs) !!!
add_borders_and_rivers(PIL_image,
cw,
area_tuple,
add_borders=in_msg.add_borders,
border_color=in_msg.border_color,
add_rivers=in_msg.add_rivers,
river_color=in_msg.river_color,
resolution=in_msg.resolution,
verbose=in_msg.verbose)
# indicate mask
if in_msg.indicate_mask:
PIL_image = indicate_mask(rgb, PIL_image, data,
in_msg.verbose)
#if area.find("EuropeCanary") != -1 or area.find("ccs4") != -1:
dc = DecoratorAGG(PIL_image)
# add title to image
if in_msg.add_title:
add_title(PIL_image,
in_msg.title,
HRV_enhance_str + rgb,
in_msg.sat_str(),
data.sat_nr(),
in_msg.datetime,
area,
dc,
in_msg.font_file,
in_msg.verbose,
title_color=in_msg.title_color,
title_y_line_nr=in_msg.title_y_line_nr
) # !!! needs change
# add MeteoSwiss and Pytroll logo
if in_msg.add_logos:
if in_msg.verbose:
print('... add logos')
dc.align_right()
if in_msg.add_colorscale:
dc.write_vertically()
if PIL_image.mode != 'L':
height = 60 # height=60.0 normal resolution
dc.add_logo(in_msg.logos_dir + "/pytroll3.jpg",
height=height) # height=60.0
dc.add_logo(in_msg.logos_dir + "/meteoSwiss3.jpg",
height=height)
dc.add_logo(
in_msg.logos_dir +
"/EUMETSAT_logo2_tiny_white_square.png",
height=height) # height=60.0
# add colorscale
if in_msg.add_colorscale and in_msg.colormap[rgb] != None:
if rgb in products.MSG_color:
unit = data[rgb.replace("c", "")].info['units']
#elif rgb in products.MSG or rgb in products.NWCSAF or rgb in products.HSAF:
# unit = data[rgb].info['units']
else:
unit = None
loaded_channels = [
chn.name for chn in data.loaded_channels()
]
if rgb in loaded_channels:
if hasattr(data[rgb], 'info'):
print(" hasattr(data[rgb], 'info')",
list(data[rgb].info.keys()))
if 'units' in list(data[rgb].info.keys()):
print(
"'units' in data[rgb].info.keys()")
unit = data[rgb].info['units']
print("... units = ", unit)
add_colorscale(dc, rgb, in_msg, unit=unit)
if in_msg.parallax_correction:
parallax_correction_str = 'pc'
else:
parallax_correction_str = ''
rgb += parallax_correction_str
# create output filename
outputDir = format_name(
in_msg.outputDir,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr()) # !!! needs change
outputFile = outputDir + "/" + format_name(
in_msg.outputFile,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr()) # !!! needs change
# check if output directory exists, if not create it
path = dirname(outputFile)
if not exists(path):
if in_msg.verbose:
print('... create output directory: ' + path)
makedirs(path)
# save file
if exists(outputFile) and not in_msg.overwrite:
if stat(outputFile).st_size > 0:
print('... outputFile ' + outputFile +
' already exists (keep old file)')
else:
print(
'*** Warning, outputFile' + outputFile +
' already exists, but is empty (overwrite file)'
)
PIL_image.save(outputFile, optimize=True
) # optimize -> minimize file size
chmod(
outputFile, 0o777
) ## FOR PYTHON3: 0o664 # give access read/write access to group members
else:
if in_msg.verbose:
print('... save final file: ' + outputFile)
PIL_image.save(
outputFile,
optimize=True) # optimize -> minimize file size
chmod(
outputFile, 0o777
) ## FOR PYTHON3: 0o664 # give access read/write access to group members
if in_msg.compress_to_8bit:
if in_msg.verbose:
print('... compress to 8 bit image: display ' +
outputFile.replace(".png", "-fs8.png") +
' &')
subprocess.call(
"/usr/bin/pngquant -force 256 " + outputFile +
" 2>&1 &",
shell=True) # 256 == "number of colors"
#if in_msg.verbose:
# print " add coastlines to "+outputFile
## alternative: reopen image and modify it (takes longer due to additional reading and saving)
#cw.add_rivers_to_file(img, area_tuple, level=5, outline='blue', width=0.5, outline_opacity=127)
#cw.add_coastlines_to_file(outputFile, obj_area, resolution=resolution, level=4)
#cw.add_borders_to_file(outputFile, obj_area, outline=outline, resolution=resolution)
# secure copy file to another place
if in_msg.scpOutput:
if (rgb in in_msg.scpProducts) or ('all' in [
x.lower()
for x in in_msg.scpProducts if type(x) == str
]):
scpOutputDir = format_name(in_msg.scpOutputDir,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr())
if in_msg.compress_to_8bit:
if in_msg.verbose:
print("... secure copy " +
outputFile.replace(
".png", "-fs8.png") + " to " +
scpOutputDir)
subprocess.call(
"scp " + in_msg.scpID + " " +
outputFile.replace(".png", "-fs8.png") +
" " + scpOutputDir + " 2>&1 &",
shell=True)
else:
if in_msg.verbose:
print("... secure copy " + outputFile +
" to " + scpOutputDir)
subprocess.call("scp " + in_msg.scpID + " " +
outputFile + " " +
scpOutputDir + " 2>&1 &",
shell=True)
if in_msg.scpOutput and in_msg.scpID2 != None and in_msg.scpOutputDir2 != None:
if (rgb in in_msg.scpProducts2) or ('all' in [
x.lower()
for x in in_msg.scpProducts2 if type(x) == str
]):
scpOutputDir2 = format_name(in_msg.scpOutputDir2,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr())
if in_msg.compress_to_8bit:
if in_msg.verbose:
print("... secure copy " +
outputFile.replace(
".png", "-fs8.png") + " to " +
scpOutputDir2)
subprocess.call(
"scp " + in_msg.scpID2 + " " +
outputFile.replace(".png", "-fs8.png") +
" " + scpOutputDir2 + " 2>&1 &",
shell=True)
else:
if in_msg.verbose:
print("... secure copy " + outputFile +
" to " + scpOutputDir2)
subprocess.call("scp " + in_msg.scpID2 + " " +
outputFile + " " +
scpOutputDir2 + " 2>&1 &",
shell=True)
if 'ninjotif' in in_msg.outputFormats:
ninjotif_file = format_name(outputDir + '/' +
in_msg.ninjotifFilename,
data.time_slot,
sat_nr=data.sat_nr(),
RSS=in_msg.RSS,
area=area,
rgb=rgb)
from plot_coalition2 import pilimage2geoimage
GEO_image = pilimage2geoimage(PIL_image, obj_area,
data.time_slot)
GEO_image.save(ninjotif_file,
fformat='mpop.imageo.formats.ninjotiff',
ninjo_product_name=rgb,
chan_id=products.ninjo_chan_id[
rgb.replace("_", "-") + "_" + area],
nbits=8)
chmod(ninjotif_file, 0o777)
print(("... save ninjotif image: display ",
ninjotif_file, " &"))
if rgb not in RGBs_done:
RGBs_done.append(rgb)
## start postprocessing
for area in in_msg.postprocessing_areas:
postprocessing(in_msg, global_data.time_slot, int(data.sat_nr()),
area)
if in_msg.verbose:
print(" ")
return RGBs_done
def load_generic(satscene, options, calibrate=True, area_extent=None):
"""Read imager data from file and load it into *satscene*.
"""
del options
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOG.debug("Channels to load from %s: %s"%(satscene.instrument_name,
satscene.channels_to_load))
# Compulsory global attribudes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
from_area = False
if area_extent is None and satscene.area is not None:
if not satscene.area_def:
satscene.area = get_area_def(satscene.area_id)
area_extent = satscene.area.area_extent
from_area = True
for chn in satscene.channels_to_load:
if from_area:
try:
metadata = xrit.sat.load(satscene.fullname, satscene.time_slot,
chn, only_metadata=True)
if(satscene.area_def.proj_dict["proj"] != "geos" or
float(satscene.area_def.proj_dict["lon_0"]) != metadata.sublon):
raise ValueError("Slicing area must be in "
"geos projection, and lon_0 should match the"
" satellite's position.")
except ReaderError, e:
# if channel can't be found, go on with next channel
LOG.error(str(e))
continue
try:
image = xrit.sat.load(satscene.fullname,
satscene.time_slot,
chn,
mask=True,
calibrate=calibrate)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOG.warning("Loading non calibrated data since calibration failed.")
image = xrit.sat.load(satscene.fullname,
satscene.time_slot,
chn,
mask=True,
calibrate=False)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except ReaderError, e:
# if channel can't be found, go on with next channel
LOG.error(str(e))
continue
parser.add_argument('--data_src',
dest='data_src',
action="store",
help="Data source",
default="EUMETCAST")
args = parser.parse_args()
if (args.input_dir is not None):
os.chdir(args.input_dir)
cfg = vars(args)
if (args.cfg is not None):
with open(args.cfg, 'r') as ymlfile:
cfg = yaml.load(ymlfile, Loader=UnsafeLoader)
narea = get_area_def(args.areadef)
global_data = Scene(sensor="images", reader="generic_image", area=narea)
global_data.load(['image'])
global_data['image'].info['area'] = narea
fname = global_data['image'].info['filename']
ofname = fname[:-3] + "tif"
# global_data.save_dataset('image', filename="out.png", writer="simple_image")
global_data.save_dataset('image',
filename=ofname,
writer="ninjotiff",
sat_id=cfg['sat_id'],
chan_id=cfg['chan_id'],
data_cat=cfg['data_cat'],
data_source=cfg['data_src'],
def geotiff_save(
self, filename, compression=6, tags=None, gdal_options=None, blocksize=0, geotransform=None, spatialref=None
):
"""Save the image to the given *filename* in geotiff_ format, with the
*compression* level in [0, 9]. 0 means not compressed. The *tags*
argument is a dict of tags to include in the image (as metadata).
By default it uses the 'area' instance to generate geotransform and spatialref
information, this can be overwritte by the arguments *geotransform* and
*spatialref*.
.. _geotiff: http://trac.osgeo.org/geotiff/
"""
from osgeo import gdal, osr
raster = gdal.GetDriverByName("GTiff")
channels, fill_value = self._finalize()
LOG.debug("Saving to GeoTiff.")
if tags is not None:
self.tags.update(tags)
if gdal_options is not None:
self.gdal_options.update(gdal_options)
g_opts = ["=".join(i) for i in self.gdal_options.items()]
if compression != 0:
g_opts.append("COMPRESS=DEFLATE")
g_opts.append("ZLEVEL=" + str(compression))
if blocksize != 0:
g_opts.append("TILED=YES")
g_opts.append("BLOCKXSIZE=" + str(blocksize))
g_opts.append("BLOCKYSIZE=" + str(blocksize))
if self.mode == "L":
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename, self.width, self.height, 1, gdal.GDT_Byte, g_opts)
else:
dst_ds = raster.Create(filename, self.width, self.height, 2, gdal.GDT_Byte, g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif self.mode == "RGB":
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename, self.width, self.height, 3, gdal.GDT_Byte, g_opts)
else:
dst_ds = raster.Create(filename, self.width, self.height, 4, gdal.GDT_Byte, g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif self.mode == "RGBA":
ensure_dir(filename)
dst_ds = raster.Create(filename, self.width, self.height, 4, gdal.GDT_Byte, g_opts)
self._gdal_write_channels(dst_ds, channels, channels[3], fill_value)
else:
raise NotImplementedError("Saving to GeoTIFF using image mode" " %s is not implemented." % self.mode)
# Create raster GeoTransform based on upper left corner and pixel
# resolution ... if not overwritten by argument geotranform.
if geotransform:
dst_ds.SetGeoTransform(geotransform)
if spatialref:
if not isinstance(spatialref, str):
spatialref = spatialref.ExportToWkt()
dst_ds.SetProjection(spatialref)
else:
try:
from pyresample import utils
from mpop.projector import get_area_def
area = get_area_def(self.area_id)
except (utils.AreaNotFound, AttributeError):
area = self.area_id
try:
adfgeotransform = [
area.area_extent[0],
area.pixel_size_x,
0,
area.area_extent[3],
0,
-area.pixel_size_y,
]
dst_ds.SetGeoTransform(adfgeotransform)
srs = osr.SpatialReference()
srs.SetProjCS(area.proj_id)
srs.ImportFromProj4(area.proj4_string)
srs = srs.ExportToWkt()
dst_ds.SetProjection(srs)
except AttributeError:
LOG.exception("Could not load geographic data, invalid area")
self.tags.update({"TIFFTAG_DATETIME": self.time_slot.strftime("%Y:%m:%d %H:%M:%S")})
dst_ds.SetMetadata(self.tags, "")
# Close the dataset
dst_ds = None
# list created with awk '/^#/ {next} /REGION/ {printf "\"%s\", ", $2}' areas.def
# removed "{" manually
#areas=["alps95", "alps95L", "ccs4", "ccs4c2", "swissXXL", "cosmo1", "cosmo7", "ticino", "nrCOALtwo1km", "nrCOALtwo750m", "swiss02", "SwitzerlandStereo500m", "swissLarge1600m", "nrEURO1km", "nrEURO3km", "nrEuro4km", "nrMET3km", "nrIODC8km", "nqceur1km", "regionH", "mpef-ceu", "baltrad_lambert", "baws", "bsea250", "bsea1000", "bsea250_new", "EastEurope", "eport", "eport1", "eport10", "eport2", "eport4", "Etna", "euro", "euro1", "euro4", "euron1", "euron0250", "EuroMercator", "europe_center", "euro_north", "EuropeCanary35", "EuropeCanary95", "EuropeOdyssey00", "EuropeOdyssey95", "EuropeOdyssey95a", "EuropeCanary", "EuropeCanaryS", "EuropeCanaryS95", "eurotv", "eurotv4n", "euroHDready", "euroHDfull", "eurol", "eurol1", "FranceSouthHyMex", "germ", "germ2", "hsaf", "hsaf_merc", "iber", "iceland", "italy", "mesanX", "mesanE", "nq0001km", "nq0003km", "nq0008km", "nqceur1km", "nqceur3km", "nqeuro1km", "nrEuro4kmEqc", "nsea", "nsea250", "nsper_swe", "nswe", "odyssey", "odysseyS2", "odysseyS25", "odysseyS3", "odysseyS4", "pifh", "pifn", "romania", "rome", "scan", "scanl", "scan1", "scan2", "scan500m", "scanice", "spain", "ssea", "ssea250", "sswe", "sve", "met07globe", "MSGHRVN", "SeviriDisk00", "SeviriDiskFull00", "SeviriDiskFull00S2", "SeviriDiskFull00S3", "SeviriDiskFull00S4", "SeviriDiskFull00S5", "SeviriDisk00Cosmo", "SeviriDisk35", "SeviriDiskFull35", "SeviriDisk95", "SeviriDiskFull95", "SeviriDiskFull95S", "SeviriDiskFull95S3", "SeviriDiskFull95S4", "SeviriDiskFull95S5", "moll", "robinson", "platecarree", "worldeqc30km", "worldeqc3km70", "worldeqc30km70", "worldeqc3km73", "worldeqc3km", "worldeqc30km", "world_plat_8192_4096", "world_plat_13500_6750", "world_plat_21600_10800", "world_plat_21600_10927", "world_plat_C1_21600_21600", "AfHorn", "afhorn", "africa", "africa_10km", "kuwait", "kuwait_phil", "kuwait_phil_small", "libya", "mali", "mali_eqc", "maspalomas", "SouthArabia", "antarctica", "arctica", "arctic_europe_1km", "arctic_europe_9km", "barents_sea", "ease_nh", "ease_sh", "sval", "afghanistan", "euroasia", "euroasia_10km", "euroasia_asia", "euroasia_asia_10km", "japan", "pacific", "stere_asia_test", "australia", "australia_pacific", "australia_pacific_10km", "brazil", "brazil2", "southamerica", "SouthAmerica", "southamerica_10km", "SouthAmerica_flat", "south_america", "GOES16", "GOES16_FULL", "NinJoGOESEregion", "NinJoGOESWregion", "northamerica", "northamerica_10km", "npp_sample_m", "npp_sample_i", "baws250", "bocheng_test", "sudeste"]
#areas= ["fullearth"] -> RuntimeError: unknown elliptical parameter name
#areas=["ccs4"]
#areas=["SeviriDisk00Cosmo"]
#areas=["Cosmo1Mercator"]
#areas=["regionB", "regionB2"] -> error message: File "_proj.pyx", line 84, in _proj.Proj.__cinit__ (_proj.c:1170) RuntimeError: k <= 0
areas = ["cosmo1eqc3km"]
for area in areas:
print(area + ":")
area_def = get_area_def(area)
print(" description: " + area_def.name)
print(" projection:")
for key in [
"proj", "lat_0", "lon_0", "lat_1", "lon_1", "lat_2", "lon_2",
"lat_ts", "ellps", "x_0", "y_0", "k_0", "a", "b", "h"
]:
if key in area_def.proj_dict:
print(" " + key + ': ', area_def.proj_dict[key])
area_def.proj_dict.pop(key, None)
for key in area_def.proj_dict:
print(" " + key + ': ', area_def.proj_dict[key])
print(" shape:")
print(" height: " + str(area_def.y_size))
try:
time_slot=datetime.datetime(YYYY,MM,DD,hh,mm)
print '\n'
print time_slot
print '\n'
except:
print "\nTIME SLOT UNDEFINED"
#Scene Configuration
try:
global_data=GeostationaryFactory.create_scene("meteosat","10","seviri",time_slot)
except:
print "\nSATELLITE DEFINITION LOAD FAILED, CHECK THAT meteosat10.cfg EXISTS IN THE MPOP FOLDER OR CHANGE ARGUMENT IF YOU USE ANOTHER SATELLITE DEFINITION."
try:
globe=get_area_def("AfSubSahara")
except:
print "\nAREA DEFINITION LOAD FAILED, CHECK THAT areas.def EXISTS IN THE MPOP FOLDER."
#Data load
try:
if MSG_FILE_TYPE=='L':
IRchannelList=['IR_039','IR_108']
global_data.load(IRchannelList,area_extent=globe.area_extent,calibrate=1)
print global_data[3.9].data.min()
print global_data[3.9].data.max()
print global_data[10.8].data.min()
print global_data[10.8].data.max()
data=global_data.project("AfSubSahara")
print "\nDATA LOAD : OK"
else :
def read(self, filename):
import tables
self.cloudiness = MsgCTTHData() # Effective cloudiness
self.temperature = MsgCTTHData()
self.height = MsgCTTHData()
self.pressure = MsgCTTHData()
self.processing_flags = MsgCTTHData()
h5f = tables.openFile(filename)
# The header
# pylint: disable-msg=W0212
self.package = h5f.root._v_attrs["PACKAGE"]
self.saf = h5f.root._v_attrs["SAF"]
self.product_name = h5f.root._v_attrs["PRODUCT_NAME"]
self.num_of_columns = h5f.root._v_attrs["NC"]
self.num_of_lines = h5f.root._v_attrs["NL"]
self.projection_name = h5f.root._v_attrs["PROJECTION_NAME"]
self.region_name = h5f.root._v_attrs["REGION_NAME"]
self.cfac = h5f.root._v_attrs["CFAC"]
self.lfac = h5f.root._v_attrs["LFAC"]
self.coff = h5f.root._v_attrs["COFF"]
self.loff = h5f.root._v_attrs["LOFF"]
self.nb_param = h5f.root._v_attrs["NB_PARAMETERS"]
self.gp_sc_id = h5f.root._v_attrs["GP_SC_ID"]
self.image_acquisition_time = h5f.root._v_attrs["IMAGE_ACQUISITION_TIME"]
self.spectral_channel_id = h5f.root._v_attrs["SPECTRAL_CHANNEL_ID"]
self.nominal_product_time = h5f.root._v_attrs["NOMINAL_PRODUCT_TIME"]
self.sgs_product_quality = h5f.root._v_attrs["SGS_PRODUCT_QUALITY"]
self.sgs_product_completeness = h5f.root._v_attrs["SGS_PRODUCT_COMPLETENESS"]
self.product_algorithm_version = h5f.root._v_attrs["PRODUCT_ALGORITHM_VERSION"]
# pylint: enable-msg=W0212
# ------------------------
# The CTTH cloudiness data
self.cloudiness.data = h5f.root.CTTH_EFFECT[:, :]
self.cloudiness.scaling_factor = \
h5f.root.CTTH_EFFECT.attrs["SCALING_FACTOR"]
self.cloudiness.offset = h5f.root.CTTH_EFFECT.attrs["OFFSET"]
self.cloudiness.num_of_lines = h5f.root.CTTH_EFFECT.attrs["N_LINES"]
self.cloudiness.num_of_columns = h5f.root.CTTH_EFFECT.attrs["N_COLS"]
self.cloudiness.product = h5f.root.CTTH_EFFECT.attrs["PRODUCT"]
self.cloudiness.id = h5f.root.CTTH_EFFECT.attrs["ID"]
self.cloudiness.data = np.ma.masked_equal(self.cloudiness.data, 255)
self.cloudiness = np.ma.masked_equal(self.cloudiness.data, 0)
# ------------------------
# The CTTH temperature data
self.temperature.data = h5f.root.CTTH_TEMPER[:, :]
self.temperature.scaling_factor = \
h5f.root.CTTH_TEMPER.attrs["SCALING_FACTOR"]
self.temperature.offset = h5f.root.CTTH_TEMPER.attrs["OFFSET"]
self.temperature.num_of_lines = h5f.root.CTTH_TEMPER.attrs["N_LINES"]
self.shape = (self.temperature.num_of_lines,
self.temperature.num_of_columns)
self.temperature.num_of_columns = h5f.root.CTTH_TEMPER.attrs["N_COLS"]
self.temperature.product = h5f.root.CTTH_TEMPER.attrs["PRODUCT"]
self.temperature.id = h5f.root.CTTH_TEMPER.attrs["ID"]
self.temperature = (np.ma.masked_equal(self.temperature.data, 0) *
self.temperature.scaling_factor +
self.temperature.offset)
# ------------------------
# The CTTH pressure data
self.pressure.data = h5f.root.CTTH_PRESS[:, :]
self.pressure.scaling_factor = \
h5f.root.CTTH_PRESS.attrs["SCALING_FACTOR"]
self.pressure.offset = h5f.root.CTTH_PRESS.attrs["OFFSET"]
self.pressure.num_of_lines = h5f.root.CTTH_PRESS.attrs["N_LINES"]
self.pressure.num_of_columns = h5f.root.CTTH_PRESS.attrs["N_COLS"]
self.pressure.product = h5f.root.CTTH_PRESS.attrs["PRODUCT"]
self.pressure.id = h5f.root.CTTH_PRESS.attrs["ID"]
self.pressure.data = np.ma.masked_equal(self.pressure.data, 255)
self.pressure = (np.ma.masked_equal(self.pressure.data, 0) *
self.pressure.scaling_factor +
self.pressure.offset)
# ------------------------
# The CTTH height data
self.height.data = h5f.root.CTTH_HEIGHT[:, :]
self.height.scaling_factor = \
h5f.root.CTTH_HEIGHT.attrs["SCALING_FACTOR"]
self.height.offset = h5f.root.CTTH_HEIGHT.attrs["OFFSET"]
self.height.num_of_lines = h5f.root.CTTH_HEIGHT.attrs["N_LINES"]
self.height.num_of_columns = h5f.root.CTTH_HEIGHT.attrs["N_COLS"]
self.height.product = h5f.root.CTTH_HEIGHT.attrs["PRODUCT"]
self.height.id = h5f.root.CTTH_HEIGHT.attrs["ID"]
self.height.data = np.ma.masked_equal(self.height.data, 255)
self.height = (np.ma.masked_equal(self.height.data, 0) *
self.height.scaling_factor +
self.height.offset)
# ------------------------
# The CTTH processing/quality flags
self.processing_flags.data = h5f.root.CTTH_QUALITY[:, :]
self.processing_flags.scaling_factor = \
h5f.root.CTTH_QUALITY.attrs["SCALING_FACTOR"]
self.processing_flags.offset = h5f.root.CTTH_QUALITY.attrs["OFFSET"]
self.processing_flags.num_of_lines = \
h5f.root.CTTH_QUALITY.attrs["N_LINES"]
self.processing_flags.num_of_columns = \
h5f.root.CTTH_QUALITY.attrs["N_COLS"]
self.processing_flags.product = h5f.root.CTTH_QUALITY.attrs["PRODUCT"]
self.processing_flags.id = h5f.root.CTTH_QUALITY.attrs["ID"]
self.processing_flags = \
np.ma.masked_equal(self.processing_flags.data, 0)
h5f.close()
self.shape = self.height.shape
self.area = get_area_def(self.region_name)
self.filled = True
def load_generic(satscene,
options,
calibrate=True,
area_extent=None,
area_def_names=None,
filenames=None):
"""Read imager data from file and load it into *satscene*.
"""
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOGGER.debug("Channels to load from %s: %s" %
(satscene.instrument_name, satscene.channels_to_load))
# Compulsory global attributes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
from_area = False
if satscene.end_time is not None:
time_slot = satscene.time_slot, satscene.end_time
else:
time_slot = satscene.time_slot
if area_extent is None and satscene.area is not None:
if not satscene.area_def:
satscene.area = get_area_def(satscene.area_id)
area_extent = satscene.area.area_extent
from_area = True
area_converted_to_extent = False
for chn in satscene.channels_to_load:
use_filenames = False
# Sort out filenames
if filenames is not None:
for section in options.keys():
if section.endswith('-level1'):
break
try:
pattern_pro = eval(options[section].get('filename_pro'))
except TypeError:
pattern_pro = None
try:
pattern_epi = eval(options[section].get('filename_epi'))
except TypeError:
pattern_epi = None
pattern = eval(options[section].get('filename'))
epilogue = None
prologue = None
image_files = []
if pattern_epi is not None:
glob_epi = satscene.time_slot.strftime(pattern_epi) % (
{
'segment': "EPI".ljust(9, '_'),
'channel': chn + '*'
})
else:
glob_epi = 'eggs_and_spam'
if pattern_pro is not None:
glob_pro = satscene.time_slot.strftime(pattern_pro) % (
{
'segment': "PRO".ljust(9, '_'),
'channel': chn + '*'
})
else:
glob_pro = 'eggs_and_spam'
glob_img = satscene.time_slot.strftime(pattern) % (
{
'segment': "*",
'channel': chn + '*'
})
for filename in filenames:
if fnmatch.fnmatch(os.path.basename(filename), glob_img):
image_files.append(filename)
elif pattern_pro is not None and fnmatch.fnmatch(
os.path.basename(filename), glob_pro):
prologue = filename
elif pattern_epi is not None and fnmatch.fnmatch(
os.path.basename(filename), glob_epi):
epilogue = filename
if len(image_files) == 0 and prologue is None and epilogue is None:
use_filenames = False
else:
use_filenames = True
if from_area:
try:
if use_filenames:
metadata = xrit.sat.load_files(
prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
only_metadata=True)
else:
metadata = xrit.sat.load(satscene.fullname,
time_slot,
chn,
only_metadata=True)
if (satscene.area_def.proj_dict["proj"] != "geos"
or float(satscene.area_def.proj_dict["lon_0"]) !=
metadata.sublon):
raise ValueError("Slicing area must be in "
"geos projection, and lon_0 should match "
"the satellite's position.")
except ReaderError, err:
# if channel can't be found, go on with next channel
LOGGER.error(str(err))
continue
# Convert area definitions to maximal area_extent
if not area_converted_to_extent and area_def_names is not None:
try:
if use_filenames:
metadata = xrit.sat.load_files(
prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
only_metadata=True)
else:
metadata = xrit.sat.load(satscene.fullname,
time_slot,
chn,
only_metadata=True)
except ReaderError as err:
LOGGER.warning(str(err))
continue
# if area_extent is given, assume it gives the maximum
# extent of the satellite view
if area_extent is not None:
area_extent = area_def_names_to_extent(area_def_names,
metadata.proj4_params,
area_extent)
# otherwise use the default value (MSG3 extent at
# lon0=0.0), that is, do not pass default_extent=area_extent
else:
area_extent = area_def_names_to_extent(area_def_names,
metadata.proj4_params,
default_extent=None)
if area_extent is None:
LOGGER.info('Could not derive area_extent from area_def_names')
area_converted_to_extent = True
try:
if use_filenames:
image = xrit.sat.load_files(prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
mask=True,
calibrate=calibrate)
else:
image = xrit.sat.load(satscene.fullname,
time_slot,
chn,
mask=True,
calibrate=calibrate)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOGGER.warning(
"Loading non calibrated data since calibration failed.")
if use_filenames:
image = xrit.sat.load_files(prologue,
image_files,
epilogue,
platform_name=satscene.fullname,
mask=True,
calibrate=False)
else:
image = xrit.sat.load(satscene.fullname,
time_slot,
chn,
mask=True,
calibrate=False)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except ReaderError as err:
# if channel can't be found, go on with next channel
LOGGER.warning(str(err))
continue
satscene[chn] = data
satscene[chn].info['units'] = metadata.calibration_unit
satscene[chn].info['sublon'] = metadata.sublon
satscene[chn].info['satname'] = satscene.satname
satscene[chn].info['satnumber'] = satscene.number
satscene[chn].info['instrument_name'] = satscene.instrument_name
satscene[chn].info['time'] = satscene.time_slot
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split(" ")
proj_dict = {}
for param in proj_params:
key, val = param.split("=")
proj_dict[key] = val
if IS_PYRESAMPLE_LOADED:
# Build area_def on-the-fly
satscene[chn].area = geometry.AreaDefinition(
satscene.satname + satscene.instrument_name +
str(metadata.area_extent) + str(data.shape), "On-the-fly area",
proj_dict["proj"], proj_dict, data.shape[1], data.shape[0],
metadata.area_extent)
else:
LOGGER.info("Could not build area, pyresample missing...")
def add_overlay(self, color=(0, 0, 0), width=0.5, resolution=None):
"""Add coastline and political borders to image, using *color* (tuple
of integers between 0 and 255).
Warning: Loses the masks !
*resolution* is chosen automatically if None (default), otherwise it should be one of:
+-----+-------------------------+---------+
| 'f' | Full resolution | 0.04 km |
| 'h' | High resolution | 0.2 km |
| 'i' | Intermediate resolution | 1.0 km |
| 'l' | Low resolution | 5.0 km |
| 'c' | Crude resolution | 25 km |
+-----+-------------------------+---------+
"""
img = self.pil_image()
import ConfigParser
conf = ConfigParser.ConfigParser()
conf.read(os.path.join(CONFIG_PATH, "mpop.cfg"))
coast_dir = conf.get('shapes', 'dir')
logger.debug("Getting area for overlay: " + str(self.area))
if self.area is None:
raise ValueError("Area of image is None, can't add overlay.")
from mpop.projector import get_area_def
if isinstance(self.area, str):
self.area = get_area_def(self.area)
logger.info("Add coastlines and political borders to image.")
logger.debug("Area = " + str(self.area))
if resolution is None:
x_resolution = ((self.area.area_extent[2] -
self.area.area_extent[0]) /
self.area.x_size)
y_resolution = ((self.area.area_extent[3] -
self.area.area_extent[1]) /
self.area.y_size)
res = min(x_resolution, y_resolution)
if res > 25000:
resolution = "c"
elif res > 5000:
resolution = "l"
elif res > 1000:
resolution = "i"
elif res > 200:
resolution = "h"
else:
resolution = "f"
logger.debug("Automagically choose resolution " + resolution)
from pycoast import ContourWriterAGG
cw_ = ContourWriterAGG(coast_dir)
cw_.add_coastlines(img, self.area, outline=color,
resolution=resolution, width=width)
cw_.add_borders(img, self.area, outline=color,
resolution=resolution, width=width)
arr = np.array(img)
if len(self.channels) == 1:
self.channels[0] = np.ma.array(arr[:, :] / 255.0)
else:
for idx in range(len(self.channels)):
self.channels[idx] = np.ma.array(arr[:, :, idx] / 255.0)
def load_constant_fields(sat_nr):
# radar threshold mask:
radar_mask = GeostationaryFactory.create_scene("odyssey", "", "radar",
datetime(1900, 1, 1, 0))
# reproject this to the desired area:
mask_rad_thres = np.load(
'../data/odyssey_mask/threshold_exceedance_mask_avg15cut2_cut04_cutmistral_201706_201707_201708.npy'
)
from mpop.projector import get_area_def
area_radar_mask = 'EuropeOdyssey00'
radar_mask.channels.append(
Channel(name='mask_radar',
wavelength_range=[0., 0., 0.],
data=mask_rad_thres[:, :]))
radar_mask['mask_radar'].area = area_radar_mask
radar_mask['mask_radar'].area_def = get_area_def(area_radar_mask)
# nominal viewing geometry
print('*** read nominal viewing geometry', "meteosat", sat_nr, "seviri")
# time_slot has NO influence at all just goes looking for the nominal position file -> will use these fields for all dates
vg = GeostationaryFactory.create_scene("meteosat", sat_nr, "seviri",
datetime(1900, 1, 1, 0))
vg.load(['vaa', 'vza', 'lon', 'lat'], reader_level="seviri-level6")
msg_area = deepcopy(vg['vaa'].area)
msg_area_def = deepcopy(vg['vaa'].area_def)
msg_resolution = deepcopy(vg['vaa'].resolution)
# read land sea mask (full SEVIRI Disk seen from 0 degree East)
ls_file = '../data/SEVIRI_data/LandSeaMask_SeviriDiskFull00.nc'
fh = Dataset(ls_file, mode='r')
lsmask = fh.variables['lsmask'][:]
# read topography (full SEVIRI Disk seen from 0 degree East)
ls_file = '../data/SEVIRI_data/SRTM_15sec_elevation_SeviriDiskFull00.nc'
fh = Dataset(ls_file, mode='r')
ele = fh.variables['elevation'][:]
# create a dummy satellite object (to reproject the land/sea mask and elevation)
ls_ele = GeostationaryFactory.create_scene("meteosat", sat_nr, "seviri",
datetime(1900, 1, 1, 0))
#ls_ele.load(['CTTH'], calibrate=True, reader_level="seviri-level3")
#convert_NWCSAF_to_radiance_format(ls_ele, None,'CTH', False, True)
# add land sea mask as a dummy channel
ls_ele.channels.append(
Channel(name='lsmask',
wavelength_range=[0., 0., 0.],
resolution=msg_resolution,
data=lsmask[::-1, :]))
#ls_ele['lsmask'].area = ls_ele['CTH'].area
#ls_ele['lsmask'].area_def = ls_ele['CTH'].area_def
ls_ele['lsmask'].area = msg_area
ls_ele['lsmask'].area_def = msg_area_def
# add elevation as a dummy channel
ls_ele.channels.append(
Channel(name='ele',
wavelength_range=[0., 0., 0.],
resolution=msg_resolution,
data=ele[::-1, :]))
#ls_ele['ele'].area = ls_ele['CTH'].area
#ls_ele['ele'].area_def = ls_ele['CTH'].area_def
ls_ele['ele'].area = msg_area
ls_ele['ele'].area_def = msg_area_def
return radar_mask, vg, ls_ele