def co2corr(self):
"""CO2 correction of the brightness temperature of the MSG 3.9um
channel::
.. math::
T4_CO2corr = (BT(IR3.9)^4 + Rcorr)^0.25
Rcorr = BT(IR10.8)^4 - (BT(IR10.8)-dt_CO2)^4
dt_CO2 = (BT(IR10.8)-BT(IR13.4))/4.0
"""
try:
self.check_channels(3.75, 10.8, 13.4)
except RuntimeError:
LOG.warning("CO2 correction not performed, channel data missing.")
return
bt039 = self[3.9].data
bt108 = self[10.8].data
bt134 = self[13.4].data
dt_co2 = (bt108-bt134)/4.0
rcorr = bt108 ** 4 - (bt108-dt_co2) ** 4
t4_co2corr = bt039 ** 4 + rcorr
t4_co2corr = np.ma.where(t4_co2corr > 0.0, t4_co2corr, 0)
t4_co2corr = t4_co2corr ** 0.25
return t4_co2corr
def save(self, resave=False):
"""Save the precomputation to disk, and overwrite existing file in case
*resave* is true.
"""
if (not os.path.exists(self._filename)) or resave:
LOG.info("Saving projection to " +
self._filename)
np.savez(self._filename, **self._cache)
def _set_reader(self, pformat):
"""Gets the reader for *pformat* format, and puts it in the `reader`
attribute.
"""
elements = pformat.split(".")
if len(elements) == 1:
reader_module = pformat
reader = "mpop.satin."+reader_module
# Loading old style plugins
reader_module = pformat
LOG.info("old style plugin: " + pformat)
try:
# Look for builtin reader
loader = __import__(reader, globals(),
locals(), ['load'])
except ImportError:
# Look for custom reader
loader = __import__(reader_module, globals(),
locals(), ['load'])
# Build a custom Reader plugin on the fly...
from mpop.plugin_base import Reader
reader_class = type(elements[-1].capitalize() + "Reader",
(Reader,),
{"pformat": elements[-1]})
reader_instance = reader_class(self)
# ... and set its "load" attribute with the "load" function of the
# loader module
loader = getattr(loader, "load")
setattr(reader_instance, "load", loader)
setattr(self, elements[-1] + "_reader", reader_instance)
else:
reader_module = ".".join(elements[:-1])
reader_class = elements[-1]
reader = "mpop.satin."+reader_module
try:
# Look for builtin reader
loader = __import__(reader, globals(),
locals(), [reader_class])
except ImportError:
# Look for custom reader
loader = __import__(reader_module, globals(),
locals(), [reader_class])
loader = getattr(loader, reader_class)
reader_instance = loader(self)
setattr(self, loader.pformat + "_reader", reader_instance)
return reader_instance
def unload(self, *channels):
"""Unloads *channels* from
memory. :meth:`mpop.scene.SatelliteInstrumentScene.load` must be called
again to reload the data.
"""
for chn in channels:
try:
self[chn].data = None
except AttributeError:
LOG.warning("Can't unload channel" + str(chn))
def project(self, coverage_instance):
"""Make a projected copy of the current channel using the given
*coverage_instance*.
See also the :mod:`mpop.projector` module.
"""
res = copy.copy(self)
res.area = coverage_instance.out_area
res.data = None
if self.is_loaded():
LOG.info("Projecting channel %s (%fμm)..."
%(self.name, self.wavelength_range[1]))
data = coverage_instance.project_array(self._data)
res.data = data
return res
else:
raise NotLoadedError("Can't project, channel %s (%fμm) not loaded."
%(self.name, self.wavelength_range[1]))
def project(self, dest_area, channels=None, precompute=False, mode=None, radius=None):
"""Make a copy of the current snapshot projected onto the
*dest_area*. Available areas are defined in the region configuration
file (ACPG). *channels* tells which channels are to be projected, and
if None, all channels are projected and copied over to the return
snapshot.
If *precompute* is set to true, the projecting data is saved on disk
for reusage. *mode* sets the mode to project in: 'quick' which works
between cartographic projections, and, as its denomination indicates,
is quick (but lower quality), and 'nearest' which uses nearest
neighbour for best projection. A *mode* set to None uses 'quick' when
possible, 'nearest' otherwise.
*radius* defines the radius of influence for neighbour search in
'nearest' mode. Setting it to None, or omitting it will fallback to
default values (5 times the channel resolution) or 10km if the
resolution is not available.
Note: channels have to be loaded to be projected, otherwise an
exception is raised.
"""
if not is_pyresample_loaded:
# Not much point in proceeding then
return self
_channels = set([])
if channels is None:
for chn in self.loaded_channels():
_channels |= set([chn])
elif(isinstance(channels, (list, tuple, set))):
for chn in channels:
try:
_channels |= set([self[chn]])
except KeyError:
LOG.warning("Channel "+str(chn)+" not found,"
"thus not projected.")
else:
raise TypeError("Channels must be a list/"
"tuple/set of channel keys!")
res = copy.copy(self)
if isinstance(dest_area, str):
dest_area = mpop.projector.get_area_def(dest_area)
res.area = dest_area
res.channels = []
if not _channels <= self.loaded_channels():
LOG.warning("Cannot project nonloaded channels: %s."
%(_channels - self.loaded_channels()))
LOG.info("Will project the other channels though.")
_channels = _channels and self.loaded_channels()
cov = {}
for chn in _channels:
if chn.area is None:
if self.area is None:
area_name = ("swath_" + self.fullname + "_" +
str(self.time_slot) + "_"
+ str(chn.shape))
chn.area = area_name
else:
if is_pyresample_loaded:
try:
chn.area = AreaDefinition(
self.area.area_id + str(chn.shape),
self.area.name,
self.area.proj_id,
self.area.proj_dict,
chn.shape[1],
chn.shape[0],
self.area.area_extent,
self.area.nprocs)
except AttributeError:
try:
dummy = self.area.lons
dummy = self.area.lats
chn.area = self.area
area_name = ("swath_" + self.fullname + "_" +
str(self.time_slot) + "_"
+ str(chn.shape))
chn.area.area_id = area_name
except AttributeError:
chn.area = self.area + str(chn.shape)
else:
chn.area = self.area + str(chn.shape)
else: #chn.area is not None
if is_pyresample_loaded and isinstance(chn.area,
SwathDefinition):
area_name = ("swath_" + self.fullname + "_" +
str(self.time_slot) + "_"
+ str(chn.shape) + "_"
+ str(chn.name))
chn.area.area_id = area_name
if chn.area == dest_area:
res.channels.append(chn)
else:
if isinstance(chn.area, str):
area_id = chn.area
else:
area_id = chn.area_id or chn.area.area_id
if area_id not in cov:
if radius is None:
if chn.resolution > 0:
radius = 5 * chn.resolution
else:
radius = 10000
cov[area_id] = mpop.projector.Projector(chn.area,
dest_area,
mode=mode,
radius=radius)
if precompute:
try:
cov[area_id].save()
except IOError:
LOG.exception("Could not save projection.")
try:
res.channels.append(chn.project(cov[area_id]))
except NotLoadedError:
LOG.warning("Channel "+str(chn.name)+" not loaded, "
"thus not projected.")
# Compose with image object
try:
if res._CompositerClass is not None:
# Pass weak ref to compositor to allow garbage collection
res.image = res._CompositerClass(weakref.proxy(res))
except AttributeError:
pass
return res
def load(self, channels=None, load_again=False, area_extent=None, **kwargs):
"""Load instrument data into the *channels*. *Channels* is a list or a
tuple containing channels we will load data into, designated by there
center wavelength (float), resolution (integer) or name (string). If
None, all channels are loaded.
The *load_again* boolean flag allows to reload the channels even they
have already been loaded, to mirror changes on disk for example. This
is false by default.
The *area_extent* keyword lets you specify which part of the data to
load. Given as a 4-element sequence, it defines the area extent to load
in satellite projection.
The other keyword arguments are passed as is to the reader
plugin. Check the corresponding documentation for more details.
"""
# Set up the list of channels to load.
if channels is None:
for chn in self.channels:
self.channels_to_load |= set([chn.name])
elif(isinstance(channels, (list, tuple, set))):
for chn in channels:
try:
self.channels_to_load |= set([self[chn].name])
except KeyError:
self.channels_to_load |= set([chn])
else:
raise TypeError("Channels must be a list/"
"tuple/set of channel keys!")
loaded_channels = [chn.name for chn in self.loaded_channels()]
if load_again:
for chn in self.channels_to_load:
if chn in loaded_channels:
self.unload(chn)
loaded_channels = []
else:
for chn in loaded_channels:
self.channels_to_load -= set([chn])
# find the plugin to use from the config file
conf = ConfigParser.ConfigParser()
try:
conf.read(os.path.join(CONFIG_PATH, self.fullname + ".cfg"))
if len(conf.sections()) == 0:
raise ConfigParser.NoSectionError(("Config file did "
"not make sense"))
levels = [section for section in conf.sections()
if section.startswith(self.instrument_name+"-level")]
except ConfigParser.NoSectionError:
LOG.warning("Can't load data, no config file for " + self.fullname)
self.channels_to_load = set()
return
levels.sort()
if levels[0] == self.instrument_name+"-level1":
levels = levels[1:]
if len(levels) == 0:
raise ConfigParser.NoSectionError(
self.instrument_name+"-levelN (N>1) to tell me how to"+
" read data... Not reading anything.")
for level in levels:
if len(self.channels_to_load) == 0:
return
LOG.debug("Looking for sources in section "+level)
reader_name = conf.get(level, 'format')
try:
reader_name = eval(reader_name)
except NameError:
reader_name = str(reader_name)
LOG.debug("Using plugin mpop.satin."+reader_name)
# read the data
reader = "mpop.satin."+reader_name
try:
try:
# Look for builtin reader
reader_module = __import__(reader, globals(),
locals(), ['load'])
except ImportError:
# Look for custom reader
reader_module = __import__(reader_name, globals(),
locals(), ['load'])
if area_extent is not None:
if(isinstance(area_extent, (tuple, list)) and
len(area_extent) == 4):
kwargs["area_extent"] = area_extent
else:
raise ValueError("Area extent must be a sequence of "
"four numbers.")
reader_module.load(self, **kwargs)
except ImportError:
LOG.exception("ImportError while loading "+reader+".")
continue
loaded_channels = [chn.name for chn in self.loaded_channels()]
self.channels_to_load = set([chn for chn in self.channels_to_load
if not chn in loaded_channels])
LOG.debug("Successfully loaded: "+str(loaded_channels))
if len(self.channels_to_load) > 0:
LOG.warning("Unable to import channels "
+ str(self.channels_to_load))
self.channels_to_load = set()
from mpop import CONFIG_PATH
from mpop.channel import Channel, NotLoadedError
from mpop.logger import LOG
from mpop.utils import OrderedConfigParser
try:
# Work around for on demand import of pyresample. pyresample depends
# on scipy.spatial which memory leaks on multiple imports
is_pyresample_loaded = False
from pyresample.geometry import AreaDefinition, SwathDefinition
import mpop.projector
is_pyresample_loaded = True
except ImportError:
LOG.warning("pyresample missing. Can only work in satellite projection")
class Satellite(object):
"""This is the satellite class. It contains information on the satellite.
"""
def __init__(self, (satname, number, variant)=(None, None, None)):
try:
self.satname = satname or "" or self.satname
except AttributeError:
self.satname = satname or ""
try:
self.number = number or "" or self.number
except AttributeError:
self.number = number or ""
def __init__(self, in_area, out_area,
in_latlons=None, mode=None,
radius=10000):
# TODO:
# - Rework so that in_area and out_area can be lonlats.
# - Add a recompute flag ?
# Setting up the input area
try:
self.in_area = get_area_def(in_area)
in_id = in_area
except (utils.AreaNotFound, AttributeError):
if isinstance(in_area, geometry.AreaDefinition):
self.in_area = in_area
in_id = in_area.area_id
elif isinstance(in_area, geometry.SwathDefinition):
self.in_area = in_area
in_id = in_area.area_id
elif in_latlons is not None:
self.in_area = geometry.SwathDefinition(lons=in_latlons[0],
lats=in_latlons[1])
in_id = in_area
else:
raise utils.AreaNotFound("Input area " +
str(in_area) +
" must be defined in " +
AREA_FILE + ", be an area object"
" or longitudes/latitudes must be "
"provided.")
# Setting up the output area
try:
self.out_area = get_area_def(out_area)
out_id = out_area
except (utils.AreaNotFound, AttributeError):
if isinstance(out_area, (geometry.AreaDefinition,
geometry.SwathDefinition)):
self.out_area = out_area
out_id = out_area.area_id
else:
raise utils.AreaNotFound("Output area " +
str(out_area) +
" must be defined in " +
AREA_FILE + " or "
"be an area object.")
if self.in_area == self.out_area:
return
# choosing the right mode if necessary
if(mode is None):
if (isinstance(in_area, geometry.AreaDefinition) and
isinstance(out_area, geometry.AreaDefinition)):
self.mode = "quick"
else:
self.mode = "nearest"
else:
self.mode = mode
filename = (in_id + "2" + out_id + "_" + self.mode + ".npz")
projections_directory = "/var/tmp"
try:
projections_directory = CONF.get("projector",
"projections_directory")
except ConfigParser.NoSectionError:
pass
self._filename = os.path.join(projections_directory, filename)
if(not os.path.exists(self._filename)):
LOG.info("Computing projection from %s to %s..."
%(in_id, out_id))
if self.mode == "nearest":
valid_index, valid_output_index, index_array, distance_array = \
kd_tree.get_neighbour_info(self.in_area,
self.out_area,
radius,
neighbours=1)
del distance_array
self._cache = {}
self._cache['valid_index'] = valid_index
self._cache['valid_output_index'] = valid_output_index
self._cache['index_array'] = index_array
elif self.mode == "quick":
ridx, cidx = \
utils.generate_quick_linesample_arrays(self.in_area,
self.out_area)
self._cache = {}
self._cache['row_idx'] = ridx
self._cache['col_idx'] = cidx
else:
raise ValueError("Unrecognised mode " + str(self.mode) + ".")
else:
self._cache = {}
self._file_cache = np.load(self._filename)
import numpy as np
from pyresample import image, utils, geometry, kd_tree
from mpop import CONFIG_PATH
from mpop.logger import LOG
CONF = ConfigParser.ConfigParser()
CONF.read(os.path.join(CONFIG_PATH, "mpop.cfg"))
try:
AREA_FILE = os.path.join(CONF.get("projector", "area_directory") or
CONFIG_PATH,
CONF.get("projector", "area_file"))
except ConfigParser.NoSectionError:
AREA_FILE = ""
LOG.warning("Couldn't find the mpop.cfg file. "
"Do you have one ? is it in $PPP_CONFIG_DIR ?")
def get_area_def(area_name):
"""Get the definition of *area_name* from file. The file is defined to use
is to be placed in the $PPP_CONFIG_DIR directory, and its name is defined
in mpop's configuration file.
"""
return utils.parse_area_file(AREA_FILE, area_name)[0]
class Projector(object):
"""This class define projector objects. They contain the mapping
information necessary for projection purposes. For efficiency reasons,
generated projectors can be saved to disk for later reuse. Use the
:meth:`save` method for this.
To define a projector object, on has to specify *in_area* and *out_area*,
def average(self, downscaling_factor=2, average_window=None):
"""
Makes a mean convolution of an image.
:Parameters:
`downscaling_factor` : int
image downscaling factor, default is a factor 2.
`average_window` : int
window size for calculating mean values, default is
the same as downscaling_factor.
:Returns:
`image` : GeoImage
mean convoluted image.
"""
from mpop.imageo.geo_image import GeoImage
from pyresample import geometry
import scipy.ndimage as ndi
self.check_channels(9.65)
if average_window == None:
average_window = downscaling_factor
LOG.info("Downsampling a factor %d and averaging "%downscaling_factor +
"in a window of %dx%d"%(average_window, average_window))
ch = self[9.65]
# If average window and downscale factor is the same
# the following could be used:
#
# data = data.reshape([shight, hight/shight,
# swidth, width/swidth]).mean(3).mean(1)
# avg kernel
kernel = np.ones((average_window, average_window), dtype=np.float) / \
(average_window*average_window)
# do convolution
data = ndi.filters.correlate(ch.data.astype(np.float), kernel, mode='nearest')
# downscale
data = data[1::downscaling_factor, 1::downscaling_factor]
# New area, and correct for integer truncation.
p_size_x, p_size_y = (ch.area.pixel_size_x*downscaling_factor,
ch.area.pixel_size_y*downscaling_factor)
area_extent = (ch.area.area_extent[0],
ch.area.area_extent[1],
ch.area.area_extent[0] + data.shape[1]*p_size_x,
ch.area.area_extent[1] + data.shape[0]*p_size_y)
area = geometry.AreaDefinition(self._data_holder.satname +
self._data_holder.instrument_name +
str(area_extent) +
str(data.shape),
"On-the-fly area",
ch.area.proj_id, ch.area.proj_dict,
data.shape[1], data.shape[0],
area_extent)
return GeoImage(data, area, self.time_slot,
fill_value=(0,), mode='L')