def concatenate(self, channels=None):
"""Concatenate the granules into a swath. Returns a loaded satellite
scene.
"""
self.granules.sort()
conffile = (self.granules[0].variant +
self.granules[0].satname +
self.granules[0].number +
".cfg")
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, conffile))
try:
# call concat
reader_name = conf.get(self.granules[0].instrument_name +
"-level1", 'format')
str_granules = [str(granule) for granule in self.granules]
LOG.debug("Concatenating "+str(str_granules))
try:
reader_name = eval(reader_name)
except NameError:
reader_name = str(reader_name)
# read the data
reader = "mpop.satin."+reader_name
try:
reader_module = __import__(reader, globals(), locals(),
['concatenate'])
except ImportError:
LOG.exception("ImportError when loading plugin for format " +
str(reader_name))
raise ImportError("No "+reader+" reader found.")
scene = reader_module.concatenate(self.granules, channels)
except NoSectionError:
#concatenate loaded granules.
scenes = [GenericFactory.create_scene(granule.satname,
granule.number,
granule.instrument_name,
granule.time_slot,
None,
None,
granule.variant)
for granule in self.granules]
for granule in scenes:
granule.load(channels)
scene = mpop.scene.assemble_segments(scenes)
return scene
def assemble_segments(segments):
"""Assemble the scene objects listed in *segment_list* and returns the
resulting scene object.
"""
from mpop.satellites import GenericFactory
channels = set([])
for seg in segments:
channels |= set([chn.name for chn in seg.loaded_channels()])
seg = segments[0]
new_scene = GenericFactory.create_scene(seg.satname, seg.number,
seg.instrument_name, seg.time_slot,
seg.orbit, variant=seg.variant)
for seg in segments:
for chn in channels:
if not seg[chn].is_loaded():
# this makes the assumption that all channels have the same
# shape.
seg[chn] = np.ma.masked_all_like(
list(seg.loaded_channels())[0].data)
for chn in channels:
new_scene[chn] = np.ma.concatenate([seg[chn].data for seg in segments])
try:
lons = np.ma.concatenate([seg.area.lons[:] for seg in segments])
lats = np.ma.concatenate([seg.area.lats[:] for seg in segments])
new_scene.area = SwathDefinition(lons=lons, lats=lats)
for chn in channels:
if any([seg[chn].area for seg in segments]):
try:
lon_arrays = []
lat_arrays = []
for seg in segments:
if seg[chn].area is not None:
lon_arrays.append(seg[chn].area.lons[:])
lat_arrays.append(seg[chn].area.lats[:])
else:
lon_arrays.append(seg.area.lons[:])
lat_arrays.append(seg.area.lats[:])
lons = np.ma.concatenate(lon_arrays)
lats = np.ma.concatenate(lat_arrays)
new_scene[chn].area = SwathDefinition(lons=lons, lats=lats)
except AttributeError:
pass
except AttributeError:
pass
return new_scene
def concatenate(self, channels=None):
"""Concatenate the granules into a swath. Returns a loaded satellite
scene.
"""
self.granules.sort()
conffile = (self.granules[0].variant + self.granules[0].satname +
self.granules[0].number + ".cfg")
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, conffile))
try:
# call concat
reader_name = conf.get(
self.granules[0].instrument_name + "-level1", 'format')
str_granules = [str(granule) for granule in self.granules]
LOG.debug("Concatenating " + str(str_granules))
try:
reader_name = eval(reader_name)
except NameError:
reader_name = str(reader_name)
# read the data
reader = "mpop.satin." + reader_name
try:
reader_module = __import__(reader, globals(), locals(),
['concatenate'])
except ImportError:
LOG.exception("ImportError when loading plugin for format " +
str(reader_name))
raise ImportError("No " + reader + " reader found.")
scene = reader_module.concatenate(self.granules, channels)
except NoSectionError:
#concatenate loaded granules.
scenes = [
GenericFactory.create_scene(granule.satname, granule.number,
granule.instrument_name,
granule.time_slot, None, None,
granule.variant)
for granule in self.granules
]
for granule in scenes:
granule.load(channels)
scene = mpop.scene.assemble_segments(scenes)
return scene
def do_processing(msg):
from xml_read import read_product_file
pl, mda = read_product_file('/local_disk/usr/src/mpop-smhi.old/etc/noaa15_products.xml')
prods = pl["euron1"]
info = msg.data
from mpop.satellites import GenericFactory
from datetime import datetime
g = GenericFactory.create_scene(str(info["satellite"]), str(info["number"]), str(info["instrument"]), datetime.utcnow(), None)
g.load(get_prerequisites(g.image, prods.keys()))
for prod, files in prods.items():
img = getattr(g.image, prod)()
print "saving", img, "to", files
def run_from_cmd(self, hook=None):
"""Batch run mpop.
"""
time_slots, mode, areas, composites = parse_options()
tasklist = self.tasklist.shape(self.klass, mode, areas, composites)
for time_slot in time_slots:
self.data = GenericFactory.create_scene(self.satellite,
self.number,
self.instrument,
time_slot,
orbit=None,
variant=self.variant)
prerequisites = tasklist.get_prerequisites(self.klass)
self.data.load(prerequisites)
self.run_from_data(tasklist, hook)
def run_from_cmd(self, hook=None):
"""Batch run mpop.
"""
time_slots, mode, areas, composites = parse_options()
tasklist = self.tasklist.shape(self.klass, mode, areas, composites)
for time_slot in time_slots:
self.data = GenericFactory.create_scene(self.satellite,
self.number,
self.instrument,
time_slot,
orbit=None,
variant=self.variant)
prerequisites = tasklist.get_prerequisites(self.klass)
self.data.load(prerequisites)
self.run_from_data(tasklist, hook)
def create_scene_from_mda(self, mda):
"""Read the metadata *mda* and return a corresponding MPOP scene.
"""
time_slot = (mda.get('start_time') or
mda.get('nominal_time') or
mda.get('end_time'))
# orbit is not given for GEO satellites, use None
if 'orbit_number' not in mda:
mda['orbit_number'] = None
platform = mda["platform_name"]
LOGGER.info("platform %s time %s",
str(platform), str(time_slot))
if isinstance(mda['sensor'], (list, tuple, set)):
sensor = mda['sensor'][0]
else:
sensor = mda['sensor']
# Create satellite scene
global_data = GF.create_scene(satname=str(platform),
satnumber='',
instrument=str(sensor),
time_slot=time_slot,
orbit=mda['orbit_number'],
variant=mda.get('variant', ''))
LOGGER.debug("Creating scene for satellite %s and time %s",
str(platform), str(time_slot))
if mda['orbit_number'] is not None or mda.get('orbit_type') == "polar":
global_data.overpass = Pass(platform,
mda['start_time'],
mda['end_time'],
instrument=sensor)
# Update missing information to global_data.info{}
# TODO: this should be fixed in mpop.
global_data.info.update(mda)
global_data.info['time'] = time_slot
return global_data
def do_processing(msg):
from xml_read import read_product_file
pl, mda = read_product_file(
'/local_disk/usr/src/mpop-smhi.old/etc/noaa15_products.xml')
prods = pl["euron1"]
info = msg.data
from mpop.satellites import GenericFactory
from datetime import datetime
g = GenericFactory.create_scene(str(info["satellite"]),
str(info["number"]),
str(info["instrument"]), datetime.utcnow(),
None)
g.load(get_prerequisites(g.image, prods.keys()))
for prod, files in prods.items():
img = getattr(g.image, prod)()
print "saving", img, "to", files
def create_scene_from_mda(self, mda):
"""Read the metadata *mda* and return a corresponding MPOP scene.
"""
time_slot = (mda.get('start_time') or mda.get('nominal_time')
or mda.get('end_time'))
# orbit is not given for GEO satellites, use None
if 'orbit_number' not in mda:
mda['orbit_number'] = None
platform = mda["platform_name"]
LOGGER.info("platform %s time %s", str(platform), str(time_slot))
if isinstance(mda['sensor'], (list, tuple, set)):
sensor = mda['sensor'][0]
else:
sensor = mda['sensor']
# Create satellite scene
global_data = GF.create_scene(satname=str(platform),
satnumber='',
instrument=str(sensor),
time_slot=time_slot,
orbit=mda['orbit_number'],
variant=mda.get('variant', ''))
LOGGER.debug("Creating scene for satellite %s and time %s",
str(platform), str(time_slot))
if mda['orbit_number'] is not None or mda.get('orbit_type') == "polar":
global_data.overpass = Pass(platform,
mda['start_time'],
mda['end_time'],
instrument=sensor)
# Update missing information to global_data.info{}
# TODO: this should be fixed in mpop.
global_data.info.update(mda)
global_data.info['time'] = time_slot
return global_data
def create_scene_from_mda(self, mda, instruments):
"""Read the metadata *mda* and return a corresponding MPOP scene.
"""
time_slot = (mda.get('start_time') or mda.get('nominal_time')
or mda.get('end_time'))
# orbit is not given for GEO satellites, use None
if 'orbit_number' not in mda:
mda['orbit_number'] = None
platform = mda["platform_name"]
self.logger.debug("platform %s time %s", str(platform), str(time_slot))
if isinstance(mda['sensor'], (list, tuple, set)):
sensor = mda['sensor'][0]
else:
sensor = mda['sensor']
if sensor not in instruments:
self.logger.debug("Unknown sensor, skipping data.")
return None
# Create satellite scene
global_data = GF.create_scene(satname=str(platform),
satnumber='',
instrument=str(sensor),
time_slot=time_slot,
orbit=mda['orbit_number'],
variant=mda.get('variant', ''))
self.logger.info("Creating scene for satellite %s and time %s",
str(platform), str(time_slot))
# Update missing information to global_data.info{}
# TODO: this should be fixed in mpop.
global_data.info.update(mda)
global_data.info['time'] = time_slot
return global_data
def _load02(filename):
"""Load data from a netcdf4 file, cf-satellite v0.2 (2012-02-03).
"""
rootgrp = Dataset(filename, 'r')
# processed variables
processed = set()
# Currently MPOP does not like unicode (so much).
satellite_name, satellite_number = [str(i) for i in rootgrp.platform.rsplit("-", 1)]
time_slot = rootgrp.variables["time"].getValue()[0]
time_slot = num2date(time_slot, TIME_UNITS)
processed |= set(["time"])
try:
service = str(rootgrp.service)
except AttributeError:
service = ""
instrument_name = str(rootgrp.instrument)
try:
orbit = str(rootgrp.orbit)
except AttributeError:
orbit = None
try:
scene = GenericFactory.create_scene(satellite_name,
satellite_number,
instrument_name,
time_slot,
orbit,
None,
service)
except NoSectionError:
scene = VisirCompositer(time_slot=time_slot)
scene.satname = satellite_name
scene.number = satellite_number
scene.service = service
dim_chart = {}
for var_name, var in rootgrp.variables.items():
varname = None
try:
varname = var.standard_name
except AttributeError:
try:
varname = var.long_name
except AttributeError:
pass
if varname in ["band_data", "Band data"]:
LOG.debug("Found some data: " + var_name)
dims = var.dimensions
for dim in dims:
dim_chart[dim] = var_name
for cnt, dim in enumerate(dims):
if dim.startswith("band"):
break
data = var
data.set_auto_maskandscale(False)
area = None
try:
area_var_name = getattr(var,"grid_mapping")
area_var = rootgrp.variables[area_var_name]
proj4_dict = {}
for attr, projattr in MAPPING_ATTRIBUTES.items():
try:
the_attr = getattr(area_var, attr)
if projattr == "proj":
proj4_dict[projattr] = PROJNAME[the_attr]
elif(isinstance(projattr, (list, tuple))):
try:
for i, subattr in enumerate(the_attr):
proj4_dict[projattr[i]] = subattr
except TypeError:
proj4_dict[projattr[0]] = the_attr
else:
proj4_dict[projattr] = the_attr
except AttributeError:
pass
y_name, x_name = dims[:cnt] + dims[cnt + 1:]
x__ = rootgrp.variables[x_name][:]
y__ = rootgrp.variables[y_name][:]
if proj4_dict["proj"] == "geos":
x__ *= proj4_dict["h"]
y__ *= proj4_dict["h"]
x_pixel_size = abs((np.diff(x__)).mean())
y_pixel_size = abs((np.diff(y__)).mean())
llx = x__[0] - x_pixel_size / 2.0
lly = y__[-1] - y_pixel_size / 2.0
urx = x__[-1] + x_pixel_size / 2.0
ury = y__[0] + y_pixel_size / 2.0
area_extent = (llx, lly, urx, ury)
try:
# create the pyresample areadef
from pyresample.geometry import AreaDefinition
area = AreaDefinition("myareaid", "myareaname",
"myprojid", proj4_dict,
len(x__), len(y__),
area_extent)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
processed |= set([area_var_name, x_name, y_name])
LOG.debug("Grid mapping found and used.")
except AttributeError:
LOG.debug("No grid mapping found.")
try:
area_var = getattr(var,"coordinates")
coordinates_vars = area_var.split(" ")
lons = None
lats = None
for coord_var_name in coordinates_vars:
coord_var = rootgrp.variables[coord_var_name]
units = getattr(coord_var, "units")
if(coord_var_name.lower().startswith("lon") or
units.lower().endswith("east") or
units.lower().endswith("west")):
lons = coord_var[:]
elif(coord_var_name.lower().startswith("lat") or
units.lower().endswith("north") or
units.lower().endswith("south")):
lats = coord_var[:]
if lons.any() and lats.any():
try:
from pyresample.geometry import SwathDefinition
area = SwathDefinition(lons=lons, lats=lats)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
processed |= set(coordinates_vars)
LOG.debug("Lon/lat found and used.")
except AttributeError:
LOG.debug("No lon/lat found.")
names = rootgrp.variables[dim][:]
scales = data.scale_factor
offsets = data.add_offset
if len(names) == 1:
scales = np.array([scales])
offsets = np.array([offsets])
LOG.info("Scales and offsets: %s %s %s" % (str(names), str(scales), str(offsets)))
for nbr, name in enumerate(names):
name = str(name)
try:
if cnt == 0:
chn_data = data[nbr, :, :].squeeze()
if cnt == 1:
chn_data = data[:, nbr, :].squeeze()
if cnt == 2:
chn_data = data[:, :, nbr].squeeze()
scene[name] = (np.ma.masked_equal(chn_data, data._FillValue)
* scales[nbr] + offsets[nbr])
scene[name].info["units"] = var.units
except KeyError:
from mpop.channel import Channel
scene.channels.append(Channel(name))
if area is not None:
scene[name].area = area
processed |= set([var_name, dim])
non_processed = set(rootgrp.variables.keys()) - processed
for var_name in non_processed:
var = rootgrp.variables[var_name]
if not (hasattr(var, "standard_name") or
hasattr(var, "long_name")):
LOG.info("Delayed processing of " + var_name)
continue
dims = var.dimensions
if len(dims) != 1:
LOG.info("Don't know what to do with " + var_name)
continue
dim = dims[0]
if var.standard_name == "radiation_wavelength":
names = rootgrp.variables[dim][:]
for nbr, name in enumerate(names):
name = str(name)
scene[name].wavelength_range[1] = var[nbr]
try:
bnds = rootgrp.variables[var.bounds][:]
for nbr, name in enumerate(names):
name = str(name)
scene[name].wavelength_range[0] = bnds[nbr, 0]
scene[name].wavelength_range[2] = bnds[nbr, 1]
processed |= set([var.bounds])
except AttributeError:
pass
processed |= set([var_name])
non_processed = set(rootgrp.variables.keys()) - processed
if len(non_processed) > 0:
LOG.warning("Remaining non-processed variables: " + str(non_processed))
return scene
def assemble_segments(segments):
"""Assemble the scene objects listed in *segment_list* and returns the
resulting scene object.
"""
from mpop.satellites import GenericFactory
channels = set([])
for seg in segments:
channels |= set([chn.name for chn in seg.loaded_channels()])
seg = segments[0]
new_scene = GenericFactory.create_scene(seg.satname, seg.number,
seg.instrument_name, seg.time_slot,
seg.orbit, variant=seg.variant)
swath_definitions = {}
for chn in channels:
new_scene[chn] = np.ma.concatenate([seg[chn].data
for seg in segments
if seg[chn].is_loaded()])
try:
area_names = tuple([seg[chn].area.area_id
for seg in segments
if seg[chn].is_loaded()])
if area_names not in swath_definitions:
lons = np.ma.concatenate([seg[chn].area.lons[:]
for seg in segments
if seg[chn].is_loaded()])
lats = np.ma.concatenate([seg[chn].area.lats[:]
for seg in segments
if seg[chn].is_loaded()])
new_scene[chn].area = SwathDefinition(lons=lons, lats=lats)
area_name = "+".join(area_names)
new_scene[chn].area.area_id = area_name
new_scene[chn].area_id = area_name
swath_definitions[area_names] = new_scene[chn].area
else:
new_scene[chn].area = swath_definitions[area_names]
new_scene[chn].area_id = new_scene[chn].area.area_id
except AttributeError:
pass
try:
lons = np.ma.concatenate([seg.area.lons[:] for seg in segments])
lats = np.ma.concatenate([seg.area.lats[:] for seg in segments])
new_scene.area = SwathDefinition(lons=lons, lats=lats)
for chn in channels:
if any([seg[chn].area for seg in segments]):
try:
lon_arrays = []
lat_arrays = []
for seg in segments:
if seg[chn].area is not None:
lon_arrays.append(seg[chn].area.lons[:])
lat_arrays.append(seg[chn].area.lats[:])
else:
lon_arrays.append(seg.area.lons[:])
lat_arrays.append(seg.area.lats[:])
lons = np.ma.concatenate(lon_arrays)
lats = np.ma.concatenate(lat_arrays)
new_scene[chn].area = SwathDefinition(lons=lons, lats=lats)
except AttributeError:
pass
except AttributeError:
pass
return new_scene
def load_from_nc4(filename):
"""Load data from a netcdf4 file.
"""
rootgrp = Dataset(filename, 'r')
time_slot = rootgrp.variables["time"].getValue()[0]
time_slot = num2date(time_slot, TIME_UNITS)
if not isinstance(rootgrp.satellite_number, str):
satellite_number = "%02d" % rootgrp.satellite_number
else:
satellite_number = str(rootgrp.satellite_number)
service = str(rootgrp.service)
satellite_name = str(rootgrp.satellite_name)
instrument_name = str(rootgrp.instrument_name)
try:
orbit = str(rootgrp.orbit)
except AttributeError:
orbit = None
try:
scene = GenericFactory.create_scene(satellite_name,
satellite_number,
instrument_name,
time_slot,
orbit,
None,
service)
except NoSectionError:
scene = VisirCompositer(time_slot=time_slot)
scene.satname = satellite_name
scene.number = satellite_number
scene.service = service
for var_name, var in rootgrp.variables.items():
area = None
if var_name.startswith("band_data"):
resolution = var.resolution
str_res = str(int(resolution)) + "m"
names = rootgrp.variables["bandname"+str_res][:]
data = var[:, :, :].astype(var.dtype)
data = np.ma.masked_outside(data,
var.valid_range[0],
var.valid_range[1])
try:
area_var = getattr(var,"grid_mapping")
area_var = rootgrp.variables[area_var]
proj4_dict = {}
for attr, projattr in MAPPING_ATTRIBUTES.items():
try:
the_attr = getattr(area_var, attr)
if projattr == "proj":
proj4_dict[projattr] = PROJNAME[the_attr]
elif(isinstance(projattr, (list, tuple))):
try:
for i, subattr in enumerate(the_attr):
proj4_dict[projattr[i]] = subattr
except TypeError:
proj4_dict[projattr[0]] = the_attr
else:
proj4_dict[projattr] = the_attr
except AttributeError:
pass
x__ = rootgrp.variables["x"+str_res][:]
y__ = rootgrp.variables["y"+str_res][:]
x_pixel_size = abs((x__[1] - x__[0]))
y_pixel_size = abs((y__[1] - y__[0]))
llx = x__[0] - x_pixel_size / 2.0
lly = y__[-1] - y_pixel_size / 2.0
urx = x__[-1] + x_pixel_size / 2.0
ury = y__[0] + y_pixel_size / 2.0
area_extent = (llx, lly, urx, ury)
try:
# create the pyresample areadef
from pyresample.geometry import AreaDefinition
area = AreaDefinition("myareaid", "myareaname",
"myprojid", proj4_dict,
data.shape[1], data.shape[0],
area_extent)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
except AttributeError:
LOG.debug("No grid mapping found.")
try:
area_var = getattr(var,"coordinates")
coordinates_vars = area_var.split(" ")
lons = None
lats = None
for coord_var_name in coordinates_vars:
coord_var = rootgrp.variables[coord_var_name]
units = getattr(coord_var, "units")
if(coord_var_name.lower().startswith("lon") or
units.lower().endswith("east") or
units.lower().endswith("west")):
lons = coord_var[:]
elif(coord_var_name.lower().startswith("lat") or
units.lower().endswith("north") or
units.lower().endswith("south")):
lats = coord_var[:]
if lons and lats:
try:
from pyresample.geometry import SwathDefinition
area = SwathDefinition(lons=lons, lats=lats)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
except AttributeError:
LOG.debug("No lon/lat found.")
for i, name in enumerate(names):
try:
scene[name] = (data[:, :, i] *
rootgrp.variables["scale"+str_res][i] +
rootgrp.variables["offset"+str_res][i])
#FIXME complete this
#scene[name].info
except KeyError:
# build the channel on the fly
from mpop.channel import Channel
wv_var = rootgrp.variables["nominal_wavelength"+str_res]
wb_var = rootgrp.variables[getattr(wv_var, "bounds")]
minmax = wb_var[i]
scene.channels.append(Channel(name,
resolution,
(minmax[0],
wv_var[i][0],
minmax[1])))
scene[name] = (data[:, :, i] *
rootgrp.variables["scale"+str_res][i] +
rootgrp.variables["offset"+str_res][i])
if area is not None:
scene[name].area = area
area = None
for attr in rootgrp.ncattrs():
scene.info[attr] = getattr(rootgrp, attr)
scene.add_to_history("Loaded from netcdf4/cf by mpop")
return scene
def assemble_segments(segments):
"""Assemble the scene objects listed in *segment_list* and returns the
resulting scene object.
"""
from mpop.satellites import GenericFactory
channels = set([])
for seg in segments:
channels |= set([chn.name for chn in seg.loaded_channels()])
seg = segments[0]
new_scene = GenericFactory.create_scene(seg.satname,
seg.number,
seg.instrument_name,
seg.time_slot,
seg.orbit,
variant=seg.variant)
swath_definitions = {}
for chn in channels:
new_scene[chn] = np.ma.concatenate(
[seg[chn].data for seg in segments if seg[chn].is_loaded()])
try:
area_names = tuple([
seg[chn].area.area_id for seg in segments
if seg[chn].is_loaded()
])
if area_names not in swath_definitions:
lons = np.ma.concatenate([
seg[chn].area.lons[:] for seg in segments
if seg[chn].is_loaded()
])
lats = np.ma.concatenate([
seg[chn].area.lats[:] for seg in segments
if seg[chn].is_loaded()
])
new_scene[chn].area = SwathDefinition(lons=lons, lats=lats)
area_name = "+".join(area_names)
new_scene[chn].area.area_id = area_name
new_scene[chn].area_id = area_name
swath_definitions[area_names] = new_scene[chn].area
else:
new_scene[chn].area = swath_definitions[area_names]
new_scene[chn].area_id = new_scene[chn].area.area_id
except AttributeError:
pass
try:
lons = np.ma.concatenate([seg.area.lons[:] for seg in segments])
lats = np.ma.concatenate([seg.area.lats[:] for seg in segments])
new_scene.area = SwathDefinition(lons=lons, lats=lats)
for chn in channels:
if any([seg[chn].area for seg in segments]):
try:
lon_arrays = []
lat_arrays = []
for seg in segments:
if seg[chn].area is not None:
lon_arrays.append(seg[chn].area.lons[:])
lat_arrays.append(seg[chn].area.lats[:])
else:
lon_arrays.append(seg.area.lons[:])
lat_arrays.append(seg.area.lats[:])
lons = np.ma.concatenate(lon_arrays)
lats = np.ma.concatenate(lat_arrays)
new_scene[chn].area = SwathDefinition(lons=lons, lats=lats)
except AttributeError:
pass
except AttributeError:
pass
return new_scene
def load_from_nc4(filename):
"""Load data from a netcdf4 file, cf-satellite v0.1
"""
rootgrp = Dataset(filename, 'r')
try:
rootgrp.satellite_number
warnings.warn("You are loading old style netcdf files...",
DeprecationWarning)
except AttributeError:
return _load02(filename)
if not isinstance(rootgrp.satellite_number, str):
satellite_number = "%02d" % rootgrp.satellite_number
else:
satellite_number = str(rootgrp.satellite_number)
time_slot = rootgrp.variables["time"].getValue()[0]
time_slot = num2date(time_slot, TIME_UNITS)
service = str(rootgrp.service)
satellite_name = str(rootgrp.satellite_name)
instrument_name = str(rootgrp.instrument_name)
try:
orbit = str(rootgrp.orbit)
except AttributeError:
orbit = None
try:
scene = GenericFactory.create_scene(satellite_name, satellite_number,
instrument_name, time_slot, orbit,
None, service)
except NoSectionError:
scene = VisirCompositer(time_slot=time_slot)
scene.satname = satellite_name
scene.number = satellite_number
scene.service = service
for var_name, var in rootgrp.variables.items():
area = None
if var_name.startswith("band_data"):
resolution = var.resolution
str_res = str(int(resolution)) + "m"
names = rootgrp.variables["bandname" + str_res][:]
data = var[:, :, :].astype(var.dtype)
data = np.ma.masked_outside(data, var.valid_range[0],
var.valid_range[1])
try:
area_var = getattr(var, "grid_mapping")
area_var = rootgrp.variables[area_var]
proj4_dict = {}
for attr, projattr in MAPPING_ATTRIBUTES.items():
try:
the_attr = getattr(area_var, attr)
if projattr == "proj":
proj4_dict[projattr] = PROJNAME[the_attr]
elif (isinstance(projattr, (list, tuple))):
try:
for i, subattr in enumerate(the_attr):
proj4_dict[projattr[i]] = subattr
except TypeError:
proj4_dict[projattr[0]] = the_attr
else:
proj4_dict[projattr] = the_attr
except AttributeError:
pass
x__ = rootgrp.variables["x" + str_res][:]
y__ = rootgrp.variables["y" + str_res][:]
x_pixel_size = abs((x__[1] - x__[0]))
y_pixel_size = abs((y__[1] - y__[0]))
llx = x__[0] - x_pixel_size / 2.0
lly = y__[-1] - y_pixel_size / 2.0
urx = x__[-1] + x_pixel_size / 2.0
ury = y__[0] + y_pixel_size / 2.0
area_extent = (llx, lly, urx, ury)
try:
# create the pyresample areadef
from pyresample.geometry import AreaDefinition
area = AreaDefinition("myareaid", "myareaname", "myprojid",
proj4_dict, data.shape[1],
data.shape[0], area_extent)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
except AttributeError:
LOG.debug("No grid mapping found.")
try:
area_var = getattr(var, "coordinates")
coordinates_vars = area_var.split(" ")
lons = None
lats = None
for coord_var_name in coordinates_vars:
coord_var = rootgrp.variables[coord_var_name]
units = getattr(coord_var, "units")
if (coord_var_name.lower().startswith("lon")
or units.lower().endswith("east")
or units.lower().endswith("west")):
lons = coord_var[:]
elif (coord_var_name.lower().startswith("lat")
or units.lower().endswith("north")
or units.lower().endswith("south")):
lats = coord_var[:]
if lons and lats:
try:
from pyresample.geometry import SwathDefinition
area = SwathDefinition(lons=lons, lats=lats)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
except AttributeError:
LOG.debug("No lon/lat found.")
for i, name in enumerate(names):
name = str(name)
if var.dimensions[0].startswith("band"):
chn_data = data[i, :, :]
elif var.dimensions[1].startswith("band"):
chn_data = data[:, i, :]
elif var.dimensions[2].startswith("band"):
chn_data = data[:, :, i]
else:
raise ValueError("Invalid dimension names for band data")
try:
scene[name] = (
chn_data * rootgrp.variables["scale" + str_res][i] +
rootgrp.variables["offset" + str_res][i])
#FIXME complete this
#scene[name].info
except KeyError:
# build the channel on the fly
from mpop.channel import Channel
wv_var = rootgrp.variables["nominal_wavelength" + str_res]
wb_var = rootgrp.variables[getattr(wv_var, "bounds")]
minmax = wb_var[i]
scene.channels.append(
Channel(name, resolution,
(minmax[0], wv_var[i][0], minmax[1])))
scene[name] = (
chn_data * rootgrp.variables["scale" + str_res][i] +
rootgrp.variables["offset" + str_res][i])
if area is not None:
scene[name].area = area
area = None
for attr in rootgrp.ncattrs():
scene.info[attr] = getattr(rootgrp, attr)
scene.add_to_history("Loaded from netcdf4/cf by mpop")
return scene
import datetime as dt
from mpop.satellites import GenericFactory as GF
tslot = dt.datetime(2015, 7, 7, 16)
glbl = GF.create_scene("Meteosat-9", "", "seviri", tslot, None)
glbl.load([0.6, 0.8, 10.8], area_def_names=[
"euro4",
])
#lcl_bil = glbl.project('euro4', nprocs=1, mode="bilinear", radius=50e3, precompute=True)
lcl_bil = glbl.project('euro4', nprocs=1, mode="ewa", precompute=True)
img = lcl_bil.image.overview()
img.show()
def run_single(self):
'''Run image production without threading.
'''
# TODO: Get relevant preprocessing function for this
# production chain type: single/multi, or
# swath, geo, granule, global_polar, global_geo, global_mixed
# That is, gatherer for the multi-image/multi-granule types
# preproc_func = getattr(preprocessing, self.production_type)
while True:
# wait for new messages
msg = self.listener_parent_conn.recv()
print msg
# shutdown trollduction
if msg.subject == '/StopTrollduction':
self.cleanup()
break
#self.shutdown()
# update trollduction config
elif msg.subject == '/NewTrollductionConfig':
self.update_td_config(msg.data)
# update product lists
elif msg.subject == '/NewProductConfig':
self.udpate_product_config(msg.data)
# process new file
elif '/NewFileArrived' in msg.subject:
self.time_slot = dt.datetime(msg.data['year'],
msg.data['month'],
msg.data['day'],
msg.data['hour'],
msg.data['minute'])
# orbit is empty string for meteosat, change it to None
if msg.data['orbit'] == '': msg.data['orbit'] = None
t1a = time.time()
self.global_data = GF.create_scene(satname=str(msg.data['satellite']),
satnumber=str(msg.data['satnumber']),
instrument=str(msg.data['instrument']),
time_slot=self.time_slot,
orbit=str(msg.data['orbit']))
# Find maximum extent that is needed for all the
# products to be made.
self.get_maximum_extent()
# Make images for each area
for area_name in self.area_def_names:
t1b = time.time()
# Check which channels are needed. Unload
# unnecessary channels and load those that are not
# already available.
self.load_unload_channels(self.product_list[area_name])
# TODO: or something
# reproject to local domain
self.local_data = self.global_data.project(area_name, mode='nearest')
print "Data reprojected for area:", area_name
# Draw requested images for this area.
self.draw_images(area_name)
print "Single area time elapsed time:", time.time()-t1b, 's'
self.local_data = None
self.global_data = None
# self.loaded_channels = []
print "Full time elapsed time:", time.time()-t1a, 's'
else:
# Unhandled message types end up here
# No need to log these?
pass
def _load02(filename):
"""Load data from a netcdf4 file, cf-satellite v0.2 (2012-02-03).
"""
rootgrp = Dataset(filename, 'r')
# processed variables
processed = set()
# Currently MPOP does not like unicode (so much).
satellite_name, satellite_number = [
str(i) for i in rootgrp.platform.rsplit("-", 1)
]
time_slot = rootgrp.variables["time"].getValue()[0]
time_slot = num2date(time_slot, TIME_UNITS)
processed |= set(["time"])
try:
service = str(rootgrp.service)
except AttributeError:
service = ""
instrument_name = str(rootgrp.instrument)
try:
orbit = str(rootgrp.orbit)
except AttributeError:
orbit = None
try:
scene = GenericFactory.create_scene(satellite_name, satellite_number,
instrument_name, time_slot, orbit,
None, service)
except NoSectionError:
scene = VisirCompositer(time_slot=time_slot)
scene.satname = satellite_name
scene.number = satellite_number
scene.service = service
dim_chart = {}
for var_name, var in rootgrp.variables.items():
varname = None
try:
varname = var.standard_name
except AttributeError:
try:
varname = var.long_name
except AttributeError:
pass
if varname in ["band_data", "Band data"]:
LOG.debug("Found some data: " + var_name)
dims = var.dimensions
for dim in dims:
dim_chart[dim] = var_name
for cnt, dim in enumerate(dims):
if dim.startswith("band"):
break
data = var
data.set_auto_maskandscale(False)
area = None
try:
area_var_name = getattr(var, "grid_mapping")
area_var = rootgrp.variables[area_var_name]
proj4_dict = {}
for attr, projattr in MAPPING_ATTRIBUTES.items():
try:
the_attr = getattr(area_var, attr)
if projattr == "proj":
proj4_dict[projattr] = PROJNAME[the_attr]
elif (isinstance(projattr, (list, tuple))):
try:
for i, subattr in enumerate(the_attr):
proj4_dict[projattr[i]] = subattr
except TypeError:
proj4_dict[projattr[0]] = the_attr
else:
proj4_dict[projattr] = the_attr
except AttributeError:
pass
y_name, x_name = dims[:cnt] + dims[cnt + 1:]
x__ = rootgrp.variables[x_name][:]
y__ = rootgrp.variables[y_name][:]
if proj4_dict["proj"] == "geos":
x__ *= proj4_dict["h"]
y__ *= proj4_dict["h"]
x_pixel_size = abs((np.diff(x__)).mean())
y_pixel_size = abs((np.diff(y__)).mean())
llx = x__[0] - x_pixel_size / 2.0
lly = y__[-1] - y_pixel_size / 2.0
urx = x__[-1] + x_pixel_size / 2.0
ury = y__[0] + y_pixel_size / 2.0
area_extent = (llx, lly, urx, ury)
try:
# create the pyresample areadef
from pyresample.geometry import AreaDefinition
area = AreaDefinition("myareaid", "myareaname",
"myprojid", proj4_dict, len(x__),
len(y__), area_extent)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
processed |= set([area_var_name, x_name, y_name])
LOG.debug("Grid mapping found and used.")
except AttributeError:
LOG.debug("No grid mapping found.")
try:
area_var = getattr(var, "coordinates")
coordinates_vars = area_var.split(" ")
lons = None
lats = None
for coord_var_name in coordinates_vars:
coord_var = rootgrp.variables[coord_var_name]
units = getattr(coord_var, "units")
if (coord_var_name.lower().startswith("lon")
or units.lower().endswith("east")
or units.lower().endswith("west")):
lons = coord_var[:]
elif (coord_var_name.lower().startswith("lat")
or units.lower().endswith("north")
or units.lower().endswith("south")):
lats = coord_var[:]
if lons.any() and lats.any():
try:
from pyresample.geometry import SwathDefinition
area = SwathDefinition(lons=lons, lats=lats)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
processed |= set(coordinates_vars)
LOG.debug("Lon/lat found and used.")
except AttributeError:
LOG.debug("No lon/lat found.")
names = rootgrp.variables[dim][:]
scales = data.scale_factor
offsets = data.add_offset
if len(names) == 1:
scales = np.array([scales])
offsets = np.array([offsets])
LOG.info("Scales and offsets: %s %s %s" %
(str(names), str(scales), str(offsets)))
for nbr, name in enumerate(names):
name = str(name)
try:
if cnt == 0:
chn_data = data[nbr, :, :].squeeze()
if cnt == 1:
chn_data = data[:, nbr, :].squeeze()
if cnt == 2:
chn_data = data[:, :, nbr].squeeze()
scene[name] = (
np.ma.masked_equal(chn_data, data._FillValue) *
scales[nbr] + offsets[nbr])
scene[name].info["units"] = var.units
except KeyError:
from mpop.channel import Channel
scene.channels.append(Channel(name))
if area is not None:
scene[name].area = area
processed |= set([var_name, dim])
non_processed = set(rootgrp.variables.keys()) - processed
for var_name in non_processed:
var = rootgrp.variables[var_name]
if not (hasattr(var, "standard_name") or hasattr(var, "long_name")):
LOG.info("Delayed processing of " + var_name)
continue
dims = var.dimensions
if len(dims) != 1:
LOG.info("Don't know what to do with " + var_name)
continue
dim = dims[0]
if var.standard_name == "radiation_wavelength":
names = rootgrp.variables[dim][:]
for nbr, name in enumerate(names):
name = str(name)
scene[name].wavelength_range[1] = var[nbr]
try:
bnds = rootgrp.variables[var.bounds][:]
for nbr, name in enumerate(names):
name = str(name)
scene[name].wavelength_range[0] = bnds[nbr, 0]
scene[name].wavelength_range[2] = bnds[nbr, 1]
processed |= set([var.bounds])
except AttributeError:
pass
processed |= set([var_name])
non_processed = set(rootgrp.variables.keys()) - processed
if len(non_processed) > 0:
LOG.warning("Remaining non-processed variables: " + str(non_processed))
return scene
def run_single(self):
'''Run image production without threading.
'''
# TODO: Get relevant preprocessing function for this
# production chain type: single/multi, or
# swath, geo, granule, global_polar, global_geo, global_mixed
# That is, gatherer for the multi-image/multi-granule types
# preproc_func = getattr(preprocessing, self.production_type)
while True:
# wait for new messages
msg = self.listener_parent_conn.recv()
print msg
# shutdown trollduction
if msg.subject == '/StopTrollduction':
self.cleanup()
break
#self.shutdown()
# update trollduction config
elif msg.subject == '/NewTrollductionConfig':
self.update_td_config(msg.data)
# update product lists
elif msg.subject == '/NewProductConfig':
self.udpate_product_config(msg.data)
# process new file
elif '/NewFileArrived' in msg.subject:
self.time_slot = dt.datetime(msg.data['year'],
msg.data['month'],
msg.data['day'], msg.data['hour'],
msg.data['minute'])
# orbit is empty string for meteosat, change it to None
if msg.data['orbit'] == '': msg.data['orbit'] = None
t1a = time.time()
self.global_data = GF.create_scene(
satname=str(msg.data['satellite']),
satnumber=str(msg.data['satnumber']),
instrument=str(msg.data['instrument']),
time_slot=self.time_slot,
orbit=str(msg.data['orbit']))
# Find maximum extent that is needed for all the
# products to be made.
self.get_maximum_extent()
# Make images for each area
for area_name in self.area_def_names:
t1b = time.time()
# Check which channels are needed. Unload
# unnecessary channels and load those that are not
# already available.
self.load_unload_channels(self.product_list[area_name])
# TODO: or something
# reproject to local domain
self.local_data = self.global_data.project(area_name,
mode='nearest')
print "Data reprojected for area:", area_name
# Draw requested images for this area.
self.draw_images(area_name)
print "Single area time elapsed time:", time.time(
) - t1b, 's'
self.local_data = None
self.global_data = None
# self.loaded_channels = []
print "Full time elapsed time:", time.time() - t1a, 's'
else:
# Unhandled message types end up here
# No need to log these?
pass
