def concatenate(self, channels=None): """Concatenate the granules into a swath. Returns a loaded satellite scene. """ self.granules.sort() conffile = (self.granules[0].variant + self.granules[0].satname + self.granules[0].number + ".cfg") conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, conffile)) try: # call concat reader_name = conf.get(self.granules[0].instrument_name + "-level1", 'format') str_granules = [str(granule) for granule in self.granules] LOG.debug("Concatenating "+str(str_granules)) try: reader_name = eval(reader_name) except NameError: reader_name = str(reader_name) # read the data reader = "mpop.satin."+reader_name try: reader_module = __import__(reader, globals(), locals(), ['concatenate']) except ImportError: LOG.exception("ImportError when loading plugin for format " + str(reader_name)) raise ImportError("No "+reader+" reader found.") scene = reader_module.concatenate(self.granules, channels) except NoSectionError: #concatenate loaded granules. scenes = [GenericFactory.create_scene(granule.satname, granule.number, granule.instrument_name, granule.time_slot, None, None, granule.variant) for granule in self.granules] for granule in scenes: granule.load(channels) scene = mpop.scene.assemble_segments(scenes) return scene
def assemble_segments(segments): """Assemble the scene objects listed in *segment_list* and returns the resulting scene object. """ from mpop.satellites import GenericFactory channels = set([]) for seg in segments: channels |= set([chn.name for chn in seg.loaded_channels()]) seg = segments[0] new_scene = GenericFactory.create_scene(seg.satname, seg.number, seg.instrument_name, seg.time_slot, seg.orbit, variant=seg.variant) for seg in segments: for chn in channels: if not seg[chn].is_loaded(): # this makes the assumption that all channels have the same # shape. seg[chn] = np.ma.masked_all_like( list(seg.loaded_channels())[0].data) for chn in channels: new_scene[chn] = np.ma.concatenate([seg[chn].data for seg in segments]) try: lons = np.ma.concatenate([seg.area.lons[:] for seg in segments]) lats = np.ma.concatenate([seg.area.lats[:] for seg in segments]) new_scene.area = SwathDefinition(lons=lons, lats=lats) for chn in channels: if any([seg[chn].area for seg in segments]): try: lon_arrays = [] lat_arrays = [] for seg in segments: if seg[chn].area is not None: lon_arrays.append(seg[chn].area.lons[:]) lat_arrays.append(seg[chn].area.lats[:]) else: lon_arrays.append(seg.area.lons[:]) lat_arrays.append(seg.area.lats[:]) lons = np.ma.concatenate(lon_arrays) lats = np.ma.concatenate(lat_arrays) new_scene[chn].area = SwathDefinition(lons=lons, lats=lats) except AttributeError: pass except AttributeError: pass return new_scene
def concatenate(self, channels=None): """Concatenate the granules into a swath. Returns a loaded satellite scene. """ self.granules.sort() conffile = (self.granules[0].variant + self.granules[0].satname + self.granules[0].number + ".cfg") conf = ConfigParser() conf.read(os.path.join(CONFIG_PATH, conffile)) try: # call concat reader_name = conf.get( self.granules[0].instrument_name + "-level1", 'format') str_granules = [str(granule) for granule in self.granules] LOG.debug("Concatenating " + str(str_granules)) try: reader_name = eval(reader_name) except NameError: reader_name = str(reader_name) # read the data reader = "mpop.satin." + reader_name try: reader_module = __import__(reader, globals(), locals(), ['concatenate']) except ImportError: LOG.exception("ImportError when loading plugin for format " + str(reader_name)) raise ImportError("No " + reader + " reader found.") scene = reader_module.concatenate(self.granules, channels) except NoSectionError: #concatenate loaded granules. scenes = [ GenericFactory.create_scene(granule.satname, granule.number, granule.instrument_name, granule.time_slot, None, None, granule.variant) for granule in self.granules ] for granule in scenes: granule.load(channels) scene = mpop.scene.assemble_segments(scenes) return scene
def do_processing(msg): from xml_read import read_product_file pl, mda = read_product_file('/local_disk/usr/src/mpop-smhi.old/etc/noaa15_products.xml') prods = pl["euron1"] info = msg.data from mpop.satellites import GenericFactory from datetime import datetime g = GenericFactory.create_scene(str(info["satellite"]), str(info["number"]), str(info["instrument"]), datetime.utcnow(), None) g.load(get_prerequisites(g.image, prods.keys())) for prod, files in prods.items(): img = getattr(g.image, prod)() print "saving", img, "to", files
def run_from_cmd(self, hook=None): """Batch run mpop. """ time_slots, mode, areas, composites = parse_options() tasklist = self.tasklist.shape(self.klass, mode, areas, composites) for time_slot in time_slots: self.data = GenericFactory.create_scene(self.satellite, self.number, self.instrument, time_slot, orbit=None, variant=self.variant) prerequisites = tasklist.get_prerequisites(self.klass) self.data.load(prerequisites) self.run_from_data(tasklist, hook)
def run_from_cmd(self, hook=None): """Batch run mpop. """ time_slots, mode, areas, composites = parse_options() tasklist = self.tasklist.shape(self.klass, mode, areas, composites) for time_slot in time_slots: self.data = GenericFactory.create_scene(self.satellite, self.number, self.instrument, time_slot, orbit=None, variant=self.variant) prerequisites = tasklist.get_prerequisites(self.klass) self.data.load(prerequisites) self.run_from_data(tasklist, hook)
def create_scene_from_mda(self, mda): """Read the metadata *mda* and return a corresponding MPOP scene. """ time_slot = (mda.get('start_time') or mda.get('nominal_time') or mda.get('end_time')) # orbit is not given for GEO satellites, use None if 'orbit_number' not in mda: mda['orbit_number'] = None platform = mda["platform_name"] LOGGER.info("platform %s time %s", str(platform), str(time_slot)) if isinstance(mda['sensor'], (list, tuple, set)): sensor = mda['sensor'][0] else: sensor = mda['sensor'] # Create satellite scene global_data = GF.create_scene(satname=str(platform), satnumber='', instrument=str(sensor), time_slot=time_slot, orbit=mda['orbit_number'], variant=mda.get('variant', '')) LOGGER.debug("Creating scene for satellite %s and time %s", str(platform), str(time_slot)) if mda['orbit_number'] is not None or mda.get('orbit_type') == "polar": global_data.overpass = Pass(platform, mda['start_time'], mda['end_time'], instrument=sensor) # Update missing information to global_data.info{} # TODO: this should be fixed in mpop. global_data.info.update(mda) global_data.info['time'] = time_slot return global_data
def do_processing(msg): from xml_read import read_product_file pl, mda = read_product_file( '/local_disk/usr/src/mpop-smhi.old/etc/noaa15_products.xml') prods = pl["euron1"] info = msg.data from mpop.satellites import GenericFactory from datetime import datetime g = GenericFactory.create_scene(str(info["satellite"]), str(info["number"]), str(info["instrument"]), datetime.utcnow(), None) g.load(get_prerequisites(g.image, prods.keys())) for prod, files in prods.items(): img = getattr(g.image, prod)() print "saving", img, "to", files
def create_scene_from_mda(self, mda): """Read the metadata *mda* and return a corresponding MPOP scene. """ time_slot = (mda.get('start_time') or mda.get('nominal_time') or mda.get('end_time')) # orbit is not given for GEO satellites, use None if 'orbit_number' not in mda: mda['orbit_number'] = None platform = mda["platform_name"] LOGGER.info("platform %s time %s", str(platform), str(time_slot)) if isinstance(mda['sensor'], (list, tuple, set)): sensor = mda['sensor'][0] else: sensor = mda['sensor'] # Create satellite scene global_data = GF.create_scene(satname=str(platform), satnumber='', instrument=str(sensor), time_slot=time_slot, orbit=mda['orbit_number'], variant=mda.get('variant', '')) LOGGER.debug("Creating scene for satellite %s and time %s", str(platform), str(time_slot)) if mda['orbit_number'] is not None or mda.get('orbit_type') == "polar": global_data.overpass = Pass(platform, mda['start_time'], mda['end_time'], instrument=sensor) # Update missing information to global_data.info{} # TODO: this should be fixed in mpop. global_data.info.update(mda) global_data.info['time'] = time_slot return global_data
def create_scene_from_mda(self, mda, instruments): """Read the metadata *mda* and return a corresponding MPOP scene. """ time_slot = (mda.get('start_time') or mda.get('nominal_time') or mda.get('end_time')) # orbit is not given for GEO satellites, use None if 'orbit_number' not in mda: mda['orbit_number'] = None platform = mda["platform_name"] self.logger.debug("platform %s time %s", str(platform), str(time_slot)) if isinstance(mda['sensor'], (list, tuple, set)): sensor = mda['sensor'][0] else: sensor = mda['sensor'] if sensor not in instruments: self.logger.debug("Unknown sensor, skipping data.") return None # Create satellite scene global_data = GF.create_scene(satname=str(platform), satnumber='', instrument=str(sensor), time_slot=time_slot, orbit=mda['orbit_number'], variant=mda.get('variant', '')) self.logger.info("Creating scene for satellite %s and time %s", str(platform), str(time_slot)) # Update missing information to global_data.info{} # TODO: this should be fixed in mpop. global_data.info.update(mda) global_data.info['time'] = time_slot return global_data
def _load02(filename): """Load data from a netcdf4 file, cf-satellite v0.2 (2012-02-03). """ rootgrp = Dataset(filename, 'r') # processed variables processed = set() # Currently MPOP does not like unicode (so much). satellite_name, satellite_number = [str(i) for i in rootgrp.platform.rsplit("-", 1)] time_slot = rootgrp.variables["time"].getValue()[0] time_slot = num2date(time_slot, TIME_UNITS) processed |= set(["time"]) try: service = str(rootgrp.service) except AttributeError: service = "" instrument_name = str(rootgrp.instrument) try: orbit = str(rootgrp.orbit) except AttributeError: orbit = None try: scene = GenericFactory.create_scene(satellite_name, satellite_number, instrument_name, time_slot, orbit, None, service) except NoSectionError: scene = VisirCompositer(time_slot=time_slot) scene.satname = satellite_name scene.number = satellite_number scene.service = service dim_chart = {} for var_name, var in rootgrp.variables.items(): varname = None try: varname = var.standard_name except AttributeError: try: varname = var.long_name except AttributeError: pass if varname in ["band_data", "Band data"]: LOG.debug("Found some data: " + var_name) dims = var.dimensions for dim in dims: dim_chart[dim] = var_name for cnt, dim in enumerate(dims): if dim.startswith("band"): break data = var data.set_auto_maskandscale(False) area = None try: area_var_name = getattr(var,"grid_mapping") area_var = rootgrp.variables[area_var_name] proj4_dict = {} for attr, projattr in MAPPING_ATTRIBUTES.items(): try: the_attr = getattr(area_var, attr) if projattr == "proj": proj4_dict[projattr] = PROJNAME[the_attr] elif(isinstance(projattr, (list, tuple))): try: for i, subattr in enumerate(the_attr): proj4_dict[projattr[i]] = subattr except TypeError: proj4_dict[projattr[0]] = the_attr else: proj4_dict[projattr] = the_attr except AttributeError: pass y_name, x_name = dims[:cnt] + dims[cnt + 1:] x__ = rootgrp.variables[x_name][:] y__ = rootgrp.variables[y_name][:] if proj4_dict["proj"] == "geos": x__ *= proj4_dict["h"] y__ *= proj4_dict["h"] x_pixel_size = abs((np.diff(x__)).mean()) y_pixel_size = abs((np.diff(y__)).mean()) llx = x__[0] - x_pixel_size / 2.0 lly = y__[-1] - y_pixel_size / 2.0 urx = x__[-1] + x_pixel_size / 2.0 ury = y__[0] + y_pixel_size / 2.0 area_extent = (llx, lly, urx, ury) try: # create the pyresample areadef from pyresample.geometry import AreaDefinition area = AreaDefinition("myareaid", "myareaname", "myprojid", proj4_dict, len(x__), len(y__), area_extent) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") processed |= set([area_var_name, x_name, y_name]) LOG.debug("Grid mapping found and used.") except AttributeError: LOG.debug("No grid mapping found.") try: area_var = getattr(var,"coordinates") coordinates_vars = area_var.split(" ") lons = None lats = None for coord_var_name in coordinates_vars: coord_var = rootgrp.variables[coord_var_name] units = getattr(coord_var, "units") if(coord_var_name.lower().startswith("lon") or units.lower().endswith("east") or units.lower().endswith("west")): lons = coord_var[:] elif(coord_var_name.lower().startswith("lat") or units.lower().endswith("north") or units.lower().endswith("south")): lats = coord_var[:] if lons.any() and lats.any(): try: from pyresample.geometry import SwathDefinition area = SwathDefinition(lons=lons, lats=lats) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") processed |= set(coordinates_vars) LOG.debug("Lon/lat found and used.") except AttributeError: LOG.debug("No lon/lat found.") names = rootgrp.variables[dim][:] scales = data.scale_factor offsets = data.add_offset if len(names) == 1: scales = np.array([scales]) offsets = np.array([offsets]) LOG.info("Scales and offsets: %s %s %s" % (str(names), str(scales), str(offsets))) for nbr, name in enumerate(names): name = str(name) try: if cnt == 0: chn_data = data[nbr, :, :].squeeze() if cnt == 1: chn_data = data[:, nbr, :].squeeze() if cnt == 2: chn_data = data[:, :, nbr].squeeze() scene[name] = (np.ma.masked_equal(chn_data, data._FillValue) * scales[nbr] + offsets[nbr]) scene[name].info["units"] = var.units except KeyError: from mpop.channel import Channel scene.channels.append(Channel(name)) if area is not None: scene[name].area = area processed |= set([var_name, dim]) non_processed = set(rootgrp.variables.keys()) - processed for var_name in non_processed: var = rootgrp.variables[var_name] if not (hasattr(var, "standard_name") or hasattr(var, "long_name")): LOG.info("Delayed processing of " + var_name) continue dims = var.dimensions if len(dims) != 1: LOG.info("Don't know what to do with " + var_name) continue dim = dims[0] if var.standard_name == "radiation_wavelength": names = rootgrp.variables[dim][:] for nbr, name in enumerate(names): name = str(name) scene[name].wavelength_range[1] = var[nbr] try: bnds = rootgrp.variables[var.bounds][:] for nbr, name in enumerate(names): name = str(name) scene[name].wavelength_range[0] = bnds[nbr, 0] scene[name].wavelength_range[2] = bnds[nbr, 1] processed |= set([var.bounds]) except AttributeError: pass processed |= set([var_name]) non_processed = set(rootgrp.variables.keys()) - processed if len(non_processed) > 0: LOG.warning("Remaining non-processed variables: " + str(non_processed)) return scene
def assemble_segments(segments): """Assemble the scene objects listed in *segment_list* and returns the resulting scene object. """ from mpop.satellites import GenericFactory channels = set([]) for seg in segments: channels |= set([chn.name for chn in seg.loaded_channels()]) seg = segments[0] new_scene = GenericFactory.create_scene(seg.satname, seg.number, seg.instrument_name, seg.time_slot, seg.orbit, variant=seg.variant) swath_definitions = {} for chn in channels: new_scene[chn] = np.ma.concatenate([seg[chn].data for seg in segments if seg[chn].is_loaded()]) try: area_names = tuple([seg[chn].area.area_id for seg in segments if seg[chn].is_loaded()]) if area_names not in swath_definitions: lons = np.ma.concatenate([seg[chn].area.lons[:] for seg in segments if seg[chn].is_loaded()]) lats = np.ma.concatenate([seg[chn].area.lats[:] for seg in segments if seg[chn].is_loaded()]) new_scene[chn].area = SwathDefinition(lons=lons, lats=lats) area_name = "+".join(area_names) new_scene[chn].area.area_id = area_name new_scene[chn].area_id = area_name swath_definitions[area_names] = new_scene[chn].area else: new_scene[chn].area = swath_definitions[area_names] new_scene[chn].area_id = new_scene[chn].area.area_id except AttributeError: pass try: lons = np.ma.concatenate([seg.area.lons[:] for seg in segments]) lats = np.ma.concatenate([seg.area.lats[:] for seg in segments]) new_scene.area = SwathDefinition(lons=lons, lats=lats) for chn in channels: if any([seg[chn].area for seg in segments]): try: lon_arrays = [] lat_arrays = [] for seg in segments: if seg[chn].area is not None: lon_arrays.append(seg[chn].area.lons[:]) lat_arrays.append(seg[chn].area.lats[:]) else: lon_arrays.append(seg.area.lons[:]) lat_arrays.append(seg.area.lats[:]) lons = np.ma.concatenate(lon_arrays) lats = np.ma.concatenate(lat_arrays) new_scene[chn].area = SwathDefinition(lons=lons, lats=lats) except AttributeError: pass except AttributeError: pass return new_scene
def load_from_nc4(filename): """Load data from a netcdf4 file. """ rootgrp = Dataset(filename, 'r') time_slot = rootgrp.variables["time"].getValue()[0] time_slot = num2date(time_slot, TIME_UNITS) if not isinstance(rootgrp.satellite_number, str): satellite_number = "%02d" % rootgrp.satellite_number else: satellite_number = str(rootgrp.satellite_number) service = str(rootgrp.service) satellite_name = str(rootgrp.satellite_name) instrument_name = str(rootgrp.instrument_name) try: orbit = str(rootgrp.orbit) except AttributeError: orbit = None try: scene = GenericFactory.create_scene(satellite_name, satellite_number, instrument_name, time_slot, orbit, None, service) except NoSectionError: scene = VisirCompositer(time_slot=time_slot) scene.satname = satellite_name scene.number = satellite_number scene.service = service for var_name, var in rootgrp.variables.items(): area = None if var_name.startswith("band_data"): resolution = var.resolution str_res = str(int(resolution)) + "m" names = rootgrp.variables["bandname"+str_res][:] data = var[:, :, :].astype(var.dtype) data = np.ma.masked_outside(data, var.valid_range[0], var.valid_range[1]) try: area_var = getattr(var,"grid_mapping") area_var = rootgrp.variables[area_var] proj4_dict = {} for attr, projattr in MAPPING_ATTRIBUTES.items(): try: the_attr = getattr(area_var, attr) if projattr == "proj": proj4_dict[projattr] = PROJNAME[the_attr] elif(isinstance(projattr, (list, tuple))): try: for i, subattr in enumerate(the_attr): proj4_dict[projattr[i]] = subattr except TypeError: proj4_dict[projattr[0]] = the_attr else: proj4_dict[projattr] = the_attr except AttributeError: pass x__ = rootgrp.variables["x"+str_res][:] y__ = rootgrp.variables["y"+str_res][:] x_pixel_size = abs((x__[1] - x__[0])) y_pixel_size = abs((y__[1] - y__[0])) llx = x__[0] - x_pixel_size / 2.0 lly = y__[-1] - y_pixel_size / 2.0 urx = x__[-1] + x_pixel_size / 2.0 ury = y__[0] + y_pixel_size / 2.0 area_extent = (llx, lly, urx, ury) try: # create the pyresample areadef from pyresample.geometry import AreaDefinition area = AreaDefinition("myareaid", "myareaname", "myprojid", proj4_dict, data.shape[1], data.shape[0], area_extent) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") except AttributeError: LOG.debug("No grid mapping found.") try: area_var = getattr(var,"coordinates") coordinates_vars = area_var.split(" ") lons = None lats = None for coord_var_name in coordinates_vars: coord_var = rootgrp.variables[coord_var_name] units = getattr(coord_var, "units") if(coord_var_name.lower().startswith("lon") or units.lower().endswith("east") or units.lower().endswith("west")): lons = coord_var[:] elif(coord_var_name.lower().startswith("lat") or units.lower().endswith("north") or units.lower().endswith("south")): lats = coord_var[:] if lons and lats: try: from pyresample.geometry import SwathDefinition area = SwathDefinition(lons=lons, lats=lats) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") except AttributeError: LOG.debug("No lon/lat found.") for i, name in enumerate(names): try: scene[name] = (data[:, :, i] * rootgrp.variables["scale"+str_res][i] + rootgrp.variables["offset"+str_res][i]) #FIXME complete this #scene[name].info except KeyError: # build the channel on the fly from mpop.channel import Channel wv_var = rootgrp.variables["nominal_wavelength"+str_res] wb_var = rootgrp.variables[getattr(wv_var, "bounds")] minmax = wb_var[i] scene.channels.append(Channel(name, resolution, (minmax[0], wv_var[i][0], minmax[1]))) scene[name] = (data[:, :, i] * rootgrp.variables["scale"+str_res][i] + rootgrp.variables["offset"+str_res][i]) if area is not None: scene[name].area = area area = None for attr in rootgrp.ncattrs(): scene.info[attr] = getattr(rootgrp, attr) scene.add_to_history("Loaded from netcdf4/cf by mpop") return scene
def assemble_segments(segments): """Assemble the scene objects listed in *segment_list* and returns the resulting scene object. """ from mpop.satellites import GenericFactory channels = set([]) for seg in segments: channels |= set([chn.name for chn in seg.loaded_channels()]) seg = segments[0] new_scene = GenericFactory.create_scene(seg.satname, seg.number, seg.instrument_name, seg.time_slot, seg.orbit, variant=seg.variant) swath_definitions = {} for chn in channels: new_scene[chn] = np.ma.concatenate( [seg[chn].data for seg in segments if seg[chn].is_loaded()]) try: area_names = tuple([ seg[chn].area.area_id for seg in segments if seg[chn].is_loaded() ]) if area_names not in swath_definitions: lons = np.ma.concatenate([ seg[chn].area.lons[:] for seg in segments if seg[chn].is_loaded() ]) lats = np.ma.concatenate([ seg[chn].area.lats[:] for seg in segments if seg[chn].is_loaded() ]) new_scene[chn].area = SwathDefinition(lons=lons, lats=lats) area_name = "+".join(area_names) new_scene[chn].area.area_id = area_name new_scene[chn].area_id = area_name swath_definitions[area_names] = new_scene[chn].area else: new_scene[chn].area = swath_definitions[area_names] new_scene[chn].area_id = new_scene[chn].area.area_id except AttributeError: pass try: lons = np.ma.concatenate([seg.area.lons[:] for seg in segments]) lats = np.ma.concatenate([seg.area.lats[:] for seg in segments]) new_scene.area = SwathDefinition(lons=lons, lats=lats) for chn in channels: if any([seg[chn].area for seg in segments]): try: lon_arrays = [] lat_arrays = [] for seg in segments: if seg[chn].area is not None: lon_arrays.append(seg[chn].area.lons[:]) lat_arrays.append(seg[chn].area.lats[:]) else: lon_arrays.append(seg.area.lons[:]) lat_arrays.append(seg.area.lats[:]) lons = np.ma.concatenate(lon_arrays) lats = np.ma.concatenate(lat_arrays) new_scene[chn].area = SwathDefinition(lons=lons, lats=lats) except AttributeError: pass except AttributeError: pass return new_scene
def load_from_nc4(filename): """Load data from a netcdf4 file, cf-satellite v0.1 """ rootgrp = Dataset(filename, 'r') try: rootgrp.satellite_number warnings.warn("You are loading old style netcdf files...", DeprecationWarning) except AttributeError: return _load02(filename) if not isinstance(rootgrp.satellite_number, str): satellite_number = "%02d" % rootgrp.satellite_number else: satellite_number = str(rootgrp.satellite_number) time_slot = rootgrp.variables["time"].getValue()[0] time_slot = num2date(time_slot, TIME_UNITS) service = str(rootgrp.service) satellite_name = str(rootgrp.satellite_name) instrument_name = str(rootgrp.instrument_name) try: orbit = str(rootgrp.orbit) except AttributeError: orbit = None try: scene = GenericFactory.create_scene(satellite_name, satellite_number, instrument_name, time_slot, orbit, None, service) except NoSectionError: scene = VisirCompositer(time_slot=time_slot) scene.satname = satellite_name scene.number = satellite_number scene.service = service for var_name, var in rootgrp.variables.items(): area = None if var_name.startswith("band_data"): resolution = var.resolution str_res = str(int(resolution)) + "m" names = rootgrp.variables["bandname" + str_res][:] data = var[:, :, :].astype(var.dtype) data = np.ma.masked_outside(data, var.valid_range[0], var.valid_range[1]) try: area_var = getattr(var, "grid_mapping") area_var = rootgrp.variables[area_var] proj4_dict = {} for attr, projattr in MAPPING_ATTRIBUTES.items(): try: the_attr = getattr(area_var, attr) if projattr == "proj": proj4_dict[projattr] = PROJNAME[the_attr] elif (isinstance(projattr, (list, tuple))): try: for i, subattr in enumerate(the_attr): proj4_dict[projattr[i]] = subattr except TypeError: proj4_dict[projattr[0]] = the_attr else: proj4_dict[projattr] = the_attr except AttributeError: pass x__ = rootgrp.variables["x" + str_res][:] y__ = rootgrp.variables["y" + str_res][:] x_pixel_size = abs((x__[1] - x__[0])) y_pixel_size = abs((y__[1] - y__[0])) llx = x__[0] - x_pixel_size / 2.0 lly = y__[-1] - y_pixel_size / 2.0 urx = x__[-1] + x_pixel_size / 2.0 ury = y__[0] + y_pixel_size / 2.0 area_extent = (llx, lly, urx, ury) try: # create the pyresample areadef from pyresample.geometry import AreaDefinition area = AreaDefinition("myareaid", "myareaname", "myprojid", proj4_dict, data.shape[1], data.shape[0], area_extent) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") except AttributeError: LOG.debug("No grid mapping found.") try: area_var = getattr(var, "coordinates") coordinates_vars = area_var.split(" ") lons = None lats = None for coord_var_name in coordinates_vars: coord_var = rootgrp.variables[coord_var_name] units = getattr(coord_var, "units") if (coord_var_name.lower().startswith("lon") or units.lower().endswith("east") or units.lower().endswith("west")): lons = coord_var[:] elif (coord_var_name.lower().startswith("lat") or units.lower().endswith("north") or units.lower().endswith("south")): lats = coord_var[:] if lons and lats: try: from pyresample.geometry import SwathDefinition area = SwathDefinition(lons=lons, lats=lats) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") except AttributeError: LOG.debug("No lon/lat found.") for i, name in enumerate(names): name = str(name) if var.dimensions[0].startswith("band"): chn_data = data[i, :, :] elif var.dimensions[1].startswith("band"): chn_data = data[:, i, :] elif var.dimensions[2].startswith("band"): chn_data = data[:, :, i] else: raise ValueError("Invalid dimension names for band data") try: scene[name] = ( chn_data * rootgrp.variables["scale" + str_res][i] + rootgrp.variables["offset" + str_res][i]) #FIXME complete this #scene[name].info except KeyError: # build the channel on the fly from mpop.channel import Channel wv_var = rootgrp.variables["nominal_wavelength" + str_res] wb_var = rootgrp.variables[getattr(wv_var, "bounds")] minmax = wb_var[i] scene.channels.append( Channel(name, resolution, (minmax[0], wv_var[i][0], minmax[1]))) scene[name] = ( chn_data * rootgrp.variables["scale" + str_res][i] + rootgrp.variables["offset" + str_res][i]) if area is not None: scene[name].area = area area = None for attr in rootgrp.ncattrs(): scene.info[attr] = getattr(rootgrp, attr) scene.add_to_history("Loaded from netcdf4/cf by mpop") return scene
import datetime as dt from mpop.satellites import GenericFactory as GF tslot = dt.datetime(2015, 7, 7, 16) glbl = GF.create_scene("Meteosat-9", "", "seviri", tslot, None) glbl.load([0.6, 0.8, 10.8], area_def_names=[ "euro4", ]) #lcl_bil = glbl.project('euro4', nprocs=1, mode="bilinear", radius=50e3, precompute=True) lcl_bil = glbl.project('euro4', nprocs=1, mode="ewa", precompute=True) img = lcl_bil.image.overview() img.show()
def run_single(self): '''Run image production without threading. ''' # TODO: Get relevant preprocessing function for this # production chain type: single/multi, or # swath, geo, granule, global_polar, global_geo, global_mixed # That is, gatherer for the multi-image/multi-granule types # preproc_func = getattr(preprocessing, self.production_type) while True: # wait for new messages msg = self.listener_parent_conn.recv() print msg # shutdown trollduction if msg.subject == '/StopTrollduction': self.cleanup() break #self.shutdown() # update trollduction config elif msg.subject == '/NewTrollductionConfig': self.update_td_config(msg.data) # update product lists elif msg.subject == '/NewProductConfig': self.udpate_product_config(msg.data) # process new file elif '/NewFileArrived' in msg.subject: self.time_slot = dt.datetime(msg.data['year'], msg.data['month'], msg.data['day'], msg.data['hour'], msg.data['minute']) # orbit is empty string for meteosat, change it to None if msg.data['orbit'] == '': msg.data['orbit'] = None t1a = time.time() self.global_data = GF.create_scene(satname=str(msg.data['satellite']), satnumber=str(msg.data['satnumber']), instrument=str(msg.data['instrument']), time_slot=self.time_slot, orbit=str(msg.data['orbit'])) # Find maximum extent that is needed for all the # products to be made. self.get_maximum_extent() # Make images for each area for area_name in self.area_def_names: t1b = time.time() # Check which channels are needed. Unload # unnecessary channels and load those that are not # already available. self.load_unload_channels(self.product_list[area_name]) # TODO: or something # reproject to local domain self.local_data = self.global_data.project(area_name, mode='nearest') print "Data reprojected for area:", area_name # Draw requested images for this area. self.draw_images(area_name) print "Single area time elapsed time:", time.time()-t1b, 's' self.local_data = None self.global_data = None # self.loaded_channels = [] print "Full time elapsed time:", time.time()-t1a, 's' else: # Unhandled message types end up here # No need to log these? pass
def _load02(filename): """Load data from a netcdf4 file, cf-satellite v0.2 (2012-02-03). """ rootgrp = Dataset(filename, 'r') # processed variables processed = set() # Currently MPOP does not like unicode (so much). satellite_name, satellite_number = [ str(i) for i in rootgrp.platform.rsplit("-", 1) ] time_slot = rootgrp.variables["time"].getValue()[0] time_slot = num2date(time_slot, TIME_UNITS) processed |= set(["time"]) try: service = str(rootgrp.service) except AttributeError: service = "" instrument_name = str(rootgrp.instrument) try: orbit = str(rootgrp.orbit) except AttributeError: orbit = None try: scene = GenericFactory.create_scene(satellite_name, satellite_number, instrument_name, time_slot, orbit, None, service) except NoSectionError: scene = VisirCompositer(time_slot=time_slot) scene.satname = satellite_name scene.number = satellite_number scene.service = service dim_chart = {} for var_name, var in rootgrp.variables.items(): varname = None try: varname = var.standard_name except AttributeError: try: varname = var.long_name except AttributeError: pass if varname in ["band_data", "Band data"]: LOG.debug("Found some data: " + var_name) dims = var.dimensions for dim in dims: dim_chart[dim] = var_name for cnt, dim in enumerate(dims): if dim.startswith("band"): break data = var data.set_auto_maskandscale(False) area = None try: area_var_name = getattr(var, "grid_mapping") area_var = rootgrp.variables[area_var_name] proj4_dict = {} for attr, projattr in MAPPING_ATTRIBUTES.items(): try: the_attr = getattr(area_var, attr) if projattr == "proj": proj4_dict[projattr] = PROJNAME[the_attr] elif (isinstance(projattr, (list, tuple))): try: for i, subattr in enumerate(the_attr): proj4_dict[projattr[i]] = subattr except TypeError: proj4_dict[projattr[0]] = the_attr else: proj4_dict[projattr] = the_attr except AttributeError: pass y_name, x_name = dims[:cnt] + dims[cnt + 1:] x__ = rootgrp.variables[x_name][:] y__ = rootgrp.variables[y_name][:] if proj4_dict["proj"] == "geos": x__ *= proj4_dict["h"] y__ *= proj4_dict["h"] x_pixel_size = abs((np.diff(x__)).mean()) y_pixel_size = abs((np.diff(y__)).mean()) llx = x__[0] - x_pixel_size / 2.0 lly = y__[-1] - y_pixel_size / 2.0 urx = x__[-1] + x_pixel_size / 2.0 ury = y__[0] + y_pixel_size / 2.0 area_extent = (llx, lly, urx, ury) try: # create the pyresample areadef from pyresample.geometry import AreaDefinition area = AreaDefinition("myareaid", "myareaname", "myprojid", proj4_dict, len(x__), len(y__), area_extent) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") processed |= set([area_var_name, x_name, y_name]) LOG.debug("Grid mapping found and used.") except AttributeError: LOG.debug("No grid mapping found.") try: area_var = getattr(var, "coordinates") coordinates_vars = area_var.split(" ") lons = None lats = None for coord_var_name in coordinates_vars: coord_var = rootgrp.variables[coord_var_name] units = getattr(coord_var, "units") if (coord_var_name.lower().startswith("lon") or units.lower().endswith("east") or units.lower().endswith("west")): lons = coord_var[:] elif (coord_var_name.lower().startswith("lat") or units.lower().endswith("north") or units.lower().endswith("south")): lats = coord_var[:] if lons.any() and lats.any(): try: from pyresample.geometry import SwathDefinition area = SwathDefinition(lons=lons, lats=lats) except ImportError: LOG.warning("Pyresample not found, " "cannot load area descrition") processed |= set(coordinates_vars) LOG.debug("Lon/lat found and used.") except AttributeError: LOG.debug("No lon/lat found.") names = rootgrp.variables[dim][:] scales = data.scale_factor offsets = data.add_offset if len(names) == 1: scales = np.array([scales]) offsets = np.array([offsets]) LOG.info("Scales and offsets: %s %s %s" % (str(names), str(scales), str(offsets))) for nbr, name in enumerate(names): name = str(name) try: if cnt == 0: chn_data = data[nbr, :, :].squeeze() if cnt == 1: chn_data = data[:, nbr, :].squeeze() if cnt == 2: chn_data = data[:, :, nbr].squeeze() scene[name] = ( np.ma.masked_equal(chn_data, data._FillValue) * scales[nbr] + offsets[nbr]) scene[name].info["units"] = var.units except KeyError: from mpop.channel import Channel scene.channels.append(Channel(name)) if area is not None: scene[name].area = area processed |= set([var_name, dim]) non_processed = set(rootgrp.variables.keys()) - processed for var_name in non_processed: var = rootgrp.variables[var_name] if not (hasattr(var, "standard_name") or hasattr(var, "long_name")): LOG.info("Delayed processing of " + var_name) continue dims = var.dimensions if len(dims) != 1: LOG.info("Don't know what to do with " + var_name) continue dim = dims[0] if var.standard_name == "radiation_wavelength": names = rootgrp.variables[dim][:] for nbr, name in enumerate(names): name = str(name) scene[name].wavelength_range[1] = var[nbr] try: bnds = rootgrp.variables[var.bounds][:] for nbr, name in enumerate(names): name = str(name) scene[name].wavelength_range[0] = bnds[nbr, 0] scene[name].wavelength_range[2] = bnds[nbr, 1] processed |= set([var.bounds]) except AttributeError: pass processed |= set([var_name]) non_processed = set(rootgrp.variables.keys()) - processed if len(non_processed) > 0: LOG.warning("Remaining non-processed variables: " + str(non_processed)) return scene
def run_single(self): '''Run image production without threading. ''' # TODO: Get relevant preprocessing function for this # production chain type: single/multi, or # swath, geo, granule, global_polar, global_geo, global_mixed # That is, gatherer for the multi-image/multi-granule types # preproc_func = getattr(preprocessing, self.production_type) while True: # wait for new messages msg = self.listener_parent_conn.recv() print msg # shutdown trollduction if msg.subject == '/StopTrollduction': self.cleanup() break #self.shutdown() # update trollduction config elif msg.subject == '/NewTrollductionConfig': self.update_td_config(msg.data) # update product lists elif msg.subject == '/NewProductConfig': self.udpate_product_config(msg.data) # process new file elif '/NewFileArrived' in msg.subject: self.time_slot = dt.datetime(msg.data['year'], msg.data['month'], msg.data['day'], msg.data['hour'], msg.data['minute']) # orbit is empty string for meteosat, change it to None if msg.data['orbit'] == '': msg.data['orbit'] = None t1a = time.time() self.global_data = GF.create_scene( satname=str(msg.data['satellite']), satnumber=str(msg.data['satnumber']), instrument=str(msg.data['instrument']), time_slot=self.time_slot, orbit=str(msg.data['orbit'])) # Find maximum extent that is needed for all the # products to be made. self.get_maximum_extent() # Make images for each area for area_name in self.area_def_names: t1b = time.time() # Check which channels are needed. Unload # unnecessary channels and load those that are not # already available. self.load_unload_channels(self.product_list[area_name]) # TODO: or something # reproject to local domain self.local_data = self.global_data.project(area_name, mode='nearest') print "Data reprojected for area:", area_name # Draw requested images for this area. self.draw_images(area_name) print "Single area time elapsed time:", time.time( ) - t1b, 's' self.local_data = None self.global_data = None # self.loaded_channels = [] print "Full time elapsed time:", time.time() - t1a, 's' else: # Unhandled message types end up here # No need to log these? pass