def load(scene, *args, **kwargs):
"""Loads the *channels* into the satellite *scene*.
A possible *calibrate* keyword argument is passed to the AAPP reader
Should be 0 for off, 1 for default, and 2 for radiances only.
However, as the AAPP-lvl1b file contains radiances this reader cannot
return counts, so calibrate=0 is not allowed/supported. The radiance to
counts conversion is not possible.
"""
del args
calibrate = kwargs.get("calibrate", True)
if calibrate == 0:
raise ValueError('calibrate=0 is not supported! ' +
'This reader cannot return counts')
filename = (kwargs.get("filename", None) or
get_filename(scene, "level2"))
LOG.debug("Using file " + filename)
reader = EpsAvhrrL1bReader(filename)
for chname, arr in reader.get_channels(scene.channels_to_load,
calibrate).items():
scene[chname] = arr
scene.orbit = str(int(reader["ORBIT_START"]))
lons, lats = reader.get_full_lonlats()
try:
scene.area = geometry.SwathDefinition(lons, lats)
except NameError:
scene.lons, scene.lats = lons, lats
def read(self, filename, **kwargs):
"""Read the data"""
from pyhdf.SD import SD
import datetime
#print "*** >>> Read the hdf-eos file!"
root = SD(filename)
# Get all the Attributes:
# Common Attributes, Data Time,
# Data Structure and Scene Coordinates
for key in root.attributes().keys():
self._eoshdf_info[key] = root.attributes()[key]
# Start Time - datetime object
starttime = datetime.datetime.strptime(self._eoshdf_info['Start Time'][0:13],
"%Y%j%H%M%S")
msec = float(self._eoshdf_info['Start Time'][13:16])/1000.
self.starttime = starttime + datetime.timedelta(seconds=msec)
# End Time - datetime object
endtime = datetime.datetime.strptime(self._eoshdf_info['End Time'][0:13],
"%Y%j%H%M%S")
msec = float(self._eoshdf_info['End Time'][13:16])/1000.
self.endtime = endtime + datetime.timedelta(seconds=msec)
# What is the leading 'H' doing here?
sensor_name = self._eoshdf_info['Sensor Name'][1:-1].lower()
try:
self.satid = EOS_SATELLITE[sensor_name]
except KeyError:
LOG.error("Failed setting the satellite id - sat-name = ",
sensor_name)
self.orbit = self._eoshdf_info['Orbit Number']
self.shape = (self._eoshdf_info['Number of Scan Control Points'],
self._eoshdf_info['Number of Pixel Control Points'])
#try:
if 1:
value = root.select(self.name)
attr = value.attributes()
data = value.get()
self.attr = attr
band = data
if self.name in FLAGS_QUALITY:
self.data = band
else:
nodata = attr['bad_value_scaled']
self.data = (np.ma.masked_equal(band, nodata) *
attr['slope'] + attr['intercept'])
value.endaccess()
#except:
# pass
root.end()
self.filled= True
def load_avhrr(satscene, options):
"""Read avhrr data from file and load it into *satscene*.
"""
if "filename" not in options:
raise IOError("No filename given, cannot load.")
values = {"INSTRUMENT": satscene.instrument_name[:4].upper(),
"FNAME": satscene.satname[0].upper() + satscene.number
}
filename = os.path.join(
options["dir"],
(satscene.time_slot.strftime(options["filename"])%values))
LOG.debug("Looking for file %s"%satscene.time_slot.strftime(filename))
file_list = glob.glob(satscene.time_slot.strftime(filename))
if len(file_list) > 1:
raise IOError("More than one l1b file matching!")
elif len(file_list) == 0:
raise IOError("No l1b file matching!")
try:
fdes = open(file_list[0])
channels, lats, lons, g3a, g3b, orbit = read(fdes)
finally:
fdes.close()
channels = np.ma.masked_invalid(channels)
satscene["1"] = channels[0, :, :]
satscene["2"] = channels[1, :, :]
satscene["4"] = channels[4, :, :]
satscene["5"] = channels[5, :, :]
if g3a:
satscene["3A"] = channels[2, :, :]
if g3b:
satscene["3B"] = channels[3, :, :]
print "Inside eps_avhrr.load_avhrr: orbit = ", orbit
#satscene.orbit = str(int(orbit) + 1)
satscene.orbit = str(int(orbit))
try:
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lons, lats=lats)
except ImportError:
satscene.area = None
satscene.lat = lats
satscene.lon = lons
def project(self, coverage):
"""Remaps the Modis EOS-HDF level2 ocean products to cartographic
map-projection on a user defined area.
"""
LOG.info("Projecting product %s..."%(self.name))
#print("Inside project...")
retv = ModisEosHdfLevel2(self.name)
retv.data = coverage.project_array(self.data)
retv.area = coverage.out_area
retv.shape = retv.data.shape
retv.resolution = self.resolution
retv.info = self.info
retv.filled = True
valid_min = retv.data.min()
valid_max = retv.data.max()
retv.info['valid_range'] = np.array([valid_min, valid_max])
retv.info['var_data'] = retv.data
return retv
def load_generic(satscene, options, calibrate=True):
"""Read sar data from file and load it into *satscene*.
"""
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOG.debug("Channels to load from %s: %s"%(satscene.instrument_name,
satscene.channels_to_load))
# Compulsory global attribudes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "No institution."
satscene.add_to_history("SAR data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
for chn in satscene.channels_to_load:
try:
metadata, data = xsar.sat.load(satscene.fullname,
satscene.time_slot,
chn,
mask=True,
calibrate=calibrate)
except CalibrationError:
LOG.warning("Loading non calibrated data since calibration failed.")
metadata, data = xsar.sat.load(satscene.fullname,
satscene.time_slot,
chn,
mask=True,
calibrate=False)
except ReaderError:
# if channel can't be found, go on with next channel
continue
satscene[chn] = data
satscene[chn].info['units'] = metadata.calibration_unit
if is_pyresample_loaded:
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split()
proj_dict = {}
for param in proj_params:
key, val = [i.strip() for i in param.split("=")]
proj_dict[key] = val
# Build area_def on-the-fly
satscene[chn].area = geometry.AreaDefinition(
satscene.satname + satscene.instrument_name +
str(metadata.area_extent) +
str(data.shape),
"On-the-fly area",
proj_dict["proj"],
proj_dict,
data.shape[1],
data.shape[0],
metadata.area_extent)
else:
LOG.info("Could not build area, pyresample missing...")
def get_filename(satscene, level):
"""Get the filename.
"""
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
options = {}
for option, value in conf.items(satscene.instrument_name + "-" + level,
raw = True):
options[option] = value
values = {"INSTRUMENT": satscene.instrument_name[:4].upper(),
"FNAME": satscene.satname[0].upper() + satscene.number
}
filename = os.path.join(
options["dir"],
(satscene.time_slot.strftime(options["filename"])%values))
LOG.debug("Looking for file %s"%satscene.time_slot.strftime(filename))
file_list = glob.glob(satscene.time_slot.strftime(filename))
if len(file_list) > 1:
raise IOError("More than one l1b file matching!")
elif len(file_list) == 0:
raise IOError("No l1b file matching!")
return file_list[0]
def load(satscene, **kwargs):
"""Read data from file and load it into *satscene*. Load data into the
*channels*. *Channels* is a list or a tuple containing channels we will
load data into. If None, all channels are loaded.
"""
del kwargs
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
options = {}
for option, value in conf.items(satscene.instrument_name+"-level3",
raw = True):
options[option] = value
pathname = os.path.join(options["dir"], options['filename'])
filename = satscene.time_slot.strftime(pathname)
for prodname in GEO_PHYS_PRODUCTS + FLAGS_QUALITY:
if prodname in satscene.channels_to_load:
prod_chan = ModisEosHdfLevel2(prodname)
prod_chan.read(filename)
prod_chan.satid = satscene.satname.capitalize()
prod_chan.resolution = 1000.0
prod_chan.shape = prod_chan.data.shape
# All this for the netCDF writer:
prod_chan.info['var_name'] = prodname
prod_chan.info['var_data'] = prod_chan.data
resolution_str = str(int(prod_chan.resolution))+'m'
prod_chan.info['var_dim_names'] = ('y'+resolution_str,
'x'+resolution_str)
prod_chan.info['long_name'] = prod_chan.attr['long_name'][:-1]
try:
prod_chan.info['standard_name'] = prod_chan.attr['standard_name'][:-1]
except KeyError:
pass
valid_min = np.min(prod_chan.data)
valid_max = np.max(prod_chan.data)
prod_chan.info['valid_range'] = np.array([valid_min, valid_max])
prod_chan.info['resolution'] = prod_chan.resolution
if prodname == 'l2_flags':
# l2 flags definitions
for i in range(1, 33):
key = "f%02d_name"%i
prod_chan.info[key] = prod_chan.attr[key][:-1]
satscene.channels.append(prod_chan)
if prodname in CHANNELS:
satscene[prodname].info['units'] = '%'
else:
satscene[prodname].info['units'] = prod_chan.attr['units'][:-1]
LOG.info("Loading modis lvl2 product '%s' done"%prodname)
# Check if there are any bands to load:
channels_to_load = False
for bandname in CHANNELS:
if bandname in satscene.channels_to_load:
channels_to_load = True
break
if channels_to_load:
#print "FILE: ", filename
eoshdf = SD(filename)
# Get all the Attributes:
# Common Attributes, Data Time,
# Data Structure and Scene Coordinates
info = {}
for key in eoshdf.attributes().keys():
info[key] = eoshdf.attributes()[key]
dsets = eoshdf.datasets()
selected_dsets = []
for bandname in CHANNELS:
if (bandname in satscene.channels_to_load and
bandname in dsets):
value = eoshdf.select(bandname)
selected_dsets.append(value)
# Get only the selected datasets
attr = value.attributes()
band = value.get()
nodata = attr['bad_value_scaled']
mask = np.equal(band, nodata)
satscene[bandname] = (np.ma.masked_where(mask, band) *
attr['slope'] + attr['intercept'])
satscene[bandname].info['units'] = '%'
satscene[bandname].info['long_name'] = attr['long_name'][:-1]
for dset in selected_dsets:
dset.endaccess()
LOG.info("Loading modis lvl2 Remote Sensing Reflectances done")
eoshdf.end()
lat, lon = get_lat_lon(satscene, None)
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lon, lats=lat)
#print "Variant: ", satscene.variant
satscene.variant = 'regional' # Temporary fix!
LOG.info("Loading modis data done.")
def load_avhrr(satscene, options):
"""Read avhrr data from file and load it into *satscene*.
"""
if "filename" not in options:
raise IOError("No filename given, cannot load.")
chns = satscene.channels_to_load & set(["1", "2", "3A", "3B", "4", "5"])
if len(chns) == 0:
return
values = {"orbit": satscene.orbit,
"satname": satscene.satname,
"number": satscene.number,
"instrument": satscene.instrument_name,
"satellite": satscene.fullname
}
filename = os.path.join(satscene.time_slot.strftime(options["dir"])%values,
satscene.time_slot.strftime(options["filename"])
%values)
file_list = glob.glob(filename)
if len(file_list) > 1:
raise IOError("More than one l1b file matching!")
elif len(file_list) == 0:
raise IOError("No l1b file matching!: "+
filename)
filename = file_list[0]
LOG.debug("Loading from " + filename)
import avhrr # AHAMAP module
avh = avhrr.avhrr(filename)
avh.get_unprojected()
instrument_data = avh.build_raw()
available_channels = set([])
data_channels = {}
for chn in instrument_data.data:
channel_name = chn.info.info["channel_id"][3:].upper()
available_channels |= set([channel_name])
data_channels[channel_name] = chn.data
for chn in satscene.channels_to_load:
if chn in available_channels:
if chn in ["1", "2", "3A"]:
gain = instrument_data.info["vis_gain"]
intercept = instrument_data.info["vis_intercept"]
units = "%"
else:
gain = instrument_data.info["ir_gain"]
intercept = instrument_data.info["ir_intercept"]
units = "K"
chn_array = np.ma.array(data_channels[chn])
missing_data = instrument_data.info["missing_data"]
chn_array = np.ma.masked_inside(chn_array,
missing_data - EPSILON,
missing_data + EPSILON)
no_data = instrument_data.info["nodata"]
chn_array = np.ma.masked_inside(chn_array,
no_data - EPSILON,
no_data + EPSILON)
satscene[chn] = chn_array
satscene[chn].data = np.ma.masked_less(satscene[chn].data *
gain +
intercept,
0)
satscene[chn].info['units'] = units
else:
LOG.warning("Channel "+str(chn)+" not available, not loaded.")
# Compulsory global attribudes
satscene.info["title"] = (satscene.satname.capitalize() + satscene.number +
" satellite, " +
satscene.instrument_name.capitalize() +
" instrument.")
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
lons = instrument_data.londata / math.pi * 180
lats = instrument_data.latdata / math.pi * 180
try:
from pyresample import geometry
satscene.area = geometry.SwathDefinition(lons=lons, lats=lats)
except ImportError:
satscene.area = None
satscene.lat = lats
satscene.lon = lons
def get_channels(self, channels, calib_type):
"""Get calibrated channel data.
*calib_type* = 0: Counts
*calib_type* = 1: Reflectances and brightness temperatures
*calib_type* = 2: Radiances
"""
if calib_type == 0:
raise ValueError('calibrate=0 is not supported! ' +
'This reader cannot return counts')
elif calib_type != 1 and calib_type != 2:
raise ValueError('calibrate=' + str(calib_type) +
'is not supported!')
if ("3a" in channels or
"3A" in channels or
"3b" in channels or
"3B" in channels):
three_a = ((self["FRAME_INDICATOR"] & 2**16) == 2**16)
three_b = ((self["FRAME_INDICATOR"] & 2**16) == 0)
chans = {}
for chan in channels:
if chan not in ["1", "2", "3a", "3A", "3b", "3B", "4", "5"]:
LOG.info("Can't load channel in eps_l1b: " + str(chan))
if chan == "1":
if calib_type == 1:
chans[chan] = np.ma.array(
to_refl(self["SCENE_RADIANCES"][:, 0, :],
self["CH1_SOLAR_FILTERED_IRRADIANCE"]))
else:
chans[chan] = np.ma.array(
self["SCENE_RADIANCES"][:, 0, :])
if chan == "2":
if calib_type == 1:
chans[chan] = np.ma.array(
to_refl(self["SCENE_RADIANCES"][:, 1, :],
self["CH2_SOLAR_FILTERED_IRRADIANCE"]))
else:
chans[chan] = np.ma.array(
self["SCENE_RADIANCES"][:, 1, :])
if chan.lower() == "3a":
if calib_type == 1:
chans[chan] = np.ma.array(
to_refl(self["SCENE_RADIANCES"][:, 2, :],
self["CH2_SOLAR_FILTERED_IRRADIANCE"]))
else:
chans[chan] = np.ma.array(self["SCENE_RADIANCES"][:, 2, :])
chans[chan][three_b, :] = np.nan
chans[chan] = np.ma.masked_invalid(chans[chan])
if chan.lower() == "3b":
if calib_type == 1:
chans[chan] = np.ma.array(
to_bt(self["SCENE_RADIANCES"][:, 2, :],
self["CH3B_CENTRAL_WAVENUMBER"],
self["CH3B_CONSTANT1"],
self["CH3B_CONSTANT2_SLOPE"]))
else:
chans[chan] = self["SCENE_RADIANCES"][:, 2, :]
chans[chan][three_a, :] = np.nan
chans[chan] = np.ma.masked_invalid(chans[chan])
if chan == "4":
if calib_type == 1:
chans[chan] = np.ma.array(
to_bt(self["SCENE_RADIANCES"][:, 3, :],
self["CH4_CENTRAL_WAVENUMBER"],
self["CH4_CONSTANT1"],
self["CH4_CONSTANT2_SLOPE"]))
else:
chans[chan] = np.ma.array(
self["SCENE_RADIANCES"][:, 3, :])
if chan == "5":
if calib_type == 1:
chans[chan] = np.ma.array(
to_bt(self["SCENE_RADIANCES"][:, 4, :],
self["CH5_CENTRAL_WAVENUMBER"],
self["CH5_CONSTANT1"],
self["CH5_CONSTANT2_SLOPE"]))
else:
chans[chan] = np.ma.array(self["SCENE_RADIANCES"][:, 4, :])
return chans
def load_generic(satscene, options, calibrate=True, area_extent=None):
"""Read seviri data from file and load it into *satscene*.
"""
os.environ["PPP_CONFIG_DIR"] = CONFIG_PATH
LOG.debug("Channels to load from %s: %s" % (satscene.instrument_name, satscene.channels_to_load))
# Compulsory global attribudes
satscene.info["title"] = (
satscene.satname.capitalize()
+ satscene.number
+ " satellite, "
+ satscene.instrument_name.capitalize()
+ " instrument."
)
satscene.info["institution"] = "Original data disseminated by EumetCast."
satscene.add_to_history("HRIT/LRIT data read by mipp/mpop.")
satscene.info["references"] = "No reference."
satscene.info["comments"] = "No comment."
from_area = False
if area_extent is None and satscene.area is not None:
if not satscene.area_def:
satscene.area = get_area_def(satscene.area_id)
area_extent = satscene.area.area_extent
from_area = True
for chn in satscene.channels_to_load:
if from_area:
try:
metadata = xrit.sat.load(satscene.fullname, satscene.time_slot, chn, only_metadata=True)
if (
satscene.area_def.proj_dict["proj"] != "geos"
or float(satscene.area_def.proj_dict["lon_0"]) != metadata.sublon
):
raise ValueError(
"Slicing area must be in "
"geos projection, and lon_0 should match the"
" satellite's position."
)
except SatReaderError:
# if channel can't be found, go on with next channel
continue
try:
image = xrit.sat.load(satscene.fullname, satscene.time_slot, chn, mask=True, calibrate=calibrate)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except CalibrationError:
LOG.warning("Loading non calibrated data since calibration failed.")
image = xrit.sat.load(satscene.fullname, satscene.time_slot, chn, mask=True, calibrate=False)
if area_extent:
metadata, data = image(area_extent)
else:
metadata, data = image()
except SatReaderError:
# if channel can't be found, go on with next channel
continue
satscene[chn] = data
satscene[chn].info["units"] = metadata.calibration_unit
# Build an area on the fly from the mipp metadata
proj_params = getattr(metadata, "proj4_params").split(" ")
proj_dict = {}
for param in proj_params:
key, val = param.split("=")
proj_dict[key] = val
if is_pyresample_loaded:
# Build area_def on-the-fly
satscene[chn].area = geometry.AreaDefinition(
satscene.satname + satscene.instrument_name + str(metadata.area_extent) + str(data.shape),
"On-the-fly area",
proj_dict["proj"],
proj_dict,
data.shape[1],
data.shape[0],
metadata.area_extent,
)
else:
LOG.info("Could not build area, pyresample missing...")
import xrit.sat
from xrit import CalibrationError, SatReaderError
from mpop import CONFIG_PATH
from mpop.satin.logger import LOG
try:
# Work around for on demand import of pyresample. pyresample depends
# on scipy.spatial which memory leaks on multiple imports
is_pyresample_loaded = False
from pyresample import geometry
from mpop.projector import get_area_def
is_pyresample_loaded = True
except ImportError:
LOG.warning("pyresample missing. Can only work in satellite projection")
def load(satscene, calibrate=True, area_extent=None):
"""Read data from file and load it into *satscene*. The *calibrate*
argument is passed to mipp (should be 0 for off, 1 for default, and 2 for
radiances only).
"""
conf = ConfigParser.ConfigParser()
conf.read(os.path.join(CONFIG_PATH, satscene.fullname + ".cfg"))
options = {}
for option, value in conf.items(satscene.instrument_name + "-level2"):
options[option] = value
for section in conf.sections():
if (
#!/usr/bin/env python2
import os
import fnmatch
import datetime
import sys
import glob
from mpop.satellites import PolarFactory
from mpop.scene import assemble_segments
from mpop.satin.logger import LOG
LOG.setLevel("DEBUG")
LOG.warning("FOO")
#Looks at the filesize of the granule and rejects it if any
# of the files for the given bands are too small, or from 1958
# This should prevent geolocation errors when the
# granules are combined.
def isvalidgranule(granule,bands,path):
pattern = "SV%(band)s_%(satellite)s_d%Y%m%d_t%H%M???_e???????_b%(orbit)s_c*h5"
values = { "orbit": granule.orbit,
"satname": granule.satname,
"instrument": granule.instrument_name,
"satellite": granule.satname }
if(granule.time_slot.strftime("%Y") == "1958"):
print "Blast from the past"
return False
for band in bands:
values["band"] = band
filename = granule.time_slot.strftime( pattern ) % values
files = glob.glob(os.path.join(path,filename))
for file in files:
