def test_is_loaded(self):
"""Check load status of a channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name = "newchan")
self.assert_(not self.chan.is_loaded())
self.chan = Channel(name = "newchan", data = data)
self.assert_(self.chan.is_loaded())
def test_is_loaded(self):
"""Check load status of a channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan")
self.assert_(not self.chan.is_loaded())
self.chan = Channel(name="newchan", data=data)
self.assert_(self.chan.is_loaded())
def test_as_image(self):
"""Check the geo_image version of the channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
img = self.chan.as_image(False)
self.assert_(np.allclose(img.channels[0], data))
self.assertEqual(img.mode, "L")
img = self.chan.as_image(True)
self.assertEqual(img.channels[0].max(), 1)
self.assertEqual(img.channels[0].min(), 0)
def test_check_range(self):
"""Check the range of a channel.
"""
self.chan = Channel(name = "newchan")
self.assertRaises(ValueError, self.chan.check_range)
numb = np.random.uniform(10)
self.assertRaises(ValueError, self.chan.check_range, numb)
# ndarray
data = np.random.rand(3, 3)
self.chan = Channel(name = "newchan", data = data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
# masked array
mask = np.array(np.random.rand(3, 3) * 2, dtype = int)
mask[1, 1] = False
data = np.ma.array(data, mask = mask)
self.chan = Channel(name = "newchan", data = data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
data = np.ma.array(data, mask = True)
self.chan = Channel(name = "newchan", data = data)
self.assertEquals(0,
self.chan.check_range(min_range).count())
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
# Wrong type arguments
self.assertRaises(TypeError, self.chan.check_range, random_string(4))
self.assertRaises(TypeError,
self.chan.check_range,
[np.random.uniform()])
def __init__(self,
time_slot=None,
area_id=None,
area=None,
orbit=None,
satellite=(None, None, None),
instrument=None):
SatelliteScene.__init__(self, time_slot, area_id, area, orbit,
satellite)
try:
self.instrument_name = instrument or self.instrument_name
except AttributeError:
self.instrument_name = None
self.channels = []
try:
conf = OrderedConfigParser()
conf.read(os.path.join(CONFIG_PATH, self.fullname + ".cfg"))
for section in conf.sections():
if (not section[:-1].endswith("level")
and not section.endswith("granules")
and section.startswith(self.instrument_name)):
name = eval(conf.get(section, "name"))
try:
w_range = eval(conf.get(section, "frequency"))
except ConfigParser.NoOptionError:
w_range = (-np.inf, -np.inf, -np.inf)
try:
resolution = eval(conf.get(section, "resolution"))
except ConfigParser.NoOptionError:
resolution = 0
self.channels.append(
Channel(name=name,
wavelength_range=w_range,
resolution=resolution))
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
for name, w_range, resolution in self.channel_list:
self.channels.append(
Channel(name=name,
wavelength_range=w_range,
resolution=resolution))
self.channels_to_load = set([])
def __setitem__(self, key, data):
# Add a channel if it is not already in the scene. Works only if key is
# a string.
try:
if key not in self:
# if it's a blob with name and data, add it as is.
if hasattr(data, "name") and hasattr(data, "data"):
self.channels.append(data)
else:
kwargs = {"name": key}
for attr in ["wavelength_range", "resolution"]:
try:
kwargs[attr] = getattr(data, attr)
except (AttributeError, NameError):
pass
self.channels.append(Channel(**kwargs))
except AttributeError:
pass
# Add the data.
if isinstance(data, np.ma.core.MaskedArray):
self[key].data = data
else:
try:
self[key].data = data.data
except AttributeError:
self[key].data = data
def _dwd_get_day_night_alpha_channel(self,
sz_night_limit=None,
sz_day_limit=None):
"""Returns the alpha values depending on the sun zenith angles.
Lower angles result in lower alpha values
so this data has to be inverted for the day image.
"""
if sz_night_limit is None:
sz_night_limit = SUN_ZEN_NIGHT_LIMIT
if sz_day_limit is None:
sz_day_limit = SUN_ZEN_DAY_LIMIT
ch_name = "DAY_NIGHT_ALPHA_" + str(sz_day_limit) \
+ "_" + str(sz_night_limit)
try:
self.check_channels(ch_name)
if self[ch_name].data.shape != self.area.shape:
self._data_holder.channels.remove(self[ch_name])
raise Exception
except:
sun_zen_chn = self._dwd_get_sun_zenith_angles_channel()
data = sun_zen_chn.data
alpha = np.ma.zeros(data.shape, dtype=np.int)
y, x = np.where((data <= sz_night_limit) & (data >= sz_day_limit))
alpha[y, x] = (((data[y, x] - sz_day_limit) /
(sz_night_limit - sz_day_limit)) * (254 - 1) + 1)
alpha[np.where(data > sz_night_limit)] += 255
alpha_chn = Channel(name=ch_name, data=alpha)
self._data_holder.channels.append(alpha_chn)
return self[ch_name]
def _dwd_get_hrvc_channel(self):
"""Returns the combination of HRV and VIS008 channel data
if there are gaps in HRV data; otherwise HRV only.
"""
if np.ma.is_masked(self["HRV"].data):
try:
self.check_channels("HRVC")
if self["HRVC"].data.shape != self.area.shape:
self._data_holder.channels.remove(self["HRVC"])
raise Exception()
hrvc_chn = self["HRVC"]
except:
hrv_chn = self["HRV"]
hrvc_data = np.ma.where(hrv_chn.data.mask, self[0.85].data,
hrv_chn.data)
hrvc_chn = Channel(name="HRVC",
resolution=hrv_chn.resolution,
wavelength_range=hrv_chn.wavelength_range,
data=hrvc_data,
calibration_unit=hrv_chn.unit)
self._data_holder.channels.append(hrvc_chn)
else:
hrvc_chn = self["HRV"]
return hrvc_chn
def test_str(self):
"""String output for a channel.
"""
self.chan = Channel(name="newchan",
wavelength_range=(1., 2., 3.),
resolution=1000)
self.assertEqual(
str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, resolution 1000m,"
" not loaded'")
self.chan.data = np.random.rand(3, 3)
self.assertEqual(
str(self.chan), "'newchan: (1.000,2.000,3.000)μm, "
"shape (3, 3), "
"resolution 1000m'")
def test_as_image(self):
"""Check the geo_image version of the channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
img = self.chan.as_image(False)
self.assert_(np.allclose(img.channels[0], data))
self.assertEqual(img.mode, "L")
img = self.chan.as_image(True)
self.assertEqual(img.channels[0].max(), 1)
self.assertEqual(img.channels[0].min(), 0)
def test_check_range(self):
"""Check the range of a channel.
"""
self.chan = Channel(name="newchan")
self.assertRaises(ValueError, self.chan.check_range)
numb = np.random.uniform(10)
self.assertRaises(ValueError, self.chan.check_range, numb)
# ndarray
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
# masked array
mask = np.array(np.random.rand(3, 3) * 2, dtype=int)
mask[1, 1] = False
data = np.ma.array(data, mask=mask)
self.chan = Channel(name="newchan", data=data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
data = np.ma.array(data, mask=True)
self.chan = Channel(name="newchan", data=data)
self.assertEquals(0, self.chan.check_range(min_range).count())
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
# Wrong type arguments
self.assertRaises(TypeError, self.chan.check_range, random_string(4))
self.assertRaises(TypeError, self.chan.check_range,
[np.random.uniform()])
def test_cmp(self):
"""Comparison of channels.
"""
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "mychan")
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = "mychan"
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "newchan")
self.assert_(self.chan == self.chan2)
self.chan = Channel(wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name = "newchan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "_mychan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "_newchan")
self.chan2 = Channel(name = "mychan")
self.assert_(self.chan > self.chan2)
self.chan = Channel(name = random_string(4),
wavelength_range = (1., 2., 3.))
self.chan2 = Channel(name = random_string(4),
wavelength_range = (4., 5., 6.))
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "_" + random_string(4),
wavelength_range = (1., 2., 3.))
self.chan2 = Channel(name = random_string(4),
wavelength_range = (4., 5., 6.))
self.assert_(self.chan > self.chan2)
def _wait_for_view_zenith_ch_sublon(self):
# wait for the satellite zenith angle calculation worker_process
if self.processing is True:
self.sublon_processor.join()
self.view_zen_data_cache[self.area_def_name] = \
self.sublon_processor.get_result(self.area_def_name)
LOGGER.debug("finished view zenith angle data processing for " +
self.area_def_name)
# provide satellite zenith angle data as channel in local_data
vza_chn = Channel(name=self.area_def_name + "_VZA",
data=self.view_zen_data_cache.get(
self.area_def_name, None))
return vza_chn
def test_str(self):
"""String output for a channel.
"""
self.chan = Channel(name="newchan",
wavelength_range=(1., 2., 3.),
resolution=1000)
self.assertEqual(str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, resolution 1000m,"
" not loaded'")
self.chan.data = np.random.rand(3, 3)
self.assertEqual(str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, "
"shape (3, 3), "
"resolution 1000m'")
def _dwd_get_sun_zenith_angles_channel(self):
"""Returns the sun zenith angles for the area of interest as a channel.
"""
LOGGER.info('Retrieve sun zenith angles')
try:
self.check_channels("SUN_ZEN_CHN")
if self["SUN_ZEN_CHN"].data.shape != self.area.shape:
self._data_holder.channels.remove(self["SUN_ZEN_CHN"])
raise Exception()
except:
if self.area.lons is None or self.area.lats is None:
self.area.lons, self.area.lats = self.area.get_lonlats()
sun_zen_chn_data = np.zeros(shape=self.area.lons.shape)
q = 500
for start in xrange(0, sun_zen_chn_data.shape[1], q):
sun_zen_chn_data[:, start:start + q] = sza(
get_first(self.time_slot), self.area.lons[:, start:start + q],
self.area.lats[:, start:start + q])
sun_zen_chn = Channel(name="SUN_ZEN_CHN", data=sun_zen_chn_data)
self._data_holder.channels.append(sun_zen_chn)
return self["SUN_ZEN_CHN"]
def test_init(self):
"""Creation of a channel.
"""
self.assertRaises(ValueError, Channel)
# Name
self.chan = Channel(name="newchan")
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name=numb)
numb = np.random.uniform() * 100000
self.assertRaises(TypeError, Channel, name=numb)
# Resolution
numb = int(np.random.uniform(100000))
self.assertRaises(ValueError, Channel, resolution=numb)
numb = int(np.random.uniform(100000))
self.chan = Channel(name="newchan", resolution=numb)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, numb)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel, name="newchan", resolution="a")
# Wavelength
numbs = [
np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)
]
numbs.sort()
self.chan = Channel(wavelength_range=numbs)
self.assertEqual(self.chan.name, None)
self.assertEqual(self.chan.wavelength_range, numbs)
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel, wavelength_range=numbs[0:1])
numbs.sort(reverse=True)
self.assertRaises(ValueError, Channel, wavelength_range=numbs)
numbs = [
int(np.random.uniform(100)),
int(np.random.uniform(100)),
int(np.random.uniform(100))
]
numbs.sort()
self.assertRaises(TypeError, Channel, wavelength_range=numbs)
self.assertRaises(TypeError,
Channel,
wavelength_range=random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel, wavelength_range=numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, wavelength_range=numb)
# Data
data = np.random.rand(3, 3)
self.assertRaises(ValueError, Channel, data=data)
self.chan = Channel(name="newchan", data=data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
mask = np.array(np.random.rand(3, 3) * 2, dtype=int)
data = np.ma.array(data, mask=mask)
self.chan = Channel(name="newchan", data=data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
self.assertRaises(TypeError,
Channel,
name="newchan",
data=random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel, name="newchan", data=numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name="newchan", data=numb)
numbs = [
np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)
]
self.assertRaises(TypeError, Channel, name="newchan", data=numbs)
class TestChannel(unittest.TestCase):
"""Class for testing the Channel class.
"""
chan = None
chan2 = None
def test_init(self):
"""Creation of a channel.
"""
self.assertRaises(ValueError, Channel)
# Name
self.chan = Channel(name = "newchan")
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name = numb)
numb = np.random.uniform() * 100000
self.assertRaises(TypeError, Channel, name = numb)
# Resolution
numb = int(np.random.uniform(100000))
self.assertRaises(ValueError, Channel, resolution = numb)
numb = int(np.random.uniform(100000))
self.chan = Channel(name = "newchan", resolution = numb)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, numb)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel,
name = "newchan",
resolution = "a")
# Wavelength
numbs = [np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)]
numbs.sort()
self.chan = Channel(wavelength_range = numbs)
self.assertEqual(self.chan.name, None)
self.assertEqual(self.chan.wavelength_range, numbs)
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel,
wavelength_range = numbs[0:1])
numbs.sort(reverse = True)
self.assertRaises(ValueError, Channel,
wavelength_range = numbs)
numbs = [int(np.random.uniform(100)),
int(np.random.uniform(100)),
int(np.random.uniform(100))]
numbs.sort()
self.assertRaises(TypeError, Channel,
wavelength_range = numbs)
self.assertRaises(TypeError, Channel,
wavelength_range = random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel,
wavelength_range = numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel,
wavelength_range = numb)
# Data
data = np.random.rand(3, 3)
self.assertRaises(ValueError, Channel, data = data)
self.chan = Channel(name = "newchan", data = data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
mask = np.array(np.random.rand(3, 3) * 2, dtype = int)
data = np.ma.array(data, mask = mask)
self.chan = Channel(name = "newchan", data = data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numb)
numbs = [np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)]
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numbs)
def test_cmp(self):
"""Comparison of channels.
"""
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "mychan")
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = "mychan"
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "newchan")
self.assert_(self.chan == self.chan2)
self.chan = Channel(wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name = "newchan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "_mychan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "_newchan")
self.chan2 = Channel(name = "mychan")
self.assert_(self.chan > self.chan2)
self.chan = Channel(name = random_string(4),
wavelength_range = (1., 2., 3.))
self.chan2 = Channel(name = random_string(4),
wavelength_range = (4., 5., 6.))
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "_" + random_string(4),
wavelength_range = (1., 2., 3.))
self.chan2 = Channel(name = random_string(4),
wavelength_range = (4., 5., 6.))
self.assert_(self.chan > self.chan2)
def test_str(self):
"""String output for a channel.
"""
self.chan = Channel(name="newchan",
wavelength_range=(1., 2., 3.),
resolution=1000)
self.assertEqual(str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, resolution 1000m,"
" not loaded'")
self.chan.data = np.random.rand(3, 3)
self.assertEqual(str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, "
"shape (3, 3), "
"resolution 1000m'")
def test_is_loaded(self):
"""Check load status of a channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name = "newchan")
self.assert_(not self.chan.is_loaded())
self.chan = Channel(name = "newchan", data = data)
self.assert_(self.chan.is_loaded())
def test_as_image(self):
"""Check the geo_image version of the channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
img = self.chan.as_image(False)
self.assert_(np.allclose(img.channels[0], data))
self.assertEqual(img.mode, "L")
img = self.chan.as_image(True)
self.assertEqual(img.channels[0].max(), 1)
self.assertEqual(img.channels[0].min(), 0)
def test_check_range(self):
"""Check the range of a channel.
"""
self.chan = Channel(name = "newchan")
self.assertRaises(ValueError, self.chan.check_range)
numb = np.random.uniform(10)
self.assertRaises(ValueError, self.chan.check_range, numb)
# ndarray
data = np.random.rand(3, 3)
self.chan = Channel(name = "newchan", data = data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
# masked array
mask = np.array(np.random.rand(3, 3) * 2, dtype = int)
mask[1, 1] = False
data = np.ma.array(data, mask = mask)
self.chan = Channel(name = "newchan", data = data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
data = np.ma.array(data, mask = True)
self.chan = Channel(name = "newchan", data = data)
self.assertEquals(0,
self.chan.check_range(min_range).count())
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
# Wrong type arguments
self.assertRaises(TypeError, self.chan.check_range, random_string(4))
self.assertRaises(TypeError,
self.chan.check_range,
[np.random.uniform()])
def test_sunzen_corr(self):
'''Test Sun zenith angle correction.
'''
import datetime as dt
chan = Channel(name='test')
original_value = 10.
chan.data = original_value * np.ones((2, 11))
lats = np.zeros((2, 11)) # equator
lons = np.array([np.linspace(-90, 90, 11), np.linspace(-90, 90, 11)])
# Equinox, so the Sun is at the equator
time_slot = dt.datetime(2014, 3, 20, 16, 57)
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
# Test minimum after correction, accuracy of three decimals is enough
#self.assertTrue(np.abs(10.000 - np.min(new_ch.data)) < 10**-3)
self.assertAlmostEqual(10.000, np.min(new_ch.data), places=3)
# Test maximum after correction
self.assertAlmostEqual(57.588, np.max(new_ch.data), places=3)
# There should be ten values at zenith angle >= 80 deg, and
# these are all equal
self.assertTrue(np.where(new_ch.data == \
np.max(new_ch.data))[0].shape[0] == 10)
# All values should be larger than the starting values
self.assertTrue(np.all(new_ch.data > original_value))
# Channel name
self.assertEqual(new_ch.name, chan.name + '_SZC')
# Test channel name in the info dict
self.assertEqual(new_ch.name, chan.info['sun_zen_corrected'])
# Test with several locations and arbitrary data
chan = Channel(name='test2')
chan.data = np.array(
[[0., 67.31614275, 49.96271995, 99.41046645, 29.08660989],
[87.61007584, 79.6683524, 53.20397351, 29.88260374, 62.33623915],
[60.49283004, 54.04267222, 32.72365906, 91.44995651, 32.27232955],
[63.71580638, 69.57673795, 7.63064373, 32.15683105, 9.05786335],
[65.61434337, 33.2317155, 18.77672384, 30.13527574, 23.22572904]])
lons = np.array([[
116.28695847, 164.1125604, 40.77223701, -113.54699788, 133.15558442
], [-17.18990601, 75.17472034, 12.81618371, -40.75524952, 40.70898002],
[
42.74662341, 164.05671859, -166.58469404,
-58.16684483, -144.97963063
],
[
46.26303645, -167.48682034, 170.28131412,
-17.80502488, -63.9031154
],
[
-107.14829679, -147.66665952, -0.75970554,
77.701768, -130.48677807
]])
lats = np.array([
[-51.53681682, -83.21762788, 5.91008672, 22.51730385, 66.83356427],
[82.78543163, 23.1529456, -7.16337152, -68.23118425, 28.72194953],
[31.03440852, 70.55322517, -83.61780288, 29.88413938, 25.7214828],
[-19.02517922, -19.20958728, -14.7825735, 22.66967876, 67.6089238],
[45.12202477, 61.79674149, 58.71037615, -62.04350423, 13.06405864]
])
time_slot = dt.datetime(1998, 8, 1, 10, 0)
# These are the expected results
results = np.array([[
0., 387.65821593, 51.74080022, 572.48205988, 138.96586013
], [
227.24857818, 105.53045776, 62.24134162, 172.0870564, 64.12902666
], [
63.08646652, 311.21934562, 188.44804188, 526.63931022, 185.84893885
], [82.86856236, 400.6764648, 43.9431259, 46.58056343, 36.04457644],
[
377.85794388, 191.3738223, 27.55002934,
173.54213642, 133.75164285
]])
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
self.assertAlmostEqual(np.max(results - new_ch.data), 0.000, places=3)
def test_cmp(self):
"""Comparison of channels.
"""
self.chan = Channel(name="newchan")
self.chan2 = Channel(name="mychan")
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name="newchan")
self.chan2 = "mychan"
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name="newchan")
self.chan2 = Channel(name="newchan")
self.assert_(self.chan == self.chan2)
self.chan = Channel(wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name="newchan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name="newchan")
self.chan2 = Channel(name="_mychan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name="_newchan")
self.chan2 = Channel(name="mychan")
self.assert_(self.chan > self.chan2)
self.chan = Channel(name=random_string(4),
wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name=random_string(4),
wavelength_range=(4., 5., 6.))
self.assert_(self.chan < self.chan2)
self.chan = Channel(name="_" + random_string(4),
wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name=random_string(4),
wavelength_range=(4., 5., 6.))
self.assert_(self.chan > self.chan2)
def load_from_nc4(filename):
"""Load data from a netcdf4 file, cf-satellite v0.1
"""
rootgrp = Dataset(filename, 'r')
try:
rootgrp.satellite_number
warnings.warn("You are loading old style netcdf files...",
DeprecationWarning)
except AttributeError:
return _load02(filename)
if not isinstance(rootgrp.satellite_number, str):
satellite_number = "%02d" % rootgrp.satellite_number
else:
satellite_number = str(rootgrp.satellite_number)
time_slot = rootgrp.variables["time"].getValue()[0]
time_slot = num2date(time_slot, TIME_UNITS)
service = str(rootgrp.service)
satellite_name = str(rootgrp.satellite_name)
instrument_name = str(rootgrp.instrument_name)
try:
orbit = str(rootgrp.orbit)
except AttributeError:
orbit = None
try:
scene = GenericFactory.create_scene(satellite_name, satellite_number,
instrument_name, time_slot, orbit,
None, service)
except NoSectionError:
scene = VisirCompositer(time_slot=time_slot)
scene.satname = satellite_name
scene.number = satellite_number
scene.service = service
for var_name, var in rootgrp.variables.items():
area = None
if var_name.startswith("band_data"):
resolution = var.resolution
str_res = str(int(resolution)) + "m"
names = rootgrp.variables["bandname" + str_res][:]
data = var[:, :, :].astype(var.dtype)
data = np.ma.masked_outside(data, var.valid_range[0],
var.valid_range[1])
try:
area_var = getattr(var, "grid_mapping")
area_var = rootgrp.variables[area_var]
proj4_dict = {}
for attr, projattr in MAPPING_ATTRIBUTES.items():
try:
the_attr = getattr(area_var, attr)
if projattr == "proj":
proj4_dict[projattr] = PROJNAME[the_attr]
elif (isinstance(projattr, (list, tuple))):
try:
for i, subattr in enumerate(the_attr):
proj4_dict[projattr[i]] = subattr
except TypeError:
proj4_dict[projattr[0]] = the_attr
else:
proj4_dict[projattr] = the_attr
except AttributeError:
pass
x__ = rootgrp.variables["x" + str_res][:]
y__ = rootgrp.variables["y" + str_res][:]
x_pixel_size = abs((x__[1] - x__[0]))
y_pixel_size = abs((y__[1] - y__[0]))
llx = x__[0] - x_pixel_size / 2.0
lly = y__[-1] - y_pixel_size / 2.0
urx = x__[-1] + x_pixel_size / 2.0
ury = y__[0] + y_pixel_size / 2.0
area_extent = (llx, lly, urx, ury)
try:
# create the pyresample areadef
from pyresample.geometry import AreaDefinition
area = AreaDefinition("myareaid", "myareaname", "myprojid",
proj4_dict, data.shape[1],
data.shape[0], area_extent)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
except AttributeError:
LOG.debug("No grid mapping found.")
try:
area_var = getattr(var, "coordinates")
coordinates_vars = area_var.split(" ")
lons = None
lats = None
for coord_var_name in coordinates_vars:
coord_var = rootgrp.variables[coord_var_name]
units = getattr(coord_var, "units")
if (coord_var_name.lower().startswith("lon")
or units.lower().endswith("east")
or units.lower().endswith("west")):
lons = coord_var[:]
elif (coord_var_name.lower().startswith("lat")
or units.lower().endswith("north")
or units.lower().endswith("south")):
lats = coord_var[:]
if lons and lats:
try:
from pyresample.geometry import SwathDefinition
area = SwathDefinition(lons=lons, lats=lats)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
except AttributeError:
LOG.debug("No lon/lat found.")
for i, name in enumerate(names):
name = str(name)
if var.dimensions[0].startswith("band"):
chn_data = data[i, :, :]
elif var.dimensions[1].startswith("band"):
chn_data = data[:, i, :]
elif var.dimensions[2].startswith("band"):
chn_data = data[:, :, i]
else:
raise ValueError("Invalid dimension names for band data")
try:
scene[name] = (
chn_data * rootgrp.variables["scale" + str_res][i] +
rootgrp.variables["offset" + str_res][i])
#FIXME complete this
#scene[name].info
except KeyError:
# build the channel on the fly
from mpop.channel import Channel
wv_var = rootgrp.variables["nominal_wavelength" + str_res]
wb_var = rootgrp.variables[getattr(wv_var, "bounds")]
minmax = wb_var[i]
scene.channels.append(
Channel(name, resolution,
(minmax[0], wv_var[i][0], minmax[1])))
scene[name] = (
chn_data * rootgrp.variables["scale" + str_res][i] +
rootgrp.variables["offset" + str_res][i])
if area is not None:
scene[name].area = area
area = None
for attr in rootgrp.ncattrs():
scene.info[attr] = getattr(rootgrp, attr)
scene.add_to_history("Loaded from netcdf4/cf by mpop")
return scene
class TestChannel(unittest.TestCase):
"""Class for testing the Channel class.
"""
chan = None
chan2 = None
def test_init(self):
"""Creation of a channel.
"""
self.assertRaises(ValueError, Channel)
# Name
self.chan = Channel(name="newchan")
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name=numb)
numb = np.random.uniform() * 100000
self.assertRaises(TypeError, Channel, name=numb)
# Resolution
numb = int(np.random.uniform(100000))
self.assertRaises(ValueError, Channel, resolution=numb)
numb = int(np.random.uniform(100000))
self.chan = Channel(name="newchan", resolution=numb)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, numb)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel, name="newchan", resolution="a")
# Wavelength
numbs = [
np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)
]
numbs.sort()
self.chan = Channel(wavelength_range=numbs)
self.assertEqual(self.chan.name, None)
self.assertEqual(self.chan.wavelength_range, numbs)
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel, wavelength_range=numbs[0:1])
numbs.sort(reverse=True)
self.assertRaises(ValueError, Channel, wavelength_range=numbs)
numbs = [
int(np.random.uniform(100)),
int(np.random.uniform(100)),
int(np.random.uniform(100))
]
numbs.sort()
self.assertRaises(TypeError, Channel, wavelength_range=numbs)
self.assertRaises(TypeError,
Channel,
wavelength_range=random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel, wavelength_range=numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, wavelength_range=numb)
# Data
data = np.random.rand(3, 3)
self.assertRaises(ValueError, Channel, data=data)
self.chan = Channel(name="newchan", data=data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
mask = np.array(np.random.rand(3, 3) * 2, dtype=int)
data = np.ma.array(data, mask=mask)
self.chan = Channel(name="newchan", data=data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
self.assertRaises(TypeError,
Channel,
name="newchan",
data=random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel, name="newchan", data=numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name="newchan", data=numb)
numbs = [
np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)
]
self.assertRaises(TypeError, Channel, name="newchan", data=numbs)
def test_cmp(self):
"""Comparison of channels.
"""
self.chan = Channel(name="newchan")
self.chan2 = Channel(name="mychan")
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name="newchan")
self.chan2 = "mychan"
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name="newchan")
self.chan2 = Channel(name="newchan")
self.assert_(self.chan == self.chan2)
self.chan = Channel(wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name="newchan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name="newchan")
self.chan2 = Channel(name="_mychan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name="_newchan")
self.chan2 = Channel(name="mychan")
self.assert_(self.chan > self.chan2)
self.chan = Channel(name=random_string(4),
wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name=random_string(4),
wavelength_range=(4., 5., 6.))
self.assert_(self.chan < self.chan2)
self.chan = Channel(name="_" + random_string(4),
wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name=random_string(4),
wavelength_range=(4., 5., 6.))
self.assert_(self.chan > self.chan2)
def test_str(self):
"""String output for a channel.
"""
self.chan = Channel(name="newchan",
wavelength_range=(1., 2., 3.),
resolution=1000)
self.assertEqual(
str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, resolution 1000m,"
" not loaded'")
self.chan.data = np.random.rand(3, 3)
self.assertEqual(
str(self.chan), "'newchan: (1.000,2.000,3.000)μm, "
"shape (3, 3), "
"resolution 1000m'")
def test_is_loaded(self):
"""Check load status of a channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan")
self.assert_(not self.chan.is_loaded())
self.chan = Channel(name="newchan", data=data)
self.assert_(self.chan.is_loaded())
def test_as_image(self):
"""Check the geo_image version of the channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
img = self.chan.as_image(False)
self.assert_(np.allclose(img.channels[0], data))
self.assertEqual(img.mode, "L")
img = self.chan.as_image(True)
self.assertEqual(img.channels[0].max(), 1)
self.assertEqual(img.channels[0].min(), 0)
def test_check_range(self):
"""Check the range of a channel.
"""
self.chan = Channel(name="newchan")
self.assertRaises(ValueError, self.chan.check_range)
numb = np.random.uniform(10)
self.assertRaises(ValueError, self.chan.check_range, numb)
# ndarray
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
# masked array
mask = np.array(np.random.rand(3, 3) * 2, dtype=int)
mask[1, 1] = False
data = np.ma.array(data, mask=mask)
self.chan = Channel(name="newchan", data=data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
data = np.ma.array(data, mask=True)
self.chan = Channel(name="newchan", data=data)
self.assertEquals(0, self.chan.check_range(min_range).count())
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
# Wrong type arguments
self.assertRaises(TypeError, self.chan.check_range, random_string(4))
self.assertRaises(TypeError, self.chan.check_range,
[np.random.uniform()])
def test_sunzen_corr(self):
'''Test Sun zenith angle correction.
'''
import datetime as dt
chan = Channel(name='test')
original_value = 10.
chan.data = original_value * np.ones((2, 11))
lats = np.zeros((2, 11)) # equator
lons = np.array([np.linspace(-90, 90, 11), np.linspace(-90, 90, 11)])
# Equinox, so the Sun is at the equator
time_slot = dt.datetime(2014, 3, 20, 16, 57)
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
# Test minimum after correction, accuracy of three decimals is enough
#self.assertTrue(np.abs(10.000 - np.min(new_ch.data)) < 10**-3)
self.assertAlmostEqual(10.000, np.min(new_ch.data), places=3)
# Test maximum after correction
self.assertAlmostEqual(57.588, np.max(new_ch.data), places=3)
# There should be ten values at zenith angle >= 80 deg, and
# these are all equal
self.assertTrue(np.where(new_ch.data == \
np.max(new_ch.data))[0].shape[0] == 10)
# All values should be larger than the starting values
self.assertTrue(np.all(new_ch.data > original_value))
# Channel name
self.assertEqual(new_ch.name, chan.name + '_SZC')
# Test channel name in the info dict
self.assertEqual(new_ch.name, chan.info['sun_zen_corrected'])
# Test with several locations and arbitrary data
chan = Channel(name='test2')
chan.data = np.array(
[[0., 67.31614275, 49.96271995, 99.41046645, 29.08660989],
[87.61007584, 79.6683524, 53.20397351, 29.88260374, 62.33623915],
[60.49283004, 54.04267222, 32.72365906, 91.44995651, 32.27232955],
[63.71580638, 69.57673795, 7.63064373, 32.15683105, 9.05786335],
[65.61434337, 33.2317155, 18.77672384, 30.13527574, 23.22572904]])
lons = np.array([[
116.28695847, 164.1125604, 40.77223701, -113.54699788, 133.15558442
], [-17.18990601, 75.17472034, 12.81618371, -40.75524952, 40.70898002],
[
42.74662341, 164.05671859, -166.58469404,
-58.16684483, -144.97963063
],
[
46.26303645, -167.48682034, 170.28131412,
-17.80502488, -63.9031154
],
[
-107.14829679, -147.66665952, -0.75970554,
77.701768, -130.48677807
]])
lats = np.array([
[-51.53681682, -83.21762788, 5.91008672, 22.51730385, 66.83356427],
[82.78543163, 23.1529456, -7.16337152, -68.23118425, 28.72194953],
[31.03440852, 70.55322517, -83.61780288, 29.88413938, 25.7214828],
[-19.02517922, -19.20958728, -14.7825735, 22.66967876, 67.6089238],
[45.12202477, 61.79674149, 58.71037615, -62.04350423, 13.06405864]
])
time_slot = dt.datetime(1998, 8, 1, 10, 0)
# These are the expected results
results = np.array([[
0., 387.65821593, 51.74080022, 572.48205988, 138.96586013
], [
227.24857818, 105.53045776, 62.24134162, 172.0870564, 64.12902666
], [
63.08646652, 311.21934562, 188.44804188, 526.63931022, 185.84893885
], [82.86856236, 400.6764648, 43.9431259, 46.58056343, 36.04457644],
[
377.85794388, 191.3738223, 27.55002934,
173.54213642, 133.75164285
]])
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
self.assertAlmostEqual(np.max(results - new_ch.data), 0.000, places=3)
def test_init(self):
"""Creation of a channel.
"""
self.assertRaises(ValueError, Channel)
# Name
self.chan = Channel(name = "newchan")
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name = numb)
numb = np.random.uniform() * 100000
self.assertRaises(TypeError, Channel, name = numb)
# Resolution
numb = int(np.random.uniform(100000))
self.assertRaises(ValueError, Channel, resolution = numb)
numb = int(np.random.uniform(100000))
self.chan = Channel(name = "newchan", resolution = numb)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, numb)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel,
name = "newchan",
resolution = "a")
# Wavelength
numbs = [np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)]
numbs.sort()
self.chan = Channel(wavelength_range = numbs)
self.assertEqual(self.chan.name, None)
self.assertEqual(self.chan.wavelength_range, numbs)
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel,
wavelength_range = numbs[0:1])
numbs.sort(reverse = True)
self.assertRaises(ValueError, Channel,
wavelength_range = numbs)
numbs = [int(np.random.uniform(100)),
int(np.random.uniform(100)),
int(np.random.uniform(100))]
numbs.sort()
self.assertRaises(TypeError, Channel,
wavelength_range = numbs)
self.assertRaises(TypeError, Channel,
wavelength_range = random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel,
wavelength_range = numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel,
wavelength_range = numb)
# Data
data = np.random.rand(3, 3)
self.assertRaises(ValueError, Channel, data = data)
self.chan = Channel(name = "newchan", data = data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
mask = np.array(np.random.rand(3, 3) * 2, dtype = int)
data = np.ma.array(data, mask = mask)
self.chan = Channel(name = "newchan", data = data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numb)
numbs = [np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)]
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numbs)
# get_area_extent_for_subset(self, row_LR, col_LR, row_UL, col_UL):
aex = area_def.get_area_extent_for_subset(y2, 0, y1, area_def.x_size - 1)
from pyresample.geometry import AreaDefinition
area = "on_the_fly"
# redefine area_def (using still the original area_def on the right side)
area_def = AreaDefinition(area, area, area_def.proj_id, area_def.proj_dict,
area_def.x_size, y2 - y1, aex)
print(" ")
print('*** some info about the NEW area definition')
print(area_def)
# defining a new channels named 'var', the wavenlegth range is not that critial (for radar data I usually use [0.,0.,0.])
data.channels.append(
Channel(name='var', wavelength_range=[0.0, 0.0, 0.0], data=HRV))
data['var'].area = area
data['var'].area_def = area_def
# here we replace the already defined channels
data['HRV'].data = HRV
data['HRV'].area = area
data['HRV'].area_def = area_def
data['VIS006'].data = VIS006
data['VIS006'].area = area
data['VIS006'].area_def = area_def
data['VIS008'].data = VIS008
data['VIS008'].area = area
data['VIS008'].area_def = area_def
data['IR_108'].data = IR_108
data['IR_108'].area = area
class TestChannel(unittest.TestCase):
"""Class for testing the Channel class.
"""
chan = None
chan2 = None
def test_init(self):
"""Creation of a channel.
"""
self.assertRaises(ValueError, Channel)
# Name
self.chan = Channel(name = "newchan")
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel, name = numb)
numb = np.random.uniform() * 100000
self.assertRaises(TypeError, Channel, name = numb)
# Resolution
numb = int(np.random.uniform(100000))
self.assertRaises(ValueError, Channel, resolution = numb)
numb = int(np.random.uniform(100000))
self.chan = Channel(name = "newchan", resolution = numb)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, numb)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel,
name = "newchan",
resolution = "a")
# Wavelength
numbs = [np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)]
numbs.sort()
self.chan = Channel(wavelength_range = numbs)
self.assertEqual(self.chan.name, None)
self.assertEqual(self.chan.wavelength_range, numbs)
self.assertEqual(self.chan.resolution, 0)
self.assert_(self.chan.data is None)
self.assertRaises(TypeError, Channel,
wavelength_range = numbs[0:1])
numbs.sort(reverse = True)
self.assertRaises(ValueError, Channel,
wavelength_range = numbs)
numbs = [int(np.random.uniform(100)),
int(np.random.uniform(100)),
int(np.random.uniform(100))]
numbs.sort()
self.assertRaises(TypeError, Channel,
wavelength_range = numbs)
self.assertRaises(TypeError, Channel,
wavelength_range = random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError, Channel,
wavelength_range = numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError, Channel,
wavelength_range = numb)
# Data
data = np.random.rand(3, 3)
self.assertRaises(ValueError, Channel, data = data)
self.chan = Channel(name = "newchan", data = data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
mask = np.array(np.random.rand(3, 3) * 2, dtype = int)
data = np.ma.array(data, mask = mask)
self.chan = Channel(name = "newchan", data = data)
self.assertEqual(self.chan.name, "newchan")
self.assertEqual(self.chan.wavelength_range,
[-np.inf, -np.inf, -np.inf])
self.assertEqual(self.chan.resolution, 0)
self.assert_(np.all(self.chan.data == data))
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = random_string(4))
numb = np.random.uniform(100000)
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numb)
numb = int(np.random.uniform(100000))
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numb)
numbs = [np.random.uniform(100),
np.random.uniform(100),
np.random.uniform(100)]
self.assertRaises(TypeError,
Channel,
name = "newchan",
data = numbs)
def test_cmp(self):
"""Comparison of channels.
"""
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "mychan")
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = "mychan"
self.assertTrue(self.chan > self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "newchan")
self.assert_(self.chan == self.chan2)
self.chan = Channel(wavelength_range=(1., 2., 3.))
self.chan2 = Channel(name = "newchan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "newchan")
self.chan2 = Channel(name = "_mychan")
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "_newchan")
self.chan2 = Channel(name = "mychan")
self.assert_(self.chan > self.chan2)
self.chan = Channel(name = random_string(4),
wavelength_range = (1., 2., 3.))
self.chan2 = Channel(name = random_string(4),
wavelength_range = (4., 5., 6.))
self.assert_(self.chan < self.chan2)
self.chan = Channel(name = "_" + random_string(4),
wavelength_range = (1., 2., 3.))
self.chan2 = Channel(name = random_string(4),
wavelength_range = (4., 5., 6.))
self.assert_(self.chan > self.chan2)
def test_str(self):
"""String output for a channel.
"""
self.chan = Channel(name="newchan",
wavelength_range=(1., 2., 3.),
resolution=1000)
self.assertEqual(str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, resolution 1000m,"
" not loaded'")
self.chan.data = np.random.rand(3, 3)
self.assertEqual(str(self.chan),
"'newchan: (1.000,2.000,3.000)μm, "
"shape (3, 3), "
"resolution 1000m'")
def test_is_loaded(self):
"""Check load status of a channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name = "newchan")
self.assert_(not self.chan.is_loaded())
self.chan = Channel(name = "newchan", data = data)
self.assert_(self.chan.is_loaded())
def test_as_image(self):
"""Check the geo_image version of the channel.
"""
data = np.random.rand(3, 3)
self.chan = Channel(name="newchan", data=data)
img = self.chan.as_image(False)
self.assert_(np.allclose(img.channels[0], data))
self.assertEqual(img.mode, "L")
img = self.chan.as_image(True)
self.assertEqual(img.channels[0].max(), 1)
self.assertEqual(img.channels[0].min(), 0)
def test_check_range(self):
"""Check the range of a channel.
"""
self.chan = Channel(name = "newchan")
self.assertRaises(ValueError, self.chan.check_range)
numb = np.random.uniform(10)
self.assertRaises(ValueError, self.chan.check_range, numb)
# ndarray
data = np.random.rand(3, 3)
self.chan = Channel(name = "newchan", data = data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
# masked array
mask = np.array(np.random.rand(3, 3) * 2, dtype = int)
mask[1, 1] = False
data = np.ma.array(data, mask = mask)
self.chan = Channel(name = "newchan", data = data)
min_range = (data.max() - data.min()) / 2
self.assert_(np.all(data == self.chan.check_range(min_range)))
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
zeros = np.zeros_like(data)
min_range = (data.max() - data.min()) + E
self.assert_(np.all(zeros == self.chan.check_range(min_range)))
data = np.ma.array(data, mask = True)
self.chan = Channel(name = "newchan", data = data)
self.assertEquals(0,
self.chan.check_range(min_range).count())
self.assertEquals(data.count(),
self.chan.check_range(min_range).count())
# Wrong type arguments
self.assertRaises(TypeError, self.chan.check_range, random_string(4))
self.assertRaises(TypeError,
self.chan.check_range,
[np.random.uniform()])
def test_sunzen_corr(self):
'''Test Sun zenith angle correction.
'''
import datetime as dt
chan = Channel(name='test')
original_value = 10.
chan.data = original_value * np.ones((2,11))
lats = np.zeros((2,11)) # equator
lons = np.array([np.linspace(-90, 90, 11), np.linspace(-90, 90, 11)])
# Equinox, so the Sun is at the equator
time_slot = dt.datetime(2014,3,20,16,57)
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
# Test minimum after correction, accuracy of three decimals is enough
#self.assertTrue(np.abs(10.000 - np.min(new_ch.data)) < 10**-3)
self.assertAlmostEqual(10.000, np.min(new_ch.data), places=3)
# Test maximum after correction
self.assertAlmostEqual(57.588, np.max(new_ch.data), places=3)
# There should be ten values at zenith angle >= 80 deg, and
# these are all equal
self.assertTrue(np.where(new_ch.data == \
np.max(new_ch.data))[0].shape[0] == 10)
# All values should be larger than the starting values
self.assertTrue(np.all(new_ch.data > original_value))
# Channel name
self.assertEqual(new_ch.name, chan.name+'_SZC')
# Test channel name in the info dict
self.assertEqual(new_ch.name, chan.info['sun_zen_corrected'])
# Test with several locations and arbitrary data
chan = Channel(name='test2')
chan.data = np.array([[0., 67.31614275, 49.96271995,
99.41046645, 29.08660989],
[87.61007584, 79.6683524, 53.20397351,
29.88260374, 62.33623915],
[60.49283004, 54.04267222, 32.72365906,
91.44995651, 32.27232955],
[63.71580638, 69.57673795, 7.63064373,
32.15683105, 9.05786335],
[65.61434337, 33.2317155, 18.77672384,
30.13527574, 23.22572904]])
lons = np.array([[116.28695847, 164.1125604, 40.77223701,
-113.54699788, 133.15558442],
[-17.18990601, 75.17472034, 12.81618371,
-40.75524952, 40.70898002],
[42.74662341, 164.05671859, -166.58469404,
-58.16684483, -144.97963063],
[46.26303645, -167.48682034, 170.28131412,
-17.80502488, -63.9031154],
[-107.14829679, -147.66665952, -0.75970554,
77.701768, -130.48677807]])
lats = np.array([[-51.53681682, -83.21762788, 5.91008672,
22.51730385, 66.83356427],
[82.78543163, 23.1529456 , -7.16337152,
-68.23118425, 28.72194953],
[31.03440852, 70.55322517, -83.61780288,
29.88413938, 25.7214828],
[-19.02517922, -19.20958728, -14.7825735,
22.66967876, 67.6089238],
[45.12202477, 61.79674149, 58.71037615,
-62.04350423, 13.06405864]])
time_slot = dt.datetime(1998, 8, 1, 10, 0)
# These are the expected results
results = np.array([[0., 387.65821593, 51.74080022,
572.48205988, 138.96586013],
[227.24857818, 105.53045776, 62.24134162,
172.0870564, 64.12902666],
[63.08646652, 311.21934562, 188.44804188,
526.63931022, 185.84893885],
[82.86856236, 400.6764648, 43.9431259,
46.58056343, 36.04457644],
[377.85794388, 191.3738223, 27.55002934,
173.54213642, 133.75164285]])
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
self.assertAlmostEqual(np.max(results-new_ch.data), 0.000, places=3)
def test_sunzen_corr(self):
'''Test Sun zenith angle correction.
'''
import datetime as dt
chan = Channel(name='test')
original_value = 10.
chan.data = original_value * np.ones((2,11))
lats = np.zeros((2,11)) # equator
lons = np.array([np.linspace(-90, 90, 11), np.linspace(-90, 90, 11)])
# Equinox, so the Sun is at the equator
time_slot = dt.datetime(2014,3,20,16,57)
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
# Test minimum after correction, accuracy of three decimals is enough
#self.assertTrue(np.abs(10.000 - np.min(new_ch.data)) < 10**-3)
self.assertAlmostEqual(10.000, np.min(new_ch.data), places=3)
# Test maximum after correction
self.assertAlmostEqual(57.588, np.max(new_ch.data), places=3)
# There should be ten values at zenith angle >= 80 deg, and
# these are all equal
self.assertTrue(np.where(new_ch.data == \
np.max(new_ch.data))[0].shape[0] == 10)
# All values should be larger than the starting values
self.assertTrue(np.all(new_ch.data > original_value))
# Channel name
self.assertEqual(new_ch.name, chan.name+'_SZC')
# Test channel name in the info dict
self.assertEqual(new_ch.name, chan.info['sun_zen_corrected'])
# Test with several locations and arbitrary data
chan = Channel(name='test2')
chan.data = np.array([[0., 67.31614275, 49.96271995,
99.41046645, 29.08660989],
[87.61007584, 79.6683524, 53.20397351,
29.88260374, 62.33623915],
[60.49283004, 54.04267222, 32.72365906,
91.44995651, 32.27232955],
[63.71580638, 69.57673795, 7.63064373,
32.15683105, 9.05786335],
[65.61434337, 33.2317155, 18.77672384,
30.13527574, 23.22572904]])
lons = np.array([[116.28695847, 164.1125604, 40.77223701,
-113.54699788, 133.15558442],
[-17.18990601, 75.17472034, 12.81618371,
-40.75524952, 40.70898002],
[42.74662341, 164.05671859, -166.58469404,
-58.16684483, -144.97963063],
[46.26303645, -167.48682034, 170.28131412,
-17.80502488, -63.9031154],
[-107.14829679, -147.66665952, -0.75970554,
77.701768, -130.48677807]])
lats = np.array([[-51.53681682, -83.21762788, 5.91008672,
22.51730385, 66.83356427],
[82.78543163, 23.1529456 , -7.16337152,
-68.23118425, 28.72194953],
[31.03440852, 70.55322517, -83.61780288,
29.88413938, 25.7214828],
[-19.02517922, -19.20958728, -14.7825735,
22.66967876, 67.6089238],
[45.12202477, 61.79674149, 58.71037615,
-62.04350423, 13.06405864]])
time_slot = dt.datetime(1998, 8, 1, 10, 0)
# These are the expected results
results = np.array([[0., 387.65821593, 51.74080022,
572.48205988, 138.96586013],
[227.24857818, 105.53045776, 62.24134162,
172.0870564, 64.12902666],
[63.08646652, 311.21934562, 188.44804188,
526.63931022, 185.84893885],
[82.86856236, 400.6764648, 43.9431259,
46.58056343, 36.04457644],
[377.85794388, 191.3738223, 27.55002934,
173.54213642, 133.75164285]])
new_ch = chan.sunzen_corr(time_slot, lonlats=(lons, lats), limit=80.)
self.assertAlmostEqual(np.max(results-new_ch.data), 0.000, places=3)
def plot_msg_minus_cosmo(in_msg):
# do statistics for the last full hour (minutes=0, seconds=0)
in_msg.datetime = datetime(in_msg.datetime.year, in_msg.datetime.month,
in_msg.datetime.day, in_msg.datetime.hour, 0, 0)
area_loaded = choose_area_loaded_msg(in_msg.sat, in_msg.sat_nr,
in_msg.datetime)
# define contour write for coasts, borders, rivers
cw = ContourWriterAGG(in_msg.mapDir)
# check if input data is complete
if in_msg.verbose:
print("*** check input data for ", in_msg.sat_str())
RGBs = check_input(in_msg,
in_msg.sat_str(layout="%(sat)s") + in_msg.sat_nr_str(),
in_msg.datetime)
# in_msg.sat_nr might be changed to backup satellite
if in_msg.verbose:
print('*** Create plots for ')
print(' Satellite/Sensor: ' + in_msg.sat_str())
print(' Satellite number: ' + in_msg.sat_nr_str() + ' // ' +
str(in_msg.sat_nr))
print(' Satellite instrument: ' + in_msg.instrument)
print(' Date/Time: ' + str(in_msg.datetime))
print(' RGBs: ', in_msg.RGBs)
print(' Area: ', in_msg.areas)
print(' reader level: ', in_msg.reader_level)
# define satellite data object
#global_data = GeostationaryFactory.create_scene(in_msg.sat, in_msg.sat_nr_str(), "seviri", in_msg.datetime)
global_data = GeostationaryFactory.create_scene(in_msg.sat_str(),
in_msg.sat_nr_str(),
in_msg.instrument,
in_msg.datetime)
# global_data = GeostationaryFactory.create_scene("msg-ot", "", "Overshooting_Tops", in_msg.datetime)
if len(RGBs) == 0 and len(in_msg.postprocessing_areas) == 0:
return RGBs
if in_msg.verbose:
print(
"*** load satellite channels for " + in_msg.sat_str() +
in_msg.sat_nr_str() + " ", global_data.fullname)
# initialize processed RGBs
RGBs_done = []
# -------------------------------------------------------------------
# load reflectivities, brightness temperatures, NWC-SAF products ...
# -------------------------------------------------------------------
area_loaded = load_products(global_data, RGBs, in_msg, area_loaded)
cosmo_input_file = "input_cosmo_cronjob.py"
print("... read COSMO input file: ", cosmo_input_file)
in_cosmo = parse_commandline_and_read_inputfile(
input_file=cosmo_input_file)
# add composite
in_msg.scpOutput = True
in_msg.resize_montage = 70
in_msg.postprocessing_montage = [[
"MSG_IR-108cpc", "COSMO_SYNMSG-BT-CL-IR10.8",
"MSG_IR-108-COSMO-minus-MSGpc"
]]
in_msg.scpProducts = [[
"MSG_IR-108cpc", "COSMO_SYNMSG-BT-CL-IR10.8",
"MSG_IR-108-COSMO-minus-MSGpc"
]]
#in_msg.scpProducts = ["all"]
# define satellite data object
cosmo_data = GeostationaryFactory.create_scene(in_cosmo.sat_str(),
in_cosmo.sat_nr_str(),
in_cosmo.instrument,
in_cosmo.datetime)
area_loaded_cosmo = load_products(cosmo_data, ['SYNMSG_BT_CL_IR10.8'],
in_cosmo, area_loaded)
# preprojecting the data to another area
# --------------------------------------
if len(RGBs) > 0:
for area in in_msg.areas:
print("")
obj_area = get_area_def(area)
if area != 'ccs4':
print("*** WARNING, diff MSG-COSMO only implemented for ccs4")
continue
# reproject data to new area
print(area_loaded)
if obj_area == area_loaded:
if in_msg.verbose:
print("*** Use data for the area loaded: ", area)
#obj_area = area_loaded
data = global_data
resolution = 'l'
else:
if in_msg.verbose:
print("*** Reproject data to area: ", area,
"(org projection: ", area_loaded.name, ")")
obj_area = get_area_def(area)
# PROJECT data to new area
data = global_data.project(area, precompute=True)
resolution = 'i'
if in_msg.parallax_correction:
loaded_products = [chn.name for chn in data.loaded_channels()]
if 'CTH' not in loaded_products:
print("*** Error in plot_msg (" +
inspect.getfile(inspect.currentframe()) + ")")
print(
" Cloud Top Height is needed for parallax correction "
)
print(
" either load CTH or specify the estimation of the CTH in the input file (load 10.8 in this case)"
)
quit()
if in_msg.verbose:
print(
" perform parallax correction for loaded channels: ",
loaded_products)
data = data.parallax_corr(fill=in_msg.parallax_gapfilling,
estimate_cth=in_msg.estimate_cth,
replace=True)
# save reprojected data
if area in in_msg.save_reprojected_data:
save_reprojected_data(data, area, in_msg)
# apply a mask to the data (switched off at the moment)
if False:
mask_data(data, area)
# save average values
if in_msg.save_statistics:
mean_array = zeros(len(RGBs))
#statisticFile = '/data/COALITION2/database/meteosat/ccs4/'+yearS+'/'+monthS+'/'+dayS+'/MSG_'+area+'_'+yearS[2:]+monthS+dayS+'.txt'
statisticFile = './' + yearS + '-' + monthS + '-' + dayS + '/MSG_' + area + '_' + yearS[
2:] + monthS + dayS + '.txt'
if in_msg.verbose:
print("*** write statistics (average values) to " +
statisticFile)
f1 = open(statisticFile, 'a') # mode append
i_rgb = 0
for rgb in RGBs:
if rgb in products.MSG_color:
mean_array[i_rgb] = data[rgb.replace("c",
"")].data.mean()
i_rgb = i_rgb + 1
# create string to write
str2write = dateS + ' ' + hourS + ' : ' + minS + ' UTC '
for mm in mean_array:
str2write = str2write + ' ' + "%7.2f" % mm
str2write = str2write + "\n"
f1.write(str2write)
f1.close()
# creating plots/images
if in_msg.make_plots:
# choose map resolution
in_msg.resolution = choose_map_resolution(
area, in_msg.mapResolution)
# define area
proj4_string = obj_area.proj4_string
# e.g. proj4_string = '+proj=geos +lon_0=0.0 +a=6378169.00 +b=6356583.80 +h=35785831.0'
area_extent = obj_area.area_extent
# e.g. area_extent = (-5570248.4773392612, -5567248.074173444, 5567248.074173444, 5570248.4773392612)
area_tuple = (proj4_string, area_extent)
RGBs = ['IR_108-COSMO-minus-MSG']
print(data['IR_108'].data.shape)
print(cosmo_data['SYNMSG_BT_CL_IR10.8'].data.shape)
diff_MSG_COSMO = cosmo_data['SYNMSG_BT_CL_IR10.8'].data - data[
'IR_108'].data
HRV_enhance_str = ''
# add IR difference as "channel object" to satellite regional "data" object
data.channels.append(
Channel(name=RGBs[0],
wavelength_range=[0., 0., 0.],
resolution=data['IR_108'].resolution,
data=diff_MSG_COSMO))
for rgb in RGBs:
if not check_loaded_channels(rgb, data):
continue
PIL_image = create_PIL_image(rgb,
data,
in_msg,
obj_area=obj_area)
# !!! in_msg.colorbar[rgb] is initialized inside (give attention to rgbs) !!!
add_borders_and_rivers(PIL_image,
cw,
area_tuple,
add_borders=in_msg.add_borders,
border_color=in_msg.border_color,
add_rivers=in_msg.add_rivers,
river_color=in_msg.river_color,
resolution=in_msg.resolution,
verbose=in_msg.verbose)
# indicate mask
if in_msg.indicate_mask:
PIL_image = indicate_mask(rgb, PIL_image, data,
in_msg.verbose)
#if area.find("EuropeCanary") != -1 or area.find("ccs4") != -1:
dc = DecoratorAGG(PIL_image)
# add title to image
if in_msg.add_title:
add_title(PIL_image,
in_msg.title,
HRV_enhance_str + rgb,
in_msg.sat_str(),
data.sat_nr(),
in_msg.datetime,
area,
dc,
in_msg.font_file,
in_msg.verbose,
title_color=in_msg.title_color,
title_y_line_nr=in_msg.title_y_line_nr
) # !!! needs change
# add MeteoSwiss and Pytroll logo
if in_msg.add_logos:
if in_msg.verbose:
print('... add logos')
dc.align_right()
if in_msg.add_colorscale:
dc.write_vertically()
if PIL_image.mode != 'L':
height = 60 # height=60.0 normal resolution
dc.add_logo(in_msg.logos_dir + "/pytroll3.jpg",
height=height) # height=60.0
dc.add_logo(in_msg.logos_dir + "/meteoSwiss3.jpg",
height=height)
dc.add_logo(
in_msg.logos_dir +
"/EUMETSAT_logo2_tiny_white_square.png",
height=height) # height=60.0
# add colorscale
if in_msg.add_colorscale and in_msg.colormap[rgb] != None:
if rgb in products.MSG_color:
unit = data[rgb.replace("c", "")].info['units']
#elif rgb in products.MSG or rgb in products.NWCSAF or rgb in products.HSAF:
# unit = data[rgb].info['units']
else:
unit = None
loaded_channels = [
chn.name for chn in data.loaded_channels()
]
if rgb in loaded_channels:
if hasattr(data[rgb], 'info'):
print(" hasattr(data[rgb], 'info')",
list(data[rgb].info.keys()))
if 'units' in list(data[rgb].info.keys()):
print(
"'units' in data[rgb].info.keys()")
unit = data[rgb].info['units']
print("... units = ", unit)
add_colorscale(dc, rgb, in_msg, unit=unit)
if in_msg.parallax_correction:
parallax_correction_str = 'pc'
else:
parallax_correction_str = ''
rgb += parallax_correction_str
# create output filename
outputDir = format_name(
in_msg.outputDir,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr()) # !!! needs change
outputFile = outputDir + "/" + format_name(
in_msg.outputFile,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr()) # !!! needs change
# check if output directory exists, if not create it
path = dirname(outputFile)
if not exists(path):
if in_msg.verbose:
print('... create output directory: ' + path)
makedirs(path)
# save file
if exists(outputFile) and not in_msg.overwrite:
if stat(outputFile).st_size > 0:
print('... outputFile ' + outputFile +
' already exists (keep old file)')
else:
print(
'*** Warning, outputFile' + outputFile +
' already exists, but is empty (overwrite file)'
)
PIL_image.save(outputFile, optimize=True
) # optimize -> minimize file size
chmod(
outputFile, 0o777
) ## FOR PYTHON3: 0o664 # give access read/write access to group members
else:
if in_msg.verbose:
print('... save final file: ' + outputFile)
PIL_image.save(
outputFile,
optimize=True) # optimize -> minimize file size
chmod(
outputFile, 0o777
) ## FOR PYTHON3: 0o664 # give access read/write access to group members
if in_msg.compress_to_8bit:
if in_msg.verbose:
print('... compress to 8 bit image: display ' +
outputFile.replace(".png", "-fs8.png") +
' &')
subprocess.call(
"/usr/bin/pngquant -force 256 " + outputFile +
" 2>&1 &",
shell=True) # 256 == "number of colors"
#if in_msg.verbose:
# print " add coastlines to "+outputFile
## alternative: reopen image and modify it (takes longer due to additional reading and saving)
#cw.add_rivers_to_file(img, area_tuple, level=5, outline='blue', width=0.5, outline_opacity=127)
#cw.add_coastlines_to_file(outputFile, obj_area, resolution=resolution, level=4)
#cw.add_borders_to_file(outputFile, obj_area, outline=outline, resolution=resolution)
# secure copy file to another place
if in_msg.scpOutput:
if (rgb in in_msg.scpProducts) or ('all' in [
x.lower()
for x in in_msg.scpProducts if type(x) == str
]):
scpOutputDir = format_name(in_msg.scpOutputDir,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr())
if in_msg.compress_to_8bit:
if in_msg.verbose:
print("... secure copy " +
outputFile.replace(
".png", "-fs8.png") + " to " +
scpOutputDir)
subprocess.call(
"scp " + in_msg.scpID + " " +
outputFile.replace(".png", "-fs8.png") +
" " + scpOutputDir + " 2>&1 &",
shell=True)
else:
if in_msg.verbose:
print("... secure copy " + outputFile +
" to " + scpOutputDir)
subprocess.call("scp " + in_msg.scpID + " " +
outputFile + " " +
scpOutputDir + " 2>&1 &",
shell=True)
if in_msg.scpOutput and in_msg.scpID2 != None and in_msg.scpOutputDir2 != None:
if (rgb in in_msg.scpProducts2) or ('all' in [
x.lower()
for x in in_msg.scpProducts2 if type(x) == str
]):
scpOutputDir2 = format_name(in_msg.scpOutputDir2,
data.time_slot,
area=area,
rgb=rgb,
sat=data.satname,
sat_nr=data.sat_nr())
if in_msg.compress_to_8bit:
if in_msg.verbose:
print("... secure copy " +
outputFile.replace(
".png", "-fs8.png") + " to " +
scpOutputDir2)
subprocess.call(
"scp " + in_msg.scpID2 + " " +
outputFile.replace(".png", "-fs8.png") +
" " + scpOutputDir2 + " 2>&1 &",
shell=True)
else:
if in_msg.verbose:
print("... secure copy " + outputFile +
" to " + scpOutputDir2)
subprocess.call("scp " + in_msg.scpID2 + " " +
outputFile + " " +
scpOutputDir2 + " 2>&1 &",
shell=True)
if 'ninjotif' in in_msg.outputFormats:
ninjotif_file = format_name(outputDir + '/' +
in_msg.ninjotifFilename,
data.time_slot,
sat_nr=data.sat_nr(),
RSS=in_msg.RSS,
area=area,
rgb=rgb)
from plot_coalition2 import pilimage2geoimage
GEO_image = pilimage2geoimage(PIL_image, obj_area,
data.time_slot)
GEO_image.save(ninjotif_file,
fformat='mpop.imageo.formats.ninjotiff',
ninjo_product_name=rgb,
chan_id=products.ninjo_chan_id[
rgb.replace("_", "-") + "_" + area],
nbits=8)
chmod(ninjotif_file, 0o777)
print(("... save ninjotif image: display ",
ninjotif_file, " &"))
if rgb not in RGBs_done:
RGBs_done.append(rgb)
## start postprocessing
for area in in_msg.postprocessing_areas:
postprocessing(in_msg, global_data.time_slot, int(data.sat_nr()),
area)
if in_msg.verbose:
print(" ")
return RGBs_done
def _load02(filename):
"""Load data from a netcdf4 file, cf-satellite v0.2 (2012-02-03).
"""
rootgrp = Dataset(filename, 'r')
# processed variables
processed = set()
# Currently MPOP does not like unicode (so much).
satellite_name, satellite_number = [
str(i) for i in rootgrp.platform.rsplit("-", 1)
]
time_slot = rootgrp.variables["time"].getValue()[0]
time_slot = num2date(time_slot, TIME_UNITS)
processed |= set(["time"])
try:
service = str(rootgrp.service)
except AttributeError:
service = ""
instrument_name = str(rootgrp.instrument)
try:
orbit = str(rootgrp.orbit)
except AttributeError:
orbit = None
try:
scene = GenericFactory.create_scene(satellite_name, satellite_number,
instrument_name, time_slot, orbit,
None, service)
except NoSectionError:
scene = VisirCompositer(time_slot=time_slot)
scene.satname = satellite_name
scene.number = satellite_number
scene.service = service
dim_chart = {}
for var_name, var in rootgrp.variables.items():
varname = None
try:
varname = var.standard_name
except AttributeError:
try:
varname = var.long_name
except AttributeError:
pass
if varname in ["band_data", "Band data"]:
LOG.debug("Found some data: " + var_name)
dims = var.dimensions
for dim in dims:
dim_chart[dim] = var_name
for cnt, dim in enumerate(dims):
if dim.startswith("band"):
break
data = var
data.set_auto_maskandscale(False)
area = None
try:
area_var_name = getattr(var, "grid_mapping")
area_var = rootgrp.variables[area_var_name]
proj4_dict = {}
for attr, projattr in MAPPING_ATTRIBUTES.items():
try:
the_attr = getattr(area_var, attr)
if projattr == "proj":
proj4_dict[projattr] = PROJNAME[the_attr]
elif (isinstance(projattr, (list, tuple))):
try:
for i, subattr in enumerate(the_attr):
proj4_dict[projattr[i]] = subattr
except TypeError:
proj4_dict[projattr[0]] = the_attr
else:
proj4_dict[projattr] = the_attr
except AttributeError:
pass
y_name, x_name = dims[:cnt] + dims[cnt + 1:]
x__ = rootgrp.variables[x_name][:]
y__ = rootgrp.variables[y_name][:]
if proj4_dict["proj"] == "geos":
x__ *= proj4_dict["h"]
y__ *= proj4_dict["h"]
x_pixel_size = abs((np.diff(x__)).mean())
y_pixel_size = abs((np.diff(y__)).mean())
llx = x__[0] - x_pixel_size / 2.0
lly = y__[-1] - y_pixel_size / 2.0
urx = x__[-1] + x_pixel_size / 2.0
ury = y__[0] + y_pixel_size / 2.0
area_extent = (llx, lly, urx, ury)
try:
# create the pyresample areadef
from pyresample.geometry import AreaDefinition
area = AreaDefinition("myareaid", "myareaname",
"myprojid", proj4_dict, len(x__),
len(y__), area_extent)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
processed |= set([area_var_name, x_name, y_name])
LOG.debug("Grid mapping found and used.")
except AttributeError:
LOG.debug("No grid mapping found.")
try:
area_var = getattr(var, "coordinates")
coordinates_vars = area_var.split(" ")
lons = None
lats = None
for coord_var_name in coordinates_vars:
coord_var = rootgrp.variables[coord_var_name]
units = getattr(coord_var, "units")
if (coord_var_name.lower().startswith("lon")
or units.lower().endswith("east")
or units.lower().endswith("west")):
lons = coord_var[:]
elif (coord_var_name.lower().startswith("lat")
or units.lower().endswith("north")
or units.lower().endswith("south")):
lats = coord_var[:]
if lons.any() and lats.any():
try:
from pyresample.geometry import SwathDefinition
area = SwathDefinition(lons=lons, lats=lats)
except ImportError:
LOG.warning("Pyresample not found, "
"cannot load area descrition")
processed |= set(coordinates_vars)
LOG.debug("Lon/lat found and used.")
except AttributeError:
LOG.debug("No lon/lat found.")
names = rootgrp.variables[dim][:]
scales = data.scale_factor
offsets = data.add_offset
if len(names) == 1:
scales = np.array([scales])
offsets = np.array([offsets])
LOG.info("Scales and offsets: %s %s %s" %
(str(names), str(scales), str(offsets)))
for nbr, name in enumerate(names):
name = str(name)
try:
if cnt == 0:
chn_data = data[nbr, :, :].squeeze()
if cnt == 1:
chn_data = data[:, nbr, :].squeeze()
if cnt == 2:
chn_data = data[:, :, nbr].squeeze()
scene[name] = (
np.ma.masked_equal(chn_data, data._FillValue) *
scales[nbr] + offsets[nbr])
scene[name].info["units"] = var.units
except KeyError:
from mpop.channel import Channel
scene.channels.append(Channel(name))
if area is not None:
scene[name].area = area
processed |= set([var_name, dim])
non_processed = set(rootgrp.variables.keys()) - processed
for var_name in non_processed:
var = rootgrp.variables[var_name]
if not (hasattr(var, "standard_name") or hasattr(var, "long_name")):
LOG.info("Delayed processing of " + var_name)
continue
dims = var.dimensions
if len(dims) != 1:
LOG.info("Don't know what to do with " + var_name)
continue
dim = dims[0]
if var.standard_name == "radiation_wavelength":
names = rootgrp.variables[dim][:]
for nbr, name in enumerate(names):
name = str(name)
scene[name].wavelength_range[1] = var[nbr]
try:
bnds = rootgrp.variables[var.bounds][:]
for nbr, name in enumerate(names):
name = str(name)
scene[name].wavelength_range[0] = bnds[nbr, 0]
scene[name].wavelength_range[2] = bnds[nbr, 1]
processed |= set([var.bounds])
except AttributeError:
pass
processed |= set([var_name])
non_processed = set(rootgrp.variables.keys()) - processed
if len(non_processed) > 0:
LOG.warning("Remaining non-processed variables: " + str(non_processed))
return scene
def load_constant_fields(sat_nr):
# radar threshold mask:
radar_mask = GeostationaryFactory.create_scene("odyssey", "", "radar",
datetime(1900, 1, 1, 0))
# reproject this to the desired area:
mask_rad_thres = np.load(
'../data/odyssey_mask/threshold_exceedance_mask_avg15cut2_cut04_cutmistral_201706_201707_201708.npy'
)
from mpop.projector import get_area_def
area_radar_mask = 'EuropeOdyssey00'
radar_mask.channels.append(
Channel(name='mask_radar',
wavelength_range=[0., 0., 0.],
data=mask_rad_thres[:, :]))
radar_mask['mask_radar'].area = area_radar_mask
radar_mask['mask_radar'].area_def = get_area_def(area_radar_mask)
# nominal viewing geometry
print('*** read nominal viewing geometry', "meteosat", sat_nr, "seviri")
# time_slot has NO influence at all just goes looking for the nominal position file -> will use these fields for all dates
vg = GeostationaryFactory.create_scene("meteosat", sat_nr, "seviri",
datetime(1900, 1, 1, 0))
vg.load(['vaa', 'vza', 'lon', 'lat'], reader_level="seviri-level6")
msg_area = deepcopy(vg['vaa'].area)
msg_area_def = deepcopy(vg['vaa'].area_def)
msg_resolution = deepcopy(vg['vaa'].resolution)
# read land sea mask (full SEVIRI Disk seen from 0 degree East)
ls_file = '../data/SEVIRI_data/LandSeaMask_SeviriDiskFull00.nc'
fh = Dataset(ls_file, mode='r')
lsmask = fh.variables['lsmask'][:]
# read topography (full SEVIRI Disk seen from 0 degree East)
ls_file = '../data/SEVIRI_data/SRTM_15sec_elevation_SeviriDiskFull00.nc'
fh = Dataset(ls_file, mode='r')
ele = fh.variables['elevation'][:]
# create a dummy satellite object (to reproject the land/sea mask and elevation)
ls_ele = GeostationaryFactory.create_scene("meteosat", sat_nr, "seviri",
datetime(1900, 1, 1, 0))
#ls_ele.load(['CTTH'], calibrate=True, reader_level="seviri-level3")
#convert_NWCSAF_to_radiance_format(ls_ele, None,'CTH', False, True)
# add land sea mask as a dummy channel
ls_ele.channels.append(
Channel(name='lsmask',
wavelength_range=[0., 0., 0.],
resolution=msg_resolution,
data=lsmask[::-1, :]))
#ls_ele['lsmask'].area = ls_ele['CTH'].area
#ls_ele['lsmask'].area_def = ls_ele['CTH'].area_def
ls_ele['lsmask'].area = msg_area
ls_ele['lsmask'].area_def = msg_area_def
# add elevation as a dummy channel
ls_ele.channels.append(
Channel(name='ele',
wavelength_range=[0., 0., 0.],
resolution=msg_resolution,
data=ele[::-1, :]))
#ls_ele['ele'].area = ls_ele['CTH'].area
#ls_ele['ele'].area_def = ls_ele['CTH'].area_def
ls_ele['ele'].area = msg_area
ls_ele['ele'].area_def = msg_area_def
return radar_mask, vg, ls_ele