def test_modis_overview_1000m(self):
"""Test a modis overview dependency calculation with resolution fixed to 1000m."""
from satpy.config import PACKAGE_CONFIG_PATH
from satpy.readers.yaml_reader import FileYAMLReader
from satpy import DataQuery
from satpy.composites import GenericCompositor
from satpy.modifiers.geometry import SunZenithCorrector
from satpy.dataset import DatasetDict
config_file = os.path.join(PACKAGE_CONFIG_PATH, 'readers', 'modis_l1b.yaml')
self.reader_instance = FileYAMLReader.from_config_files(config_file)
overview = {'_satpy_id': make_dataid(name='overview'),
'name': 'overview',
'optional_prerequisites': [],
'prerequisites': [DataQuery(name='1', modifiers=('sunz_corrected',)),
DataQuery(name='2', modifiers=('sunz_corrected',)),
DataQuery(name='31')],
'standard_name': 'overview'}
compositors = {'modis': DatasetDict()}
compositors['modis']['overview'] = GenericCompositor(**overview)
modifiers = {'modis': {'sunz_corrected': (SunZenithCorrector,
{'optional_prerequisites': ['solar_zenith_angle'],
'name': 'sunz_corrected',
'prerequisites': []})}}
dep_tree = DependencyTree({'modis_l1b': self.reader_instance}, compositors, modifiers)
dep_tree.populate_with_keys({'overview'}, DataQuery(resolution=1000))
for key in dep_tree._all_nodes.keys():
assert key.get('resolution', 1000) == 1000
def __call__(self, projectables, **info):
"""Create the composite."""
if len(projectables) != 3:
raise ValueError("Expected 3 datasets, got %d" %
(len(projectables), ))
data, palette, status = projectables
fill_value_color = palette.attrs.get("fill_value_color", [0, 0, 0])
colormap, palette = self.build_colormap(palette, data.attrs)
mapped_channels = colormap.colorize(data.data)
valid = status != status.attrs['_FillValue']
# cloud-free pixels are marked invalid (fill_value in ctth_alti) but have status set to 1.
status_not_cloud_free = status % 2 == 0
not_cloud_free = np.logical_or(status_not_cloud_free,
np.logical_not(valid))
channels = []
for (channel, cloud_free_color) in zip(mapped_channels,
fill_value_color):
channel_data = self._create_masked_dataarray_like(
channel, data, valid)
# Set cloud-free pixels as fill_value_color
channels.append(
channel_data.where(not_cloud_free, cloud_free_color))
res = GenericCompositor.__call__(self, channels, **data.attrs)
res.attrs['_FillValue'] = np.nan
return res
def setUp(self):
"""Create test data."""
from satpy.composites import GenericCompositor
self.comp = GenericCompositor(name='test')
self.comp2 = GenericCompositor(name='test2', common_channel_mask=False)
all_valid = np.ones((1, 2, 2))
self.all_valid = xr.DataArray(all_valid, dims=['bands', 'y', 'x'])
first_invalid = np.reshape(np.array([np.nan, 1., 1., 1.]), (1, 2, 2))
self.first_invalid = xr.DataArray(first_invalid,
dims=['bands', 'y', 'x'])
second_invalid = np.reshape(np.array([1., np.nan, 1., 1.]), (1, 2, 2))
self.second_invalid = xr.DataArray(second_invalid,
dims=['bands', 'y', 'x'])
wrong_shape = np.reshape(np.array([1., 1., 1.]), (1, 3, 1))
self.wrong_shape = xr.DataArray(wrong_shape, dims=['bands', 'y', 'x'])
def __call__(self, projectables, **info):
"""Create the composite."""
if len(projectables) != 3:
raise ValueError("Expected 3 datasets, got %d" %
(len(projectables), ))
data, palette, status = projectables
colormap, palette = self.build_colormap(palette, data.attrs)
channels = colormap.colorize(np.asanyarray(data))
not_nan = np.logical_and(self._get_mask_from_data(data),
status != status.attrs['_FillValue'])
not_cloud_free = np.logical_or((status + 1) % 2,
np.logical_not(not_nan))
chans = []
for channel in channels:
chan = self._create_masked_dataarray_like(channel, data, not_nan)
# Set cloud-free pixels as black
chans.append(chan.where(not_cloud_free, 0))
res = GenericCompositor.__call__(self, chans, **data.attrs)
res.attrs['_FillValue'] = np.nan
return res
class TestGenericCompositor(unittest.TestCase):
"""Test generic compositor."""
def setUp(self):
"""Create test data."""
from satpy.composites import GenericCompositor
self.comp = GenericCompositor(name='test')
self.comp2 = GenericCompositor(name='test2', common_channel_mask=False)
all_valid = np.ones((1, 2, 2))
self.all_valid = xr.DataArray(all_valid, dims=['bands', 'y', 'x'])
first_invalid = np.reshape(np.array([np.nan, 1., 1., 1.]), (1, 2, 2))
self.first_invalid = xr.DataArray(first_invalid,
dims=['bands', 'y', 'x'])
second_invalid = np.reshape(np.array([1., np.nan, 1., 1.]), (1, 2, 2))
self.second_invalid = xr.DataArray(second_invalid,
dims=['bands', 'y', 'x'])
wrong_shape = np.reshape(np.array([1., 1., 1.]), (1, 3, 1))
self.wrong_shape = xr.DataArray(wrong_shape, dims=['bands', 'y', 'x'])
def test_masking(self):
"""Test masking in generic compositor."""
# Single channel
res = self.comp([self.all_valid])
np.testing.assert_allclose(res.data, 1., atol=1e-9)
# Three channels, one value invalid
res = self.comp([self.all_valid, self.all_valid, self.first_invalid])
correct = np.reshape(np.array([np.nan, 1., 1., 1.]), (2, 2))
for i in range(3):
np.testing.assert_almost_equal(res.data[i, :, :], correct)
# Three channels, two values invalid
res = self.comp(
[self.all_valid, self.first_invalid, self.second_invalid])
correct = np.reshape(np.array([np.nan, np.nan, 1., 1.]), (2, 2))
for i in range(3):
np.testing.assert_almost_equal(res.data[i, :, :], correct)
def test_concat_datasets(self):
"""Test concatenation of datasets."""
from satpy.composites import IncompatibleAreas
res = self.comp._concat_datasets([self.all_valid], 'L')
num_bands = len(res.bands)
self.assertEqual(num_bands, 1)
self.assertEqual(res.shape[0], num_bands)
self.assertTrue(res.bands[0] == 'L')
res = self.comp._concat_datasets([self.all_valid, self.all_valid],
'LA')
num_bands = len(res.bands)
self.assertEqual(num_bands, 2)
self.assertEqual(res.shape[0], num_bands)
self.assertTrue(res.bands[0] == 'L')
self.assertTrue(res.bands[1] == 'A')
self.assertRaises(IncompatibleAreas, self.comp._concat_datasets,
[self.all_valid, self.wrong_shape], 'LA')
def test_get_sensors(self):
"""Test getting sensors from the dataset attributes."""
res = self.comp._get_sensors([self.all_valid])
self.assertIsNone(res)
dset1 = self.all_valid
dset1.attrs['sensor'] = 'foo'
res = self.comp._get_sensors([dset1])
self.assertEqual(res, 'foo')
dset2 = self.first_invalid
dset2.attrs['sensor'] = 'bar'
res = self.comp._get_sensors([dset1, dset2])
self.assertTrue('foo' in res)
self.assertTrue('bar' in res)
self.assertEqual(len(res), 2)
self.assertTrue(isinstance(res, set))
@mock.patch('satpy.composites.GenericCompositor._get_sensors')
@mock.patch('satpy.composites.combine_metadata')
@mock.patch('satpy.composites.check_times')
@mock.patch('satpy.composites.GenericCompositor.match_data_arrays')
def test_call_with_mock(self, match_data_arrays, check_times,
combine_metadata, get_sensors):
"""Test calling generic compositor."""
from satpy.composites import IncompatibleAreas
combine_metadata.return_value = dict()
get_sensors.return_value = 'foo'
# One dataset, no mode given
res = self.comp([self.all_valid])
self.assertEqual(res.shape[0], 1)
self.assertEqual(res.attrs['mode'], 'L')
match_data_arrays.assert_not_called()
# This compositor has been initialized without common masking, so the
# masking shouldn't have been called
projectables = [
self.all_valid, self.first_invalid, self.second_invalid
]
match_data_arrays.return_value = projectables
res = self.comp2(projectables)
match_data_arrays.assert_called_once()
match_data_arrays.reset_mock()
# Dataset for alpha given, so shouldn't be masked
projectables = [self.all_valid, self.all_valid]
match_data_arrays.return_value = projectables
res = self.comp(projectables)
match_data_arrays.assert_called_once()
match_data_arrays.reset_mock()
# When areas are incompatible, masking shouldn't happen
match_data_arrays.side_effect = IncompatibleAreas()
self.assertRaises(IncompatibleAreas, self.comp,
[self.all_valid, self.wrong_shape])
match_data_arrays.assert_called_once()
def test_call(self):
"""Test calling generic compositor."""
# Multiple datasets with extra attributes
all_valid = self.all_valid
all_valid.attrs['sensor'] = 'foo'
attrs = {'foo': 'bar', 'resolution': 333}
self.comp.attrs['resolution'] = None
res = self.comp([self.all_valid, self.first_invalid], **attrs)
# Verify attributes
self.assertEqual(res.attrs.get('sensor'), 'foo')
self.assertTrue('foo' in res.attrs)
self.assertEqual(res.attrs.get('foo'), 'bar')
self.assertTrue('units' not in res.attrs)
self.assertTrue('calibration' not in res.attrs)
self.assertTrue('modifiers' not in res.attrs)
self.assertIsNone(res.attrs['wavelength'])
self.assertEqual(res.attrs['mode'], 'LA')
self.assertEquals(res.attrs['resolution'], 333)
flags.append((trainData[iTRK])['flag'])
#Process image and label files if at least 1 ship track satisfies the cropped region condition
if np.sum(flags) < len(trainData):
#Create image
pngFile = path_output_images + str(CT).zfill(4) + '.png'
if os.path.isfile(pngFile):
print('exists: ' + pngFile)
else:
global_scene = (Scene(reader="hdfeos_l1b",
filenames=[file02, file03]))
global_scene.load(['1', '20', '32'], resolution=1000)
global_scene = global_scene[0:yN,
cropped_area:xN - cropped_area]
compositor = GenericCompositor("rgb")
composite = compositor([
global_scene['1'], global_scene['20'], global_scene['32']
])
img = to_image(composite)
img.stretch_hist_equalize("linear")
img.save(pngFile)
print(pngFile)
#Create labels - valid (within cropped area) training data only
txtFile = path_output_labels + str(CT).zfill(4) + '.txt'
text_file = open(txtFile, "w")
for iTRK in range(len(trainData)):
if (trainData[iTRK])['flag'] == 0:
text_file.write((trainData[iTRK])['type'] + ' ' +
(trainData[iTRK])['truncated'] + ' ' +
print global_scene.available_writers()
if save_overview:
#local_scene.show('overview')
local_scene.save_dataset('overview',
'./overview_' + area + '.png',
overlay={
'coast_dir': '/data/OWARNA/hau/maps_pytroll/',
'color': (255, 255, 255),
'resolution': 'i'
})
print 'display ./overview_' + area + '.png &'
if save_overview_composite:
from satpy.composites import GenericCompositor
compositor = GenericCompositor("overview")
composite = compositor(
[local_scene[0.6], local_scene[0.8], local_scene["IR_108"]])
img = to_image(composite)
img.invert([False, False, True])
img.stretch("linear")
img.gamma(1.7)
img.save("./overview_" + area + "_composite.png")
#img.show()
local_scene["ndvi"] = (local_scene[0.8] - local_scene[0.6]) / (
local_scene[0.8] + local_scene[0.6])
#local_scene["ndvi"].area = local_scene[0.8].area
print "local_scene[\"ndvi\"].min()", local_scene["ndvi"].compute().min()
print "local_scene[\"ndvi\"].max()", local_scene["ndvi"].compute().max()