def test_remove_feature(self):
bands = np.arange(2 * 3 * 3 * 2).reshape(2, 3, 3, 2)
names = ['bands1', 'bands2', 'bands3']
eop = EOPatch()
eop.add_feature(FeatureType.DATA, names[0], bands)
eop.data[names[1]] = bands
eop[FeatureType.DATA][names[2]] = bands
for feature_name in names:
self.assertTrue(
feature_name in eop.data,
"Feature {} was not added to EOPatch".format(feature_name))
self.assertTrue(
np.array_equal(eop.data[feature_name], bands),
"Data of feature {} is "
"incorrect".format(feature_name))
eop.remove_feature(FeatureType.DATA, names[0])
del eop.data[names[1]]
del eop[FeatureType.DATA][names[2]]
for feature_name in names:
self.assertFalse(feature_name in eop.data,
msg="Feature {} should be deleted from "
"EOPatch".format(feature_name))
def test_add_ndvi(self):
a = np.arange(2 * 3 * 3 * 13).reshape(2, 3, 3, 13)
eop = EOPatch()
eop.add_feature(attr_type=FeatureType.DATA, field='bands', value=a)
eotask_ndvi = FeatureExtractionTask('I(B4, B8A)', 'bands', 'ndvi')
eop_ndvi = eotask_ndvi(eop)
in_shape = eop.data['bands'].shape
out_shape = in_shape[:-1] + (1, )
self.assertEqual(eop_ndvi.data['ndvi'].shape, out_shape)
def test_temporal_indices(self):
""" Test case for computation of argmax/argmin of NDVI and another band
Cases with and without data masking are tested
"""
# EOPatch
eopatch = EOPatch()
t, h, w, c = 5, 3, 3, 2
# NDVI
ndvi_shape = (t, h, w, 1)
# VAlid data mask
valid_data = np.ones(ndvi_shape, np.bool)
valid_data[0] = 0
valid_data[-1] = 0
# Fill in eopatch
eopatch.add_feature(FeatureType.DATA, 'NDVI',
np.arange(np.prod(ndvi_shape)).reshape(ndvi_shape))
eopatch.add_feature(FeatureType.MASK, 'IS_DATA',
np.ones(ndvi_shape, dtype=np.int16))
eopatch.add_feature(FeatureType.MASK, 'VALID_DATA', valid_data)
# Task
add_ndvi = AddMaxMinTemporalIndicesTask(mask_data=False)
# Run task
new_eopatch = add_ndvi(eopatch)
# Asserts
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['ARGMIN_NDVI'],
np.zeros((h, w, 1))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['ARGMAX_NDVI'],
(t - 1) * np.ones((h, w, 1))))
del add_ndvi, new_eopatch
# Repeat with valid dat amask
add_ndvi = AddMaxMinTemporalIndicesTask(mask_data=True)
new_eopatch = add_ndvi(eopatch)
# Asserts
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['ARGMIN_NDVI'],
np.ones((h, w, 1))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['ARGMAX_NDVI'],
(t - 2) * np.ones((h, w, 1))))
del add_ndvi, new_eopatch, valid_data
# BANDS
bands_shape = (t, h, w, c)
eopatch.add_feature(
FeatureType.DATA, 'BANDS',
np.arange(np.prod(bands_shape)).reshape(bands_shape))
add_bands = AddMaxMinTemporalIndicesTask(data_feature='BANDS',
data_index=1,
amax_data_feature='ARGMAX_B1',
amin_data_feature='ARGMIN_B1',
mask_data=False)
new_eopatch = add_bands(eopatch)
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['ARGMIN_B1'],
np.zeros((h, w, 1))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['ARGMAX_B1'],
(t - 1) * np.ones((h, w, 1))))
def tif_to_npz(files):
patch = EOPatch()
tasks = [
ImportFromTiff((FeatureType.DATA, f'LOADED_DATA'), path)
for path in files[::-1]
]
for task in tqdm(tasks):
task.execute(patch)
if 'bands' in patch[FeatureType.DATA]:
patch.add_feature(
FeatureType.DATA, 'bands',
EOPatch.concatenate_data(
patch[FeatureType.DATA][f'LOADED_DATA'],
patch[FeatureType.DATA]['bands'],
))
patch.remove_feature(FeatureType.DATA, f'LOADED_DATA')
else:
patch.rename_feature(FeatureType.DATA, f'LOADED_DATA', 'bands')
np.savez_compressed('data', data=patch[FeatureType.DATA]['bands'])
def test_stf_task(self):
""" Test case for computation of spatio-temporal features
The NDVI is a sinusoid over 0-pi over the temporal dimension, while bands is an array with values equal to
the temporal index
"""
# Timestamps
timestamp = perdelta(date(2018, 3, 1), date(2018, 3, 11),
timedelta(days=1))
# EOPatch
eopatch = EOPatch(timestamp=list(timestamp))
# Shape of arrays
t, h, w, c = 10, 3, 3, 2
# NDVI is a sinusoid where max slope is at index 1 and min slope at index 8
ndvi_shape = (t, h, w, 1)
bands_shape = (t, h, w, c)
xx = np.zeros(ndvi_shape, np.float32)
x = np.linspace(0, np.pi, t)
xx[:, :, :, :] = x[:, None, None, None]
# Bands are arrays with values equal to the temporal index
bands = np.ones(bands_shape) * np.arange(t)[:, None, None, None]
# Add features to eopatch
eopatch.add_feature(FeatureType.DATA, 'NDVI', np.sin(xx))
eopatch.add_feature(FeatureType.DATA, 'BANDS', bands)
eopatch.add_feature(FeatureType.MASK, 'IS_DATA',
np.ones(ndvi_shape, np.bool))
# Tasks
add_ndvi = AddMaxMinTemporalIndicesTask(mask_data=False)
add_bands = AddMaxMinTemporalIndicesTask(data_feature='BANDS',
data_index=1,
amax_data_feature='ARGMAX_B1',
amin_data_feature='ARGMIN_B1',
mask_data=False)
add_ndvi_slope = AddMaxMinNDVISlopeIndicesTask(mask_data=False)
add_stf = AddSpatioTemporalFeaturesTask(argmax_red='ARGMAX_B1',
data_feature='BANDS',
indices=[0, 1])
# Run tasks
new_eopatch = add_stf(add_ndvi_slope(add_bands(add_ndvi(eopatch))))
# Asserts
self.assertTrue(new_eopatch.data_timeless['STF'].shape == (h, w,
c * 5))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['STF'][:, :, 0:c],
4 * np.ones((h, w, c))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['STF'][:, :, c:2 * c],
9 * np.ones((h, w, c))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['STF'][:, :, 2 * c:3 * c],
np.ones((h, w, c))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['STF'][:, :, 3 * c:4 * c],
8 * np.ones((h, w, c))))
self.assertTrue(
np.array_equal(new_eopatch.data_timeless['STF'][:, :, 4 * c:5 * c],
9 * np.ones((h, w, c))))
def save_as_eopatch(data, label, field_id, dates, folder):
"""
Creates an EOPatch and adds data:
* S2 L2A bands will be stored in DATA feature under name `S2-BANDS-L2A`
* sen2cor cloud probabilities will be stored in DATA feature under name `CLOUD_PROB`
* labels will be stored in MASK_TIMELESS under name `CROP_ID`
* field ids will be stored in MASK_TIMELESS under name `FIELD_ID`
* dates of observations will be added to timestamps
EOPatch is saved to specified folder.
"""
eopatch = EOPatch()
eopatch.add_feature(FeatureType.DATA, 'S2-BANDS-L2A', data[..., :12])
eopatch.add_feature(FeatureType.DATA, 'CLOUD_PROB', data[..., -1][...,np.newaxis])
eopatch.add_feature(FeatureType.MASK_TIMELESS, 'CROP_ID', label[...,np.newaxis])
eopatch.add_feature(FeatureType.MASK_TIMELESS, 'FIELD_ID', field_id[...,np.newaxis])
eopatch.timestamp = dates
eopatch.save(folder)
return eopatch
def test_ndvi_slope_indices(self):
""" Test case for computation of argmax/argmin of NDVI slope
The NDVI is a sinusoid over 0-pi over the temporal dimension
Cases with and without data masking are tested
"""
# Slope needs timestamps
timestamp = perdelta(date(2018, 3, 1), date(2018, 3, 11), timedelta(days=1))
# EOPatch
eopatch = EOPatch(timestamp=list(timestamp))
t, h, w, = 10, 3, 3
# NDVI is a sinusoid where max slope is at index 1 and min slope at index 8
ndvi_shape = (t, h, w, 1)
xx = np.zeros(ndvi_shape, np.float32)
x = np.linspace(0, np.pi, t)
xx[:, :, :, :] = x[:, None, None, None]
# Valid data mask
valid_data = np.ones(ndvi_shape, np.uint8)
valid_data[1] = 0
valid_data[-1] = 0
valid_data[4] = 0
# Fill EOPatch
eopatch.add_feature(FeatureType.DATA, 'NDVI', np.sin(xx))
eopatch.add_feature(FeatureType.MASK, 'IS_DATA', np.ones(ndvi_shape, np.bool))
eopatch.add_feature(FeatureType.MASK, 'VALID_DATA', valid_data)
# Tasks
add_ndvi = AddMaxMinTemporalIndicesTask(mask_data=False)
add_ndvi_slope = AddMaxMinNDVISlopeIndicesTask(mask_data=False)
# Run
new_eopatch = add_ndvi_slope(add_ndvi(eopatch))
# Assert
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMIN_NDVI'], (t-1)*np.ones((h, w, 1))))
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMAX_NDVI'], (t//2-1)*np.ones((h, w, 1))))
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMIN_NDVI_SLOPE'], (t-2)*np.ones((h, w, 1))))
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMAX_NDVI_SLOPE'], np.ones((h, w, 1))))
del add_ndvi_slope, add_ndvi, new_eopatch
# Run on valid data only now
add_ndvi = AddMaxMinTemporalIndicesTask(mask_data=True)
add_ndvi_slope = AddMaxMinNDVISlopeIndicesTask(mask_data=True)
# Run
new_eopatch = add_ndvi_slope(add_ndvi(eopatch))
# Assert
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMIN_NDVI'], 0 * np.ones((h, w, 1))))
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMAX_NDVI'], (t // 2) * np.ones((h, w, 1))))
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMIN_NDVI_SLOPE'], (t - 3) * np.ones((h, w, 1))))
self.assertTrue(np.array_equal(new_eopatch.data_timeless['ARGMAX_NDVI_SLOPE'], 2 * np.ones((h, w, 1))))
def test_registration(self):
# Set up a dummy EOPatch to test execution of registration
bands = np.zeros((2, 20, 20, 1))
bands[1] = np.arange(400).reshape(1, 20, 20, 1) / 400
bands[0] = bands[1]
bands[1, 5:15, 5:15, :] = .5
bands[0, 7:17, 5:15, :] = .5
mask = np.ones((2, 20, 20, 1))
ndvi = np.ones((2, 20, 20, 1))
dem = np.ones((20, 20, 1))
eop = EOPatch()
eop.add_feature(attr_type=FeatureType.DATA, field='bands', value=bands)
eop.add_feature(attr_type=FeatureType.DATA, field='ndvi', value=ndvi)
eop.add_feature(attr_type=FeatureType.MASK, field='cm', value=mask)
eop.add_feature(attr_type=FeatureType.DATA_TIMELESS, field='dem', value=dem)
reg = ECCRegistration(FeatureType.DATA, 'bands', interpolation=Interpolation.NEAREST)
reop = reg.execute(eop)
self.assertEqual(eop.data['bands'].shape, reop.data['bands'].shape,
msg='Shapes of .data[''bands''] do not match')
self.assertEqual(eop.data['ndvi'].shape, reop.data['ndvi'].shape,
msg='Shapes of .data[''ndvi''] do not match')
self.assertEqual(eop.mask['cm'].shape, reop.mask['cm'].shape,
msg='Shapes of .mask[''cm''] do not match')
self.assertEqual(eop.data_timeless['dem'].shape, reop.data_timeless['dem'].shape,
msg='Shapes of .data[''bands''] do not match')
self.assertFalse(np.allclose(eop.data['bands'], reop.data['bands']),
msg='Registration did not warp .data[''bands'']')
self.assertFalse(np.allclose(eop.data['ndvi'], reop.data['ndvi']),
msg='Registration did not warp .data[''ndvi'']')
self.assertFalse(np.allclose(eop.mask['cm'], reop.mask['cm']),
msg='Registration did not warp .mask[''cm'']')
self.assertTrue(np.allclose(eop.data_timeless['dem'], reop.data_timeless['dem']),
msg='Registration did warp data_timeless')
def save_as_eopatch(data, label, field_id, dates, folder):
"""
Creates an EOPatch and adds data:
* S2 L2A bands will be stored in DATA feature under name `S2-BANDS-L2A`
* Individual bands will be stored in DATA features under their respective channel names
* sen2cor cloud probabilities will be stored in DATA feature under name `CLOUD_PROB`
* labels will be stored in MASK_TIMELESS under name `CROP_ID`
* field ids will be stored in MASK_TIMELESS under name `FIELD_ID`
* dates of observations will be added to timestamps
EOPatch is saved to specified folder.
"""
eopatch = EOPatch()
# Bands
eopatch.add_feature(FeatureType.DATA, 'B01', data[..., 0][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B02', data[..., 1][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B03', data[..., 2][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B04', data[..., 3][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B05', data[..., 4][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B06', data[..., 5][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B07', data[..., 6][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B08', data[..., 7][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B8A', data[..., 8][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B09', data[..., 9][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B11', data[..., 10][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'B12', data[..., 11][..., np.newaxis])
eopatch.add_feature(FeatureType.DATA, 'S2-BANDS-L2A', data[..., :12])
eopatch.add_feature(FeatureType.DATA, 'CLOUD_PROB', data[..., -1][..., np.newaxis])
eopatch.add_feature(FeatureType.MASK_TIMELESS, 'CROP_ID', label[..., np.newaxis])
eopatch.add_feature(FeatureType.MASK_TIMELESS, 'FIELD_ID', field_id[..., np.newaxis])
eopatch.timestamp = dates
eopatch.save(folder)
return eopatch