def test_fill(self):
array = np.array([[np.NaN] * 4 + [1., 2., 3., 4.] + [np.NaN] * 3,
[
1., np.NaN, np.NaN, 2., 3., np.NaN, np.NaN,
np.NaN, 4., np.NaN, 5.
]])
array_ffill = ValueFilloutTask.fill(array, operation='f')
array_bfill = ValueFilloutTask.fill(array, operation='b')
array_fbfill = ValueFilloutTask.fill(array_ffill, operation='b')
self.assertTrue(
np.array_equal(array_ffill,
np.array(
[[np.NaN] * 4 + [1., 2., 3., 4., 4., 4., 4.],
[1., 1., 1., 2., 3., 3., 3., 3., 4., 4., 5.]]),
equal_nan=True))
self.assertTrue(
np.array_equal(
array_bfill,
np.array([[1., 1., 1., 1., 1., 2., 3., 4.] + [np.NaN] * 3,
[1., 2., 2., 2., 3., 4., 4., 4., 4., 5., 5.]]),
equal_nan=True))
self.assertTrue(
np.array_equal(array_fbfill,
np.array(
[[1., 1., 1., 1., 1., 2., 3., 4., 4., 4., 4.],
[1., 1., 1., 2., 3., 3., 3., 3., 4., 4., 5.]]),
equal_nan=True))
def test_fill():
array = np.array([
[np.NaN] * 4 + [1.0, 2.0, 3.0, 4.0] + [np.NaN] * 3,
[
1.0, np.NaN, np.NaN, 2.0, 3.0, np.NaN, np.NaN, np.NaN, 4.0, np.NaN,
5.0
],
])
array_ffill = ValueFilloutTask.fill(array, operation="f")
array_bfill = ValueFilloutTask.fill(array, operation="b")
array_fbfill = ValueFilloutTask.fill(array_ffill, operation="b")
assert_allclose(
array_ffill,
np.array([
[np.NaN] * 4 + [1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 4.0],
[1.0, 1.0, 1.0, 2.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 5.0],
]),
equal_nan=True,
)
assert_allclose(
array_bfill,
np.array([
[1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 3.0, 4.0] + [np.NaN] * 3,
[1.0, 2.0, 2.0, 2.0, 3.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0],
]),
equal_nan=True,
)
assert_allclose(
array_fbfill,
np.array([
[1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 3.0, 4.0, 4.0, 4.0, 4.0],
[1.0, 1.0, 1.0, 2.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 5.0],
]),
equal_nan=True,
)
def execute_fillout(eopatch, feature, **kwargs):
input_array = eopatch[feature]
eopatch = ValueFilloutTask(feature, **kwargs)(eopatch)
output_array = eopatch[feature]
return eopatch, input_array, output_array
def test_value_fillout(self):
feature = (FeatureType.DATA, 'TEST')
shape = (8, 10, 10, 5)
data = np.random.randint(0, 100, size=shape).astype(np.float)
eopatch = EOPatch(data={'TEST': data})
self.assertRaises(ValueError,
ValueFilloutTask,
feature,
operations='x')
self.assertRaises(ValueError, ValueFilloutTask, feature, operations=4)
self.assertRaises(ValueError,
ValueFilloutTask.fill,
None,
operation='f')
self.assertRaises(ValueError,
ValueFilloutTask.fill,
np.zeros((4, 5)),
operation='x')
# nothing to be filled, return the same eopatch object immediately
eopatch_new = ValueFilloutTask(feature, operations='fb',
axis=0)(eopatch)
self.assertEqual(eopatch, eopatch_new)
eopatch[feature][0, 0, 0, :] = np.nan
def execute_fillout(eopatch, feature, **kwargs):
input_array = eopatch[feature]
eopatch = ValueFilloutTask(feature, **kwargs)(eopatch)
output_array = eopatch[feature]
return eopatch, input_array, output_array
# filling forward temporally should not fill nans
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations='f',
axis=0)
compare_mask = ~np.isnan(input_array)
self.assertTrue(np.isnan(output_array[0, 0, 0, :]).all())
self.assertTrue(
np.array_equal(input_array[compare_mask],
output_array[compare_mask]))
self.assertNotEqual(id(input_array), id(output_array))
# filling in any direction along axis=-1 should also not fill nans since all neighbors are nans
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations='fb',
axis=-1)
self.assertTrue(np.isnan(output_array[0, 0, 0, :]).all())
self.assertNotEqual(id(input_array), id(output_array))
# filling nans backwards temporally should fill nans
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations='b',
axis=0)
self.assertFalse(np.isnan(output_array).any())
self.assertNotEqual(id(input_array), id(output_array))
# try filling something else than nan (e.g.: -1)
eopatch[feature][0, :, 0, 0] = -1
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations='b',
value=-1,
axis=-1)
self.assertFalse((output_array == -1).any())
self.assertNotEqual(id(input_array), id(output_array))
# [nan, 1, nan, 2, ... ] ---('fb')---> [1, 1, 1, 2, ... ]
eopatch[feature][0, 0, 0, 0:4] = [np.nan, 1, np.nan, 2]
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations='fb',
axis=-1)
self.assertTrue(
np.array_equal(output_array[0, 0, 0, 0:4], [1, 1, 1, 2]))
self.assertNotEqual(id(input_array), id(output_array))
# [nan, 1, nan, 2, ... ] ---('bf')---> [1, 1, 2, 2, ... ]
eopatch[feature][0, 0, 0, 0:4] = [np.nan, 1, np.nan, 2]
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations='bf',
axis=-1)
self.assertTrue(
np.array_equal(output_array[0, 0, 0, 0:4], [1, 1, 2, 2]))
self.assertNotEqual(id(input_array), id(output_array))
(FeatureType.DATA, 'MSR'),
modified_simple_ratio)
nonlinear_vegetation_index_task = ZipFeatureTask({FeatureType.DATA: ['B08', 'B04']},
(FeatureType.DATA, 'NVI'),
nonlinear_vegetation_index)
renormalized_difference_vegetation_index_task = ZipFeatureTask({FeatureType.DATA: ['B08', 'B04']},
(FeatureType.DATA, 'RDVI'),
renormalized_difference_vegetation_index)
# sample only fields
sample = SampleValid((FeatureType.MASK_TIMELESS, 'FIELD_ID'), fraction=1.0, no_data_value=0)
# if invalid data are at beginning or end of time-series, pad with nearest value
fillout = ValueFilloutTask((FeatureType.DATA, 'S2-BANDS-L2A'))
# Create workflow and preprocessing
workflow = ComposeTask([cloud_mask,
cloud_dilation,
add_cc,
count_train_fields,
count_test_fields])
preprocess = ComposeTask([valid_data,
ndvi_task,
false_color_infrared_task,
corrected_transform_vegetation_index_task,
global_environment_monitoring_index_task,
infrared_agriculture_display_task,
barren_soil_index_task,
def test_bad_input(input_feature, operation):
with pytest.raises(ValueError):
ValueFilloutTask(input_feature, operations=operation)
def test_value_fillout():
feature = (FeatureType.DATA, "TEST")
shape = (8, 10, 10, 5)
data = np.random.randint(0, 100, size=shape).astype(float)
eopatch = EOPatch(data={"TEST": data})
# nothing to be filled, return the same eopatch object immediately
eopatch_new = ValueFilloutTask(feature, operations="fb", axis=0)(eopatch)
assert eopatch == eopatch_new
eopatch[feature][0, 0, 0, :] = np.nan
def execute_fillout(eopatch, feature, **kwargs):
input_array = eopatch[feature]
eopatch = ValueFilloutTask(feature, **kwargs)(eopatch)
output_array = eopatch[feature]
return eopatch, input_array, output_array
# filling forward temporally should not fill nans
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations="f",
axis=0)
compare_mask = ~np.isnan(input_array)
assert np.isnan(output_array[0, 0, 0, :]).all()
assert_array_equal(input_array[compare_mask], output_array[compare_mask])
assert id(input_array) != id(output_array)
# filling in any direction along axis=-1 should also not fill nans since all neighbors are nans
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations="fb",
axis=-1)
assert np.isnan(output_array[0, 0, 0, :]).all()
assert id(input_array) != id(output_array)
# filling nans backwards temporally should fill nans
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations="b",
axis=0)
assert not np.isnan(output_array).any()
assert id(input_array) != id(output_array)
# try filling something else than nan (e.g.: -1)
eopatch[feature][0, :, 0, 0] = -1
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations="b",
value=-1,
axis=-1)
assert not (output_array == -1).any()
assert id(input_array) != id(output_array)
# [nan, 1, nan, 2, ... ] ---('fb')---> [1, 1, 1, 2, ... ]
eopatch[feature][0, 0, 0, 0:4] = [np.nan, 1, np.nan, 2]
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations="fb",
axis=-1)
assert_array_equal(output_array[0, 0, 0, 0:4], [1, 1, 1, 2])
assert id(input_array) != id(output_array)
# [nan, 1, nan, 2, ... ] ---('bf')---> [1, 1, 2, 2, ... ]
eopatch[feature][0, 0, 0, 0:4] = [np.nan, 1, np.nan, 2]
eopatch, input_array, output_array = execute_fillout(eopatch,
feature,
operations="bf",
axis=-1)
assert_array_equal(output_array[0, 0, 0, 0:4], [1, 1, 2, 2])
assert id(input_array) != id(output_array)