def test_export_import(self):
for test_case in self.test_cases:
with self.subTest(msg='Test case {}'.format(test_case.name)):
self.eopatch[test_case.feature_type][test_case.name] = test_case.data
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
feature = test_case.feature_type, test_case.name
export_task = ExportToTiff(feature, folder=tmp_dir_name,
band_indices=test_case.bands, date_indices=test_case.times)
export_task.execute(self.eopatch, filename=tmp_file_name)
import_task = ImportFromTiff(feature, folder=tmp_dir_name,
timestamp_size=test_case.get_expected_timestamp_size())
expected_raster = test_case.get_expected()
new_eop = import_task.execute(filename=tmp_file_name)
old_eop = import_task.execute(self.eopatch, filename=tmp_file_name)
self.assertTrue(np.array_equal(expected_raster, new_eop[test_case.feature_type][test_case.name]),
msg='Tiff imported into new EOPatch is not the same as expected')
self.assertTrue(np.array_equal(expected_raster, old_eop[test_case.feature_type][test_case.name]),
msg='Tiff imported into old EOPatch is not the same as expected')
self.assertEqual(expected_raster.dtype, new_eop[test_case.feature_type][test_case.name].dtype,
msg='Tiff imported into new EOPatch has different dtype as expected')
def test_export2tiff_data_band_tuple_time_tuple(self):
data = np.arange(10 * 3 * 2 * 6, dtype=float).reshape(10, 3, 2, 6)
bands = (1, 4)
times = (2, 8)
bands_selection = np.arange(bands[0], bands[1] + 1)
times_selection = np.arange(times[0], times[1] + 1)
subset = data[times_selection][..., bands_selection].squeeze()
eop = EOPatch.load(self.PATCH_FILENAME)
eop.data['data'] = data
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.DATA, 'data'),
folder=tmp_dir_name,
band_indices=bands,
date_indices=times,
image_dtype=data.dtype)
task.execute(eop, filename=tmp_file_name)
# split times and bands in raster and mimic the initial shape
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
raster = raster.reshape(raster.shape[0], raster.shape[1],
len(times_selection), len(bands_selection))
raster = np.moveaxis(raster, -2, 0)
self.assertTrue(np.all(subset == raster))
def save_to_tiff(self, file_path, feature=None, no_data_value=None, merge_method="last", padding=0):
"""
Save indexed EOPatches to a complete tiff.
:param feature: Feature which will be exported
:type feature: (FeatureType, str)
:param file_path: path to save tiff
:type file_path: str
:param no_data_value: Value of pixels of tiff image with no data in EOPatch
:type no_data_value: int or float
:param merge_method: How to merge overlap EOPatches. "last" mean latter array overwrite former array,
"first" mean former array overwrite latter array.
:type merge_method: str
"""
if not feature:
feature = self.feature
if not self._is_loaded():
self._load_with_index(feature=feature)
union_patch = self._patch_joint(self.patch_index, feature=feature, merge_method=merge_method, padding=padding)
self._assure_folder_exist(path=file_path, path_type="file")
temp_file = tempfile.mktemp(suffix=".tiff")
try:
export_tiff = ExportToTiff(feature, no_data_value=no_data_value)
export_tiff.execute(union_patch, filename=temp_file)
self._cog_translate(src_path=temp_file, dst_path=file_path)
except Exception as e:
raise PatchSetError(e.__str__())
finally:
if os.path.exists(temp_file):
os.remove(temp_file)
def test_export2tiff_order(self):
data = np.arange(10 * 3 * 2 * 6, dtype=float).reshape(10, 3, 2, 6)
bands = [2, 3, 0]
times = [1, 7]
# create ordered subset
ordered_subset = []
for t in times:
for b in bands:
ordered_subset.append(data[t][..., b])
ordered_subset = np.array(ordered_subset)
ordered_subset = np.moveaxis(ordered_subset, 0, -1)
eop = EOPatch.load(self.PATCH_FILENAME)
eop.data['data'] = data
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.DATA, 'data'),
folder=tmp_dir_name,
band_indices=bands,
date_indices=times,
image_dtype=data.dtype)
task.execute(eop, filename=tmp_file_name)
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
self.assertTrue(np.all(ordered_subset == raster))
def test_time_dependent_feature(self):
feature = FeatureType.DATA, 'NDVI'
filename_export = 'relative-path/*.tiff'
filename_import = [
f'relative-path/{timestamp.strftime("%Y%m%dT%H%M%S")}.tiff'
for timestamp in self.eopatch.timestamp
]
export_task = ExportToTiff(feature, folder=self.path)
import_task = ImportFromTiff(feature,
folder=self.path,
timestamp_size=68)
export_task.execute(self.eopatch, filename=filename_export)
new_eopatch = import_task.execute(filename=filename_import)
self.assertTrue(
np.array_equal(new_eopatch[feature], self.eopatch[feature]))
self.eopatch.timestamp[-1] = datetime.datetime(2020, 10, 10)
filename_import = [
f'relative-path/{timestamp.strftime("%Y%m%dT%H%M%S")}.tiff'
for timestamp in self.eopatch.timestamp
]
with self.assertRaises(ResourceNotFound):
import_task.execute(filename=filename_import)
def test_export2tiff_mask_tuple_string(self):
eop = EOPatch.load(self.PATCH_FILENAME)
dates = np.array(eop.timestamp)
mask = np.arange(len(dates) * 3 * 2 * 1).reshape(len(dates), 3, 2, 1)
eop.mask['mask'] = mask
indices = [2, 4]
# day time gets floored
times = (datetime_to_iso(dates[indices[0]]),
datetime_to_iso(dates[indices[1]]))
selection = np.nonzero(
np.where((dates >= iso_to_datetime(times[0])) &
(dates <= iso_to_datetime(times[1])), dates, 0))
subset = mask[selection].squeeze()
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.MASK, 'mask'),
folder=tmp_dir_name,
date_indices=times)
task.execute(eop, filename=tmp_file_name)
# rasterio saves `bands` to the last dimension, move it up front
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
raster = np.moveaxis(raster, -1, 0)
self.assertTrue(np.all(subset == raster))
def test_timeless_feature(self):
feature = FeatureType.DATA_TIMELESS, 'DEM'
filename = 'relative-path/my-filename.tiff'
export_task = ExportToTiff(feature, folder=self.path)
import_task = ImportFromTiff(feature, folder=self.path)
export_task.execute(self.eopatch, filename=filename)
new_eopatch = import_task.execute(self.eopatch, filename=filename)
self.assertTrue(
np.array_equal(new_eopatch[feature], self.eopatch[feature]))
def test_time_dependent_feature_with_timestamps(self):
feature = FeatureType.DATA, 'NDVI'
filename = 'relative-path/%Y%m%dT%H%M%S.tiff'
export_task = ExportToTiff(feature, folder=self.path)
import_task = ImportFromTiff(feature, folder=self.path)
export_task.execute(self.eopatch, filename=filename)
new_eopatch = import_task.execute(self.eopatch, filename=filename)
self.assertTrue(
np.array_equal(new_eopatch[feature], self.eopatch[feature]))
def test_export2tiff_wrong_format(self):
data = np.arange(10*3*2*6, dtype=float).reshape(10, 3, 2, 6)
self.eopatch.data['data'] = data
for bands, times in [([2, 'string', 1, 0], [1, 7, 0, 2, 3]),
([2, 3, 1, 0], [1, 7, 'string', 2, 3])]:
with tempfile.TemporaryDirectory() as tmp_dir_name, self.assertRaises(ValueError):
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.DATA, 'data'), folder=tmp_dir_name,
band_indices=bands, date_indices=times, image_dtype=data.dtype)
task.execute(self.eopatch, filename=tmp_file_name)
def test_export2tiff_mask_timeless(self):
mask_timeless = np.arange(3 * 3 * 1).reshape(3, 3, 1)
subset = mask_timeless.squeeze()
eop = EOPatch.load(self.PATCH_FILENAME)
eop.mask_timeless['mask_timeless'] = mask_timeless
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.MASK_TIMELESS, 'mask_timeless'),
folder=tmp_dir_name)
task.execute(eop, filename=tmp_file_name)
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
self.assertTrue(np.all(subset == raster))
def test_export2tiff_mask_single(self):
mask = np.arange(5 * 3 * 3 * 1).reshape(5, 3, 3, 1)
times = [4]
subset = mask[times].squeeze()
eop = EOPatch.load(self.PATCH_FILENAME)
eop.mask['mask'] = mask
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.MASK, 'mask'),
folder=tmp_dir_name,
date_indices=times)
task.execute(eop, filename=tmp_file_name)
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
self.assertTrue(np.all(subset == raster))
def test_export2tiff_wrong_dates_format(self):
data = np.arange(10 * 3 * 2 * 6, dtype=float).reshape(10, 3, 2, 6)
bands = [2, 3, 1, 0]
times = [1, 7, 'string', 2, 3]
eop = EOPatch.load(self.PATCH_FILENAME)
eop.data['data'] = data
with tempfile.TemporaryDirectory() as tmp_dir_name, self.assertRaises(
ValueError):
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.DATA, 'data'),
folder=tmp_dir_name,
band_indices=bands,
date_indices=times,
image_dtype=data.dtype)
task.execute(eop, filename=tmp_file_name)
def test_export2tiff_scalar_timeless_list(self):
scalar_timeless = np.arange(5)
bands = [3, 0, 2]
subset = scalar_timeless[bands]
eop = EOPatch.load(self.PATCH_FILENAME)
eop.scalar_timeless['scalar_timeless'] = scalar_timeless
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff(
(FeatureType.SCALAR_TIMELESS, 'scalar_timeless'),
folder=tmp_dir_name,
band_indices=bands)
task.execute(eop, filename=tmp_file_name)
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
self.assertTrue(np.all(subset == raster))
def test_export2tiff_data_timeless_band_list(self):
data_timeless = np.arange(3 * 2 * 5, dtype=float).reshape(3, 2, 5)
bands = [2, 4, 1, 0]
subset = data_timeless[..., bands].squeeze()
eop = EOPatch.load(self.PATCH_FILENAME)
eop.data_timeless['data_timeless'] = data_timeless
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.DATA_TIMELESS, 'data_timeless'),
folder=tmp_dir_name,
band_indices=bands,
image_dtype=data_timeless.dtype)
task.execute(eop, filename=tmp_file_name)
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
self.assertTrue(np.all(subset == raster))
def test_export2tiff_mask_list(self):
mask = np.arange(5 * 3 * 2 * 1).reshape(5, 3, 2, 1)
times = [4, 2]
subset = mask[times].squeeze()
eop = EOPatch.load(self.PATCH_FILENAME)
eop.mask['mask'] = mask
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.MASK, 'mask'),
folder=tmp_dir_name,
date_indices=times)
task.execute(eop, filename=tmp_file_name)
# rasterio saves `bands` to the last dimension, move it up front
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
raster = np.moveaxis(raster, -1, 0)
self.assertTrue(np.all(subset == raster))
def test_export2tiff_wrong_feature(self, mocked_logger):
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
feature = FeatureType.MASK_TIMELESS, 'feature-not-present'
export_task = ExportToTiff(feature,
folder=tmp_dir_name,
fail_on_missing=False)
export_task.execute(self.eopatch, filename=tmp_file_name)
assert mocked_logger.call_count == 1
val_err_tup, _ = mocked_logger.call_args
val_err, = val_err_tup
assert str(val_err) == 'Feature feature-not-present of type FeatureType.MASK_TIMELESS ' \
'was not found in EOPatch'
with self.assertRaises(ValueError):
export_task_fail = ExportToTiff(feature,
folder=tmp_dir_name,
fail_on_missing=True)
export_task_fail.execute(self.eopatch, filename=tmp_file_name)
def test_export2tiff_scalar_band_single_time_single(self):
scalar = np.arange(10 * 6, dtype=float).reshape(10, 6)
bands = [3]
times = [7]
subset = scalar[times][..., bands].squeeze()
eop = EOPatch.load(self.PATCH_FILENAME)
eop.scalar['scalar'] = scalar
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file.tiff'
task = ExportToTiff((FeatureType.SCALAR, 'scalar'),
folder=tmp_dir_name,
band_indices=bands,
date_indices=times,
image_dtype=scalar.dtype)
task.execute(eop, filename=tmp_file_name)
raster = read_data(os.path.join(tmp_dir_name, tmp_file_name))
self.assertTrue(np.all(subset == raster))
def test_export2tiff_separate_timestamps(self):
test_case = self.test_cases[-1]
eopatch = copy.deepcopy(self.eopatch)
eopatch[test_case.feature_type][test_case.name] = test_case.data
eopatch.timestamp = self.eopatch.timestamp[:test_case.data.shape[0]]
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_file_name = 'temp_file_*'
tmp_file_name_reproject = 'temp_file_4326_%Y%m%d.tif'
feature = test_case.feature_type, test_case.name
export_task = ExportToTiff(feature,
band_indices=test_case.bands,
date_indices=test_case.times)
full_path = os.path.join(tmp_dir_name, tmp_file_name)
export_task.execute(eopatch, filename=full_path)
for timestamp in eopatch.timestamp:
expected_path = os.path.join(
tmp_dir_name,
timestamp.strftime('temp_file_%Y%m%dT%H%M%S.tif'))
self.assertTrue(os.path.exists(expected_path),
f'Path {expected_path} does not exist')
full_path = os.path.join(tmp_dir_name, tmp_file_name_reproject)
export_task = ExportToTiff(feature,
folder=full_path,
band_indices=test_case.bands,
date_indices=test_case.times,
crs='EPSG:4326',
compress='lzw')
export_task.execute(eopatch)
for timestamp in eopatch.timestamp:
expected_path = os.path.join(
tmp_dir_name, timestamp.strftime(tmp_file_name_reproject))
self.assertTrue(os.path.exists(expected_path),
f'Path {expected_path} does not exist')
}
})
executor = EOExecutor(workflow, execution_args, save_logs=True)
executor.run(workers=5, multiprocess=False)
# should install graphviz
# executor.make_report()
# Load GeogeniusEOPatch
eopatch = GeogeniusEOPatch.load(path=os.path.join(path_out,
'eopatch_{}'.format(0)),
lazy_loading=True)
print(eopatch)
# Print data
print(eopatch.get_feature(FeatureType.DATA, 'BANDS'))
# Convert all patches to tiff
tiff_out = get_current_folder("tiff")
if not os.path.isdir(tiff_out):
os.makedirs(tiff_out)
export_to_tiff = ExportToTiff(feature=(FeatureType.DATA, 'BANDS'),
folder=tiff_out)
for idx, bbox in enumerate(bbox_list[patchIDs]):
patch_patch = os.path.join(path_out, 'eopatch_{}'.format(idx))
sub_patch = GeogeniusEOPatch.load(path=os.path.join(
path_out, 'eopatch_{}'.format(idx)),
lazy_loading=True)
export_to_tiff.execute(eopatch=sub_patch,
filename='eopatch_{}.tiff'.format(idx))
execution_args.append({
add_data: {'pixelbox': bbox},
save: {'eopatch_folder': 'eopatch_{}'.format(idx)}
})
executor = EOExecutor(workflow, execution_args, save_logs=True)
executor.run(workers=1, multiprocess=False)
# Load GeogeniusEOPatch
print("First EOPatch")
eopatch = GeogeniusEOPatch.load(path=os.path.join(path_out, 'eopatch_{}'.format(0)), lazy_loading=True)
print(eopatch.get_feature(FeatureType.DATA, 'BANDS').shape)
print(eopatch.bbox)
print("Second EOPatch")
eopatch1 = GeogeniusEOPatch.load(path=os.path.join(path_out, 'eopatch_{}'.format(1)), lazy_loading=True)
print(eopatch1.get_feature(FeatureType.DATA, 'BANDS').shape)
print(eopatch1.bbox)
# Convert all patches to tiff
tiff_out = get_current_folder("pixel_range_tiff")
if not os.path.isdir(tiff_out):
os.makedirs(tiff_out)
export_to_tiff = ExportToTiff(feature=(FeatureType.DATA, 'BANDS'), folder=tiff_out)
for idx, bbox in enumerate(bbox_list):
info = info_list[idx]
patch_patch = os.path.join(path_out, 'eopatch_{}'.format(idx))
sub_patch = GeogeniusEOPatch.load(path=os.path.join(path_out, 'eopatch_{}'.format(idx)), lazy_loading=True)
export_to_tiff.execute(
eopatch=sub_patch, filename='eopatch_{}_{}.tiff'.format(info['index_x'], info['index_y']))