def prepare_mnt(self):
"""
Prepare the srtm files.
:return: Path to the full resolution DEM file.gsw
:rtype: str
"""
# Find/Download SRTM archives:
srtm_archives = self.get_raw_data()
# Unzip the downloaded/found srtm zip files:
unzipped = []
for arch in srtm_archives:
basename = os.path.splitext(os.path.basename(arch))[0]
FileSystem.unzip(arch, self.wdir)
fn_unzipped = FileSystem.find_single(pattern=basename + ".tif", path=self.wdir)
unzipped.append(fn_unzipped)
# Fusion of all SRTM files
concat = ImageTools.gdal_buildvrt(*unzipped, vrtnodata=-32768)
# Set nodata to 0
nodata = ImageTools.gdal_warp(concat,
srcnodata=-32768,
dstnodata=0,
multi=True)
# Combine to image of fixed extent
srtm_full_res = os.path.join(self.wdir, "srtm_%sm.tif" % int(self.site.res_x))
ImageTools.gdal_warp(nodata, dst=srtm_full_res,
r="cubic",
te=self.site.te_str,
t_srs=self.site.epsg_str,
tr=self.site.tr_str,
dstnodata=0,
srcnodata=0,
multi=True)
return srtm_full_res
def test_gdal_tile_untile(self):
img = np.arange(0., 100.).reshape((10, 10))
path = os.path.join(os.getcwd(), "test_gdal_retile.tif")
tile_folder = os.path.join(os.getcwd(), "tiled")
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
# Add parasitic file - It should not cause problems
path_parasite = os.path.join(tile_folder, "tile_01_01.tif")
FileSystem.create_directory(tile_folder)
ImageIO.write_geotiff(img, path_parasite, self.projection, self.coordinates)
ds_in = GDalDatasetWrapper.from_file(path)
self.assertTrue(os.path.exists(path))
tiles = ImageTools.gdal_retile(ds_in, tile_folder,
TileWidth=2,
TileHeight=2,
Overlap=1)
self.assertTrue(os.path.isdir(tile_folder))
self.assertEqual(len(tiles), 81)
img_read = np.array(ImageIO.tiff_to_array(tiles[-1]))
expected = np.array([[88, 89],
[98, 99]])
# Some gdal_retile versions are producing the following image:
# [[87, 89], [97, 99]].
np.testing.assert_allclose(expected, img_read, atol=1)
# Untile
ds_untiled = ImageTools.gdal_buildvrt(*tiles)
np.testing.assert_allclose(img, ds_untiled.array, atol=1)
FileSystem.remove_file(path)
FileSystem.remove_directory(tile_folder)
def _reproject_to_epsg(self, img, outpath, epsg):
tmpfile = tempfile.mktemp(prefix="reproject_", suffix=".tif")
ImageTools.gdal_warp(tmpfile,
img,
t_srs="EPSG:%s" % epsg,
tr=" ".join(map(str, self.base_resolution)),
q=True)
shutil.move(tmpfile, outpath)
def test_gdal_buildvrt_concatenate(self):
from StartMaja.Common import FileSystem
paths = []
for i in range(1, 3, 1):
img = np.ones((i, i, 2), np.int16) * i
path = os.path.join(os.getcwd(), "test_gdal_merge_%s.tif" % i)
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
self.assertTrue(os.path.exists(path))
paths.append(path)
empty = os.path.join(os.getcwd(), "empty.vrt")
driver = ImageTools.gdal_buildvrt(*paths, dst=empty,
separate=True,
srcnodata=0)
expected = np.array([[[1, 0],
[0, 0]],
[[2, 2],
[2, 2]]], dtype=np.int16)
np.testing.assert_almost_equal(driver.array, expected)
self.assertEqual(driver.nodata_value, 0)
self.assertEqual(driver.epsg, 32631)
[FileSystem.remove_file(path) for path in paths]
FileSystem.remove_file(empty)
[self.assertFalse(os.path.exists(path)) for path in paths]
self.assertFalse(os.path.exists(empty))
def test_merge_then_translate(self):
datasets = []
init = np.zeros((2, 2), np.int16)
path = os.path.join(os.getcwd(), "test_gdal_merge.tif")
ImageIO.write_geotiff(init, path, self.projection, self.coordinates)
ds_in = GDalDatasetWrapper.from_file(path)
for i in range(1, 3, 1):
img = np.ones((i*2, i*2), np.int16) * i
ds_n = GDalDatasetWrapper(ds=ds_in.get_ds(), array=img)
self.assertTrue(os.path.exists(path))
datasets.append(ds_n)
ds_merged = ImageTools.gdal_merge(*datasets, dst="out.tif",
separate=True,
q=True,
a_nodata=0)
# Array of shape (4, 4, 2):
expected = np.array([[[1, 2],
[1, 2],
[0, 2],
[0, 2]],
[[1, 2],
[1, 2],
[0, 2],
[0, 2]],
[[0, 2],
[0, 2],
[0, 2],
[0, 2]],
[[0, 2],
[0, 2],
[0, 2],
[0, 2]]], dtype=np.int16)
FileSystem.remove_file("out.tif")
np.testing.assert_equal(expected.dtype, ds_merged.array.dtype)
np.testing.assert_almost_equal(expected, ds_merged.array)
self.assertEqual(ds_merged.nodata_value, 0)
self.assertEqual(ds_merged.epsg, 32631)
ds_translate = ImageTools.gdal_translate(ds_merged,
a_nodata=0)
FileSystem.remove_file(path)
np.testing.assert_equal(expected.dtype, ds_translate.array.dtype)
np.testing.assert_almost_equal(ds_translate.array, expected)
self.assertEqual(ds_translate.nodata_value, 0)
self.assertEqual(ds_translate.epsg, 32631)
def test_extract_class_bit(self):
mask = np.arange(0, 9).reshape(3, 3)
class_to_extract = [3]
value_type = "bit"
expected = np.array([[0, 0, 0],
[0, 0, 0],
[0, 0, 1]])
calculated = ImageTools.extract_class(mask, class_to_extract, value_type)
np.testing.assert_array_almost_equal(expected, calculated)
def get_ndvi(red, nir, vrange=(-1, 1), dtype=np.float32):
"""
Calculate the NDVI (Normalized-Difference Vegetation Index)
:param red: The red band dataset
:type red: :class:`Common.GDalDatasetWrapper.GDalDatasetWrapper`
:param nir: The nir band dataset
:type nir: :class:`Common.GDalDatasetWrapper.GDalDatasetWrapper`
:param vrange: The range of output values as tuple. By default: (-1, 1).
:type vrange: tuple of int
:param dtype: The output dtype.
:type dtype: :class`np.dtype`
:return: The NDVI as numpy array.
:rtype: :class:`Common.GDalDatasetWrapper.GDalDatasetWrapper`
"""
if nir.extent != red.extent or nir.epsg != red.epsg:
raise ValueError("Cannot calculate NDSI on two different extents.")
if nir.resolution != red.resolution:
# Resize to nir resolution in this case.
tr = " ".join([str(i) for i in nir.resolution])
ds_red = ImageTools.gdal_translate(red, tr=tr, r="cubic")
else:
ds_red = red
# TODO Add new test
img_red = np.array(ds_red.array, dtype=np.float32)
img_nir = np.array(nir.array, dtype=np.float32)
# Compensate for nan:
np.seterr(divide='ignore', invalid='ignore')
img_ndvi = np.where((img_red + img_nir) != 0,
(img_red - img_nir) / (img_red + img_nir), -1)
# Scale to vrange
img_ndvi_scaled = ImageTools.normalize(img_ndvi,
value_range_out=vrange,
value_range_in=(-1, 1),
dtype=dtype,
clip=True)
return GDalDatasetWrapper(ds=nir.get_ds(), array=img_ndvi_scaled)
def test_get_nodata(self):
expected_nodata = 42.0
img = np.ones((self.height, self.width), np.int16)
path = os.path.join(os.getcwd(), "test_get_nodata_init.tif")
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
ds = ImageTools.gdal_buildvrt(path, VRTNodata=expected_nodata)
self.assertEqual(expected_nodata, ds.nodata_value)
np.testing.assert_almost_equal(ds.nodata_mask, np.ones_like(img))
FileSystem.remove_file(path)
self.assertFalse(os.path.exists(path))
def test_gdal_merge_optfile(self):
datasets, written = [], []
init = np.zeros((2, 2), np.int16)
path = os.path.join(os.getcwd(), "test_gdal_merge_optfile.tif")
ImageIO.write_geotiff(init, path, self.projection, self.coordinates)
ds_in = GDalDatasetWrapper.from_file(path)
for i in range(1, 3, 1):
img = np.ones((i*2, i*2), np.int16) * i
ds_n = GDalDatasetWrapper(ds=ds_in.get_ds(), array=img)
p_out = "test_gdal_merge_optfile_%s.tif" % i
ImageIO.write_geotiff_existing(img, p_out, ds_in.get_ds())
self.assertTrue(os.path.exists(path))
datasets.append(ds_n)
written.append(p_out)
optfile = "test_gdal_merge_optfile.txt"
with open(optfile, 'w') as file_handler:
for item in written:
file_handler.write("{}\n".format(item))
ds_merged = ImageTools.gdal_merge(*datasets,
q=True,
a_nodata=0)
ds_optfile = ImageTools.gdal_merge(optfile=optfile,
q=True,
a_nodata=0)
# Array of shape (4, 4):
expected = np.array([[2, 2, 2, 2],
[2, 2, 2, 2],
[2, 2, 2, 2],
[2, 2, 2, 2]], dtype=np.int16)
FileSystem.remove_file(path)
[FileSystem.remove_file(p) for p in written]
FileSystem.remove_file(optfile)
np.testing.assert_equal(expected.dtype, ds_merged.array.dtype)
np.testing.assert_almost_equal(expected, ds_merged.array)
self.assertEqual(ds_merged.nodata_value, 0)
self.assertEqual(ds_merged.epsg, 32631)
np.testing.assert_equal(expected.dtype, ds_optfile.array.dtype)
np.testing.assert_almost_equal(expected, ds_optfile.array)
self.assertEqual(ds_optfile.nodata_value, 0)
self.assertEqual(ds_optfile.epsg, 32631)
def test_normalize(self):
img = np.arange(0, 9).reshape(3, 3)
value_range_in = (0, 10)
value_range_out = (0, 1)
expected = np.array([[0, .1, .2],
[.3, .4, .5],
[.6, .7, .8]])
calculated = ImageTools.normalize(img,
value_range_in=value_range_in,
value_range_out=value_range_out)
np.testing.assert_array_almost_equal(expected, calculated)
def test_extract_class_threshold(self):
mask = np.arange(0, 25).reshape(5, 5)
class_to_extract = 10
value_type = "threshold"
expected = np.array([[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1]])
calculated = ImageTools.extract_class(mask, class_to_extract, value_type)
np.testing.assert_array_almost_equal(expected, calculated)
def test_normalize_0to255(self):
img = np.arange(-4, 5).reshape(3, 3) / 5
value_range_in = (-1, 1)
value_range_out = (0, 255)
expected = np.array([[25.5, 51., 76.5],
[102., 127.5, 153.],
[178.5, 204., 229.5]])
calculated = ImageTools.normalize(img,
value_range_in=value_range_in,
value_range_out=value_range_out)
np.testing.assert_array_almost_equal(expected, calculated)
def test_gdal_buildvrt(self):
path = os.path.join(os.getcwd(), "test_gdal_buildvrt.tif")
vrt = os.path.join(os.getcwd(), "test_vrt.vrt")
img = np.arange(-4, 5).reshape(3, 3) / 5
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
self.assertTrue(os.path.exists(path))
driver = ImageTools.gdal_buildvrt(path, dst=vrt)
self.assertTrue(os.path.exists(vrt))
np.testing.assert_almost_equal(driver.array, img)
FileSystem.remove_file(vrt)
FileSystem.remove_file(path)
def test_normalize_clip(self):
img = np.arange(0, 9).reshape(3, 3)
value_range_in = (0, 5)
value_range_out = (0, 1)
expected = np.array([[0, .2, .4],
[.6, .8, 1],
[1, 1, 1]])
calculated = ImageTools.normalize(img,
value_range_in=value_range_in,
value_range_out=value_range_out,
clip=True)
np.testing.assert_array_almost_equal(expected, calculated)
def prepare_mnt(self):
"""
Prepare the eudem files.
:return: Path to the full resolution DEM file.gsw
:rtype: str
"""
# Find/Download EuDEM archives:
eudem_files = self.get_raw_data()
# Unzip the downloaded/found EuDEM zip files:
unzipped = []
for arch in eudem_files:
basename = os.path.splitext(os.path.basename(arch))[0]
FileSystem.unzip(arch, self.wdir)
fn_unzipped = FileSystem.find_single(pattern=basename + ".TIF$",
path=self.wdir)
unzipped.append(fn_unzipped)
# Fusion of all EuDEM files
ds_cropped = []
for fn in unzipped:
ds = ImageTools.gdal_warp(fn,
of="GTiff",
ot="Int16",
r="cubic",
te=self.site.te_str,
t_srs=self.site.epsg_str,
tr=self.site.tr_str,
multi=True)
ds.array[ds.array < self.lowest_allowed_height] = -32767
ds_cropped.append(ds)
eudem_full_res = os.path.join(self.wdir,
"eudem_%sm.tif" % int(self.site.res_x))
ImageTools.gdal_merge(*ds_cropped,
dst=eudem_full_res,
n=-32767,
a_nodata=0,
q=True)
return eudem_full_res
def test_gdal_translate(self):
img = np.ones((self.height, self.width, 2), np.int16)
path = os.path.join(os.getcwd(), "test_gdal_translate.tif")
scaled = os.path.join(os.getcwd(), "scaled.tif")
out_resolution = (20, -20)
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
self.assertTrue(os.path.exists(path))
ds = ImageTools.gdal_translate(path, scaled,
tr=" ".join([str(i) for i in out_resolution]),
scale="0 1 0 255")
self.assertEqual(ds.resolution, out_resolution)
self.assertEqual(ds.array.shape, (self.height // 2, self.width // 2, 2))
np.testing.assert_almost_equal(ds.array, 255)
FileSystem.remove_file(path)
FileSystem.remove_file(scaled)
def get_synthetic_band(self, synthetic_band, **kwargs):
wdir = kwargs.get("wdir", self.fpath)
output_folder = os.path.join(wdir, self.base)
output_bname = "_".join(
[self.base.split(".")[0],
synthetic_band.upper() + ".tif"])
output_filename = kwargs.get("output_filename",
os.path.join(output_folder, output_bname))
max_value = kwargs.get("max_value", 10000.)
# Skip existing:
if os.path.exists(output_filename):
return output_filename
if synthetic_band.lower() == "ndvi":
FileSystem.create_directory(output_folder)
b4 = self.find_file(pattern=r"*B0?4(_10m)?.jp2$")[0]
b8 = self.find_file(pattern=r"*B0?8(_10m)?.jp2$")[0]
ds_red = GDalDatasetWrapper.from_file(b4)
ds_nir = GDalDatasetWrapper.from_file(b8)
ds_ndvi = ImageApps.get_ndvi(ds_red,
ds_nir,
vrange=(0, max_value),
dtype=np.int16)
ds_ndvi.write(output_filename, options=["COMPRESS=DEFLATE"])
elif synthetic_band.lower() == "ndsi":
FileSystem.create_directory(output_folder)
b3 = self.find_file(pattern=r"*B0?3(_10m)?.jp2$")[0]
b11 = self.find_file(pattern=r"*B11(_20m)?.jp2$")[0]
ds_green = ImageTools.gdal_translate(b3, tr="20 20", r="cubic")
ds_swir = GDalDatasetWrapper.from_file(b11)
ds_ndsi = ImageApps.get_ndsi(ds_green,
ds_swir,
vrange=(0, max_value),
dtype=np.int16)
ds_ndsi.write(output_filename, options=["COMPRESS=DEFLATE"])
elif synthetic_band.lower() == "mca_sim":
FileSystem.create_directory(output_folder)
b4 = self.find_file(pattern=r"*B0?4(_10m)?.jp2$")[0]
b3 = self.find_file(pattern=r"*B0?3(_10m)?.jp2$")[0]
img_red, drv = ImageIO.tiff_to_array(b4, array_only=False)
img_green = ImageIO.tiff_to_array(b3)
img_mcasim = (img_red + img_green) / 2
ImageIO.write_geotiff_existing(img_mcasim,
output_filename,
drv,
options=["COMPRESS=DEFLATE"])
else:
raise ValueError("Unknown synthetic band %s" % synthetic_band)
return output_filename
def test_gdal_warp(self):
img = np.ones((self.height, self.width, 2), np.int16)
img_rescaled = np.ones((int(self.height/2), int(self.width/2), 2), np.int16)
out_resolution = (20, -20)
path = os.path.join(os.getcwd(), "test_gdal_warp.tif")
scaled = os.path.join(os.getcwd(), "res_changed.tif")
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
self.assertTrue(os.path.exists(path))
ds = ImageTools.gdal_warp(path, scaled,
tr=" ".join(str(e) for e in out_resolution),
r="max", q=True)
self.assertTrue(os.path.isfile(scaled))
self.assertEqual(ds.resolution, out_resolution)
self.assertEqual(ds.array.shape, (self.height // 2, self.width // 2, 2))
np.testing.assert_almost_equal(ds.array, img_rescaled)
FileSystem.remove_file(path)
FileSystem.remove_file(scaled)