def test_gdal_tile_untile(self): img = np.arange(0., 100.).reshape((10, 10)) path = os.path.join(os.getcwd(), "test_gdal_retile.tif") tile_folder = os.path.join(os.getcwd(), "tiled") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) # Add parasitic file - It should not cause problems path_parasite = os.path.join(tile_folder, "tile_01_01.tif") FileSystem.create_directory(tile_folder) ImageIO.write_geotiff(img, path_parasite, self.projection, self.coordinates) ds_in = GDalDatasetWrapper.from_file(path) self.assertTrue(os.path.exists(path)) tiles = ImageTools.gdal_retile(ds_in, tile_folder, TileWidth=2, TileHeight=2, Overlap=1) self.assertTrue(os.path.isdir(tile_folder)) self.assertEqual(len(tiles), 81) img_read = np.array(ImageIO.tiff_to_array(tiles[-1])) expected = np.array([[88, 89], [98, 99]]) # Some gdal_retile versions are producing the following image: # [[87, 89], [97, 99]]. np.testing.assert_allclose(expected, img_read, atol=1) # Untile ds_untiled = ImageTools.gdal_buildvrt(*tiles) np.testing.assert_allclose(img, ds_untiled.array, atol=1) FileSystem.remove_file(path) FileSystem.remove_directory(tile_folder)
def test_gdal_buildvrt_concatenate(self): from Common import FileSystem paths = [] for i in range(1, 3, 1): img = np.ones((i, i, 2), np.int16) * i path = os.path.join(os.getcwd(), "test_gdal_merge_%s.tif" % i) ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) paths.append(path) empty = os.path.join(os.getcwd(), "empty.vrt") driver = ImageTools.gdal_buildvrt(*paths, dst=empty, separate=True, srcnodata=0) expected = np.array([[[1, 0], [0, 0]], [[2, 2], [2, 2]]], dtype=np.int16) np.testing.assert_almost_equal(driver.array, expected) self.assertEqual(driver.nodata_value, 0) self.assertEqual(driver.epsg, 32631) [FileSystem.remove_file(path) for path in paths] FileSystem.remove_file(empty) [self.assertFalse(os.path.exists(path)) for path in paths] self.assertFalse(os.path.exists(empty))
def test_merge_then_translate(self): datasets = [] init = np.zeros((2, 2), np.int16) path = os.path.join(os.getcwd(), "test_gdal_merge.tif") ImageIO.write_geotiff(init, path, self.projection, self.coordinates) ds_in = GDalDatasetWrapper.from_file(path) for i in range(1, 3, 1): img = np.ones((i*2, i*2), np.int16) * i ds_n = GDalDatasetWrapper(ds=ds_in.get_ds(), array=img) self.assertTrue(os.path.exists(path)) datasets.append(ds_n) ds_merged = ImageTools.gdal_merge(*datasets, dst="out.tif", separate=True, q=True, a_nodata=0) # Array of shape (4, 4, 2): expected = np.array([[[1, 2], [1, 2], [0, 2], [0, 2]], [[1, 2], [1, 2], [0, 2], [0, 2]], [[0, 2], [0, 2], [0, 2], [0, 2]], [[0, 2], [0, 2], [0, 2], [0, 2]]], dtype=np.int16) FileSystem.remove_file("out.tif") np.testing.assert_equal(expected.dtype, ds_merged.array.dtype) np.testing.assert_almost_equal(expected, ds_merged.array) self.assertEqual(ds_merged.nodata_value, 0) self.assertEqual(ds_merged.epsg, 32631) ds_translate = ImageTools.gdal_translate(ds_merged, a_nodata=0) FileSystem.remove_file(path) np.testing.assert_equal(expected.dtype, ds_translate.array.dtype) np.testing.assert_almost_equal(ds_translate.array, expected) self.assertEqual(ds_translate.nodata_value, 0) self.assertEqual(ds_translate.epsg, 32631)
def test_extract_class_bit(self): mask = np.arange(0, 9).reshape(3, 3) class_to_extract = [3] value_type = "bit" expected = np.array([[0, 0, 0], [0, 0, 0], [0, 0, 1]]) calculated = ImageTools.extract_class(mask, class_to_extract, value_type) np.testing.assert_array_almost_equal(expected, calculated)
def test_get_nodata(self): expected_nodata = 42.0 img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_nodata_init.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) ds = ImageTools.gdal_buildvrt(path, VRTNodata=expected_nodata) self.assertEqual(expected_nodata, ds.nodata_value) np.testing.assert_almost_equal(ds.nodata_mask, np.ones_like(img)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_gdal_merge_optfile(self): datasets, written = [], [] init = np.zeros((2, 2), np.int16) path = os.path.join(os.getcwd(), "test_gdal_merge_optfile.tif") ImageIO.write_geotiff(init, path, self.projection, self.coordinates) ds_in = GDalDatasetWrapper.from_file(path) for i in range(1, 3, 1): img = np.ones((i*2, i*2), np.int16) * i ds_n = GDalDatasetWrapper(ds=ds_in.get_ds(), array=img) p_out = "test_gdal_merge_optfile_%s.tif" % i ImageIO.write_geotiff_existing(img, p_out, ds_in.get_ds()) self.assertTrue(os.path.exists(path)) datasets.append(ds_n) written.append(p_out) optfile = "test_gdal_merge_optfile.txt" with open(optfile, 'w') as file_handler: for item in written: file_handler.write("{}\n".format(item)) ds_merged = ImageTools.gdal_merge(*datasets, q=True, a_nodata=0) ds_optfile = ImageTools.gdal_merge(optfile=optfile, q=True, a_nodata=0) # Array of shape (4, 4): expected = np.array([[2, 2, 2, 2], [2, 2, 2, 2], [2, 2, 2, 2], [2, 2, 2, 2]], dtype=np.int16) FileSystem.remove_file(path) [FileSystem.remove_file(p) for p in written] FileSystem.remove_file(optfile) np.testing.assert_equal(expected.dtype, ds_merged.array.dtype) np.testing.assert_almost_equal(expected, ds_merged.array) self.assertEqual(ds_merged.nodata_value, 0) self.assertEqual(ds_merged.epsg, 32631) np.testing.assert_equal(expected.dtype, ds_optfile.array.dtype) np.testing.assert_almost_equal(expected, ds_optfile.array) self.assertEqual(ds_optfile.nodata_value, 0) self.assertEqual(ds_optfile.epsg, 32631)
def test_extract_class_threshold(self): mask = np.arange(0, 25).reshape(5, 5) class_to_extract = 10 value_type = "threshold" expected = np.array([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]]) calculated = ImageTools.extract_class(mask, class_to_extract, value_type) np.testing.assert_array_almost_equal(expected, calculated)
def test_normalize(self): img = np.arange(0, 9).reshape(3, 3) value_range_in = (0, 10) value_range_out = (0, 1) expected = np.array([[0, .1, .2], [.3, .4, .5], [.6, .7, .8]]) calculated = ImageTools.normalize(img, value_range_in=value_range_in, value_range_out=value_range_out) np.testing.assert_array_almost_equal(expected, calculated)
def test_normalize_0to255(self): img = np.arange(-4, 5).reshape(3, 3) / 5 value_range_in = (-1, 1) value_range_out = (0, 255) expected = np.array([[25.5, 51., 76.5], [102., 127.5, 153.], [178.5, 204., 229.5]]) calculated = ImageTools.normalize(img, value_range_in=value_range_in, value_range_out=value_range_out) np.testing.assert_array_almost_equal(expected, calculated)
def test_normalize_clip(self): img = np.arange(0, 9).reshape(3, 3) value_range_in = (0, 5) value_range_out = (0, 1) expected = np.array([[0, .2, .4], [.6, .8, 1], [1, 1, 1]]) calculated = ImageTools.normalize(img, value_range_in=value_range_in, value_range_out=value_range_out, clip=True) np.testing.assert_array_almost_equal(expected, calculated)
def test_gdal_buildvrt(self): path = os.path.join(os.getcwd(), "test_gdal_buildvrt.tif") vrt = os.path.join(os.getcwd(), "test_vrt.vrt") img = np.arange(-4, 5).reshape(3, 3) / 5 ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) driver = ImageTools.gdal_buildvrt(path, dst=vrt) self.assertTrue(os.path.exists(vrt)) np.testing.assert_almost_equal(driver.array, img) FileSystem.remove_file(vrt) FileSystem.remove_file(path)
def get_ndvi(red, nir, vrange=(-1, 1), dtype=np.float32): """ Calculate the NDVI (Normalized-Difference Vegetation Index) :param red: The red band dataset :type red: :class:`Common.GDalDatasetWrapper.GDalDatasetWrapper` :param nir: The nir band dataset :type nir: :class:`Common.GDalDatasetWrapper.GDalDatasetWrapper` :param vrange: The range of output values as tuple. By default: (-1, 1). :type vrange: tuple of int :param dtype: The output dtype. :type dtype: :class`np.dtype` :return: The NDVI as numpy array. :rtype: :class:`Common.GDalDatasetWrapper.GDalDatasetWrapper` """ if nir.extent != red.extent or nir.epsg != red.epsg: raise ValueError("Cannot calculate NDSI on two different extents.") if nir.resolution != red.resolution: # Resize to nir resolution in this case. tr = " ".join([str(i) for i in nir.resolution]) ds_red = ImageTools.gdal_translate(red, tr=tr, r="cubic") else: ds_red = red # TODO Add new test img_red = np.array(ds_red.array, dtype=np.float32) img_nir = np.array(nir.array, dtype=np.float32) # Compensate for nan: np.seterr(divide='ignore', invalid='ignore') img_ndvi = np.where((img_red + img_nir) != 0, (img_red - img_nir) / (img_red + img_nir), -1) # Scale to vrange img_ndvi_scaled = ImageTools.normalize(img_ndvi, value_range_out=vrange, value_range_in=(-1, 1), dtype=dtype, clip=True) return GDalDatasetWrapper(ds=nir.get_ds(), array=img_ndvi_scaled)
def prepare_mnt(self): """ Prepare the eudem files. :return: Path to the full resolution DEM file.gsw :rtype: str """ # Find/Download EuDEM archives: eudem_files = self.get_raw_data() # Unzip the downloaded/found EuDEM zip files: unzipped = [] for arch in eudem_files: basename = os.path.splitext(os.path.basename(arch))[0] FileSystem.unzip(arch, self.wdir) fn_unzipped = FileSystem.find_single(pattern=basename + ".TIF$", path=self.wdir) unzipped.append(fn_unzipped) # Fusion of all EuDEM files ds_cropped = [] for fn in unzipped: ds = ImageTools.gdal_warp(fn, of="GTiff", ot="Int16", r="cubic", te=self.site.te_str, t_srs=self.site.epsg_str, tr=self.site.tr_str, multi=True) ds.array[ds.array < self.lowest_allowed_height] = -32767 ds_cropped.append(ds) eudem_full_res = os.path.join(self.wdir, "eudem_%sm.tif" % int(self.site.res_x)) ImageTools.gdal_merge(*ds_cropped, dst=eudem_full_res, n=-32767, a_nodata=0, q=True) return eudem_full_res
def prepare_water_data(self): """ Prepare the water mask constituing of a set of gsw files. :return: Writes the tiles water_mask to the self.gsw_dst path. """ occ_files = self.get_raw_water_data() vrt_path = os.path.join(self.wdir, "vrt_%s.vrt" % self.site.nom) ImageTools.gdal_buildvrt(*occ_files, dst=vrt_path) # Overlay occurrence image with same extent as the given site. # Should the occurrence files not be complete, this sets all areas not covered by the occurrence to 0. ds_warped = ImageTools.gdal_warp(vrt_path, r="near", te=self.site.te_str, t_srs=self.site.epsg_str, tr=self.site.tr_str, dstnodata=0, multi=True) # Threshold the final image and write to destination: image_bin = ds_warped.array > self.gsw_threshold FileSystem.remove_file(vrt_path) ImageIO.write_geotiff_existing(image_bin, self.gsw_dst, ds_warped.get_ds())
def prepare_mnt(self): """ Prepare the srtm files. :return: Path to the full resolution DEM file.gsw :rtype: str """ # Find/Download SRTM archives: srtm_archives = self.get_raw_data() # Unzip the downloaded/found srtm zip files: unzipped = [] for arch in srtm_archives: basename = os.path.splitext(os.path.basename(arch))[0] FileSystem.unzip(arch, self.wdir) fn_unzipped = FileSystem.find_single(pattern=basename + ".tif", path=self.wdir) unzipped.append(fn_unzipped) # Fusion of all SRTM files concat = ImageTools.gdal_buildvrt(*unzipped, vrtnodata=-32768) # Set nodata to 0 nodata = ImageTools.gdal_warp(concat, srcnodata=-32768, dstnodata=0, multi=True) # Combine to image of fixed extent srtm_full_res = os.path.join(self.wdir, "srtm_%sm.tif" % int(self.site.res_x)) ImageTools.gdal_warp(nodata, dst=srtm_full_res, r="cubic", te=self.site.te_str, t_srs=self.site.epsg_str, tr=self.site.tr_str, dstnodata=0, srcnodata=0, multi=True) return srtm_full_res
def test_gdal_translate(self): img = np.ones((self.height, self.width, 2), np.int16) path = os.path.join(os.getcwd(), "test_gdal_translate.tif") scaled = os.path.join(os.getcwd(), "scaled.tif") out_resolution = (20, -20) ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = ImageTools.gdal_translate(path, scaled, tr=" ".join([str(i) for i in out_resolution]), scale="0 1 0 255") self.assertEqual(ds.resolution, out_resolution) self.assertEqual(ds.array.shape, (self.height // 2, self.width // 2, 2)) np.testing.assert_almost_equal(ds.array, 255) FileSystem.remove_file(path) FileSystem.remove_file(scaled)
def test_gdal_warp(self): img = np.ones((self.height, self.width, 2), np.int16) img_rescaled = np.ones((int(self.height/2), int(self.width/2), 2), np.int16) out_resolution = (20, -20) path = os.path.join(os.getcwd(), "test_gdal_warp.tif") scaled = os.path.join(os.getcwd(), "res_changed.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = ImageTools.gdal_warp(path, scaled, tr=" ".join(str(e) for e in out_resolution), r="max", q=True) self.assertTrue(os.path.isfile(scaled)) self.assertEqual(ds.resolution, out_resolution) self.assertEqual(ds.array.shape, (self.height // 2, self.width // 2, 2)) np.testing.assert_almost_equal(ds.array, img_rescaled) FileSystem.remove_file(path) FileSystem.remove_file(scaled)
def get_synthetic_band(self, synthetic_band, **kwargs): wdir = kwargs.get("wdir", self.fpath) output_folder = os.path.join(wdir, self.base) output_bname = "_".join([self.base.split(".")[0], synthetic_band.upper() + ".tif"]) output_filename = kwargs.get("output_filename", os.path.join(output_folder, output_bname)) max_value = kwargs.get("max_value", 10000.) # Skip existing: if os.path.exists(output_filename): return output_filename if synthetic_band.lower() == "ndvi": FileSystem.create_directory(output_folder) b4 = self.find_file(pattern=r"*B0?4(_10m)?.jp2$")[0] b8 = self.find_file(pattern=r"*B0?8(_10m)?.jp2$")[0] ds_red = GDalDatasetWrapper.from_file(b4) ds_nir = GDalDatasetWrapper.from_file(b8) ds_ndvi = ImageApps.get_ndvi(ds_red, ds_nir, vrange=(0, max_value), dtype=np.int16) ds_ndvi.write(output_filename, options=["COMPRESS=DEFLATE"]) elif synthetic_band.lower() == "ndsi": FileSystem.create_directory(output_folder) b3 = self.find_file(pattern=r"*B0?3(_10m)?.jp2$")[0] b11 = self.find_file(pattern=r"*B11(_20m)?.jp2$")[0] ds_green = ImageTools.gdal_translate(b3, tr="20 20", r="cubic") ds_swir = GDalDatasetWrapper.from_file(b11) ds_ndsi = ImageApps.get_ndsi(ds_green, ds_swir, vrange=(0, max_value), dtype=np.int16) ds_ndsi.write(output_filename, options=["COMPRESS=DEFLATE"]) elif synthetic_band.lower() == "mca_sim": FileSystem.create_directory(output_folder) b4 = self.find_file(pattern=r"*B0?4(_10m)?.jp2$")[0] b3 = self.find_file(pattern=r"*B0?3(_10m)?.jp2$")[0] img_red, drv = ImageIO.tiff_to_array(b4, array_only=False) img_green = ImageIO.tiff_to_array(b3) img_mcasim = (img_red + img_green) / 2 ImageIO.write_geotiff_existing(img_mcasim, output_filename, drv, options=["COMPRESS=DEFLATE"]) else: raise ValueError("Unknown synthetic band %s" % synthetic_band) return output_filename
def _reproject_to_epsg(self, img, outpath, epsg): tmpfile = tempfile.mktemp(prefix="reproject_", suffix=".tif") ImageTools.gdal_warp(tmpfile, img, t_srs="EPSG:%s" % epsg, tr=" ".join(map(str, self.base_resolution)), q=True) shutil.move(tmpfile, outpath)
def to_maja_format(self, platform_id, mission_field, mnt_resolutions, coarse_res, full_res_only=False): """ Writes an MNT in Maja (=EarthExplorer) format: A folder .DBL.DIR containing the rasters and an accompanying .HDR xml-file. The two files follow the maja syntax:: *AUX_REFDE2*.(HDR|DBL.DIR) :param platform_id: The platform ID of two digits (e.g. S2_ for Sentinel2A/B; VS for Venus) :param mission_field: Similar to the platform ID, this is used in the <Mission>-field for the HDR file. e.g. SENTINEL-2 for S2 :param mnt_resolutions: A dict containing the resolutions for the given sensor. E.g.:: {"XS": (10, -10)} :param coarse_res: A tuple of int describing the coarse resolution. E.g.:: (240, -240). :param full_res_only: If True, no coarse_res rasters will be created. :return: Writes the .DBL.DIR and .HDR into the specified self.dem_dir """ assert len(mnt_resolutions) >= 1 basename = str( "%s_TEST_AUX_REFDE2_%s_%s" % (platform_id, self.site.nom, str(self.dem_version).zfill(4))) # Get mnt data mnt_max_res = self.prepare_mnt() # Water mask not needed with optional coarse_res writing: if coarse_res and not full_res_only: # Get water data self.prepare_water_data() mnt_res = (self.site.res_x, self.site.res_y) dbl_base = basename + ".DBL.DIR" dbl_dir = os.path.join(self.dem_dir, dbl_base) FileSystem.create_directory(dbl_dir) hdr = os.path.join(self.dem_dir, basename + ".HDR") # Calulate gradient mask at MNT resolution: mnt_in, drv = ImageIO.tiff_to_array(mnt_max_res, array_only=False) grad_y_mnt, grad_x_mnt = self.calc_gradient(mnt_in, self.site.res_x, self.site.res_y) full_res = (int(mnt_resolutions[0]["val"].split(" ")[0]), int(mnt_resolutions[0]["val"].split(" ")[1])) grad_x = self.resample_to_full_resolution(grad_x_mnt, mnt_resolution=mnt_res, full_resolution=full_res, order=3) grad_y = self.resample_to_full_resolution(grad_y_mnt, mnt_resolution=mnt_res, full_resolution=full_res, order=3) slope, aspect = self.calc_slope_aspect(grad_y, grad_x) # Write full res slope and aspect geotransform = list(drv.GetGeoTransform()) geotransform[1] = float(full_res[0]) geotransform[-1] = float(full_res[1]) projection = drv.GetProjection() tmp_asp = tempfile.mktemp(dir=self.wdir, suffix="_asp.tif") ImageIO.write_geotiff(aspect, tmp_asp, projection, tuple(geotransform)) tmp_slp = tempfile.mktemp(dir=self.wdir, suffix="_slp.tif") ImageIO.write_geotiff(slope, tmp_slp, projection, tuple(geotransform)) # Full resolution: write_resolution_name = True if len(mnt_resolutions) > 1 else False # Names for R1, R2 etc. rasters_written = [] path_alt, path_asp, path_slp = "", "", "" all_paths_alt = [] for res in mnt_resolutions: # ALT: bname_alt = basename + "_ALT" bname_alt += "_" + str( res["name"]) if write_resolution_name else "" bname_alt += ".TIF" rel_alt = os.path.join(dbl_base, bname_alt) path_alt = os.path.join(self.dem_dir, rel_alt) all_paths_alt.append(path_alt) ImageTools.gdal_warp(mnt_max_res, dst=path_alt, tr=res["val"], r="cubic", multi=True) rasters_written.append(rel_alt) # ASP: bname_asp = basename + "_ASP" bname_asp += "_" + res["name"] if write_resolution_name else "" bname_asp += ".TIF" rel_asp = os.path.join(dbl_base, bname_asp) path_asp = os.path.join(self.dem_dir, rel_asp) ImageTools.gdal_warp(tmp_asp, dst=path_asp, tr=res["val"], r="cubic", multi=True) rasters_written.append(rel_asp) # SLP: bname_slp = basename + "_SLP" bname_slp += "_" + res["name"] if write_resolution_name else "" bname_slp += ".TIF" rel_slp = os.path.join(dbl_base, bname_slp) path_slp = os.path.join(self.dem_dir, rel_slp) ImageTools.gdal_warp(tmp_slp, dst=path_slp, tr=res["val"], r="cubic", multi=True) rasters_written.append(rel_slp) # Optional coarse_res writing: if coarse_res and not full_res_only: # Resize all rasters for coarse res. coarse_res_str = str(coarse_res[0]) + " " + str(coarse_res[1]) # ALC: bname_alc = basename + "_ALC.TIF" rel_alc = os.path.join(dbl_base, bname_alc) path_alc = os.path.join(self.dem_dir, rel_alc) ImageTools.gdal_warp(path_alt, dst=path_alc, tr=coarse_res_str, multi=True) rasters_written.append(rel_alc) # ALC: bname_asc = basename + "_ASC.TIF" rel_asc = os.path.join(dbl_base, bname_asc) path_asc = os.path.join(self.dem_dir, rel_asc) ImageTools.gdal_warp(path_asp, dst=path_asc, tr=coarse_res_str, multi=True) rasters_written.append(rel_asc) # ALC: bname_slc = basename + "_SLC.TIF" rel_slc = os.path.join(dbl_base, bname_slc) path_slc = os.path.join(self.dem_dir, rel_slc) ImageTools.gdal_warp(path_slp, dst=path_slc, tr=coarse_res_str, multi=True) rasters_written.append(rel_slc) # Water mask: bname_msk = basename + "_MSK.TIF" rel_msk = os.path.join(dbl_base, bname_msk) path_msk = os.path.join(self.dem_dir, rel_msk) ImageTools.gdal_warp(self.gsw_dst, dst=path_msk, tr=coarse_res_str, multi=True) rasters_written.append(rel_msk) # Write HDR Metadata: date_start = datetime(1970, 1, 1) date_end = datetime(2100, 1, 1) dem_info = DEMInfo(self.site, all_paths_alt[0]) root = self._get_root() self._create_hdr(root, mission_field, basename, rasters_written, dem_info, date_start, date_end, self.dem_version) XMLTools.write_xml(root, hdr) # Remove temp files: FileSystem.remove_file(tmp_asp) FileSystem.remove_file(tmp_slp) FileSystem.remove_file(mnt_max_res) return hdr, dbl_dir