def test_gdal_buildvrt_concatenate(self): from StartMaja.Common import FileSystem paths = [] for i in range(1, 3, 1): img = np.ones((i, i, 2), np.int16) * i path = os.path.join(os.getcwd(), "test_gdal_merge_%s.tif" % i) ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) paths.append(path) empty = os.path.join(os.getcwd(), "empty.vrt") driver = ImageTools.gdal_buildvrt(*paths, dst=empty, separate=True, srcnodata=0) expected = np.array([[[1, 0], [0, 0]], [[2, 2], [2, 2]]], dtype=np.int16) np.testing.assert_almost_equal(driver.array, expected) self.assertEqual(driver.nodata_value, 0) self.assertEqual(driver.epsg, 32631) [FileSystem.remove_file(path) for path in paths] FileSystem.remove_file(empty) [self.assertFalse(os.path.exists(path)) for path in paths] self.assertFalse(os.path.exists(empty))
def test_write_read_geotiff_kwoptions(self): img = np.ones((self.height, self.width), np.int16) nodata = 42 path = os.path.join(os.getcwd(), "test_write_read_geotiff.tif") ImageIO.gdal_write("GTiff", img, path, self.projection, self.coordinates, options=["COMPRESS=DEFLATE"], nodata=nodata) self.assertTrue(os.path.exists(path)) arr, ds = ImageIO.tiff_to_array(path, array_only=False) self.assertTrue((arr == img).all()) self.assertEqual( nodata, gdal.Info(ds, format="json")["bands"][0]["noDataValue"]) self.assertEqual( gdal.Info( ds, format="json")["metadata"]["IMAGE_STRUCTURE"]["COMPRESSION"], "DEFLATE") self.assertEqual(ds.GetGeoTransform(), self.coordinates) # Compare projections by removing all spaces cause of multiline string self.assertEqual(ds.GetProjection().replace(" ", ""), self.projection.replace(" ", "")) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_reg_l8_lc2(self): tiles = ["199029"] dates = ["20170527T120000"] levels = ["l1c"] for prod, tile, date, level in zip(self.prod_l8_lc2, tiles, dates, levels): p = MajaProduct.factory(prod) self.assertIsInstance(p, Landsat8LC2) self.assertEqual(p.level, level) self.assertEqual(p.platform, "landsat8") self.assertEqual(p.type, "natif") self.assertEqual(p.tile, tile) self.assertEqual(p.nodata, 0) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertEqual(os.path.basename(p.metadata_file), prod.split(".")[0] + "_MTL.txt") self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, True) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{ 'name': 'XS', 'val': '30 -30' }]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_l8_lc1 + self.prod_l8_nat + self.prod_l8_mus + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, Landsat8LC2)
def test_get_s2_epsg_code(self): epsg_ref = 32631 projection = 'PROJCS["WGS 84 / UTM zone 31N",\ GEOGCS["WGS 84",DATUM["WGS_1984",\ SPHEROID["WGS 84",6378137,298.257223563,\ AUTHORITY["EPSG","7030"]],\ AUTHORITY["EPSG","6326"]],\ PRIMEM["Greenwich",0,\ AUTHORITY["EPSG","8901"]],\ UNIT["degree",0.0174532925199433,\ AUTHORITY["EPSG","9122"]],\ AUTHORITY["EPSG","4326"]],\ PROJECTION["Transverse_Mercator"],\ PARAMETER["latitude_of_origin",0],\ PARAMETER["central_meridian",3],\ PARAMETER["scale_factor",0.9996],\ PARAMETER["false_easting",500000],\ PARAMETER["false_northing",0],\ UNIT["metre",1,AUTHORITY["EPSG","9001"]],\ AXIS["Easting",EAST],AXIS["Northing",NORTH],\ AUTHORITY["EPSG","%s"]]' % epsg_ref img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_epsg.tif") ImageIO.write_geotiff(img, path, projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) epsg_new = ds.epsg self.assertEqual(epsg_new, epsg_ref) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def check_completeness(self): """ Check if the Gipp-folder exists already :return: True if existing. False if not. """ from StartMaja.Common import FileSystem n_files_per_lut = 4 try: found_models = sorted(self.get_models()) n_models = len(found_models) except ValueError: return False if not os.path.isdir(self.out_path): return False if found_models not in self.expected_models: return False try: hdrs = FileSystem.find("*.HDR", self.out_path) dbls = FileSystem.find("*.DBL.DIR", self.out_path) eefs = FileSystem.find("*.EEF", self.out_path) except ValueError: return False if len(eefs) < 4: return False # LUTs = 4 (TOCR, DIRT, DIFT, ALBD) + 1 constant for WATV per satellite if len(hdrs) != len( dbls ) != n_files_per_lut * self.n_sat * n_models + 1 * self.n_sat: return False return True
def test_reg_l8_muscate(self): tiles = ["31TCH", "31TCH"] levels = ["l1c", "l2a"] dates = ["20170501T103532", "20170501T103532"] validity = [True, False] for prod, tile, date, level, valid in zip(self.prod_l8_mus, tiles, dates, levels, validity): p = MajaProduct.factory(prod) self.assertIsInstance(p, Landsat8Muscate) self.assertEqual(p.level, level) self.assertEqual(p.nodata, 0) self.assertEqual(p.platform, "landsat8") self.assertEqual(p.type, "muscate") self.assertEqual(p.tile, tile) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertTrue( os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml")) self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, valid) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{ 'name': 'XS', 'val': '30 -30' }]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_l8_lc1 + self.prod_l8_lc2 + self.prod_l8_nat + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, Landsat8Muscate)
def test_override_array(self): img = np.ones((self.height, self.width), np.int16) img_new = np.array(np.arange(0, 40200).reshape(201, 200), dtype=np.uint8) path = os.path.join(os.getcwd(), "test_write_read_geotiff.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) self.assertTrue((ds.array == img).all()) self.assertEqual(ds.geotransform, self.coordinates) self.assertEqual(ds.projection.replace(" ", ""), self.projection.replace(" ", "")) self.assertIsNone(ds.nodata_value) self.assertEqual(ds.epsg, 32631) ds_new = GDalDatasetWrapper(ds=ds.get_ds(), array=img_new, nodata_value=123) FileSystem.remove_file(path) np.testing.assert_almost_equal(img_new, ds_new.array) self.assertEqual(ds_new.geotransform, self.coordinates) self.assertEqual(ds_new.projection.replace(" ", ""), self.projection.replace(" ", "")) self.assertEqual(ds_new.nodata_value, 123) self.assertEqual(ds_new.epsg, 32631) self.assertFalse(os.path.exists(path))
def prepare_mnt(self): """ Prepare the srtm files. :return: Path to the full resolution DEM file.gsw :rtype: str """ # Find/Download SRTM archives: srtm_archives = self.get_raw_data() # Unzip the downloaded/found srtm zip files: unzipped = [] for arch in srtm_archives: basename = os.path.splitext(os.path.basename(arch))[0] FileSystem.unzip(arch, self.wdir) fn_unzipped = FileSystem.find_single(pattern=basename + ".tif", path=self.wdir) unzipped.append(fn_unzipped) # Fusion of all SRTM files concat = ImageTools.gdal_buildvrt(*unzipped, vrtnodata=-32768) # Set nodata to 0 nodata = ImageTools.gdal_warp(concat, srcnodata=-32768, dstnodata=0, multi=True) # Combine to image of fixed extent srtm_full_res = os.path.join(self.wdir, "srtm_%sm.tif" % int(self.site.res_x)) ImageTools.gdal_warp(nodata, dst=srtm_full_res, r="cubic", te=self.site.te_str, t_srs=self.site.epsg_str, tr=self.site.tr_str, dstnodata=0, srcnodata=0, multi=True) return srtm_full_res
def generate(self, **kwargs): import random from StartMaja.Chain import Product from StartMaja.Common import TestFunctions, FileSystem platform_specifier = random.choice( self.platform_options["L1C"][self.platform]) orbit = kwargs.get("orbit", random.randint(0, 999)) version_orbit = kwargs.get("version", random.randint(0, 9)) if "sentinel2" in self.platform: date_str = self.date.strftime("%Y%m%dT%H%M%S") product_name = "_".join([ platform_specifier, "MSIL1C", date_str, "N" + str(orbit).zfill(4), "R" + str(version_orbit).zfill(3), self.tile, date_str + ".SAFE" ]) product_path = os.path.join(self.root, product_name) metadata_path = os.path.join(product_path, "MTD_MSIL1C.xml") else: date_str = self.date.strftime("%Y%m%d-%H%M%S") product_name = "_".join([ platform_specifier, date_str + "-000", "L2A", self.tile, random.choice("DC"), "V" + str(version_orbit) + "-" + str(version_orbit) ]) product_path = os.path.join(self.root, product_name) metadata_path = os.path.join(product_path, product_name + "_MTD_ALL.xml") self.prod = product_path self.mtd = metadata_path FileSystem.create_directory(product_path) TestFunctions.touch(metadata_path) return Product.MajaProduct.factory(self.prod)
def test_run_nonexisting_app(self): import subprocess cmd = "non_existing_app" args = [""] err = subprocess.CalledProcessError with self.assertRaises(err): FileSystem.run_external_app(cmd, args)
def test_reg_vs_muscate(self): tiles = ["KHUMBU", "KHUMBU", "KHUMBU"] levels = ["l1c", "l2a", "l3a"] dates = ["20180201T051359", "20180201T051359", "20180201T000000"] validity = [True, False, False] for prod, tile, date, level, valid in zip(self.prod_vs_mus, tiles, dates, levels, validity): p = MajaProduct.factory(prod) self.assertIsInstance(p, VenusMuscate) self.assertEqual(p.level, level) self.assertEqual(p.platform, "venus") self.assertEqual(p.type, "muscate") self.assertEqual(p.tile, tile) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertTrue(os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml")) self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, valid) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_vs_nat + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, VenusMuscate)
def test_reg_vs_natif(self): tiles = ["ISRAW906", "UNH", "SUDOUE_6", "CHILE"] dates = ["20180317T120000", "20180329T120000", "20191110T120000", "20200311T120000"] levels = ["l2a", "l1c", "l1c", "l1c"] for prod, tile, date, level in zip(self.prod_vs_nat, tiles, dates, levels): p = MajaProduct.factory(prod) self.assertIsInstance(p, VenusNatif) self.assertEqual(p.level, level) self.assertEqual(p.platform, "venus") self.assertEqual(p.type, "natif") self.assertEqual(p.tile, tile) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertEqual(os.path.basename(p.metadata_file), prod.split(".")[0] + ".HDR") self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, True) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_vs_mus + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, VenusNatif)
def test_srtm_get_maja_format_s2_31tcj(self): site = SiteInfo.Site("T31TCJ", 32631, res_x=90, res_y=-90, ul=(300000.000, 4900020.000), lr=(409800.000, 4790220.000)) dem_dir = os.path.join(os.getcwd(), "test_srtm_get_maja_format_s2_31tcj") s = SRTM.SRTM(site, dem_dir=dem_dir, raw_dem=self.raw_srtm, raw_gsw=self.raw_gsw, wdir=dem_dir) self.assertTrue(os.path.isdir(dem_dir)) hdr, dbl = s.to_maja_format(platform_id="S2_", mission_field="SENTINEL-2_", coarse_res=(240, -240), mnt_resolutions=[{ "name": "R1", "val": "10 -10" }, { "name": "R2", "val": "20 -20" }]) self.assertTrue(os.path.exists(hdr)) self.assertTrue(os.path.isdir(dbl)) FileSystem.remove_directory(dem_dir) FileSystem.remove_file(hdr)
def test_create_remove_file(self): path = os.path.join(os.getcwd(), "test_create_remove_file") self.assertFalse(os.path.exists(path)) TestFunctions.touch(path) self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.isfile(path)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def tearDownClass(cls): from StartMaja.Common import FileSystem # In case there's duplicates, remove them: FileSystem.remove_directory(cls.product_root) FileSystem.remove_file(cls.folders_file) FileSystem.remove_directory(cls.cams) FileSystem.remove_directory(cls.mnt.dbl) FileSystem.remove_file(cls.mnt.hdr)
def test_create_remove_dir(self): path = os.path.join(os.getcwd(), "test_create_remove_dir") # This throws a log message FileSystem.create_directory(path) self.assertTrue(os.path.isdir(path)) FileSystem.remove_directory(path) self.assertFalse(os.path.isdir(path)) self.assertFalse(os.path.exists(path))
def test_make_symlink_folder(self): origin = os.path.join(self.root, self.subdir_prefix + "0") destination = os.path.join(self.root, "symlink2") self.assertTrue(os.path.exists(origin)) self.assertFalse(os.path.exists(destination)) FileSystem.symlink(origin, destination) self.assertTrue(os.path.islink(destination)) os.remove(destination) self.assertFalse(os.path.exists(destination)) self.assertTrue(os.path.exists(origin))
def test_make_symlink_file(self): origin = os.path.join(self.root, self.file_a1) destination = os.path.join(self.root, "symlink1") self.assertTrue(os.path.exists(origin)) self.assertFalse(os.path.exists(destination)) FileSystem.symlink(origin, destination) self.assertTrue(os.path.islink(destination)) os.remove(destination) self.assertFalse(os.path.exists(destination)) self.assertTrue(os.path.exists(origin))
def test_get_nodata(self): expected_nodata = 42.0 img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_nodata_init.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) ds = ImageTools.gdal_buildvrt(path, VRTNodata=expected_nodata) self.assertEqual(expected_nodata, ds.nodata_value) np.testing.assert_almost_equal(ds.nodata_mask, np.ones_like(img)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_get_utm_description(self): img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_utm_description.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) utm = ds.utm_description utm_expected = "WGS 84 / UTM zone 31N" self.assertEqual(utm_expected, utm) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_gdal_buildvrt(self): path = os.path.join(os.getcwd(), "test_gdal_buildvrt.tif") vrt = os.path.join(os.getcwd(), "test_vrt.vrt") img = np.arange(-4, 5).reshape(3, 3) / 5 ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) driver = ImageTools.gdal_buildvrt(path, dst=vrt) self.assertTrue(os.path.exists(vrt)) np.testing.assert_almost_equal(driver.array, img) FileSystem.remove_file(vrt) FileSystem.remove_file(path)
def link(self, dest): """ Symlink a set of Gipps to a given destination :param dest: The destination directory :return: """ from StartMaja.Common import FileSystem eefs = FileSystem.find(GIPPFile.regex, self.out_path) dbls = FileSystem.find(GIPPFile.regex_dbl, self.out_path) for f in eefs + dbls: base = os.path.basename(f) FileSystem.symlink(f, os.path.join(dest, base))
def test_get_resolution(self): img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_resolution.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) res_expected = (self.coordinates[1], self.coordinates[-1]) self.assertEqual(res_expected, ds.resolution) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_get_models_cams(self): from StartMaja.Common import FileSystem g = GippSet(self.root, "sentinel2", "tm", cams=True) with self.assertRaises(ValueError): g.get_models() self.assertFalse(g.check_completeness()) g.download() self.assertTrue(g.check_completeness()) self.assertTrue(g.get_models() in g.expected_models) FileSystem.remove_file(os.path.join(self.root, "wget-log")) if not os.getcwd() == g.out_path: FileSystem.remove_directory(g.out_path) self.assertFalse(os.path.exists(g.out_path))
def test_get_ul_lr(self): img = np.ones((1000, 1000), np.int16) path = os.path.join(os.getcwd(), "test_get_ul_lr.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) ulx, uly, lrx, lry = ds.ul_lr self.assertEqual((ulx, uly), (300000.0, 4900020.0)) self.assertEqual((lrx, lry), (310000.0, 4890020.0)) self.assertEqual(ds.extent, (300000.0, 4890020.0, 310000.0, 4900020.0)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def link(self, dest): """ Symlink a file to the working dir without copying it :param dest: The destination folder :return: """ from StartMaja.Common import FileSystem hdr_realpath = os.path.realpath(os.path.expanduser(self.hdr)) dbl_realpath = os.path.realpath(os.path.expanduser(self.dbl)) FileSystem.symlink(hdr_realpath, os.path.join(dest, os.path.basename(hdr_realpath))) FileSystem.symlink(dbl_realpath, os.path.join(dest, os.path.basename(dbl_realpath)))
def generate(self, **kwargs): from datetime import datetime import random from StartMaja.Common import FileSystem from StartMaja.Chain.GippFile import GippSet mission_param = kwargs.get("mission", random.choice(["muscate", "natif"])) if mission_param == "tm": self.platform = "sentinel2" mission = self.mission_choices[mission_param][self.platform] satellites = [self.mission_short[self.platform] ] if self.platform != "sentinel2" else ["S2A", "S2B"] with_cams = kwargs.get("cams", True) cams_suffix = "_CAMS" if with_cams else "" out_path = os.path.join( self.root, "_".join([self.platform.upper(), mission_param.upper()]) + cams_suffix) FileSystem.create_directory(out_path) if with_cams: models = ["CONTINEN"] + [ "ORGANICM", "BLACKCAR", "DUST", "SEASALT", "SULPHATE" ] else: models = ["CONTINEN"] allsites = "ALLSITES" hdr_types = ["ALBD", "DIFT", "DIRT", "TOCR", "WATV"] eef_types = ["COMM", "SITE", "SMAC", "EXTL", "QLTL"] tm_types = ["COMM", "EXTL", "QLTL"] version = random.randint(0, 9999) version_str = str(version).zfill(5) start_date = datetime(2014, 12, 30) for sat in satellites: for name in eef_types: self._create_hdr(out_path, sat, name, start_date, version_str, allsites, mission, ".EEF") for name in hdr_types: for model in models: basename = self._create_hdr(out_path, sat, name, start_date, version_str, model, mission, ".HDR") dbl_name = os.path.join(out_path, basename + ".DBL.DIR") FileSystem.create_directory(dbl_name) # For TM: Add an additional set of COMM, EXTL and QLTL files with muscate mission: for name in tm_types: if mission_param != "tm": continue tm_mission = "SENTINEL2" tm_version_str = str(version + 10000).zfill(5) self._create_hdr(out_path, sat, name, start_date, tm_version_str, allsites, tm_mission, ".EEF") return GippSet(self.root, self.platform, mission_param, cams=with_cams)
def test_get_file_full(self): expected = os.path.join(self.root, "b.jpg") dirnames_e = p.normpath(expected).split(os.sep) calculated = FileSystem.find(path=self.root, pattern="b.jpg", depth=1) calculated2 = FileSystem.find(path=self.root, pattern="b.jpg", depth=1, ftype="file") self.assertEqual(calculated, calculated2) self.assertEqual(len(calculated), 1) dirnames_c = p.normpath(calculated[0]).split(os.sep) for exp, calc in zip(dirnames_c[-1:], dirnames_e[-1:]): self.assertEqual(exp[:-1], calc[:-1]) self.assertEqual(expected, calculated[0])
def test_write_read_geotiff(self): img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_write_read_geotiff.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) arr, ds = ImageIO.tiff_to_array(path, array_only=False) self.assertTrue((arr == img).all()) self.assertEqual(ds.GetGeoTransform(), self.coordinates) # Compare projections by removing all spaces cause of multiline string self.assertEqual(ds.GetProjection().replace(" ", ""), self.projection.replace(" ", "")) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_write_read_memory(self): img = np.ones((self.height, self.width), np.int16) path = "/vsimem/test_write_read_memory.tif" ds = ImageIO.write_to_memory(img, path, self.projection, self.coordinates) arr = ds.ReadAsArray() self.assertTrue((arr == img).all()) self.assertEqual(ds.GetGeoTransform(), self.coordinates) # Compare projections by removing all spaces cause of multiline string self.assertEqual(ds.GetProjection().replace(" ", ""), self.projection.replace(" ", "")) ds = None # Always remember to dereference :) FileSystem.remove_file(path)