def check_completeness(self): """ Check if the Gipp-folder exists already :return: True if existing. False if not. """ from Common import FileSystem n_files_per_lut = 4 try: found_models = sorted(self.get_models()) n_models = len(found_models) except ValueError: return False if not os.path.isdir(self.out_path): return False if found_models not in self.expected_models: return False try: hdrs = FileSystem.find("*.HDR", self.out_path) dbls = FileSystem.find("*.DBL.DIR", self.out_path) eefs = FileSystem.find("*.EEF", self.out_path) except ValueError: return False if len(eefs) < 4: return False # LUTs = 4 (TOCR, DIRT, DIFT, ALBD) + 1 constant for WATV per satellite if len(hdrs) != len( dbls ) != n_files_per_lut * self.n_sat * n_models + 1 * self.n_sat: return False return True
def test_srtm_get_maja_format_s2_31tcj(self): site = SiteInfo.Site("T31TCJ", 32631, res_x=90, res_y=-90, ul=(300000.000, 4900020.000), lr=(409800.000, 4790220.000)) dem_dir = os.path.join(os.getcwd(), "test_srtm_get_maja_format_s2_31tcj") s = SRTM.SRTM(site, dem_dir=dem_dir, raw_dem=self.raw_srtm, raw_gsw=self.raw_gsw, wdir=dem_dir) self.assertTrue(os.path.isdir(dem_dir)) hdr, dbl = s.to_maja_format(platform_id="S2_", mission_field="SENTINEL-2_", coarse_res=(240, -240), mnt_resolutions=[{ "name": "R1", "val": "10 -10" }, { "name": "R2", "val": "20 -20" }]) self.assertTrue(os.path.exists(hdr)) self.assertTrue(os.path.isdir(dbl)) FileSystem.remove_directory(dem_dir) FileSystem.remove_file(hdr)
def test_reg_l8_lc2(self): tiles = ["199029"] dates = ["20170527T120000"] levels = ["l1c"] for prod, tile, date, level in zip(self.prod_l8_lc2, tiles, dates, levels): p = MajaProduct.factory(prod) self.assertIsInstance(p, Landsat8LC2) self.assertEqual(p.level, level) self.assertEqual(p.platform, "landsat8") self.assertEqual(p.type, "natif") self.assertEqual(p.tile, tile) self.assertEqual(p.nodata, 0) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertEqual(os.path.basename(p.metadata_file), prod.split(".")[0] + "_MTL.txt") self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, True) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{ 'name': 'XS', 'val': '30 -30' }]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_l8_lc1 + self.prod_l8_nat + self.prod_l8_mus + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, Landsat8LC2)
def test_reg_l8_muscate(self): tiles = ["31TCH", "31TCH"] levels = ["l1c", "l2a"] dates = ["20170501T103532", "20170501T103532"] validity = [True, False] for prod, tile, date, level, valid in zip(self.prod_l8_mus, tiles, dates, levels, validity): p = MajaProduct.factory(prod) self.assertIsInstance(p, Landsat8Muscate) self.assertEqual(p.level, level) self.assertEqual(p.nodata, 0) self.assertEqual(p.platform, "landsat8") self.assertEqual(p.type, "muscate") self.assertEqual(p.tile, tile) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertTrue( os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml")) self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, valid) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{ 'name': 'XS', 'val': '30 -30' }]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_l8_lc1 + self.prod_l8_lc2 + self.prod_l8_nat + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, Landsat8Muscate)
def test_get_s2_epsg_code(self): epsg_ref = 32631 projection = 'PROJCS["WGS 84 / UTM zone 31N",\ GEOGCS["WGS 84",DATUM["WGS_1984",\ SPHEROID["WGS 84",6378137,298.257223563,\ AUTHORITY["EPSG","7030"]],\ AUTHORITY["EPSG","6326"]],\ PRIMEM["Greenwich",0,\ AUTHORITY["EPSG","8901"]],\ UNIT["degree",0.0174532925199433,\ AUTHORITY["EPSG","9122"]],\ AUTHORITY["EPSG","4326"]],\ PROJECTION["Transverse_Mercator"],\ PARAMETER["latitude_of_origin",0],\ PARAMETER["central_meridian",3],\ PARAMETER["scale_factor",0.9996],\ PARAMETER["false_easting",500000],\ PARAMETER["false_northing",0],\ UNIT["metre",1,AUTHORITY["EPSG","9001"]],\ AXIS["Easting",EAST],AXIS["Northing",NORTH],\ AUTHORITY["EPSG","%s"]]' % epsg_ref img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_epsg.tif") ImageIO.write_geotiff(img, path, projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) epsg_new = ds.epsg self.assertEqual(epsg_new, epsg_ref) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def generate(self, **kwargs): import random from Chain import Product from Common import TestFunctions, FileSystem platform_specifier = random.choice( self.platform_options["L1C"][self.platform]) orbit = kwargs.get("orbit", random.randint(0, 999)) version_orbit = kwargs.get("version", random.randint(0, 9)) if "sentinel2" in self.platform: date_str = self.date.strftime("%Y%m%dT%H%M%S") product_name = "_".join([ platform_specifier, "MSIL1C", date_str, "N" + str(orbit).zfill(4), "R" + str(version_orbit).zfill(3), self.tile, date_str + ".SAFE" ]) product_path = os.path.join(self.root, product_name) metadata_path = os.path.join(product_path, "MTD_MSIL1C.xml") else: date_str = self.date.strftime("%Y%m%d-%H%M%S") product_name = "_".join([ platform_specifier, date_str + "-000", "L2A", self.tile, random.choice("DC"), "V" + str(version_orbit) + "-" + str(version_orbit) ]) product_path = os.path.join(self.root, product_name) metadata_path = os.path.join(product_path, product_name + "_MTD_ALL.xml") self.prod = product_path self.mtd = metadata_path FileSystem.create_directory(product_path) TestFunctions.touch(metadata_path) return Product.MajaProduct.factory(self.prod)
def test_reg_vs_muscate(self): tiles = ["KHUMBU", "KHUMBU", "KHUMBU"] levels = ["l1c", "l2a", "l3a"] dates = ["20180201T051359", "20180201T051359", "20180201T000000"] validity = [True, False, False] for prod, tile, date, level, valid in zip(self.prod_vs_mus, tiles, dates, levels, validity): p = MajaProduct.factory(prod) self.assertIsInstance(p, VenusMuscate) self.assertEqual(p.level, level) self.assertEqual(p.platform, "venus") self.assertEqual(p.type, "muscate") self.assertEqual(p.tile, tile) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertTrue(os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml")) self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, valid) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_vs_nat + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, VenusMuscate)
def test_reg_vs_natif(self): tiles = ["ISRAW906", "UNH", "SUDOUE_6", "CHILE"] dates = ["20180317T120000", "20180329T120000", "20191110T120000", "20200311T120000"] levels = ["l2a", "l1c", "l1c", "l1c"] for prod, tile, date, level in zip(self.prod_vs_nat, tiles, dates, levels): p = MajaProduct.factory(prod) self.assertIsInstance(p, VenusNatif) self.assertEqual(p.level, level) self.assertEqual(p.platform, "venus") self.assertEqual(p.type, "natif") self.assertEqual(p.tile, tile) self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date) self.assertEqual(os.path.basename(p.metadata_file), prod.split(".")[0] + ".HDR") self.assertTrue(os.path.exists(p.metadata_file)) self.assertEqual(p.validity, True) link_dir = "linkdir" FileSystem.create_directory(link_dir) p.link(link_dir) self.assertTrue(os.path.islink(os.path.join(link_dir, p.base))) self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}]) self.assertEqual(p, p) FileSystem.remove_directory(link_dir) # Other prods: for prod in self.prod_vs_mus + self.prods_other: p = MajaProduct.factory(prod) self.assertNotIsInstance(p, VenusNatif)
def test_write_read_geotiff_kwoptions(self): img = np.ones((self.height, self.width), np.int16) nodata = 42 path = os.path.join(os.getcwd(), "test_write_read_geotiff.tif") ImageIO.gdal_write("GTiff", img, path, self.projection, self.coordinates, options=["COMPRESS=DEFLATE"], nodata=nodata) self.assertTrue(os.path.exists(path)) arr, ds = ImageIO.tiff_to_array(path, array_only=False) self.assertTrue((arr == img).all()) self.assertEqual( nodata, gdal.Info(ds, format="json")["bands"][0]["noDataValue"]) self.assertEqual( gdal.Info( ds, format="json")["metadata"]["IMAGE_STRUCTURE"]["COMPRESSION"], "DEFLATE") self.assertEqual(ds.GetGeoTransform(), self.coordinates) # Compare projections by removing all spaces cause of multiline string self.assertEqual(ds.GetProjection().replace(" ", ""), self.projection.replace(" ", "")) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_run_nonexisting_app(self): import subprocess cmd = "non_existing_app" args = [""] err = subprocess.CalledProcessError with self.assertRaises(err): FileSystem.run_external_app(cmd, args)
def test_gdal_buildvrt_concatenate(self): from Common import FileSystem paths = [] for i in range(1, 3, 1): img = np.ones((i, i, 2), np.int16) * i path = os.path.join(os.getcwd(), "test_gdal_merge_%s.tif" % i) ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) paths.append(path) empty = os.path.join(os.getcwd(), "empty.vrt") driver = ImageTools.gdal_buildvrt(*paths, dst=empty, separate=True, srcnodata=0) expected = np.array([[[1, 0], [0, 0]], [[2, 2], [2, 2]]], dtype=np.int16) np.testing.assert_almost_equal(driver.array, expected) self.assertEqual(driver.nodata_value, 0) self.assertEqual(driver.epsg, 32631) [FileSystem.remove_file(path) for path in paths] FileSystem.remove_file(empty) [self.assertFalse(os.path.exists(path)) for path in paths] self.assertFalse(os.path.exists(empty))
def test_create_remove_dir(self): path = os.path.join(os.getcwd(), "test_create_remove_dir") # This throws a log message FileSystem.create_directory(path) self.assertTrue(os.path.isdir(path)) FileSystem.remove_directory(path) self.assertFalse(os.path.isdir(path)) self.assertFalse(os.path.exists(path))
def test_create_remove_file(self): path = os.path.join(os.getcwd(), "test_create_remove_file") self.assertFalse(os.path.exists(path)) TestFunctions.touch(path) self.assertTrue(os.path.exists(path)) self.assertTrue(os.path.isfile(path)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def tearDownClass(cls): from Common import FileSystem # In case there's duplicates, remove them: FileSystem.remove_directory(cls.product_root) FileSystem.remove_file(cls.folders_file) FileSystem.remove_directory(cls.cams) FileSystem.remove_directory(cls.mnt.dbl) FileSystem.remove_file(cls.mnt.hdr)
def test_get_nodata(self): expected_nodata = 42.0 img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_nodata_init.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) ds = ImageTools.gdal_buildvrt(path, VRTNodata=expected_nodata) self.assertEqual(expected_nodata, ds.nodata_value) np.testing.assert_almost_equal(ds.nodata_mask, np.ones_like(img)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_make_symlink_file(self): origin = os.path.join(self.root, self.file_a1) destination = os.path.join(self.root, "symlink1") self.assertTrue(os.path.exists(origin)) self.assertFalse(os.path.exists(destination)) FileSystem.symlink(origin, destination) self.assertTrue(os.path.islink(destination)) os.remove(destination) self.assertFalse(os.path.exists(destination)) self.assertTrue(os.path.exists(origin))
def test_make_symlink_folder(self): origin = os.path.join(self.root, self.subdir_prefix + "0") destination = os.path.join(self.root, "symlink2") self.assertTrue(os.path.exists(origin)) self.assertFalse(os.path.exists(destination)) FileSystem.symlink(origin, destination) self.assertTrue(os.path.islink(destination)) os.remove(destination) self.assertFalse(os.path.exists(destination)) self.assertTrue(os.path.exists(origin))
def test_get_utm_description(self): img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_utm_description.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) utm = ds.utm_description utm_expected = "WGS 84 / UTM zone 31N" self.assertEqual(utm_expected, utm) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_gdal_buildvrt(self): path = os.path.join(os.getcwd(), "test_gdal_buildvrt.tif") vrt = os.path.join(os.getcwd(), "test_vrt.vrt") img = np.arange(-4, 5).reshape(3, 3) / 5 ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) driver = ImageTools.gdal_buildvrt(path, dst=vrt) self.assertTrue(os.path.exists(vrt)) np.testing.assert_almost_equal(driver.array, img) FileSystem.remove_file(vrt) FileSystem.remove_file(path)
def test_get_resolution(self): img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_get_resolution.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) res_expected = (self.coordinates[1], self.coordinates[-1]) self.assertEqual(res_expected, ds.resolution) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def link(self, dest): """ Symlink a set of Gipps to a given destination :param dest: The destination directory :return: """ from Common import FileSystem eefs = FileSystem.find(GIPPFile.regex, self.out_path) dbls = FileSystem.find(GIPPFile.regex_dbl, self.out_path) for f in eefs + dbls: base = os.path.basename(f) FileSystem.symlink(f, os.path.join(dest, base))
def test_get_models_cams(self): from Common import FileSystem g = GippSet(self.root, "sentinel2", "tm", cams=True) with self.assertRaises(ValueError): g.get_models() self.assertFalse(g.check_completeness()) g.download() self.assertTrue(g.check_completeness()) self.assertTrue(g.get_models() in g.expected_models) FileSystem.remove_file(os.path.join(self.root, "wget-log")) if not os.getcwd() == g.out_path: FileSystem.remove_directory(g.out_path) self.assertFalse(os.path.exists(g.out_path))
def link(self, dest): """ Symlink a file to the working dir without copying it :param dest: The destination folder :return: """ from Common import FileSystem hdr_realpath = os.path.realpath(os.path.expanduser(self.hdr)) dbl_realpath = os.path.realpath(os.path.expanduser(self.dbl)) FileSystem.symlink(hdr_realpath, os.path.join(dest, os.path.basename(hdr_realpath))) FileSystem.symlink(dbl_realpath, os.path.join(dest, os.path.basename(dbl_realpath)))
def test_get_ul_lr(self): img = np.ones((1000, 1000), np.int16) path = os.path.join(os.getcwd(), "test_get_ul_lr.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) ds = GDalDatasetWrapper.from_file(path) ulx, uly, lrx, lry = ds.ul_lr self.assertEqual((ulx, uly), (300000.0, 4900020.0)) self.assertEqual((lrx, lry), (310000.0, 4890020.0)) self.assertEqual(ds.extent, (300000.0, 4890020.0, 310000.0, 4900020.0)) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def generate(self, **kwargs): from datetime import datetime import random from Common import FileSystem from Chain.GippFile import GippSet mission_param = kwargs.get("mission", random.choice(["muscate", "natif"])) if mission_param == "tm": self.platform = "sentinel2" mission = self.mission_choices[mission_param][self.platform] satellites = [self.mission_short[self.platform] ] if self.platform != "sentinel2" else ["S2A", "S2B"] with_cams = kwargs.get("cams", True) cams_suffix = "_CAMS" if with_cams else "" out_path = os.path.join( self.root, "_".join([self.platform.upper(), mission_param.upper()]) + cams_suffix) FileSystem.create_directory(out_path) if with_cams: models = ["CONTINEN"] + [ "ORGANICM", "BLACKCAR", "DUST", "SEASALT", "SULPHATE" ] else: models = ["CONTINEN"] allsites = "ALLSITES" hdr_types = ["ALBD", "DIFT", "DIRT", "TOCR", "WATV"] eef_types = ["COMM", "SITE", "SMAC", "EXTL", "QLTL"] tm_types = ["COMM", "EXTL", "QLTL"] version = random.randint(0, 9999) version_str = str(version).zfill(5) start_date = datetime(2014, 12, 30) for sat in satellites: for name in eef_types: self._create_hdr(out_path, sat, name, start_date, version_str, allsites, mission, ".EEF") for name in hdr_types: for model in models: basename = self._create_hdr(out_path, sat, name, start_date, version_str, model, mission, ".HDR") dbl_name = os.path.join(out_path, basename + ".DBL.DIR") FileSystem.create_directory(dbl_name) # For TM: Add an additional set of COMM, EXTL and QLTL files with muscate mission: for name in tm_types: if mission_param != "tm": continue tm_mission = "SENTINEL2" tm_version_str = str(version + 10000).zfill(5) self._create_hdr(out_path, sat, name, start_date, tm_version_str, allsites, tm_mission, ".EEF") return GippSet(self.root, self.platform, mission_param, cams=with_cams)
def test_write_read_memory(self): img = np.ones((self.height, self.width), np.int16) path = "/vsimem/test_write_read_memory.tif" ds = ImageIO.write_to_memory(img, path, self.projection, self.coordinates) arr = ds.ReadAsArray() self.assertTrue((arr == img).all()) self.assertEqual(ds.GetGeoTransform(), self.coordinates) # Compare projections by removing all spaces cause of multiline string self.assertEqual(ds.GetProjection().replace(" ", ""), self.projection.replace(" ", "")) ds = None # Always remember to dereference :) FileSystem.remove_file(path)
def __init__(self, site, **kwargs): if not int(gdal.VersionInfo()) >= 2000000: raise ImportError("MNT creation needs Gdal >2.0!") self.site = site self.dem_dir = kwargs.get( "dem_dir", os.path.join(tempfile.gettempdir(), "maja_dem_files")) if not os.path.isdir(self.dem_dir): FileSystem.create_directory(self.dem_dir) self.wdir = kwargs.get("wdir", None) if not self.wdir: self.wdir = tempfile.mkdtemp(prefix="raw_dem_") else: FileSystem.create_directory(self.wdir) # This needs to be provided for EuDEM: self.raw_dem = kwargs.get("raw_dem", None) if not self.raw_dem: self.raw_dem = tempfile.mkdtemp(prefix="raw_dem_") else: FileSystem.create_directory(self.raw_dem) self.raw_gsw = kwargs.get("raw_gsw", None) if not self.raw_gsw: self.raw_gsw = tempfile.mkdtemp(prefix="raw_gsw_") else: FileSystem.create_directory(self.raw_gsw) self.gsw_codes = self.get_gsw_codes(self.site) self.dem_version = kwargs.get("dem_version", None) self.gsw_threshold = kwargs.get("gsw_threshold", 30.) self.gsw_dst = kwargs.get( "gsw_dst", os.path.join(self.wdir, "surface_water_mask.tif")) self.quiet = not kwargs.get("verbose", False)
def test_write_read_geotiff(self): img = np.ones((self.height, self.width), np.int16) path = os.path.join(os.getcwd(), "test_write_read_geotiff.tif") ImageIO.write_geotiff(img, path, self.projection, self.coordinates) self.assertTrue(os.path.exists(path)) arr, ds = ImageIO.tiff_to_array(path, array_only=False) self.assertTrue((arr == img).all()) self.assertEqual(ds.GetGeoTransform(), self.coordinates) # Compare projections by removing all spaces cause of multiline string self.assertEqual(ds.GetProjection().replace(" ", ""), self.projection.replace(" ", "")) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_faulty_geotransform_projection(self): coordinates = (652594.9112913811, 10.00887639510383, 0, 5072876.717295351, 0) # Missing one value projection = '' # Empty height = 200 width = 100 img = np.ones((height, width), np.int16) path = os.path.join(os.getcwd(), "test_faulty_geotransform_projection.tif") # Check geotransform wrong: with self.assertRaises(TypeError): ImageIO.write_geotiff(img, path, projection, coordinates) FileSystem.remove_file(path) self.assertFalse(os.path.exists(path))
def test_get_file_full(self): expected = os.path.join(self.root, "b.jpg") dirnames_e = p.normpath(expected).split(os.sep) calculated = FileSystem.find(path=self.root, pattern="b.jpg", depth=1) calculated2 = FileSystem.find(path=self.root, pattern="b.jpg", depth=1, ftype="file") self.assertEqual(calculated, calculated2) self.assertEqual(len(calculated), 1) dirnames_c = p.normpath(calculated[0]).split(os.sep) for exp, calc in zip(dirnames_c[-1:], dirnames_e[-1:]): self.assertEqual(exp[:-1], calc[:-1]) self.assertEqual(expected, calculated[0])