def test_reg_l8_muscate(self):
tiles = ["31TCH", "31TCH"]
levels = ["l1c", "l2a"]
dates = ["20170501T103532", "20170501T103532"]
validity = [True, False]
for prod, tile, date, level, valid in zip(self.prod_l8_mus, tiles,
dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Landsat8Muscate)
self.assertEqual(p.level, level)
self.assertEqual(p.nodata, 0)
self.assertEqual(p.platform, "landsat8")
self.assertEqual(p.type, "muscate")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(
os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml"))
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
'name': 'XS',
'val': '30 -30'
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_l8_lc1 + self.prod_l8_lc2 + self.prod_l8_nat + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Landsat8Muscate)
def test_reg_vs_natif(self):
tiles = ["ISRAW906", "UNH", "SUDOUE_6", "CHILE"]
dates = ["20180317T120000", "20180329T120000", "20191110T120000", "20200311T120000"]
levels = ["l2a", "l1c", "l1c", "l1c"]
for prod, tile, date, level in zip(self.prod_vs_nat, tiles, dates, levels):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, VenusNatif)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "venus")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertEqual(os.path.basename(p.metadata_file), prod.split(".")[0] + ".HDR")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, True)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_vs_mus + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, VenusNatif)
def test_gdal_tile_untile(self):
img = np.arange(0., 100.).reshape((10, 10))
path = os.path.join(os.getcwd(), "test_gdal_retile.tif")
tile_folder = os.path.join(os.getcwd(), "tiled")
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
# Add parasitic file - It should not cause problems
path_parasite = os.path.join(tile_folder, "tile_01_01.tif")
FileSystem.create_directory(tile_folder)
ImageIO.write_geotiff(img, path_parasite, self.projection, self.coordinates)
ds_in = GDalDatasetWrapper.from_file(path)
self.assertTrue(os.path.exists(path))
tiles = ImageTools.gdal_retile(ds_in, tile_folder,
TileWidth=2,
TileHeight=2,
Overlap=1)
self.assertTrue(os.path.isdir(tile_folder))
self.assertEqual(len(tiles), 81)
img_read = np.array(ImageIO.tiff_to_array(tiles[-1]))
expected = np.array([[88, 89],
[98, 99]])
# Some gdal_retile versions are producing the following image:
# [[87, 89], [97, 99]].
np.testing.assert_allclose(expected, img_read, atol=1)
# Untile
ds_untiled = ImageTools.gdal_buildvrt(*tiles)
np.testing.assert_allclose(img, ds_untiled.array, atol=1)
FileSystem.remove_file(path)
FileSystem.remove_directory(tile_folder)
def test_srtm_get_maja_format_s2_31tcj(self):
site = SiteInfo.Site("T31TCJ",
32631,
res_x=90,
res_y=-90,
ul=(300000.000, 4900020.000),
lr=(409800.000, 4790220.000))
dem_dir = os.path.join(os.getcwd(),
"test_srtm_get_maja_format_s2_31tcj")
s = SRTM.SRTM(site,
dem_dir=dem_dir,
raw_dem=self.raw_srtm,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
hdr, dbl = s.to_maja_format(platform_id="S2_",
mission_field="SENTINEL-2_",
coarse_res=(240, -240),
mnt_resolutions=[{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertTrue(os.path.exists(hdr))
self.assertTrue(os.path.isdir(dbl))
FileSystem.remove_directory(dem_dir)
FileSystem.remove_file(hdr)
def test_reg_vs_muscate(self):
tiles = ["KHUMBU", "KHUMBU", "KHUMBU"]
levels = ["l1c", "l2a", "l3a"]
dates = ["20180201T051359", "20180201T051359", "20180201T000000"]
validity = [True, False, False]
for prod, tile, date, level, valid in zip(self.prod_vs_mus, tiles, dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, VenusMuscate)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "venus")
self.assertEqual(p.type, "muscate")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml"))
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_vs_nat + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, VenusMuscate)
def test_reg_l8_lc2(self):
tiles = ["199029"]
dates = ["20170527T120000"]
levels = ["l1c"]
for prod, tile, date, level in zip(self.prod_l8_lc2, tiles, dates,
levels):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Landsat8LC2)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "landsat8")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.nodata, 0)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertEqual(os.path.basename(p.metadata_file),
prod.split(".")[0] + "_MTL.txt")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, True)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
'name': 'XS',
'val': '30 -30'
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_l8_lc1 + self.prod_l8_nat + self.prod_l8_mus + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Landsat8LC2)
def test_create_remove_dir(self):
path = os.path.join(os.getcwd(), "test_create_remove_dir")
# This throws a log message
FileSystem.create_directory(path)
self.assertTrue(os.path.isdir(path))
FileSystem.remove_directory(path)
self.assertFalse(os.path.isdir(path))
self.assertFalse(os.path.exists(path))
def __clean_up(self):
"""
Clean up the download directory.
:return:
"""
from StartMaja.Common import FileSystem
FileSystem.remove_directory(self.temp_folder)
FileSystem.remove_file(self.lut_archive)
FileSystem.remove_file(self.gipp_archive)
def test_get_models_cams(self):
from StartMaja.Common import FileSystem
g = GippSet(self.root, "sentinel2", "tm", cams=True)
with self.assertRaises(ValueError):
g.get_models()
self.assertFalse(g.check_completeness())
g.download()
self.assertTrue(g.check_completeness())
self.assertTrue(g.get_models() in g.expected_models)
FileSystem.remove_file(os.path.join(self.root, "wget-log"))
if not os.getcwd() == g.out_path:
FileSystem.remove_directory(g.out_path)
self.assertFalse(os.path.exists(g.out_path))
def download(self):
"""
Download a specific set of Gipps to the given folder.
First, attempt to download the most recent git archive containing the .EEF files as well as the
url to download the LUTs. Then, the latter will be downloaded separately.
:return:
"""
import shutil
from StartMaja.Common import FileSystem
self.out_path = os.path.join(self.fpath, self.gipp_folder_name)
FileSystem.download_file(self.url, self.gipp_archive, self.log_level)
FileSystem.unzip(self.gipp_archive, self.temp_folder)
gipp_maja_git = os.path.join(self.temp_folder, "maja-gipp-develop")
# TODO Remove second WATV for Venus-Natif
platform_folder = FileSystem.find_single(path=gipp_maja_git,
pattern="^" +
self.gipp_folder_name + "$")
if not platform_folder:
self.__clean_up()
raise OSError("Cannot find any gipp folder for platform %s" %
self.gipp_folder_name)
readme = FileSystem.find_single(path=platform_folder,
pattern="readme*")
if not readme:
self.__clean_up()
raise OSError("Cannot find download-file for LUT-Download in %s" %
platform_folder)
lut_url = FileSystem.find_in_file(readme, self.zenodo_reg)
if not lut_url:
self.__clean_up()
raise OSError("Cannot find url to download LUTs")
FileSystem.download_file(lut_url, self.lut_archive, self.log_level)
FileSystem.unzip(self.lut_archive, platform_folder)
lut_folder = FileSystem.find_single(path=platform_folder,
pattern="LUTs")
if not lut_folder:
self.__clean_up()
raise OSError("Cannot find 'LUTs' folder in %s" % self.temp_folder)
for f in os.listdir(lut_folder):
shutil.move(os.path.join(lut_folder, f), platform_folder)
if os.path.isdir(self.out_path):
FileSystem.remove_directory(self.out_path)
shutil.move(platform_folder, self.out_path)
self.__clean_up()
FileSystem.remove_directory(lut_folder)
# Sanity check:
if not self.check_completeness():
raise ValueError(
"GIPP download failed. Please delete gipp/ folder and try again."
)
def test_symlink(self):
from StartMaja.Common import FileSystem
g = GippSet(self.root, "sentinel2", "tm")
self.assertFalse(g.check_completeness())
g.download()
self.assertTrue(g.check_completeness())
symlink_dir = os.path.join(self.root, "symlinks")
FileSystem.create_directory(symlink_dir)
with self.assertRaises(ValueError):
self.assertTrue(len(FileSystem.find("*EEF", symlink_dir)), 0)
g.link(symlink_dir)
self.assertEqual(len(FileSystem.find("*EEF", symlink_dir)), 15)
FileSystem.remove_directory(symlink_dir)
self.assertFalse(os.path.isdir(symlink_dir))
FileSystem.remove_file(os.path.join(self.root, "wget-log"))
if not os.getcwd() == g.out_path:
FileSystem.remove_directory(g.out_path)
self.assertFalse(os.path.exists(g.out_path))
def test_download_s2_muscate_nocams(self):
from StartMaja.Common import FileSystem
g = GippSet(self.root, "sentinel2", "muscate")
self.assertFalse(g.check_completeness())
g.download()
self.assertTrue(g.check_completeness())
n_l2comm = len(FileSystem.find("*L2COMM*", g.out_path))
self.assertEqual(n_l2comm, 2)
n_qltl = len(FileSystem.find("*CKQLTL*", g.out_path))
self.assertEqual(n_qltl, 2)
n_extl = len(FileSystem.find("*CKEXTL*", g.out_path))
self.assertEqual(n_extl, 2)
n_extl = len(FileSystem.find("*EEF", g.out_path))
self.assertEqual(n_extl, 9)
FileSystem.remove_file(os.path.join(self.root, "wget-log"))
if not os.getcwd() == g.out_path:
FileSystem.remove_directory(g.out_path)
self.assertFalse(os.path.exists(g.out_path))
def test_get_raw_data(self):
site = SiteInfo.Site("T31TCJ",
32631,
ul=(300000.000, 4900020.000),
lr=(409800.000, 4790220.000))
dem_dir = os.path.join(os.getcwd(), "srtm_dir")
s = SRTM.SRTM(site,
dem_dir=dem_dir,
raw_dem=self.raw_srtm,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
srtm_codes = SRTM.SRTM.get_srtm_codes(site)
s.get_raw_data()
for code in srtm_codes:
filepath = os.path.join(self.raw_srtm, code + ".zip")
self.assertTrue(os.path.isfile(filepath))
FileSystem.remove_directory(dem_dir)
def test_srtm_prepare_mnt_vns_maccanw2(self):
resx, resy = 5000, -5000
px, py = 11, 14
site = SiteInfo.Site("MACCANW2",
32633,
px=px,
py=py,
ul=(246439.500, 4672656.500),
lr=(299769.500, 4604231.500),
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(),
"test_srtm_prepare_mnt_vns_maccanw2")
s = SRTM.SRTM(site,
dem_dir=dem_dir,
raw_dem=self.raw_srtm,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
srtm = s.prepare_mnt()
self.assertTrue(os.path.isfile(srtm))
driver = GDalDatasetWrapper.from_file(srtm)
expected_img = [[
293, 210, 337, 390, 238, 218, 237, 245, 270, 208, 132
], [302, 293, 277, 302, 182, 172, 237, 270, 262, 171, 60],
[155, 239, 239, 231, 238, 199, 164, 173, 137, 85, 33],
[0, 28, 94, 138, 150, 131, 116, 117, 72, 37, 73],
[0, 0, 18, 67, 81, 79, 84, 117, 72, 36, 59],
[0, 0, 0, 11, 38, 58, 68, 87, 68, 38, 33],
[0, 0, 0, 0, 6, 41, 52, 62, 52, 43, 41],
[0, 0, 0, 0, 1, 0, 30, 34, 30, 48, 68],
[0, 0, 0, 0, 0, -1, 12, 26, 42, 70, 101],
[0, 0, 0, 0, 0, 4, 11, 50, 59, 89, 136],
[0, 0, 0, 0, 0, 0, 0, 30, 76, 97, 130],
[0, 0, 0, 0, 0, 0, 0, 0, 44, 72, 86],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 59],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 54]]
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.array.shape, (site.py, site.px))
self.assertEqual(driver.nodata_value, 0)
np.testing.assert_allclose(expected_img, driver.array, atol=1.5)
FileSystem.remove_directory(dem_dir)
def test_get_water_data_tls_l8(self):
resx, resy = 15000, -15000
px, py = 15, 14
site = SiteInfo.Site("19080",
32631,
px=px,
py=py,
ul=(285431.584, 4884950.507),
lr=(510124.885, 4680403.373),
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(), "test_get_water_data_tls_l8")
mnt = MNTBase.MNT(site,
dem_dir=dem_dir,
raw_dem=self.raw_dem,
raw_gsw=self.raw_gsw)
water_mask = mnt.gsw_dst
self.assertTrue(os.path.exists(dem_dir))
mnt.prepare_water_data()
self.assertTrue(os.path.exists(water_mask))
driver = GDalDatasetWrapper.from_file(water_mask)
img_expected = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.epsg, 32631)
self.assertEqual(driver.array.shape, (py, px))
self.assertIsNone(driver.nodata_value)
np.testing.assert_almost_equal(img_expected, driver.array)
FileSystem.remove_directory(dem_dir)
def test_get_water_data_maccanw2_vns(self):
resx, resy = 5000, -5000
px, py = 11, 14
site = SiteInfo.Site("MACCANW2",
32633,
px=px,
py=py,
ul=(246439.500, 4672656.500),
lr=(299769.500, 4604231.500),
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(), "test_get_water_data_maccanw2_vns")
mnt = MNTBase.MNT(site,
dem_dir=dem_dir,
raw_dem=self.raw_dem,
raw_gsw=self.raw_gsw)
water_mask = mnt.gsw_dst
self.assertTrue(os.path.exists(dem_dir))
mnt.prepare_water_data()
self.assertTrue(os.path.exists(water_mask))
driver = GDalDatasetWrapper.from_file(water_mask)
img_expected = [[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0],
[1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]]
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.epsg, 32633)
self.assertEqual(driver.array.shape, (py, px))
np.testing.assert_almost_equal(img_expected, driver.array)
self.assertIsNone(driver.nodata_value)
FileSystem.remove_directory(dem_dir)
def test_get_water_data_spain_s2(self):
"""
Download the given gsw file and project it to a 10km x 10km resolution (11x11 image)
"""
resx, resy = 10000, -10000
px, py = 11, 11
site = SiteInfo.Site("T30SYJ",
32630,
px=px,
py=py,
ul=(699960.000, 4400040.000),
lr=(809760.000, 4290240.000),
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(), "test_get_water_data_spain_s2")
mnt = MNTBase.MNT(site,
dem_dir=dem_dir,
raw_dem=self.raw_dem,
raw_gsw=self.raw_gsw)
water_mask = mnt.gsw_dst
self.assertTrue(os.path.exists(dem_dir))
mnt.prepare_water_data()
self.assertTrue(os.path.exists(water_mask))
driver = GDalDatasetWrapper.from_file(water_mask)
img_expected = [[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1]]
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.epsg, 32630)
self.assertEqual(driver.array.shape, (py, px))
np.testing.assert_almost_equal(img_expected, driver.array)
self.assertIsNone(driver.nodata_value)
FileSystem.remove_directory(dem_dir)
def setUpClass(cls):
from StartMaja.Chain import DummyFiles
from StartMaja.Common import FileSystem
cls.product_root = os.path.join(cls.root, cls.tile)
FileSystem.remove_directory(cls.root)
FileSystem.create_directory(cls.root)
FileSystem.create_directory(cls.product_root)
DummyFiles.L1Generator(cls.product_root,
tile=cls.tile,
date=cls.start_product,
platform="sentinel2").generate()
DummyFiles.L1Generator(cls.product_root,
tile=cls.tile,
date=cls.end_product,
platform="sentinel2").generate()
for d in set([DummyFiles.random_date() for _ in range(cls.n_dummies)]):
DummyFiles.L1Generator(cls.product_root, tile=cls.tile, date=d, platform="sentinel2").generate()
DummyFiles.L2Generator(cls.product_root, tile=cls.tile, date=d, platform="sentinel2").generate()
for d in set([DummyFiles.random_date() for _ in range(cls.n_not_used)]):
DummyFiles.L1Generator(cls.product_root, date=d, platform="sentinel2").generate()
DummyFiles.L2Generator(cls.product_root, date=d, platform="sentinel2").generate()
cls.folders_file = os.path.join(cls.root, "test_working_folders_file.txt")
modify_folders_file(cls.template_folders_file, new_file=cls.folders_file,
exeMaja=os.getcwd(),
repWork=os.getcwd(),
repGIPP=os.getcwd(),
repL1=os.getcwd(),
repL2=os.getcwd(),
repMNT=os.getcwd())
cls.mnt = DummyFiles.MNTGenerator(root=cls.root, tile=cls.tile, platform="sentinel2")
cls.mnt.generate()
cls.cams = os.path.join(cls.root, "CAMS")
os.makedirs(cls.cams)
DummyFiles.CAMSGenerator(cls.cams, cls.start_product).generate()
DummyFiles.CAMSGenerator(cls.cams, cls.end_product).generate()
for _ in range(cls.n_not_used):
DummyFiles.CAMSGenerator(cls.cams).generate()
assert os.path.isfile(cls.folders_file)
def test_gsw_download(self):
site = SiteInfo.Site("Ecuador",
32619,
ul=(-250000.000, 250000.000),
lr=(400000.000, -120000.000))
gsw_dir = os.path.join(os.getcwd(), "test_gsw_download")
m = MNTBase.MNT(site,
dem_dir=gsw_dir,
raw_gsw=self.raw_gsw,
raw_dem=self.raw_dem)
fnames = m.get_raw_water_data()
self.assertEqual(len(fnames), 4)
fnames_expected = [
'occurrence_70W_0N_v1_1.tif', 'occurrence_70W_10N_v1_1.tif',
'occurrence_80W_0N_v1_1.tif', 'occurrence_80W_10N_v1_1.tif'
]
for fn, expected in zip(fnames, fnames_expected):
self.assertTrue(os.path.exists(fn))
self.assertEqual(os.path.basename(fn), expected)
FileSystem.remove_directory(gsw_dir)
self.assertFalse(os.path.exists(gsw_dir))
def tearDownClass(cls):
from StartMaja.Common import FileSystem
# In case there's duplicates, remove them:
FileSystem.remove_directory(cls.product_root)
FileSystem.remove_file(cls.folders_file)
FileSystem.remove_directory(cls.cams)
FileSystem.remove_directory(cls.mnt.dbl)
FileSystem.remove_file(cls.mnt.hdr)
def test_reg_s2_natif(self):
tiles = ["29RPQ", "32TMR", "55HBC", "55HBD"]
dates = [
"20170412T110621", "20180316T103021", "20160814T002112",
"20160926T003552"
]
levels = ["l1c", "l1c", "l2a", "l2a"]
for prod, tile, date, level in zip(self.prod_s2_nat, tiles, dates,
levels):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Sentinel2Natif)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "sentinel2")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(os.path.basename(p.metadata_file),
"MTD_MSIL1C.xml")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, True)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_s2_prd + self.prod_s2_ssc + self.prod_s2_mus + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Sentinel2Natif)
def test_srtm_prepare_mnt_s2_tls(self):
resx, resy = 10000, -10000
site = SiteInfo.Site("T31TCJ",
32631,
px=11,
py=11,
ul=(300000.000, 4900020.000),
lr=(409800.000, 4790220.000),
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(), "test_srtm_prepare_mnt_s2_tls")
s = SRTM.SRTM(site,
dem_dir=dem_dir,
raw_dem=self.raw_srtm,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
srtm = s.prepare_mnt()
self.assertTrue(os.path.isfile(srtm))
driver = GDalDatasetWrapper.from_file(srtm)
expected_img = [
[85, 85, 115, 142, 138, 147, 151, 163, 162, 258, 282],
[136, 98, 96, 90, 101, 91, 120, 118, 121, 243, 302],
[137, 133, 135, 137, 101, 89, 92, 131, 162, 246, 294],
[139, 144, 170, 184, 157, 115, 108, 132, 176, 210, 212],
[148, 140, 161, 162, 206, 138, 133, 134, 150, 164, 160],
[169, 173, 168, 171, 222, 168, 128, 158, 162, 151, 164],
[193, 199, 181, 186, 186, 193, 158, 147, 189, 204, 199],
[209, 223, 207, 195, 215, 215, 166, 161, 187, 214, 228],
[240, 247, 225, 206, 268, 211, 172, 203, 195, 213, 242],
[281, 268, 246, 272, 282, 216, 216, 208, 231, 211, 220],
[335, 302, 319, 311, 265, 234, 262, 251, 236, 259, 250]
]
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.array.shape, (site.py, site.px))
self.assertEqual(driver.nodata_value, 0)
np.testing.assert_allclose(expected_img, driver.array, atol=1.5)
FileSystem.remove_directory(dem_dir)
def test_reg_s2_muscate(self):
tiles = ["31TCH", "31TCH", "31UFR"]
levels = ["l1c", "l2a", "l3a"]
dates = ["20171008T105012", "20161206T105012", "20190415T000000"]
validity = [True, False, False]
for prod, tile, date, level, valid in zip(self.prod_s2_mus, tiles,
dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Sentinel2Muscate)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "sentinel2")
self.assertEqual(p.type, "muscate")
self.assertEqual(p.nodata, -10000)
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(
os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml"))
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_s2_prd + self.prod_s2_ssc + self.prod_s2_nat + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Sentinel2Muscate)
def test_eudem_prepare_mnt_s2_tls(self):
resx, resy = 10000, -10000
site = SiteInfo.Site("T31TCJ",
32631,
ul=(300000.000, 4900020.000),
lr=(409800.000, 4790220.000),
px=11,
py=11,
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(), "test_eudem_prepare_mnt_s2_31ucr")
s = EuDEM.EuDEM(site,
dem_dir=dem_dir,
raw_dem=self.raw_eudem,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
srtm = s.prepare_mnt()
self.assertTrue(os.path.isfile(srtm))
driver = GDalDatasetWrapper.from_file(srtm)
expected_img = [[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]]
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.array.shape, (site.py, site.px))
self.assertEqual(driver.nodata_value, 0)
np.testing.assert_allclose(expected_img, driver.array)
FileSystem.remove_directory(dem_dir)
def test_srtm_get_maja_format_tls_l8(self):
site = SiteInfo.Site("19080",
32631,
res_x=90,
res_y=-90,
ul=(285431.584, 4884950.507),
lr=(510124.885, 4680403.373))
dem_dir = os.path.join(os.getcwd(), "test_srtm_get_maja_format_tls_l8")
s = SRTM.SRTM(site,
dem_dir=dem_dir,
raw_dem=self.raw_srtm,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
hdr, dbl = s.to_maja_format(platform_id="L8_",
mission_field="LANDSAT8",
coarse_res=(240, -240),
mnt_resolutions=[{
"name": "R1",
"val": "30 -30"
}])
self.assertTrue(os.path.exists(hdr))
self.assertTrue(os.path.isdir(dbl))
FileSystem.remove_directory(dem_dir)
def test_reg_s2_ssc(self):
tiles = ["36JTT", "21MXT"]
dates = ["20160914T120000", "20180925T120000"]
levels = ["l2a", "l1c"]
validity = [True, True]
for prod, tile, date, level, valid in zip(self.prod_s2_ssc, tiles,
dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Sentinel2SSC)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "sentinel2")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertEqual(os.path.basename(p.metadata_file),
prod.split(".")[0] + ".HDR")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_s2_prd + self.prod_s2_nat + self.prod_s2_mus:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Sentinel2SSC)
def tearDownClass(cls):
FileSystem.remove_directory(cls.raw_eudem)
FileSystem.remove_directory(cls.raw_gsw)
def test_srtm_prepare_mnt_l8_tls(self):
resx, resy = 15000, -15000
px, py = 15, 14
site = SiteInfo.Site("19080",
32631,
px=px,
py=py,
ul=(285431.584, 4884950.507),
lr=(510124.885, 4680403.373),
res_x=resx,
res_y=resy)
dem_dir = os.path.join(os.getcwd(), "test_srtm_prepare_mnt_l8_tls")
s = SRTM.SRTM(site,
dem_dir=dem_dir,
raw_dem=self.raw_srtm,
raw_gsw=self.raw_gsw,
wdir=dem_dir)
self.assertTrue(os.path.isdir(dem_dir))
srtm = s.prepare_mnt()
self.assertTrue(os.path.isfile(srtm))
driver = GDalDatasetWrapper.from_file(srtm)
expected_img = np.array([[
126, 134, 125, 114, 88, 104, 144, 265, 266, 283, 388, 487, 592,
571, 617
],
[
144, 139, 161, 177, 134, 117, 161, 198,
189, 205, 328, 487, 457, 444, 634
],
[
166, 163, 164, 189, 177, 137, 156, 153,
198, 231, 384, 619, 698, 707, 638
],
[
194, 201, 194, 188, 198, 154, 178, 205,
218, 196, 303, 634, 898, 925, 586
],
[
216, 235, 223, 223, 226, 175, 194, 220,
234, 318, 462, 637, 752, 547, 275
],
[
273, 287, 268, 283, 234, 222, 226, 227,
215, 423, 581, 548, 404, 246, 86
],
[
429, 383, 352, 315, 277, 283, 265, 274,
269, 170, 148, 120, 72, 43, 27
],
[
598, 472, 402, 387, 427, 379, 362, 328,
365, 291, 270, 328, 186, 114, 42
],
[
1151, 897, 977, 763, 647, 814, 708, 596,
514, 458, 458, 576, 383, 231, 0
],
[
1591, 1303, 1697, 1728, 1229, 1342, 1085,
1173, 1137, 829, 685, 513, 289, 158, 27
],
[
2261, 2075, 1964, 2052, 1916, 2087, 1923,
1634, 1741, 1576, 1185, 490, 198, 36, -24
],
[
1671, 1898, 2127, 2148, 1475, 1992, 2126,
2301, 2098, 1749, 1235, 1060, 448, 135,
159
],
[
1264, 1283, 1496, 1520, 1529, 1398, 1684,
1735, 1472, 2044, 1949, 1344, 684, 374,
302
],
[
871, 980, 988, 1052, 1401, 1051, 1526,
1479, 1487, 1457, 1253, 753, 567, 179, 32
]])
self.assertEqual(driver.resolution, (resx, resy))
self.assertEqual(driver.array.shape, (site.py, site.px))
self.assertEqual(driver.nodata_value, 0)
np.testing.assert_allclose(expected_img, driver.array, atol=1.5)
FileSystem.remove_directory(dem_dir)
def tearDown(self):
FileSystem.remove_directory(self.wdir)
FileSystem.remove_directory(self.outdir)
FileSystem.remove_directory(self.l1.fpath)
FileSystem.remove_directory(self.l2.fpath)
for p in self.l1_list:
FileSystem.remove_directory(p.fpath)
for p in self.other_l1:
FileSystem.remove_directory(p.fpath)
for p in self.other_l2:
FileSystem.remove_directory(p.fpath)
FileSystem.remove_directory(self.cams.dbl)
FileSystem.remove_file(self.cams.hdr)
FileSystem.remove_directory(self.gipp_dir)
def tearDownClass(cls):
from StartMaja.Common import FileSystem
FileSystem.remove_directory(cls.root)