def test_reg_l8_lc2(self):
tiles = ["199029"]
dates = ["20170527T120000"]
levels = ["l1c"]
for prod, tile, date, level in zip(self.prod_l8_lc2, tiles, dates,
levels):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Landsat8LC2)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "landsat8")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.nodata, 0)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertEqual(os.path.basename(p.metadata_file),
prod.split(".")[0] + "_MTL.txt")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, True)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
'name': 'XS',
'val': '30 -30'
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_l8_lc1 + self.prod_l8_nat + self.prod_l8_mus + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Landsat8LC2)
def test_reg_vs_natif(self):
tiles = ["ISRAW906", "UNH", "SUDOUE_6", "CHILE"]
dates = ["20180317T120000", "20180329T120000", "20191110T120000", "20200311T120000"]
levels = ["l2a", "l1c", "l1c", "l1c"]
for prod, tile, date, level in zip(self.prod_vs_nat, tiles, dates, levels):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, VenusNatif)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "venus")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertEqual(os.path.basename(p.metadata_file), prod.split(".")[0] + ".HDR")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, True)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_vs_mus + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, VenusNatif)
def test_reg_vs_muscate(self):
tiles = ["KHUMBU", "KHUMBU", "KHUMBU"]
levels = ["l1c", "l2a", "l3a"]
dates = ["20180201T051359", "20180201T051359", "20180201T000000"]
validity = [True, False, False]
for prod, tile, date, level, valid in zip(self.prod_vs_mus, tiles, dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, VenusMuscate)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "venus")
self.assertEqual(p.type, "muscate")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml"))
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{'name': 'XS', 'val': '5 -5'}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_vs_nat + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, VenusMuscate)
def test_gdal_tile_untile(self):
img = np.arange(0., 100.).reshape((10, 10))
path = os.path.join(os.getcwd(), "test_gdal_retile.tif")
tile_folder = os.path.join(os.getcwd(), "tiled")
ImageIO.write_geotiff(img, path, self.projection, self.coordinates)
# Add parasitic file - It should not cause problems
path_parasite = os.path.join(tile_folder, "tile_01_01.tif")
FileSystem.create_directory(tile_folder)
ImageIO.write_geotiff(img, path_parasite, self.projection, self.coordinates)
ds_in = GDalDatasetWrapper.from_file(path)
self.assertTrue(os.path.exists(path))
tiles = ImageTools.gdal_retile(ds_in, tile_folder,
TileWidth=2,
TileHeight=2,
Overlap=1)
self.assertTrue(os.path.isdir(tile_folder))
self.assertEqual(len(tiles), 81)
img_read = np.array(ImageIO.tiff_to_array(tiles[-1]))
expected = np.array([[88, 89],
[98, 99]])
# Some gdal_retile versions are producing the following image:
# [[87, 89], [97, 99]].
np.testing.assert_allclose(expected, img_read, atol=1)
# Untile
ds_untiled = ImageTools.gdal_buildvrt(*tiles)
np.testing.assert_allclose(img, ds_untiled.array, atol=1)
FileSystem.remove_file(path)
FileSystem.remove_directory(tile_folder)
def generate(self, **kwargs):
import random
from StartMaja.Chain import Product
from StartMaja.Common import TestFunctions, FileSystem
platform_specifier = random.choice(
self.platform_options["L1C"][self.platform])
orbit = kwargs.get("orbit", random.randint(0, 999))
version_orbit = kwargs.get("version", random.randint(0, 9))
if "sentinel2" in self.platform:
date_str = self.date.strftime("%Y%m%dT%H%M%S")
product_name = "_".join([
platform_specifier, "MSIL1C", date_str,
"N" + str(orbit).zfill(4), "R" + str(version_orbit).zfill(3),
self.tile, date_str + ".SAFE"
])
product_path = os.path.join(self.root, product_name)
metadata_path = os.path.join(product_path, "MTD_MSIL1C.xml")
else:
date_str = self.date.strftime("%Y%m%d-%H%M%S")
product_name = "_".join([
platform_specifier, date_str + "-000", "L2A", self.tile,
random.choice("DC"),
"V" + str(version_orbit) + "-" + str(version_orbit)
])
product_path = os.path.join(self.root, product_name)
metadata_path = os.path.join(product_path,
product_name + "_MTD_ALL.xml")
self.prod = product_path
self.mtd = metadata_path
FileSystem.create_directory(product_path)
TestFunctions.touch(metadata_path)
return Product.MajaProduct.factory(self.prod)
def test_reg_l8_muscate(self):
tiles = ["31TCH", "31TCH"]
levels = ["l1c", "l2a"]
dates = ["20170501T103532", "20170501T103532"]
validity = [True, False]
for prod, tile, date, level, valid in zip(self.prod_l8_mus, tiles,
dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Landsat8Muscate)
self.assertEqual(p.level, level)
self.assertEqual(p.nodata, 0)
self.assertEqual(p.platform, "landsat8")
self.assertEqual(p.type, "muscate")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(
os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml"))
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
'name': 'XS',
'val': '30 -30'
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_l8_lc1 + self.prod_l8_lc2 + self.prod_l8_nat + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Landsat8Muscate)
def test_create_remove_dir(self):
path = os.path.join(os.getcwd(), "test_create_remove_dir")
# This throws a log message
FileSystem.create_directory(path)
self.assertTrue(os.path.isdir(path))
FileSystem.remove_directory(path)
self.assertFalse(os.path.isdir(path))
self.assertFalse(os.path.exists(path))
def generate(self, **kwargs):
from datetime import datetime
import random
from StartMaja.Common import FileSystem
from StartMaja.Chain.GippFile import GippSet
mission_param = kwargs.get("mission",
random.choice(["muscate", "natif"]))
if mission_param == "tm":
self.platform = "sentinel2"
mission = self.mission_choices[mission_param][self.platform]
satellites = [self.mission_short[self.platform]
] if self.platform != "sentinel2" else ["S2A", "S2B"]
with_cams = kwargs.get("cams", True)
cams_suffix = "_CAMS" if with_cams else ""
out_path = os.path.join(
self.root, "_".join([self.platform.upper(),
mission_param.upper()]) + cams_suffix)
FileSystem.create_directory(out_path)
if with_cams:
models = ["CONTINEN"] + [
"ORGANICM", "BLACKCAR", "DUST", "SEASALT", "SULPHATE"
]
else:
models = ["CONTINEN"]
allsites = "ALLSITES"
hdr_types = ["ALBD", "DIFT", "DIRT", "TOCR", "WATV"]
eef_types = ["COMM", "SITE", "SMAC", "EXTL", "QLTL"]
tm_types = ["COMM", "EXTL", "QLTL"]
version = random.randint(0, 9999)
version_str = str(version).zfill(5)
start_date = datetime(2014, 12, 30)
for sat in satellites:
for name in eef_types:
self._create_hdr(out_path, sat, name, start_date, version_str,
allsites, mission, ".EEF")
for name in hdr_types:
for model in models:
basename = self._create_hdr(out_path, sat, name,
start_date, version_str, model,
mission, ".HDR")
dbl_name = os.path.join(out_path, basename + ".DBL.DIR")
FileSystem.create_directory(dbl_name)
# For TM: Add an additional set of COMM, EXTL and QLTL files with muscate mission:
for name in tm_types:
if mission_param != "tm":
continue
tm_mission = "SENTINEL2"
tm_version_str = str(version + 10000).zfill(5)
self._create_hdr(out_path, sat, name, start_date,
tm_version_str, allsites, tm_mission, ".EEF")
return GippSet(self.root, self.platform, mission_param, cams=with_cams)
def reproject(self, **kwargs):
out_dir = kwargs.get("out_dir", self.fpath)
assert os.path.isdir(out_dir)
out_dir = os.path.join(out_dir, self.base)
FileSystem.create_directory(out_dir)
patterns = kwargs.get("patterns", [r".(tif|jp2)$"])
imgs = [self.find_file(pattern=p) for p in patterns]
epsg = kwargs.get("epsg", None)
# Flatten
imgs = [i for img in imgs for i in img]
for img in imgs:
if not epsg:
drv = GDalDatasetWrapper.from_file(img)
epsg = drv.epsg
outpath = os.path.join(out_dir, os.path.basename(img))
self._reproject_to_epsg(img, outpath, epsg)
return out_dir
def generate(self, **kwargs):
import random
from StartMaja.Common import FileSystem
from StartMaja.Chain.AuxFile import DTMFile
mission_param = kwargs.get("mission", self.get_mission())
mission_specifier = "_" if self.platform == "sentinel2" else ""
basename = "_".join([
self.mission_short[self.platform] + mission_specifier, "TEST",
"AUX", "REFDE2", self.tile,
str(random.randint(0, 1000)).zfill(4)
])
self.dbl = os.path.join(self.root, basename + ".DBL.DIR")
self.hdr = os.path.join(self.root, basename + ".HDR")
FileSystem.create_directory(self.dbl)
self.create_dummy_hdr(self.hdr,
mission=mission_param + mission_specifier)
return DTMFile(self.dbl)
def test_symlink(self):
from StartMaja.Common import FileSystem
g = GippSet(self.root, "sentinel2", "tm")
self.assertFalse(g.check_completeness())
g.download()
self.assertTrue(g.check_completeness())
symlink_dir = os.path.join(self.root, "symlinks")
FileSystem.create_directory(symlink_dir)
with self.assertRaises(ValueError):
self.assertTrue(len(FileSystem.find("*EEF", symlink_dir)), 0)
g.link(symlink_dir)
self.assertEqual(len(FileSystem.find("*EEF", symlink_dir)), 15)
FileSystem.remove_directory(symlink_dir)
self.assertFalse(os.path.isdir(symlink_dir))
FileSystem.remove_file(os.path.join(self.root, "wget-log"))
if not os.getcwd() == g.out_path:
FileSystem.remove_directory(g.out_path)
self.assertFalse(os.path.exists(g.out_path))
def generate(self, **kwargs):
from datetime import datetime
from StartMaja.Common import FileSystem
from StartMaja.Chain.AuxFile import CAMSFile
end_date = datetime(2099, 1, 1, 23, 59, 59)
mission_param = kwargs.get("mission", self.get_mission())
mission_specifier = "_" if self.platform == "sentinel2" else ""
basename = "_".join([
self.mission_short[self.platform], "TEST", "EXO", "CAMS",
self.date.strftime("%Y%m%dT%H%M%S"),
end_date.strftime("%Y%m%dT%H%M%S")
])
self.dbl = os.path.join(self.root, basename + ".DBL.DIR")
self.hdr = os.path.join(self.root, basename + ".HDR")
FileSystem.create_directory(self.dbl)
self.create_dummy_hdr(self.hdr,
mission=mission_param + mission_specifier)
return CAMSFile(self.dbl)
def setUpClass(cls):
from StartMaja.Chain import DummyFiles
from StartMaja.Common import FileSystem
cls.product_root = os.path.join(cls.root, cls.tile)
FileSystem.remove_directory(cls.root)
FileSystem.create_directory(cls.root)
FileSystem.create_directory(cls.product_root)
DummyFiles.L1Generator(cls.product_root,
tile=cls.tile,
date=cls.start_product,
platform="sentinel2").generate()
DummyFiles.L1Generator(cls.product_root,
tile=cls.tile,
date=cls.end_product,
platform="sentinel2").generate()
for d in set([DummyFiles.random_date() for _ in range(cls.n_dummies)]):
DummyFiles.L1Generator(cls.product_root, tile=cls.tile, date=d, platform="sentinel2").generate()
DummyFiles.L2Generator(cls.product_root, tile=cls.tile, date=d, platform="sentinel2").generate()
for d in set([DummyFiles.random_date() for _ in range(cls.n_not_used)]):
DummyFiles.L1Generator(cls.product_root, date=d, platform="sentinel2").generate()
DummyFiles.L2Generator(cls.product_root, date=d, platform="sentinel2").generate()
cls.folders_file = os.path.join(cls.root, "test_working_folders_file.txt")
modify_folders_file(cls.template_folders_file, new_file=cls.folders_file,
exeMaja=os.getcwd(),
repWork=os.getcwd(),
repGIPP=os.getcwd(),
repL1=os.getcwd(),
repL2=os.getcwd(),
repMNT=os.getcwd())
cls.mnt = DummyFiles.MNTGenerator(root=cls.root, tile=cls.tile, platform="sentinel2")
cls.mnt.generate()
cls.cams = os.path.join(cls.root, "CAMS")
os.makedirs(cls.cams)
DummyFiles.CAMSGenerator(cls.cams, cls.start_product).generate()
DummyFiles.CAMSGenerator(cls.cams, cls.end_product).generate()
for _ in range(cls.n_not_used):
DummyFiles.CAMSGenerator(cls.cams).generate()
assert os.path.isfile(cls.folders_file)
def setUpClass(cls):
# Note those directories are not destroyed after executing tests.
# This is in order to avoid multiple downloads amongst different test classes.
FileSystem.create_directory(cls.raw_gsw)
FileSystem.create_directory(cls.raw_eudem)
# Setup dummy EuDEM file of reduced resolution
arr = np.zeros((400, 400), dtype=np.float32)
to_zip = [
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N20.TIF"),
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N20.TIF.ovr"),
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N20.TIF.aux.xml"),
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N20.TFw")
]
write_geotiff(arr, to_zip[0], cls.projection, cls.geotransform_e30n20)
[TestFunctions.touch(f) for f in to_zip[1:]]
with zipfile.ZipFile(
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N20.zip"),
'w') as zip_archive:
for file in to_zip:
zip_archive.write(file, compress_type=zipfile.ZIP_DEFLATED)
FileSystem.remove_file(to_zip[0])
# Setup dummy EuDEM file of reduced resolution
arr = np.ones((400, 400), dtype=np.float32)
# Add some other value to around half of the image
arr[:, 260:] = 10
to_zip = [
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N30.TIF"),
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N30.TIF.ovr"),
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N30.TIF.aux.xml"),
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N30.TFw")
]
write_geotiff(arr, to_zip[0], cls.projection, cls.geotransform_e30n30)
[TestFunctions.touch(f) for f in to_zip[1:]]
with zipfile.ZipFile(
os.path.join(cls.raw_eudem, "eu_dem_v11_E30N30.zip"),
'w') as zip_archive:
for file in to_zip:
zip_archive.write(file, compress_type=zipfile.ZIP_DEFLATED)
FileSystem.remove_file(to_zip[0])
def generate(self, **kwargs):
import random
from StartMaja.Chain import Product
from StartMaja.Common import TestFunctions, FileSystem
platform_specifier = random.choice(
self.platform_options["L2A"][self.platform])
ms = kwargs.get("ms", random.randint(0, 999))
version = kwargs.get("version", random.randint(0, 9))
date_str = self.date.strftime("%Y%m%d-%H%M%S-") + str(ms).zfill(3)
product_name = "_".join([
platform_specifier, date_str, "L2A", self.tile,
random.choice("DC"), "V" + str(version) + "-" + str(version)
])
product_path = os.path.join(self.root, product_name)
self.prod = product_path
metadata_path = os.path.join(product_path,
product_name + "_MTD_ALL.xml")
self.mtd = metadata_path
FileSystem.create_directory(product_path)
TestFunctions.touch(metadata_path)
return Product.MajaProduct.factory(self.prod)
def create_archive(output_dir, netcdf, output_file_basename):
"""
Create the DBL.DIR folder
:param output_dir: The output directory
:param netcdf: The list of netcdf files
:param output_file_basename: The basename of the cams file of format .._EXO_CAMS_...
:return: The DBL folder path as well as the relative path to the cams files.
"""
import shutil
from StartMaja.Common import FileSystem
destination_filename = "{}.DBL".format(output_file_basename)
destination_filepath = os.path.join(output_dir, destination_filename)
dbl_dir = os.path.join(output_dir, output_file_basename + ".DBL.DIR")
FileSystem.create_directory(dbl_dir)
cams_file_to_return = []
for ncf in netcdf:
shutil.copy(ncf, os.path.join(dbl_dir, os.path.basename(ncf)))
cams_file_to_return.append(
os.path.join(destination_filename + ".DIR",
os.path.basename(ncf)))
return destination_filepath, cams_file_to_return
def setUp(self):
self.wdir = os.path.join(os.getcwd(), "wp_wdir")
self.outdir = os.path.join(os.getcwd(), "wp_outdir")
self.l1 = DummyFiles.L1Generator(root=os.getcwd(),
tile="T11ABC",
platform="sentinel2").generate()
self.l1_list = [
DummyFiles.L1Generator(root=os.getcwd(),
tile="T11ABC",
platform="sentinel2").generate()
for _ in range(3)
]
self.other_l1 = [
DummyFiles.L1Generator(root=os.getcwd()).generate()
for _ in range(3)
]
self.l2 = DummyFiles.L2Generator(root=self.outdir,
tile="T11ABC",
platform="sentinel2").generate()
self.other_l2 = [
DummyFiles.L2Generator(root=self.outdir).generate()
for _ in range(3)
]
self.gipp_dir = os.path.join(os.getcwd(), "gipp_dir")
FileSystem.create_directory(self.gipp_dir)
self.gipp = DummyFiles.GippGenerator(root=self.gipp_dir,
platform="sentinel2").generate()
self.cams = DummyFiles.CAMSGenerator(root=os.getcwd(),
date=self.l1.date,
platform="sentinel2").generate()
FileSystem.create_directory(self.wdir)
FileSystem.create_directory(self.outdir)
def test_reg_s2_natif(self):
tiles = ["29RPQ", "32TMR", "55HBC", "55HBD"]
dates = [
"20170412T110621", "20180316T103021", "20160814T002112",
"20160926T003552"
]
levels = ["l1c", "l1c", "l2a", "l2a"]
for prod, tile, date, level in zip(self.prod_s2_nat, tiles, dates,
levels):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Sentinel2Natif)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "sentinel2")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(os.path.basename(p.metadata_file),
"MTD_MSIL1C.xml")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, True)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_s2_prd + self.prod_s2_ssc + self.prod_s2_mus + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Sentinel2Natif)
def test_reg_s2_muscate(self):
tiles = ["31TCH", "31TCH", "31UFR"]
levels = ["l1c", "l2a", "l3a"]
dates = ["20171008T105012", "20161206T105012", "20190415T000000"]
validity = [True, False, False]
for prod, tile, date, level, valid in zip(self.prod_s2_mus, tiles,
dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Sentinel2Muscate)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "sentinel2")
self.assertEqual(p.type, "muscate")
self.assertEqual(p.nodata, -10000)
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertTrue(
os.path.basename(p.metadata_file).endswith("_MTD_ALL.xml"))
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_s2_prd + self.prod_s2_ssc + self.prod_s2_nat + self.prods_other:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Sentinel2Muscate)
def test_reg_s2_ssc(self):
tiles = ["36JTT", "21MXT"]
dates = ["20160914T120000", "20180925T120000"]
levels = ["l2a", "l1c"]
validity = [True, True]
for prod, tile, date, level, valid in zip(self.prod_s2_ssc, tiles,
dates, levels, validity):
p = MajaProduct.factory(prod)
self.assertIsInstance(p, Sentinel2SSC)
self.assertEqual(p.level, level)
self.assertEqual(p.platform, "sentinel2")
self.assertEqual(p.type, "natif")
self.assertEqual(p.tile, tile)
self.assertEqual(p.date.strftime("%Y%m%dT%H%M%S"), date)
self.assertEqual(os.path.basename(p.metadata_file),
prod.split(".")[0] + ".HDR")
self.assertTrue(os.path.exists(p.metadata_file))
self.assertEqual(p.validity, valid)
link_dir = "linkdir"
FileSystem.create_directory(link_dir)
p.link(link_dir)
self.assertTrue(os.path.islink(os.path.join(link_dir, p.base)))
self.assertEqual(p.mnt_resolutions_dict, [{
"name": "R1",
"val": "10 -10"
}, {
"name": "R2",
"val": "20 -20"
}])
self.assertEqual(p, p)
FileSystem.remove_directory(link_dir)
# Other prods:
for prod in self.prod_s2_prd + self.prod_s2_nat + self.prod_s2_mus:
p = MajaProduct.factory(prod)
self.assertNotIsInstance(p, Sentinel2SSC)
def __init__(self, root, platform, gtype, cams=False, **kwargs):
"""
Set the path to the root gipp folder
:param root: The full path to the root gipp folder
:param platform: The platform name. Has to be in ["sentinel2", "landsat8", "venus"]
:param gtype: The gipp type. Has to be in ["muscate", "natif", "tm"]
:param cams: Build GIPP with CAMS models
:param log_level: The log level for the messages displayed.
"""
from StartMaja.Common import FileSystem
self.fpath = os.path.realpath(root)
if platform not in self.platforms:
raise ValueError("Unknown platform found: %s" % platform)
if gtype not in self.plugins[platform]:
raise ValueError("No Plugin of type %s existing for platform %s" %
(gtype, platform))
if gtype == "tm" and platform in ["landsat8", "venus"]:
self.gtype = "natif"
else:
self.gtype = gtype
self.platform = platform
self.cams_suffix = "_CAMS" if cams else ""
self.log_level = kwargs.get("log_level", logging.INFO)
self.lut_date = kwargs.get("date", datetime.now())
self.cams_46r1 = datetime(2019, 7, 10)
self.n_sat = 2 if platform == "sentinel2" else 1
# Create root if not existing:
FileSystem.create_directory(self.fpath)
# Create folder names:
self.gipp_archive = os.path.join(self.fpath, "archive.zip")
self.lut_archive = os.path.join(self.fpath, "lut_archive.zip")
self.temp_folder = os.path.join(self.fpath, "tempdir")
self.gipp_folder_name = "%s_%s" % (
self.platform.upper(), self.gtype.upper()) + self.cams_suffix
self.out_path = os.path.join(self.fpath, self.gipp_folder_name)
def get_synthetic_band(self, synthetic_band, **kwargs):
wdir = kwargs.get("wdir", self.fpath)
output_folder = os.path.join(wdir, self.base)
output_bname = "_".join(
[self.base.split(".")[0],
synthetic_band.upper() + ".tif"])
output_filename = kwargs.get("output_filename",
os.path.join(output_folder, output_bname))
max_value = kwargs.get("max_value", 10000.)
# Skip existing:
if os.path.exists(output_filename):
return output_filename
if synthetic_band.lower() == "ndvi":
FileSystem.create_directory(output_folder)
b4 = self.find_file(pattern=r"*B0?4(_10m)?.jp2$")[0]
b8 = self.find_file(pattern=r"*B0?8(_10m)?.jp2$")[0]
ds_red = GDalDatasetWrapper.from_file(b4)
ds_nir = GDalDatasetWrapper.from_file(b8)
ds_ndvi = ImageApps.get_ndvi(ds_red,
ds_nir,
vrange=(0, max_value),
dtype=np.int16)
ds_ndvi.write(output_filename, options=["COMPRESS=DEFLATE"])
elif synthetic_band.lower() == "ndsi":
FileSystem.create_directory(output_folder)
b3 = self.find_file(pattern=r"*B0?3(_10m)?.jp2$")[0]
b11 = self.find_file(pattern=r"*B11(_20m)?.jp2$")[0]
ds_green = ImageTools.gdal_translate(b3, tr="20 20", r="cubic")
ds_swir = GDalDatasetWrapper.from_file(b11)
ds_ndsi = ImageApps.get_ndsi(ds_green,
ds_swir,
vrange=(0, max_value),
dtype=np.int16)
ds_ndsi.write(output_filename, options=["COMPRESS=DEFLATE"])
elif synthetic_band.lower() == "mca_sim":
FileSystem.create_directory(output_folder)
b4 = self.find_file(pattern=r"*B0?4(_10m)?.jp2$")[0]
b3 = self.find_file(pattern=r"*B0?3(_10m)?.jp2$")[0]
img_red, drv = ImageIO.tiff_to_array(b4, array_only=False)
img_green = ImageIO.tiff_to_array(b3)
img_mcasim = (img_red + img_green) / 2
ImageIO.write_geotiff_existing(img_mcasim,
output_filename,
drv,
options=["COMPRESS=DEFLATE"])
else:
raise ValueError("Unknown synthetic band %s" % synthetic_band)
return output_filename
def __set_input_paths(self):
"""
Set the available L1 and L2 input paths.
:return: The L1-, L2- folders (creating the latter if needed).
An info-string whether site and/or tile is used.
"""
if self.site:
site_l1 = FileSystem.find_single(r"^%s$" % self.site, self.rep_l1)
self.logger.warning(self.tile)
path_input_l1 = FileSystem.find_single(r"^T?%s$" % self.tile,
site_l1)
try:
site_l2 = FileSystem.find_single(r"^%s$" % self.site,
self.rep_l2)
except ValueError:
site_l2 = os.path.join(self.rep_l2, self.site)
FileSystem.create_directory(site_l2)
try:
path_input_l2 = FileSystem.find_single(r"^T?%s$" % self.tile,
site_l2)
except ValueError:
path_input_l2 = os.path.join(self.rep_l2, self.site, self.tile)
FileSystem.create_directory(path_input_l2)
site_info = "site %s and tile %s" % (self.site, self.tile)
else:
path_input_l1 = FileSystem.find_single(r"^T?%s$" % self.tile,
self.rep_l1)
try:
path_input_l2 = FileSystem.find_single(r"^T?%s$" % self.tile,
self.rep_l2)
except ValueError:
path_input_l2 = os.path.join(self.rep_l2, self.tile)
FileSystem.create_directory(path_input_l2)
site_info = "tile %s" % self.tile
return path_input_l1, path_input_l2, site_info
time = ["00", "12"]
# Step du forecast voulu
# step = 3 indique qu'on telecharge les previsions a 3h apres l'heure de l'analyse.
# Exemples : time = 00 et step = 3 => 03:00:00 UTC
# time = 12 et step = 3 => 15:00:00 UTC
step_choice = "3"
# Choix des fichiers a telecharger
# Surface : AOT (aerosol optical thickness)
# Pressure : RH (relative humidity)
# Model : MR (mixing ratios)
ftype = {'surface': True, 'pressure': True, 'model': True}
# Create directories
from StartMaja.Common import FileSystem
FileSystem.create_directory(args.archive_dir)
FileSystem.create_directory(args.write_dir)
# Boucle sur les jours a telecharger
for i in range(nb_days):
date = dt1 + datetime.timedelta(days=i)
print("==================================")
print("Downloading files for date %s" % date)
print("==================================")
for t in range(len(time)):
aot, mr, rh = download_files(date, ftype, time[t], step_choice,
args.write_dir)
# Conversion to MAJA DBL/HDR format
RawCAMSArchive.process_one_file(args.archive_dir, aot, rh, mr,
args.platform)
if not args.keep:
def setUpClass(cls):
from StartMaja.Common import FileSystem
FileSystem.create_directory(cls.root)
def parse_config(self, cfg_file):
"""
Read contents of the config/folders.txt file containing:
Required params:
repWork, repL1, repL2, repMNT, exeMaja
Optional params:
repCAMS
:param cfg_file: The path to the file
:return: The parsed paths for each of the directories. None for the optional ones if not given.
"""
import configparser as cfg
# Parsing configuration file
config = cfg.ConfigParser()
config.read(cfg_file)
# Maja_Inputs
rep_work = os.path.realpath(
os.path.expanduser(config.get("Maja_Inputs", "repWork")))
if not p.isdir(rep_work):
FileSystem.create_directory(rep_work)
rep_gipp = os.path.realpath(
os.path.expanduser(config.get("Maja_Inputs", "repGipp")))
if not p.isdir(rep_gipp):
FileSystem.create_directory(rep_gipp)
rep_l1 = StartMaja.__read_config_param(config, "Maja_Inputs", "repL1")
rep_l2 = os.path.realpath(
os.path.expanduser(config.get("Maja_Inputs", "repL2")))
if not p.isdir(rep_l2):
FileSystem.create_directory(rep_l2)
exe_maja = StartMaja.__read_config_param(config, "Maja_Inputs",
"exeMaja")
rep_mnt = os.path.realpath(
os.path.expanduser(config.get("Maja_Inputs", "repMNT")))
if not p.isdir(rep_mnt):
FileSystem.create_directory(rep_mnt)
# CAMS is optional:
try:
rep_cams = os.path.realpath(
os.path.expanduser(config.get("Maja_Inputs", "repCAMS")))
except cfg.NoOptionError:
self.logger.warning("repCAMS is missing. Processing without CAMS")
rep_cams = None
# DTM_Creation
rep_raw = os.path.realpath(
os.path.expanduser(config.get("DTM_Creation", "repRAW")))
if not p.isdir(rep_raw):
FileSystem.create_directory(rep_raw)
rep_gsw = os.path.realpath(
os.path.expanduser(config.get("DTM_Creation", "repGSW")))
if not p.isdir(rep_gsw):
FileSystem.create_directory(rep_gsw)
return rep_work, rep_gipp, rep_l1, rep_l2, exe_maja, rep_cams, rep_mnt, rep_raw, rep_gsw
def setUpClass(cls):
# Note those directories are not destroyed after executing tests.
# This is in order to avoid multiple downloads amongst different test classes.
FileSystem.create_directory(cls.raw_gsw)
FileSystem.create_directory(cls.raw_dem)
