def check_completeness(self):
"""
Check if the Gipp-folder exists already
:return: True if existing. False if not.
"""
from StartMaja.Common import FileSystem
n_files_per_lut = 4
try:
found_models = sorted(self.get_models())
n_models = len(found_models)
except ValueError:
return False
if not os.path.isdir(self.out_path):
return False
if found_models not in self.expected_models:
return False
try:
hdrs = FileSystem.find("*.HDR", self.out_path)
dbls = FileSystem.find("*.DBL.DIR", self.out_path)
eefs = FileSystem.find("*.EEF", self.out_path)
except ValueError:
return False
if len(eefs) < 4:
return False
# LUTs = 4 (TOCR, DIRT, DIFT, ALBD) + 1 constant for WATV per satellite
if len(hdrs) != len(
dbls
) != n_files_per_lut * self.n_sat * n_models + 1 * self.n_sat:
return False
return True
def link(self, dest):
"""
Symlink a set of Gipps to a given destination
:param dest: The destination directory
:return:
"""
from StartMaja.Common import FileSystem
eefs = FileSystem.find(GIPPFile.regex, self.out_path)
dbls = FileSystem.find(GIPPFile.regex_dbl, self.out_path)
for f in eefs + dbls:
base = os.path.basename(f)
FileSystem.symlink(f, os.path.join(dest, base))
def test_get_file_full(self):
expected = os.path.join(self.root, "b.jpg")
dirnames_e = p.normpath(expected).split(os.sep)
calculated = FileSystem.find(path=self.root, pattern="b.jpg", depth=1)
calculated2 = FileSystem.find(path=self.root,
pattern="b.jpg",
depth=1,
ftype="file")
self.assertEqual(calculated, calculated2)
self.assertEqual(len(calculated), 1)
dirnames_c = p.normpath(calculated[0]).split(os.sep)
for exp, calc in zip(dirnames_c[-1:], dirnames_e[-1:]):
self.assertEqual(exp[:-1], calc[:-1])
self.assertEqual(expected, calculated[0])
def get_dtm(self, type_dem):
"""
Find DTM folder for tile and search for associated HDR and DBL files
A DTM folder has the following naming structure:
*_AUX_REFDE2_TILEID_*DBL.DIR with TILEID e.g. T31TCH, KHUMBU ...
A single .HDR file and an associated .DBL.DIR file
has to be found. OSError is thrown otherwise.
:param type_dem: The DEM-type as str, e.g. 'srtm'
:return: The full path to the hdr and dbl.dir. None if they're not found.
"""
regexes = [
"%s%s_%s.DBL(.DIR)?$" %
(AuxFile.DTMFile.get_specifiable_regex(), self.tile, nbr)
for nbr in AuxFile.DTMFile.mnt_version[type_dem]
]
mnt_folders = []
for regex in regexes:
try:
mnt_folders += FileSystem.find(regex, self.rep_mnt)
except ValueError:
pass
if not mnt_folders:
return None
mnts = [AuxFile.DTMFile(mnt) for mnt in mnt_folders]
mnts = [mnt for mnt in mnts if mnt is not None]
return mnts[0]
def test_get_folder(self):
expected = os.path.join(self.root, "subdir0")
dirnames_e = p.normpath(expected).split(os.sep)
filename = p.basename(expected)
calculated = FileSystem.find(path=self.root,
pattern="subdir0",
depth=1)
calculated2 = FileSystem.find(path=self.root,
pattern="subdir0",
depth=1,
ftype="folder")
self.assertEqual(calculated, calculated2)
self.assertEqual(len(calculated), 1)
dirnames_c = p.normpath(calculated[0]).split(os.sep)
for exp, calc in zip(dirnames_c[-1:], dirnames_e[-1:]):
self.assertEqual(exp[:-1], calc[:-1])
self.assertEqual(filename, p.basename(calculated[0]))
def test_cams_date(self):
from StartMaja.Common import FileSystem
dbl = FileSystem.find("*EXO_CAMS*DBL.DIR", self.cams_dir)[0]
c = CAMSFile(dbl)
base = os.path.basename(dbl).split(".")[0]
date = datetime.strptime(base.split("_")[-2], "%Y%m%dT%H%M%S")
self.assertEqual(c.get_date(), date)
def test_download_s2_muscate_nocams(self):
from StartMaja.Common import FileSystem
g = GippSet(self.root, "sentinel2", "muscate")
self.assertFalse(g.check_completeness())
g.download()
self.assertTrue(g.check_completeness())
n_l2comm = len(FileSystem.find("*L2COMM*", g.out_path))
self.assertEqual(n_l2comm, 2)
n_qltl = len(FileSystem.find("*CKQLTL*", g.out_path))
self.assertEqual(n_qltl, 2)
n_extl = len(FileSystem.find("*CKEXTL*", g.out_path))
self.assertEqual(n_extl, 2)
n_extl = len(FileSystem.find("*EEF", g.out_path))
self.assertEqual(n_extl, 9)
FileSystem.remove_file(os.path.join(self.root, "wget-log"))
if not os.getcwd() == g.out_path:
FileSystem.remove_directory(g.out_path)
self.assertFalse(os.path.exists(g.out_path))
def test_get_file_ending(self):
expected = os.path.join(self.root, "c.xml")
dirnames_e = p.normpath(expected).split(os.sep)
calculated = FileSystem.find(path=self.root, pattern="*xml", depth=1)
self.assertEqual(len(calculated), 1)
dirnames_c = p.normpath(calculated[0]).split(os.sep)
for exp, calc in zip(dirnames_c[-1:], dirnames_e[-1:]):
self.assertEqual(exp[:-1], calc[:-1])
self.assertEqual(expected, calculated[0])
def test_symlink(self):
from StartMaja.Common import FileSystem
g = GippSet(self.root, "sentinel2", "tm")
self.assertFalse(g.check_completeness())
g.download()
self.assertTrue(g.check_completeness())
symlink_dir = os.path.join(self.root, "symlinks")
FileSystem.create_directory(symlink_dir)
with self.assertRaises(ValueError):
self.assertTrue(len(FileSystem.find("*EEF", symlink_dir)), 0)
g.link(symlink_dir)
self.assertEqual(len(FileSystem.find("*EEF", symlink_dir)), 15)
FileSystem.remove_directory(symlink_dir)
self.assertFalse(os.path.isdir(symlink_dir))
FileSystem.remove_file(os.path.join(self.root, "wget-log"))
if not os.getcwd() == g.out_path:
FileSystem.remove_directory(g.out_path)
self.assertFalse(os.path.exists(g.out_path))
def test_get_file_depth2(self):
expected = os.path.join(self.root, "subdir1", "a")
dirnames_e = p.normpath(expected).split(os.sep)
calculated = FileSystem.find(path=self.root, pattern="^a$", depth=2)
self.assertEqual(len(calculated), 3)
dirnames_c = p.normpath(calculated[0]).split(os.sep)
for exp, calc in zip(dirnames_c[-1:], dirnames_e[-1:]):
self.assertEqual(exp[:-1], calc[:-1])
self.assertEqual(expected, sorted(calculated)[-1])
def test_cams_creation(self):
from StartMaja.Common import FileSystem
dbl = FileSystem.find("DBL.DIR", self.cams_dir)[0]
c = CAMSFile(dbl)
self.assertIsNotNone(c)
base = os.path.basename(dbl).split(".")[0]
hdr = os.path.join(os.path.dirname(dbl), base + ".HDR")
self.assertEqual(hdr, c.hdr)
def get_models(self):
"""
Get the list of models present in the gipp-set.
:return: List of models in alphabetical order.
"""
from StartMaja.Common import FileSystem
import re
hdr_reg = os.path.splitext(GIPPFile.regex)[0] + ".HDR"
hdrs = FileSystem.find(hdr_reg, self.out_path, depth=1)
raw_models = [
re.search(hdr_reg, h).group(3).replace("_", "").upper()
for h in hdrs
]
models = list(set(raw_models))
return sorted(models)
def get_list_of_cams_files(input_dir):
"""
Get all available cams files inside a directory tree sorting out the duplicates.
:return: The list of available cams files.
"""
from StartMaja.Common import FileSystem
regex = r"CAMS_(AOT|MR|RH)_(\d{8}UTC\d{6})\.nc$"
raw_files = FileSystem.find(regex, input_dir)
unique_files, basenames = [], []
# Sort out duplicates by basename
for i in raw_files:
bname = os.path.basename(i)
if bname not in basenames:
basenames.append(bname)
unique_files.append(i)
return unique_files
def run_tiling(ds_or_path, **kwargs):
"""
Run the tiling process
:param ds_or_path: The :class:`gdal.Dataset` or a path to a file on disk to be tiled
:type ds_or_path: :class:`gdal.Dataset` or str
:param kwargs: Optional arguments.
:return: Creates the tiles in the given location.
"""
cmd_dict = {"Verbose": "-v", "Quiet": "-q", "CreationOptions": "-co", "TileSize": "-ps",
"Overlap": "-overlap", "Format": "-of", "BandType": "-ot", "TileIndexFieldName": "-tileIndexField",
"TileIndexName": "-tileIndex", "CsvDelimiter": "-csvDelim", "Source_SRS": "-s_srs",
"TargetDir": "-targetDir", "-ResamplingMethod": "-r", "Levels": "-levels",
"PyramidOnly": "-pyramidOnly", "UseDirForEachRow": "-useDirForEachRow"}
gdal.AllRegister()
gdal.UseExceptions()
options = []
if type(ds_or_path) != str:
tmppath = os.path.join(tempfile.gettempdir(), next(tempfile._get_candidate_names()))
ds_or_path.write(tmppath)
del_tmp = True
else:
tmppath = ds_or_path
del_tmp = False
tx, ty = None, None
for key in kwargs.keys():
if key == "TileWidth":
ty = kwargs[key]
elif key == "TileHeight":
tx = kwargs[key]
else:
options += [cmd_dict[key], kwargs[key]]
if not tx or not ty:
raise ValueError("Must provide tile width and height: (%s, %s)" % (ty, tx))
options += [cmd_dict["TileSize"], "%s %s" % (ty, tx)]
options += [tmppath]
assert FileSystem.run_external_app("gdal_retile.py", options) == 0, "Error running gdal_retile"
bname = os.path.basename(tmppath)
outpath = kwargs["TargetDir"]
# TODO Discard previously written files from `find`:
files_written = sorted(FileSystem.find(r"%s_\d+_\d+.tif" % bname, path=outpath))
if del_tmp:
FileSystem.remove_file(tmppath)
return files_written
def get_cams_files(self):
"""
Find all associated CAMS- HDR and DBL files
A CAMS folder has the following naming structure:
MMM_TEST_EXO_CAMS_YYYYMMDDThhmmss_YYYYMMDDThhmmss
with MMM = mission (see regex tests)
For each CAMS a single .HDR file and an associated .DBL.DIR/.DBL file
has to be found. Otherwise it gets discarded
"""
try:
cams_folders = FileSystem.find(AuxFile.CAMSFile.regex,
self.rep_cams)
except ValueError:
return []
cams = [AuxFile.CAMSFile(c) for c in cams_folders]
cams = [c for c in cams if c is not None]
return cams
def test_wrong_cams_creation(self):
from StartMaja.Common import FileSystem
dbl = FileSystem.find("*AUX_REFDE2*DBL.DIR", self.root)[0]
self.assertIsNone(CAMSFile(dbl))
def test_wrong_mnt_creation(self):
from StartMaja.Common import FileSystem
dbl = FileSystem.find("DBL.DIR", self.cams_dir)[0]
self.assertIsNone(DTMFile(dbl))
def test_find_get_subsubfolder(self):
expected = "subdir1"
calculated = FileSystem.find(path=self.root, pattern=expected)
self.assertEqual(expected, p.basename(calculated[0]))
def test_find_get_file(self):
expected = "a.jpg"
calculated = FileSystem.find(path=self.root, pattern=expected)
self.assertEqual(expected, p.basename(calculated[0]))