def download(self):
"""
Download a specific set of Gipps to the given folder.
First, attempt to download the most recent git archive containing the .EEF files as well as the
url to download the LUTs. Then, the latter will be downloaded separately.
:return:
"""
import shutil
from StartMaja.Common import FileSystem
self.out_path = os.path.join(self.fpath, self.gipp_folder_name)
FileSystem.download_file(self.url, self.gipp_archive, self.log_level)
FileSystem.unzip(self.gipp_archive, self.temp_folder)
gipp_maja_git = os.path.join(self.temp_folder, "maja-gipp-develop")
# TODO Remove second WATV for Venus-Natif
platform_folder = FileSystem.find_single(path=gipp_maja_git,
pattern="^" +
self.gipp_folder_name + "$")
if not platform_folder:
self.__clean_up()
raise OSError("Cannot find any gipp folder for platform %s" %
self.gipp_folder_name)
readme = FileSystem.find_single(path=platform_folder,
pattern="readme*")
if not readme:
self.__clean_up()
raise OSError("Cannot find download-file for LUT-Download in %s" %
platform_folder)
lut_url = FileSystem.find_in_file(readme, self.zenodo_reg)
if not lut_url:
self.__clean_up()
raise OSError("Cannot find url to download LUTs")
FileSystem.download_file(lut_url, self.lut_archive, self.log_level)
FileSystem.unzip(self.lut_archive, platform_folder)
lut_folder = FileSystem.find_single(path=platform_folder,
pattern="LUTs")
if not lut_folder:
self.__clean_up()
raise OSError("Cannot find 'LUTs' folder in %s" % self.temp_folder)
for f in os.listdir(lut_folder):
shutil.move(os.path.join(lut_folder, f), platform_folder)
if os.path.isdir(self.out_path):
FileSystem.remove_directory(self.out_path)
shutil.move(platform_folder, self.out_path)
self.__clean_up()
FileSystem.remove_directory(lut_folder)
# Sanity check:
if not self.check_completeness():
raise ValueError(
"GIPP download failed. Please delete gipp/ folder and try again."
)
def mnt_site(self):
try:
band_b2 = FileSystem.find_single(pattern=r"*B0?2(_10m)?.jp2$",
path=self.fpath)
except IOError as e:
raise e
return Site.from_raster(self.tile, band_b2)
def prepare_mnt(self):
"""
Prepare the srtm files.
:return: Path to the full resolution DEM file.gsw
:rtype: str
"""
# Find/Download SRTM archives:
srtm_archives = self.get_raw_data()
# Unzip the downloaded/found srtm zip files:
unzipped = []
for arch in srtm_archives:
basename = os.path.splitext(os.path.basename(arch))[0]
FileSystem.unzip(arch, self.wdir)
fn_unzipped = FileSystem.find_single(pattern=basename + ".tif", path=self.wdir)
unzipped.append(fn_unzipped)
# Fusion of all SRTM files
concat = ImageTools.gdal_buildvrt(*unzipped, vrtnodata=-32768)
# Set nodata to 0
nodata = ImageTools.gdal_warp(concat,
srcnodata=-32768,
dstnodata=0,
multi=True)
# Combine to image of fixed extent
srtm_full_res = os.path.join(self.wdir, "srtm_%sm.tif" % int(self.site.res_x))
ImageTools.gdal_warp(nodata, dst=srtm_full_res,
r="cubic",
te=self.site.te_str,
t_srs=self.site.epsg_str,
tr=self.site.tr_str,
dstnodata=0,
srcnodata=0,
multi=True)
return srtm_full_res
def __init__(self, dbl, regex=None):
from StartMaja.Common import FileSystem
self.regex = self.regex if regex is None else regex
self.dbl = dbl
self.base = os.path.basename(dbl).split(".")[0]
# Find associated HDR
self.hdr = FileSystem.find_single(path=os.path.join(dbl, "../"),
pattern=self.base + ".HDR")
assert os.path.isfile(self.hdr)
def validity(self):
if self.level == "l1c" and os.path.exists(self.metadata_file):
return True
if self.level == "l2a":
try:
jpi = FileSystem.find_single("*JPI_ALL.xml", self.fpath)
except ValueError:
return False
validity_xpath = "./Processing_Flags_And_Modes_List/Processing_Flags_And_Modes/Value"
processing_flags = XMLTools.get_xpath(jpi, validity_xpath)
validity_flags = [flg.text for flg in processing_flags]
if "L2VALD" in validity_flags:
return True
return False
def prepare_mnt(self):
"""
Prepare the eudem files.
:return: Path to the full resolution DEM file.gsw
:rtype: str
"""
# Find/Download EuDEM archives:
eudem_files = self.get_raw_data()
# Unzip the downloaded/found EuDEM zip files:
unzipped = []
for arch in eudem_files:
basename = os.path.splitext(os.path.basename(arch))[0]
FileSystem.unzip(arch, self.wdir)
fn_unzipped = FileSystem.find_single(pattern=basename + ".TIF$",
path=self.wdir)
unzipped.append(fn_unzipped)
# Fusion of all EuDEM files
ds_cropped = []
for fn in unzipped:
ds = ImageTools.gdal_warp(fn,
of="GTiff",
ot="Int16",
r="cubic",
te=self.site.te_str,
t_srs=self.site.epsg_str,
tr=self.site.tr_str,
multi=True)
ds.array[ds.array < self.lowest_allowed_height] = -32767
ds_cropped.append(ds)
eudem_full_res = os.path.join(self.wdir,
"eudem_%sm.tif" % int(self.site.res_x))
ImageTools.gdal_merge(*ds_cropped,
dst=eudem_full_res,
n=-32767,
a_nodata=0,
q=True)
return eudem_full_res
def __set_input_paths(self):
"""
Set the available L1 and L2 input paths.
:return: The L1-, L2- folders (creating the latter if needed).
An info-string whether site and/or tile is used.
"""
if self.site:
site_l1 = FileSystem.find_single(r"^%s$" % self.site, self.rep_l1)
self.logger.warning(self.tile)
path_input_l1 = FileSystem.find_single(r"^T?%s$" % self.tile,
site_l1)
try:
site_l2 = FileSystem.find_single(r"^%s$" % self.site,
self.rep_l2)
except ValueError:
site_l2 = os.path.join(self.rep_l2, self.site)
FileSystem.create_directory(site_l2)
try:
path_input_l2 = FileSystem.find_single(r"^T?%s$" % self.tile,
site_l2)
except ValueError:
path_input_l2 = os.path.join(self.rep_l2, self.site, self.tile)
FileSystem.create_directory(path_input_l2)
site_info = "site %s and tile %s" % (self.site, self.tile)
else:
path_input_l1 = FileSystem.find_single(r"^T?%s$" % self.tile,
self.rep_l1)
try:
path_input_l2 = FileSystem.find_single(r"^T?%s$" % self.tile,
self.rep_l2)
except ValueError:
path_input_l2 = os.path.join(self.rep_l2, self.tile)
FileSystem.create_directory(path_input_l2)
site_info = "tile %s" % self.tile
return path_input_l1, path_input_l2, site_info