def _execute(self):
self._setup()
if (os.path.exists(self.user_download_cache_path)
and os.path.isdir(self.user_download_cache_path)):
executable = "/usr/bin/du"
args = ["-sb", self.user_download_cache_path]
self._run_process(
Process(exec_type="exec",
executable=executable,
id="compute_download_cache_size",
executable_params=args))
print("Disk usage ", self.module_output_log[0]["stdout"])
dc_size = int(self.module_output_log[0]["stdout"].split("\t")[0])
quota_size = int(self.config.DOWNLOAD_CACHE_QUOTA * 1024 * 1024 *
1024)
model = StorageModel(
used=dc_size,
free=quota_size - dc_size,
quota=quota_size,
free_percent=int(100 * (quota_size - dc_size) / quota_size))
self.module_results = model
self.finish_message = "Download cache size successfully computed"
else:
raise AsyncProcessError(
"Download cache directory <%s> does not exist." %
self.user_download_cache_path)
def get_wget_process(source, url):
"""The function returns a wget Process for the given source and url
Args:
source (str): The source file name where to download the data
url (str): The url which should be downloaded
Returns:
p (Process): process for wget
"""
wget = "/usr/bin/wget"
wget_params = list()
wget_params.append("-t5")
wget_params.append("-c")
wget_params.append("-q")
wget_params.append("-O")
wget_params.append("%s" % source)
wget_params.append(url)
p = Process(exec_type="exec",
executable=wget,
executable_params=wget_params,
id=f"importer_wget_{os.path.basename(source)}",
skip_permission_check=True)
return p
def get_vector_import_command(input_source, vector_name, layer_name=None):
"""Generate raster import process list that makes use of v.import
Args:
input_source (str): The input source can be a file path or a database string
vector_name (str): The name of the new vector layer
layer_name (str): The layer name or comma separated list of layer names
that should be imported from the input source
Returns:
Process
"""
if layer_name is not None:
exec_params = ["input=%s" % input_source,
"output=%s" % vector_name,
"layer=%s" % layer_name,
"--q"]
else:
exec_params = ["input=%s" % input_source,
"output=%s" % vector_name,
"--q"]
p = Process(exec_type="grass",
executable="v.import",
executable_params=exec_params,
id=f"v_import_{os.path.basename(input_source)}",
skip_permission_check=True)
return p
def _export_strds(self, strds_name, format="GTiff"):
"""Export a specific strds layer with t.rast.export.
The result is stored in a temporary directory
that is located in the temporary grass database.
Args:
strds_name (str): The name of the strds layer
format (str): GTiff (only option)
Returns:
tuple: A tuple (file_name, output_path)
"""
suffix = ".tar.gz"
file_name = strds_name.split("@")[0] + suffix
output_path = os.path.join(self.temp_file_path, file_name)
if format != 'GTiff':
format = 'GTiff'
self.message_logger.info(
"Only GTiff driver is supported for STRDS export.")
module_name = "t.rast.export"
args = [
"input=%s" % strds_name,
"format=%s" % format,
"output=%s" % output_path,
"compression=%s" % "gzip"
]
# optimized for GTiff
create_opts = "createopt=BIGTIFF=YES,COMPRESS=LZW,TILED=YES"
args.append(create_opts)
os.environ['COMPRESS_OVERVIEW'] = "LZW"
p = Process(exec_type="grass",
executable=module_name,
executable_params=args,
id=f"exporter_strds_{strds_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_module(p)
return file_name, output_path
def get_file_rename_command(file_path, file_name):
"""Generate the file-rename process list so that the input file has a specific
file name that is accessible in the process chain via file_id
Args:
file_path:
file_name:
Returns:
Process
"""
p = Process(exec_type="exec", executable="/bin/mv",
executable_params=[file_path, file_name],
id=f"mv_{os.path.basename(file_path)}_to_"
f"{os.path.basename(file_name)}",
skip_permission_check=True)
return p
def _stac_import(self, stac_collection_id=None, semantic_label=None,
interval=None, bbox=None, filter=None):
if has_plugin:
try:
stac_name = stac_collection_id.split(".")[3]
except Exception:
raise AsyncProcessError("The source has not the right structure")
stac_root = self._get_search_root(stac_collection_id)
stac_filtered = self._apply_filter(stac_root, stac_name,
interval, bbox, filter)
stac_result = self._get_filtered_bands(stac_filtered, semantic_label)
stac_processes = []
for key, value in stac_result.items():
for name_id, url in value.items():
output_name = stac_name + "_" + key + "_" + name_id
# From Here Onwards, the Process build starts
exec_params = ["input=%s" % "/vsicurl/"+url,
"output=%s" % output_name,
"-o"]
p = Process(
exec_type="grass",
executable="r.in.gdal",
executable_params=exec_params,
id=f"r_gdal_{os.path.basename(output_name)}",
skip_permission_check=True
)
stac_processes.append(p)
else:
raise AsyncProcessError("Actinia STAC plugin is not installed")
return stac_processes
def get_raster_import_command(file_path, raster_name):
"""Generate raster import process list that makes use of r.import
Args:
file_path:
raster_name:
Returns:
Process
"""
p = Process(exec_type="grass",
executable="r.import",
executable_params=["input=%s" % file_path,
"output=%s" % raster_name,
"--q"],
id=f"r_import_{os.path.basename(file_path)}",
skip_permission_check=True)
return p
def _export_file(self, tmp_file, file_name):
"""Export a specific file
The result is stored in a temporary directory
that is located in the temporary grass database.
The output file will always be compressed using zip
Args:
tmp_file (str): The name of the temporary file generated by a module
file_name (str): The file name to be used for export
Returns:
tuple: A tuple (file_name, output_path)
Raises:
AsyncProcessError: If a GRASS module return status is not 0
"""
# Export the file
archive_name = file_name + ".zip"
# switch into the temporary working directory to use relative path for zip
os.chdir(self.temp_file_path)
# Compression
compressed_output_path = os.path.join(self.temp_file_path,
archive_name)
executable = "/usr/bin/zip"
args = ["-r", archive_name, tmp_file]
p = Process(exec_type="exec",
executable=executable,
executable_params=args,
id=f"exporter_file_{file_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_process(p)
return archive_name, compressed_output_path
def _execute(self):
self._setup()
if (os.path.exists(self.user_download_cache_path)
and os.path.isdir(self.user_download_cache_path)):
executable = "/bin/rm"
args = ["-rf", self.user_download_cache_path]
self._run_process(
Process(exec_type="exec",
executable=executable,
id="delete_download_cache_directory",
executable_params=args))
os.mkdir(self.user_download_cache_path)
self.finish_message = "Download cache successfully removed."
else:
raise AsyncProcessError(
"Download cache directory <%s> does not exist." %
self.user_download_cache_path)
def get_mv_process(source, dest):
"""The function returns a move Process for the given source and dest
Args:
source (str): The source file name
dest (str): The destination file name
Returns:
p (Process): process for mv
"""
copy = "/bin/mv"
copy_params = list()
copy_params.append(source)
copy_params.append(dest)
p = Process(exec_type="exec",
executable=copy,
executable_params=copy_params,
id=f"importer_mv_{os.path.basename(source)}",
skip_permission_check=True)
return p
def _export_postgis(self,
vector_name,
dbstring,
output_layer=None,
additional_options=[]):
"""Export a specific vector layer with v.out.postgis to a PostGIS database
Args:
vector_name (str): The name of the raster layer
dbstring (str): The PostgreSQL database string to connect to the
output database
output_layer (str): The name of the PostgreSQL database table
additional_options (list): Unused
Raises:
AsyncProcessError: If a GRASS module return status is not 0
"""
module_name = "v.out.postgis"
args = ["-l", "input=%s" % vector_name, "output=%s" % dbstring]
if output_layer:
args.append("output_layer=%s" % output_layer)
if additional_options:
args.extend(additional_options)
# Export
p = Process(exec_type="grass",
executable=module_name,
executable_params=args,
id=f"exporter_postgis_{vector_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_module(p)
def _execute(self, skip_permission_check=False):
"""Overwrite this function in subclasses
Setup the user credentials, check the executable access and run the process
"""
# Setup the user credentials and logger
self._setup(init_grass=False)
# Check if the user has access to the required process executable
resp = check_location_mapset_module_access(self.user_credentials,
self.config,
module_name=self.executable)
if resp is not None:
raise AsyncProcessError("Executable <%s> is not supported" %
self.executable)
p = Process(exec_type="exec",
executable=self.executable,
executable_params=self.executable_params,
id=f"exec_{self.executable}",
stdin_source=None)
self._run_process(p)
def _export_vector(self,
vector_name,
format="GPKG",
additional_options=[]):
"""Export a specific vector layer with v.out.ogr using a specific output format
The result is stored in a temporary directory
that is located in the temporary grass database.
The resulting vector file will always be compressed using zip
Args:
vector_name (str): The name of the raster layer
format (str): GPKG (default), GML, GeoJSON, ESRI_Shapefile, SQLite, CSV
additional_options (list): Unused
Returns:
tuple: A tuple (file_name, output_path)
Raises:
AsyncProcessError: If a GRASS module return status is not 0
"""
# Export the layer
prefix = ""
if format == "GPKG":
prefix = ".gpkg"
if format == "GML":
prefix = ".gml"
if format == "GeoJSON":
prefix = ".json"
if format == "ESRI_Shapefile":
prefix = ""
if format == "SQLite":
prefix = ".sqlite"
if format == "CSV":
prefix = ".csv"
# Remove a potential mapset
file_name = vector_name.split("@")[0] + prefix
archive_name = file_name + ".zip"
# switch into the temporary working directory to use relative path for zip
os.chdir(self.temp_file_path)
module_name = "v.out.ogr"
args = [
"-e",
"input=%s" % vector_name,
"format=%s" % format,
"output=%s" % file_name
]
if additional_options:
args.extend(additional_options)
# Export
p = Process(exec_type="grass",
executable=module_name,
executable_params=args,
id=f"exporter_vector_{vector_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_module(p)
# Compression
compressed_output_path = os.path.join(self.temp_file_path,
archive_name)
executable = "/usr/bin/zip"
args = ["-r", archive_name, file_name]
p = Process(exec_type="exec",
executable=executable,
executable_params=args,
id=f"exporter_zip_{vector_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_process(p)
return archive_name, compressed_output_path
def _export_raster(self,
raster_name,
format="COG",
additional_options=[],
use_raster_region=False):
"""Export a specific raster layer with r.out.gdal as GeoTiff.
The result is stored in a temporary directory
that is located in the temporary grass database.
The region of the raster layer can be used for export. In this case a
temporary region will be used for export, so that the original region
of the mapset is not modified.
COG-Driver: https://gdal.org/drivers/raster/cog.html
Args:
raster_name (str): The name of the raster layer
format (str): COG (default; requires GDAL >= 3.1 on server), GTiff
additional_options (list): Unused
use_raster_region (bool): Use the region of the raster layer for export
Returns:
tuple: A tuple (file_name, output_path)
Raises:
AsyncProcessError: If a GRASS module return status is not 0
"""
# Export the layer
suffix = ".tif"
# Remove a potential mapset
file_name = raster_name.split("@")[0] + suffix
if use_raster_region is True:
p = Process(exec_type="grass",
executable="g.region",
executable_params=["raster=%s" % raster_name, "-g"],
id=f"exporter_region_{raster_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_module(p)
if format == 'COG':
# check if GDAL has COG driver
from osgeo import gdal
driver_list = [
gdal.GetDriver(i).ShortName
for i in range(gdal.GetDriverCount())
]
if 'COG' not in driver_list:
format = 'GTiff'
self.message_logger.info(
"COG driver not available, using GTiff driver")
# Save the file in the temporary directory of the temporary gisdb
output_path = os.path.join(self.temp_file_path, file_name)
module_name = "r.out.gdal"
args = [
"-fmt",
"input=%s" % raster_name,
"format=%s" % format,
"output=%s" % output_path
]
create_opts = "createopt=BIGTIFF=YES,COMPRESS=LZW"
if format == "GTiff":
# generate overviews with compression:
os.environ['COMPRESS_OVERVIEW'] = "LZW"
args.append("overviews=5")
create_opts += ",TILED=YES"
args.append(create_opts)
# current workaround due to color table export
# COG bug in GDAL, see https://github.com/OSGeo/gdal/issues/2946
# TODO: DELETE AND TEST ONCE GDAL 3.1.4 HAS BEEN RELEASED
if format == "COG":
args.append("-c")
if additional_options:
args.extend(additional_options)
p = Process(exec_type="grass",
executable=module_name,
executable_params=args,
id=f"exporter_raster_{raster_name}",
stdin_source=None)
self._update_num_of_steps(1)
self._run_module(p)
return file_name, output_path
