def _execute(self): self._setup() if (os.path.exists(self.user_download_cache_path) and os.path.isdir(self.user_download_cache_path)): executable = "/usr/bin/du" args = ["-sb", self.user_download_cache_path] self._run_process( Process(exec_type="exec", executable=executable, id="compute_download_cache_size", executable_params=args)) print("Disk usage ", self.module_output_log[0]["stdout"]) dc_size = int(self.module_output_log[0]["stdout"].split("\t")[0]) quota_size = int(self.config.DOWNLOAD_CACHE_QUOTA * 1024 * 1024 * 1024) model = StorageModel( used=dc_size, free=quota_size - dc_size, quota=quota_size, free_percent=int(100 * (quota_size - dc_size) / quota_size)) self.module_results = model self.finish_message = "Download cache size successfully computed" else: raise AsyncProcessError( "Download cache directory <%s> does not exist." % self.user_download_cache_path)
def get_wget_process(source, url): """The function returns a wget Process for the given source and url Args: source (str): The source file name where to download the data url (str): The url which should be downloaded Returns: p (Process): process for wget """ wget = "/usr/bin/wget" wget_params = list() wget_params.append("-t5") wget_params.append("-c") wget_params.append("-q") wget_params.append("-O") wget_params.append("%s" % source) wget_params.append(url) p = Process(exec_type="exec", executable=wget, executable_params=wget_params, id=f"importer_wget_{os.path.basename(source)}", skip_permission_check=True) return p
def get_vector_import_command(input_source, vector_name, layer_name=None): """Generate raster import process list that makes use of v.import Args: input_source (str): The input source can be a file path or a database string vector_name (str): The name of the new vector layer layer_name (str): The layer name or comma separated list of layer names that should be imported from the input source Returns: Process """ if layer_name is not None: exec_params = ["input=%s" % input_source, "output=%s" % vector_name, "layer=%s" % layer_name, "--q"] else: exec_params = ["input=%s" % input_source, "output=%s" % vector_name, "--q"] p = Process(exec_type="grass", executable="v.import", executable_params=exec_params, id=f"v_import_{os.path.basename(input_source)}", skip_permission_check=True) return p
def _export_strds(self, strds_name, format="GTiff"): """Export a specific strds layer with t.rast.export. The result is stored in a temporary directory that is located in the temporary grass database. Args: strds_name (str): The name of the strds layer format (str): GTiff (only option) Returns: tuple: A tuple (file_name, output_path) """ suffix = ".tar.gz" file_name = strds_name.split("@")[0] + suffix output_path = os.path.join(self.temp_file_path, file_name) if format != 'GTiff': format = 'GTiff' self.message_logger.info( "Only GTiff driver is supported for STRDS export.") module_name = "t.rast.export" args = [ "input=%s" % strds_name, "format=%s" % format, "output=%s" % output_path, "compression=%s" % "gzip" ] # optimized for GTiff create_opts = "createopt=BIGTIFF=YES,COMPRESS=LZW,TILED=YES" args.append(create_opts) os.environ['COMPRESS_OVERVIEW'] = "LZW" p = Process(exec_type="grass", executable=module_name, executable_params=args, id=f"exporter_strds_{strds_name}", stdin_source=None) self._update_num_of_steps(1) self._run_module(p) return file_name, output_path
def get_file_rename_command(file_path, file_name): """Generate the file-rename process list so that the input file has a specific file name that is accessible in the process chain via file_id Args: file_path: file_name: Returns: Process """ p = Process(exec_type="exec", executable="/bin/mv", executable_params=[file_path, file_name], id=f"mv_{os.path.basename(file_path)}_to_" f"{os.path.basename(file_name)}", skip_permission_check=True) return p
def _stac_import(self, stac_collection_id=None, semantic_label=None, interval=None, bbox=None, filter=None): if has_plugin: try: stac_name = stac_collection_id.split(".")[3] except Exception: raise AsyncProcessError("The source has not the right structure") stac_root = self._get_search_root(stac_collection_id) stac_filtered = self._apply_filter(stac_root, stac_name, interval, bbox, filter) stac_result = self._get_filtered_bands(stac_filtered, semantic_label) stac_processes = [] for key, value in stac_result.items(): for name_id, url in value.items(): output_name = stac_name + "_" + key + "_" + name_id # From Here Onwards, the Process build starts exec_params = ["input=%s" % "/vsicurl/"+url, "output=%s" % output_name, "-o"] p = Process( exec_type="grass", executable="r.in.gdal", executable_params=exec_params, id=f"r_gdal_{os.path.basename(output_name)}", skip_permission_check=True ) stac_processes.append(p) else: raise AsyncProcessError("Actinia STAC plugin is not installed") return stac_processes
def get_raster_import_command(file_path, raster_name): """Generate raster import process list that makes use of r.import Args: file_path: raster_name: Returns: Process """ p = Process(exec_type="grass", executable="r.import", executable_params=["input=%s" % file_path, "output=%s" % raster_name, "--q"], id=f"r_import_{os.path.basename(file_path)}", skip_permission_check=True) return p
def _export_file(self, tmp_file, file_name): """Export a specific file The result is stored in a temporary directory that is located in the temporary grass database. The output file will always be compressed using zip Args: tmp_file (str): The name of the temporary file generated by a module file_name (str): The file name to be used for export Returns: tuple: A tuple (file_name, output_path) Raises: AsyncProcessError: If a GRASS module return status is not 0 """ # Export the file archive_name = file_name + ".zip" # switch into the temporary working directory to use relative path for zip os.chdir(self.temp_file_path) # Compression compressed_output_path = os.path.join(self.temp_file_path, archive_name) executable = "/usr/bin/zip" args = ["-r", archive_name, tmp_file] p = Process(exec_type="exec", executable=executable, executable_params=args, id=f"exporter_file_{file_name}", stdin_source=None) self._update_num_of_steps(1) self._run_process(p) return archive_name, compressed_output_path
def _execute(self): self._setup() if (os.path.exists(self.user_download_cache_path) and os.path.isdir(self.user_download_cache_path)): executable = "/bin/rm" args = ["-rf", self.user_download_cache_path] self._run_process( Process(exec_type="exec", executable=executable, id="delete_download_cache_directory", executable_params=args)) os.mkdir(self.user_download_cache_path) self.finish_message = "Download cache successfully removed." else: raise AsyncProcessError( "Download cache directory <%s> does not exist." % self.user_download_cache_path)
def get_mv_process(source, dest): """The function returns a move Process for the given source and dest Args: source (str): The source file name dest (str): The destination file name Returns: p (Process): process for mv """ copy = "/bin/mv" copy_params = list() copy_params.append(source) copy_params.append(dest) p = Process(exec_type="exec", executable=copy, executable_params=copy_params, id=f"importer_mv_{os.path.basename(source)}", skip_permission_check=True) return p
def _export_postgis(self, vector_name, dbstring, output_layer=None, additional_options=[]): """Export a specific vector layer with v.out.postgis to a PostGIS database Args: vector_name (str): The name of the raster layer dbstring (str): The PostgreSQL database string to connect to the output database output_layer (str): The name of the PostgreSQL database table additional_options (list): Unused Raises: AsyncProcessError: If a GRASS module return status is not 0 """ module_name = "v.out.postgis" args = ["-l", "input=%s" % vector_name, "output=%s" % dbstring] if output_layer: args.append("output_layer=%s" % output_layer) if additional_options: args.extend(additional_options) # Export p = Process(exec_type="grass", executable=module_name, executable_params=args, id=f"exporter_postgis_{vector_name}", stdin_source=None) self._update_num_of_steps(1) self._run_module(p)
def _execute(self, skip_permission_check=False): """Overwrite this function in subclasses Setup the user credentials, check the executable access and run the process """ # Setup the user credentials and logger self._setup(init_grass=False) # Check if the user has access to the required process executable resp = check_location_mapset_module_access(self.user_credentials, self.config, module_name=self.executable) if resp is not None: raise AsyncProcessError("Executable <%s> is not supported" % self.executable) p = Process(exec_type="exec", executable=self.executable, executable_params=self.executable_params, id=f"exec_{self.executable}", stdin_source=None) self._run_process(p)
def _export_vector(self, vector_name, format="GPKG", additional_options=[]): """Export a specific vector layer with v.out.ogr using a specific output format The result is stored in a temporary directory that is located in the temporary grass database. The resulting vector file will always be compressed using zip Args: vector_name (str): The name of the raster layer format (str): GPKG (default), GML, GeoJSON, ESRI_Shapefile, SQLite, CSV additional_options (list): Unused Returns: tuple: A tuple (file_name, output_path) Raises: AsyncProcessError: If a GRASS module return status is not 0 """ # Export the layer prefix = "" if format == "GPKG": prefix = ".gpkg" if format == "GML": prefix = ".gml" if format == "GeoJSON": prefix = ".json" if format == "ESRI_Shapefile": prefix = "" if format == "SQLite": prefix = ".sqlite" if format == "CSV": prefix = ".csv" # Remove a potential mapset file_name = vector_name.split("@")[0] + prefix archive_name = file_name + ".zip" # switch into the temporary working directory to use relative path for zip os.chdir(self.temp_file_path) module_name = "v.out.ogr" args = [ "-e", "input=%s" % vector_name, "format=%s" % format, "output=%s" % file_name ] if additional_options: args.extend(additional_options) # Export p = Process(exec_type="grass", executable=module_name, executable_params=args, id=f"exporter_vector_{vector_name}", stdin_source=None) self._update_num_of_steps(1) self._run_module(p) # Compression compressed_output_path = os.path.join(self.temp_file_path, archive_name) executable = "/usr/bin/zip" args = ["-r", archive_name, file_name] p = Process(exec_type="exec", executable=executable, executable_params=args, id=f"exporter_zip_{vector_name}", stdin_source=None) self._update_num_of_steps(1) self._run_process(p) return archive_name, compressed_output_path
def _export_raster(self, raster_name, format="COG", additional_options=[], use_raster_region=False): """Export a specific raster layer with r.out.gdal as GeoTiff. The result is stored in a temporary directory that is located in the temporary grass database. The region of the raster layer can be used for export. In this case a temporary region will be used for export, so that the original region of the mapset is not modified. COG-Driver: https://gdal.org/drivers/raster/cog.html Args: raster_name (str): The name of the raster layer format (str): COG (default; requires GDAL >= 3.1 on server), GTiff additional_options (list): Unused use_raster_region (bool): Use the region of the raster layer for export Returns: tuple: A tuple (file_name, output_path) Raises: AsyncProcessError: If a GRASS module return status is not 0 """ # Export the layer suffix = ".tif" # Remove a potential mapset file_name = raster_name.split("@")[0] + suffix if use_raster_region is True: p = Process(exec_type="grass", executable="g.region", executable_params=["raster=%s" % raster_name, "-g"], id=f"exporter_region_{raster_name}", stdin_source=None) self._update_num_of_steps(1) self._run_module(p) if format == 'COG': # check if GDAL has COG driver from osgeo import gdal driver_list = [ gdal.GetDriver(i).ShortName for i in range(gdal.GetDriverCount()) ] if 'COG' not in driver_list: format = 'GTiff' self.message_logger.info( "COG driver not available, using GTiff driver") # Save the file in the temporary directory of the temporary gisdb output_path = os.path.join(self.temp_file_path, file_name) module_name = "r.out.gdal" args = [ "-fmt", "input=%s" % raster_name, "format=%s" % format, "output=%s" % output_path ] create_opts = "createopt=BIGTIFF=YES,COMPRESS=LZW" if format == "GTiff": # generate overviews with compression: os.environ['COMPRESS_OVERVIEW'] = "LZW" args.append("overviews=5") create_opts += ",TILED=YES" args.append(create_opts) # current workaround due to color table export # COG bug in GDAL, see https://github.com/OSGeo/gdal/issues/2946 # TODO: DELETE AND TEST ONCE GDAL 3.1.4 HAS BEEN RELEASED if format == "COG": args.append("-c") if additional_options: args.extend(additional_options) p = Process(exec_type="grass", executable=module_name, executable_params=args, id=f"exporter_raster_{raster_name}", stdin_source=None) self._update_num_of_steps(1) self._run_module(p) return file_name, output_path