def get_default_destination_values(param: QgsProcessingDestinationParameter,
default: Optional[str]) -> Tuple[str, str]:
""" Get default value from parameter
"""
defval = param.defaultValue()
ext = param.defaultFileExtension()
# Check for default extensions override ?
if isinstance(param, DESTINATION_VECTOR_LAYER_TYPES):
ext = confservice.get('processing', 'vector.fileext') or ext
elif isinstance(param, DESTINATION_RASTER_LAYER_TYPES):
ext = confservice.get('processing', 'raster.fileext') or ext
if isinstance(defval, QgsProcessingOutputLayerDefinition):
sink = defval.sink
if sink:
# Try to get extension from the sink value
sink = sink.staticValue()
if sink:
# Remove extra open options
url = urlparse(sink.split('|', 1)[0])
if url.path and url.scheme.lower() in ('', 'file'):
p = Path(url.path)
ext = p.suffix.lstrip('.') or ext
defval = defval.destinationName or p.stem
else:
defval = default
return ext, defval
def __init__(self, workdir: str, map_uri: Optional[str]=None) -> None:
super().__init__()
self.workdir = workdir
self.rootdir = Path(confservice.get('projects.cache','rootdir'))
if map_uri is not None:
self.map_uri = map_uri
project = cacheservice.lookup(map_uri)[0]
# Get the CRS of the project
project_crs = project.crs()
if project_crs.isValid():
self.setProject(project)
else:
LOGGER.warning("Invalid CRS for project '%s'", map_uri)
else:
LOGGER.warning("No map url defined, inputs may be incorrect !")
self.map_uri = None
default_crs = confservice.get('processing','default_crs')
project_crs = QgsCoordinateReferenceSystem()
project_crs.createFromUserInput(default_crs)
if not project_crs.isValid():
LOGGER.error("Invalid default CRS '%s'",default_crs)
# Create the destination project
destination_project = QgsProject()
if project_crs.isValid():
adjust_ellipsoid = confservice.getboolean('processing','adjust_ellipsoid')
destination_project.setCrs(project_crs, adjust_ellipsoid)
self.destination_project = destination_project
def delete_results(self, uuid):
""" Delete process results and status
:param uuid: the uuid of the required process.
If set to None, return all the stored status.
:return: True if the status has been deleted.
"""
rec = logstore.get_status(uuid)
if rec is None:
raise FileNotFoundError(uuid)
try:
if STATUS[rec['status']] < STATUS.DONE_STATUS:
return False
except KeyError:
# Handle legacy status
pass
workdir = os.path.abspath(confservice.get('server', 'workdir'))
# Delete the working directory
uuid_str = rec['uuid']
workdir = os.path.join(workdir, uuid_str)
LOGGER.info("Cleaning response status: %s", uuid_str)
try:
if os.path.isdir(workdir):
shutil.rmtree(workdir)
except Exception as err:
LOGGER.error('Unable to remove directory: %s: %s', workdir, err)
# Delete the record/response
logstore.delete_response(uuid_str)
return True
def __init__(self, process, wps_request, uuid, status_url=None):
"""constructor
:param pyqgiswps.app.process.Process process:
:param pyqgiswps.app.request.WPSRequest wps_request:
:param uuid: string this request uuid
:param status_url: url to retrieve the status from
"""
store_url = confservice.get('server', 'store_url')
store_url = store_url.format(host_url=wps_request.host_url,
uuid=uuid,
file='{file}')
self.process = process
self.wps_request = wps_request
self.outputs = {o.identifier: o for o in process.outputs}
self.message = ''
self.status = WPSResponse.STATUS.NO_STATUS
self.status_percentage = -1
self.status_url = status_url
self.store_url = store_url
self.uuid = uuid
self.document = None
self.store = False
def calculate_max_input_size(self) -> int:
"""Calculates maximal size for input file based on configuration
and units
:return: maximum file size bytes
"""
max_size = confservice.get('server','maxinputsize')
self.max_size = get_size_bytes(max_size)
return self.max_size
def chconfig(section, key, value):
""" Use configuration setting
"""
prev = confservice.get(section, key)
try:
confservice.set(section, key, value)
yield prev
finally:
confservice.set(section, key, prev)
def init_restart_handler(self):
restartmon = confservice.get('server', 'restartmon')
if restartmon:
LOGGER.info(f"Reload handler monitoring: {restartmon}")
check_time = confservice.getint('server', 'restartmon_check_time',
3000)
self._reload_handler = watchfiles(
[restartmon], lambda *_args: self.reload_qgis_processes(),
check_time)
self._reload_handler.start()
def __init__(self, workdir: str, map_uri: str = None) -> None:
super().__init__()
self.workdir = workdir
self.rootdir = Path(confservice.get('cache', 'rootdir'))
if map_uri is not None:
self.map_uri = map_uri
self.setProject(cacheservice.lookup(map_uri)[0])
else:
LOGGER.warning("No map url defined, inputs may be incorrect !")
self.map_uri = None
# Create the destination project
self.destination_project = QgsProject()
async def execute(self, identifier, wps_request, uuid, **context):
"""Parse and perform Execute WPS request call
:param identifier: process identifier string
:param wps_request: pyqgiswps.WPSRequest structure with parsed inputs, still in memory
:param uuid: string identifier of the request
"""
try:
process = self.get_process(identifier, **context)
except UnknownProcessError:
raise InvalidParameterValue("Unknown process '%r'" % identifier,
'Identifier')
# make deep copy of the process instance
# so that processes are not overriding each other
# just for execute
process = copy.deepcopy(process)
self._parse(process, wps_request)
workdir = os.path.abspath(confservice.get('server', 'workdir'))
workdir = os.path.join(workdir, str(uuid))
# Create working directory if it does not exists
os.makedirs(workdir, exist_ok=True)
process.set_workdir(workdir)
# Get status url
status_url = self._status_url(uuid, wps_request)
# Create response object
wps_response = WPSResponse(process,
wps_request,
uuid,
status_url=status_url)
if wps_request.store_execute == 'true':
# Setting STORE_AND_UPDATE_STATUS will trigger
# asynchronous requests
wps_response.status = STATUS.STORE_AND_UPDATE_STATUS
LOGGER.debug("Update status enabled")
if wps_request.raw:
raise NotImplementedError("Raw output is not implemented")
document = await self.executor.execute(wps_request, wps_response)
return document
def init(self, filepath: Union[str, Path] = None) -> None:
""" Load policy file
"""
if not isinstance(filepath, Path):
filepath = Path(filepath
or confservice.get('processing', 'accesspolicy'))
if not filepath.exists():
return
LOGGER.info("Loading access policy from %s", filepath.as_posix())
with filepath.open('r') as f:
policy = yaml.load(f, yaml.SafeLoader)
self._deny = _validate_policy(policy.get('deny', []))
self._allow = _validate_policy(policy.get('allow', []))
def start_qgis(self):
""" Set up qgis
"""
# Do not intialize twice
if self.qgisapp is not None:
return
logprefix = "[qgis:%s]" % os.getpid()
settings = {
"Processing/Configuration/PREFER_FILENAME_AS_LAYER_NAME":"false"
}
def _folders_setting( setting, folders ):
folders = folders.split(';')
folders = chain( *(glob(f) for f in folders) )
folders = ';'.join( f for f in folders if os.path.isdir(f) )
if folders:
LOGGER.info("%s = %s", setting, folders)
settings[setting] = folders
# Set up folder settings
# XXX Note that if scripts folder is not set then ScriptAlgorithmProvider will crash !
for setting, value in confservice.items('qgis.settings.folders'):
LOGGER.debug("*** Folder settings: %s = %s", setting, value)
_folders_setting(setting, value)
# Load other settings from configuration file
# Init qgis application
self.qgisapp = start_qgis_application(
enable_processing=True,
verbose=confservice.get('logging','level')=='DEBUG',
logger=LOGGER, logprefix=logprefix,
settings=settings
)
# Load plugins
self._wps_interface.register_providers()
return self.qgisapp
def parse_root_destination_path(param: QgsProcessingDestinationParameter,
value: str,
default_extension: str) -> Tuple[str, str]:
""" Parse input value as sink
In this situation, value is interpreted as the output sink of the destination layer,
It may be any source url supported by Qgis (but with options stripped)
The layername may be specified by appending the '|layername=<name>' to the input string.
"""
value, *rest = value.split('|', 2)
destinationName = None
# Check for layername option
if len(rest) > 0 and rest[0].lower().startswith('layername='):
destinationName = rest[0].split('=')[1].strip()
url = urlparse(value)
if url.path and url.scheme.lower() in ('', 'file'):
p = Path(url.path)
if p.is_absolute():
root = Path(confservice.get('processing', 'destination_root_path'))
p = root.joinpath(p.relative_to('/'))
# Check for extension:
if not p.suffix:
p = p.with_suffix(f'.{default_extension}')
destinationName = destinationName or p.stem
sink = str(p)
else:
destinationName = destinationName or param.name()
sink = value
return sink, destinationName
def __init__(self, map_uri: str = None, **create_context):
self.rootdir = Path(confservice.get('cache', 'rootdir'))
self.map_uri = map_uri
self._create_context = create_context
def get_capabilities(self, wps_request, accesspolicy=None):
""" Handle getcapbabilities request
"""
process_elements = [
p.capabilities_xml() for p in self.processes
if accesspolicy.allow(p.identifier)
]
doc = WPS.Capabilities()
doc.attrib['service'] = 'WPS'
doc.attrib['version'] = '1.0.0'
doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US'
doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \
'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsGetCapabilities_response.xsd'
# TODO: check Table 7 in OGC 05-007r7
doc.attrib['updateSequence'] = '1'
metadata = confservice['metadata:main']
# Service Identification
service_ident_doc = OWS.ServiceIdentification(
OWS.Title(metadata.get('identification_title')))
if metadata.get('identification_abstract'):
service_ident_doc.append(
OWS.Abstract(metadata.get('identification_abstract')))
if metadata.get('identification_keywords'):
keywords_doc = OWS.Keywords()
for k in metadata.get('identification_keywords').split(','):
if k:
keywords_doc.append(OWS.Keyword(k))
service_ident_doc.append(keywords_doc)
if metadata.get('identification_keywords_type'):
keywords_type = OWS.Type(
metadata.get('identification_keywords_type'))
keywords_type.attrib['codeSpace'] = 'ISOTC211/19115'
keywords_doc.append(keywords_type)
service_ident_doc.append(OWS.ServiceType('WPS'))
# TODO: set proper version support
service_ident_doc.append(OWS.ServiceTypeVersion('1.0.0'))
service_ident_doc.append(OWS.Fees(metadata.get('identification_fees')))
for con in metadata.get('identification_accessconstraints').split(','):
service_ident_doc.append(OWS.AccessConstraints(con))
if metadata.get('identification_profile'):
service_ident_doc.append(
OWS.Profile(metadata.get('identification_profile')))
doc.append(service_ident_doc)
# Service Provider
service_prov_doc = OWS.ServiceProvider(
OWS.ProviderName(metadata.get('provider_name')))
if metadata.get('provider_url'):
service_prov_doc.append(
OWS.ProviderSite({
'{http://www.w3.org/1999/xlink}href':
metadata.get('provider_url')
}))
# Service Contact
service_contact_doc = OWS.ServiceContact()
# Add Contact information only if a name is set
if metadata.get('contact_name'):
service_contact_doc.append(
OWS.IndividualName(metadata.get('contact_name')))
if metadata.get('contact_position'):
service_contact_doc.append(
OWS.PositionName(metadata.get('contact_position')))
contact_info_doc = OWS.ContactInfo()
phone_doc = OWS.Phone()
if metadata.get('contact_phone'):
phone_doc.append(OWS.Voice(metadata.get('contact_phone')))
# Add Phone if not empty
if len(phone_doc):
contact_info_doc.append(phone_doc)
address_doc = OWS.Address()
if metadata.get('deliveryPoint'):
address_doc.append(
OWS.DeliveryPoint(metadata.get('contact_address')))
if metadata.get('city'):
address_doc.append(OWS.City(metadata.get('contact_city')))
if metadata.get('contact_stateorprovince'):
address_doc.append(
OWS.AdministrativeArea(
metadata.get('contact_stateorprovince')))
if metadata.get('contact_postalcode'):
address_doc.append(
OWS.PostalCode(metadata.get('contact_postalcode')))
if metadata.get('contact_country'):
address_doc.append(OWS.Country(
metadata.get('contact_country')))
if metadata.get('contact_email'):
address_doc.append(
OWS.ElectronicMailAddress(metadata.get('contact_email')))
# Add Address if not empty
if len(address_doc):
contact_info_doc.append(address_doc)
if metadata.get('contact_url'):
contact_info_doc.append(
OWS.OnlineResource({
'{http://www.w3.org/1999/xlink}href':
metadata.get('contact_url')
}))
if metadata.get('contact_hours'):
contact_info_doc.append(
OWS.HoursOfService(metadata.get('contact_hours')))
if metadata.get('contact_instructions'):
contact_info_doc.append(
OWS.ContactInstructions(
metadata.get('contact_instructions')))
# Add Contact information if not empty
if len(contact_info_doc):
service_contact_doc.append(contact_info_doc)
if metadata.get('contact_role'):
service_contact_doc.append(
OWS.Role(metadata.get('contact_role')))
# Add Service Contact only if ProviderName and PositionName are set
if len(service_contact_doc):
service_prov_doc.append(service_contact_doc)
doc.append(service_prov_doc)
server_href = {
'{http://www.w3.org/1999/xlink}href':
confservice.get('server',
'url').format(host_url=wps_request.host_url)
}
# Operations Metadata
operations_metadata_doc = OWS.OperationsMetadata(
OWS.Operation(OWS.DCP(
OWS.HTTP(OWS.Get(server_href), OWS.Post(server_href))),
name="GetCapabilities"),
OWS.Operation(OWS.DCP(
OWS.HTTP(OWS.Get(server_href), OWS.Post(server_href))),
name="DescribeProcess"),
OWS.Operation(OWS.DCP(
OWS.HTTP(OWS.Get(server_href), OWS.Post(server_href))),
name="Execute"))
doc.append(operations_metadata_doc)
doc.append(WPS.ProcessOfferings(*process_elements))
languages = confservice.get('server', 'language').split(',')
languages_doc = WPS.Languages(WPS.Default(OWS.Language(languages[0])))
lang_supported_doc = WPS.Supported()
for l in languages:
lang_supported_doc.append(OWS.Language(l))
languages_doc.append(lang_supported_doc)
doc.append(languages_doc)
return doc
def __init__(self, map_uri: Optional[str]=None):
self.rootdir = Path(confservice.get('projects.cache','rootdir'))
self.map_uri = map_uri
def _construct_doc(self):
doc = WPS.ExecuteResponse()
doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \
'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_response.xsd'
doc.attrib['service'] = 'WPS'
doc.attrib['version'] = '1.0.0'
doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US'
doc.attrib['serviceInstance'] = '%s%s' % (confservice.get(
'server', 'url').format(host_url=self.wps_request.host_url
), '?service=WPS&request=GetCapabilities')
if self.status >= STATUS.STORE_STATUS:
doc.attrib['statusLocation'] = self.status_url
# Process XML
process_doc = WPS.Process(OWS.Identifier(self.process.identifier),
OWS.Title(self.process.title))
if self.process.abstract:
process_doc.append(OWS.Abstract(self.process.abstract))
# TODO: See Table 32 Metadata in OGC 06-121r3
# for m in self.process.metadata:
# process_doc.append(OWS.Metadata(m))
if self.process.profile:
process_doc.append(OWS.Profile(self.process.profile))
process_doc.attrib[
'{http://www.opengis.net/wps/1.0.0}processVersion'] = self.process.version
doc.append(process_doc)
# Status XML
# return the correct response depending on the progress of the process
if self.status == STATUS.STORE_AND_UPDATE_STATUS:
if self.status_percentage == -1:
status_doc = self._process_accepted()
doc.append(status_doc)
return doc
elif self.status_percentage >= 0:
status_doc = self._process_started()
doc.append(status_doc)
return doc
# check if process failed and display fail message
if self.status == STATUS.ERROR_STATUS:
status_doc = self._process_failed()
doc.append(status_doc)
return doc
# TODO: add paused status
if self.status == STATUS.DONE_STATUS:
status_doc = self._process_succeeded()
doc.append(status_doc)
# DataInputs and DataOutputs definition XML if lineage=true
if self.wps_request.lineage == 'true':
data_inputs = [
self.wps_request.inputs[i][0].execute_xml()
for i in self.wps_request.inputs
]
doc.append(WPS.DataInputs(*data_inputs))
output_definitions = [
self.outputs[o].execute_xml_lineage() for o in self.outputs
]
doc.append(WPS.OutputDefinitions(*output_definitions))
# Process outputs XML
output_elements = [
self.outputs[o].execute_xml() for o in self.outputs
]
doc.append(WPS.ProcessOutputs(*output_elements))
return doc