def start_qgis(self):
""" Set up qgis
"""
# Do not intialize twice
if self.qgisapp is not None:
return
logprefix = "[qgis:%s]" % os.getpid()
settings = {}
def _folders_setting( setting, folders ):
folders = folders.split(';')
folders = chain( *(glob(f) for f in folders) )
folders = ';'.join( f for f in folders if os.path.isdir(f) )
if folders:
LOGGER.info("%s = %s", setting, folders)
settings[setting] = folders
# Set up folder settings
# XXX Note that if scripts folder is not set then ScriptAlgorithmProvider will crash !
for setting, value in config.get_config().items('qgis.settings.folders'):
LOGGER.debug("*** Folder settings: %s = %s", setting, value)
_folders_setting(setting, value)
# Init qgis application
self.qgisapp = start_qgis_application( enable_processing=True,
verbose=config.get_config('logging').get('level')=='DEBUG',
logger=LOGGER, logprefix=logprefix,
settings=settings)
# Load plugins
self._wps_interface.register_providers()
return self.qgisapp
def format_output_url(response: WPSResponse, file_name: str) -> str:
""" Build output/store url for output file name
"""
outputurl = config.get_config('server')['outputurl']
return outputurl.format(host_url=response.wps_request.host_url,
uuid=response.uuid,
file=file_name)
def delete_results(self, uuid):
""" Delete process results and status
:param uuid: the uuid of the required process.
If set to None, return all the stored status.
:return: True if the status has been deleted.
"""
rec = logstore.get_status(uuid)
if rec is None:
raise FileNotFoundError(uuid)
try:
if STATUS[rec['status']] < STATUS.DONE_STATUS:
return False
except KeyError:
# Handle legacy status
pass
cfg = config.get_config('server')
rootdir = os.path.abspath(cfg['workdir'])
# Delete the working directory
uuid_str = rec['uuid']
workdir = os.path.join(rootdir, uuid_str)
LOGGER.info("Cleaning response status: %s", uuid_str)
try:
if os.path.isdir(workdir):
shutil.rmtree(workdir)
except Exception as err:
LOGGER.error('Unable to remove directory: %s: %s', workdir, err)
# Delete the record/response
logstore.delete_response(uuid_str)
return True
def _status_url(self, uuid, request):
""" Return the status_url for the process <uuid>
"""
cfg = config.get_config('server')
status_url = cfg['status_url']
proxy_host = cfg['host_proxy']
if not proxy_host:
# Need to return the 'real' host
proxy_host = request.host_url if request else '{host_url}'
return status_url.format(host_url=proxy_host,uuid=uuid)
def __init__(self, processes=[]):
self.processes = {}
self._context_processes = lrucache(50)
cfg = config.get_config('server')
maxqueuesize = cfg.getint('maxqueuesize')
self._pool = create_client(maxqueuesize)
self._factory = get_process_factory()
self.install_processes(processes)
# Launch the cleanup task
self.schedule_cleanup()
def init(self, filepath: Union[str,Path]=None) -> None:
""" Load policy file
"""
if not isinstance(filepath,Path):
filepath = Path(filepath or get_config('processing')['accesspolicy'])
if not filepath.exists():
return
LOGGER.info("Loading access policy from %s", filepath.as_posix())
with filepath.open('r') as f:
policy = yaml.load(f, yaml.SafeLoader)
self._deny = _validate_policy(policy.get('deny' ,[]))
self._allow = _validate_policy(policy.get('allow',[]))
def _handler(request: WPSRequest, response: WPSResponse,
create_context: Mapping[str, Any]) -> WPSResponse:
""" WPS process handler
"""
uuid_str = str(response.uuid)
LOGGER.info("Starting task %s:%s", uuid_str[:8], request.identifier)
alg = QgsApplication.processingRegistry().createAlgorithmById(
request.identifier, create_context)
workdir = response.process.workdir
context = ProcessingContext(workdir, map_uri=request.map_uri)
feedback = Feedback(response, alg.id(), uuid_str=uuid_str)
context.setFeedback(feedback)
context.setInvalidGeometryCheck(QgsFeatureRequest.GeometrySkipInvalid)
# Convert parameters from WPS inputs
parameters = dict(
input_to_processing(ident, inp, alg, context)
for ident, inp in request.inputs.items())
try:
# XXX Warning, a new instance of the algorithm without create_context will be created at the 'run' call
# see https://qgis.org/api/qgsprocessingalgorithm_8cpp_source.html#l00414
# We can deal with that because we will have a QgsProcessingContext and we should not
# rely on the create_context at this time.
results = Processing.runAlgorithm(
alg,
parameters=parameters,
onFinish=handle_algorithm_results,
feedback=feedback,
context=context)
except QgsProcessingException as e:
raise ProcessException("%s" % e)
LOGGER.info("Task finished %s:%s", request.identifier, uuid_str)
handle_layer_outputs(results, context)
# Get WMS output uri
output_uri = config.get_config('server')['wms_response_uri'].format(
host_url=request.host_url, uuid=response.uuid, name=alg.name())
context.response = response
write_outputs(alg, results, response.outputs, output_uri, context)
return response
def _clean_processes():
""" Clean up all processes
Remove status and delete processes workdir
Dangling tasks will be removed: these are tasks no marked finished but
for which the timeout has expired. Usually this is task for wich the process
as died (memory exhaustion, segfault....)
"""
# Iterate over done processes
cfg = config.get_config('server')
rootdir = os.path.abspath(cfg['workdir'])
# The response expiration in seconds
expire_default = cfg.getint('response_expiration')
now_ts = datetime.utcnow().timestamp()
for _, rec in list(logstore.records):
timestamp = rec.get('timestamp')
dangling = timestamp is None
try:
if not dangling and STATUS[rec['status']] < STATUS.DONE_STATUS:
# Check that the task is not in dangling state
timeout = rec.get('timeout')
dangling = timeout is None or (now_ts -
int(timestamp)) >= timeout
if not dangling:
continue
except KeyError:
# Handle legacy status
pass
expiration = rec.get('expiration', expire_default)
notpinned = not rec.get('pinned', False)
if notpinned and (dangling or
(now_ts - int(timestamp)) >= expiration):
# Delete the working directory
uuid_str = rec['uuid']
workdir = os.path.join(rootdir, uuid_str)
LOGGER.info("Cleaning response status: %s", uuid_str)
try:
if os.path.isdir(workdir):
shutil.rmtree(workdir)
except Exception as err:
LOGGER.error('Unable to remove directory: %s', err)
# Delete the record/response
logstore.delete_response(uuid_str)
def schedule_cleanup(self):
""" Schedule a periodic cleanup
"""
interval = config.get_config('server').getint('cleanup_interval')
loop = asyncio.get_event_loop()
async def _run_cleanup():
while True:
await asyncio.sleep(interval)
try:
self._clean_processes()
except Exception as e:
traceback.print_exc()
LOGGER.error("Cleanup task failed: %s", e)
# Schedule the task
self._cleanup_task = asyncio.ensure_future(_run_cleanup())
def initialize(self):
""" Initialize the factory
Should be called once
"""
assert not self._initialized
self._config = config.get_config('processing')
plugin_path = self._config.get('providers_module_path')
exposed_providers = self._config.get('exposed_providers','').split(',')
setup_qgis_paths()
self._wps_interface = WPSServerInterfaceImpl(plugin_path, with_providers=exposed_providers)
self._wps_interface.initialize()
self._create_pool()
def init_session(self):
""" Initialize store session
see https://redis-py.readthedocs.io/en/latest/ for redis options
"""
LOGGER.debug("LOGSTORE: Initializing REDIS session")
cfg = config.get_config('logstorage:redis')
self._config = cfg
self._prefix = cfg.get('prefix', 'pyggiswps')
self._hstatus = "%s:status" % self._prefix
if use_fakeredis:
LOGGER.warning("LOGSTORE: Simulating REDIS connection")
self._db = fakeredis.FakeStrictRedis()
else:
self._db = redis.StrictRedis(host=cfg.get('host', 'localhost'),
port=cfg.getint('port',
fallback=6379),
db=cfg.getint('dbnum', fallback=0))
def __init__(self):
self.operation = None
self.version = None
self.language = None
self.identifiers = None
self.identifier = None
self.store_execute = None
self.status = None
self.lineage = None
self.inputs = None
self.outputs = None
self.raw = None
self.map_uri = None
self.host_url = None
cfg = config.get_config('server')
self.timeout = cfg.getint('response_timeout')
self.expiration = cfg.getint('response_expiration')
async def execute(self, identifier, wps_request, uuid, **context):
"""Parse and perform Execute WPS request call
:param identifier: process identifier string
:param wps_request: pyqgiswps.WPSRequest structure with parsed inputs, still in memory
:param uuid: string identifier of the request
"""
try:
process = self.get_process(identifier, **context)
except UnknownProcessError:
raise InvalidParameterValue("Unknown process '%r'" % identifier, 'Identifier')
# make deep copy of the process instance
# so that processes are not overriding each other
# just for execute
process = copy.deepcopy(process)
self._parse( process, wps_request )
workdir = os.path.abspath(config.get_config('server').get('workdir'))
workdir = os.path.join(workdir, str(uuid))
process.set_workdir(workdir)
# Get status url
status_url = self._status_url(uuid, wps_request)
# Create response object
wps_response = WPSResponse( process, wps_request, uuid, status_url=status_url)
if wps_request.store_execute == 'true':
# Setting STORE_AND_UPDATE_STATUS will trigger
# asynchronous requests
wps_response.status = STATUS.STORE_AND_UPDATE_STATUS
LOGGER.debug("Update status enabled")
if wps_request.raw:
raise NotImplementedError("Raw output is not implemented")
document = await self.executor.execute(wps_request, wps_response)
return document
def _create_pool(self):
""" Initialize the worker pool
"""
cfg = config.get_config('server')
maxparallel = cfg.getint('parallelprocesses')
processlifecycle = cfg.getint('processlifecycle')
response_timeout = cfg.getint('response_timeout')
maxqueuesize = cfg.getint('maxqueuesize')
# Initialize logstore (redis)
logstore.init_session()
# 0 mean eternal life
if processlifecycle == 0:
processlifecycle=None
# Need to be initialized as soon as possible
self._poolserver = create_poolserver( maxparallel, maxcycles = processlifecycle,
initializer = self.worker_initializer,
timeout = response_timeout)
self._initialized = True
def get_capabilities(self, wps_request, accesspolicy=None):
""" Handle getcapbabilities request
"""
process_elements = [p.capabilities_xml()
for p in self.processes if accesspolicy.allow(p.identifier)]
doc = WPS.Capabilities()
doc.attrib['service'] = 'WPS'
doc.attrib['version'] = '1.0.0'
doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US'
doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \
'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsGetCapabilities_response.xsd'
# TODO: check Table 7 in OGC 05-007r7
doc.attrib['updateSequence'] = '1'
metadata = config.get_config('metadata:main')
# Service Identification
service_ident_doc = OWS.ServiceIdentification(
OWS.Title(metadata.get('identification_title'))
)
if metadata.get('identification_abstract'):
service_ident_doc.append(
OWS.Abstract(metadata.get('identification_abstract')))
if metadata.get('identification_keywords'):
keywords_doc = OWS.Keywords()
for k in metadata.get('identification_keywords').split(','):
if k:
keywords_doc.append(OWS.Keyword(k))
service_ident_doc.append(keywords_doc)
if metadata.get('identification_keywords_type'):
keywords_type = OWS.Type(metadata.get('identification_keywords_type'))
keywords_type.attrib['codeSpace'] = 'ISOTC211/19115'
keywords_doc.append(keywords_type)
service_ident_doc.append(OWS.ServiceType('WPS'))
# TODO: set proper version support
service_ident_doc.append(OWS.ServiceTypeVersion('1.0.0'))
service_ident_doc.append(
OWS.Fees(metadata.get('identification_fees')))
for con in metadata.get('identification_accessconstraints').split(','):
service_ident_doc.append(OWS.AccessConstraints(con))
if metadata.get('identification_profile'):
service_ident_doc.append(
OWS.Profile(metadata.get('identification_profile')))
doc.append(service_ident_doc)
# Service Provider
service_prov_doc = OWS.ServiceProvider(
OWS.ProviderName(metadata.get('provider_name')))
if metadata.get('provider_url'):
service_prov_doc.append(OWS.ProviderSite(
{'{http://www.w3.org/1999/xlink}href': metadata.get('provider_url')})
)
# Service Contact
service_contact_doc = OWS.ServiceContact()
# Add Contact information only if a name is set
if metadata.get('contact_name'):
service_contact_doc.append(
OWS.IndividualName(metadata.get('contact_name')))
if metadata.get('contact_position'):
service_contact_doc.append(
OWS.PositionName(metadata.get('contact_position')))
contact_info_doc = OWS.ContactInfo()
phone_doc = OWS.Phone()
if metadata.get('contact_phone'):
phone_doc.append(
OWS.Voice(metadata.get('contact_phone')))
# Add Phone if not empty
if len(phone_doc):
contact_info_doc.append(phone_doc)
address_doc = OWS.Address()
if metadata.get('deliveryPoint'):
address_doc.append(
OWS.DeliveryPoint(metadata.get('contact_address')))
if metadata.get('city'):
address_doc.append(
OWS.City(metadata.get('contact_city')))
if metadata.get('contact_stateorprovince'):
address_doc.append(
OWS.AdministrativeArea(metadata.get('contact_stateorprovince')))
if metadata.get('contact_postalcode'):
address_doc.append(
OWS.PostalCode(metadata.get('contact_postalcode')))
if metadata.get('contact_country'):
address_doc.append(
OWS.Country(metadata.get('contact_country')))
if metadata.get('contact_email'):
address_doc.append(
OWS.ElectronicMailAddress(
metadata.get('contact_email'))
)
# Add Address if not empty
if len(address_doc):
contact_info_doc.append(address_doc)
if metadata.get('contact_url'):
contact_info_doc.append(OWS.OnlineResource(
{'{http://www.w3.org/1999/xlink}href': metadata.get('contact_url')})
)
if metadata.get('contact_hours'):
contact_info_doc.append(
OWS.HoursOfService(metadata.get('contact_hours')))
if metadata.get('contact_instructions'):
contact_info_doc.append(OWS.ContactInstructions(
metadata.get('contact_instructions')))
# Add Contact information if not empty
if len(contact_info_doc):
service_contact_doc.append(contact_info_doc)
if metadata.get('contact_role'):
service_contact_doc.append(
OWS.Role(metadata.get('contact_role')))
# Add Service Contact only if ProviderName and PositionName are set
if len(service_contact_doc):
service_prov_doc.append(service_contact_doc)
doc.append(service_prov_doc)
server_href = {'{http://www.w3.org/1999/xlink}href': config.get_config('server').get('url').format(host_url=wps_request.host_url)}
# Operations Metadata
operations_metadata_doc = OWS.OperationsMetadata(
OWS.Operation(
OWS.DCP(
OWS.HTTP(
OWS.Get(server_href),
OWS.Post(server_href)
)
),
name="GetCapabilities"
),
OWS.Operation(
OWS.DCP(
OWS.HTTP(
OWS.Get(server_href),
OWS.Post(server_href)
)
),
name="DescribeProcess"
),
OWS.Operation(
OWS.DCP(
OWS.HTTP(
OWS.Get(server_href),
OWS.Post(server_href)
)
),
name="Execute"
)
)
doc.append(operations_metadata_doc)
doc.append(WPS.ProcessOfferings(*process_elements))
languages = config.get_config('server').get('language').split(',')
languages_doc = WPS.Languages(
WPS.Default(
OWS.Language(languages[0])
)
)
lang_supported_doc = WPS.Supported()
for l in languages:
lang_supported_doc.append(OWS.Language(l))
languages_doc.append(lang_supported_doc)
doc.append(languages_doc)
return doc
def _construct_doc(self):
doc = WPS.ExecuteResponse()
doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \
'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_response.xsd'
doc.attrib['service'] = 'WPS'
doc.attrib['version'] = '1.0.0'
doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US'
doc.attrib['serviceInstance'] = '%s%s' % (
config.get_config('server').get('url').format(
host_url=self.wps_request.host_url),
'?service=WPS&request=GetCapabilities')
if self.status >= STATUS.STORE_STATUS:
doc.attrib['statusLocation'] = self.status_url
# Process XML
process_doc = WPS.Process(OWS.Identifier(self.process.identifier),
OWS.Title(self.process.title))
if self.process.abstract:
process_doc.append(OWS.Abstract(self.process.abstract))
# TODO: See Table 32 Metadata in OGC 06-121r3
# for m in self.process.metadata:
# process_doc.append(OWS.Metadata(m))
if self.process.profile:
process_doc.append(OWS.Profile(self.process.profile))
process_doc.attrib[
'{http://www.opengis.net/wps/1.0.0}processVersion'] = self.process.version
doc.append(process_doc)
# Status XML
# return the correct response depending on the progress of the process
if self.status == STATUS.STORE_AND_UPDATE_STATUS:
if self.status_percentage == -1:
status_doc = self._process_accepted()
doc.append(status_doc)
return doc
elif self.status_percentage >= 0:
status_doc = self._process_started()
doc.append(status_doc)
return doc
# check if process failed and display fail message
if self.status == STATUS.ERROR_STATUS:
status_doc = self._process_failed()
doc.append(status_doc)
return doc
# TODO: add paused status
if self.status == STATUS.DONE_STATUS:
status_doc = self._process_succeeded()
doc.append(status_doc)
# DataInputs and DataOutputs definition XML if lineage=true
if self.wps_request.lineage == 'true':
data_inputs = [
self.wps_request.inputs[i][0].execute_xml()
for i in self.wps_request.inputs
]
doc.append(WPS.DataInputs(*data_inputs))
output_definitions = [
self.outputs[o].execute_xml_lineage() for o in self.outputs
]
doc.append(WPS.OutputDefinitions(*output_definitions))
# Process outputs XML
output_elements = [
self.outputs[o].execute_xml() for o in self.outputs
]
doc.append(WPS.ProcessOutputs(*output_elements))
return doc
