def start_qgis(self): """ Set up qgis """ # Do not intialize twice if self.qgisapp is not None: return logprefix = "[qgis:%s]" % os.getpid() settings = {} def _folders_setting( setting, folders ): folders = folders.split(';') folders = chain( *(glob(f) for f in folders) ) folders = ';'.join( f for f in folders if os.path.isdir(f) ) if folders: LOGGER.info("%s = %s", setting, folders) settings[setting] = folders # Set up folder settings # XXX Note that if scripts folder is not set then ScriptAlgorithmProvider will crash ! for setting, value in config.get_config().items('qgis.settings.folders'): LOGGER.debug("*** Folder settings: %s = %s", setting, value) _folders_setting(setting, value) # Init qgis application self.qgisapp = start_qgis_application( enable_processing=True, verbose=config.get_config('logging').get('level')=='DEBUG', logger=LOGGER, logprefix=logprefix, settings=settings) # Load plugins self._wps_interface.register_providers() return self.qgisapp
def format_output_url(response: WPSResponse, file_name: str) -> str: """ Build output/store url for output file name """ outputurl = config.get_config('server')['outputurl'] return outputurl.format(host_url=response.wps_request.host_url, uuid=response.uuid, file=file_name)
def delete_results(self, uuid): """ Delete process results and status :param uuid: the uuid of the required process. If set to None, return all the stored status. :return: True if the status has been deleted. """ rec = logstore.get_status(uuid) if rec is None: raise FileNotFoundError(uuid) try: if STATUS[rec['status']] < STATUS.DONE_STATUS: return False except KeyError: # Handle legacy status pass cfg = config.get_config('server') rootdir = os.path.abspath(cfg['workdir']) # Delete the working directory uuid_str = rec['uuid'] workdir = os.path.join(rootdir, uuid_str) LOGGER.info("Cleaning response status: %s", uuid_str) try: if os.path.isdir(workdir): shutil.rmtree(workdir) except Exception as err: LOGGER.error('Unable to remove directory: %s: %s', workdir, err) # Delete the record/response logstore.delete_response(uuid_str) return True
def _status_url(self, uuid, request): """ Return the status_url for the process <uuid> """ cfg = config.get_config('server') status_url = cfg['status_url'] proxy_host = cfg['host_proxy'] if not proxy_host: # Need to return the 'real' host proxy_host = request.host_url if request else '{host_url}' return status_url.format(host_url=proxy_host,uuid=uuid)
def __init__(self, processes=[]): self.processes = {} self._context_processes = lrucache(50) cfg = config.get_config('server') maxqueuesize = cfg.getint('maxqueuesize') self._pool = create_client(maxqueuesize) self._factory = get_process_factory() self.install_processes(processes) # Launch the cleanup task self.schedule_cleanup()
def init(self, filepath: Union[str,Path]=None) -> None: """ Load policy file """ if not isinstance(filepath,Path): filepath = Path(filepath or get_config('processing')['accesspolicy']) if not filepath.exists(): return LOGGER.info("Loading access policy from %s", filepath.as_posix()) with filepath.open('r') as f: policy = yaml.load(f, yaml.SafeLoader) self._deny = _validate_policy(policy.get('deny' ,[])) self._allow = _validate_policy(policy.get('allow',[]))
def _handler(request: WPSRequest, response: WPSResponse, create_context: Mapping[str, Any]) -> WPSResponse: """ WPS process handler """ uuid_str = str(response.uuid) LOGGER.info("Starting task %s:%s", uuid_str[:8], request.identifier) alg = QgsApplication.processingRegistry().createAlgorithmById( request.identifier, create_context) workdir = response.process.workdir context = ProcessingContext(workdir, map_uri=request.map_uri) feedback = Feedback(response, alg.id(), uuid_str=uuid_str) context.setFeedback(feedback) context.setInvalidGeometryCheck(QgsFeatureRequest.GeometrySkipInvalid) # Convert parameters from WPS inputs parameters = dict( input_to_processing(ident, inp, alg, context) for ident, inp in request.inputs.items()) try: # XXX Warning, a new instance of the algorithm without create_context will be created at the 'run' call # see https://qgis.org/api/qgsprocessingalgorithm_8cpp_source.html#l00414 # We can deal with that because we will have a QgsProcessingContext and we should not # rely on the create_context at this time. results = Processing.runAlgorithm( alg, parameters=parameters, onFinish=handle_algorithm_results, feedback=feedback, context=context) except QgsProcessingException as e: raise ProcessException("%s" % e) LOGGER.info("Task finished %s:%s", request.identifier, uuid_str) handle_layer_outputs(results, context) # Get WMS output uri output_uri = config.get_config('server')['wms_response_uri'].format( host_url=request.host_url, uuid=response.uuid, name=alg.name()) context.response = response write_outputs(alg, results, response.outputs, output_uri, context) return response
def _clean_processes(): """ Clean up all processes Remove status and delete processes workdir Dangling tasks will be removed: these are tasks no marked finished but for which the timeout has expired. Usually this is task for wich the process as died (memory exhaustion, segfault....) """ # Iterate over done processes cfg = config.get_config('server') rootdir = os.path.abspath(cfg['workdir']) # The response expiration in seconds expire_default = cfg.getint('response_expiration') now_ts = datetime.utcnow().timestamp() for _, rec in list(logstore.records): timestamp = rec.get('timestamp') dangling = timestamp is None try: if not dangling and STATUS[rec['status']] < STATUS.DONE_STATUS: # Check that the task is not in dangling state timeout = rec.get('timeout') dangling = timeout is None or (now_ts - int(timestamp)) >= timeout if not dangling: continue except KeyError: # Handle legacy status pass expiration = rec.get('expiration', expire_default) notpinned = not rec.get('pinned', False) if notpinned and (dangling or (now_ts - int(timestamp)) >= expiration): # Delete the working directory uuid_str = rec['uuid'] workdir = os.path.join(rootdir, uuid_str) LOGGER.info("Cleaning response status: %s", uuid_str) try: if os.path.isdir(workdir): shutil.rmtree(workdir) except Exception as err: LOGGER.error('Unable to remove directory: %s', err) # Delete the record/response logstore.delete_response(uuid_str)
def schedule_cleanup(self): """ Schedule a periodic cleanup """ interval = config.get_config('server').getint('cleanup_interval') loop = asyncio.get_event_loop() async def _run_cleanup(): while True: await asyncio.sleep(interval) try: self._clean_processes() except Exception as e: traceback.print_exc() LOGGER.error("Cleanup task failed: %s", e) # Schedule the task self._cleanup_task = asyncio.ensure_future(_run_cleanup())
def initialize(self): """ Initialize the factory Should be called once """ assert not self._initialized self._config = config.get_config('processing') plugin_path = self._config.get('providers_module_path') exposed_providers = self._config.get('exposed_providers','').split(',') setup_qgis_paths() self._wps_interface = WPSServerInterfaceImpl(plugin_path, with_providers=exposed_providers) self._wps_interface.initialize() self._create_pool()
def init_session(self): """ Initialize store session see https://redis-py.readthedocs.io/en/latest/ for redis options """ LOGGER.debug("LOGSTORE: Initializing REDIS session") cfg = config.get_config('logstorage:redis') self._config = cfg self._prefix = cfg.get('prefix', 'pyggiswps') self._hstatus = "%s:status" % self._prefix if use_fakeredis: LOGGER.warning("LOGSTORE: Simulating REDIS connection") self._db = fakeredis.FakeStrictRedis() else: self._db = redis.StrictRedis(host=cfg.get('host', 'localhost'), port=cfg.getint('port', fallback=6379), db=cfg.getint('dbnum', fallback=0))
def __init__(self): self.operation = None self.version = None self.language = None self.identifiers = None self.identifier = None self.store_execute = None self.status = None self.lineage = None self.inputs = None self.outputs = None self.raw = None self.map_uri = None self.host_url = None cfg = config.get_config('server') self.timeout = cfg.getint('response_timeout') self.expiration = cfg.getint('response_expiration')
async def execute(self, identifier, wps_request, uuid, **context): """Parse and perform Execute WPS request call :param identifier: process identifier string :param wps_request: pyqgiswps.WPSRequest structure with parsed inputs, still in memory :param uuid: string identifier of the request """ try: process = self.get_process(identifier, **context) except UnknownProcessError: raise InvalidParameterValue("Unknown process '%r'" % identifier, 'Identifier') # make deep copy of the process instance # so that processes are not overriding each other # just for execute process = copy.deepcopy(process) self._parse( process, wps_request ) workdir = os.path.abspath(config.get_config('server').get('workdir')) workdir = os.path.join(workdir, str(uuid)) process.set_workdir(workdir) # Get status url status_url = self._status_url(uuid, wps_request) # Create response object wps_response = WPSResponse( process, wps_request, uuid, status_url=status_url) if wps_request.store_execute == 'true': # Setting STORE_AND_UPDATE_STATUS will trigger # asynchronous requests wps_response.status = STATUS.STORE_AND_UPDATE_STATUS LOGGER.debug("Update status enabled") if wps_request.raw: raise NotImplementedError("Raw output is not implemented") document = await self.executor.execute(wps_request, wps_response) return document
def _create_pool(self): """ Initialize the worker pool """ cfg = config.get_config('server') maxparallel = cfg.getint('parallelprocesses') processlifecycle = cfg.getint('processlifecycle') response_timeout = cfg.getint('response_timeout') maxqueuesize = cfg.getint('maxqueuesize') # Initialize logstore (redis) logstore.init_session() # 0 mean eternal life if processlifecycle == 0: processlifecycle=None # Need to be initialized as soon as possible self._poolserver = create_poolserver( maxparallel, maxcycles = processlifecycle, initializer = self.worker_initializer, timeout = response_timeout) self._initialized = True
def get_capabilities(self, wps_request, accesspolicy=None): """ Handle getcapbabilities request """ process_elements = [p.capabilities_xml() for p in self.processes if accesspolicy.allow(p.identifier)] doc = WPS.Capabilities() doc.attrib['service'] = 'WPS' doc.attrib['version'] = '1.0.0' doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US' doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \ 'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsGetCapabilities_response.xsd' # TODO: check Table 7 in OGC 05-007r7 doc.attrib['updateSequence'] = '1' metadata = config.get_config('metadata:main') # Service Identification service_ident_doc = OWS.ServiceIdentification( OWS.Title(metadata.get('identification_title')) ) if metadata.get('identification_abstract'): service_ident_doc.append( OWS.Abstract(metadata.get('identification_abstract'))) if metadata.get('identification_keywords'): keywords_doc = OWS.Keywords() for k in metadata.get('identification_keywords').split(','): if k: keywords_doc.append(OWS.Keyword(k)) service_ident_doc.append(keywords_doc) if metadata.get('identification_keywords_type'): keywords_type = OWS.Type(metadata.get('identification_keywords_type')) keywords_type.attrib['codeSpace'] = 'ISOTC211/19115' keywords_doc.append(keywords_type) service_ident_doc.append(OWS.ServiceType('WPS')) # TODO: set proper version support service_ident_doc.append(OWS.ServiceTypeVersion('1.0.0')) service_ident_doc.append( OWS.Fees(metadata.get('identification_fees'))) for con in metadata.get('identification_accessconstraints').split(','): service_ident_doc.append(OWS.AccessConstraints(con)) if metadata.get('identification_profile'): service_ident_doc.append( OWS.Profile(metadata.get('identification_profile'))) doc.append(service_ident_doc) # Service Provider service_prov_doc = OWS.ServiceProvider( OWS.ProviderName(metadata.get('provider_name'))) if metadata.get('provider_url'): service_prov_doc.append(OWS.ProviderSite( {'{http://www.w3.org/1999/xlink}href': metadata.get('provider_url')}) ) # Service Contact service_contact_doc = OWS.ServiceContact() # Add Contact information only if a name is set if metadata.get('contact_name'): service_contact_doc.append( OWS.IndividualName(metadata.get('contact_name'))) if metadata.get('contact_position'): service_contact_doc.append( OWS.PositionName(metadata.get('contact_position'))) contact_info_doc = OWS.ContactInfo() phone_doc = OWS.Phone() if metadata.get('contact_phone'): phone_doc.append( OWS.Voice(metadata.get('contact_phone'))) # Add Phone if not empty if len(phone_doc): contact_info_doc.append(phone_doc) address_doc = OWS.Address() if metadata.get('deliveryPoint'): address_doc.append( OWS.DeliveryPoint(metadata.get('contact_address'))) if metadata.get('city'): address_doc.append( OWS.City(metadata.get('contact_city'))) if metadata.get('contact_stateorprovince'): address_doc.append( OWS.AdministrativeArea(metadata.get('contact_stateorprovince'))) if metadata.get('contact_postalcode'): address_doc.append( OWS.PostalCode(metadata.get('contact_postalcode'))) if metadata.get('contact_country'): address_doc.append( OWS.Country(metadata.get('contact_country'))) if metadata.get('contact_email'): address_doc.append( OWS.ElectronicMailAddress( metadata.get('contact_email')) ) # Add Address if not empty if len(address_doc): contact_info_doc.append(address_doc) if metadata.get('contact_url'): contact_info_doc.append(OWS.OnlineResource( {'{http://www.w3.org/1999/xlink}href': metadata.get('contact_url')}) ) if metadata.get('contact_hours'): contact_info_doc.append( OWS.HoursOfService(metadata.get('contact_hours'))) if metadata.get('contact_instructions'): contact_info_doc.append(OWS.ContactInstructions( metadata.get('contact_instructions'))) # Add Contact information if not empty if len(contact_info_doc): service_contact_doc.append(contact_info_doc) if metadata.get('contact_role'): service_contact_doc.append( OWS.Role(metadata.get('contact_role'))) # Add Service Contact only if ProviderName and PositionName are set if len(service_contact_doc): service_prov_doc.append(service_contact_doc) doc.append(service_prov_doc) server_href = {'{http://www.w3.org/1999/xlink}href': config.get_config('server').get('url').format(host_url=wps_request.host_url)} # Operations Metadata operations_metadata_doc = OWS.OperationsMetadata( OWS.Operation( OWS.DCP( OWS.HTTP( OWS.Get(server_href), OWS.Post(server_href) ) ), name="GetCapabilities" ), OWS.Operation( OWS.DCP( OWS.HTTP( OWS.Get(server_href), OWS.Post(server_href) ) ), name="DescribeProcess" ), OWS.Operation( OWS.DCP( OWS.HTTP( OWS.Get(server_href), OWS.Post(server_href) ) ), name="Execute" ) ) doc.append(operations_metadata_doc) doc.append(WPS.ProcessOfferings(*process_elements)) languages = config.get_config('server').get('language').split(',') languages_doc = WPS.Languages( WPS.Default( OWS.Language(languages[0]) ) ) lang_supported_doc = WPS.Supported() for l in languages: lang_supported_doc.append(OWS.Language(l)) languages_doc.append(lang_supported_doc) doc.append(languages_doc) return doc
def _construct_doc(self): doc = WPS.ExecuteResponse() doc.attrib['{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'] = \ 'http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_response.xsd' doc.attrib['service'] = 'WPS' doc.attrib['version'] = '1.0.0' doc.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = 'en-US' doc.attrib['serviceInstance'] = '%s%s' % ( config.get_config('server').get('url').format( host_url=self.wps_request.host_url), '?service=WPS&request=GetCapabilities') if self.status >= STATUS.STORE_STATUS: doc.attrib['statusLocation'] = self.status_url # Process XML process_doc = WPS.Process(OWS.Identifier(self.process.identifier), OWS.Title(self.process.title)) if self.process.abstract: process_doc.append(OWS.Abstract(self.process.abstract)) # TODO: See Table 32 Metadata in OGC 06-121r3 # for m in self.process.metadata: # process_doc.append(OWS.Metadata(m)) if self.process.profile: process_doc.append(OWS.Profile(self.process.profile)) process_doc.attrib[ '{http://www.opengis.net/wps/1.0.0}processVersion'] = self.process.version doc.append(process_doc) # Status XML # return the correct response depending on the progress of the process if self.status == STATUS.STORE_AND_UPDATE_STATUS: if self.status_percentage == -1: status_doc = self._process_accepted() doc.append(status_doc) return doc elif self.status_percentage >= 0: status_doc = self._process_started() doc.append(status_doc) return doc # check if process failed and display fail message if self.status == STATUS.ERROR_STATUS: status_doc = self._process_failed() doc.append(status_doc) return doc # TODO: add paused status if self.status == STATUS.DONE_STATUS: status_doc = self._process_succeeded() doc.append(status_doc) # DataInputs and DataOutputs definition XML if lineage=true if self.wps_request.lineage == 'true': data_inputs = [ self.wps_request.inputs[i][0].execute_xml() for i in self.wps_request.inputs ] doc.append(WPS.DataInputs(*data_inputs)) output_definitions = [ self.outputs[o].execute_xml_lineage() for o in self.outputs ] doc.append(WPS.OutputDefinitions(*output_definitions)) # Process outputs XML output_elements = [ self.outputs[o].execute_xml() for o in self.outputs ] doc.append(WPS.ProcessOutputs(*output_elements)) return doc