def test_wps_describeprocess_emu_all(): # Initialize WPS client wps = WebProcessingService('http://localhost:8094/wps', skip_caps=True) # Execute fake invocation of DescribeProcess operation by parsing cached response from xml = open(resource_file('wps_EmuDescribeProcess_all.xml'), 'rb').read() process = wps.describeprocess('nap', xml=xml) processes = wps.describeprocess('all', xml=xml) assert isinstance(process, Process) assert isinstance(processes, list)
def test_describe(): """Check that owslib can parse the processes' description.""" from owslib.wps import WebProcessingService wps = WebProcessingService(URL, skip_caps=True) client = client_for(Service(processes=processes)) resp = client.get(service="wps", request="getcapabilities", version="1.0.0") wps.describeprocess("all", xml=resp.data)
def run_wps(process_id,input,output): #choose the first wps engine my_engine = WebProcessingService('http://appsdev.hydroshare.org:8282/wps/WebProcessingService', verbose=False, skip_caps=True) my_engine.getcapabilities() my_process = my_engine.describeprocess(process_id) my_inputs = my_process.dataInputs input_names = [] #getting list of input for input1 in my_inputs: input_names.append(input1) #executing the process.. execution = my_engine.execute(process_id, input, output) request = execution.request #set store executeresponse to false request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"') url_wps = 'http://appsdev.hydroshare.org:8282/wps/WebProcessingService' wps_request = urllib2.Request(url_wps,request) wps_open = urllib2.urlopen(wps_request) wps_read = wps_open.read() if 'href' in wps_read: tag = 'href="' location = wps_read.find(tag) new= wps_read[location+len(tag):len(wps_read)] tag2 = '"/>\n </wps:Output>\n </wps:ProcessOutputs>\n</wps:' location2 = new.find(tag2) final = new[0:location2] split = final.split() wps_request1 = urllib2.Request(split[0]) wps_open1 = urllib2.urlopen(wps_request1) wps_read1 = wps_open1.read() #return [final_output_url, final_data] return [wps_read1, split]
def create_from_process(cls, url, identifier, request_template, username=None, password=None, output_hook=None): try: wps = WebProcessingService(url, username=username, password=password, verbose=False, skip_caps=False) process = wps.describeprocess(identifier) if process: _f = open(request_template) _run = cls.objects.create(identifier=identifier, title=process.title, abstract=process.abstract, version=wps.version, url=url, service_instance=wps.url, username=username, password=password, request_template=File(_f), output_hook=output_hook) _run._initialized = True _run._wps = wps return _run except: log.exception("Could not create Process!") raise return wps.id
def test_wps_describeprocess_ceda(): # Initialize WPS client wps = WebProcessingService('http://ceda-wps2.badc.rl.ac.uk/wps', skip_caps=True) # Execute fake invocation of DescribeProcess operation by parsing cached response from CEDA service xml = open(resource_file('wps_CEDADescribeProcess.xml'), 'rb').read() process = wps.describeprocess('Doubleit', xml=xml) # Check process description assert process.identifier == 'DoubleIt' assert process.title == 'Doubles the input number and returns value' assert process.abstract == 'This is test process used to demonstrate how the WPS and the WPS User Interface work. The process accepts an integer or floating point number and returns some XML containing the input number double.' # NOQA # Check process properties assert process.statusSupported is False assert process.storeSupported is True # Check process inputs # Example Input: # identifier=NumberToDouble, title=NumberToDouble, abstract=NumberToDouble, data type=LiteralData # Any value allowed # Default Value: None # minOccurs=1, maxOccurs=-1 for input in process.dataInputs: assert input.identifier == 'NumberToDouble' assert input.dataType == 'LiteralData' # Example Output: # identifier=OutputXML, title=OutputXML, abstract=OutputXML, data type=ComplexData # Supported Value: mimeType=text/XML, encoding=UTF-8, schema=NONE # Default Value: None # reference=None, mimeType=None # Check process outputs for output in process.processOutputs: assert output.identifier == 'OutputXML' assert output.dataType == 'ComplexData'
def test_wps_describeprocess_bbox(): # Initialize WPS client wps = WebProcessingService('http://localhost:8094/wps', skip_caps=True) # Execute fake invocation of DescribeProcess operation by parsing cached response from Emu service xml = open(resource_file('wps_bbox_DescribeProcess.xml'), 'rb').read() process = wps.describeprocess('bbox', xml=xml) # Check process description assert process.identifier == 'bbox' assert process.title == 'Bounding Box' # Check process inputs # Example Input: # identifier=bbox, title=Bounding Box, abstract=None, data type=BoundingBoxData # Supported Value: EPSG:4326 # Supported Value: EPSG:3035 # Default Value: EPSG:4326 # minOccurs=1, maxOccurs=1 for input in process.dataInputs: assert input.identifier == 'bbox' assert input.dataType == 'BoundingBoxData' # Example Output: # identifier=bbox, title=Bounding Box, abstract=None, data type=BoundingBoxData # Supported Value: EPSG:4326 # Default Value: EPSG:4326 # reference=None, mimeType=None # Check process outputs for output in process.processOutputs: assert output.identifier == 'bbox' assert output.dataType == 'BoundingBoxData'
def test_wps_describeprocess_ceda(): # Initialize WPS client wps = WebProcessingService('http://ceda-wps2.badc.rl.ac.uk/wps', skip_caps=True) # Execute fake invocation of DescribeProcess operation by parsing cached response from CEDA service xml = open(resource_file('wps_CEDADescribeProcess.xml'), 'rb').read() process = wps.describeprocess('DoubleIt', xml=xml) # Check process description assert process.identifier == 'DoubleIt' assert process.title == 'Doubles the input number and returns value' assert process.abstract == 'This is test process used to demonstrate how the WPS and the WPS User Interface work. The process accepts an integer or floating point number and returns some XML containing the input number double.' # NOQA # Check process properties assert process.statusSupported is False assert process.storeSupported is True # Check process inputs # Example Input: # identifier=NumberToDouble, title=NumberToDouble, abstract=NumberToDouble, data type=LiteralData # Any value allowed # Default Value: None # minOccurs=1, maxOccurs=-1 for input in process.dataInputs: assert input.identifier == 'NumberToDouble' assert input.dataType == 'LiteralData' # Example Output: # identifier=OutputXML, title=OutputXML, abstract=OutputXML, data type=ComplexData # Supported Value: mimeType=text/XML, encoding=UTF-8, schema=NONE # Default Value: None # reference=None, mimeType=None # Check process outputs for output in process.processOutputs: assert output.identifier == 'OutputXML' assert output.dataType == 'ComplexData'
class ExecuteProcess(Processes): def __init__(self, request): self.wps_id = request.params.get("wps") self.wps = WebProcessingService(url=wps_url(request, self.wps_id)) identifier = request.params.get("process") # TODO: need to fix owslib to handle special identifiers self.process = self.wps.describeprocess(identifier) super(ExecuteProcess, self).__init__(request, name="processes_execute", title="") def breadcrumbs(self): breadcrumbs = super(ExecuteProcess, self).breadcrumbs() route_path = self.request.route_path("processes_list", _query=[("wps", self.wps_id)]) breadcrumbs.append(dict(route_path=route_path, title=self.wps.identification.title)) breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.process.title)) return breadcrumbs def appstruct(self): return {} def generate_form(self, formid="deform"): from phoenix.schema.wps import WPSSchema schema = WPSSchema(process=self.process, user=self.get_user()) return Form(schema, buttons=("submit",), formid=formid) def process_form(self, form): controls = self.request.POST.items() try: appstruct = form.validate(controls) self.execute(appstruct) except ValidationFailure, e: logger.exception("validation of exectue view failed.") self.session.flash("There are errors on this page.", queue="danger") return dict(description=getattr(self.process, "abstract", ""), form=e.render()) return HTTPFound(location=self.request.route_url("monitor"))
class ComplexInputs(Wizard): def __init__(self, request): super(ComplexInputs, self).__init__( request, name='wizard_complex_inputs', title="Choose Input Parameter") from owslib.wps import WebProcessingService self.wps = WebProcessingService(wps_url(request, self.wizard_state.get('wizard_wps')['identifier']), verify=False) self.process = self.wps.describeprocess(self.wizard_state.get('wizard_process')['identifier']) self.title = "Choose Input Parameter of {0}".format(self.process.title) def breadcrumbs(self): breadcrumbs = super(ComplexInputs, self).breadcrumbs() breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title)) return breadcrumbs def schema(self): return Schema().bind(process=self.process) def success(self, appstruct): for input in self.process.dataInputs: if input.identifier == appstruct.get('identifier'): appstruct['mime_types'] = [value.mimeType for value in input.supportedValues] super(ComplexInputs, self).success(appstruct) def next_success(self, appstruct): self.success(appstruct) return self.next('wizard_source') @view_config(route_name='wizard_complex_inputs', renderer='../templates/wizard/default.pt') def view(self): return super(ComplexInputs, self).view()
def pinned_processes(request): from owslib.wps import WebProcessingService settings = request.db.settings.find_one() or {} processes = [] if 'pinned_processes' in settings: for pinned in settings.get('pinned_processes'): try: service_name, identifier = pinned.split('.', 1) url = request.route_path('processes_execute', _query=[('wps', service_name), ('process', identifier)]) wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=service_name), verify=False) # TODO: need to fix owslib to handle special identifiers process = wps.describeprocess(identifier) description = headline(process.abstract) except Exception: LOGGER.warn("could not add pinned process %s", pinned) else: processes.append( dict(title=process.identifier, description=description, url=url, service_title=wps.identification.title)) return processes
def call_wps(args): if not args.token: user_id = args.user if not user_id: logging.error("No user id found on the call, aborting!") sys.exit(1) user_token_file = os.path.join("/etc/d4science/", user_id) with open(user_token_file, "r") as f: gcube_vre_token = f.read() else: gcube_vre_token = args.token.encode("utf-8") logging.info("User: %s", args.user) logging.info("Token: (SHA256) %s", hashlib.sha256(gcube_vre_token).hexdigest()) gcube_vre_token_header = {"gcube-token": gcube_vre_token} dataminer_url = ( "http://dataminer-prototypes.d4science.org/wps/" "WebProcessingService" ) wps = WebProcessingService(dataminer_url, headers=gcube_vre_token_header) process_id = args.process process = wps.describeprocess(process_id) inputs = build_inputs(process, args.input, args.inputdata, gcube_vre_token) outputs = [(o.identifier, True) for o in process.processOutputs] # execute the process execution = wps.execute(process_id, inputs, outputs) monitorExecution(execution, sleepSecs=5, download=True) logging.info("Execution status: %s", execution.status) exit_code = 0 if execution.status == "ProcessSucceeded" else 1 logging.info("Exit code: %d", exit_code) produce_output(execution, args.output, args.outdir, gcube_vre_token_header) return exit_code
def describe_process(identifier, wps_host=None, wps_client=None, version='1.0.0'): """WPS Describe Process response. Parameters ---------- identifer : string wps_host : string wps_client : pywps.tests.WpsClient version : string Returns ------- out : list of ? """ if wps_host: wps = WebProcessingService(wps_host, version) return wps.describeprocess(identifier) else: response = wps_client.get( ('?service=WPS&request=DescribeProcess&version={0}&' 'identifier={1}').format(version, identifier)) wps_reader = WPSReader() element = wps_reader.readFromString(response.get_data()) wps = WebProcessingService(None, version, skip_caps=True) return wps._parseProcessMetadata(element)
def run_wps(process_id,input,output): #choose the first wps engine #my_engine = WebProcessingService('http://appsdev.hydroshare.org:8282/wps/WebProcessingService', verbose=False, skip_caps=True) my_engine = WebProcessingService('http://appsdev.hydroshare.org:8282/wps/WebProcessingService',verbose=False, skip_caps=True) my_engine.getcapabilities() #wps_engines = list_wps_service_engines() #my_engine = wps_engines[0] #choose the r.time-series-converter my_process = my_engine.describeprocess(process_id) my_inputs = my_process.dataInputs input_names = [] #getting list of input for input1 in my_inputs: input_names.append(input1) #executing the process.. execution = my_engine.execute(process_id, input, output) request = execution.request #set store executeresponse to false request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"') url_wps = 'http://appsdev.hydroshare.org:8282/wps/WebProcessingService' wps_request = urllib2.Request(url_wps,request) wps_open = urllib2.urlopen(wps_request) wps_read = wps_open.read() if 'href' in wps_read: tag = 'href="' location = wps_read.find(tag) new= wps_read[location+len(tag):len(wps_read)] tag2 = '"/>\n </wps:Output>\n </wps:ProcessOutputs>\n</wps:' location2 = new.find(tag2) final = new[0:location2] split = final.split() wps_request1 = urllib2.Request(split[0]) wps_open1 = urllib2.urlopen(wps_request1) wps_read1 = wps_open1.read() #now we must use our own method to send the request1 #we need to use the request #this code is for the normal wps which is not working right now # monitorExecution(execution) # output_data = execution.processOutputs # final_output_url = output_data[0].reference # final_data = read_final_data(final_output_url) #return [final_output_url, final_data] return [wps_read1, split]
class pyGDPwebProcessing(): """ This class allows interactive calls to be made into the GDP. """ def __init__(self, wfs_url=WFS_URL): # if WFS_URL is None: # from pygdp.namespaces import WFS_URL # wfsUrl = WFS_URL self.wfsUrl = wfs_url self.wpsUrl = WPS_URL self.version = '1.1.0' self.wps = WebProcessingService(self.wpsUrl) def WPSgetCapabilities(self, xml=None): """ Returns a list of capabilities. """ self.wps.getcapabilities(xml) def WPSdescribeprocess(self, identifier, xml=None): """ Returns a list describing a specific identifier/process. """ self.wps.describeprocess(identifier, xml) # pyGDP Submit Feature def dodsReplace(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return _execute_request.dodsReplace(dataSetURI, verbose) def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage=True, outputfname=None, sleepSecs=10, async=False): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCoverageOPenDAP(geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs, async=async)
def describe_process(identifier, wps_host=None, wps_client=None, version='1.0.0'): if wps_host: wps = WebProcessingService(wps_host, version) return wps.describeprocess(identifier) else: response = wps_client.get( ('?service=WPS&request=DescribeProcess&version={0}&' 'identifier={1}').format(version, identifier)) wps_reader = WPSReader() element = wps_reader.readFromString(response.get_data()) wps = WebProcessingService(None, version, skip_caps=True) return wps._parseProcessMetadata(element)
def run(self): responseToReturn = Response() if self.identifier != "": try: wps = WebProcessingService(self.url) process = wps.describeprocess(self.identifier) responseToReturn.status = 200 responseToReturn.data = process except: responseToReturn.status = 500 else: responseToReturn.status = 500 self.statusChanged.emit(responseToReturn)
def run_wps(res_ids,gap): if (gap ==''): gap = "linear" # checks if there is two resource IDs resources = res_ids.split("_") process_id = 'org.n52.wps.server.r.series_gap_filler_3' process_input = [('resource_id',str(resources[0])),('fill_function',str(gap))] #setting the WPS URL is set in app.py url_wps = GapFillerTool.wps_url + '/WebProcessingService' my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True) my_process = my_engine.describeprocess(process_id) #executing the process.. # build execution execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username, password=my_engine.password, verbose=my_engine.verbose) requestElement = execution.buildRequest(process_id, process_input, 'output') request = etree.tostring(requestElement) #set store executeresponse to false request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"') execution = my_engine.execute(process_id, process_input, 'output', request) monitorExecution(execution) status = execution.status # if the status is successful... if status == 'ProcessSucceeded': outputs = execution.processOutputs output0 = outputs[0] reference0 = output0.reference # retrieve the data from the reference output_data = requests.get(reference0) resp = HttpResponse(output_data, content_type="application/json") return resp else: return JsonResponse({'status': 'wps request failed'})
class ChooseSource(Wizard): def __init__(self, request): super(ChooseSource, self).__init__(request, name='wizard_source', title="Choose Data Source") self.wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.wizard_state.get('wizard_wps')['identifier']), verify=False, skip_caps=True) self.process = self.wps.describeprocess( self.wizard_state.get('wizard_process')['identifier']) for data_input in self.process.dataInputs: if data_input.identifier == self.wizard_state.get( 'wizard_complex_inputs')['identifier']: self.title = "Choose Data Source for %s" % data_input.title break def breadcrumbs(self): breadcrumbs = super(ChooseSource, self).breadcrumbs() breadcrumbs.append( dict(route_path=self.request.route_path(self.name), title=self.title)) return breadcrumbs def schema(self): return Schema().bind(request=self.request) def next_success(self, appstruct): self.success(appstruct) # TODO: that is a dirty way to init esgf search if appstruct.get('source') == 'wizard_esgf_search': defaults = dict(constraints=process_constraints(self.process)) query = query_params_from_appstruct( self.wizard_state.get('wizard_esgf_search'), defaults) if not self.request.cert_ok: msg = """<strong>Error:</strong> You are not allowed to access ESGF data. Please <a href="{}" class="alert-link">update</a> your ESGF credentials.""" callback = self.request.current_route_path() self.session.flash(msg.format( self.request.route_path('esgflogon', _query=[('callback', callback)])), queue='danger') return self.next(self.name) else: query = None return self.next(appstruct.get('source'), query=query) def view(self): return super(ChooseSource, self).view()
def describe_provider_process(request): # type: (Request) -> Process """ Obtains a remote service process description in a compatible local process format. Note: this processes won't be stored to the local process storage. """ provider_id = request.matchdict.get("provider_id") process_id = request.matchdict.get("process_id") store = get_db(request).get_store(StoreServices) service = store.fetch_by_name(provider_id) wps = WebProcessingService(url=service.url, headers=get_cookie_headers(request.headers)) set_wps_language(wps, request=request) process = wps.describeprocess(process_id) return Process.from_ows(service, process, get_settings(request))
def __init__(self, request): super(ChooseSource, self).__init__(request, name='wizard_source', title="Choose Data Source") wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.wizard_state.get('wizard_wps')['identifier']), verify=False, skip_caps=True) process = wps.describeprocess( self.wizard_state.get('wizard_process')['identifier']) for data_input in process.dataInputs: if data_input.identifier == self.wizard_state.get( 'wizard_complex_inputs')['identifier']: self.title = "Choose Data Source for %s" % data_input.title break
def run_wps(res_ids): print"launch wps" print res_ids # checks if there is two resource IDs resources = res_ids.split("_") if len(resources) < 2: return JsonResponse({'status': 'wps request failed. 2 resources are required, found ' + str(len(resources))}) process_id = 'org.n52.wps.server.r.linear_regression' process_input = [('x_resource_id',str(resources[0])),('y_resource_id',str(resources[1]))] #setting the WPS URL is set in app.py url_wps = CorrelationPlot.wps_url + '/WebProcessingService' my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True) my_process = my_engine.describeprocess(process_id) #executing the process.. # build execution execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username, password=my_engine.password, verbose=my_engine.verbose) requestElement = execution.buildRequest(process_id, process_input, 'output') request = etree.tostring(requestElement) #set store executeresponse to false request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"') print request execution = my_engine.execute(process_id, process_input, 'output', request) monitorExecution(execution) status = execution.status print status # if the status is successful... if status == 'ProcessSucceeded': outputs = execution.processOutputs output0 = outputs[0] reference0 = output0.reference # retrieve the data from the reference output_data = requests.get(reference0) resp = HttpResponse(output_data, content_type="application/json") return resp else: return JsonResponse({'status': 'wps request failed'})
class ComplexInputs(Wizard): def __init__(self, request): super(ComplexInputs, self).__init__(request, name='wizard_complex_inputs', title="Choose Input Parameter") self.wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.wizard_state.get('wizard_wps')['identifier']), verify=False, skip_caps=True) self.process = self.wps.describeprocess( self.wizard_state.get('wizard_process')['identifier']) self.title = "Choose Input Parameter of {0}".format(self.process.title) def breadcrumbs(self): breadcrumbs = super(ComplexInputs, self).breadcrumbs() breadcrumbs.append( dict(route_path=self.request.route_path(self.name), title=self.title)) return breadcrumbs def schema(self): return Schema().bind(request=self.request, process=self.process) def success(self, appstruct): for inp in self.process.dataInputs: if inp.identifier == appstruct.get('identifier'): appstruct['mime_types'] = [ value.mimeType for value in inp.supportedValues ] super(ComplexInputs, self).success(appstruct) def next_success(self, appstruct): self.success(appstruct) return self.next('wizard_source') def view(self): return super(ComplexInputs, self).view() def custom_view(self): return dict(summary_title=self.process.title, summary=getattr(self.process, 'abstract', 'No summary'), url=wps_describe_url(self.wps.url, self.process.identifier), metadata=self.process.metadata)
def fill_section(section, gcube_vre_token, tool_dir): gcube_vre_token_header = {"gcube-token": gcube_vre_token} dataminer_url = ("http://dataminer-prototypes.d4science.org/wps/" "WebProcessingService") wps = WebProcessingService(dataminer_url, headers=gcube_vre_token_header) tools = {} tools["CSV extractor"] = {"file": os.path.join(tool_dir, "extract.xml")} for process in wps.processes: descr = wps.describeprocess(process.identifier) tools[descr.title] = {"descr": descr, "process": process, "file": None} for i, t in enumerate(sorted(tools)): tool_file = os.path.join(tool_dir, "tool%02d.xml" % i) if tools[t]["file"]: shutil.copyfile(tools[t]["file"], tool_file) else: generate_tool_description(tools[t]["process"], tools[t]["descr"], tool_file) etree.SubElement(section, "tool", attrib={"file": tool_file})
class ExecuteProcess(Processes): def __init__(self, request): self.wps_id = request.params.get('wps') self.wps = WebProcessingService(url=wps_url(request, self.wps_id), verify=False) identifier = request.params.get('process') # TODO: need to fix owslib to handle special identifiers self.process = self.wps.describeprocess(identifier) super(ExecuteProcess, self).__init__(request, name='processes_execute', title='') def breadcrumbs(self): breadcrumbs = super(ExecuteProcess, self).breadcrumbs() route_path = self.request.route_path('processes_list', _query=[('wps', self.wps_id)]) breadcrumbs.append(dict(route_path=route_path, title=self.wps.identification.title)) breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.process.title)) return breadcrumbs def appstruct(self): return {} def generate_form(self, formid='deform'): schema = WPSSchema(request=self.request, process=self.process, user=self.get_user()) return Form( schema, buttons=('submit',), formid=formid, ) def process_form(self, form): controls = self.request.POST.items() try: logger.debug("before validate") appstruct = form.validate(controls) logger.debug("before execute %s", appstruct) self.execute(appstruct) except ValidationFailure, e: logger.exception('validation of exectue view failed.') self.session.flash("There are errors on this page.", queue='danger') return dict(description=getattr(self.process, 'abstract', ''), form = e.render()) return HTTPFound(location=self.request.route_url('monitor'))
class LiteralInputs(Wizard): def __init__(self, request): super(LiteralInputs, self).__init__(request, name='wizard_literal_inputs', title="Literal Inputs") self.wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.wizard_state.get('wizard_wps')['identifier']), verify=False, skip_caps=True) self.process = self.wps.describeprocess( self.wizard_state.get('wizard_process')['identifier']) self.title = "Literal inputs of {0}".format(self.process.title) def breadcrumbs(self): breadcrumbs = super(LiteralInputs, self).breadcrumbs() breadcrumbs.append( dict(route_path=self.request.route_path(self.name), title=self.title)) return breadcrumbs def schema(self): return WPSSchema(request=self.request, hide_complex=True, process=self.process).bind(request=self.request) def next_success(self, appstruct): self.success(appstruct) return self.next('wizard_complex_inputs') def view(self): return super(LiteralInputs, self).view() def custom_view(self): return dict(summary_title=self.process.title, summary=getattr(self.process, 'abstract', 'No summary'), url=wps_describe_url(self.wps.url, self.process.identifier), metadata=self.process.metadata)
class RookWPS: def __init__(self, url): self.wps = WebProcessingService(url, verbose=False, skip_caps=True) def getcapabilities(self): self.wps.getcapabilities() return self.wps def describeprocess(self, identifier): return self.wps.describeprocess(identifier) def execute(self, identifier, inputs): outputs = [("output", True, None)] execution = self.wps.execute(identifier, inputs, output=outputs) monitorExecution(execution) print(execution.errors) assert execution.isSucceded() is True assert len(execution.processOutputs) > 0 ml_url = execution.processOutputs[0].reference xml = requests.get(ml_url).text urls = parse_metalink(xml) return urls
def initialize(self): if not self.is_initialized(): try: wps = WebProcessingService(self.url, username=self.username, password=self.password, verbose=False, skip_caps=False) process = wps.describeprocess(self.identifier) if process: self.title = self.title or process.title self.abstract = self.abstract or process.abstract self.version = self.version or wps.version self.service_instance = wps.url self.save() self._initialized = True self._wps = wps self._execution_response = WebProcessingServiceRun.getOWSLibExecutionResponse( self) except: self._initialized = False self._wps = None
def pinned_processes(self): settings = self.request.db.settings.find_one() or {} processes = [] if 'pinned_processes' in settings: for pinned in settings.get('pinned_processes'): try: service_name, identifier = pinned.split('.', 1) url = self.request.route_path( 'processes_execute', _query=[('wps', service_name), ('process', identifier)]) wps = WebProcessingService( url=self.request.route_url('owsproxy', service_name=service_name), verify=False) # TODO: need to fix owslib to handle special identifiers process = wps.describeprocess(identifier) description = headline(process.abstract) except Exception: LOGGER.warn("could not add pinned process %s", pinned) else: processes.append(dict( title=process.identifier, description=description, url=url, service_title=wps.identification.title)) return processes
class LiteralInputs(Wizard): def __init__(self, request): super(LiteralInputs, self).__init__(request, name='wizard_literal_inputs', title="Literal Inputs") from owslib.wps import WebProcessingService self.wps = WebProcessingService(wps_url(request, self.wizard_state.get('wizard_wps')['identifier'])) self.process = self.wps.describeprocess(self.wizard_state.get('wizard_process')['identifier']) self.title = "Literal inputs of {0}".format(self.process.title) def breadcrumbs(self): breadcrumbs = super(LiteralInputs, self).breadcrumbs() breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title)) return breadcrumbs def schema(self): from phoenix.schema.wps import WPSSchema return WPSSchema(hide_complex=True, process = self.process) def next_success(self, appstruct): self.success(appstruct) return self.next('wizard_complex_inputs') @view_config(route_name='wizard_literal_inputs', renderer='../templates/wizard/default.pt') def view(self): return super(LiteralInputs, self).view()
class pyGDPwebProcessing(): """ This class allows interactive calls to be made into the GDP. """ def _init_(self, wfsUrl=WFS_URL, wpsUrl=WPS_URL, version='1.1.0'): self.wfsUrl = wfsUrl self.wpsUrl = wpsUrl self.version = version self.wps = WebProcessingService(wpsUrl) def WPSgetCapbilities(self, xml=None): """ Returns a list of capabilities. """ self.wps.getcapabilities(xml) def WPSdescribeprocess(self, identifier, xml=None): """ Returns a list describing a specific identifier/process. """ self.wps.describeprocess(identifier, xml) def _encodeZipFolder(self, filename): """ This function will encode a zipfile and return the filename. """ #check extension if not filename.endswith('.zip'): print 'Wrong filetype.' return #encode the file with open(filename, 'rb') as fin: bytesRead = fin.read() encode= base64.b64encode(bytesRead) #renames the file and saves it onto local drive filename = filename.split('.') filename = str(filename[0]) + '_copy.' + str(filename[-1]) fout = open(filename, "w") fout.write(encode) fout.close() return filename def uploadShapeFile(self, filePath): """ Given a file, this function encodes the file and uploads it onto geoserver. """ # encodes the file, opens it, reads it, and closes it # returns a filename in form of: filename_copy.zip filePath = self._encodeZipFolder(filePath) if filePath is None: return filehandle = open(filePath, 'r') filedata = filehandle.read() filehandle.close() os.remove(filePath) # deletes the encoded file # this if for naming the file on geoServer filename = filePath.split("/") # gets rid of filepath, keeps only filename eg: file.zip filename = filename[len(filename) - 1] filename = filename.replace("_copy.zip", "") # check to make sure a file with the same name does not exist fileCheckString = "upload:" + filename shapefiles = self.getShapefiles() if fileCheckString in shapefiles: print 'File exists already.' return xmlGen = gdpXMLGenerator() root = xmlGen.getUploadXMLtree(filename, upload_URL, filedata) # now we have a complete XML upload request uploadRequest = etree.tostring(root) POST = WebProcessingService(WPS_Service) execution = POST.execute(None, [], request=uploadRequest) monitorExecution(execution) return "upload:"+filename def getTuples(self, shapefile, attribute): """ Will return the dictionary tuples only. """ return self.getValues(shapefile, attribute, getTuples='only') def _getGMLIDString(self, GMLbeginterm, line, GMLstopterm, valBeginTerm, valStopTerm): """ This function is specific to the output documents from the GDP. This function parses the XML document, to find the correct GMLID associated with a feature. Returns the list of values, and a dictionary [feature:id]. """ # we are searching for attr-value, gml:id pair value = [] ntuple = [] begin_index = 0 end_index = len(line) tmpline = line # start at the beginning while begin_index < len(line): begin_index = tmpline.find(GMLbeginterm) if begin_index != -1: end_index = tmpline.find(GMLstopterm, begin_index) # we get the gml term gmlterm = tmpline[begin_index + len(GMLbeginterm) : end_index ] # now we get the attribute value begin_index2 = tmpline.find(valBeginTerm) end_index2 = tmpline.find(valStopTerm, begin_index2) valTerm = tmpline[begin_index2 + len(valBeginTerm) : end_index2 ] #tuple: attr-value, gml:id tmpTuple = valTerm, gmlterm ntuple.append(tmpTuple) tmpline = tmpline[end_index2 :] if valTerm not in value: value.append(valTerm) begin_index = end_index #print begin_index else: break return value, ntuple def _urlen(self, typename): """ Sets up a cgi request to the wfs for features specified. """ service_url = GDP_URL qs = [] if service_url.find('?') != -1: qs = cgi.parse_qsl(service_url.split('?')[1]) params = [x[0] for x in qs] if 'service' not in params: qs.append(('service', 'WFS')) if 'request' not in params: qs.append(('request', 'DescribeFeatureType')) if 'version' not in params: qs.append(('version', '1.1.0')) if 'typename' not in params: qs.append(('typename', typename)) urlqs = urlencode(tuple(qs)) return service_url.split('?')[0] + '?' + urlqs def _getStringBetween(self, beginterm, line, stopterm): """ Helper function. Gets the string between beginterm and stopterm. Line is the line or superstring to be examined. returns the string inbetween. """ begin_index = line.find(beginterm) end_index = line.find(stopterm, begin_index) return line[begin_index + len(beginterm) : end_index ] def _getLinesContaining(self, linesToParse, term): """ Given a document, goes through the document and for each line with the occurence of the specified term, add that line to a list. Returns the list. """ line_list = [] for line in linesToParse: if term in line: line_list.append(line) linesToParse.close() return line_list def _getFilterID(self,tuples, value): """ Given a the tuples generated by getTuples and a value, will return a list of gmlIDs associated with the value specified. """ value = str(value) filterID = [] for item in tuples: if item[0] == value: filterID.append(item[1]) return filterID def _parseXMLNodesForTagText(self, xml, tag): """ Parses through a XML tree for text associated with specified tag. Returns a list of the text. """ tag_text = [] for node in xml.iter(): if node.tag == tag: tag_text.append(node.text) return tag_text def _generateRequest(self, dataSetURI, algorithm, method, varID=None, verbose=False): """ Takes a dataset uri, algorithm, method, and datatype. This function will generate a simple XML document to make the request specified. (Only works for ListOpendapGrids and GetGridTimeRange). Will return a list containing the info requested for (either data types or time range). """ wps_Service = 'http://cida.usgs.gov/gdp/utility/WebProcessingService' POST = WebProcessingService(wps_Service, verbose=False) xmlGen = gdpXMLGenerator() root = xmlGen.getXMLRequestTree(dataSetURI, algorithm, method, varID, verbose) # change standard output to not display waiting status if not verbose: old_stdout = sys.stdout result = StringIO() sys.stdout = result request = etree.tostring(root) execution = POST.execute(None, [], request=request) if method == 'getDataSetTime': seekterm = 'time' else: seekterm = 'name' if not verbose: sys.stdout = old_stdout return self._parseXMLNodesForTagText(execution.response, seekterm) def _generateFeatureRequest(self, typename, attribute=None): """ This function, given a attribute and a typename or filename will return a list of values associated with the file and the attribute chosen. """ service_url = GDP_URL qs = [] if service_url.find('?') != -1: qs = cgi.parse_qsl(service_url.split('?')[1]) params = [x[0] for x in qs] if 'service' not in params: qs.append(('service', 'WFS')) if 'request' not in params: if attribute is None: qs.append(('request', 'DescribeFeatureType')) else: qs.append(('request', 'GetFeature')) if 'version' not in params: qs.append(('version', '1.1.0')) if 'typename' not in params: qs.append(('typename', typename)) if attribute is not None: if 'propertyname' not in params: qs.append(('propertyname', attribute)) urlqs = urlencode(tuple(qs)) return service_url.split('?')[0] + '?' + urlqs def getAttributes(self, shapefile): """ Given a valid shapefile, this function will create a cgi call returning a list of attributes associated with the shapefile. """ # makes a call to get an xml document containing list of shapefiles urlen = self._generateFeatureRequest(shapefile) linesToParse = urlopen(urlen) # gets back from the linesToParse document, all lines with 2nd arg lines = self._getLinesContaining(linesToParse, 'xsd:element maxOccurs=') attributes = [] # search the line for item in lines: word = self._getStringBetween('name=', item, ' ') # for attributes, will return "attribute", qoutes included, strip qoutes if word[1:len(word) - 1] != "the_geom": attributes.append(word[1: len(word) - 1]) return attributes def getShapefiles(self): """ Returns a list of available files currently on geoserver. """ wfs = WebFeatureService(GDP_URL) shapefiles = wfs.contents.keys() return shapefiles def getValues(self, shapefile, attribute, getTuples='false'): """ Similar to get attributes, given a shapefile and a valid attribute this function will make a call to the Web Feature Services returning a list of values associated with the shapefile and attribute. If getTuples = True, will also return the tuples of [feature:id] along with values [feature] """ urlen = self._generateFeatureRequest(shapefile, attribute) inputObject = urlopen(urlen) shapefileterm = shapefile.split(':') strinx = inputObject.read() lines = strinx.split('\n') # gets the tag/namespace name stringSnippet = self._getStringBetween('<', lines[1], ':'+attribute+'>') stringSnippet = stringSnippet.split('<') shapefileterm[0] = stringSnippet[len(stringSnippet) - 1] # look for this pattern: <term[0]:attribute>SOUGHTWORD</term[0]:attribute> values, tuples = self._getGMLIDString('gml:id="', lines[1], '">', '<'+shapefileterm[0] + ':' + attribute + '>', '</' +shapefileterm[0] +':' + attribute + '>') if getTuples=='true': return sorted(values), sorted(tuples) elif getTuples=='only': return sorted(tuples) else: return sorted(values) def getDataType(self, dataSetURI, verbose=False): """ Set up a get Data type request given a dataSetURI. Returns a list of all available data types. If verbose = True, will print on screen the waiting seq. for response document. """ algorithm = 'gov.usgs.cida.gdp.wps.algorithm.discovery.ListOpendapGrids' return self._generateRequest(dataSetURI, algorithm, method='getDataType', varID=None, verbose=verbose) def getDataSetURI(self): """ This function will not be implemented. This function is only implemented to give a few dataset URIs which may not work with certain datasets and will with others within the bounding box requirements. """ print 'The dataSetURI outputs a select few URIs and may not work with the specific shapefile you are providing.' print 'To ensure compatibility, we recommend selecting a dataSetURI that is specific to the shapefile.' print 'Or you may utilize the web gdp @ http://cida.usgs.gov/gdp/ to get a dataSet matching your specified shapefile.' print dataSetURIs = ['http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/NCEP/merged/monthly/RegCM3_A2_monthly_merged_NCEP.ncml', 'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml', 'http://cida.usgs.gov/qa/thredds/dodsC/prism', 'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml', 'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/alaska_grid.w_meta.ncml', 'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/gmo/GMO_w_meta.ncml'] return dataSetURIs def getGMLIDs(self, shapefile, attribute, value): """ This function returns the gmlID associated with a particular attribute value. """ tuples = self.getTuples(shapefile, attribute) return self._getFilterID(tuples, value) def getTimeRange(self, dataSetURI, varID, verbose=False): """ Set up a get dataset time range request given a datatype and dataset uri. Returns the range of the earliest and latest time. If verbose = True, will print on screen the waiting seq. for response document. """ algorithm = 'gov.usgs.cida.gdp.wps.algorithm.discovery.GetGridTimeRange' return self._generateRequest(dataSetURI, algorithm, method='getDataSetTime', varID=varID, verbose=verbose) def _getFeatureCollectionGeoType(self, geoType, attribute='the_geom', value=None, gmlIDs=None): """ This function returns a featurecollection. It takes a geotype and determines if the geotype is a shapfile or polygon. """ # This is a polygon if isinstance(geoType, list): return GMLMultiPolygonFeatureCollection( [geoType] ) elif isinstance(geoType, str): if value==None and gmlIDs==None: print 'must input a value and attribute for shapefile' return else: tmpID = [] if gmlIDs is None: if type(value) == type(tmpID): gmlIDs = [] for v in value: tuples = self.getTuples(geoType, attribute) tmpID = self._getFilterID(tuples, v) gmlIDs = gmlIDs + tmpID else: tuples = self.getTuples(geoType, attribute) gmlIDs = self._getFilterID(tuples, value) query = WFSQuery(geoType, propertyNames=["the_geom", attribute], filters=gmlIDs) return WFSFeatureCollection(WFS_URL, query) else: print 'Geotype is not a shapefile or a recognizable polygon.' return None def _executeRequest(self, processid, inputs, verbose): """ This function makes a call to the Web Processing Service with the specified user inputs. """ wps = WebProcessingService(WPS_URL) # if verbose=True, then will we will monitor the status of the call. # if verbose=False, then we will return only the file outputpath. if not verbose: # redirects the standard output to avoid printing request status old_stdout = sys.stdout result = StringIO() sys.stdout = result # executes the request execution = wps.execute(processid, inputs, output = "OUTPUT") monitorExecution(execution, download=True) # sets the standard output back to original sys.stdout = old_stdout result_string = result.getvalue() #parses the redirected output to get the filepath of the saved file output = result_string.split('\n') tmp = output[len(output) - 2].split(' ') return tmp[len(tmp)-1] # executes the request execution = wps.execute(processid, inputs, output = "OUTPUT") monitorExecution(execution, download=True) def submitFeatureWeightedGridStatistics(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=None, coverage='true', delim='COMMA', stat='MEAN', grpby='STATISTIC', timeStep='false', summAttr='false'): """ Makes a featureWeightedGridStatistics algorithm call. """ featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs) if featureCollection is None: return processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm' inputs = [("FEATURE_ATTRIBUTE_NAME",attribute), ("DATASET_URI", dataSetURI), ("DATASET_ID", varID), ("TIME_START",startTime), ("TIME_END",endTime), ("REQUIRE_FULL_COVERAGE",coverage), ("DELIMITER",delim), ("STATISTICS",stat), ("GROUP_BY", grpby), ("SUMMARIZE_TIMESTEP", timeStep), ("SUMMARIZE_FEATURE_ATTRIBUTE",summAttr), ("FEATURE_COLLECTION", featureCollection)] return self._executeRequest(processid, inputs, verbose) def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true'): """ Makes a featureCoverageOPenDAP algorithm call. """ featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs) if featureCollection is None: return processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureCoverageOPeNDAPIntersectionAlgorithm' inputs = [ ("DATASET_URI", dataSetURI), ("DATASET_ID", varID), ("TIME_START",startTime), ("TIME_END",endTime), ("REQUIRE_FULL_COVERAGE",coverage), ("FEATURE_COLLECTION", featureCollection)] return self._executeRequest(processid, inputs, verbose) def submitFeatureCoverageWCSIntersection(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true'): """ Makes a featureCoverageWCSIntersection algorithm call. """ featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs) if featureCollection is None: return processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureCoverageIntersectionAlgorithm' inputs = [("DATASET_URI", dataSetURI), ("DATASET_ID", varID), ("REQUIRE_FULL_COVERAGE",coverage), ("FEATURE_COLLECTION", featureCollection)] return self._executeRequest(processid, inputs, verbose) def submitFeatureCategoricalGridCoverage(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', delim='COMMA'): """ Makes a featureCategoricalGridCoverage algorithm call. """ featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs) if featureCollection is None: return processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureCategoricalGridCoverageAlgorithm' inputs = [ ("FEATURE_ATTRIBUTE_NAME",attribute), ("DATASET_URI", dataSetURI), ("DATASET_ID", varID), ("DELIMITER", delim), ("REQUIRE_FULL_COVERAGE",coverage), ("FEATURE_COLLECTION", featureCollection)] return self._executeRequest(processid, inputs, verbose)
class GenericWPS(ProgressMonitorPE): """ Wrap the execution of a WPS process into a dispel4py PE """ STATUS_NAME = 'status' STATUS_LOCATION_NAME = 'status_location' OUTPUT_NAME = 'outputs' DUMMY_INPUT_NAME = 'None' def __init__(self, name, url, identifier, inputs=None, linked_inputs=None, monitor=None, progress_range=None, headers=None, **_): """ :param name: task name :param url: url of the WPS provider :param identifier: identifier of the WPS task :param inputs: dict of static inputs for the wps (multiple values can be passed for each key by using an array) :param linked_inputs: dict of dynamic inputs of the wps obtained from upstream tasks (multiple values can be passed for each key by using an array) :param monitor: status and progress monitor :param progress_provider: :param progress_range: progress range of this task in the overall workflow :param headers: headers to be included in the wps call :param _: Accept any others parameters coming from the workflow description """ ProgressMonitorPE.__init__(self, name, monitor) # Convert dict of values or array of values to a flat list of key, value tuple inputs = list(self.iter_inputs(inputs)) linked_inputs = list(self.iter_inputs(linked_inputs)) # Defaults argument (with mutable type) if not progress_range: progress_range = [0, 100] self.set_progress_provider( RangeProgress(progress_range[0], progress_range[1])) self.wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers) self.identifier = identifier self.proc_desc = self.wps.describeprocess(identifier) self._validate_inputs(inputs, linked_inputs) # These are the static inputs # (linked inputs will be appended to inputs just before execution by the _set_inputs function) self.static_inputs = inputs self.dynamic_inputs = [] # Will be filled as PE are connected to us (by the get_output function) self.outputs = [] for _input in linked_inputs: # Here we add PE input that will need to be connected if _input[0] not in self.inputconnections: self._add_input(_input[0]) self._add_linked_input(_input[0], _input[1]) def try_connect(self, graph, linked_input, downstream_task, downstream_task_input): """ Override TaskPE fct. See TaskPE.try_connect for details. Add the WPS outputs upon connection """ if ProgressMonitorPE.try_connect(self, graph, linked_input, downstream_task, downstream_task_input): # Here we added the WPS output name and its as_reference request self.outputs.append((linked_input.get('output', self._get_default_output()), linked_input.get('as_reference', False))) return True return False def _process(self, inputs): """ Implement the GenericPE _process function. This function is called multiple time if more than one input must be set :param inputs: What has been outputted by the previous task :return: None because the WPS execute function will only be called in _postprocess fct """ # Assign the input internally and wait for all inputs before launching the wps execution for key, value in self._read_inputs(inputs): self.dynamic_inputs.append((key, value)) def _postprocess(self): """ Implement the GenericPE _postprocess function. Call when this PE has received all its inputs and thus are ready to execute the wps """ try: self._check_inputs() self._send_outputs(self._execute()) except Exception: logger.exception("process failed!") raise def _connect(self, from_connection, to_node, to_connection, graph): """ Override TaskPE fct. See TaskPE._connect for details. We add the PE output just before completing the connection in the graph """ # Here we add a PE output that is required before completing the actual connection self._add_output(from_connection) ProgressMonitorPE._connect(self, from_connection, to_node, to_connection, graph) def _get_default_output(self): """ Implement TaskPE fct. If the WPS has only one output it can be set as the default one. Else it has to be set. """ if len(self.proc_desc.processOutputs) == 1: return self.proc_desc.processOutputs[0].identifier return None def get_input_desc(self, input_name): """ Implement TaskPE fct. See TaskPE.get_input_desc for details. """ for wps_input in self.proc_desc.dataInputs: if wps_input.identifier == input_name: return wps_input return None def get_output_desc(self, output_name): """ Implement TaskPE fct. See TaskPE.get_output_desc for details. """ for wps_output in self.proc_desc.processOutputs: if wps_output.identifier == output_name: return wps_output return None def _validate_inputs(self, inputs, linked_inputs): """ Check that given inputs and linked inputs are valid :param inputs: static input (value is available now) :param linked_inputs: dynamic inputs (value will be obtained from upstream task) """ # Validate that given inputs exist in the wps valid_inputs = [ wps_input.identifier for wps_input in self.proc_desc.dataInputs ] # Allow a process not requiring any input to be linked to a previous one by using a dummy ("None") input name valid_inputs.append(self.DUMMY_INPUT_NAME) for submitted_input in inputs + linked_inputs: if submitted_input[0] not in valid_inputs: raise WorkflowException( "Invalid workflow : Input '{input}' of process '{proc}' is unknown." .format(input=submitted_input[0], proc=self.identifier)) def _check_inputs(self): """ Check if all the required inputs have been set (from workflow static inputs or from upstream tasks) """ ready_inputs = [ _input[0] for _input in self.static_inputs + self.dynamic_inputs ] # Consider the dummy input to be always ready! ready_inputs.append(self.DUMMY_INPUT_NAME) for linked_input in self.linked_inputs: if linked_input[0] not in ready_inputs: msg = "Workflow cannot complete because of a missing input '{input}' of task {task}".format( input=linked_input[0], task=self.name) raise WorkflowException(msg) @staticmethod def _check_status(execution): """ Try to read the xml status of the underlying WPS process, raise Exception if the url cannot be read properly """ reader = WPSExecuteReader(verbose=execution.verbose) # override status location logger.info('Checking execution status : %s' % execution.statusLocation) try: response = reader.readFromUrl(execution.statusLocation, username=execution.username, password=execution.password, verify=execution.verify, headers=execution.headers) response = etree.tostring(response) except Exception: raise else: execution.checkStatus(response=response, sleepSecs=3) def _monitor_execution(self, execution): """ Monitor the execution of the underlying WPS and return only when the process end (successfully or not) """ progress = self.progress(execution) self.monitor("status_location={0.statusLocation}".format(execution), progress) xml_doc_read_failure = 0 last_status_message = None last_progress = None while execution.isNotComplete(): try: # Check the status of the wps execution self._check_status(execution) except Exception as e: # Try XML_DOC_READING_MAX_ATTEMPT time before raising an exception xml_doc_read_failure += 1 if xml_doc_read_failure > XML_DOC_READING_MAX_ATTEMPT: logger.error( "Failed to read status xml document after %s attempts : %s", XML_DOC_READING_MAX_ATTEMPT, str(e)) raise else: # Sleep 5 seconds to give a chance sleep(5) else: progress = self.progress(execution) if execution.statusMessage != last_status_message or progress != last_progress: last_status_message = execution.statusMessage last_progress = progress self.monitor(execution.statusMessage, progress) self.monitor(execution.statusMessage, progress) # In case of success log all output value if execution.isSucceded(): for output in execution.processOutputs: if output.reference is not None: self.monitor( '{0.identifier}={0.reference} ({0.mimeType})'.format( output), progress) else: self.monitor( '{0}={1}'.format(output.identifier, ", ".join(output.data)), progress) # Or log the errors else: self.monitor( '\n'.join([ 'ERROR: {0.text} code={0.code} locator={0.locator})'. format(ex) for ex in execution.errors ]), progress) def get_output_datatype(self, output): """ Extract the correct datatype from the datatype structure :param output: output as parsed from the wps xml status (ows.wps.Output) :return: The datatype of the output as a string (ComplexData, string, etc.) """ # outputs from execution structure do not always carry the dataType # so find it from the process description if not output.dataType: for desc_wps_output in self.proc_desc.processOutputs: if desc_wps_output.identifier == output.identifier: return desc_wps_output.dataType # Also the datatype is not always stripped correctly so do the job here return output.dataType.split(':')[-1] def _jsonify_output(self, output): """ Utility method to jsonify an output element. """ json_output = dict(identifier=output.identifier, title=output.title, dataType=self.get_output_datatype(output)) # WPS standard v1.0.0 specify that either a reference or a data field has to be provided if output.reference: json_output['reference'] = output.reference else: # WPS standard v1.0.0 specify that Output data field has Zero or one value json_output['data'] = output.data[0] if output.data else None if json_output['dataType'] == 'ComplexData': json_output['mimeType'] = output.mimeType return json_output def _execute(self): """ This is the function doing the actual WPS process call, monitoring its execution and parsing the output. :return: Return the data that is send to the downstream PE """ logger.debug( "execute with static inputs=%s and dynamic inputs=%s to get outputs=%s", self.static_inputs, self.dynamic_inputs, self.outputs) self.wps.headers['machineid'] = ''.join( random.choice(string.ascii_lowercase + string.digits) for _ in range(16)) execution = self.wps.execute(identifier=self.identifier, inputs=self.static_inputs + self.dynamic_inputs, output=self.outputs, lineage=True) self.set_headers(self.data_headers) self._monitor_execution(execution) # Reset dynamic inputs for the next iteration self.dynamic_inputs = [] execution_result = { self.STATUS_NAME: execution.status, self.STATUS_LOCATION_NAME: execution.statusLocation } if execution.isSucceded(): execution_result[self.OUTPUT_NAME] = \ [self._jsonify_output(output) for output in execution.processOutputs] self.save_result(execution_result) result = {} # NOTE: only set workflow output if specific output was needed for wps_output in self.outputs: for output in execution.processOutputs: if wps_output[0] == output.identifier: # outputs from execution structure do not always carry the dataType # so find it from the process description because the _adapt fct needs it if not output.dataType: for desc_wps_output in self.proc_desc.processOutputs: if desc_wps_output.identifier == output.identifier: output.dataType = desc_wps_output.dataType # Also the datatype is not always stripped correctly so do the job here else: output.dataType = output.dataType.split(':')[-1] # Send directly the wps output object to the downstream PE # Note: output.data is always an array since a wps output is appended to processOutputs[x].data result[wps_output[0]] = output break return result else: failure_msg = '\n'.join( ['{0.text}'.format(ex) for ex in execution.errors]) raise WorkflowException(failure_msg) @staticmethod def iter_inputs(inputs): """ Convert dictionary of inputs where the value is either an array or a value to a flat array of tuple (key, value) :param inputs: Dictionnary of inputs looking like this {key1: val1, key2: val2, key3: [val3.1, val3.2, val3.3]} :return: yield a tuple for each key/value pair looking like this [(key1, val1), (key2, val2), (key3, val3.1), (key3, val3.2), (key3, val3.3)] """ if not inputs: return for key in inputs: if isinstance(inputs[key], list): for value in inputs[key]: yield (key, value) else: yield (key, inputs[key])
# 1) GetCapabilities # GET request: http://ceda-wps2.badc.rl.ac.uk/wps?Service=WPS&Request=GetCapabilities&Format=text/xml wps.getcapabilities() print('WPS Identification type: %s' % wps.identification.type) print('WPS Identification title: %s' % wps.identification.title) print('WPS Identification abstract: %s' % wps.identification.abstract) for operation in wps.operations: print('WPS Operation: %s' % operation.name) for process in wps.processes: print('WPS Process: identifier=%s title=%s' % (process.identifier, process.title)) # 2) DescribeProcess # GET request: http://ceda-wps2.badc.rl.ac.uk/wps?identifier=DoubleIt&version=1.0.0&request=DescribeProcess&service=WPS process = wps.describeprocess('DoubleIt') print('WPS Process: identifier=%s' % process.identifier) print('WPS Process: title=%s' % process.title) print('WPS Process: abstract=%s' % process.abstract) for input in process.dataInputs: print('Process input:') printInputOutput(input, indent='\t') for output in process.processOutputs: print('Process output:') printInputOutput(output, indent='\t') # 3) Execute # POST request: # Note: not working, requires openid login ? #processid = "DoubleIt" #inputs = [ ("NumberToDouble","1") ]
verbose=verbose, skip_caps=True) # 1) GetCapabilities wps.getcapabilities() print('WPS Identification type: %s' % wps.identification.type) print('WPS Identification title: %s' % wps.identification.title) print('WPS Identification abstract: %s' % wps.identification.abstract) for operation in wps.operations: print('WPS Operation: %s' % operation.name) for process in wps.processes: print('WPS Process: identifier=%s title=%s' % (process.identifier, process.title)) # 2) DescribeProcess process = wps.describeprocess('v.net.path') # alternatively, read process description from XML file (no live request to WPS server) #xml = open('../tests/USGSDescribeProcess.xml', 'rb').read() #process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml) print('WPS Process: identifier=%s' % process.identifier) print('WPS Process: title=%s' % process.title) print('WPS Process: abstract=%s' % process.abstract) for input in process.dataInputs: print( 'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d' % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs)) for output in process.processOutputs: print('Process output: identifier=%s, data type=%s' % (output.identifier, output.dataType)) # 3) Execute
class pyGDPwebProcessing(): """ This class allows interactive calls to be made into the GDP. """ def __init__(self, WFS_URL=None): if WFS_URL==None: from pygdp.namespaces import WFS_URL wfsUrl=WFS_URL self.wfsUrl = wfsUrl self.wpsUrl = WPS_URL self.version = '1.1.0' self.wps = WebProcessingService(WPS_URL) def WPSgetCapbilities(self, xml=None): """ Returns a list of capabilities. """ self.wps.getcapabilities(xml) def WPSdescribeprocess(self, identifier, xml=None): """ Returns a list describing a specific identifier/process. """ self.wps.describeprocess(identifier, xml) #pyGDP Submit Feature def dodsReplace(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return _execute_request.dodsReplace(dataSetURI, verbose) def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCoverageOPenDAP(geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs) def submitFeatureCoverageWCSIntersection(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCoverageWCSIntersection(geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs) def submitFeatureCategoricalGridCoverage(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', delim='COMMA', outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCategoricalGridCoverage(geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose, coverage, delim, self.wfsUrl, outputfname, sleepSecs) def submitFeatureWeightedGridStatistics(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=None, coverage=True, delim='COMMA', stat='MEAN', grpby='STATISTIC', timeStep=False, summAttr=False, weighted=True, outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return fwgs.submitFeatureWeightedGridStatistics(geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs, verbose, coverage, delim, stat, grpby, timeStep, summAttr, weighted, self.wfsUrl, outputfname, sleepSecs) #pyGDP File Utilities def shapeToZip(self, inShape, outZip=None, allFiles=True): return shape_to_zip.shapeToZip(inShape, outZip=None, allFiles=True) def uploadShapeFile(self, filePath): value, ntuple = upload_shapefile.uploadShapefile(filePath) return value, ntuple #pyGDP WFS Utilities def getTuples(self, shapefile, attribute): return shapefile_id_handle.getTuples(shapefile, attribute) def getShapefiles(self): return shapefile_value_handle.getShapefiles(self.wfsUrl) def getAttributes(self, shapefile): return shapefile_value_handle.getAttributes(shapefile, self.wfsUrl) def getValues(self, shapefile, attribute, getTuples='false', limitFeatures=None): return shapefile_value_handle.getValues(shapefile, attribute, getTuples, limitFeatures, self.wfsUrl) def getGMLIDs(self, shapefile, attribute, value): return shapefile_id_handle.getGMLIDs(shapefile, attribute, value, WFS_URL=self.wfsUrl) def _getFilterID(self, tuples, value): return shapefile_id_handle._getFilterID(tuples, value) def _getFeatureCollectionGeoType(self, geoType, attribute='the_geom', value=None, gmlIDs=None): return _get_geotype._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs, self.wfsUrl) def _generateRequest(self, dataSetURI, algorithm, method, varID=None, verbose=False): return _webdata_xml_generate._generateRequest(dataSetURI, algorithm, method, varID, verbose) #pyGDP WebData Utilities def getDataLongName(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getDataLongName(dataSetURI, verbose) def getDataType(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getDataType(dataSetURI, verbose) def getDataUnits(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getDataUnits(dataSetURI, verbose) def getDataSetURI(self, anyText='',CSWURL=CSWURL,BBox=None): return webdata_handle.getDataSetURI(anyText, CSWURL, BBox) def getTimeRange(self, dataSetURI, varID, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getTimeRange(dataSetURI, varID, verbose) #Pull up docstrings. #dodsReplace.__doc__ = _execute_request.dodsReplace.__doc__ getAttributes.__doc__ = shapefile_value_handle.getAttributes.__doc__ getDataLongName.__doc__ = webdata_handle.getDataLongName.__doc__ getDataSetURI.__doc__ = webdata_handle.getDataSetURI.__doc__ getDataType.__doc__ = webdata_handle.getDataType.__doc__ getDataUnits.__doc__ = webdata_handle.getDataUnits.__doc__ getGMLIDs.__doc__ = shapefile_id_handle.getGMLIDs.__doc__ getShapefiles.__doc__ = shapefile_value_handle.getShapefiles.__doc__ getTimeRange.__doc__ = webdata_handle.getTimeRange.__doc__ getTuples.__doc__ = shapefile_id_handle.getTuples.__doc__ getValues.__doc__ = shapefile_value_handle.getValues.__doc__ shapeToZip.__doc__ = shape_to_zip.shapeToZip.__doc__ submitFeatureCategoricalGridCoverage.__doc__ = feature_coverage.submitFeatureCategoricalGridCoverage.__doc__ submitFeatureCoverageOPenDAP.__doc__ = feature_coverage.submitFeatureCoverageOPenDAP.__doc__ submitFeatureCoverageWCSIntersection.__doc__ = feature_coverage.submitFeatureCoverageWCSIntersection.__doc__ submitFeatureWeightedGridStatistics.__doc__ = fwgs.submitFeatureWeightedGridStatistics.__doc__ uploadShapeFile.__doc__ = upload_shapefile.uploadShapeFile.__doc__
def check_wps(serverCursor): ''' Check all WPS services, and update the database accordingly ''' # Get the server list, but ignore servers marked as deleted serverCursor.execute("SELECT DISTINCT url FROM " + tables["wpsServers"]) upsert_list = [] sqlList = [] # Mark all our records as dead; we'll mark them as alive as we process them. Note that the database won't # actually be udpated until we commit all our transactions at the end, so we'll never see this value # for a server/process/input that is in fact alive. sqlList.append("UPDATE " + tables["wpsServers"] + " SET alive = false") sqlList.append("UPDATE " + tables["processes"] + " SET alive = false") # We'll delete the parameter list completely; There is no benefit of keeping older, but now disused module # parameters around... it just confuses things. # If the parameters reappaear in the process in the future, they will be added back, and any values # associated with them will be retained. sqlList.append("DELETE FROM " + tables["processParams"]) for row in serverCursor: server_url = row[0] # Run a GetCapabilities query on the WPS server -- could fail if URL is bogus try: wps = WebProcessingService(server_url, version = wpsVersion) except: print "Could not load WPS data from url " + server_url # If URL is bogus, will raise a URLError... but whatever... no errors are reoverable at this point continue # Update the server title and abstract sqlList.append(prepare_update_wps_server_info(server_url, wps)) # Iterate over the processes available on this server for proc in wps.processes: abstract = get_process_abstract(proc) sqlList.append(prepare_update_wps_process(server_url, proc.identifier, proc.title, abstract)) # Need to do this here so that the SELECT below will find a record if the upsert inserts... a little messy run_queries(db_conn, upsert_list, sqlList) upsert_list = [] sqlList = [] # Get all processes associated with server that match the specified identifier -- could be more than one update_cursor.execute(prepare_select_processes(server_url, proc.identifier)) for procrow in update_cursor: procId = procrow[0] try: procDescr = wps.describeprocess(proc.identifier) # Call to OWSLib except: print "Could not describe process ", proc.identifier, " on server ", server_url for input in procDescr.dataInputs: upsert_list.append((tables["processParams"], "wps_process_id", procId, input.identifier)) sqlList.append(prepare_update_wps_param(procId, input, True)) for output in procDescr.processOutputs: upsert_list.append((tables["processParams"], "wps_process_id", procId, output.identifier)) sqlList.append(prepare_update_wps_param(procId, output, False)) # Run and commit WPS transactions run_queries(db_conn, upsert_list, sqlList) upsert_list = []
def test_wps_describeprocess_usgs(): # Initialize WPS client wps = WebProcessingService('http://cida.usgs.gov/gdp/process/WebProcessingService', skip_caps=True) # Execute fake invocation of DescribeProcess operation by parsing cached response from xml = open(resource_file('wps_USGSDescribeProcess.xml'), 'rb').read() process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml) # Check process description assert process.identifier == 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm' assert process.title == 'Feature Weighted Grid Statistics' assert process.abstract == 'This algorithm generates area weighted statistics of a gridded dataset for a set of vector polygon features. Using the bounding-box that encloses the feature data and the time range, if provided, a subset of the gridded dataset is requested from the remote gridded data server. Polygon representations are generated for cells in the retrieved grid. The polygon grid-cell representations are then projected to the feature data coordinate reference system. The grid-cells are used to calculate per grid-cell feature coverage fractions. Area-weighted statistics are then calculated for each feature using the grid values and fractions as weights. If the gridded dataset has a time range the last step is repeated for each time step within the time range or all time steps if a time range was not supplied.' # NOQA # Check process inputs # Expected Input: # Process input: # identifier=FEATURE_COLLECTION, title=Feature Collection, abstract=A feature collection encoded as a WFS request or one of the supported GML profiles., data type=ComplexData # NOQA # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.0.0/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.1.1/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.1.2/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.1.2.1/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.0.0/base/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.0.1/base/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.1.0/base/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.1.1/base/feature.xsd # Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.2.1/base/feature.xsd # Default Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.0.0/feature.xsd # minOccurs=1, maxOccurs=1 # Process input: # identifier=DATASET_URI, title=Dataset URI, abstract=The base data web service URI for the dataset of interest., data type=anyURI # NOQA # Any value allowed # Default Value: None # minOccurs=1, maxOccurs=1 # Process input: # identifier=DATASET_ID, title=Dataset Identifier, abstract=The unique identifier for the data type or variable of interest., data type=string # NOQA # Any value allowed # Default Value: None # minOccurs=1, maxOccurs=2147483647 # Process input: # identifier=REQUIRE_FULL_COVERAGE, title=Require Full Coverage, abstract=If turned on, the service will require that the dataset of interest fully cover the polygon analysis zone data., data type=boolean # NOQA # Any value allowed # Default Value: True # minOccurs=1, maxOccurs=1 # Process input: # identifier=TIME_START, title=Time Start, abstract=The date to begin analysis., data type=dateTime # Any value allowed # Default Value: None # minOccurs=0, maxOccurs=1 # Process input: # identifier=TIME_END, title=Time End, abstract=The date to end analysis., data type=dateTime # Any value allowed # Default Value: None # minOccurs=0, maxOccurs=1 # Process input: # identifier=FEATURE_ATTRIBUTE_NAME, title=Feature Attribute Name, abstract=The attribute that will be used to label column headers in processing output., data type=string # NOQA # Any value allowed # Default Value: None # minOccurs=1, maxOccurs=1 # Process input: # identifier=DELIMITER, title=Delimiter, abstract=The delimiter that will be used to separate columns in the processing output., data type=string # NOQA # Allowed Value: COMMA # Allowed Value: TAB # Allowed Value: SPACE # Default Value: COMMA # minOccurs=1, maxOccurs=1 # Process input: # identifier=STATISTICS, title=Statistics, abstract=Statistics that will be returned for each feature in the processing output., data type=string # NOQA # Allowed Value: MEAN # Allowed Value: MINIMUM # Allowed Value: MAXIMUM # Allowed Value: VARIANCE # Allowed Value: STD_DEV # Allowed Value: WEIGHT_SUM # Allowed Value: COUNT # Default Value: None # minOccurs=1, maxOccurs=7 # Process input: # identifier=GROUP_BY, title=Group By, abstract=If multiple features and statistics are selected, this will change whether the processing output columns are sorted according to statistics or feature attributes., data type=string # NOQA # Allowed Value: STATISTIC # Allowed Value: FEATURE_ATTRIBUTE # Default Value: None # minOccurs=1, maxOccurs=1 # Process input: # identifier=SUMMARIZE_TIMESTEP, title=Summarize Timestep, abstract=If selected, processing output will include columns with summarized statistics for all feature attribute values for each timestep, data type=boolean # NOQA # Any value allowed # Default Value: True # minOccurs=0, maxOccurs=1 # Process input: # identifier=SUMMARIZE_FEATURE_ATTRIBUTE, title=Summarize Feature Attribute, abstract=If selected, processing output will include a final row of statistics summarizing all timesteps for each feature attribute value, data type=boolean # NOQA # Any value allowed # Default Value: True # minOccurs=0, maxOccurs=1 assert len(process.dataInputs) == 12 input = process.dataInputs[0] assert input.identifier == 'FEATURE_COLLECTION' assert input.dataType == 'ComplexData' # Expected Output: # identifier=OUTPUT, title=Output File, abstract=A delimited text file containing requested process output., data type=ComplexData # NOQA # Supported Value: mimeType=text/csv, encoding=UTF-8, schema=None # Default Value: mimeType=text/csv, encoding=UTF-8, schema=None # reference=None, mimeType=None # Check process outputs assert len(process.processOutputs) == 1 output = process.processOutputs[0] assert output.identifier == 'OUTPUT' assert output.dataType == 'ComplexData'
def register_wps_processes_from_config(wps_processes_file_path, container): # type: (Optional[FileSystemPathType], AnySettingsContainer) -> None """ Loads a `wps_processes.yml` file and registers `WPS-1` providers processes to the current `Weaver` instance as equivalent `WPS-2` processes. References listed under ``processes`` are registered. When the reference is a service (provider), registration of each WPS process is done individually for each of the specified providers with ID ``[service]_[process]`` per listed process by ``GetCapabilities``. .. versionadded:: 1.14.0 When references are specified using ``providers`` section instead of ``processes``, the registration only saves the remote WPS provider endpoint to dynamically populate WPS processes on demand. .. seealso:: - `weaver.wps_processes.yml.example` for additional file format details """ if wps_processes_file_path is None: warnings.warn("No file specified for WPS-1 providers registration.", RuntimeWarning) wps_processes_file_path = get_weaver_config_file( "", WEAVER_DEFAULT_WPS_PROCESSES_CONFIG) elif wps_processes_file_path == "": warnings.warn( "Configuration file for WPS-1 providers registration explicitly defined as empty in settings. " "Not loading anything.", RuntimeWarning) return # reprocess the path in case it is relative to default config directory wps_processes_file_path = get_weaver_config_file( wps_processes_file_path, WEAVER_DEFAULT_WPS_PROCESSES_CONFIG, generate_default_from_example=False) if wps_processes_file_path == "": warnings.warn("No file specified for WPS-1 providers registration.", RuntimeWarning) return LOGGER.info("Using WPS-1 provider processes file: [%s]", wps_processes_file_path) try: with open(wps_processes_file_path, "r") as f: processes_config = yaml.safe_load(f) processes = processes_config.get("processes") or [] providers = processes_config.get("providers") or [] if not processes and not providers: LOGGER.warning("Nothing to process from file: [%s]", wps_processes_file_path) return db = get_db(container) process_store = db.get_store(StoreProcesses) service_store = db.get_store(StoreServices) # either 'service' references to register every underlying 'process' individually # or explicit 'process' references to register by themselves for cfg_service in processes: svc_name, svc_url, svc_proc, svc_vis = parse_wps_process_config( cfg_service) # fetch data LOGGER.info("Fetching WPS-1: [%s]", svc_url) wps = WebProcessingService(url=svc_url) if LooseVersion(wps.version) >= LooseVersion("2.0"): LOGGER.warning("Invalid WPS-1 provider, version was [%s]", wps.version) continue wps_processes = [wps.describeprocess(p) for p in svc_proc] or wps.processes for wps_process in wps_processes: proc_id = "{}_{}".format(svc_name, get_sane_name(wps_process.identifier)) try: process_store.fetch_by_id(proc_id) except ProcessNotFound: pass else: LOGGER.warning( "Process already registered: [%s]. Skipping...", proc_id) continue proc_url = "{}?service=WPS&request=DescribeProcess&identifier={}&version={}" \ .format(svc_url, wps_process.identifier, wps.version) svc_vis = VISIBILITY_PUBLIC if svc_vis else VISIBILITY_PRIVATE payload = { "processDescription": { "process": { "id": proc_id, "visibility": svc_vis } }, "executionUnit": [{ "href": proc_url }], "deploymentProfileName": "http://www.opengis.net/profiles/eoc/wpsApplication", } try: resp = deploy_process_from_payload(payload, container) if resp.status_code == HTTPOk.code: LOGGER.info("Process registered: [%s]", proc_id) else: raise RuntimeError( "Process registration failed: [{}]".format( proc_id)) except Exception as ex: LOGGER.exception( "Exception during process registration: [%r]. Skipping...", ex) continue # direct WPS providers to register for cfg_service in providers: svc_name, svc_url, _, svc_vis = parse_wps_process_config( cfg_service) LOGGER.info("Register WPS-1 provider: [%s]", svc_url) WebProcessingService( url=svc_url) # only attempt fetch to validate it exists try: service_store.fetch_by_name(svc_name) except ServiceNotFound: pass else: LOGGER.warning( "Provider already registered: [%s]. Skipping...", svc_name) continue try: service_store.save_service( Service(name=svc_name, url=svc_url, public=svc_vis)) except Exception as ex: LOGGER.exception( "Exception during provider registration: [%r]. Skipping...", ex) continue LOGGER.info("Finished processing configuration file [%s].", wps_processes_file_path) except Exception as exc: msg = "Invalid WPS-1 providers configuration file [{!r}].".format(exc) LOGGER.exception(msg) raise RuntimeError(msg)
class WPSClient(object): """Returns a class where every public method is a WPS process available at the given url. Example: >>> emu = WPSClient(url='<server url>') >>> emu.hello('stranger') 'Hello stranger' """ def __init__( self, url, processes=None, converters=None, username=None, password=None, headers=None, verify=True, cert=None, verbose=False, progress=False, version=WPS_DEFAULT_VERSION, ): """ Args: url (str): Link to WPS provider. config (Config): an instance processes: Specify a subset of processes to bind. Defaults to all processes. converters (dict): Correspondence of {mimetype: class} to convert this mimetype to a python object. username (str): passed to :class:`owslib.wps.WebProcessingService` password (str): passed to :class:`owslib.wps.WebProcessingService` headers (str): passed to :class:`owslib.wps.WebProcessingService` verify (bool): passed to :class:`owslib.wps.WebProcessingService` cert (str): passed to :class:`owslib.wps.WebProcessingService` verbose (str): passed to :class:`owslib.wps.WebProcessingService` progress (bool): If True, enable interactive user mode. version (str): WPS version to use. """ self._converters = converters self._interactive = progress self._mode = ASYNC if progress else SYNC self._notebook = notebook.is_notebook() self._inputs = {} self._outputs = {} if not verify: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) self._wps = WebProcessingService( url, version=version, username=username, password=password, verbose=verbose, headers=headers, verify=verify, cert=cert, skip_caps=True, ) try: self._wps.getcapabilities() except ServiceException as e: if "AccessForbidden" in str(e): raise UnauthorizedException( "You are not authorized to do a request of type: GetCapabilities" ) raise self._processes = self._get_process_description(processes) # Build the methods for pid in self._processes: setattr(self, sanitize(pid), types.MethodType(self._method_factory(pid), self)) self.logger = logging.getLogger('WPSClient') if progress: self._setup_logging() self.__doc__ = utils.build_wps_client_doc(self._wps, self._processes) def _get_process_description(self, processes): """Return the description for each process. Sends the server a `describeProcess` request for each process. Parameters ---------- processes: str, list, None A process name, a list of process names or None (for all processes). Returns ------- OrderedDict A dictionary keyed by the process identifier of process descriptions. """ all_wps_processes = [p.identifier for p in self._wps.processes] if processes is None: if owslib.__version__ > '0.17.0': # Get the description for all processes in one request. ps = self._wps.describeprocess('all') return OrderedDict((p.identifier, p) for p in ps) else: processes = all_wps_processes # Check for invalid process names, i.e. not matching the getCapabilities response. process_names, missing = utils.filter_case_insensitive( processes, all_wps_processes) if missing: message = "These process names were not found on the WPS server: {}" raise ValueError(message.format(", ".join(missing))) # Get the description for each process. ps = [self._wps.describeprocess(pid) for pid in process_names] return OrderedDict((p.identifier, p) for p in ps) def _setup_logging(self): self.logger.setLevel(logging.INFO) import sys fh = logging.StreamHandler(sys.stdout) fh.setFormatter(logging.Formatter('%(asctime)s: %(message)s')) self.logger.addHandler(fh) def _method_factory(self, pid): """Create a custom function signature with docstring, instantiate it and pass it to a wrapper which will actually call the process. Parameters ---------- pid: str Identifier of the WPS process. Returns ------- func A Python function calling the remote process, complete with docstring and signature. """ process = self._processes[pid] input_defaults = OrderedDict() for inpt in process.dataInputs: iid = sanitize(inpt.identifier) default = getattr(inpt, "defaultValue", None) if inpt.dataType != 'ComplexData' else None input_defaults[iid] = utils.from_owslib(default, inpt.dataType) body = dedent(""" inputs = locals() inputs.pop('self') return self._execute('{pid}', **inputs) """).format(pid=pid) func_builder = FunctionBuilder( name=sanitize(pid), doc=utils.build_process_doc(process), args=["self"] + list(input_defaults), defaults=tuple(input_defaults.values()), body=body, filename=__file__, module=self.__module__, ) self._inputs[pid] = {} if hasattr(process, "dataInputs"): self._inputs[pid] = OrderedDict( (i.identifier, i) for i in process.dataInputs ) self._outputs[pid] = {} if hasattr(process, "processOutputs"): self._outputs[pid] = OrderedDict( (o.identifier, o) for o in process.processOutputs ) func = func_builder.get_func() return func def _execute(self, pid, **kwargs): """Execute the process.""" wps_inputs = [] for name, input_param in self._inputs[pid].items(): value = kwargs.get(sanitize(name)) if value is not None: if isinstance(input_param.defaultValue, ComplexData): encoding = input_param.defaultValue.encoding mimetype = input_param.defaultValue.mimeType if isinstance(value, ComplexData): inp = value else: if utils.is_embedded_in_request(self._wps.url, value): # If encoding is None, this will return the actual encoding used (utf-8 or base64). value, encoding = embed(value, mimetype, encoding=encoding) else: value = fix_url(value) inp = utils.to_owslib(value, data_type=input_param.dataType, encoding=encoding, mimetype=mimetype) else: inp = utils.to_owslib(value, data_type=input_param.dataType) wps_inputs.append((name, inp)) wps_outputs = [ (o.identifier, "ComplexData" in o.dataType) for o in self._outputs[pid].values() ] mode = self._mode if self._processes[pid].storeSupported else SYNC try: wps_response = self._wps.execute( pid, inputs=wps_inputs, output=wps_outputs, mode=mode ) if self._interactive and self._processes[pid].statusSupported: if self._notebook: notebook.monitor(wps_response, sleep=.2) else: self._console_monitor(wps_response) except ServiceException as e: if "AccessForbidden" in str(e): raise UnauthorizedException( "You are not authorized to do a request of type: Execute" ) raise # Add the convenience methods of WPSResult to the WPSExecution class. This adds a `get` method. utils.extend_instance(wps_response, WPSResult) wps_response.attach(wps_outputs=self._outputs[pid], converters=self._converters) return wps_response def _console_monitor(self, execution, sleep=3): """Monitor the execution of a process. Parameters ---------- execution : WPSExecution instance The execute response to monitor. sleep: float Number of seconds to wait before each status check. """ import signal # Intercept CTRL-C def sigint_handler(signum, frame): self.cancel() signal.signal(signal.SIGINT, sigint_handler) while not execution.isComplete(): execution.checkStatus(sleepSecs=sleep) self.logger.info("{} [{}/100] - {} ".format( execution.process.identifier, execution.percentCompleted, execution.statusMessage[:50],)) if execution.isSucceded(): self.logger.info("{} done.".format(execution.process.identifier)) else: self.logger.info("{} failed.".format(execution.process.identifier))
verbose=verbose, skip_caps=True) # 1) GetCapabilities wps.getcapabilities() print('WPS Identification type: %s' % wps.identification.type) print('WPS Identification title: %s' % wps.identification.title) print('WPS Identification abstract: %s' % wps.identification.abstract) for operation in wps.operations: print('WPS Operation: %s' % operation.name) for process in wps.processes: print('WPS Process: identifier=%s title=%s' % (process.identifier, process.title)) # 2) DescribeProcess process = wps.describeprocess('reprojectImage') print('WPS Process: identifier=%s' % process.identifier) print('WPS Process: title=%s' % process.title) print('WPS Process: abstract=%s' % process.abstract) for input in process.dataInputs: print( 'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d' % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs)) for output in process.processOutputs: print('Process output: identifier=%s, data type=%s' % (output.identifier, output.dataType)) # 3a) Execute # GET request: http://rsg.pml.ac.uk/wps/generic.cgi?request=Execute&service=wps&version=1.0.0&identifier=reprojectImage&datainputs=[inputImage=http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img;outputSRS=EPSG:4326]&responsedocument=outputImage=@asreference=true processid = "reprojectImage" inputs = [("inputImage",
class pyGDPwebProcessing(): """ This class allows interactive calls to be made into the GDP. """ def _init_(self, wfsUrl=WFS_URL, wpsUrl=WPS_URL, version='1.1.0'): self.wfsUrl = wfsUrl self.wpsUrl = wpsUrl self.version = version self.wps = WebProcessingService(wpsUrl) def WPSgetCapbilities(self, xml=None): """ Returns a list of capabilities. """ self.wps.getcapabilities(xml) def WPSdescribeprocess(self, identifier, xml=None): """ Returns a list describing a specific identifier/process. """ self.wps.describeprocess(identifier, xml) def _encodeZipFolder(self, filename): """ This function will encode a zipfile and return the filename. """ #check extension if not filename.endswith('.zip'): raise Exception('Wrong filetype.') #encode the file with open(filename, 'rb') as fin: bytesRead = fin.read() encode= base64.b64encode(bytesRead) #renames the file and saves it onto local drive filename = filename.split('.') filename = str(filename[0]) + '_copy.' + str(filename[-1]) fout = open(filename, "w") fout.write(encode) fout.close() return filename def shapeToZip(self,inShape, outZip=None, allFiles=True): """Packs a shapefile to ZIP format. arguments -inShape - input shape file -outZip - output ZIP file (optional) default: <inShapeName>.zip in same folder as inShape (If full path not specified, output is written to to same folder as inShape) -allFiles - Include all files? (optional) True (default) - all shape file components False - just .shp,.shx,.dbf,.prj,shp.xml files reference: Esri, Inc, 1998, Esri Shapefile Technical Description http://www.esri.com/library/whitepapers/pdfs/shapefile.pdf author: Curtis Price, [email protected]""" if not os.path.splitext(inShape)[1] == ".shp": raise Exception, "inShape must be a *.shp" if not os.path.exists(inShape): raise Exception, "%s not found" % inShape # get shapefile root name "path/file.shp" -> "file" # and shapefile path rootName = os.path.splitext(os.path.basename(inShape))[0] inShape = os.path.realpath(inShape) inDir = os.path.dirname(inShape) # output zip file path if outZip in [None,""]: # default output: shapefilepath/shapefilename.zip outDir = inDir outZip = os.path.join(outDir,rootName) + ".zip" else: outDir = os.path.dirname(outZip) if outDir.strip() in ["","."]: # if full path not specified, use input shapefile folder outDir = os.path.dirname(os.path.realpath(inShape)) else: # if output path does exist, raise an exception if not os.path.exists(outDir): raise Exception, "Output folder %s not found" % outDir outZip = os.path.join(outDir,outZip) # enforce .zip extension outZip = os.path.splitext(outZip)[0] + ".zip" if not os.access(outDir, os.W_OK): raise Exception, "Output directory %s not writeable" % outDir if os.path.exists(outZip): os.unlink(outZip) try: # open zipfile zf = zipfile.ZipFile(outZip, 'w', zipfile.ZIP_DEFLATED) # write shapefile parts to zipfile ShapeExt = ["shp","shx","dbf","prj","shp.xml"] if allFiles: ShapeExt += ["sbn","sbx","fbn","fbx", "ain","aih","isx","mxs","atx","cpg"] for f in ["%s.%s" % (os.path.join(inDir,rootName),ext) for ext in ShapeExt]: if os.path.exists(f): zf.write(f) ##print f # debug print return outZip except Exception, msg: raise Exception, \ "Could not write zipfile " + outZip + "\n" + str(msg) finally:
# instantiate WPS client verbose = False wps = WebProcessingService('http://rsg.pml.ac.uk/wps/vector.cgi', verbose=verbose, skip_caps=True) # 1) GetCapabilities wps.getcapabilities() print 'WPS Identification type: %s' % wps.identification.type print 'WPS Identification title: %s' % wps.identification.title print 'WPS Identification abstract: %s' % wps.identification.abstract for operation in wps.operations: print 'WPS Operation: %s' % operation.name for process in wps.processes: print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title) # 2) DescribeProcess process = wps.describeprocess('v.net.path') # alternatively, read process description from XML file (no live request to WPS server) #xml = open('../tests/USGSDescribeProcess.xml', 'r').read() #process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml) print 'WPS Process: identifier=%s' % process.identifier print 'WPS Process: title=%s' % process.title print 'WPS Process: abstract=%s' % process.abstract for input in process.dataInputs: print 'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d' % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs) for output in process.processOutputs: print 'Process output: identifier=%s, data type=%s' % (output.identifier, output.dataType) # 3) Execute # GET request: http://rsg.pml.ac.uk/wps/vector.cgi?request=execute&service=WPS&version=1.0.0&identifier=v.net.path&datainputs=[input=http://rsg.pml.ac.uk/wps/example/graph.gml;file=1%20-960123.1421801624%204665723.56559387%20-101288.65106088226%205108200.011823481] processid = "v.net.path" inputs = [ ("input","http://rsg.pml.ac.uk/wps/example/graph.gml"),
# instantiate WPS client verbose = False wps = WebProcessingService('http://rsg.pml.ac.uk/wps/generic.cgi', verbose=verbose) # 1) GetCapabilities wps.getcapabilities() print 'WPS Identification type: %s' % wps.identification.type print 'WPS Identification title: %s' % wps.identification.title print 'WPS Identification abstract: %s' % wps.identification.abstract for operation in wps.operations: print 'WPS Operation: %s' % operation.name for process in wps.processes: print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title) # 2) DescribeProcess process = wps.describeprocess('reprojectImage') print 'WPS Process: identifier=%s' % process.identifier print 'WPS Process: title=%s' % process.title print 'WPS Process: abstract=%s' % process.abstract for input in process.dataInputs: print 'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d' % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs) for output in process.processOutputs: print 'Process output: identifier=%s, data type=%s' % (output.identifier, output.dataType) # 3a) Execute # GET request: http://rsg.pml.ac.uk/wps/generic.cgi?request=Execute&service=wps&version=1.0.0&identifier=reprojectImage&datainputs=[inputImage=http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img;outputSRS=EPSG:4326]&responsedocument=outputImage=@asreference=true processid = "reprojectImage" inputs = [ ("inputImage","http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img"), ("outputSRS", "EPSG:4326") ] output = "outputImage" execution = wps.execute(processid, inputs, output)
if request == "GetCapabilities": wps.getcapabilities() print("WPS Identification type: %s" % wps.identification.type) print("WPS Identification title: %s" % wps.identification.title) print("WPS Identification abstract: %s" % wps.identification.abstract) for operation in wps.operations: print("WPS Operation: %s" % operation.name) for process in wps.processes: print("WPS Process: identifier=%s title=%s" % (process.identifier, process.title)) elif request == "DescribeProcess": if identifier is None: print('\nERROR: missing mandatory "-i (or --identifier)" argument') usage() sys.exit(4) process = wps.describeprocess(identifier) print("WPS Process: identifier=%s" % process.identifier) print("WPS Process: title=%s" % process.title) print("WPS Process: abstract=%s" % process.abstract) for input in process.dataInputs: print( "Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d" % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs) ) for output in process.processOutputs: print("Process output: identifier=%s, data type=%s" % (output.identifier, output.dataType)) elif request == "Execute": if xml is None: print('\nERROR: missing mandatory "-x (or --xml)" argument') usage()
return ref # Hummingbird WPS url wps_url = 'https://pavics.ouranos.ca/twitcher/ows/proxy/hummingbird/wps' # connection wps = WebProcessingService(url=wps_url) # print wps title print(wps.identification.title) for process in wps.processes: print('%s \t : %s \n' % (process.identifier, process.abstract)) # ncdump proc_name = 'ncdump' process = wps.describeprocess(proc_name) # get process info print(process.abstract) print("Inputs:") for inputs in process.dataInputs: print(' * ', inputs.identifier) # Example netcdf url to NRCAN daily - tasmin 2013 nc_url = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/nrcan/nrcan_canada_daily/tasmin/nrcan_canada_daily_tasmin_2013.nc' print(nc_url) myinputs = [] myinputs.append( ('dataset_opendap', nc_url) ) # inputs : use opendap link of a single netcdf file from catalogue search to erun ncdump print(myinputs)
class BirdyCLI(click.MultiCommand): """BirdyCLI is an implementation of :class:`click.MultiCommand`. It adds each process of a Web Processing Service as command to the command-line interface. :param url: URL of the Web Processing Service. :param xml: A WPS GetCapabilities response for testing. """ def __init__(self, name=None, url=None, xml=None, **attrs): click.MultiCommand.__init__(self, name, **attrs) self.url = os.environ.get('WPS_SERVICE') or url self.verify = get_ssl_verify() self.xml = xml self.wps = WebProcessingService(self.url, verify=self.verify, skip_caps=True) self.commands = OrderedDict() def _update_commands(self): if not self.commands: try: self.wps.getcapabilities(xml=self.xml) except SSLError: raise ConnectionError('SSL verfication of server certificate failed. Set WPS_SSL_VERIFY=false.') except Exception: raise ConnectionError("Web Processing Service not available.") for process in self.wps.processes: self.commands[process.identifier] = dict( name=process.identifier, url=self.wps.url, version=process.processVersion, help=BirdyCLI.format_command_help(process), options=[]) def list_commands(self, ctx): ctx.obj = True self._update_commands() return self.commands.keys() def get_command(self, ctx, name): self._update_commands() cmd_templ = template_env.get_template('cmd.py.j2') rendered_cmd = cmd_templ.render(self._get_command_info(name, details=ctx.obj is None or False)) ns = {} code = compile(rendered_cmd, filename='<string>', mode='exec') eval(code, ns, ns) return ns['cli'] def _get_command_info(self, name, details=False): cmd = self.commands.get(name) if details: pp = self.wps.describeprocess(name) for inp in pp.dataInputs: help = inp.title or '' default = BirdyCLI.get_param_default(inp) if default: help = "{}. Default: {}".format(help, default) cmd['options'].append(dict( name=inp.identifier, # default=BirdyCLI.get_param_default(inp), help=help, type=BirdyCLI.get_param_type(inp), multiple=inp.maxOccurs > 1)) return cmd @staticmethod def format_command_help(process): return "{}: {}".format(process.title or process.identifier, process.abstract or '') @staticmethod def get_param_default(param): if 'ComplexData' in param.dataType: # TODO: get default value of complex type default = None elif 'BoundingBoxData' in param.dataType: # TODO: get default value of bbox default = None else: default = getattr(param, 'defaultValue', None) return default @staticmethod def get_param_type(param): if param.dataType is None: param_type = click.STRING elif 'boolean' in param.dataType: param_type = click.BOOL elif 'integer' in param.dataType: param_type = click.INT elif 'float' in param.dataType: param_type = click.FLOAT elif 'ComplexData' in param.dataType: param_type = COMPLEX else: param_type = click.STRING return param_type
def execute(self, workflow_inputs, out_dir, expected_outputs): self.update_status("Preparing execute request for remote WPS1 provider.", REMOTE_JOB_PROGRESS_REQ_PREP, status.STATUS_RUNNING) LOGGER.debug("Execute process WPS request for %s", self.process) try: try: wps = WebProcessingService(url=self.provider, headers=self.cookies, verify=self.verify) raise_on_xml_exception(wps._capabilities) # noqa: W0212 except Exception as ex: raise OWSNoApplicableCode("Failed to retrieve WPS capabilities. Error: [{}].".format(str(ex))) try: process = wps.describeprocess(self.process) except Exception as ex: raise OWSNoApplicableCode("Failed to retrieve WPS process description. Error: [{}].".format(str(ex))) # prepare inputs complex_inputs = [] for process_input in process.dataInputs: if WPS_COMPLEX_DATA in process_input.dataType: complex_inputs.append(process_input.identifier) # remove any 'null' input, should employ the 'default' of the remote WPS process inputs_provided_keys = filter(lambda i: workflow_inputs[i] != "null", workflow_inputs) wps_inputs = [] for input_key in inputs_provided_keys: input_val = workflow_inputs[input_key] # in case of array inputs, must repeat (id,value) # in case of complex input (File), obtain location, otherwise get data value if not isinstance(input_val, list): input_val = [input_val] input_values = [] for val in input_val: if isinstance(val, dict): val = val["location"] # owslib only accepts strings, not numbers directly if isinstance(val, (int, float)): val = str(val) if val.startswith("file://"): # we need to host file starting with file:// scheme val = self.host_file(val) input_values.append(val) # need to use ComplexDataInput structure for complex input # TODO: BoundingBox not supported for input_value in input_values: if input_key in complex_inputs: input_value = ComplexDataInput(input_value) wps_inputs.append((input_key, input_value)) # prepare outputs outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA) for o in process.processOutputs if o.identifier in expected_outputs] self.update_status("Executing job on remote WPS1 provider.", REMOTE_JOB_PROGRESS_EXECUTION, status.STATUS_RUNNING) mode = EXECUTE_MODE_ASYNC execution = wps.execute(self.process, inputs=wps_inputs, output=outputs, mode=mode, lineage=True) if not execution.process and execution.errors: raise execution.errors[0] self.update_status("Monitoring job on remote WPS1 provider : [{0}]".format(self.provider), REMOTE_JOB_PROGRESS_MONITORING, status.STATUS_RUNNING) max_retries = 5 num_retries = 0 run_step = 0 job_id = "<undefined>" while execution.isNotComplete() or run_step == 0: if num_retries >= max_retries: raise Exception("Could not read status document after {} retries. Giving up.".format(max_retries)) try: execution = check_wps_status(location=execution.statusLocation, verify=self.verify, sleep_secs=wait_secs(run_step)) job_id = execution.statusLocation.replace(".xml", "").split("/")[-1] LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()), execution.percentCompleted, execution.statusMessage, execution.statusLocation)) self.update_status(get_job_log_msg(status=status.map_status(execution.getStatus()), message=execution.statusMessage, progress=execution.percentCompleted, duration=None), # get if available map_progress(execution.percentCompleted, REMOTE_JOB_PROGRESS_MONITORING, REMOTE_JOB_PROGRESS_FETCH_OUT), status.STATUS_RUNNING) except Exception as exc: num_retries += 1 LOGGER.debug("Exception raised: %r", exc) sleep(1) else: num_retries = 0 run_step += 1 if not execution.isSucceded(): exec_msg = execution.statusMessage or "Job failed." LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()), execution.percentCompleted, exec_msg, execution.statusLocation)) raise Exception(execution.statusMessage or "Job failed.") self.update_status("Fetching job outputs from remote WPS1 provider.", REMOTE_JOB_PROGRESS_FETCH_OUT, status.STATUS_RUNNING) results = [ows2json_output(output, process) for output in execution.processOutputs] for result in results: result_id = get_any_id(result) result_val = get_any_value(result) if result_id in expected_outputs: # This is where cwl expect the output file to be written # TODO We will probably need to handle multiple output value... dst_fn = "/".join([out_dir.rstrip("/"), expected_outputs[result_id]]) # TODO Should we handle other type than File reference? resp = request_extra("get", result_val, allow_redirects=True, settings=self.settings) LOGGER.debug("Fetching result output from [%s] to cwl output destination: [%s]", result_val, dst_fn) with open(dst_fn, mode="wb") as dst_fh: dst_fh.write(resp.content) except Exception as exc: exception_class = "{}.{}".format(type(exc).__module__, type(exc).__name__) errors = "{0}: {1!s}".format(exception_class, exc) LOGGER.exception(exc) raise Exception(errors) self.update_status("Execution on remote WPS1 provider completed.", REMOTE_JOB_PROGRESS_COMPLETED, status.STATUS_SUCCEEDED)
class WPSClient(object): """Returns a class where every public method is a WPS process available at the given url. Example: >>> emu = WPSClient(url='<server url>') >>> emu.hello('stranger') 'Hello stranger' """ def __init__( self, url, processes=None, converters=None, username=None, password=None, headers=None, auth=None, verify=True, cert=None, verbose=False, progress=False, version=WPS_DEFAULT_VERSION, caps_xml=None, desc_xml=None, language=None, ): """ Args: url (str): Link to WPS provider. config (Config): an instance processes: Specify a subset of processes to bind. Defaults to all processes. converters (dict): Correspondence of {mimetype: class} to convert this mimetype to a python object. username (str): passed to :class:`owslib.wps.WebProcessingService` password (str): passed to :class:`owslib.wps.WebProcessingService` headers (str): passed to :class:`owslib.wps.WebProcessingService` auth (requests.auth.AuthBase): requests-style auth class to authenticate, see https://2.python-requests.org/en/master/user/authentication/ verify (bool): passed to :class:`owslib.wps.WebProcessingService` cert (str): passed to :class:`owslib.wps.WebProcessingService` verbose (str): passed to :class:`owslib.wps.WebProcessingService` progress (bool): If True, enable interactive user mode. version (str): WPS version to use. language (str): passed to :class:`owslib.wps.WebProcessingService` ex: 'fr-CA', 'en_US'. """ self._converters = converters self._interactive = progress self._mode = ASYNC if progress else SYNC self._notebook = notebook.is_notebook() self._inputs = {} self._outputs = {} if not verify: import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) if headers is None: headers = {} if auth is not None: if isinstance(auth, tuple) and len(auth) == 2: # special-case basic HTTP auth auth = requests.auth.HTTPBasicAuth(*auth) # We only need some headers from the requests.auth.AuthBase implementation # We prepare a dummy request, call the auth object with it, and get its headers dummy_request = requests.Request("get", "http://localhost") r = auth(dummy_request.prepare()) auth_headers = ["Authorization", "Proxy-Authorization", "Cookie"] headers.update( {h: r.headers[h] for h in auth_headers if h in r.headers}) self._wps = WebProcessingService(url, version=version, username=username, password=password, verbose=verbose, headers=headers, verify=verify, cert=cert, skip_caps=True, language=language) try: self._wps.getcapabilities(xml=caps_xml) except ServiceException as e: if "AccessForbidden" in str(e): raise UnauthorizedException( "You are not authorized to do a request of type: GetCapabilities" ) raise self._processes = self._get_process_description(processes, xml=desc_xml) # Build the methods for pid in self._processes: setattr(self, sanitize(pid), types.MethodType(self._method_factory(pid), self)) self.logger = logging.getLogger('WPSClient') if progress: self._setup_logging() self.__doc__ = utils.build_wps_client_doc(self._wps, self._processes) @property def language(self): return self._wps.language @language.setter def language(self, value): self._wps.language = value @property def languages(self): return self._wps.languages def _get_process_description(self, processes=None, xml=None): """Return the description for each process. Sends the server a `describeProcess` request for each process. Parameters ---------- processes: str, list, None A process name, a list of process names or None (for all processes). Returns ------- OrderedDict A dictionary keyed by the process identifier of process descriptions. """ all_wps_processes = [p.identifier for p in self._wps.processes] if processes is None: if owslib.__version__ > '0.17.0': # Get the description for all processes in one request. ps = self._wps.describeprocess('all', xml=xml) return OrderedDict((p.identifier, p) for p in ps) else: processes = all_wps_processes # Check for invalid process names, i.e. not matching the getCapabilities response. process_names, missing = utils.filter_case_insensitive( processes, all_wps_processes) if missing: message = "These process names were not found on the WPS server: {}" raise ValueError(message.format(", ".join(missing))) # Get the description for each process. ps = [self._wps.describeprocess(pid, xml=xml) for pid in process_names] return OrderedDict((p.identifier, p) for p in ps) def _setup_logging(self): self.logger.setLevel(logging.INFO) import sys fh = logging.StreamHandler(sys.stdout) fh.setFormatter(logging.Formatter('%(asctime)s: %(message)s')) self.logger.addHandler(fh) def _method_factory(self, pid): """Create a custom function signature with docstring, instantiate it and pass it to a wrapper which will actually call the process. Parameters ---------- pid: str Identifier of the WPS process. Returns ------- func A Python function calling the remote process, complete with docstring and signature. """ process = self._processes[pid] required_inputs_first = sorted(process.dataInputs, key=sort_inputs_key) input_names = [] # defaults will be set to the function's __defaults__: # A tuple containing default argument values for those arguments that have defaults, # or None if no arguments have a default value. defaults = [] for inpt in required_inputs_first: input_names.append(sanitize(inpt.identifier)) if inpt.minOccurs == 0 or inpt.defaultValue is not None: default = inpt.defaultValue if inpt.dataType != "ComplexData" else None defaults.append(utils.from_owslib(default, inpt.dataType)) defaults = tuple(defaults) if defaults else None body = dedent(""" inputs = locals() inputs.pop('self') return self._execute('{pid}', **inputs) """).format(pid=pid) func_builder = FunctionBuilder( name=sanitize(pid), doc=utils.build_process_doc(process), args=["self"] + input_names, defaults=defaults, body=body, filename=__file__, module=self.__module__, ) self._inputs[pid] = {} if hasattr(process, "dataInputs"): self._inputs[pid] = OrderedDict( (i.identifier, i) for i in process.dataInputs) self._outputs[pid] = {} if hasattr(process, "processOutputs"): self._outputs[pid] = OrderedDict( (o.identifier, o) for o in process.processOutputs) func = func_builder.get_func() return func def _build_inputs(self, pid, **kwargs): """Build the input sequence from the function arguments.""" wps_inputs = [] for name, input_param in list(self._inputs[pid].items()): arg = kwargs.get(sanitize(name)) if arg is None: continue values = [ arg, ] if not isinstance(arg, (list, tuple)) else arg supported_mimetypes = [ v.mimeType for v in input_param.supportedValues ] for value in values: # if input_param.dataType == "ComplexData": seems simpler if isinstance(input_param.defaultValue, ComplexData): # Guess the mimetype of the input value mimetype, encoding = guess_type(value, supported_mimetypes) if encoding is None: encoding = input_param.defaultValue.encoding if isinstance(value, ComplexData): inp = value # Either embed the file content or just the reference. else: if utils.is_embedded_in_request(self._wps.url, value): # If encoding is None, this will return the actual encoding used (utf-8 or base64). value, encoding = embed(value, mimetype, encoding=encoding) else: value = fix_url(str(value)) inp = utils.to_owslib(value, data_type=input_param.dataType, encoding=encoding, mimetype=mimetype) else: inp = utils.to_owslib(value, data_type=input_param.dataType) wps_inputs.append((name, inp)) return wps_inputs def _execute(self, pid, **kwargs): """Execute the process.""" wps_inputs = self._build_inputs(pid, **kwargs) wps_outputs = [(o.identifier, "ComplexData" in o.dataType) for o in list(self._outputs[pid].values())] mode = self._mode if self._processes[pid].storeSupported else SYNC try: wps_response = self._wps.execute(pid, inputs=wps_inputs, output=wps_outputs, mode=mode) if self._interactive and self._processes[pid].statusSupported: if self._notebook: notebook.monitor(wps_response, sleep=.2) else: self._console_monitor(wps_response) except ServiceException as e: if "AccessForbidden" in str(e): raise UnauthorizedException( "You are not authorized to do a request of type: Execute") raise # Add the convenience methods of WPSResult to the WPSExecution class. This adds a `get` method. utils.extend_instance(wps_response, WPSResult) wps_response.attach(wps_outputs=self._outputs[pid], converters=self._converters) return wps_response def _console_monitor(self, execution, sleep=3): """Monitor the execution of a process. Parameters ---------- execution : WPSExecution instance The execute response to monitor. sleep: float Number of seconds to wait before each status check. """ import signal # Intercept CTRL-C def sigint_handler(signum, frame): self.cancel() signal.signal(signal.SIGINT, sigint_handler) while not execution.isComplete(): execution.checkStatus(sleepSecs=sleep) self.logger.info("{} [{}/100] - {} ".format( execution.process.identifier, execution.percentCompleted, execution.statusMessage[:50], )) if execution.isSucceded(): self.logger.info("{} done.".format(execution.process.identifier)) else: self.logger.info("{} failed.".format(execution.process.identifier))
def execute(self): Outputs={} sys.stdout = open(config.getConfigValue("server","logFile"),'w') try: wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True) identifier = 'dummyprocess' description = wps.describeprocess(identifier) wps_outputs=[] for item in description.processOutputs: if item.dataType == 'ComplexOutput': wps_outputs.append((item.identifier, True)) else: wps_outputs.append((item.identifier, False)) inputs = [('i1',str(self.Input1.getValue())),('i2',str(self.Input2.getValue())),('i3',str(self.Input3.getValue()))] execution = wps.execute(identifier, inputs, output=wps_outputs) monitorExecution(execution) execution.checkStatus(sleepSecs=1) Outputs['dummyprocess_1']={} if "ComplexData" in description.processOutputs: i=0 for output in execution.processOutputs: if description.processOutputs[i].dataType == 'ComplexData': Outputs['dummyprocess_1'][output.identifier]=output.reference else: Outputs['dummyprocess_1'][output.identifier]=output.data[0] i=i+1 else: for output in execution.processOutputs: Outputs['dummyprocess_1'][output.identifier]=output.data[0] x= Outputs['dummyprocess_1' ]['output1'] except: wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True) identifier = 'dummyprocess' description = wps.describeprocess(identifier) wps_outputs=[] for item in description.processOutputs: if item.dataType == 'ComplexOutput': wps_outputs.append((item.identifier, True)) else: wps_outputs.append((item.identifier, False)) inputs = [('i1',str(self.Input1.getValue())),('i2',str(self.Input2.getValue())),('i3',str(self.Input3.getValue()))] execution = wps.execute(identifier, inputs, output=wps_outputs) monitorExecution(execution) execution.checkStatus(sleepSecs=1) Outputs['dummyprocess_1']={} if "ComplexData" in description.processOutputs: i=0 for output in execution.processOutputs: if description.processOutputs[i].dataType == 'ComplexData': Outputs['dummyprocess_1'][output.identifier]=output.reference else: Outputs['dummyprocess_1'][output.identifier]=output.data[0] i=i+1 else: for output in execution.processOutputs: Outputs['dummyprocess_1'][output.identifier]=output.data[0] x= Outputs['dummyprocess_1' ]['output1'] try: wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True) identifier = 'dummyprocess' description = wps.describeprocess(identifier) wps_outputs=[] for item in description.processOutputs: if item.dataType == 'ComplexOutput': wps_outputs.append((item.identifier, True)) else: wps_outputs.append((item.identifier, False)) inputs = [('i1',Outputs['dummyprocess_1']['output1']),('i2',x),('i3',Outputs['dummyprocess_1']['output2'])] execution = wps.execute(identifier, inputs) Outputs['dummyprocess_2']={} if "ComplexData" in description.processOutputs: i=0 for output in execution.processOutputs: if description.processOutputs[i].dataType == 'ComplexData': Outputs['dummyprocess_2'][output.identifier]=output.reference else: Outputs['dummyprocess_2'][output.identifier]=output.data[0] i=i+1 else: for output in execution.processOutputs: Outputs['dummyprocess_2'][output.identifier]=output.data[0] except: wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True) identifier = 'dummyprocess' description = wps.describeprocess(identifier) wps_outputs=[] for item in description.processOutputs: if item.dataType == 'ComplexOutput': wps_outputs.append((item.identifier, True)) else: wps_outputs.append((item.identifier, False)) inputs = [('i1',Outputs['dummyprocess_1']['output1']),('i2',x),('i3',Outputs['dummyprocess_1']['output2'])] execution = wps.execute(identifier, inputs) Outputs['dummyprocess_2']={} if "ComplexData" in description.processOutputs: i=0 for output in execution.processOutputs: if description.processOutputs[i].dataType == 'ComplexData': Outputs['dummyprocess_2'][output.identifier]=output.reference else: Outputs['dummyprocess_2'][output.identifier]=output.data[0] i=i+1 else: for output in execution.processOutputs: Outputs['dummyprocess_2'][output.identifier]=output.data[0] self.dummyprocess_1_output1.setValue(Outputs['dummyprocess_1']['output1']) self.dummyprocess_2_output2.setValue(Outputs['dummyprocess_2']['output2']) self.x.setValue(x) sys.stdout.close() sys.stdout = sys.__stdout__ return
class GenericWPS(MonitorPE): STATUS_NAME = 'status' STATUS_LOCATION_NAME = 'status_location' def __init__(self, url, identifier, resource='resource', inputs=[], output=None, headers=None): MonitorPE.__init__(self) self._add_output(self.STATUS_NAME) self._add_output(self.STATUS_LOCATION_NAME) self.wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers) self.identifier = identifier self.wps_resource = resource self.wps_inputs = inputs self.wps_output = output def progress(self, execution): return int(self._pstart + ((self._pend - self._pstart) / 100.0 * execution.percentCompleted)) def monitor_execution(self, execution): progress = self.progress(execution) self.monitor( "status_location={0.statusLocation}".format(execution), progress) while execution.isNotComplete(): try: execution.checkStatus(sleepSecs=3) except Exception: LOGGER.exception("Could not read status xml document.") else: progress = self.progress(execution) self.monitor(execution.statusMessage, progress) if execution.isSucceded(): for output in execution.processOutputs: self.monitor('ouput={0.identifier}'.format(output), progress) else: self.monitor('\n'.join( ['ERROR: {0.text} code={0.code} locator={0.locator})'. format(ex) for ex in execution.errors]), progress) def _build_wps_inputs(self): process = self.wps.describeprocess(self.identifier) complex_inpts = [] bbox_inpts = [] for inpt in process.dataInputs: if 'ComplexData' in inpt.dataType: complex_inpts.append(inpt.identifier) elif 'BoundingBoxData' in inpt.dataType: bbox_inpts.append(inpt.identifier) inputs = [] for inpt in self.wps_inputs: LOGGER.debug("input=%s", inpt) if inpt[0] in complex_inpts: inputs.append((inpt[0], ComplexDataInput(inpt[1]))) elif inpt[0] in bbox_inpts: inputs.append((inpt[0], BoundingBoxDataInput(inpt[1]))) else: inputs.append(inpt) return inputs def _build_wps_outputs(self): outputs = [] if self.wps_output is not None: outputs = [(self.wps_output, True)] return outputs def execute(self): LOGGER.debug("execute inputs=%s", self.wps_inputs) execution = self.wps.execute( identifier=self.identifier, inputs=self._build_wps_inputs(), output=self._build_wps_outputs(), lineage=True) self.monitor_execution(execution) result = {self.STATUS_NAME: execution.status, self.STATUS_LOCATION_NAME: execution.statusLocation} if execution.isSucceded(): # NOTE: only set workflow output if specific output was requested if self.wps_output is not None: for output in execution.processOutputs: if self.wps_output == output.identifier: result[self.OUTPUT_NAME] = output.reference break return result else: failure_msg = '\n'.join(['{0.text}'. format(ex) for ex in execution.errors]) raise Exception(failure_msg) def _set_inputs(self, inputs): if self.INPUT_NAME in inputs: for value in inputs[self.INPUT_NAME]: self.wps_inputs.append((self.wps_resource, value)) def process(self, inputs): try: result = self._process(inputs) if result is not None: return result except Exception: LOGGER.exception("process failed!") raise def _process(self, inputs): self._set_inputs(inputs) return self.execute()
wps.getcapabilities() # alternatively, read capabilities from XML file (no live request to WPS server) #xml = open('../tests/USGSCapabilities.xml', 'r').read() #wps.getcapabilities(xml=xml) print 'WPS Identification type: %s' % wps.identification.type print 'WPS Identification title: %s' % wps.identification.title print 'WPS Identification abstract: %s' % wps.identification.abstract for operation in wps.operations: print 'WPS Operation: %s' % operation.name for process in wps.processes: print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title) # 2) DescribeProcess # Submits an HTTP GET "DescribeProcess" request to the WPS service and parses the HTTP response process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm') # alternatively, read process description from XML file (no live request to WPS server) #xml = open('../tests/USGSDescribeProcess.xml', 'r').read() #process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml) print 'WPS Process: identifier=%s' % process.identifier print 'WPS Process: title=%s' % process.title print 'WPS Process: abstract=%s' % process.abstract for input in process.dataInputs: print 'Process input:' printInputOutput(input, indent='\t') for output in process.processOutputs: print 'Process output:' printInputOutput(output, indent='\t') # 3a) Execute # Submits an HTTP POST "Execute" process request to the WPS service, keeps checking the status of the request,
class ExecuteProcess(MyView): def __init__(self, request): self.request = request self.execution = None self.service_name = None self.processid = None self.process = None if 'job_id' in request.params: job = request.db.jobs.find_one( {'identifier': request.params['job_id']}) self.service_name = job.get('service_name') self.execution = check_status(url=job.get('status_location'), response=job.get('response'), verify=False, sleep_secs=0) self.processid = self.execution.process.identifier elif 'wps' in request.params: self.service_name = request.params.get('wps') self.processid = request.params.get('process') if self.service_name: # TODO: avoid getcaps self.wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.service_name), verify=False) # TODO: need to fix owslib to handle special identifiers self.process = self.wps.describeprocess(self.processid) super(ExecuteProcess, self).__init__(request, name='processes_execute', title='') def breadcrumbs(self): breadcrumbs = super(ExecuteProcess, self).breadcrumbs() breadcrumbs.append( dict(route_path=self.request.route_path('processes'), title='Processes')) breadcrumbs.append( dict(route_path=self.request.route_path('processes_list', _query=[('wps', self.service_name) ]), title=self.service_name)) breadcrumbs.append( dict(route_path=self.request.route_path(self.name), title=self.process.identifier)) return breadcrumbs def appstruct(self): # TODO: not a nice way to get inputs ... should be cleaned up in owslib result = {} if self.execution: for inp in self.execution.dataInputs: if inp.data or inp.reference: if inp.identifier not in result: # init result for param with empty list result[inp.identifier] = [] if inp.data: # add literal input, inp.data is a list result[inp.identifier].extend(inp.data) elif inp.reference: # add reference to complex input result[inp.identifier].append(inp.reference) for inp in self.process.dataInputs: # TODO: dupliate code in wizard.start # convert boolean if 'boolean' in inp.dataType and inp.identifier in result: result[inp.identifier] = [ val.lower() == 'true' for val in result[inp.identifier] ] elif 'dateTime' in inp.dataType and inp.identifier in result: result[inp.identifier] = [ dateparser.parse(val) for val in result[inp.identifier] ] elif 'date' in inp.dataType and inp.identifier in result: result[inp.identifier] = [ dateparser.parse(val) for val in result[inp.identifier] ] elif 'time' in inp.dataType and inp.identifier in result: result[inp.identifier] = [ dateparser.parse(val) for val in result[inp.identifier] ] elif inp.dataType == 'BoundingBoxData' and inp.identifier in result: result[inp.identifier] = [ "{0.minx},{0.miny},{0.maxx},{0.maxy}".format(bbox) for bbox in result[inp.identifier] ] # TODO: very dirty ... if single value then take the first if inp.maxOccurs < 2 and inp.identifier in result: result[inp.identifier] = result[inp.identifier][0] return result def generate_form(self, formid='deform'): schema = WPSSchema(request=self.request, process=self.process, use_async=self.request.has_permission('admin'), user=self.request.user) submit_button = Button(name='submit', title='Submit', css_class='btn btn-success btn-lg btn-block', disabled=not has_execute_permission( self.request, self.service_name)) return Form( schema.bind(request=self.request), buttons=(submit_button, ), formid=formid, ) def process_form(self, form): controls = list(self.request.POST.items()) try: # TODO: uploader puts qqfile in controls controls = [ control for control in controls if 'qqfile' not in control[0] ] LOGGER.debug("before validate %s", controls) appstruct = form.validate(controls) LOGGER.debug("before execute %s", appstruct) job_id = self.execute(appstruct) except ValidationFailure as e: self.session.flash("Page validation failed.", queue='danger') return dict(process=self.process, url=wps_describe_url(self.wps.url, self.processid), form=e.render()) else: if not self.request.user: # not logged-in return HTTPFound(location=self.request.route_url( 'job_status', job_id=job_id)) else: return HTTPFound(location=self.request.route_url('monitor')) def execute(self, appstruct): inputs = appstruct_to_inputs(self.request, appstruct) # need to use ComplexDataInput complex_inpts = {} bbox_inpts = [] for inpt in self.process.dataInputs: if 'ComplexData' in inpt.dataType: complex_inpts[inpt.identifier] = inpt elif 'BoundingBoxData' in inpt.dataType: bbox_inpts.append(inpt.identifier) new_inputs = [] for inpt in inputs: identifier = inpt[0] value = inpt[1] if identifier in complex_inpts: new_inputs.append((identifier, ComplexDataInput(value))) if is_reference(value): if value not in self.request.cart: if complex_inpts[identifier].supportedValues: mime_type = complex_inpts[ identifier].supportedValues[0].mimeType else: mime_type = None LOGGER.debug("add input to cart: %s %s", identifier, mime_type) self.request.cart.add_item( value, abstract= "Automatically added in process execution.", mime_type=mime_type) elif identifier in bbox_inpts: new_inputs.append((identifier, BoundingBoxDataInput(value))) else: new_inputs.append(inpt) inputs = new_inputs # prepare outputs outputs = [] for output in self.process.processOutputs: outputs.append( (output.identifier, output.dataType == 'ComplexData')) from phoenix.tasks.execute import execute_process result = execute_process.delay( userid=self.request.unauthenticated_userid, url=self.wps.url, service_name=self.service_name, identifier=self.process.identifier, inputs=inputs, outputs=outputs, async=appstruct.get('_async_check', True)) self.request.registry.notify(JobStarted(self.request, result.id)) return result.id @view_config(route_name='processes_execute', renderer='phoenix:processes/templates/processes/execute.pt', accept='text/html') def view(self): form = self.generate_form() if 'submit' in self.request.POST: check_csrf_token(self.request) return self.process_form(form) if not has_execute_permission(self.request, self.service_name): msg = """<strong>Warning:</strong> You are not allowed to run this process. Please <a href="{0}" class="alert-link">sign in</a> and wait for account activation.""" msg = msg.format(self.request.route_path('sign_in')) self.session.flash(msg, queue='warning') return dict(process=self.process, url=wps_describe_url(self.wps.url, self.processid), form=form.render(self.appstruct()))
class pyGDPwebProcessing(): """ This class allows interactive calls to be made into the GDP. """ def __init__(self, WFS_URL=None): if WFS_URL == None: from pygdp.namespaces import WFS_URL wfsUrl = WFS_URL self.wfsUrl = wfsUrl self.wpsUrl = WPS_URL self.version = '1.1.0' self.wps = WebProcessingService(WPS_URL) def WPSgetCapbilities(self, xml=None): """ Returns a list of capabilities. """ self.wps.getcapabilities(xml) def WPSdescribeprocess(self, identifier, xml=None): """ Returns a list describing a specific identifier/process. """ self.wps.describeprocess(identifier, xml) #pyGDP Submit Feature def dodsReplace(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return _execute_request.dodsReplace(dataSetURI, verbose) def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCoverageOPenDAP( geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs) def submitFeatureCoverageWCSIntersection(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCoverageWCSIntersection( geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs) def submitFeatureCategoricalGridCoverage(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true', delim='COMMA', outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return feature_coverage.submitFeatureCategoricalGridCoverage( geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose, coverage, delim, self.wfsUrl, outputfname, sleepSecs) def submitFeatureWeightedGridStatistics(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None, verbose=None, coverage=True, delim='COMMA', stat='MEAN', grpby='STATISTIC', timeStep=False, summAttr=False, weighted=True, outputfname=None, sleepSecs=10): if verbose: ch.setLevel(logging.INFO) return fwgs.submitFeatureWeightedGridStatistics( geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs, verbose, coverage, delim, stat, grpby, timeStep, summAttr, weighted, self.wfsUrl, outputfname, sleepSecs) #pyGDP File Utilities def shapeToZip(self, inShape, outZip=None, allFiles=True): return shape_to_zip.shapeToZip(inShape, outZip=None, allFiles=True) def uploadShapeFile(self, filePath): value = upload_shapefile.uploadShapeFile(filePath) return value #pyGDP WFS Utilities def getTuples(self, shapefile, attribute): return shapefile_id_handle.getTuples(shapefile, attribute) def getShapefiles(self): return shapefile_value_handle.getShapefiles(self.wfsUrl) def getAttributes(self, shapefile): return shapefile_value_handle.getAttributes(shapefile, self.wfsUrl) def getValues(self, shapefile, attribute, getTuples='false', limitFeatures=None): return shapefile_value_handle.getValues(shapefile, attribute, getTuples, limitFeatures, self.wfsUrl) def getGMLIDs(self, shapefile, attribute, value): return shapefile_id_handle.getGMLIDs(shapefile, attribute, value, WFS_URL=self.wfsUrl) def _getFilterID(self, tuples, value): return shapefile_id_handle._getFilterID(tuples, value) def _getFeatureCollectionGeoType(self, geoType, attribute='the_geom', value=None, gmlIDs=None): return _get_geotype._getFeatureCollectionGeoType( geoType, attribute, value, gmlIDs, self.wfsUrl) def _generateRequest(self, dataSetURI, algorithm, method, varID=None, verbose=False): return _webdata_xml_generate._generateRequest(dataSetURI, algorithm, method, varID, verbose) #pyGDP WebData Utilities def getDataLongName(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getDataLongName(dataSetURI, verbose) def getDataType(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getDataType(dataSetURI, verbose) def getDataUnits(self, dataSetURI, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getDataUnits(dataSetURI, verbose) def getDataSetURI(self, anyText=None, CSWURL=CSWURL, BBox=None): """ Searches a given CSW server and returns metadata content for the datasets found. :param anyText: keywords to be passed to CSW get records :type anyText: list or None """ return webdata_handle.getDataSetURI(anyText, CSWURL, BBox) def getTimeRange(self, dataSetURI, varID, verbose=False): if verbose: ch.setLevel(logging.INFO) return webdata_handle.getTimeRange(dataSetURI, varID, verbose) #Pull up docstrings. #dodsReplace.__doc__ = _execute_request.dodsReplace.__doc__ getAttributes.__doc__ = shapefile_value_handle.getAttributes.__doc__ getDataLongName.__doc__ = webdata_handle.getDataLongName.__doc__ getDataSetURI.__doc__ = webdata_handle.getDataSetURI.__doc__ getDataType.__doc__ = webdata_handle.getDataType.__doc__ getDataUnits.__doc__ = webdata_handle.getDataUnits.__doc__ getGMLIDs.__doc__ = shapefile_id_handle.getGMLIDs.__doc__ getShapefiles.__doc__ = shapefile_value_handle.getShapefiles.__doc__ getTimeRange.__doc__ = webdata_handle.getTimeRange.__doc__ getTuples.__doc__ = shapefile_id_handle.getTuples.__doc__ getValues.__doc__ = shapefile_value_handle.getValues.__doc__ shapeToZip.__doc__ = shape_to_zip.shapeToZip.__doc__ submitFeatureCategoricalGridCoverage.__doc__ = feature_coverage.submitFeatureCategoricalGridCoverage.__doc__ submitFeatureCoverageOPenDAP.__doc__ = feature_coverage.submitFeatureCoverageOPenDAP.__doc__ submitFeatureCoverageWCSIntersection.__doc__ = feature_coverage.submitFeatureCoverageWCSIntersection.__doc__ submitFeatureWeightedGridStatistics.__doc__ = fwgs.submitFeatureWeightedGridStatistics.__doc__ uploadShapeFile.__doc__ = upload_shapefile.uploadShapeFile.__doc__
# alternatively, read capabilities from XML file (no live request to WPS server) #xml = open('../tests/USGSCapabilities.xml', 'r').read() #wps.getcapabilities(xml=xml) print 'WPS Identification type: %s' % wps.identification.type print 'WPS Identification title: %s' % wps.identification.title print 'WPS Identification abstract: %s' % wps.identification.abstract for operation in wps.operations: print 'WPS Operation: %s' % operation.name for process in wps.processes: print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title) # 2) DescribeProcess # Submits an HTTP GET "DescribeProcess" request to the WPS service and parses the HTTP response process = wps.describeprocess( 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm') # alternatively, read process description from XML file (no live request to WPS server) #xml = open('../tests/USGSDescribeProcess.xml', 'r').read() #process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml) print 'WPS Process: identifier=%s' % process.identifier print 'WPS Process: title=%s' % process.title print 'WPS Process: abstract=%s' % process.abstract for input in process.dataInputs: print 'Process input:' printInputOutput(input, indent='\t') for output in process.processOutputs: print 'Process output:' printInputOutput(output, indent='\t') # 3a) Execute # Submits an HTTP POST "Execute" process request to the WPS service, keeps checking the status of the request,
class ExecuteProcess(MyView): def __init__(self, request): self.request = request self.execution = None self.service_name = None self.processid = None self.process = None if 'job_id' in request.params: job = request.db.jobs.find_one( {'identifier': request.params['job_id']}) self.service_name = job.get('service_name') self.execution = check_status(url=job.get('status_location'), response=job.get('response'), verify=False, sleep_secs=0) self.processid = self.execution.process.identifier elif 'wps' in request.params: self.service_name = request.params.get('wps') self.processid = request.params.get('process') if self.service_name: # TODO: avoid getcaps self.wps = WebProcessingService(url=request.route_url( 'owsproxy', service_name=self.service_name), verify=False) # TODO: need to fix owslib to handle special identifiers self.process = self.wps.describeprocess(self.processid) super(ExecuteProcess, self).__init__(request, name='namewps_execute', title='') def breadcrumbs(self): breadcrumbs = super(ExecuteProcess, self).breadcrumbs() breadcrumbs.append( dict(route_path=self.request.route_path('namewps_list'), title='NAME processes')) breadcrumbs.append( dict(route_path=self.request.route_path(self.name), title=self.process.identifier)) return breadcrumbs def appstruct(self): # TODO: not a nice way to get inputs ... should be cleaned up in owslib result = {} if self.execution: for inp in self.execution.dataInputs: if inp.data or inp.reference: if inp.identifier not in result: # init result for param with empty list result[inp.identifier] = [] if inp.data: # add literal input, inp.data is a list result[inp.identifier].extend(inp.data) elif inp.reference: # add reference to complex input result[inp.identifier].append(inp.reference) for inp in self.process.dataInputs: # TODO: dupliate code in wizard.start # convert boolean if 'boolean' in inp.dataType and inp.identifier in result: result[inp.identifier] = [ val.lower() == 'true' for val in result[inp.identifier] ] elif inp.dataType in ['date', 'time', 'dateTime' ] and inp.identifier in result: result[inp.identifier] = [ dateparser.parse(val) for val in result[inp.identifier] ] elif inp.dataType == 'BoundingBoxData' and inp.identifier in result: result[inp.identifier] = [ "{0.minx},{0.miny},{0.maxx},{0.maxy}".format(bbox) for bbox in result[inp.identifier] ] # TODO: very dirty ... if single value then take the first if inp.maxOccurs < 2 and inp.identifier in result: result[inp.identifier] = result[inp.identifier][0] return result def generate_form(self, formid='deform'): schema = WPSSchema(request=self.request, process=self.process, use_async=self.request.has_permission('admin'), user=self.request.user) submit_button = Button(name='submit', title='Submit', css_class='btn btn-success btn-lg btn-block', disabled=not has_execute_permission( self.request, self.service_name)) return Form( schema.bind(request=self.request), buttons=(submit_button, ), formid=formid, ) def process_form(self, form): controls = self.request.POST.items() try: # TODO: uploader puts qqfile in controls controls = [ control for control in controls if 'qqfile' not in control[0] ] LOGGER.debug("before validate %s", controls) appstruct = form.validate(controls) LOGGER.debug("before execute %s", appstruct) job_id = self.execute(appstruct) except ValidationFailure, e: self.session.flash("Page validation failed.", queue='danger') return dict(process=self.process, url=wps_describe_url(self.wps.url, self.processid), form=e.render()) else: