def to_owslib(value, data_type, encoding=None, mimetype=None, schema=None): """Convert value into OWSlib objects.""" # owslib only accepts literaldata, complexdata and boundingboxdata if data_type == "ComplexData": return ComplexDataInput(value, encoding=encoding, mimeType=mimetype, schema=schema) if data_type == "BoundingBoxData": # TODO: return BoundingBoxDataInput(data=value, crs=crs, dimensions=2) return value else: # LiteralData return str(value)
def execute_process(self): self.setCursor(Qt.WaitCursor) # Async call: https://ouranosinc.github.io/pavics-sdi/tutorials/wps_with_python.html myinputs = [] for param, widget in self.input_items.items(): if isinstance(widget, QgsMapLayerComboBox): # TODO check input type and export into it (GML, GeoPackage, etc.) layer = widget.currentLayer() if layer is None: iface.messageBar().pushMessage( self.tr("Error"), self.tr("There is not any layer"), level=Qgis.Critical) return if layer.type() == QgsMapLayer.VectorLayer: tmp_ext = '.gml' tmp_frmt = 'GML' else: iface.messageBar().pushMessage( "Error", "Unsupported map layer type", level=Qgis.Critical) return tmp_file = QgsProcessingUtils.generateTempFilename( self.process_identifier + '_' + param) + tmp_ext QgsVectorFileWriter.writeAsVectorFormat(layer, tmp_file, fileEncoding="UTF-8", driverName=tmp_frmt) with open(tmp_file) as fd: cdi = ComplexDataInput(fd.read()) myinputs.append((param, cdi)) elif isinstance(widget, QgsFieldComboBox): myinputs.append((param, widget.currentField())) elif isinstance(widget, QgsDateTimeEdit): myinputs.append((param, widget.date().toString('yyyy-MM-dd'))) else: # TODO check also other types than just QLineEdit if widget.text() != 'None': myinputs.append((param, widget.text())) self.appendLogMessage( self.tr("Executing {} process ...".format( self.process_identifier))) self.executeProcess = ExecuteProcess() self.executeProcess.setUrl(self.service_url) self.executeProcess.setIdentifier(self.process_identifier) # print(myinputs) self.executeProcess.setInputs(myinputs) self.executeProcess.statusChanged.connect( self.on_execute_process_response) self.executeProcess.start()
def execute(self, appstruct): inputs = appstruct_to_inputs(self.request, appstruct) # need to use ComplexDataInput complex_inpts = {} bbox_inpts = [] for inpt in self.process.dataInputs: if 'ComplexData' in inpt.dataType: complex_inpts[inpt.identifier] = inpt elif 'BoundingBoxData' in inpt.dataType: bbox_inpts.append(inpt.identifier) new_inputs = [] for inpt in inputs: identifier = inpt[0] value = inpt[1] if identifier in complex_inpts: new_inputs.append((identifier, ComplexDataInput(value))) if is_reference(value): if value not in self.request.cart: if complex_inpts[identifier].supportedValues: mime_type = complex_inpts[ identifier].supportedValues[0].mimeType else: mime_type = None LOGGER.debug("add input to cart: %s %s", identifier, mime_type) self.request.cart.add_item( value, abstract= "Automatically added in process execution.", mime_type=mime_type) elif identifier in bbox_inpts: new_inputs.append((identifier, BoundingBoxDataInput(value))) else: new_inputs.append(inpt) inputs = new_inputs # prepare outputs outputs = [] for output in self.process.processOutputs: outputs.append( (output.identifier, output.dataType == 'ComplexData')) from phoenix.tasks.execute import execute_process result = execute_process.delay( userid=self.request.unauthenticated_userid, url=self.wps.url, service_name=self.service_name, identifier=self.process.identifier, inputs=inputs, outputs=outputs, async=appstruct.get('_async_check', True)) self.request.registry.notify(JobStarted(self.request, result.id)) return result.id
def build_input(arg, is_data, process_inputs, sh): k, v = arg.split("=", 1) if not v: # skip those not specified, hopefully there will be some sane default return None inp = process_inputs.get(k, None) if not inp: # weird, not one of inputs, ignore return None clean_v = util.restore_text(v) if is_data: file_url = None # really data? check if HTML of previous dataminer if magic.from_file(clean_v, mime=True) == "text/html": # html, try to read it and get the output description parser = CallerHTMLParser() parser.feed(open(clean_v).read()) outputs = parser.caller_dataminer_data() # try to guess which one is the right input if outputs: for out in outputs.get("outputs", []): # discard 'Log of the computation.csv' if out["descriptor"] == "Log of the computation": continue logging.info("Assuming %s as the right input for the process", out) file_url = out["url"] break if not file_url: # we are here, so the html was not working as expected so just copy # to the StorageHub file_url = sh.upload_file(k, clean_v) return (k, ComplexDataInput(file_url, mimeType="text/xml")) else: if inp.dataType == "ComplexData": # assume text/xml is fine always? return (k, ComplexDataInput(clean_v, mimeType="text/xml")) else: # let's assume just taking the value is ok return (k, clean_v)
def test_wps_request8(): # Process input/ouutput arguments processid = "wordcount" textdoc = ComplexDataInput("Alice was beginning to get very tired ...") inputs = [("text", textdoc), ] outputs = [("output", True), ] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=outputs, async=True, lineage=True) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest8.xml'), 'rb').read() assert compare_xml(request, _request) is True
def execute_process(process_name): data = json.loads(request.data) process_data = [] if Constants.DATA_PROPERTY in data.keys(): data_tuples = [(k, v) for k, v in data[Constants.DATA_PROPERTY].items()] process_data = process_data + data_tuples if Constants.GEO_DATA_PROPERTY in data.keys(): geo_date_tuples = [ (k, ComplexDataInput(v, Constants.GEOJSON_FORMAT)) for k, v in data[Constants.GEO_DATA_PROPERTY].items() ] process_data = process_data + geo_date_tuples execution = wps.execute(process_name, process_data) outputs = execution.processOutputs return jsonify(parse_outputs(outputs))
def test_wps_request7(): # Process input/ouutput arguments processid = "wordcount" textdoc = ComplexDataInput("http://emu.readthedocs.org/en/latest/index.html") inputs = [("text", textdoc), ] outputs = [("output", True)] # Build XML request for WPS process execution, sync request execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=outputs, mode=SYNC, lineage=False) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest7.xml'), 'rb').read() print(request) assert compare_xml(request, _request) is True
def execute(self, appstruct): inputs = appstruct_to_inputs(self.request, appstruct) # need to use ComplexDataInput complex_inpts = {} bbox_inpts = [] for inpt in self.process.dataInputs: if 'ComplexData' in inpt.dataType: complex_inpts[inpt.identifier] = inpt elif 'BoundingBoxData' in inpt.dataType: bbox_inpts.append(inpt.identifier) new_inputs = [] for inpt in inputs: identifier = inpt[0] value = inpt[1] if identifier in complex_inpts: new_inputs.append((identifier, ComplexDataInput(value))) elif identifier in bbox_inpts: crs = 'urn:ogc:def:crs:OGC:2:84' new_inputs.append((identifier, BoundingBoxDataInput(value, crs=crs))) else: new_inputs.append(inpt) inputs = new_inputs # prepare outputs outputs = [] for output in self.process.processOutputs: outputs.append( (output.identifier, output.dataType == 'ComplexData')) from phoenix.tasks.execute import execute_process result = execute_process.delay( userid=self.request.unauthenticated_userid, url=self.wps.url, service_name=self.service.title, identifier=self.process.identifier, inputs=inputs, outputs=outputs, use_async=appstruct.get('_async_check', True)) # give the job a chance to start sleep(1) self.request.registry.notify(JobStarted(self.request, result.id)) LOGGER.debug('wps url={}'.format(self.wps.url)) LOGGER.debug('request inputs = {}'.format(str(inputs))) return result.id
def _build_wps_inputs(self): process = self.wps.describeprocess(self.identifier) complex_inpts = [] bbox_inpts = [] for inpt in process.dataInputs: if 'ComplexData' in inpt.dataType: complex_inpts.append(inpt.identifier) elif 'BoundingBoxData' in inpt.dataType: bbox_inpts.append(inpt.identifier) inputs = [] for inpt in self.wps_inputs: LOGGER.debug("input=%s", inpt) if inpt[0] in complex_inpts: inputs.append((inpt[0], ComplexDataInput(inpt[1]))) elif inpt[0] in bbox_inpts: inputs.append((inpt[0], BoundingBoxDataInput(inpt[1]))) else: inputs.append(inpt) return inputs
def parse_wps_inputs(wps_process, job): """ Parses expected WPS process inputs against submitted job input values considering supported process definitions. """ complex_inputs = [] for process_input in wps_process.dataInputs: if WPS_COMPLEX_DATA in process_input.dataType: complex_inputs.append(process_input.identifier) try: wps_inputs = list() # parse both dict and list type inputs job_inputs = job.inputs.items() if isinstance(job.inputs, dict) else job.get("inputs", []) for process_input in job_inputs: if isinstance(process_input, tuple): input_id = process_input[0] process_value = process_input[1] else: input_id = get_any_id(process_input) process_value = get_any_value(process_input) # in case of array inputs, must repeat (id,value) input_values = process_value if isinstance(process_value, list) else [process_value] # we need to support file:// scheme but PyWPS doesn't like them so remove the scheme file:// input_values = [ # when value is an array of dict that each contain a file reference (get_any_value(val)[7:] if str(get_any_value(val)).startswith("file://") else get_any_value(val)) if isinstance(val, dict) else # when value is directly a single dict with file reference (val[7:] if str(val).startswith("file://") else val) for val in input_values ] # need to use ComplexDataInput structure for complex input # need to use literal String for anything else than complex # TODO: BoundingBox not supported wps_inputs.extend([ (input_id, ComplexDataInput(input_value) if input_id in complex_inputs else str(input_value)) for input_value in input_values]) except KeyError: wps_inputs = [] return wps_inputs
def test_smoke_execute_c3s_cmip6_orchestrate_metadata(wps, tmp_path): inputs = [ ("workflow", ComplexDataInput(WF_C3S_CMIP6)), ] urls = wps.execute("orchestrate", inputs) assert len(urls) == 1 assert "rlds_Amon_INM-CM5-0_ssp245_r1i1p1f1_gr1_avg-t.nc" in urls[0] ds = open_dataset(urls[0], tmp_path) assert "rlds" in ds.variables # check fill value in bounds assert "_FillValue" not in ds.lat_bnds.encoding assert "_FillValue" not in ds.lon_bnds.encoding # assert "_FillValue" not in ds.time_bnds.encoding # check fill value in coordinates # assert "_FillValue" not in ds.time.encoding assert "_FillValue" not in ds.lat.encoding assert "_FillValue" not in ds.lon.encoding # assert "_FillValue" not in ds.height.encoding # check coordinates in bounds assert "coordinates" not in ds.lat_bnds.encoding assert "coordinates" not in ds.lon_bnds.encoding
def execute_process(self): myinputs = [] for x in self.input_items: print(x) print(self.input_items[x]) if isinstance(self.input_items[x], QgsMapLayerComboBox): # TODO check input type and export into it (GML, GeoPackage, etc.) cdi = ComplexDataInput( 'http://rain.fsv.cvut.cz/geodata/test.gml') myinputs.append((x, cdi)) else: # TODO check also other types than just QLineEdit if self.input_items[x].text() != 'None': myinputs.append((x, self.input_items[x].text())) self.textEditLog.append( QApplication.translate("WPS", "Executing process ...", None)) self.executeProcess = ExecuteProcess() self.executeProcess.setUrl(self.lineEditWpsUrl.text()) self.executeProcess.setIdentifier(self.comboBoxProcesses.currentText()) self.executeProcess.setInputs(myinputs) self.executeProcess.statusChanged.connect( self.on_execute_process_response) self.executeProcess.start()
def complex_input_with_content(): """ use ComplexDataInput with a direct content """ print("\ncomplex_input_with_content ...") wps = WebProcessingService('http://localhost:8094/wps', verbose=verbose) processid = 'wordcount' textdoc = ComplexDataInput("ALICE was beginning to get very tired ...") # alice in wonderland inputs = [("text", textdoc)] # list of tuple (output identifier, asReference attribute) outputs = [("output",True)] execution = wps.execute(processid, inputs, output=outputs) monitorExecution(execution) # show status print('percent complete', execution.percentCompleted) print('status message', execution.statusMessage) for output in execution.processOutputs: print('identifier=%s, dataType=%s, data=%s, reference=%s' % (output.identifier, output.dataType, output.data, output.reference))
def complex_input_with_reference(): """ use ComplexDataInput with a reference to a document """ print("\ncomplex_input_with_reference ...") wps = WebProcessingService('http://localhost:8094/wps', verbose=verbose) processid = 'wordcount' textdoc = ComplexDataInput("http://www.gutenberg.org/files/28885/28885-h/28885-h.htm") # alice in wonderland inputs = [("text", textdoc)] # list of tuple (output identifier, asReference attribute) outputs = [("output",True)] execution = wps.execute(processid, inputs, output=outputs) monitorExecution(execution) # show status print('percent complete', execution.percentCompleted) print('status message', execution.statusMessage) for output in execution.processOutputs: print('identifier=%s, dataType=%s, data=%s, reference=%s' % (output.identifier, output.dataType, output.data, output.reference))
processid = 'org.n52.wps.python.algorithm.QuakeMLProcess' inputs = [("lonmin", "288"), ("lonmax", "292"), ("latmin", "-70"), ("latmax", "-10"), ("mmin", "6.6"), ("mmax", "8.5"), ("zmin", "5"), ("zmax", "140"), ("p", "0.1"), ("etype", "deaggregation"), ("tlon", "-71.5730623712764"), ("tlat", "-33.1299174879672")] output = "selected-rows" execution = wps.execute(processid, inputs, output) from owslib.wps import monitorExecution monitorExecution(execution) print(execution.processOutputs[0].reference) from owslib.wps import ComplexDataInput quakeMLInput = ComplexDataInput(value=execution.processOutputs[0].reference) quakeMLInput.schema = "http://quakeml.org/xmlns/quakeml/1.2/QuakeML-1.2.xsd" quakeMLInput.mimeType = "text/xml" processid2 = 'org.n52.wps.python.algorithm.ShakemapProcess' inputs2 = [("quakeml-input", quakeMLInput)] output2 = "shakemap-output" execution2 = wps.execute(processid2, inputs2, output2) monitorExecution(execution2) print(execution2.processOutputs[0].reference) import urllib.request contents = urllib.request.urlopen(
def _adapt(self, input_value, input_desc, expecting_reference): """ Try to fit the input_value to the needs of the downstream task input requirements This can involve returning the data or its reference, parsing the reference to return its content or even loading a json structure (or a reference ot it) and returning its content The function will raise an exception if the input_value cannot met the input requirements :param input_value: output data as returned by the upstream task. This object come from the WPSExecution.processOutputs array of (ows.wps.Output) object :param input_desc: input description as declared by the current task :param expecting_reference: indicate if the current task expects the input_value as reference or not :return: The data in the required form The data will be feed to the WebProcessingService.execute function which expect an array of inputs where the input can be : - LiteralData inputs are expressed as simple (key,value) tuples where key is the input identifier, value is the value - ComplexData inputs are expressed as (key, object) tuples, where key is the input identifier, and the object must contain a 'getXml()' method that returns an XML infoset to be included in the WPS request (ows.wps.ComplexDataInput or ows.wps.BoundingBoxDataInput) """ expecting_complex = input_desc.dataType == 'ComplexData' output_data = None output_datatype = [ input_value.dataType, ] if input_value.reference: # If a reference is available and we expect a reference consider it as the data from this point if expecting_reference: output_data = input_value.reference # Append the string datatype since a reference can be considered as a string too output_datatype.append('string') # If we expect the data read the reference else: output_data = self._read_reference(input_value.reference) # process output data are append into a list and # WPS standard v1.0.0 specify that Output data field has zero or one value elif input_value.data: output_data = input_value.data[0] # At this point raise an exception if we don't have data in wps_output.data if not output_data: raise TaskPE._get_exception(self.name, input_value, input_desc) # Consider the validation completed if the dataType match for non-complex data or # if the mimetype match for complex data supported_mimetypes = [ value.mimeType for value in input_desc.supportedValues ] if expecting_complex else [] if input_desc.dataType in output_datatype and \ (not expecting_complex or input_value.mimeType in supported_mimetypes): return [ output_data, ] # Remain cases are either datatypes or complex data mimetypes mismatching... # Before raising an exception we will check for a specific case that we can handle: # json array that could be fed into the downstream wps wanting an array of data. # If this specific case is detected we will simply send the json content to the downstream wps without further # validation since the json content type cannot be verified. take_array = input_desc.maxOccurs > 1 if take_array and 'ComplexData' in output_datatype and input_value.mimeType == 'application/json': # If the json data is referenced and hasn't already been read, read it now if input_value.reference and expecting_reference: output_data = self._read_reference(input_value.reference) json_data = json.loads(output_data) if isinstance(json_data, list): array = [] for value in json_data: # Be a good guy and set the mimeType to something expected... array.append( ComplexDataInput(value, mimeType=input_desc.supportedValues[0] ) if expecting_complex else value) return array # Cannot do anything else raise TaskPE._get_exception(self.name, input_value, input_desc)
def execute(self, workflow_inputs, out_dir, expected_outputs): self.update_status("Preparing execute request for remote WPS1 provider.", REMOTE_JOB_PROGRESS_REQ_PREP, status.STATUS_RUNNING) LOGGER.debug("Execute process WPS request for %s", self.process) try: try: wps = WebProcessingService(url=self.provider, headers=self.cookies, verify=self.verify) raise_on_xml_exception(wps._capabilities) # noqa: W0212 except Exception as ex: raise OWSNoApplicableCode("Failed to retrieve WPS capabilities. Error: [{}].".format(str(ex))) try: process = wps.describeprocess(self.process) except Exception as ex: raise OWSNoApplicableCode("Failed to retrieve WPS process description. Error: [{}].".format(str(ex))) # prepare inputs complex_inputs = [] for process_input in process.dataInputs: if WPS_COMPLEX_DATA in process_input.dataType: complex_inputs.append(process_input.identifier) # remove any 'null' input, should employ the 'default' of the remote WPS process inputs_provided_keys = filter(lambda i: workflow_inputs[i] != "null", workflow_inputs) wps_inputs = [] for input_key in inputs_provided_keys: input_val = workflow_inputs[input_key] # in case of array inputs, must repeat (id,value) # in case of complex input (File), obtain location, otherwise get data value if not isinstance(input_val, list): input_val = [input_val] input_values = [] for val in input_val: if isinstance(val, dict): val = val["location"] # owslib only accepts strings, not numbers directly if isinstance(val, (int, float)): val = str(val) if val.startswith("file://"): # we need to host file starting with file:// scheme val = self.host_file(val) input_values.append(val) # need to use ComplexDataInput structure for complex input # TODO: BoundingBox not supported for input_value in input_values: if input_key in complex_inputs: input_value = ComplexDataInput(input_value) wps_inputs.append((input_key, input_value)) # prepare outputs outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA) for o in process.processOutputs if o.identifier in expected_outputs] self.update_status("Executing job on remote WPS1 provider.", REMOTE_JOB_PROGRESS_EXECUTION, status.STATUS_RUNNING) mode = EXECUTE_MODE_ASYNC execution = wps.execute(self.process, inputs=wps_inputs, output=outputs, mode=mode, lineage=True) if not execution.process and execution.errors: raise execution.errors[0] self.update_status("Monitoring job on remote WPS1 provider : [{0}]".format(self.provider), REMOTE_JOB_PROGRESS_MONITORING, status.STATUS_RUNNING) max_retries = 5 num_retries = 0 run_step = 0 job_id = "<undefined>" while execution.isNotComplete() or run_step == 0: if num_retries >= max_retries: raise Exception("Could not read status document after {} retries. Giving up.".format(max_retries)) try: execution = check_wps_status(location=execution.statusLocation, verify=self.verify, sleep_secs=wait_secs(run_step)) job_id = execution.statusLocation.replace(".xml", "").split("/")[-1] LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()), execution.percentCompleted, execution.statusMessage, execution.statusLocation)) self.update_status(get_job_log_msg(status=status.map_status(execution.getStatus()), message=execution.statusMessage, progress=execution.percentCompleted, duration=None), # get if available map_progress(execution.percentCompleted, REMOTE_JOB_PROGRESS_MONITORING, REMOTE_JOB_PROGRESS_FETCH_OUT), status.STATUS_RUNNING) except Exception as exc: num_retries += 1 LOGGER.debug("Exception raised: %r", exc) sleep(1) else: num_retries = 0 run_step += 1 if not execution.isSucceded(): exec_msg = execution.statusMessage or "Job failed." LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()), execution.percentCompleted, exec_msg, execution.statusLocation)) raise Exception(execution.statusMessage or "Job failed.") self.update_status("Fetching job outputs from remote WPS1 provider.", REMOTE_JOB_PROGRESS_FETCH_OUT, status.STATUS_RUNNING) results = [ows2json_output(output, process) for output in execution.processOutputs] for result in results: result_id = get_any_id(result) result_val = get_any_value(result) if result_id in expected_outputs: # This is where cwl expect the output file to be written # TODO We will probably need to handle multiple output value... dst_fn = "/".join([out_dir.rstrip("/"), expected_outputs[result_id]]) # TODO Should we handle other type than File reference? resp = request_extra("get", result_val, allow_redirects=True, settings=self.settings) LOGGER.debug("Fetching result output from [%s] to cwl output destination: [%s]", result_val, dst_fn) with open(dst_fn, mode="wb") as dst_fh: dst_fh.write(resp.content) except Exception as exc: exception_class = "{}.{}".format(type(exc).__module__, type(exc).__name__) errors = "{0}: {1!s}".format(exception_class, exc) LOGGER.exception(exc) raise Exception(errors) self.update_status("Execution on remote WPS1 provider completed.", REMOTE_JOB_PROGRESS_COMPLETED, status.STATUS_SUCCEEDED)
def wps_execute(self, version, accept): wps_url = get_wps_url(self.settings) if version == "1.0.0": test_content = "Test file in Docker - WPS KVP" wps_method = "GET" elif version == "2.0.0": test_content = "Test file in Docker - WPS XML" wps_method = "POST" else: raise ValueError("Invalid WPS version: {}".format(version)) test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper()) with contextlib.ExitStack() as stack_exec: # setup dir_name = tempfile.gettempdir() tmp_file = stack_exec.enter_context(tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")) tmp_file.write(test_content) tmp_file.seek(0) for mock_exec in mocked_execute_process(): stack_exec.enter_context(mock_exec) # execute if version == "1.0.0": wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)] wps_params = { "service": "WPS", "request": "Execute", "version": version, "identifier": self.process_id, "DataInputs": wps_inputs, } wps_headers = {"Accept": accept} wps_data = None else: wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))] wps_outputs = [(self.out_key, True)] # as reference wps_exec = WPSExecution(version=version, url=wps_url) wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs) wps_data = xml_util.tostring(wps_req) wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML} wps_params = None resp = mocked_sub_requests(self.app, wps_method, wps_url, params=wps_params, data=wps_data, headers=wps_headers, only_local=True) assert resp.status_code in [200, 201], ( "Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text) ) # parse response status if accept == CONTENT_TYPE_APP_XML: assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content xml_body = xml_util.fromstring(str2bytes(resp.text)) status_url = xml_body.get("statusLocation") job_id = status_url.split("/")[-1].split(".")[0] elif accept == CONTENT_TYPE_APP_JSON: assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content status_url = resp.json["location"] job_id = resp.json["jobID"] assert status_url assert job_id if accept == CONTENT_TYPE_APP_XML: wps_out_url = self.settings["weaver.wps_output_url"] weaver_url = self.settings["weaver.url"] assert status_url == f"{wps_out_url}/{job_id}.xml", "Status URL should be XML file for WPS-1 request" # remap to employ JSON monitor method (could be done with XML parsing otherwise) status_url = f"{weaver_url}/jobs/{job_id}" # job monitoring results = self.monitor_job(status_url) outputs = self.get_outputs(status_url) # validate XML status is updated accordingly wps_xml_status = os.path.join(self.settings["weaver.wps_output_dir"], job_id + ".xml") assert os.path.isfile(wps_xml_status) with open(wps_xml_status, "r") as status_file: assert "ProcessSucceeded" in status_file.read() self.validate_outputs(job_id, results, outputs, test_content)
def parse_wps_inputs(wps_process, job): # type: (ProcessOWS, Job) -> List[Tuple[str, OWS_Input_Type]] """ Parses expected WPS process inputs against submitted job input values considering supported process definitions. """ complex_inputs = {} # type: Dict[str, ComplexInput] for process_input in wps_process.dataInputs: if WPS_COMPLEX_DATA in process_input.dataType: complex_inputs[process_input.identifier] = process_input try: wps_inputs = [] # parse both dict and list type inputs job_inputs = job.inputs.items() if isinstance( job.inputs, dict) else job.get("inputs", []) for job_input in job_inputs: if isinstance(job_input, tuple): input_id = job_input[0] input_val = job_input[1] job_input = input_val else: input_id = get_any_id(job_input) input_val = get_any_value(job_input) # in case of array inputs, must repeat (id,value) if isinstance(input_val, list): input_values = input_val input_details = input_val # each value has its own metadata else: input_values = [input_val] input_details = [ job_input ] # metadata directly in definition, not nested per array value # we need to support file:// scheme but PyWPS doesn't like them so remove the scheme file:// input_values = [ # when value is an array of dict that each contain a file reference (get_any_value(val)[7:] if str(get_any_value(val)).startswith( "file://") else get_any_value(val)) if isinstance( val, dict) else # when value is directly a single dict with file reference (val[7:] if str(val).startswith("file://") else val) for val in input_values ] for input_value, input_detail in zip(input_values, input_details): # need to use ComplexDataInput structure for complex input if input_id in complex_inputs: # if provided, pass down specified data input format to allow validation against supported formats ctype = get_field(input_detail, "type", default=None) encoding = None if not ctype: media_format = get_field(input_detail, "format", default=None) if isinstance(media_format, dict): ctype = get_field(input_detail, "mime_type", search_variations=True, default=None) encoding = get_field(input_detail, "encoding", search_variations=True, default=None) wps_inputs.append((input_id, ComplexDataInput(input_value, mimeType=ctype, encoding=encoding))) # need to use literal String for anything else than complex # FIXME: pre-validate allowed literal values? # TODO: BoundingBox not supported else: wps_inputs.append((input_id, str(input_value))) except KeyError: wps_inputs = [] return wps_inputs
def wps_execute(self, version, accept): wps_url = get_wps_url(self.settings) if version == "1.0.0": test_content = "Test file in Docker - WPS KVP" wps_method = "GET" elif version == "2.0.0": test_content = "Test file in Docker - WPS XML" wps_method = "POST" else: raise ValueError("Invalid WPS version: {}".format(version)) test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper()) with contextlib.ExitStack() as stack_exec: # setup dir_name = tempfile.gettempdir() tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt") tmp_file = stack_exec.enter_context(tmp_path) # noqa tmp_file.write(test_content) tmp_file.seek(0) for mock_exec in mocked_execute_process(): stack_exec.enter_context(mock_exec) # execute if version == "1.0.0": wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)] wps_params = { "service": "WPS", "request": "Execute", "version": version, "identifier": self.process_id, "DataInputs": wps_inputs, } wps_headers = {"Accept": accept} wps_data = None else: wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))] wps_outputs = [(self.out_key, True)] # as reference wps_exec = WPSExecution(version=version, url=wps_url) wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs) wps_data = lxml.etree.tostring(wps_req) wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML} wps_params = None resp = mocked_sub_requests(self.app, wps_method, wps_url, params=wps_params, data=wps_data, headers=wps_headers, only_local=True) assert resp.status_code in [200, 201], \ "Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text) # parse response status if accept == CONTENT_TYPE_APP_XML: assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content xml = lxml.etree.fromstring(str2bytes(resp.text)) status_url = xml.get("statusLocation") job_id = status_url.split("/")[-1] elif accept == CONTENT_TYPE_APP_JSON: assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content status_url = resp.json["location"] job_id = resp.json["jobID"] assert status_url assert job_id # job monitoring result = self.monitor_job(status_url) self.validate_outputs(job_id, result, test_content)
def execute_workflow(self, userid, url, service_name, workflow, caption=None): registry = app.conf['PYRAMID_REGISTRY'] db = mongodb(registry) job = add_job(db, userid=userid, task_id=self.request.id, is_workflow=True, service_name=service_name, process_id=workflow['worker']['identifier'], caption=caption) try: # generate and run dispel workflow # TODO: fix owslib wps for unicode/yaml parameters # logger.debug('workflow=%s', workflow) headers = wps_headers(userid) # TODO: handle access token in workflow # workflow['worker']['url'] = build_get_url( # workflow['worker']['url'], # {'access_token': headers.get('Access-Token', '')}) logger.debug('workflow=%s', workflow) inputs = [( 'workflow', ComplexDataInput( # TODO: pywps-4 expects base64 encoding when not set to '' dump_json(workflow), mimeType="text/yaml", encoding=""))] outputs = [('output', True), ('logfile', True)] wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers) # worker_wps = WebProcessingService(url=workflow['worker']['url'], # skip_caps=False, verify=False) execution = wps.execute(identifier='workflow', inputs=inputs, output=outputs, lineage=True) # job['service'] = worker_wps.identification.title # job['title'] = getattr(execution.process, "title") # job['abstract'] = getattr(execution.process, "abstract") job['status_location'] = execution.statusLocation job['response'] = etree.tostring(execution.response) logger.debug("job init done %s ...", self.request.id) run_step = 0 num_retries = 0 while execution.isNotComplete(): if num_retries >= 5: raise Exception( "Could not read status document after 5 retries. Giving up." ) try: execution = check_status(url=execution.statusLocation, verify=False, sleep_secs=wait_secs(run_step)) job['response'] = etree.tostring(execution.response) job['status'] = execution.getStatus() job['status_message'] = execution.statusMessage job['progress'] = execution.percentCompleted duration = datetime.now() - job.get('created', datetime.now()) job['duration'] = str(duration).split('.')[0] if execution.isComplete(): job['finished'] = datetime.now() if execution.isSucceded(): for output in execution.processOutputs: if 'output' == output.identifier: result = yaml.load( urllib.urlopen(output.reference)) job['worker_status_location'] = result[ 'worker']['status_location'] job['progress'] = 100 save_log(job) else: job['status_message'] = '\n'.join( error.text for error in execution.errors) for error in execution.errors: save_log(job, error) else: save_log(job) except Exception: num_retries += 1 logger.exception( "Could not read status xml document for job %s. Trying again ...", self.request.id) sleep(1) else: logger.debug("update job %s ...", self.request.id) num_retries = 0 run_step += 1 db.jobs.update({'identifier': job['identifier']}, job) except Exception as exc: logger.exception("Failed to run Job") job['status'] = "ProcessFailed" job['status_message'] = "Failed to run Job. %s" % exc.message finally: save_log(job) db.jobs.update({'identifier': job['identifier']}, job) registry.notify(JobFinished(job)) return job['status']
# 1. test GetCapabilities query wps = WebProcessingService('https://rain1.fsv.cvut.cz/services/wps', skip_caps=True) wps.getcapabilities() print("Test 1: GetCapabilities -> list of processes:") for process in wps.processes: print(process.identifier) processId = 'd-rain-csv' # 2. test DescribeProcess query #process = wps.describeprocess(processId) #print ("Test 2: DescribeProcess -> list of parameters:") # for input in process.dataInputs: # print (input.identifier) # for output in process.processOutputs: # print (output.identifier) # 3. test Execute query print("Test 3: Execute") inputs = [("input", ComplexDataInput('http://rain.fsv.cvut.cz/geodata/test.gml')), ("keycolumn", "HLGP_ID"), ("return_period", "N2,N5,N10"), ("rainlength", "120")] execution = wps.execute(processId, inputs) outputFile = '/tmp/output.csv' execution.getOutput(outputFile) with open(outputFile) as fd: print(fd.readlines()) o
def execute_process(self, job_id, url, headers=None): from weaver.wps.service import get_pywps_service LOGGER.debug("Job execute process called.") settings = get_settings(app) task_logger = get_task_logger(__name__) load_pywps_config(settings) task_logger.debug("Job task setup.") # reset the connection because we are in a forked celery process db = get_db(app, reset_connection=True) store = db.get_store(StoreJobs) job = store.fetch_by_id(job_id) job.task_id = self.request.id job.progress = JOB_PROGRESS_SETUP job.save_log(logger=task_logger, message="Job task setup completed.") job = store.update_job(job) try: try: job.progress = JOB_PROGRESS_DESCRIBE job.save_log( logger=task_logger, message="Execute WPS request for process [{!s}]".format( job.process)) ssl_verify = get_ssl_verify_option("get", url, settings=settings) wps = WebProcessingService(url=url, headers=get_cookie_headers(headers), verify=ssl_verify) set_wps_language(wps, accept_language=job.accept_language) raise_on_xml_exception(wps._capabilities) # noqa except Exception as ex: raise OWSNoApplicableCode( "Failed to retrieve WPS capabilities. Error: [{}].".format( str(ex))) try: process = wps.describeprocess(job.process) except Exception as ex: raise OWSNoApplicableCode( "Failed to retrieve WPS process description. Error: [{}].". format(str(ex))) # prepare inputs job.progress = JOB_PROGRESS_GET_INPUTS job.save_log(logger=task_logger, message="Fetching job input definitions.") complex_inputs = [] for process_input in process.dataInputs: if WPS_COMPLEX_DATA in process_input.dataType: complex_inputs.append(process_input.identifier) try: wps_inputs = list() for process_input in job.inputs: input_id = get_any_id(process_input) process_value = get_any_value(process_input) # in case of array inputs, must repeat (id,value) input_values = process_value if isinstance( process_value, list) else [process_value] # we need to support file:// scheme but PyWPS doesn't like them so remove the scheme file:// input_values = [ val[7:] if str(val).startswith("file://") else val for val in input_values ] # need to use ComplexDataInput structure for complex input # need to use literal String for anything else than complex # TODO: BoundingBox not supported wps_inputs.extend([ (input_id, ComplexDataInput(input_value) if input_id in complex_inputs else str(input_value)) for input_value in input_values ]) except KeyError: wps_inputs = [] # prepare outputs job.progress = JOB_PROGRESS_GET_OUTPUTS job.save_log(logger=task_logger, message="Fetching job output definitions.") wps_outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA) for o in process.processOutputs] mode = EXECUTE_MODE_ASYNC if job.execute_async else EXECUTE_MODE_SYNC job.progress = JOB_PROGRESS_EXECUTE_REQUEST job.save_log(logger=task_logger, message="Starting job process execution.") job.save_log( logger=task_logger, message= "Following updates could take a while until the Application Package answers..." ) wps_worker = get_pywps_service(environ=settings, is_worker=True) execution = wps_worker.execute_job(job.process, wps_inputs=wps_inputs, wps_outputs=wps_outputs, mode=mode, job_uuid=job.id) if not execution.process and execution.errors: raise execution.errors[0] # adjust status location wps_status_path = get_wps_local_status_location( execution.statusLocation, settings) job.progress = JOB_PROGRESS_EXECUTE_STATUS_LOCATION LOGGER.debug("WPS status location that will be queried: [%s]", wps_status_path) if not wps_status_path.startswith("http") and not os.path.isfile( wps_status_path): LOGGER.warning( "WPS status location not resolved to local path: [%s]", wps_status_path) job.save_log(logger=task_logger, level=logging.DEBUG, message="Updated job status location: [{}].".format( wps_status_path)) job.status = map_status(STATUS_STARTED) job.status_message = execution.statusMessage or "{} initiation done.".format( str(job)) job.status_location = wps_status_path job.request = execution.request job.response = execution.response job.progress = JOB_PROGRESS_EXECUTE_MONITOR_START job.save_log(logger=task_logger, message="Starting monitoring of job execution.") job = store.update_job(job) max_retries = 5 num_retries = 0 run_step = 0 while execution.isNotComplete() or run_step == 0: if num_retries >= max_retries: raise Exception( "Could not read status document after {} retries. Giving up." .format(max_retries)) try: # NOTE: # Don't actually log anything here until process is completed (success or fail) so that underlying # WPS execution logs can be inserted within the current job log and appear continuously. # Only update internal job fields in case they get referenced elsewhere. job.progress = JOB_PROGRESS_EXECUTE_MONITOR_LOOP execution = check_wps_status(location=wps_status_path, settings=settings, sleep_secs=wait_secs(run_step)) job_msg = (execution.statusMessage or "").strip() job.response = execution.response job.status = map_status(execution.getStatus()) job.status_message = "Job execution monitoring (progress: {}%, status: {})."\ .format(execution.percentCompleted, job_msg or "n/a") # job.save_log(logger=task_logger) # job = store.update_job(job) if execution.isComplete(): job.mark_finished() job.progress = JOB_PROGRESS_EXECUTE_MONITOR_END msg_progress = " (status: {})".format( job_msg) if job_msg else "" if execution.isSucceded(): job.status = map_status(STATUS_SUCCEEDED) job.status_message = "Job succeeded{}.".format( msg_progress) wps_package.retrieve_package_job_log(execution, job) job.save_log(logger=task_logger) job_results = [ ows2json_output(output, process, settings) for output in execution.processOutputs ] job.results = make_results_relative( job_results, settings) else: task_logger.debug("Job failed.") job.status_message = "Job failed{}.".format( msg_progress) wps_package.retrieve_package_job_log(execution, job) job.save_log(errors=execution.errors, logger=task_logger) task_logger.debug( "Mapping Job references with generated WPS locations.") map_locations(job, settings) except Exception as exc: num_retries += 1 task_logger.debug("Exception raised: %s", repr(exc)) job.status_message = "Could not read status XML document for {!s}. Trying again...".format( job) job.save_log(errors=execution.errors, logger=task_logger) sleep(1) else: # job.status_message = "Update {}...".format(str(job)) # job.save_log(logger=task_logger) num_retries = 0 run_step += 1 finally: job = store.update_job(job) except Exception as exc: LOGGER.exception("Failed running [%s]", job) job.status = map_status(STATUS_FAILED) job.status_message = "Failed to run {!s}.".format(job) job.progress = JOB_PROGRESS_EXECUTE_MONITOR_ERROR exception_class = "{}.{}".format( type(exc).__module__, type(exc).__name__) errors = "{0}: {1!s}".format(exception_class, exc) job.save_log(errors=errors, logger=task_logger) finally: job.progress = JOB_PROGRESS_EXECUTE_MONITOR_END job.status_message = "Job {}.".format(job.status) job.save_log(logger=task_logger) # Send email if requested if job.notification_email is not None: job.progress = JOB_PROGRESS_NOTIFY try: notify_job_complete(job, job.notification_email, settings) message = "Notification email sent successfully." job.save_log(logger=task_logger, message=message) except Exception as exc: exception_class = "{}.{}".format( type(exc).__module__, type(exc).__name__) exception = "{0}: {1!s}".format(exception_class, exc) message = "Couldn't send notification email ({})".format( exception) job.save_log(errors=message, logger=task_logger, message=message) job.progress = JOB_PROGRESS_DONE job.save_log(logger=task_logger, message="Job task complete.") job = store.update_job(job) return job.status