def test_wps_request3(): # Supply process input arguments polygon = [(-102.8184, 39.5273), (-102.8184, 37.418), (-101.2363, 37.418), (-101.2363, 39.5273), (-102.8184, 39.5273)] featureCollection = GMLMultiPolygonFeatureCollection([polygon]) processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm' inputs = [("FEATURE_ATTRIBUTE_NAME", "the_geom"), ("DATASET_URI", "dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml"), ("DATASET_ID", "ccsm3_a1b_tmax"), ("TIME_START", "1960-01-01T00:00:00.000Z"), ("TIME_END", "1960-12-31T00:00:00.000Z"), ("REQUIRE_FULL_COVERAGE", "true"), ("DELIMITER", "COMMA"), ("STATISTICS", "MEAN"), ("STATISTICS", "MINIMUM"), ("STATISTICS", "MAXIMUM"), ("STATISTICS", "WEIGHT_SUM"), ("STATISTICS", "VARIANCE"), ("STATISTICS", "STD_DEV"), ("STATISTICS", "COUNT"), ("GROUP_BY", "STATISTIC"), ("SUMMARIZE_TIMESTEP", "false"), ("SUMMARIZE_FEATURE_ATTRIBUTE", "false"), ("FEATURE_COLLECTION", featureCollection)] output = "OUTPUT" # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=output) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_USGSExecuteRequest3.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request3(): # Supply process input argument wfsUrl = "http://igsarm-cida-gdp2.er.usgs.gov:8082/geoserver/wfs" query = WFSQuery("sample:CONUS_States", propertyNames=['the_geom', "STATE"], filters=["CONUS_States.508", "CONUS_States.469"]) featureCollection = WFSFeatureCollection(wfsUrl, query) processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm' inputs = [ ("FEATURE_ATTRIBUTE_NAME", "STATE"), ("DATASET_URI", "dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml" ), ("DATASET_ID", "ccsm3_a1b_tmax"), ("DATASET_ID", "ccsm3_a1b_pr"), ("DATASET_ID", "ccsm3_a1fi_tmax"), ("TIME_START", "1960-01-01T00:00:00.000Z"), ("TIME_END", "1960-12-31T00:00:00.000Z"), ("REQUIRE_FULL_COVERAGE", "true"), ("DELIMITER", "COMMA"), ("STATISTICS", "MEAN"), ("STATISTICS", "MINIMUM"), ("STATISTICS", "MAXIMUM"), ("STATISTICS", "WEIGHT_SUM"), ("STATISTICS", "VARIANCE"), ("STATISTICS", "STD_DEV"), ("STATISTICS", "COUNT"), ("GROUP_BY", "STATISTIC"), ("SUMMARIZE_TIMESTEP", "true"), ("SUMMARIZE_FEATURE_ATTRIBUTE", "true"), ("FEATURE_COLLECTION", featureCollection) ] output = "OUTPUT" # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=output) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_USGSExecuteRequest2.xml'), 'rb').read() assert compare_xml(request, _request) is True
def compare_io(self, name, fn, fmt): """Start the dummy process, post the request and check the response matches the input data.""" # Note that `WPSRequest` calls `get_inputs_from_xml` which converts base64 input to bytes # See `_get_rawvalue_value` client = client_for(Service(processes=[create_fmt_process(name, fn, fmt)])) data = get_data(fn, fmt.encoding) wps = WPSExecution() doc = wps.buildRequest('test-fmt', inputs=[('complex', ComplexDataInput(data, mimeType=fmt.mime_type, encoding=fmt.encoding))], mode='sync') resp = client.post_xml(doc=doc) assert_response_success(resp) wps.parseResponse(resp.xml) out = wps.processOutputs[0].data[0] if 'gml' in fmt.mime_type: xml_orig = etree.tostring(etree.fromstring(data.encode('utf-8'))).decode('utf-8') xml_out = etree.tostring(etree.fromstring(out.decode('utf-8'))).decode('utf-8') # Not equal because the output includes additional namespaces compared to the origin. # self.assertEqual(xml_out, xml_orig) else: self.assertEqual(out.strip(), data.strip())
def execute_job(self, process_id, wps_inputs, wps_outputs, mode, job_uuid): """ Real execution of the process by active Celery Worker. """ execution = WPSExecution(version="2.0", url="localhost") xml_request = execution.buildRequest(process_id, wps_inputs, wps_outputs, mode=mode, lineage=True) wps_request = WPSRequest() wps_request.identifier = process_id wps_request.set_version("2.0.0") request_parser = wps_request._post_request_parser( wps_request.WPS.Execute().tag) # noqa: W0212 request_parser(xml_request) # NOTE: # Setting 'status = false' will disable async execution of 'pywps.app.Process.Process' # but this is needed since this job is running within Celery worker already async # (daemon process can't have children processes) # Because if how the code in PyWPS is made, we have to re-enable creation of status file wps_request.status = "false" wps_response = super(WorkerService, self).execute(process_id, wps_request, job_uuid) wps_response.store_status_file = True # update execution status with actual status file and apply required references execution = check_wps_status( location=wps_response.process.status_location, settings=self.settings) execution.request = xml_request return execution
def execute_job( self, job, # type: Job wps_inputs, # type: List[WPS_InputData] wps_outputs, # type: List[WPS_OutputRequested] remote_process, # type: Optional[Process] headers, # type: Optional[AnyHeadersContainer] ): # type: (...) -> WPSExecution """ Real execution of the process by active Celery Worker. """ process_id = job.process execution = WPSExecution(version="2.0", url="localhost") xml_request = execution.buildRequest(process_id, wps_inputs, wps_outputs, mode=job.execution_mode, lineage=True) wps_request = WorkerRequest(http_headers=headers) wps_request.identifier = process_id wps_request.check_and_set_language(job.accept_language) wps_request.set_version("2.0.0") request_parser = wps_request._post_request_parser( wps_request.WPS.Execute().tag) # noqa: W0212 request_parser( xml_request ) # parses the submitted inputs/outputs data and request parameters # NOTE: # Setting 'status = false' will disable async execution of 'pywps.app.Process.Process' # but this is needed since this job is running within Celery worker already async # (daemon process can't have children processes). wps_request.status = "false" # When 'execute' is called, pywps will in turn call 'prepare_process_for_execution', # which then setups and retrieves currently loaded 'local' processes. # Since only local processes were defined by 'get_pywps_service', # a temporary process must be added for remote providers execution. if not remote_process: worker_process_id = process_id else: worker_process_id = f"wps_package-{process_id}-{job.uuid}" self.dispatched_processes[worker_process_id] = remote_process wps_response = super(WorkerService, self).execute(worker_process_id, wps_request, job.uuid) # re-enable creation of status file so we can find it since we disabled 'status' earlier for sync execution wps_response.store_status_file = True # update execution status with actual status file and apply required references execution = check_wps_status( location=wps_response.process.status_location, settings=self.settings) execution.request = xml_request return execution
def test_wps_request9(): # Process input/output arguments processid = "helloworld" inputs = [("user", 'Pingu')] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest9.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request11_bbox(): processid = "bbox" bbox = BoundingBoxDataInput([51.9, 7.0, 53.0, 8.0]) inputs = [("bbox", bbox)] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest11.xml'), 'rb').read() assert compare_xml(request, _request) is True
def run_wps(res_ids,gap): if (gap ==''): gap = "linear" # checks if there is two resource IDs resources = res_ids.split("_") process_id = 'org.n52.wps.server.r.series_gap_filler_3' process_input = [('resource_id',str(resources[0])),('fill_function',str(gap))] #setting the WPS URL is set in app.py url_wps = GapFillerTool.wps_url + '/WebProcessingService' my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True) my_process = my_engine.describeprocess(process_id) #executing the process.. # build execution execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username, password=my_engine.password, verbose=my_engine.verbose) requestElement = execution.buildRequest(process_id, process_input, 'output') request = etree.tostring(requestElement) #set store executeresponse to false request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"') execution = my_engine.execute(process_id, process_input, 'output', request) monitorExecution(execution) status = execution.status # if the status is successful... if status == 'ProcessSucceeded': outputs = execution.processOutputs output0 = outputs[0] reference0 = output0.reference # retrieve the data from the reference output_data = requests.get(reference0) resp = HttpResponse(output_data, content_type="application/json") return resp else: return JsonResponse({'status': 'wps request failed'})
def test_wps_request5(): # Process input/output arguments processid = "reprojectCoords" inputs = [("coords", "http://rsg.pml.ac.uk/wps/testdata/coords.txt"), ("outputSRS", "EPSG:32630"), ("inputSRS", "EPSG:4326")] # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_PMLExecuteRequest5.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request5(): # Process input/output arguments processid = "reprojectCoords" inputs = [("coords", "http://rsg.pml.ac.uk/wps/testdata/coords.txt"), ("outputSRS", "EPSG:32630"), ("inputSRS", "EPSG:4326")] # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_PMLExecuteRequest5.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request8(): # Process input/ouutput arguments processid = "wordcount" textdoc = ComplexDataInput("Alice was beginning to get very tired ...") inputs = [("text", textdoc), ] outputs = [("output", True), ] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=outputs, mode=ASYNC, lineage=True) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest8.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request6(): # Process input/output arguments processid = "v.net.path" inputs = [("input", "http://rsg.pml.ac.uk/wps/example/graph.gml"), ("file", "1 -960123.1421801624 4665723.56559387 -101288.65106088226 5108200.011823481")] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_PMLExecuteRequest6.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request4(): # Process input/ouutput arguments processid = "reprojectImage" inputs = [("inputImage", "http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img"), ("outputSRS", "EPSG:4326")] output = "outputImage" # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=[(output, True)]) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_PMLExecuteRequest4.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request8(): # Process input/ouutput arguments processid = "wordcount" textdoc = ComplexDataInput("Alice was beginning to get very tired ...") inputs = [("text", textdoc), ] outputs = [("output", True), ] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=outputs, async=True, lineage=True) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest8.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request4(): # Process input/ouutput arguments processid = "reprojectImage" inputs = [("inputImage", "http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img"), ("outputSRS", "EPSG:4326")] output = "outputImage" # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=output) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_PMLExecuteRequest4.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request7(): # Process input/ouutput arguments processid = "wordcount" textdoc = ComplexDataInput("http://emu.readthedocs.org/en/latest/index.html") inputs = [("text", textdoc), ] outputs = [("output", True)] # Build XML request for WPS process execution, sync request execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=outputs, mode=SYNC, lineage=False) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_EmuExecuteRequest7.xml'), 'rb').read() print(request) assert compare_xml(request, _request) is True
def run_wps(res_ids): print"launch wps" print res_ids # checks if there is two resource IDs resources = res_ids.split("_") if len(resources) < 2: return JsonResponse({'status': 'wps request failed. 2 resources are required, found ' + str(len(resources))}) process_id = 'org.n52.wps.server.r.linear_regression' process_input = [('x_resource_id',str(resources[0])),('y_resource_id',str(resources[1]))] #setting the WPS URL is set in app.py url_wps = CorrelationPlot.wps_url + '/WebProcessingService' my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True) my_process = my_engine.describeprocess(process_id) #executing the process.. # build execution execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username, password=my_engine.password, verbose=my_engine.verbose) requestElement = execution.buildRequest(process_id, process_input, 'output') request = etree.tostring(requestElement) #set store executeresponse to false request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"') print request execution = my_engine.execute(process_id, process_input, 'output', request) monitorExecution(execution) status = execution.status print status # if the status is successful... if status == 'ProcessSucceeded': outputs = execution.processOutputs output0 = outputs[0] reference0 = output0.reference # retrieve the data from the reference output_data = requests.get(reference0) resp = HttpResponse(output_data, content_type="application/json") return resp else: return JsonResponse({'status': 'wps request failed'})
def test_wps_request6(): # Process input/output arguments processid = "v.net.path" inputs = [ ("input", "http://rsg.pml.ac.uk/wps/example/graph.gml"), ("file", "1 -960123.1421801624 4665723.56559387 -101288.65106088226 5108200.011823481" ) ] # Build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_PMLExecuteRequest6.xml'), 'rb').read() assert compare_xml(request, _request) is True
def test_wps_request3(): # Supply process input argument wfsUrl = "http://igsarm-cida-gdp2.er.usgs.gov:8082/geoserver/wfs" query = WFSQuery("sample:CONUS_States", propertyNames=['the_geom', "STATE"], filters=["CONUS_States.508", "CONUS_States.469"]) featureCollection = WFSFeatureCollection(wfsUrl, query) processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm' inputs = [("FEATURE_ATTRIBUTE_NAME", "STATE"), ("DATASET_URI", "dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml"), ("DATASET_ID", "ccsm3_a1b_tmax"), ("DATASET_ID", "ccsm3_a1b_pr"), ("DATASET_ID", "ccsm3_a1fi_tmax"), ("TIME_START", "1960-01-01T00:00:00.000Z"), ("TIME_END", "1960-12-31T00:00:00.000Z"), ("REQUIRE_FULL_COVERAGE", "true"), ("DELIMITER", "COMMA"), ("STATISTICS", "MEAN"), ("STATISTICS", "MINIMUM"), ("STATISTICS", "MAXIMUM"), ("STATISTICS", "WEIGHT_SUM"), ("STATISTICS", "VARIANCE"), ("STATISTICS", "STD_DEV"), ("STATISTICS", "COUNT"), ("GROUP_BY", "STATISTIC"), ("SUMMARIZE_TIMESTEP", "true"), ("SUMMARIZE_FEATURE_ATTRIBUTE", "true"), ("FEATURE_COLLECTION", featureCollection) ] output = "OUTPUT" # build XML request for WPS process execution execution = WPSExecution() requestElement = execution.buildRequest(processid, inputs, output=output) request = etree.tostring(requestElement) # Compare to cached XML request _request = open(resource_file('wps_USGSExecuteRequest2.xml'), 'rb').read() assert compare_xml(request, _request) is True
def wps_execute(self, version, accept): wps_url = get_wps_url(self.settings) if version == "1.0.0": test_content = "Test file in Docker - WPS KVP" wps_method = "GET" elif version == "2.0.0": test_content = "Test file in Docker - WPS XML" wps_method = "POST" else: raise ValueError("Invalid WPS version: {}".format(version)) test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper()) with contextlib.ExitStack() as stack_exec: # setup dir_name = tempfile.gettempdir() tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt") tmp_file = stack_exec.enter_context(tmp_path) # noqa tmp_file.write(test_content) tmp_file.seek(0) for mock_exec in mocked_execute_process(): stack_exec.enter_context(mock_exec) # execute if version == "1.0.0": wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)] wps_params = { "service": "WPS", "request": "Execute", "version": version, "identifier": self.process_id, "DataInputs": wps_inputs, } wps_headers = {"Accept": accept} wps_data = None else: wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))] wps_outputs = [(self.out_key, True)] # as reference wps_exec = WPSExecution(version=version, url=wps_url) wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs) wps_data = lxml.etree.tostring(wps_req) wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML} wps_params = None resp = mocked_sub_requests(self.app, wps_method, wps_url, params=wps_params, data=wps_data, headers=wps_headers, only_local=True) assert resp.status_code in [200, 201], \ "Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text) # parse response status if accept == CONTENT_TYPE_APP_XML: assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content xml = lxml.etree.fromstring(str2bytes(resp.text)) status_url = xml.get("statusLocation") job_id = status_url.split("/")[-1] elif accept == CONTENT_TYPE_APP_JSON: assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content status_url = resp.json["location"] job_id = resp.json["jobID"] assert status_url assert job_id # job monitoring result = self.monitor_job(status_url) self.validate_outputs(job_id, result, test_content)
def wps_execute(self, version, accept): wps_url = get_wps_url(self.settings) if version == "1.0.0": test_content = "Test file in Docker - WPS KVP" wps_method = "GET" elif version == "2.0.0": test_content = "Test file in Docker - WPS XML" wps_method = "POST" else: raise ValueError("Invalid WPS version: {}".format(version)) test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper()) with contextlib.ExitStack() as stack_exec: # setup dir_name = tempfile.gettempdir() tmp_file = stack_exec.enter_context(tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")) tmp_file.write(test_content) tmp_file.seek(0) for mock_exec in mocked_execute_process(): stack_exec.enter_context(mock_exec) # execute if version == "1.0.0": wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)] wps_params = { "service": "WPS", "request": "Execute", "version": version, "identifier": self.process_id, "DataInputs": wps_inputs, } wps_headers = {"Accept": accept} wps_data = None else: wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))] wps_outputs = [(self.out_key, True)] # as reference wps_exec = WPSExecution(version=version, url=wps_url) wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs) wps_data = xml_util.tostring(wps_req) wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML} wps_params = None resp = mocked_sub_requests(self.app, wps_method, wps_url, params=wps_params, data=wps_data, headers=wps_headers, only_local=True) assert resp.status_code in [200, 201], ( "Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text) ) # parse response status if accept == CONTENT_TYPE_APP_XML: assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content xml_body = xml_util.fromstring(str2bytes(resp.text)) status_url = xml_body.get("statusLocation") job_id = status_url.split("/")[-1].split(".")[0] elif accept == CONTENT_TYPE_APP_JSON: assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content status_url = resp.json["location"] job_id = resp.json["jobID"] assert status_url assert job_id if accept == CONTENT_TYPE_APP_XML: wps_out_url = self.settings["weaver.wps_output_url"] weaver_url = self.settings["weaver.url"] assert status_url == f"{wps_out_url}/{job_id}.xml", "Status URL should be XML file for WPS-1 request" # remap to employ JSON monitor method (could be done with XML parsing otherwise) status_url = f"{weaver_url}/jobs/{job_id}" # job monitoring results = self.monitor_job(status_url) outputs = self.get_outputs(status_url) # validate XML status is updated accordingly wps_xml_status = os.path.join(self.settings["weaver.wps_output_dir"], job_id + ".xml") assert os.path.isfile(wps_xml_status) with open(wps_xml_status, "r") as status_file: assert "ProcessSucceeded" in status_file.read() self.validate_outputs(job_id, results, outputs, test_content)