def check_status(url=None, response=None, sleep_secs=2, verify=False):
"""
Run owslib.wps check_status with additional exception handling.
:param verify: Flag to enable SSL verification. Default: False
:return: OWSLib.wps.WPSExecution object.
"""
execution = WPSExecution()
if response:
logger.debug("using response document ...")
xml = response
elif url:
logger.debug('using status_location url ...')
xml = requests.get(url, verify=verify).content
else:
raise Exception("you need to provide a status-location url or response object.")
if type(xml) is unicode:
xml = xml.encode('utf8', errors='ignore')
execution.checkStatus(response=xml, sleepSecs=sleep_secs)
if execution.response is None:
raise Exception("check_status failed!")
# TODO: workaround for owslib type change of reponse
if not isinstance(execution.response, etree._Element):
execution.response = etree.fromstring(execution.response)
return execution
def check_status(url=None, response=None, sleep_secs=2, verify=False):
"""
Run owslib.wps check_status with additional exception handling.
:param verify: Flag to enable SSL verification. Default: False
:return: OWSLib.wps.WPSExecution object.
"""
execution = WPSExecution()
if response:
LOGGER.debug("using response document ...")
xml = response
elif url:
LOGGER.debug('using status_location url ...')
xml = requests.get(url, verify=verify).content
else:
raise Exception(
"you need to provide a status-location url or response object.")
# TODO: see owslib: https://github.com/geopython/OWSLib/pull/477
execution.checkStatus(response=xml, sleepSecs=sleep_secs)
if execution.response is None:
raise Exception("check_status failed!")
# TODO: workaround for owslib type change of reponse
if not isinstance(execution.response, etree._Element):
execution.response = etree.fromstring(execution.response)
return execution
def execute_job(self, process_id, wps_inputs, wps_outputs, mode, job_uuid):
"""
Real execution of the process by active Celery Worker.
"""
execution = WPSExecution(version="2.0", url="localhost")
xml_request = execution.buildRequest(process_id,
wps_inputs,
wps_outputs,
mode=mode,
lineage=True)
wps_request = WPSRequest()
wps_request.identifier = process_id
wps_request.set_version("2.0.0")
request_parser = wps_request._post_request_parser(
wps_request.WPS.Execute().tag) # noqa: W0212
request_parser(xml_request)
# NOTE:
# Setting 'status = false' will disable async execution of 'pywps.app.Process.Process'
# but this is needed since this job is running within Celery worker already async
# (daemon process can't have children processes)
# Because if how the code in PyWPS is made, we have to re-enable creation of status file
wps_request.status = "false"
wps_response = super(WorkerService,
self).execute(process_id, wps_request, job_uuid)
wps_response.store_status_file = True
# update execution status with actual status file and apply required references
execution = check_wps_status(
location=wps_response.process.status_location,
settings=self.settings)
execution.request = xml_request
return execution
def compare_io(self, name, fn, fmt):
"""Start the dummy process, post the request and check the response matches the input data."""
# Note that `WPSRequest` calls `get_inputs_from_xml` which converts base64 input to bytes
# See `_get_rawvalue_value`
client = client_for(Service(processes=[create_fmt_process(name, fn, fmt)]))
data = get_data(fn, fmt.encoding)
wps = WPSExecution()
doc = wps.buildRequest('test-fmt',
inputs=[('complex', ComplexDataInput(data, mimeType=fmt.mime_type,
encoding=fmt.encoding))],
mode='sync')
resp = client.post_xml(doc=doc)
assert_response_success(resp)
wps.parseResponse(resp.xml)
out = wps.processOutputs[0].data[0]
if 'gml' in fmt.mime_type:
xml_orig = etree.tostring(etree.fromstring(data.encode('utf-8'))).decode('utf-8')
xml_out = etree.tostring(etree.fromstring(out.decode('utf-8'))).decode('utf-8')
# Not equal because the output includes additional namespaces compared to the origin.
# self.assertEqual(xml_out, xml_orig)
else:
self.assertEqual(out.strip(), data.strip())
def update(self):
'''Updates the job in the database from the latest WPS execution status.'''
# create a new execution from the job status URL
execution = WPSExecution()
execution.checkStatus(url=self.statusLocation, sleepSecs=0)
self._update(execution)
def test_wps_request3():
# Supply process input argument
wfsUrl = "http://igsarm-cida-gdp2.er.usgs.gov:8082/geoserver/wfs"
query = WFSQuery("sample:CONUS_States",
propertyNames=['the_geom', "STATE"],
filters=["CONUS_States.508", "CONUS_States.469"])
featureCollection = WFSFeatureCollection(wfsUrl, query)
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm'
inputs = [
("FEATURE_ATTRIBUTE_NAME", "STATE"),
("DATASET_URI",
"dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml"
), ("DATASET_ID", "ccsm3_a1b_tmax"), ("DATASET_ID", "ccsm3_a1b_pr"),
("DATASET_ID", "ccsm3_a1fi_tmax"),
("TIME_START", "1960-01-01T00:00:00.000Z"),
("TIME_END", "1960-12-31T00:00:00.000Z"),
("REQUIRE_FULL_COVERAGE", "true"), ("DELIMITER", "COMMA"),
("STATISTICS", "MEAN"), ("STATISTICS", "MINIMUM"),
("STATISTICS", "MAXIMUM"), ("STATISTICS", "WEIGHT_SUM"),
("STATISTICS", "VARIANCE"), ("STATISTICS", "STD_DEV"),
("STATISTICS", "COUNT"), ("GROUP_BY", "STATISTIC"),
("SUMMARIZE_TIMESTEP", "true"),
("SUMMARIZE_FEATURE_ATTRIBUTE", "true"),
("FEATURE_COLLECTION", featureCollection)
]
output = "OUTPUT"
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=output)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_USGSExecuteRequest2.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_response_with_lineage():
execution = WPSExecution()
xml = open(resource_file('wps_HummingbirdExecuteResponse1.xml'),
'rb').read()
execution.checkStatus(response=xml)
assert execution.isSucceded()
assert execution.creationTime == '2018-05-08T14:00:54Z'
# check lineage input with literal data
inp = execution.dataInputs[0]
assert inp.identifier == 'test'
assert inp.title == 'Select the test you want to run.'
assert inp.abstract == 'CF-1.6=Climate and Forecast Conventions (CF)'
assert inp.data[0] == 'CF-1.6'
# check lineage input with reference
inp = execution.dataInputs[1]
assert inp.identifier == 'dataset'
assert inp.title == 'Upload your NetCDF file here'
assert inp.abstract == 'or enter a URL pointing to a NetCDF file.'
assert inp.reference.startswith('https://www.esrl.noaa.gov/')
# check output with reference
outp = execution.processOutputs[0]
assert outp.identifier == 'output'
assert outp.title == 'Test Report'
assert outp.abstract == 'Compliance checker test report.'
assert outp.reference.startswith('http://localhost:8090/wpsoutputs')
def test_wps_request3():
# Supply process input arguments
polygon = [(-102.8184, 39.5273), (-102.8184, 37.418), (-101.2363, 37.418),
(-101.2363, 39.5273), (-102.8184, 39.5273)]
featureCollection = GMLMultiPolygonFeatureCollection([polygon])
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm'
inputs = [("FEATURE_ATTRIBUTE_NAME", "the_geom"),
("DATASET_URI", "dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml"),
("DATASET_ID", "ccsm3_a1b_tmax"),
("TIME_START", "1960-01-01T00:00:00.000Z"),
("TIME_END", "1960-12-31T00:00:00.000Z"),
("REQUIRE_FULL_COVERAGE", "true"),
("DELIMITER", "COMMA"),
("STATISTICS", "MEAN"),
("STATISTICS", "MINIMUM"),
("STATISTICS", "MAXIMUM"),
("STATISTICS", "WEIGHT_SUM"),
("STATISTICS", "VARIANCE"),
("STATISTICS", "STD_DEV"),
("STATISTICS", "COUNT"),
("GROUP_BY", "STATISTIC"),
("SUMMARIZE_TIMESTEP", "false"),
("SUMMARIZE_FEATURE_ATTRIBUTE", "false"),
("FEATURE_COLLECTION", featureCollection)]
output = "OUTPUT"
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=output)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_USGSExecuteRequest3.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def check_wps_status(
location=None, # type: Optional[str]
response=None, # type: Optional[XML]
sleep_secs=2, # type: int
verify=True, # type: bool
settings=None, # type: Optional[AnySettingsContainer]
): # type: (...) -> WPSExecution
"""
Run :func:`owslib.wps.WPSExecution.checkStatus` with additional exception handling.
:param location: job URL or file path where to look for job status.
:param response: WPS response document of job status.
:param sleep_secs: number of seconds to sleep before returning control to the caller.
:param verify: Flag to enable SSL verification.
:param settings: Application settings to retrieve any additional request parameters as applicable.
:return: OWSLib.wps.WPSExecution object.
"""
def _retry_file():
LOGGER.warning(
"Failed retrieving WPS status-location, attempting with local file."
)
out_path = get_wps_local_status_location(location, settings)
if not out_path:
raise HTTPNotFound(
"Could not find file resource from [{}].".format(location))
LOGGER.info("Resolved WPS status-location using local file reference.")
return open(out_path, "r").read()
execution = WPSExecution()
if response:
LOGGER.debug("Retrieving WPS status from XML response document...")
xml = response
elif location:
xml_resp = HTTPNotFound()
try:
LOGGER.debug("Attempt to retrieve WPS status-location from URL...")
xml_resp = request_extra("get",
location,
verify=verify,
settings=settings)
xml = xml_resp.content
except Exception as ex:
LOGGER.debug("Got exception during get status: [%r]", ex)
xml = _retry_file()
if xml_resp.status_code == HTTPNotFound.code:
LOGGER.debug("Got not-found during get status: [%r]", xml)
xml = _retry_file()
else:
raise Exception(
"Missing status-location URL/file reference or response with XML object."
)
if isinstance(xml, str):
xml = xml.encode("utf8", errors="ignore")
execution.checkStatus(response=xml, sleepSecs=sleep_secs)
if execution.response is None:
raise Exception("Missing response, cannot check status.")
if not isinstance(execution.response, lxml.etree._Element): # noqa
execution.response = lxml.etree.fromstring(execution.response)
return execution
def collect_outputs(status_location):
from owslib.wps import WPSExecution
execution = WPSExecution()
execution.checkStatus(url=status_location, sleepSecs=0)
outputs = {}
for output in execution.processOutputs:
outputs[output.identifier] = output
return outputs
def execute_job(
self,
job, # type: Job
wps_inputs, # type: List[WPS_InputData]
wps_outputs, # type: List[WPS_OutputRequested]
remote_process, # type: Optional[Process]
headers, # type: Optional[AnyHeadersContainer]
): # type: (...) -> WPSExecution
"""
Real execution of the process by active Celery Worker.
"""
process_id = job.process
execution = WPSExecution(version="2.0", url="localhost")
xml_request = execution.buildRequest(process_id,
wps_inputs,
wps_outputs,
mode=job.execution_mode,
lineage=True)
wps_request = WorkerRequest(http_headers=headers)
wps_request.identifier = process_id
wps_request.check_and_set_language(job.accept_language)
wps_request.set_version("2.0.0")
request_parser = wps_request._post_request_parser(
wps_request.WPS.Execute().tag) # noqa: W0212
request_parser(
xml_request
) # parses the submitted inputs/outputs data and request parameters
# NOTE:
# Setting 'status = false' will disable async execution of 'pywps.app.Process.Process'
# but this is needed since this job is running within Celery worker already async
# (daemon process can't have children processes).
wps_request.status = "false"
# When 'execute' is called, pywps will in turn call 'prepare_process_for_execution',
# which then setups and retrieves currently loaded 'local' processes.
# Since only local processes were defined by 'get_pywps_service',
# a temporary process must be added for remote providers execution.
if not remote_process:
worker_process_id = process_id
else:
worker_process_id = f"wps_package-{process_id}-{job.uuid}"
self.dispatched_processes[worker_process_id] = remote_process
wps_response = super(WorkerService,
self).execute(worker_process_id, wps_request,
job.uuid)
# re-enable creation of status file so we can find it since we disabled 'status' earlier for sync execution
wps_response.store_status_file = True
# update execution status with actual status file and apply required references
execution = check_wps_status(
location=wps_response.process.status_location,
settings=self.settings)
execution.request = xml_request
return execution
def getOWSLibExecutionResponse(cls, instance):
_e = None
if instance.execution.response:
_e = WPSExecution()
try:
_e.checkStatus(url=instance.url,
response=str(instance.execution.response),
sleepSecs=0)
except:
pass
return _e
def test_wps_request9():
# Process input/output arguments
processid = "helloworld"
inputs = [("user", 'Pingu')]
# Build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_EmuExecuteRequest9.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request11_bbox():
processid = "bbox"
bbox = BoundingBoxDataInput([51.9, 7.0, 53.0, 8.0])
inputs = [("bbox", bbox)]
# Build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_EmuExecuteRequest11.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request5():
# Process input/output arguments
processid = "reprojectCoords"
inputs = [("coords", "http://rsg.pml.ac.uk/wps/testdata/coords.txt"),
("outputSRS", "EPSG:32630"), ("inputSRS", "EPSG:4326")]
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_PMLExecuteRequest5.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def run_wps(res_ids,gap):
if (gap ==''):
gap = "linear"
# checks if there is two resource IDs
resources = res_ids.split("_")
process_id = 'org.n52.wps.server.r.series_gap_filler_3'
process_input = [('resource_id',str(resources[0])),('fill_function',str(gap))]
#setting the WPS URL is set in app.py
url_wps = GapFillerTool.wps_url + '/WebProcessingService'
my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True)
my_process = my_engine.describeprocess(process_id)
#executing the process..
# build execution
execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username,
password=my_engine.password, verbose=my_engine.verbose)
requestElement = execution.buildRequest(process_id, process_input, 'output')
request = etree.tostring(requestElement)
#set store executeresponse to false
request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"')
execution = my_engine.execute(process_id, process_input, 'output', request)
monitorExecution(execution)
status = execution.status
# if the status is successful...
if status == 'ProcessSucceeded':
outputs = execution.processOutputs
output0 = outputs[0]
reference0 = output0.reference
# retrieve the data from the reference
output_data = requests.get(reference0)
resp = HttpResponse(output_data, content_type="application/json")
return resp
else:
return JsonResponse({'status': 'wps request failed'})
def test_wps_request5():
# Process input/output arguments
processid = "reprojectCoords"
inputs = [("coords", "http://rsg.pml.ac.uk/wps/testdata/coords.txt"),
("outputSRS", "EPSG:32630"),
("inputSRS", "EPSG:4326")]
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_PMLExecuteRequest5.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request4():
# Process input/ouutput arguments
processid = "reprojectImage"
inputs = [("inputImage", "http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img"),
("outputSRS", "EPSG:4326")]
output = "outputImage"
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=[(output, True)])
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_PMLExecuteRequest4.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request4():
# Process input/ouutput arguments
processid = "reprojectImage"
inputs = [("inputImage", "http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img"),
("outputSRS", "EPSG:4326")]
output = "outputImage"
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=output)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_PMLExecuteRequest4.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request8():
# Process input/ouutput arguments
processid = "wordcount"
textdoc = ComplexDataInput("Alice was beginning to get very tired ...")
inputs = [("text", textdoc), ]
outputs = [("output", True), ]
# Build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=outputs, mode=ASYNC, lineage=True)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_EmuExecuteRequest8.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request6():
# Process input/output arguments
processid = "v.net.path"
inputs = [("input", "http://rsg.pml.ac.uk/wps/example/graph.gml"),
("file", "1 -960123.1421801624 4665723.56559387 -101288.65106088226 5108200.011823481")]
# Build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_PMLExecuteRequest6.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_request8():
# Process input/ouutput arguments
processid = "wordcount"
textdoc = ComplexDataInput("Alice was beginning to get very tired ...")
inputs = [("text", textdoc), ]
outputs = [("output", True), ]
# Build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=outputs, async=True, lineage=True)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_EmuExecuteRequest8.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def test_wps_ncml():
client = client_for(Service(processes=[NcMLAgg()], cfgfiles=CFG_FILE))
resp = client.get(service='wps',
request='execute',
version='1.0.0',
identifier='ncml')
assert_response_success(resp)
ex = WPSExecution()
ex.parseResponse(resp.xml)
d1, d2, d3 = ex.processOutputs
ncml = d3.retrieveData()
assert ncml.strip().startswith("<")
def test_wps_request7():
# Process input/ouutput arguments
processid = "wordcount"
textdoc = ComplexDataInput("http://emu.readthedocs.org/en/latest/index.html")
inputs = [("text", textdoc), ]
outputs = [("output", True)]
# Build XML request for WPS process execution, sync request
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=outputs, mode=SYNC, lineage=False)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_EmuExecuteRequest7.xml'), 'rb').read()
print(request)
assert compare_xml(request, _request) is True
def execute_process(client,
identifier,
inputs,
output_names=("output_netcdf", )) -> xr.Dataset:
"""Execute a process using the test client, and return the 'output_netcdf' output as an xarray.Dataset"""
request_doc = WPS.Execute(OWS.Identifier(identifier),
WPS.DataInputs(*inputs),
version="1.0.0")
response = client.post_xml(doc=request_doc)
assert_response_success(response)
execution = WPSExecution()
execution.parseResponse(response.xml)
outputs = get_file_outputs(execution, output_names=output_names)
return outputs
def run_wps(res_ids):
print"launch wps"
print res_ids
# checks if there is two resource IDs
resources = res_ids.split("_")
if len(resources) < 2:
return JsonResponse({'status': 'wps request failed. 2 resources are required, found ' + str(len(resources))})
process_id = 'org.n52.wps.server.r.linear_regression'
process_input = [('x_resource_id',str(resources[0])),('y_resource_id',str(resources[1]))]
#setting the WPS URL is set in app.py
url_wps = CorrelationPlot.wps_url + '/WebProcessingService'
my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True)
my_process = my_engine.describeprocess(process_id)
#executing the process..
# build execution
execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username,
password=my_engine.password, verbose=my_engine.verbose)
requestElement = execution.buildRequest(process_id, process_input, 'output')
request = etree.tostring(requestElement)
#set store executeresponse to false
request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"')
print request
execution = my_engine.execute(process_id, process_input, 'output', request)
monitorExecution(execution)
status = execution.status
print status
# if the status is successful...
if status == 'ProcessSucceeded':
outputs = execution.processOutputs
output0 = outputs[0]
reference0 = output0.reference
# retrieve the data from the reference
output_data = requests.get(reference0)
resp = HttpResponse(output_data, content_type="application/json")
return resp
else:
return JsonResponse({'status': 'wps request failed'})
def checkStatus(cls, instance):
if not instance.is_initialized():
exception = Exception("Not initialized!")
log.exception("Not initialized!")
raise exception
if instance.execution and isinstance(
instance.execution,
WebProcessingServiceExecution) and instance.execution.response:
_e = WPSExecution(version=instance.version,
url=instance.url,
username=instance.username,
password=instance.password,
verbose=False)
_e.request = instance.execution.request
_e.statusLocation = instance.execution.status_location
_e.serviceInstance = instance.service_instance
try:
_e.checkStatus(sleepSecs=3)
except:
log.exception(
"Exception while Checking WPS Execution Status {}".format(
_e))
if _e.status:
cls._update_instance_execution_status(instance, _e)
else:
raise Exception("Status check failed")
return instance.execution.status
else:
exception = Exception("No running executions!")
log.exception("No running executions!")
raise exception
def execute(identifier, inputs=[], wps_host=None, wps_client=None,
version='1.0.0'):
if wps_host:
wps = WebProcessingService(wps_host, version)
return wps.execute(identifier, inputs=inputs)
else:
y = ''
for data_input in inputs:
y += '{0}={1};'.format(data_input[0], data_input[1])
y = y[:-1]
response = wps_client.get(
('?service=WPS&request=execute&version={0}&'
'identifier={1}&DataInputs={2}').format(version, identifier, y))
wps_reader = WPSReader()
element = wps_reader.readFromString(response.get_data())
execution = WPSExecution()
execution._parseExecuteResponse(element)
return execution
def test_wps_request6():
# Process input/output arguments
processid = "v.net.path"
inputs = [
("input", "http://rsg.pml.ac.uk/wps/example/graph.gml"),
("file",
"1 -960123.1421801624 4665723.56559387 -101288.65106088226 5108200.011823481"
)
]
# Build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_PMLExecuteRequest6.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def checkStatus(self):
"""
Sends a request to the status URL checking the progress of the remote
process. If the process has finished creates the necessary Output
objects to fetch the results and stores them in the resultsLiteral and
resultsComplex attributes.
:returns: True if the process finished successfully
False if the process is still running
:rtype: Boolean
"""
self.processError = ""
self.processErrorCode = ""
self.processErrorText = ""
self.status = None
if (self.statusURL == None):
self.logger.error(self.ERR_05)
raise Exception(self.ERR_05)
try:
self.execution = WPSExecution()
self.execution.statusLocation = self.statusURL
self.execution.checkStatus(sleepSecs=0)
except Exception as ex:
mesg = "Unexpected error from OWSLib!! " + str(ex)
self.logger.error(mesg)
# This is a work arround !!!!!!!!
# It is not clear why these calls to OWSLib are failing.
# Thus RUNNING status is reported back
self.status = self.RUNNING
return False
# Check if the process has finished
if not (self.execution.isComplete()):
self.status = self.RUNNING
self.logger.debug("The process hasn't finished yet.")
self.logger.info(str(self.percentCompleted) + " % of the execution complete.")
return False
# Check if the process failed
if not (self.execution.isSucceded()):
self.status = self.ERROR
self.processError = self.execution.errors[0]
self.processErrorText = self.execution.errors[0].text
self.logger.error(self.ERR_06 + self.processErrorText)
self.logger.debug("The status URL: " + self.execution.statusLocation)
raise Exception(self.ERR_06 + self.processErrorText)
self.logger.debug(self.SUCC_01)
self.status = self.FINISHED
return True
def execute(identifier,
inputs=[],
wps_host=None,
wps_client=None,
version='1.0.0'):
if wps_host:
wps = WebProcessingService(wps_host, version)
return wps.execute(identifier, inputs=inputs)
else:
y = ''
for data_input in inputs:
y += '{0}={1};'.format(data_input[0], data_input[1])
y = y[:-1]
response = wps_client.get(
('?service=WPS&request=execute&version={0}&'
'identifier={1}&DataInputs={2}').format(version, identifier, y))
wps_reader = WPSReader()
element = wps_reader.readFromString(response.get_data())
execution = WPSExecution()
execution._parseExecuteResponse(element)
return execution
def execute_process(client, identifier, inputs,
output_names=("output", )) -> xr.Dataset:
"""Execute a process using the test client, and return the 'output_netcdf' output as an xarray.Dataset"""
request_doc = WPS.Execute(OWS.Identifier(identifier),
WPS.DataInputs(*inputs),
version="1.0.0")
response = client.post_xml(doc=request_doc)
try:
assert_response_success(response)
except AssertionError as e:
exception = response.xpath("//ows:Exception")[0]
exception_code = exception.get("exceptionCode")
message = exception[0].text
output_message = f"{exception_code}: {message}"
raise ProcessError(output_message) from e
execution = WPSExecution()
execution.parseResponse(response.xml)
outputs = get_outputs(execution, output_names=output_names)
return outputs
def test_wps_request3():
# Supply process input argument
wfsUrl = "http://igsarm-cida-gdp2.er.usgs.gov:8082/geoserver/wfs"
query = WFSQuery("sample:CONUS_States",
propertyNames=['the_geom', "STATE"],
filters=["CONUS_States.508", "CONUS_States.469"])
featureCollection = WFSFeatureCollection(wfsUrl, query)
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm'
inputs = [("FEATURE_ATTRIBUTE_NAME", "STATE"),
("DATASET_URI", "dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml"),
("DATASET_ID", "ccsm3_a1b_tmax"),
("DATASET_ID", "ccsm3_a1b_pr"),
("DATASET_ID", "ccsm3_a1fi_tmax"),
("TIME_START", "1960-01-01T00:00:00.000Z"),
("TIME_END", "1960-12-31T00:00:00.000Z"),
("REQUIRE_FULL_COVERAGE", "true"),
("DELIMITER", "COMMA"),
("STATISTICS", "MEAN"),
("STATISTICS", "MINIMUM"),
("STATISTICS", "MAXIMUM"),
("STATISTICS", "WEIGHT_SUM"),
("STATISTICS", "VARIANCE"),
("STATISTICS", "STD_DEV"),
("STATISTICS", "COUNT"),
("GROUP_BY", "STATISTIC"),
("SUMMARIZE_TIMESTEP", "true"),
("SUMMARIZE_FEATURE_ATTRIBUTE", "true"),
("FEATURE_COLLECTION", featureCollection)
]
output = "OUTPUT"
# build XML request for WPS process execution
execution = WPSExecution()
requestElement = execution.buildRequest(processid, inputs, output=output)
request = etree.tostring(requestElement)
# Compare to cached XML request
_request = open(resource_file('wps_USGSExecuteRequest2.xml'), 'rb').read()
assert compare_xml(request, _request) is True
def execute(identifier,
inputs=[],
wps_host=None,
wps_client=None,
version='1.0.0'):
"""WPS execute response.
Parameters
----------
identifer : string
inputs : list of tuples
wps_host : string
wps_client : pywps.tests.WpsClient
version : string
Returns
-------
out : list of ?
"""
if wps_host:
wps = WebProcessingService(wps_host, version)
return wps.execute(identifier, inputs=inputs)
else:
y = ''
for data_input in inputs:
y += '{0}={1};'.format(data_input[0], data_input[1])
y = y[:-1]
response = wps_client.get(
('?service=WPS&request=execute&version={0}&'
'identifier={1}&DataInputs={2}').format(version, identifier, y))
wps_reader = WPSReader()
element = wps_reader.readFromString(response.get_data())
execution = WPSExecution()
execution._parseExecuteResponse(element)
return execution
def check_status(url=None, response=None, sleep_secs=2, verify=False):
"""
Run owslib.wps check_status with additional exception handling.
:param verify: Flag to enable SSL verification. Default: False
:return: OWSLib.wps.WPSExecution object.
"""
execution = WPSExecution()
if response:
LOGGER.debug("using response document ...")
xml = response
elif url:
LOGGER.debug('using status_location url ...')
xml = requests.get(url, verify=verify).content
else:
raise Exception("you need to provide a status-location url or response object.")
# TODO: see owslib: https://github.com/geopython/OWSLib/pull/477
execution.checkStatus(response=xml, sleepSecs=sleep_secs)
if execution.response is None:
raise Exception("check_status failed!")
# TODO: workaround for owslib type change of reponse
if not isinstance(execution.response, etree._Element):
execution.response = etree.fromstring(execution.response)
return execution
def test_wps_response_with_lineage():
execution = WPSExecution()
xml = open(resource_file('wps_HummingbirdExecuteResponse1.xml'), 'rb').read()
execution.checkStatus(response=xml)
assert execution.isSucceded()
assert execution.creationTime == '2018-05-08T14:00:54Z'
# check lineage input with literal data
inp = execution.dataInputs[0]
assert inp.identifier == 'test'
assert inp.title == 'Select the test you want to run.'
assert inp.abstract == 'CF-1.6=Climate and Forecast Conventions (CF)'
assert inp.data[0] == 'CF-1.6'
# check lineage input with reference
inp = execution.dataInputs[1]
assert inp.identifier == 'dataset'
assert inp.title == 'Upload your NetCDF file here'
assert inp.abstract == 'or enter a URL pointing to a NetCDF file.'
assert inp.reference.startswith('https://www.esrl.noaa.gov/')
# check output with reference
outp = execution.processOutputs[0]
assert outp.identifier == 'output'
assert outp.title == 'Test Report'
assert outp.abstract == 'Compliance checker test report.'
assert outp.reference.startswith('http://localhost:8090/wpsoutputs')
class WPSClient:
"""
This class manages the WPS request, status checking and results
processing cycle.
.. attribute:: logger
Reference to logging object
.. attribute:: logFormat
String formating log output
.. attribute:: processName
Name of the process to invoke
.. attribute:: inputs
Array of pair lists composed by the input name and the respective value
.. attribute:: outputs
Array of pair lists composed by the output name and a boolean indicating return
by reference (True) or data (False)
.. attribute:: wps
WPSExecution object used to communicate with the WPS server
.. attribute:: execution
WebProcessingService object used to invoke process execution on the remote WPS server
.. attribute:: statusURL
URL of the remote XML file where the process updates its status and
writes its results after completion
.. attribute:: processId
Identifier of the remote process, part of statusURL used to identify
the results stored locally
.. attribute:: percentCompleted
Percentage of process completion, as reported in the status XML file
.. attribute:: statusMessage
Last status message returned during asynchronous execution
.. attribute:: map
Object of type MapFile used to generate the map file publishing complex
outputs through MapServer
.. attribute:: epsg
EPSG code of the primary coordinate system used to publish complex
outputs
.. attribute:: dataSets
Array with output dataSets, created during map file generation
.. attribute:: logFile
Path to the log file
.. attribute:: logLevel
Level of logging
.. attribute:: pathFilesGML
Path where to store the GML files with complex outputs
.. attribute:: mapServerURL
URL of the MapServer instance to use
.. attribute:: mapFilesPath
Path where to write the map file (folder used by MapServer)
.. attribute:: mapTemplate
Path to the MapServer map template folder
.. attribute:: imagePath
Path to the MapServer image folder
.. attribute:: imageURL
URL to MapServer images folder
.. attribute:: otherProjs
String listing EPSG codes of further coordinate systems with which to
publish the complex outputs
"""
logger = None
logFormat = "[WPSClient][%(asctime)s] %(levelname)s: %(message)s"
processName = None
inputs = None
outputs = None
wps = None
execution = None
statusURL = None
processId = None
percentCompleted = 0
statusMessage = None
map = None
epsg = None
dataSets = []
#Configs
logFile = None
logLevel = None
pathFilesGML = None
mapServerURL = None
mapFilesPath = None
mapTemplate = None
imagePath = None
imageURL = None
otherProjs = None
meta_fees = "none"
meta_accessconstraints = "none"
meta_keywordlist = ""
meta_addresstype = ""
meta_address = ""
meta_city = ""
meta_stateorprovince = ""
meta_postcode = ""
meta_country = ""
meta_contactelectronicmailaddress = ""
meta_contactperson = ""
meta_contactorganization = ""
meta_contactposition = ""
meta_role = ""
meta_contactvoicetelephone = ""
meta_contactfacsimiletelephone = ""
meta_contactinstructions = ""
meta_hoursofservice = ""
RUNNING = 1
FINISHED = 2
ERROR = 3
#Messages
WARN_02 = "Output "
WARN_03 = " not added to the map file, possibly non complex output."
WARN_04 = "No spatial layers found, no map file was written."
ERR_04 = "EXECUTE request failed:\n"
ERR_05 = "Incomplete request -- missing URL"
ERR_06 = "The process failed with the following message:\n"
ERR_07 = "Failed to save map file to disk:\n"
ERR_08 = "Cannot generate a map file with the outputs specified."
SUCC_01 = "The process has finished successfully.\nProcessing the results..."
SUCC_02 = "Wrote map file to disk:\n"
def __init__(self, logger = None):
self.loadConfigs()
if (logger == None):
self.setupLogging()
def init(self, serverAddress, processName, inputs, outputs):
"""
Initialises the WPSClient object with the required arguments to create
the WPS request.
:param serverAddress: string with the address of the remote WPS server
:param processName: string with process name
:param inputNames: list of strings with input names
:param inputValues: list of strings with input values
:param outputNames: list of strings with output names
"""
# Loading this stuff here probably doesn't make sense
# Check at a later data if __init__ is run from an external model (that includes this one)
self.loadConfigs()
self.setupLogging()
self.processName = processName
self.inputs = inputs
self.outputs = outputs
self.wps = WebProcessingService(serverAddress, verbose=False, skip_caps=True)
def initFromURL(self, url, outputs):
"""
Initialises the WPSClient with the status URL address of a running
remote process. Used when a request has already been sent.
:param url: string with the status URL address of a remote process
"""
# Loading this stuff here probably doesn't make sense
# Check at a later data if __init__ is run from an external model (tha includes this one)
self.loadConfigs()
self.setupLogging()
# Note that the outputs are not used
self.statusURL = url
self.outputs = outputs
self.processId = self.decodeId(url)
def loadConfigs(self):
"""
Loads default attribute values from the configuration file.
"""
parser = SafeConfigParser()
currDir = os.path.dirname(os.path.realpath(__file__))
parentDir = os.path.abspath(os.path.join(currDir, os.pardir))
parser.read(os.path.join(parentDir, 'WPSClient.cfg'))
self.logFile = parser.get('Logging', 'logFile')
self.logLevel = parser.get('Logging', 'logLevel')
self.pathFilesGML = parser.get('Data', 'GMLfilesPath')
self.mapServerURL = parser.get('MapServer', 'MapServerURL')
self.mapFilesPath = parser.get('MapServer', 'mapFilesPath')
self.mapTemplate = parser.get('MapServer', 'mapTemplate')
self.imagePath = parser.get('MapServer', 'imagePath')
self.imageURL = parser.get('MapServer', 'imageURL')
self.otherProjs = parser.get('MapServer', 'otherProjs')
if parser.has_option('MapServer', 'meta_fees'):
self.meta_fees = parser.get('MapServer', 'meta_fees')
if parser.has_option('MapServer', 'meta_accessconstraints'):
self.meta_accessconstraints = parser.get('MapServer', 'meta_accessconstraints')
if parser.has_option('MapServer', 'meta_keywordlist'):
self.meta_keywordlist = parser.get('MapServer', 'meta_keywordlist')
if parser.has_option('MapServer', 'meta_addresstype'):
self.meta_addresstype = parser.get('MapServer', 'meta_addresstype')
if parser.has_option('MapServer', 'meta_address'):
self.meta_address = parser.get('MapServer', 'meta_address')
if parser.has_option('MapServer', 'meta_city'):
self.meta_city = parser.get('MapServer', 'meta_city')
if parser.has_option('MapServer', 'meta_stateorprovince'):
self.meta_stateorprovince = parser.get('MapServer', 'meta_stateorprovince')
if parser.has_option('MapServer', 'meta_postcode'):
self.meta_postcode = parser.get('MapServer', 'meta_postcode')
if parser.has_option('MapServer', 'meta_country'):
self.meta_country = parser.get('MapServer', 'meta_country')
if parser.has_option('MapServer', 'meta_contactelectronicmailaddress'):
self.meta_contactelectronicmailaddress = parser.get('MapServer', 'meta_contactelectronicmailaddress')
if parser.has_option('MapServer', 'meta_contactperson'):
self.meta_contactperson = parser.get('MapServer', 'meta_contactperson')
if parser.has_option('MapServer', 'meta_contactorganization'):
self.meta_contactorganization = parser.get('MapServer', 'meta_contactorganization')
if parser.has_option('MapServer', 'meta_contactposition'):
self.meta_contactposition = parser.get('MapServer', 'meta_contactposition')
if parser.has_option('MapServer', 'meta_role'):
self.meta_role = parser.get('MapServer', 'meta_role')
if parser.has_option('MapServer', 'meta_contactvoicetelephone'):
self.meta_contactvoicetelephone = parser.get('MapServer', 'meta_contactvoicetelephone')
if parser.has_option('MapServer', 'meta_contactfacsimiletelephone'):
self.meta_contactfacsimiletelephone = parser.get('MapServer', 'meta_contactfacsimiletelephone')
if parser.has_option('MapServer', 'meta_contactinstructions'):
self.meta_contactinstructions = parser.get('MapServer', 'meta_contactinstructions')
if parser.has_option('MapServer', 'meta_hoursofservice'):
self.meta_hoursofservice = parser.get('MapServer', 'meta_hoursofservice')
def setupLogging(self):
"""
Sets up the logging file.
"""
formatter = logging.Formatter(self.logFormat)
self.logger = logging.getLogger(__name__)
self.logger.setLevel(self.logLevel)
if(self.logFile == None):
ch_stream = logging.StreamHandler()
#ch_stream.setLevel(logging.INFO)
ch_stream.setFormatter(formatter)
self.logger.addHandler(ch_stream)
else:
ch_file = logging.FileHandler(self.logFile, 'a')
#ch_file.setLevel(self.logLevel)
ch_file.setFormatter(formatter)
self.logger.addHandler(ch_file)
def decodeId(self, url):
"""
Decodes the remote process identifier from the status URL.
:param: string with the status URL address of a remote process
:returns: string with the process identifier
"""
s = url.split("/")
return s[len(s) - 1].split(".")[0]
def getPercentCompleted(self):
"""
:returns: process execution progress in percentage
"""
if (self.execution is not None):
return self.execution.percentCompleted
else:
return None
def getStatusMessage(self):
"""
:returns: last status message returned by the server
"""
if (self.execution is not None):
return self.execution.statusMessage
else:
return None
def getProcessErrorCode(self):
"""
:returns: last error code returned by the server
"""
if (self.processError is not None):
return self.processError.code
else:
return None
def getProcessErrorText(self):
"""
:returns: last error message returned by the server
"""
if (self.processError is not None):
return self.processError.text
else:
return None
def sendRequest(self):
"""
Uses the wps object to start the process execution. Stores the
status URL and the process in the statusURL and processId attributes.
:returns: string with the status URL, None in case of error
"""
execOutputs = []
for key in self.outputs:
execOutputs.append((key, "True"))
self.execution = self.wps.execute(self.processName, self.inputs, execOutputs)
self.logger.info("The request sent: \n" + self.execution.request)
self.logger.debug("The status URL: " + self.execution.statusLocation)
if len(self.execution.errors) > 0:
self.logger.error(self.ERR_04 + self.execution.errors[0].code + self.execution.errors[0].text)
raise Exception(self.ERR_04 + self.execution.errors[0].code + self.execution.errors[0].text)
return None
self.statusURL = self.execution.statusLocation
self.processId = self.decodeId(self.statusURL)
return self.statusURL
def checkStatus(self):
"""
Sends a request to the status URL checking the progress of the remote
process. If the process has finished creates the necessary Output
objects to fetch the results and stores them in the resultsLiteral and
resultsComplex attributes.
:returns: True if the process finished successfully
False if the process is still running
:rtype: Boolean
"""
self.processError = ""
self.processErrorCode = ""
self.processErrorText = ""
self.status = None
if (self.statusURL == None):
self.logger.error(self.ERR_05)
raise Exception(self.ERR_05)
try:
self.execution = WPSExecution()
self.execution.statusLocation = self.statusURL
self.execution.checkStatus(sleepSecs=0)
except Exception as ex:
mesg = "Unexpected error from OWSLib!! " + str(ex)
self.logger.error(mesg)
# This is a work arround !!!!!!!!
# It is not clear why these calls to OWSLib are failing.
# Thus RUNNING status is reported back
self.status = self.RUNNING
return False
# Check if the process has finished
if not (self.execution.isComplete()):
self.status = self.RUNNING
self.logger.debug("The process hasn't finished yet.")
self.logger.info(str(self.percentCompleted) + " % of the execution complete.")
return False
# Check if the process failed
if not (self.execution.isSucceded()):
self.status = self.ERROR
self.processError = self.execution.errors[0]
self.processErrorText = self.execution.errors[0].text
self.logger.error(self.ERR_06 + self.processErrorText)
self.logger.debug("The status URL: " + self.execution.statusLocation)
raise Exception(self.ERR_06 + self.processErrorText)
self.logger.debug(self.SUCC_01)
self.status = self.FINISHED
return True
def generateMapFile(self):
"""
Creates the MapFile object that encodes a map file publishing the
complex outputs and writes it to disk.
:returns: string with the path to the map file generated. None if no
map file was generated (no complex outputs present).
"""
if(self.outputs is None) or (len(self.outputs) != len(self.execution.processOutputs)):
self.logger.error(self.ERR_08)
raise Exception(self.ERR_08)
#self.map = UMN.MapFile(self.processId)
self.map = MapFile(self.processId)
self.map.shapePath = self.pathFilesGML
self.map.epsgCode = self.epsg
self.map.mapTemplate = self.mapTemplate
self.map.imagePath = self.imagePath
self.map.imageURL = self.imageURL
self.map.mapServerURL = self.mapServerURL
self.map.mapFilesPath = self.mapFilesPath
self.map.otherProjs = self.otherProjs
self.map.meta_fees = self.meta_fees
self.map.meta_accessconstraints = self.meta_accessconstraints
self.map.meta_keywordlist = self.meta_keywordlist
self.map.meta_addresstype = self.meta_addresstype
self.map.meta_address = self.meta_address
self.map.meta_city = self.meta_city
self.map.meta_stateorprovince = self.meta_stateorprovince
self.map.meta_postcode = self.meta_postcode
self.map.meta_country = self.meta_country
self.map.meta_contactelectronicmailaddress = self.meta_contactelectronicmailaddress
self.map.meta_contactperson = self.meta_contactperson
self.map.meta_contactorganization = self.meta_contactorganization
self.map.meta_contactposition = self.meta_contactposition
self.map.meta_role = self.meta_role
self.map.meta_contactvoicetelephone = self.meta_contactvoicetelephone
self.map.meta_contactfacsimiletelephone = self.meta_contactfacsimiletelephone
self.map.meta_contactinstructions = self.meta_contactinstructions
self.map.meta_hoursofservice = self.meta_hoursofservice
for output in self.execution.processOutputs:
output.writeToDisk(self.pathFilesGML);
providedTitle = self.outputs[output.identifier]
dataSet = DataSet(output.filePath, providedTitle, output.identifier)
self.dataSets.append(dataSet)
layerEPSG = dataSet.getEPSG()
if (layerEPSG == None):
layerEPSG = self.map.epsgCode
if dataSet.dataType == dataSet.TYPE_VECTOR:
#* style = UMN.MapStyle()
style = MapStyle()
#* layer = UMN.VectorLayer(
layer = VectorLayer(
output.filePath,
dataSet.getBBox(),
layerEPSG,
output.identifier,
providedTitle)
type = str(dataSet.getGeometryType())
if type <> None:
layer.layerType = type
else:
layer.layerType = "Polygon"
self.logger.debug("The layer type: " + str(dataSet.getGeometryType()))
layer.addStyle(style)
self.map.addLayer(layer)
self.logger.debug("Generated layer " + layer.name + " of type " + layer.layerType + ".")
elif dataSet.dataType == dataSet.TYPE_RASTER:
#layer = UMN.RasterLayer(
layer = RasterLayer(
output.filePath,
dataSet.getBBox(),
layerEPSG,
output.identifier,
providedTitle)
layer.setBounds(dataSet.getMaxValue(), dataSet.getMinValue())
self.map.addLayer(layer)
self.logger.debug("Generated layer " + layer.name + " of type raster.")
else:
self.logger.warning(self.WARN_02 + output.identifier + self.WARN_03)
self.logger.debug("Guessed mime type for this layer: " + str(dataSet.getMimeType()))
print "The pixel res: " + str(dataSet.getPixelRes())
if (len(self.map.layers) > 0):
try :
self.map.writeToDisk()
except Exception, e:
self.logger.error(self.ERR_07 + str(e))
raise Exception(self.ERR_07 + str(e))
return
self.logger.info(self.SUCC_02 + self.map.filePath())
return self.map.filePath()
else:
def wps_execute(self, version, accept):
wps_url = get_wps_url(self.settings)
if version == "1.0.0":
test_content = "Test file in Docker - WPS KVP"
wps_method = "GET"
elif version == "2.0.0":
test_content = "Test file in Docker - WPS XML"
wps_method = "POST"
else:
raise ValueError("Invalid WPS version: {}".format(version))
test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper())
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_path = tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt")
tmp_file = stack_exec.enter_context(tmp_path) # noqa
tmp_file.write(test_content)
tmp_file.seek(0)
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
if version == "1.0.0":
wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)]
wps_params = {
"service": "WPS",
"request": "Execute",
"version": version,
"identifier": self.process_id,
"DataInputs": wps_inputs,
}
wps_headers = {"Accept": accept}
wps_data = None
else:
wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))]
wps_outputs = [(self.out_key, True)] # as reference
wps_exec = WPSExecution(version=version, url=wps_url)
wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
wps_data = lxml.etree.tostring(wps_req)
wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
wps_params = None
resp = mocked_sub_requests(self.app, wps_method, wps_url,
params=wps_params, data=wps_data, headers=wps_headers, only_local=True)
assert resp.status_code in [200, 201], \
"Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text)
# parse response status
if accept == CONTENT_TYPE_APP_XML:
assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
xml = lxml.etree.fromstring(str2bytes(resp.text))
status_url = xml.get("statusLocation")
job_id = status_url.split("/")[-1]
elif accept == CONTENT_TYPE_APP_JSON:
assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url
assert job_id
# job monitoring
result = self.monitor_job(status_url)
self.validate_outputs(job_id, result, test_content)
def wps_execute(self, version, accept):
wps_url = get_wps_url(self.settings)
if version == "1.0.0":
test_content = "Test file in Docker - WPS KVP"
wps_method = "GET"
elif version == "2.0.0":
test_content = "Test file in Docker - WPS XML"
wps_method = "POST"
else:
raise ValueError("Invalid WPS version: {}".format(version))
test_content += " {} request - Accept {}".format(wps_method, accept.split("/")[-1].upper())
with contextlib.ExitStack() as stack_exec:
# setup
dir_name = tempfile.gettempdir()
tmp_file = stack_exec.enter_context(tempfile.NamedTemporaryFile(dir=dir_name, mode="w", suffix=".txt"))
tmp_file.write(test_content)
tmp_file.seek(0)
for mock_exec in mocked_execute_process():
stack_exec.enter_context(mock_exec)
# execute
if version == "1.0.0":
wps_inputs = ["file={}@mimeType={}".format(tmp_file.name, CONTENT_TYPE_TEXT_PLAIN)]
wps_params = {
"service": "WPS",
"request": "Execute",
"version": version,
"identifier": self.process_id,
"DataInputs": wps_inputs,
}
wps_headers = {"Accept": accept}
wps_data = None
else:
wps_inputs = [("file", ComplexDataInput(tmp_file.name, mimeType=CONTENT_TYPE_TEXT_PLAIN))]
wps_outputs = [(self.out_key, True)] # as reference
wps_exec = WPSExecution(version=version, url=wps_url)
wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
wps_data = xml_util.tostring(wps_req)
wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
wps_params = None
resp = mocked_sub_requests(self.app, wps_method, wps_url,
params=wps_params, data=wps_data, headers=wps_headers, only_local=True)
assert resp.status_code in [200, 201], (
"Failed with: [{}]\nTest: [{}]\nReason:\n{}".format(resp.status_code, test_content, resp.text)
)
# parse response status
if accept == CONTENT_TYPE_APP_XML:
assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
xml_body = xml_util.fromstring(str2bytes(resp.text))
status_url = xml_body.get("statusLocation")
job_id = status_url.split("/")[-1].split(".")[0]
elif accept == CONTENT_TYPE_APP_JSON:
assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content
status_url = resp.json["location"]
job_id = resp.json["jobID"]
assert status_url
assert job_id
if accept == CONTENT_TYPE_APP_XML:
wps_out_url = self.settings["weaver.wps_output_url"]
weaver_url = self.settings["weaver.url"]
assert status_url == f"{wps_out_url}/{job_id}.xml", "Status URL should be XML file for WPS-1 request"
# remap to employ JSON monitor method (could be done with XML parsing otherwise)
status_url = f"{weaver_url}/jobs/{job_id}"
# job monitoring
results = self.monitor_job(status_url)
outputs = self.get_outputs(status_url)
# validate XML status is updated accordingly
wps_xml_status = os.path.join(self.settings["weaver.wps_output_dir"], job_id + ".xml")
assert os.path.isfile(wps_xml_status)
with open(wps_xml_status, "r") as status_file:
assert "ProcessSucceeded" in status_file.read()
self.validate_outputs(job_id, results, outputs, test_content)
def test_wps_checkStatus():
execution = WPSExecution()
xml = open(resource_file('wps_PMLExecuteResponse5.xml'), 'rb').read()
execution.checkStatus(response=xml)
assert execution.isSucceded()
assert execution.creationTime == '2011-11-08T21:36:55Z'
def check_wps_status(
location=None, # type: Optional[str]
response=None, # type: Optional[xml_util.XML]
sleep_secs=2, # type: int
verify=True, # type: bool
settings=None, # type: Optional[AnySettingsContainer]
): # type: (...) -> WPSExecution
"""
Run :func:`owslib.wps.WPSExecution.checkStatus` with additional exception handling.
:param location: job URL or file path where to look for job status.
:param response: WPS response document of job status.
:param sleep_secs: number of seconds to sleep before returning control to the caller.
:param verify: flag to enable SSL verification.
:param settings: application settings to retrieve any additional request parameters as applicable.
:returns: OWSLib.wps.WPSExecution object.
"""
def _retry_file():
# type: () -> str
LOGGER.warning(
"Failed retrieving WPS status-location, attempting with local file."
)
out_path = get_wps_local_status_location(location, settings)
if not out_path:
raise HTTPNotFound(
f"Could not find file resource from [{location}].")
LOGGER.info("Resolved WPS status-location using local file reference.")
with open(out_path, mode="r", encoding="utf-8") as f:
return f.read()
execution = WPSExecution()
if response:
LOGGER.debug("Retrieving WPS status from XML response document...")
xml_data = response
elif location:
xml_resp = HTTPNotFound()
xml_data = None
try:
LOGGER.debug(
"Attempt to retrieve WPS status-location from URL [%s]...",
location)
xml_resp = request_extra("get",
location,
verify=verify,
settings=settings)
xml_data = xml_resp.content
except Exception as ex:
LOGGER.debug(
"Got exception during get status: [%r]. Will retry with local reference.",
ex)
if xml_resp.status_code != HTTPOk.code:
LOGGER.debug(
"WPS XML status not found: [%r]. Retrying with local reference.",
xml_data)
xml_data = _retry_file()
else:
raise Exception(
"Missing status-location URL/file reference or response with XML object."
)
if isinstance(xml_data, str):
xml_data = xml_data.encode("utf8", errors="ignore")
execution.checkStatus(response=xml_data, sleepSecs=sleep_secs)
if execution.response is None:
raise Exception("Missing response, cannot check status.")
if not isinstance(execution.response, xml_util.XML):
execution.response = xml_util.fromstring(execution.response)
return execution