def test_wps_describeprocess_emu_all():
# Initialize WPS client
wps = WebProcessingService('http://localhost:8094/wps', skip_caps=True)
# Execute fake invocation of DescribeProcess operation by parsing cached response from
xml = open(resource_file('wps_EmuDescribeProcess_all.xml'), 'rb').read()
process = wps.describeprocess('nap', xml=xml)
processes = wps.describeprocess('all', xml=xml)
assert isinstance(process, Process)
assert isinstance(processes, list)
def test_describe():
"""Check that owslib can parse the processes' description."""
from owslib.wps import WebProcessingService
wps = WebProcessingService(URL, skip_caps=True)
client = client_for(Service(processes=processes))
resp = client.get(service="wps",
request="getcapabilities",
version="1.0.0")
wps.describeprocess("all", xml=resp.data)
def run_wps(process_id,input,output):
#choose the first wps engine
my_engine = WebProcessingService('http://appsdev.hydroshare.org:8282/wps/WebProcessingService', verbose=False, skip_caps=True)
my_engine.getcapabilities()
my_process = my_engine.describeprocess(process_id)
my_inputs = my_process.dataInputs
input_names = [] #getting list of input
for input1 in my_inputs:
input_names.append(input1)
#executing the process..
execution = my_engine.execute(process_id, input, output)
request = execution.request
#set store executeresponse to false
request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"')
url_wps = 'http://appsdev.hydroshare.org:8282/wps/WebProcessingService'
wps_request = urllib2.Request(url_wps,request)
wps_open = urllib2.urlopen(wps_request)
wps_read = wps_open.read()
if 'href' in wps_read:
tag = 'href="'
location = wps_read.find(tag)
new= wps_read[location+len(tag):len(wps_read)]
tag2 = '"/>\n </wps:Output>\n </wps:ProcessOutputs>\n</wps:'
location2 = new.find(tag2)
final = new[0:location2]
split = final.split()
wps_request1 = urllib2.Request(split[0])
wps_open1 = urllib2.urlopen(wps_request1)
wps_read1 = wps_open1.read()
#return [final_output_url, final_data]
return [wps_read1, split]
def create_from_process(cls,
url,
identifier,
request_template,
username=None,
password=None,
output_hook=None):
try:
wps = WebProcessingService(url,
username=username,
password=password,
verbose=False,
skip_caps=False)
process = wps.describeprocess(identifier)
if process:
_f = open(request_template)
_run = cls.objects.create(identifier=identifier,
title=process.title,
abstract=process.abstract,
version=wps.version,
url=url,
service_instance=wps.url,
username=username,
password=password,
request_template=File(_f),
output_hook=output_hook)
_run._initialized = True
_run._wps = wps
return _run
except:
log.exception("Could not create Process!")
raise
return wps.id
def test_wps_describeprocess_ceda():
# Initialize WPS client
wps = WebProcessingService('http://ceda-wps2.badc.rl.ac.uk/wps', skip_caps=True)
# Execute fake invocation of DescribeProcess operation by parsing cached response from CEDA service
xml = open(resource_file('wps_CEDADescribeProcess.xml'), 'rb').read()
process = wps.describeprocess('Doubleit', xml=xml)
# Check process description
assert process.identifier == 'DoubleIt'
assert process.title == 'Doubles the input number and returns value'
assert process.abstract == 'This is test process used to demonstrate how the WPS and the WPS User Interface work. The process accepts an integer or floating point number and returns some XML containing the input number double.' # NOQA
# Check process properties
assert process.statusSupported is False
assert process.storeSupported is True
# Check process inputs
# Example Input:
# identifier=NumberToDouble, title=NumberToDouble, abstract=NumberToDouble, data type=LiteralData
# Any value allowed
# Default Value: None
# minOccurs=1, maxOccurs=-1
for input in process.dataInputs:
assert input.identifier == 'NumberToDouble'
assert input.dataType == 'LiteralData'
# Example Output:
# identifier=OutputXML, title=OutputXML, abstract=OutputXML, data type=ComplexData
# Supported Value: mimeType=text/XML, encoding=UTF-8, schema=NONE
# Default Value: None
# reference=None, mimeType=None
# Check process outputs
for output in process.processOutputs:
assert output.identifier == 'OutputXML'
assert output.dataType == 'ComplexData'
def test_wps_describeprocess_bbox():
# Initialize WPS client
wps = WebProcessingService('http://localhost:8094/wps', skip_caps=True)
# Execute fake invocation of DescribeProcess operation by parsing cached response from Emu service
xml = open(resource_file('wps_bbox_DescribeProcess.xml'), 'rb').read()
process = wps.describeprocess('bbox', xml=xml)
# Check process description
assert process.identifier == 'bbox'
assert process.title == 'Bounding Box'
# Check process inputs
# Example Input:
# identifier=bbox, title=Bounding Box, abstract=None, data type=BoundingBoxData
# Supported Value: EPSG:4326
# Supported Value: EPSG:3035
# Default Value: EPSG:4326
# minOccurs=1, maxOccurs=1
for input in process.dataInputs:
assert input.identifier == 'bbox'
assert input.dataType == 'BoundingBoxData'
# Example Output:
# identifier=bbox, title=Bounding Box, abstract=None, data type=BoundingBoxData
# Supported Value: EPSG:4326
# Default Value: EPSG:4326
# reference=None, mimeType=None
# Check process outputs
for output in process.processOutputs:
assert output.identifier == 'bbox'
assert output.dataType == 'BoundingBoxData'
def test_wps_describeprocess_ceda():
# Initialize WPS client
wps = WebProcessingService('http://ceda-wps2.badc.rl.ac.uk/wps',
skip_caps=True)
# Execute fake invocation of DescribeProcess operation by parsing cached response from CEDA service
xml = open(resource_file('wps_CEDADescribeProcess.xml'), 'rb').read()
process = wps.describeprocess('DoubleIt', xml=xml)
# Check process description
assert process.identifier == 'DoubleIt'
assert process.title == 'Doubles the input number and returns value'
assert process.abstract == 'This is test process used to demonstrate how the WPS and the WPS User Interface work. The process accepts an integer or floating point number and returns some XML containing the input number double.' # NOQA
# Check process properties
assert process.statusSupported is False
assert process.storeSupported is True
# Check process inputs
# Example Input:
# identifier=NumberToDouble, title=NumberToDouble, abstract=NumberToDouble, data type=LiteralData
# Any value allowed
# Default Value: None
# minOccurs=1, maxOccurs=-1
for input in process.dataInputs:
assert input.identifier == 'NumberToDouble'
assert input.dataType == 'LiteralData'
# Example Output:
# identifier=OutputXML, title=OutputXML, abstract=OutputXML, data type=ComplexData
# Supported Value: mimeType=text/XML, encoding=UTF-8, schema=NONE
# Default Value: None
# reference=None, mimeType=None
# Check process outputs
for output in process.processOutputs:
assert output.identifier == 'OutputXML'
assert output.dataType == 'ComplexData'
class ExecuteProcess(Processes):
def __init__(self, request):
self.wps_id = request.params.get("wps")
self.wps = WebProcessingService(url=wps_url(request, self.wps_id))
identifier = request.params.get("process")
# TODO: need to fix owslib to handle special identifiers
self.process = self.wps.describeprocess(identifier)
super(ExecuteProcess, self).__init__(request, name="processes_execute", title="")
def breadcrumbs(self):
breadcrumbs = super(ExecuteProcess, self).breadcrumbs()
route_path = self.request.route_path("processes_list", _query=[("wps", self.wps_id)])
breadcrumbs.append(dict(route_path=route_path, title=self.wps.identification.title))
breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.process.title))
return breadcrumbs
def appstruct(self):
return {}
def generate_form(self, formid="deform"):
from phoenix.schema.wps import WPSSchema
schema = WPSSchema(process=self.process, user=self.get_user())
return Form(schema, buttons=("submit",), formid=formid)
def process_form(self, form):
controls = self.request.POST.items()
try:
appstruct = form.validate(controls)
self.execute(appstruct)
except ValidationFailure, e:
logger.exception("validation of exectue view failed.")
self.session.flash("There are errors on this page.", queue="danger")
return dict(description=getattr(self.process, "abstract", ""), form=e.render())
return HTTPFound(location=self.request.route_url("monitor"))
class ComplexInputs(Wizard):
def __init__(self, request):
super(ComplexInputs, self).__init__(
request, name='wizard_complex_inputs',
title="Choose Input Parameter")
from owslib.wps import WebProcessingService
self.wps = WebProcessingService(wps_url(request, self.wizard_state.get('wizard_wps')['identifier']), verify=False)
self.process = self.wps.describeprocess(self.wizard_state.get('wizard_process')['identifier'])
self.title = "Choose Input Parameter of {0}".format(self.process.title)
def breadcrumbs(self):
breadcrumbs = super(ComplexInputs, self).breadcrumbs()
breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title))
return breadcrumbs
def schema(self):
return Schema().bind(process=self.process)
def success(self, appstruct):
for input in self.process.dataInputs:
if input.identifier == appstruct.get('identifier'):
appstruct['mime_types'] = [value.mimeType for value in input.supportedValues]
super(ComplexInputs, self).success(appstruct)
def next_success(self, appstruct):
self.success(appstruct)
return self.next('wizard_source')
@view_config(route_name='wizard_complex_inputs', renderer='../templates/wizard/default.pt')
def view(self):
return super(ComplexInputs, self).view()
def pinned_processes(request):
from owslib.wps import WebProcessingService
settings = request.db.settings.find_one() or {}
processes = []
if 'pinned_processes' in settings:
for pinned in settings.get('pinned_processes'):
try:
service_name, identifier = pinned.split('.', 1)
url = request.route_path('processes_execute',
_query=[('wps', service_name),
('process', identifier)])
wps = WebProcessingService(url=request.route_url(
'owsproxy', service_name=service_name),
verify=False)
# TODO: need to fix owslib to handle special identifiers
process = wps.describeprocess(identifier)
description = headline(process.abstract)
except Exception:
LOGGER.warn("could not add pinned process %s", pinned)
else:
processes.append(
dict(title=process.identifier,
description=description,
url=url,
service_title=wps.identification.title))
return processes
def call_wps(args):
if not args.token:
user_id = args.user
if not user_id:
logging.error("No user id found on the call, aborting!")
sys.exit(1)
user_token_file = os.path.join("/etc/d4science/", user_id)
with open(user_token_file, "r") as f:
gcube_vre_token = f.read()
else:
gcube_vre_token = args.token.encode("utf-8")
logging.info("User: %s", args.user)
logging.info("Token: (SHA256) %s", hashlib.sha256(gcube_vre_token).hexdigest())
gcube_vre_token_header = {"gcube-token": gcube_vre_token}
dataminer_url = (
"http://dataminer-prototypes.d4science.org/wps/" "WebProcessingService"
)
wps = WebProcessingService(dataminer_url, headers=gcube_vre_token_header)
process_id = args.process
process = wps.describeprocess(process_id)
inputs = build_inputs(process, args.input, args.inputdata, gcube_vre_token)
outputs = [(o.identifier, True) for o in process.processOutputs]
# execute the process
execution = wps.execute(process_id, inputs, outputs)
monitorExecution(execution, sleepSecs=5, download=True)
logging.info("Execution status: %s", execution.status)
exit_code = 0 if execution.status == "ProcessSucceeded" else 1
logging.info("Exit code: %d", exit_code)
produce_output(execution, args.output, args.outdir, gcube_vre_token_header)
return exit_code
def describe_process(identifier,
wps_host=None,
wps_client=None,
version='1.0.0'):
"""WPS Describe Process response.
Parameters
----------
identifer : string
wps_host : string
wps_client : pywps.tests.WpsClient
version : string
Returns
-------
out : list of ?
"""
if wps_host:
wps = WebProcessingService(wps_host, version)
return wps.describeprocess(identifier)
else:
response = wps_client.get(
('?service=WPS&request=DescribeProcess&version={0}&'
'identifier={1}').format(version, identifier))
wps_reader = WPSReader()
element = wps_reader.readFromString(response.get_data())
wps = WebProcessingService(None, version, skip_caps=True)
return wps._parseProcessMetadata(element)
def run_wps(process_id,input,output):
#choose the first wps engine
#my_engine = WebProcessingService('http://appsdev.hydroshare.org:8282/wps/WebProcessingService', verbose=False, skip_caps=True)
my_engine = WebProcessingService('http://appsdev.hydroshare.org:8282/wps/WebProcessingService',verbose=False, skip_caps=True)
my_engine.getcapabilities()
#wps_engines = list_wps_service_engines()
#my_engine = wps_engines[0]
#choose the r.time-series-converter
my_process = my_engine.describeprocess(process_id)
my_inputs = my_process.dataInputs
input_names = [] #getting list of input
for input1 in my_inputs:
input_names.append(input1)
#executing the process..
execution = my_engine.execute(process_id, input, output)
request = execution.request
#set store executeresponse to false
request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"')
url_wps = 'http://appsdev.hydroshare.org:8282/wps/WebProcessingService'
wps_request = urllib2.Request(url_wps,request)
wps_open = urllib2.urlopen(wps_request)
wps_read = wps_open.read()
if 'href' in wps_read:
tag = 'href="'
location = wps_read.find(tag)
new= wps_read[location+len(tag):len(wps_read)]
tag2 = '"/>\n </wps:Output>\n </wps:ProcessOutputs>\n</wps:'
location2 = new.find(tag2)
final = new[0:location2]
split = final.split()
wps_request1 = urllib2.Request(split[0])
wps_open1 = urllib2.urlopen(wps_request1)
wps_read1 = wps_open1.read()
#now we must use our own method to send the request1
#we need to use the request
#this code is for the normal wps which is not working right now
# monitorExecution(execution)
# output_data = execution.processOutputs
# final_output_url = output_data[0].reference
# final_data = read_final_data(final_output_url)
#return [final_output_url, final_data]
return [wps_read1, split]
class pyGDPwebProcessing():
"""
This class allows interactive calls to be made into the GDP.
"""
def __init__(self, wfs_url=WFS_URL):
# if WFS_URL is None:
# from pygdp.namespaces import WFS_URL
# wfsUrl = WFS_URL
self.wfsUrl = wfs_url
self.wpsUrl = WPS_URL
self.version = '1.1.0'
self.wps = WebProcessingService(self.wpsUrl)
def WPSgetCapabilities(self, xml=None):
"""
Returns a list of capabilities.
"""
self.wps.getcapabilities(xml)
def WPSdescribeprocess(self, identifier, xml=None):
"""
Returns a list describing a specific identifier/process.
"""
self.wps.describeprocess(identifier, xml)
# pyGDP Submit Feature
def dodsReplace(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return _execute_request.dodsReplace(dataSetURI, verbose)
def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom',
value=None, gmlIDs=None, verbose=False, coverage=True, outputfname=None,
sleepSecs=10, async=False):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCoverageOPenDAP(geoType, dataSetURI, varID, startTime,
endTime, attribute, value, gmlIDs, verbose,
coverage, self.wfsUrl, outputfname, sleepSecs, async=async)
def describe_process(identifier, wps_host=None, wps_client=None,
version='1.0.0'):
if wps_host:
wps = WebProcessingService(wps_host, version)
return wps.describeprocess(identifier)
else:
response = wps_client.get(
('?service=WPS&request=DescribeProcess&version={0}&'
'identifier={1}').format(version, identifier))
wps_reader = WPSReader()
element = wps_reader.readFromString(response.get_data())
wps = WebProcessingService(None, version, skip_caps=True)
return wps._parseProcessMetadata(element)
def run(self):
responseToReturn = Response()
if self.identifier != "":
try:
wps = WebProcessingService(self.url)
process = wps.describeprocess(self.identifier)
responseToReturn.status = 200
responseToReturn.data = process
except:
responseToReturn.status = 500
else:
responseToReturn.status = 500
self.statusChanged.emit(responseToReturn)
def run_wps(res_ids,gap):
if (gap ==''):
gap = "linear"
# checks if there is two resource IDs
resources = res_ids.split("_")
process_id = 'org.n52.wps.server.r.series_gap_filler_3'
process_input = [('resource_id',str(resources[0])),('fill_function',str(gap))]
#setting the WPS URL is set in app.py
url_wps = GapFillerTool.wps_url + '/WebProcessingService'
my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True)
my_process = my_engine.describeprocess(process_id)
#executing the process..
# build execution
execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username,
password=my_engine.password, verbose=my_engine.verbose)
requestElement = execution.buildRequest(process_id, process_input, 'output')
request = etree.tostring(requestElement)
#set store executeresponse to false
request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"')
execution = my_engine.execute(process_id, process_input, 'output', request)
monitorExecution(execution)
status = execution.status
# if the status is successful...
if status == 'ProcessSucceeded':
outputs = execution.processOutputs
output0 = outputs[0]
reference0 = output0.reference
# retrieve the data from the reference
output_data = requests.get(reference0)
resp = HttpResponse(output_data, content_type="application/json")
return resp
else:
return JsonResponse({'status': 'wps request failed'})
class ChooseSource(Wizard):
def __init__(self, request):
super(ChooseSource, self).__init__(request,
name='wizard_source',
title="Choose Data Source")
self.wps = WebProcessingService(url=request.route_url(
'owsproxy',
service_name=self.wizard_state.get('wizard_wps')['identifier']),
verify=False,
skip_caps=True)
self.process = self.wps.describeprocess(
self.wizard_state.get('wizard_process')['identifier'])
for data_input in self.process.dataInputs:
if data_input.identifier == self.wizard_state.get(
'wizard_complex_inputs')['identifier']:
self.title = "Choose Data Source for %s" % data_input.title
break
def breadcrumbs(self):
breadcrumbs = super(ChooseSource, self).breadcrumbs()
breadcrumbs.append(
dict(route_path=self.request.route_path(self.name),
title=self.title))
return breadcrumbs
def schema(self):
return Schema().bind(request=self.request)
def next_success(self, appstruct):
self.success(appstruct)
# TODO: that is a dirty way to init esgf search
if appstruct.get('source') == 'wizard_esgf_search':
defaults = dict(constraints=process_constraints(self.process))
query = query_params_from_appstruct(
self.wizard_state.get('wizard_esgf_search'), defaults)
if not self.request.cert_ok:
msg = """<strong>Error:</strong> You are not allowed to access ESGF data.
Please <a href="{}" class="alert-link">update</a> your ESGF credentials."""
callback = self.request.current_route_path()
self.session.flash(msg.format(
self.request.route_path('esgflogon',
_query=[('callback', callback)])),
queue='danger')
return self.next(self.name)
else:
query = None
return self.next(appstruct.get('source'), query=query)
def view(self):
return super(ChooseSource, self).view()
def describe_process(identifier,
wps_host=None,
wps_client=None,
version='1.0.0'):
if wps_host:
wps = WebProcessingService(wps_host, version)
return wps.describeprocess(identifier)
else:
response = wps_client.get(
('?service=WPS&request=DescribeProcess&version={0}&'
'identifier={1}').format(version, identifier))
wps_reader = WPSReader()
element = wps_reader.readFromString(response.get_data())
wps = WebProcessingService(None, version, skip_caps=True)
return wps._parseProcessMetadata(element)
def describe_provider_process(request):
# type: (Request) -> Process
"""
Obtains a remote service process description in a compatible local process format.
Note: this processes won't be stored to the local process storage.
"""
provider_id = request.matchdict.get("provider_id")
process_id = request.matchdict.get("process_id")
store = get_db(request).get_store(StoreServices)
service = store.fetch_by_name(provider_id)
wps = WebProcessingService(url=service.url,
headers=get_cookie_headers(request.headers))
set_wps_language(wps, request=request)
process = wps.describeprocess(process_id)
return Process.from_ows(service, process, get_settings(request))
def __init__(self, request):
super(ChooseSource, self).__init__(request,
name='wizard_source',
title="Choose Data Source")
wps = WebProcessingService(url=request.route_url(
'owsproxy',
service_name=self.wizard_state.get('wizard_wps')['identifier']),
verify=False,
skip_caps=True)
process = wps.describeprocess(
self.wizard_state.get('wizard_process')['identifier'])
for data_input in process.dataInputs:
if data_input.identifier == self.wizard_state.get(
'wizard_complex_inputs')['identifier']:
self.title = "Choose Data Source for %s" % data_input.title
break
def run_wps(res_ids):
print"launch wps"
print res_ids
# checks if there is two resource IDs
resources = res_ids.split("_")
if len(resources) < 2:
return JsonResponse({'status': 'wps request failed. 2 resources are required, found ' + str(len(resources))})
process_id = 'org.n52.wps.server.r.linear_regression'
process_input = [('x_resource_id',str(resources[0])),('y_resource_id',str(resources[1]))]
#setting the WPS URL is set in app.py
url_wps = CorrelationPlot.wps_url + '/WebProcessingService'
my_engine = WebProcessingService(url_wps, verbose=False, skip_caps=True)
my_process = my_engine.describeprocess(process_id)
#executing the process..
# build execution
execution = WPSExecution(version=my_engine.version, url=my_engine.url, username=my_engine.username,
password=my_engine.password, verbose=my_engine.verbose)
requestElement = execution.buildRequest(process_id, process_input, 'output')
request = etree.tostring(requestElement)
#set store executeresponse to false
request = request.replace('storeExecuteResponse="true"', 'storeExecuteResponse="false"')
print request
execution = my_engine.execute(process_id, process_input, 'output', request)
monitorExecution(execution)
status = execution.status
print status
# if the status is successful...
if status == 'ProcessSucceeded':
outputs = execution.processOutputs
output0 = outputs[0]
reference0 = output0.reference
# retrieve the data from the reference
output_data = requests.get(reference0)
resp = HttpResponse(output_data, content_type="application/json")
return resp
else:
return JsonResponse({'status': 'wps request failed'})
class ComplexInputs(Wizard):
def __init__(self, request):
super(ComplexInputs, self).__init__(request,
name='wizard_complex_inputs',
title="Choose Input Parameter")
self.wps = WebProcessingService(url=request.route_url(
'owsproxy',
service_name=self.wizard_state.get('wizard_wps')['identifier']),
verify=False,
skip_caps=True)
self.process = self.wps.describeprocess(
self.wizard_state.get('wizard_process')['identifier'])
self.title = "Choose Input Parameter of {0}".format(self.process.title)
def breadcrumbs(self):
breadcrumbs = super(ComplexInputs, self).breadcrumbs()
breadcrumbs.append(
dict(route_path=self.request.route_path(self.name),
title=self.title))
return breadcrumbs
def schema(self):
return Schema().bind(request=self.request, process=self.process)
def success(self, appstruct):
for inp in self.process.dataInputs:
if inp.identifier == appstruct.get('identifier'):
appstruct['mime_types'] = [
value.mimeType for value in inp.supportedValues
]
super(ComplexInputs, self).success(appstruct)
def next_success(self, appstruct):
self.success(appstruct)
return self.next('wizard_source')
def view(self):
return super(ComplexInputs, self).view()
def custom_view(self):
return dict(summary_title=self.process.title,
summary=getattr(self.process, 'abstract', 'No summary'),
url=wps_describe_url(self.wps.url,
self.process.identifier),
metadata=self.process.metadata)
def fill_section(section, gcube_vre_token, tool_dir):
gcube_vre_token_header = {"gcube-token": gcube_vre_token}
dataminer_url = ("http://dataminer-prototypes.d4science.org/wps/"
"WebProcessingService")
wps = WebProcessingService(dataminer_url, headers=gcube_vre_token_header)
tools = {}
tools["CSV extractor"] = {"file": os.path.join(tool_dir, "extract.xml")}
for process in wps.processes:
descr = wps.describeprocess(process.identifier)
tools[descr.title] = {"descr": descr, "process": process, "file": None}
for i, t in enumerate(sorted(tools)):
tool_file = os.path.join(tool_dir, "tool%02d.xml" % i)
if tools[t]["file"]:
shutil.copyfile(tools[t]["file"], tool_file)
else:
generate_tool_description(tools[t]["process"], tools[t]["descr"],
tool_file)
etree.SubElement(section, "tool", attrib={"file": tool_file})
class ExecuteProcess(Processes):
def __init__(self, request):
self.wps_id = request.params.get('wps')
self.wps = WebProcessingService(url=wps_url(request, self.wps_id), verify=False)
identifier = request.params.get('process')
# TODO: need to fix owslib to handle special identifiers
self.process = self.wps.describeprocess(identifier)
super(ExecuteProcess, self).__init__(request, name='processes_execute', title='')
def breadcrumbs(self):
breadcrumbs = super(ExecuteProcess, self).breadcrumbs()
route_path = self.request.route_path('processes_list', _query=[('wps', self.wps_id)])
breadcrumbs.append(dict(route_path=route_path, title=self.wps.identification.title))
breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.process.title))
return breadcrumbs
def appstruct(self):
return {}
def generate_form(self, formid='deform'):
schema = WPSSchema(request=self.request, process=self.process, user=self.get_user())
return Form(
schema,
buttons=('submit',),
formid=formid,
)
def process_form(self, form):
controls = self.request.POST.items()
try:
logger.debug("before validate")
appstruct = form.validate(controls)
logger.debug("before execute %s", appstruct)
self.execute(appstruct)
except ValidationFailure, e:
logger.exception('validation of exectue view failed.')
self.session.flash("There are errors on this page.", queue='danger')
return dict(description=getattr(self.process, 'abstract', ''),
form = e.render())
return HTTPFound(location=self.request.route_url('monitor'))
class LiteralInputs(Wizard):
def __init__(self, request):
super(LiteralInputs, self).__init__(request,
name='wizard_literal_inputs',
title="Literal Inputs")
self.wps = WebProcessingService(url=request.route_url(
'owsproxy',
service_name=self.wizard_state.get('wizard_wps')['identifier']),
verify=False,
skip_caps=True)
self.process = self.wps.describeprocess(
self.wizard_state.get('wizard_process')['identifier'])
self.title = "Literal inputs of {0}".format(self.process.title)
def breadcrumbs(self):
breadcrumbs = super(LiteralInputs, self).breadcrumbs()
breadcrumbs.append(
dict(route_path=self.request.route_path(self.name),
title=self.title))
return breadcrumbs
def schema(self):
return WPSSchema(request=self.request,
hide_complex=True,
process=self.process).bind(request=self.request)
def next_success(self, appstruct):
self.success(appstruct)
return self.next('wizard_complex_inputs')
def view(self):
return super(LiteralInputs, self).view()
def custom_view(self):
return dict(summary_title=self.process.title,
summary=getattr(self.process, 'abstract', 'No summary'),
url=wps_describe_url(self.wps.url,
self.process.identifier),
metadata=self.process.metadata)
class RookWPS:
def __init__(self, url):
self.wps = WebProcessingService(url, verbose=False, skip_caps=True)
def getcapabilities(self):
self.wps.getcapabilities()
return self.wps
def describeprocess(self, identifier):
return self.wps.describeprocess(identifier)
def execute(self, identifier, inputs):
outputs = [("output", True, None)]
execution = self.wps.execute(identifier, inputs, output=outputs)
monitorExecution(execution)
print(execution.errors)
assert execution.isSucceded() is True
assert len(execution.processOutputs) > 0
ml_url = execution.processOutputs[0].reference
xml = requests.get(ml_url).text
urls = parse_metalink(xml)
return urls
def initialize(self):
if not self.is_initialized():
try:
wps = WebProcessingService(self.url,
username=self.username,
password=self.password,
verbose=False,
skip_caps=False)
process = wps.describeprocess(self.identifier)
if process:
self.title = self.title or process.title
self.abstract = self.abstract or process.abstract
self.version = self.version or wps.version
self.service_instance = wps.url
self.save()
self._initialized = True
self._wps = wps
self._execution_response = WebProcessingServiceRun.getOWSLibExecutionResponse(
self)
except:
self._initialized = False
self._wps = None
def pinned_processes(self):
settings = self.request.db.settings.find_one() or {}
processes = []
if 'pinned_processes' in settings:
for pinned in settings.get('pinned_processes'):
try:
service_name, identifier = pinned.split('.', 1)
url = self.request.route_path(
'processes_execute', _query=[('wps', service_name), ('process', identifier)])
wps = WebProcessingService(
url=self.request.route_url('owsproxy', service_name=service_name), verify=False)
# TODO: need to fix owslib to handle special identifiers
process = wps.describeprocess(identifier)
description = headline(process.abstract)
except Exception:
LOGGER.warn("could not add pinned process %s", pinned)
else:
processes.append(dict(
title=process.identifier,
description=description,
url=url,
service_title=wps.identification.title))
return processes
class LiteralInputs(Wizard):
def __init__(self, request):
super(LiteralInputs, self).__init__(request, name='wizard_literal_inputs', title="Literal Inputs")
from owslib.wps import WebProcessingService
self.wps = WebProcessingService(wps_url(request, self.wizard_state.get('wizard_wps')['identifier']))
self.process = self.wps.describeprocess(self.wizard_state.get('wizard_process')['identifier'])
self.title = "Literal inputs of {0}".format(self.process.title)
def breadcrumbs(self):
breadcrumbs = super(LiteralInputs, self).breadcrumbs()
breadcrumbs.append(dict(route_path=self.request.route_path(self.name), title=self.title))
return breadcrumbs
def schema(self):
from phoenix.schema.wps import WPSSchema
return WPSSchema(hide_complex=True, process = self.process)
def next_success(self, appstruct):
self.success(appstruct)
return self.next('wizard_complex_inputs')
@view_config(route_name='wizard_literal_inputs', renderer='../templates/wizard/default.pt')
def view(self):
return super(LiteralInputs, self).view()
class pyGDPwebProcessing():
"""
This class allows interactive calls to be made into the GDP.
"""
def _init_(self, wfsUrl=WFS_URL, wpsUrl=WPS_URL, version='1.1.0'):
self.wfsUrl = wfsUrl
self.wpsUrl = wpsUrl
self.version = version
self.wps = WebProcessingService(wpsUrl)
def WPSgetCapbilities(self, xml=None):
"""
Returns a list of capabilities.
"""
self.wps.getcapabilities(xml)
def WPSdescribeprocess(self, identifier, xml=None):
"""
Returns a list describing a specific identifier/process.
"""
self.wps.describeprocess(identifier, xml)
def _encodeZipFolder(self, filename):
"""
This function will encode a zipfile and return the filename.
"""
#check extension
if not filename.endswith('.zip'):
print 'Wrong filetype.'
return
#encode the file
with open(filename, 'rb') as fin:
bytesRead = fin.read()
encode= base64.b64encode(bytesRead)
#renames the file and saves it onto local drive
filename = filename.split('.')
filename = str(filename[0]) + '_copy.' + str(filename[-1])
fout = open(filename, "w")
fout.write(encode)
fout.close()
return filename
def uploadShapeFile(self, filePath):
"""
Given a file, this function encodes the file and uploads it onto geoserver.
"""
# encodes the file, opens it, reads it, and closes it
# returns a filename in form of: filename_copy.zip
filePath = self._encodeZipFolder(filePath)
if filePath is None:
return
filehandle = open(filePath, 'r')
filedata = filehandle.read()
filehandle.close()
os.remove(filePath) # deletes the encoded file
# this if for naming the file on geoServer
filename = filePath.split("/")
# gets rid of filepath, keeps only filename eg: file.zip
filename = filename[len(filename) - 1]
filename = filename.replace("_copy.zip", "")
# check to make sure a file with the same name does not exist
fileCheckString = "upload:" + filename
shapefiles = self.getShapefiles()
if fileCheckString in shapefiles:
print 'File exists already.'
return
xmlGen = gdpXMLGenerator()
root = xmlGen.getUploadXMLtree(filename, upload_URL, filedata)
# now we have a complete XML upload request
uploadRequest = etree.tostring(root)
POST = WebProcessingService(WPS_Service)
execution = POST.execute(None, [], request=uploadRequest)
monitorExecution(execution)
return "upload:"+filename
def getTuples(self, shapefile, attribute):
"""
Will return the dictionary tuples only.
"""
return self.getValues(shapefile, attribute, getTuples='only')
def _getGMLIDString(self, GMLbeginterm, line, GMLstopterm, valBeginTerm, valStopTerm):
"""
This function is specific to the output documents from the GDP. This
function parses the XML document, to find the correct GMLID associated
with a feature. Returns the list of values, and a dictionary [feature:id].
"""
# we are searching for attr-value, gml:id pair
value = []
ntuple = []
begin_index = 0
end_index = len(line)
tmpline = line
# start at the beginning
while begin_index < len(line):
begin_index = tmpline.find(GMLbeginterm)
if begin_index != -1:
end_index = tmpline.find(GMLstopterm, begin_index)
# we get the gml term
gmlterm = tmpline[begin_index + len(GMLbeginterm) : end_index ]
# now we get the attribute value
begin_index2 = tmpline.find(valBeginTerm)
end_index2 = tmpline.find(valStopTerm, begin_index2)
valTerm = tmpline[begin_index2 + len(valBeginTerm) : end_index2 ]
#tuple: attr-value, gml:id
tmpTuple = valTerm, gmlterm
ntuple.append(tmpTuple)
tmpline = tmpline[end_index2 :]
if valTerm not in value:
value.append(valTerm)
begin_index = end_index
#print begin_index
else:
break
return value, ntuple
def _urlen(self, typename):
"""
Sets up a cgi request to the wfs for features specified.
"""
service_url = GDP_URL
qs = []
if service_url.find('?') != -1:
qs = cgi.parse_qsl(service_url.split('?')[1])
params = [x[0] for x in qs]
if 'service' not in params:
qs.append(('service', 'WFS'))
if 'request' not in params:
qs.append(('request', 'DescribeFeatureType'))
if 'version' not in params:
qs.append(('version', '1.1.0'))
if 'typename' not in params:
qs.append(('typename', typename))
urlqs = urlencode(tuple(qs))
return service_url.split('?')[0] + '?' + urlqs
def _getStringBetween(self, beginterm, line, stopterm):
"""
Helper function. Gets the string between beginterm and stopterm.
Line is the line or superstring to be examined.
returns the string inbetween.
"""
begin_index = line.find(beginterm)
end_index = line.find(stopterm, begin_index)
return line[begin_index + len(beginterm) : end_index ]
def _getLinesContaining(self, linesToParse, term):
"""
Given a document, goes through the document and for each line with the
occurence of the specified term, add that line to a list.
Returns the list.
"""
line_list = []
for line in linesToParse:
if term in line:
line_list.append(line)
linesToParse.close()
return line_list
def _getFilterID(self,tuples, value):
"""
Given a the tuples generated by getTuples and a value, will return a list of gmlIDs
associated with the value specified.
"""
value = str(value)
filterID = []
for item in tuples:
if item[0] == value:
filterID.append(item[1])
return filterID
def _parseXMLNodesForTagText(self, xml, tag):
"""
Parses through a XML tree for text associated with specified tag.
Returns a list of the text.
"""
tag_text = []
for node in xml.iter():
if node.tag == tag:
tag_text.append(node.text)
return tag_text
def _generateRequest(self, dataSetURI, algorithm, method, varID=None, verbose=False):
"""
Takes a dataset uri, algorithm, method, and datatype. This function will generate a simple XML document
to make the request specified. (Only works for ListOpendapGrids and GetGridTimeRange).
Will return a list containing the info requested for (either data types or time range).
"""
wps_Service = 'http://cida.usgs.gov/gdp/utility/WebProcessingService'
POST = WebProcessingService(wps_Service, verbose=False)
xmlGen = gdpXMLGenerator()
root = xmlGen.getXMLRequestTree(dataSetURI, algorithm, method, varID, verbose)
# change standard output to not display waiting status
if not verbose:
old_stdout = sys.stdout
result = StringIO()
sys.stdout = result
request = etree.tostring(root)
execution = POST.execute(None, [], request=request)
if method == 'getDataSetTime':
seekterm = 'time'
else:
seekterm = 'name'
if not verbose:
sys.stdout = old_stdout
return self._parseXMLNodesForTagText(execution.response, seekterm)
def _generateFeatureRequest(self, typename, attribute=None):
"""
This function, given a attribute and a typename or filename will return a list of values associated
with the file and the attribute chosen.
"""
service_url = GDP_URL
qs = []
if service_url.find('?') != -1:
qs = cgi.parse_qsl(service_url.split('?')[1])
params = [x[0] for x in qs]
if 'service' not in params:
qs.append(('service', 'WFS'))
if 'request' not in params:
if attribute is None:
qs.append(('request', 'DescribeFeatureType'))
else:
qs.append(('request', 'GetFeature'))
if 'version' not in params:
qs.append(('version', '1.1.0'))
if 'typename' not in params:
qs.append(('typename', typename))
if attribute is not None:
if 'propertyname' not in params:
qs.append(('propertyname', attribute))
urlqs = urlencode(tuple(qs))
return service_url.split('?')[0] + '?' + urlqs
def getAttributes(self, shapefile):
"""
Given a valid shapefile, this function will create a cgi call
returning a list of attributes associated with the shapefile.
"""
# makes a call to get an xml document containing list of shapefiles
urlen = self._generateFeatureRequest(shapefile)
linesToParse = urlopen(urlen)
# gets back from the linesToParse document, all lines with 2nd arg
lines = self._getLinesContaining(linesToParse, 'xsd:element maxOccurs=')
attributes = []
# search the line
for item in lines:
word = self._getStringBetween('name=', item, ' ')
# for attributes, will return "attribute", qoutes included, strip qoutes
if word[1:len(word) - 1] != "the_geom":
attributes.append(word[1: len(word) - 1])
return attributes
def getShapefiles(self):
"""
Returns a list of available files currently on geoserver.
"""
wfs = WebFeatureService(GDP_URL)
shapefiles = wfs.contents.keys()
return shapefiles
def getValues(self, shapefile, attribute, getTuples='false'):
"""
Similar to get attributes, given a shapefile and a valid attribute this function
will make a call to the Web Feature Services returning a list of values associated
with the shapefile and attribute.
If getTuples = True, will also return the tuples of [feature:id] along with values [feature]
"""
urlen = self._generateFeatureRequest(shapefile, attribute)
inputObject = urlopen(urlen)
shapefileterm = shapefile.split(':')
strinx = inputObject.read()
lines = strinx.split('\n')
# gets the tag/namespace name
stringSnippet = self._getStringBetween('<', lines[1], ':'+attribute+'>')
stringSnippet = stringSnippet.split('<')
shapefileterm[0] = stringSnippet[len(stringSnippet) - 1]
# look for this pattern: <term[0]:attribute>SOUGHTWORD</term[0]:attribute>
values, tuples = self._getGMLIDString('gml:id="', lines[1], '">', '<'+shapefileterm[0] +
':' + attribute + '>', '</' +shapefileterm[0] +':' +
attribute + '>')
if getTuples=='true':
return sorted(values), sorted(tuples)
elif getTuples=='only':
return sorted(tuples)
else:
return sorted(values)
def getDataType(self, dataSetURI, verbose=False):
"""
Set up a get Data type request given a dataSetURI. Returns a list of all available data types.
If verbose = True, will print on screen the waiting seq. for response document.
"""
algorithm = 'gov.usgs.cida.gdp.wps.algorithm.discovery.ListOpendapGrids'
return self._generateRequest(dataSetURI, algorithm, method='getDataType', varID=None, verbose=verbose)
def getDataSetURI(self):
"""
This function will not be implemented. This function is only implemented to give a few dataset URIs which may not work
with certain datasets and will with others within the bounding box requirements.
"""
print 'The dataSetURI outputs a select few URIs and may not work with the specific shapefile you are providing.'
print 'To ensure compatibility, we recommend selecting a dataSetURI that is specific to the shapefile.'
print 'Or you may utilize the web gdp @ http://cida.usgs.gov/gdp/ to get a dataSet matching your specified shapefile.'
print
dataSetURIs = ['http://regclim.coas.oregonstate.edu:8080/thredds/dodsC/regcmdata/NCEP/merged/monthly/RegCM3_A2_monthly_merged_NCEP.ncml',
'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/conus_grid.w_meta.ncml',
'http://cida.usgs.gov/qa/thredds/dodsC/prism',
'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml',
'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/dcp/alaska_grid.w_meta.ncml',
'dods://igsarm-cida-thredds1.er.usgs.gov:8080/thredds/dodsC/gmo/GMO_w_meta.ncml']
return dataSetURIs
def getGMLIDs(self, shapefile, attribute, value):
"""
This function returns the gmlID associated with a particular attribute value.
"""
tuples = self.getTuples(shapefile, attribute)
return self._getFilterID(tuples, value)
def getTimeRange(self, dataSetURI, varID, verbose=False):
"""
Set up a get dataset time range request given a datatype and dataset uri. Returns the range
of the earliest and latest time.
If verbose = True, will print on screen the waiting seq. for response document.
"""
algorithm = 'gov.usgs.cida.gdp.wps.algorithm.discovery.GetGridTimeRange'
return self._generateRequest(dataSetURI, algorithm, method='getDataSetTime', varID=varID, verbose=verbose)
def _getFeatureCollectionGeoType(self, geoType, attribute='the_geom', value=None, gmlIDs=None):
"""
This function returns a featurecollection. It takes a geotype and determines if
the geotype is a shapfile or polygon.
"""
# This is a polygon
if isinstance(geoType, list):
return GMLMultiPolygonFeatureCollection( [geoType] )
elif isinstance(geoType, str):
if value==None and gmlIDs==None:
print 'must input a value and attribute for shapefile'
return
else:
tmpID = []
if gmlIDs is None:
if type(value) == type(tmpID):
gmlIDs = []
for v in value:
tuples = self.getTuples(geoType, attribute)
tmpID = self._getFilterID(tuples, v)
gmlIDs = gmlIDs + tmpID
else:
tuples = self.getTuples(geoType, attribute)
gmlIDs = self._getFilterID(tuples, value)
query = WFSQuery(geoType, propertyNames=["the_geom", attribute], filters=gmlIDs)
return WFSFeatureCollection(WFS_URL, query)
else:
print 'Geotype is not a shapefile or a recognizable polygon.'
return None
def _executeRequest(self, processid, inputs, verbose):
"""
This function makes a call to the Web Processing Service with
the specified user inputs.
"""
wps = WebProcessingService(WPS_URL)
# if verbose=True, then will we will monitor the status of the call.
# if verbose=False, then we will return only the file outputpath.
if not verbose:
# redirects the standard output to avoid printing request status
old_stdout = sys.stdout
result = StringIO()
sys.stdout = result
# executes the request
execution = wps.execute(processid, inputs, output = "OUTPUT")
monitorExecution(execution, download=True)
# sets the standard output back to original
sys.stdout = old_stdout
result_string = result.getvalue()
#parses the redirected output to get the filepath of the saved file
output = result_string.split('\n')
tmp = output[len(output) - 2].split(' ')
return tmp[len(tmp)-1]
# executes the request
execution = wps.execute(processid, inputs, output = "OUTPUT")
monitorExecution(execution, download=True)
def submitFeatureWeightedGridStatistics(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None,
gmlIDs=None, verbose=None, coverage='true', delim='COMMA', stat='MEAN', grpby='STATISTIC',
timeStep='false', summAttr='false'):
"""
Makes a featureWeightedGridStatistics algorithm call.
"""
featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs)
if featureCollection is None:
return
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm'
inputs = [("FEATURE_ATTRIBUTE_NAME",attribute),
("DATASET_URI", dataSetURI),
("DATASET_ID", varID),
("TIME_START",startTime),
("TIME_END",endTime),
("REQUIRE_FULL_COVERAGE",coverage),
("DELIMITER",delim),
("STATISTICS",stat),
("GROUP_BY", grpby),
("SUMMARIZE_TIMESTEP", timeStep),
("SUMMARIZE_FEATURE_ATTRIBUTE",summAttr),
("FEATURE_COLLECTION", featureCollection)]
return self._executeRequest(processid, inputs, verbose)
def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None,
verbose=False, coverage='true'):
"""
Makes a featureCoverageOPenDAP algorithm call.
"""
featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs)
if featureCollection is None:
return
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureCoverageOPeNDAPIntersectionAlgorithm'
inputs = [ ("DATASET_URI", dataSetURI),
("DATASET_ID", varID),
("TIME_START",startTime),
("TIME_END",endTime),
("REQUIRE_FULL_COVERAGE",coverage),
("FEATURE_COLLECTION", featureCollection)]
return self._executeRequest(processid, inputs, verbose)
def submitFeatureCoverageWCSIntersection(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False, coverage='true'):
"""
Makes a featureCoverageWCSIntersection algorithm call.
"""
featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs)
if featureCollection is None:
return
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureCoverageIntersectionAlgorithm'
inputs = [("DATASET_URI", dataSetURI),
("DATASET_ID", varID),
("REQUIRE_FULL_COVERAGE",coverage),
("FEATURE_COLLECTION", featureCollection)]
return self._executeRequest(processid, inputs, verbose)
def submitFeatureCategoricalGridCoverage(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False,
coverage='true', delim='COMMA'):
"""
Makes a featureCategoricalGridCoverage algorithm call.
"""
featureCollection = self._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs)
if featureCollection is None:
return
processid = 'gov.usgs.cida.gdp.wps.algorithm.FeatureCategoricalGridCoverageAlgorithm'
inputs = [ ("FEATURE_ATTRIBUTE_NAME",attribute),
("DATASET_URI", dataSetURI),
("DATASET_ID", varID),
("DELIMITER", delim),
("REQUIRE_FULL_COVERAGE",coverage),
("FEATURE_COLLECTION", featureCollection)]
return self._executeRequest(processid, inputs, verbose)
class GenericWPS(ProgressMonitorPE):
"""
Wrap the execution of a WPS process into a dispel4py PE
"""
STATUS_NAME = 'status'
STATUS_LOCATION_NAME = 'status_location'
OUTPUT_NAME = 'outputs'
DUMMY_INPUT_NAME = 'None'
def __init__(self,
name,
url,
identifier,
inputs=None,
linked_inputs=None,
monitor=None,
progress_range=None,
headers=None,
**_):
"""
:param name: task name
:param url: url of the WPS provider
:param identifier: identifier of the WPS task
:param inputs: dict of static inputs for the wps
(multiple values can be passed for each key by using an array)
:param linked_inputs: dict of dynamic inputs of the wps obtained from upstream tasks
(multiple values can be passed for each key by using an array)
:param monitor: status and progress monitor
:param progress_provider:
:param progress_range: progress range of this task in the overall workflow
:param headers: headers to be included in the wps call
:param _: Accept any others parameters coming from the workflow description
"""
ProgressMonitorPE.__init__(self, name, monitor)
# Convert dict of values or array of values to a flat list of key, value tuple
inputs = list(self.iter_inputs(inputs))
linked_inputs = list(self.iter_inputs(linked_inputs))
# Defaults argument (with mutable type)
if not progress_range:
progress_range = [0, 100]
self.set_progress_provider(
RangeProgress(progress_range[0], progress_range[1]))
self.wps = WebProcessingService(url=url,
skip_caps=True,
verify=False,
headers=headers)
self.identifier = identifier
self.proc_desc = self.wps.describeprocess(identifier)
self._validate_inputs(inputs, linked_inputs)
# These are the static inputs
# (linked inputs will be appended to inputs just before execution by the _set_inputs function)
self.static_inputs = inputs
self.dynamic_inputs = []
# Will be filled as PE are connected to us (by the get_output function)
self.outputs = []
for _input in linked_inputs:
# Here we add PE input that will need to be connected
if _input[0] not in self.inputconnections:
self._add_input(_input[0])
self._add_linked_input(_input[0], _input[1])
def try_connect(self, graph, linked_input, downstream_task,
downstream_task_input):
"""
Override TaskPE fct. See TaskPE.try_connect for details.
Add the WPS outputs upon connection
"""
if ProgressMonitorPE.try_connect(self, graph, linked_input,
downstream_task,
downstream_task_input):
# Here we added the WPS output name and its as_reference request
self.outputs.append((linked_input.get('output',
self._get_default_output()),
linked_input.get('as_reference', False)))
return True
return False
def _process(self, inputs):
"""
Implement the GenericPE _process function.
This function is called multiple time if more than one input must be set
:param inputs: What has been outputted by the previous task
:return: None because the WPS execute function will only be called in _postprocess fct
"""
# Assign the input internally and wait for all inputs before launching the wps execution
for key, value in self._read_inputs(inputs):
self.dynamic_inputs.append((key, value))
def _postprocess(self):
"""
Implement the GenericPE _postprocess function.
Call when this PE has received all its inputs and thus are ready to execute the wps
"""
try:
self._check_inputs()
self._send_outputs(self._execute())
except Exception:
logger.exception("process failed!")
raise
def _connect(self, from_connection, to_node, to_connection, graph):
"""
Override TaskPE fct. See TaskPE._connect for details.
We add the PE output just before completing the connection in the graph
"""
# Here we add a PE output that is required before completing the actual connection
self._add_output(from_connection)
ProgressMonitorPE._connect(self, from_connection, to_node,
to_connection, graph)
def _get_default_output(self):
"""
Implement TaskPE fct.
If the WPS has only one output it can be set as the default one. Else it has to be set.
"""
if len(self.proc_desc.processOutputs) == 1:
return self.proc_desc.processOutputs[0].identifier
return None
def get_input_desc(self, input_name):
"""
Implement TaskPE fct. See TaskPE.get_input_desc for details.
"""
for wps_input in self.proc_desc.dataInputs:
if wps_input.identifier == input_name:
return wps_input
return None
def get_output_desc(self, output_name):
"""
Implement TaskPE fct. See TaskPE.get_output_desc for details.
"""
for wps_output in self.proc_desc.processOutputs:
if wps_output.identifier == output_name:
return wps_output
return None
def _validate_inputs(self, inputs, linked_inputs):
"""
Check that given inputs and linked inputs are valid
:param inputs: static input (value is available now)
:param linked_inputs: dynamic inputs (value will be obtained from upstream task)
"""
# Validate that given inputs exist in the wps
valid_inputs = [
wps_input.identifier for wps_input in self.proc_desc.dataInputs
]
# Allow a process not requiring any input to be linked to a previous one by using a dummy ("None") input name
valid_inputs.append(self.DUMMY_INPUT_NAME)
for submitted_input in inputs + linked_inputs:
if submitted_input[0] not in valid_inputs:
raise WorkflowException(
"Invalid workflow : Input '{input}' of process '{proc}' is unknown."
.format(input=submitted_input[0], proc=self.identifier))
def _check_inputs(self):
"""
Check if all the required inputs have been set (from workflow static inputs or from upstream tasks)
"""
ready_inputs = [
_input[0] for _input in self.static_inputs + self.dynamic_inputs
]
# Consider the dummy input to be always ready!
ready_inputs.append(self.DUMMY_INPUT_NAME)
for linked_input in self.linked_inputs:
if linked_input[0] not in ready_inputs:
msg = "Workflow cannot complete because of a missing input '{input}' of task {task}".format(
input=linked_input[0], task=self.name)
raise WorkflowException(msg)
@staticmethod
def _check_status(execution):
"""
Try to read the xml status of the underlying WPS process, raise Exception if the url cannot be read properly
"""
reader = WPSExecuteReader(verbose=execution.verbose)
# override status location
logger.info('Checking execution status : %s' %
execution.statusLocation)
try:
response = reader.readFromUrl(execution.statusLocation,
username=execution.username,
password=execution.password,
verify=execution.verify,
headers=execution.headers)
response = etree.tostring(response)
except Exception:
raise
else:
execution.checkStatus(response=response, sleepSecs=3)
def _monitor_execution(self, execution):
"""
Monitor the execution of the underlying WPS and return only when the process end (successfully or not)
"""
progress = self.progress(execution)
self.monitor("status_location={0.statusLocation}".format(execution),
progress)
xml_doc_read_failure = 0
last_status_message = None
last_progress = None
while execution.isNotComplete():
try:
# Check the status of the wps execution
self._check_status(execution)
except Exception as e:
# Try XML_DOC_READING_MAX_ATTEMPT time before raising an exception
xml_doc_read_failure += 1
if xml_doc_read_failure > XML_DOC_READING_MAX_ATTEMPT:
logger.error(
"Failed to read status xml document after %s attempts : %s",
XML_DOC_READING_MAX_ATTEMPT, str(e))
raise
else:
# Sleep 5 seconds to give a chance
sleep(5)
else:
progress = self.progress(execution)
if execution.statusMessage != last_status_message or progress != last_progress:
last_status_message = execution.statusMessage
last_progress = progress
self.monitor(execution.statusMessage, progress)
self.monitor(execution.statusMessage, progress)
# In case of success log all output value
if execution.isSucceded():
for output in execution.processOutputs:
if output.reference is not None:
self.monitor(
'{0.identifier}={0.reference} ({0.mimeType})'.format(
output), progress)
else:
self.monitor(
'{0}={1}'.format(output.identifier,
", ".join(output.data)), progress)
# Or log the errors
else:
self.monitor(
'\n'.join([
'ERROR: {0.text} code={0.code} locator={0.locator})'.
format(ex) for ex in execution.errors
]), progress)
def get_output_datatype(self, output):
"""
Extract the correct datatype from the datatype structure
:param output: output as parsed from the wps xml status (ows.wps.Output)
:return: The datatype of the output as a string (ComplexData, string, etc.)
"""
# outputs from execution structure do not always carry the dataType
# so find it from the process description
if not output.dataType:
for desc_wps_output in self.proc_desc.processOutputs:
if desc_wps_output.identifier == output.identifier:
return desc_wps_output.dataType
# Also the datatype is not always stripped correctly so do the job here
return output.dataType.split(':')[-1]
def _jsonify_output(self, output):
"""
Utility method to jsonify an output element.
"""
json_output = dict(identifier=output.identifier,
title=output.title,
dataType=self.get_output_datatype(output))
# WPS standard v1.0.0 specify that either a reference or a data field has to be provided
if output.reference:
json_output['reference'] = output.reference
else:
# WPS standard v1.0.0 specify that Output data field has Zero or one value
json_output['data'] = output.data[0] if output.data else None
if json_output['dataType'] == 'ComplexData':
json_output['mimeType'] = output.mimeType
return json_output
def _execute(self):
"""
This is the function doing the actual WPS process call, monitoring its execution and parsing the output.
:return: Return the data that is send to the downstream PE
"""
logger.debug(
"execute with static inputs=%s and dynamic inputs=%s to get outputs=%s",
self.static_inputs, self.dynamic_inputs, self.outputs)
self.wps.headers['machineid'] = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(16))
execution = self.wps.execute(identifier=self.identifier,
inputs=self.static_inputs +
self.dynamic_inputs,
output=self.outputs,
lineage=True)
self.set_headers(self.data_headers)
self._monitor_execution(execution)
# Reset dynamic inputs for the next iteration
self.dynamic_inputs = []
execution_result = {
self.STATUS_NAME: execution.status,
self.STATUS_LOCATION_NAME: execution.statusLocation
}
if execution.isSucceded():
execution_result[self.OUTPUT_NAME] = \
[self._jsonify_output(output) for output in execution.processOutputs]
self.save_result(execution_result)
result = {}
# NOTE: only set workflow output if specific output was needed
for wps_output in self.outputs:
for output in execution.processOutputs:
if wps_output[0] == output.identifier:
# outputs from execution structure do not always carry the dataType
# so find it from the process description because the _adapt fct needs it
if not output.dataType:
for desc_wps_output in self.proc_desc.processOutputs:
if desc_wps_output.identifier == output.identifier:
output.dataType = desc_wps_output.dataType
# Also the datatype is not always stripped correctly so do the job here
else:
output.dataType = output.dataType.split(':')[-1]
# Send directly the wps output object to the downstream PE
# Note: output.data is always an array since a wps output is appended to processOutputs[x].data
result[wps_output[0]] = output
break
return result
else:
failure_msg = '\n'.join(
['{0.text}'.format(ex) for ex in execution.errors])
raise WorkflowException(failure_msg)
@staticmethod
def iter_inputs(inputs):
"""
Convert dictionary of inputs where the value is either an array or a value to a flat array of tuple (key, value)
:param inputs: Dictionnary of inputs looking like this {key1: val1, key2: val2, key3: [val3.1, val3.2, val3.3]}
:return: yield a tuple for each key/value pair looking like this
[(key1, val1), (key2, val2), (key3, val3.1), (key3, val3.2), (key3, val3.3)]
"""
if not inputs:
return
for key in inputs:
if isinstance(inputs[key], list):
for value in inputs[key]:
yield (key, value)
else:
yield (key, inputs[key])
# 1) GetCapabilities
# GET request: http://ceda-wps2.badc.rl.ac.uk/wps?Service=WPS&Request=GetCapabilities&Format=text/xml
wps.getcapabilities()
print('WPS Identification type: %s' % wps.identification.type)
print('WPS Identification title: %s' % wps.identification.title)
print('WPS Identification abstract: %s' % wps.identification.abstract)
for operation in wps.operations:
print('WPS Operation: %s' % operation.name)
for process in wps.processes:
print('WPS Process: identifier=%s title=%s' % (process.identifier, process.title))
# 2) DescribeProcess
# GET request: http://ceda-wps2.badc.rl.ac.uk/wps?identifier=DoubleIt&version=1.0.0&request=DescribeProcess&service=WPS
process = wps.describeprocess('DoubleIt')
print('WPS Process: identifier=%s' % process.identifier)
print('WPS Process: title=%s' % process.title)
print('WPS Process: abstract=%s' % process.abstract)
for input in process.dataInputs:
print('Process input:')
printInputOutput(input, indent='\t')
for output in process.processOutputs:
print('Process output:')
printInputOutput(output, indent='\t')
# 3) Execute
# POST request:
# Note: not working, requires openid login ?
#processid = "DoubleIt"
#inputs = [ ("NumberToDouble","1") ]
verbose=verbose,
skip_caps=True)
# 1) GetCapabilities
wps.getcapabilities()
print('WPS Identification type: %s' % wps.identification.type)
print('WPS Identification title: %s' % wps.identification.title)
print('WPS Identification abstract: %s' % wps.identification.abstract)
for operation in wps.operations:
print('WPS Operation: %s' % operation.name)
for process in wps.processes:
print('WPS Process: identifier=%s title=%s' %
(process.identifier, process.title))
# 2) DescribeProcess
process = wps.describeprocess('v.net.path')
# alternatively, read process description from XML file (no live request to WPS server)
#xml = open('../tests/USGSDescribeProcess.xml', 'rb').read()
#process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml)
print('WPS Process: identifier=%s' % process.identifier)
print('WPS Process: title=%s' % process.title)
print('WPS Process: abstract=%s' % process.abstract)
for input in process.dataInputs:
print(
'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d'
% (input.identifier, input.dataType, input.minOccurs, input.maxOccurs))
for output in process.processOutputs:
print('Process output: identifier=%s, data type=%s' %
(output.identifier, output.dataType))
# 3) Execute
class pyGDPwebProcessing():
"""
This class allows interactive calls to be made into the GDP.
"""
def __init__(self, WFS_URL=None):
if WFS_URL==None:
from pygdp.namespaces import WFS_URL
wfsUrl=WFS_URL
self.wfsUrl = wfsUrl
self.wpsUrl = WPS_URL
self.version = '1.1.0'
self.wps = WebProcessingService(WPS_URL)
def WPSgetCapbilities(self, xml=None):
"""
Returns a list of capabilities.
"""
self.wps.getcapabilities(xml)
def WPSdescribeprocess(self, identifier, xml=None):
"""
Returns a list describing a specific identifier/process.
"""
self.wps.describeprocess(identifier, xml)
#pyGDP Submit Feature
def dodsReplace(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return _execute_request.dodsReplace(dataSetURI, verbose)
def submitFeatureCoverageOPenDAP(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None, gmlIDs=None,
verbose=False, coverage='true', outputfname=None, sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCoverageOPenDAP(geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs)
def submitFeatureCoverageWCSIntersection(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False,
coverage='true', outputfname=None, sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCoverageWCSIntersection(geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs)
def submitFeatureCategoricalGridCoverage(self, geoType, dataSetURI, varID, attribute='the_geom', value=None, gmlIDs=None, verbose=False,
coverage='true', delim='COMMA', outputfname=None, sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCategoricalGridCoverage(geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose, coverage, delim, self.wfsUrl, outputfname, sleepSecs)
def submitFeatureWeightedGridStatistics(self, geoType, dataSetURI, varID, startTime, endTime, attribute='the_geom', value=None,
gmlIDs=None, verbose=None, coverage=True, delim='COMMA', stat='MEAN', grpby='STATISTIC',
timeStep=False, summAttr=False, weighted=True, outputfname=None, sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return fwgs.submitFeatureWeightedGridStatistics(geoType, dataSetURI, varID, startTime, endTime, attribute, value, gmlIDs,
verbose, coverage, delim, stat, grpby, timeStep, summAttr, weighted, self.wfsUrl, outputfname, sleepSecs)
#pyGDP File Utilities
def shapeToZip(self, inShape, outZip=None, allFiles=True):
return shape_to_zip.shapeToZip(inShape, outZip=None, allFiles=True)
def uploadShapeFile(self, filePath):
value, ntuple = upload_shapefile.uploadShapefile(filePath)
return value, ntuple
#pyGDP WFS Utilities
def getTuples(self, shapefile, attribute):
return shapefile_id_handle.getTuples(shapefile, attribute)
def getShapefiles(self):
return shapefile_value_handle.getShapefiles(self.wfsUrl)
def getAttributes(self, shapefile):
return shapefile_value_handle.getAttributes(shapefile, self.wfsUrl)
def getValues(self, shapefile, attribute, getTuples='false', limitFeatures=None):
return shapefile_value_handle.getValues(shapefile, attribute, getTuples, limitFeatures, self.wfsUrl)
def getGMLIDs(self, shapefile, attribute, value):
return shapefile_id_handle.getGMLIDs(shapefile, attribute, value, WFS_URL=self.wfsUrl)
def _getFilterID(self, tuples, value):
return shapefile_id_handle._getFilterID(tuples, value)
def _getFeatureCollectionGeoType(self, geoType, attribute='the_geom', value=None, gmlIDs=None):
return _get_geotype._getFeatureCollectionGeoType(geoType, attribute, value, gmlIDs, self.wfsUrl)
def _generateRequest(self, dataSetURI, algorithm, method, varID=None, verbose=False):
return _webdata_xml_generate._generateRequest(dataSetURI, algorithm, method, varID, verbose)
#pyGDP WebData Utilities
def getDataLongName(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getDataLongName(dataSetURI, verbose)
def getDataType(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getDataType(dataSetURI, verbose)
def getDataUnits(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getDataUnits(dataSetURI, verbose)
def getDataSetURI(self, anyText='',CSWURL=CSWURL,BBox=None):
return webdata_handle.getDataSetURI(anyText, CSWURL, BBox)
def getTimeRange(self, dataSetURI, varID, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getTimeRange(dataSetURI, varID, verbose)
#Pull up docstrings.
#dodsReplace.__doc__ = _execute_request.dodsReplace.__doc__
getAttributes.__doc__ = shapefile_value_handle.getAttributes.__doc__
getDataLongName.__doc__ = webdata_handle.getDataLongName.__doc__
getDataSetURI.__doc__ = webdata_handle.getDataSetURI.__doc__
getDataType.__doc__ = webdata_handle.getDataType.__doc__
getDataUnits.__doc__ = webdata_handle.getDataUnits.__doc__
getGMLIDs.__doc__ = shapefile_id_handle.getGMLIDs.__doc__
getShapefiles.__doc__ = shapefile_value_handle.getShapefiles.__doc__
getTimeRange.__doc__ = webdata_handle.getTimeRange.__doc__
getTuples.__doc__ = shapefile_id_handle.getTuples.__doc__
getValues.__doc__ = shapefile_value_handle.getValues.__doc__
shapeToZip.__doc__ = shape_to_zip.shapeToZip.__doc__
submitFeatureCategoricalGridCoverage.__doc__ = feature_coverage.submitFeatureCategoricalGridCoverage.__doc__
submitFeatureCoverageOPenDAP.__doc__ = feature_coverage.submitFeatureCoverageOPenDAP.__doc__
submitFeatureCoverageWCSIntersection.__doc__ = feature_coverage.submitFeatureCoverageWCSIntersection.__doc__
submitFeatureWeightedGridStatistics.__doc__ = fwgs.submitFeatureWeightedGridStatistics.__doc__
uploadShapeFile.__doc__ = upload_shapefile.uploadShapeFile.__doc__
def check_wps(serverCursor):
'''
Check all WPS services, and update the database accordingly
'''
# Get the server list, but ignore servers marked as deleted
serverCursor.execute("SELECT DISTINCT url FROM " + tables["wpsServers"])
upsert_list = []
sqlList = []
# Mark all our records as dead; we'll mark them as alive as we process them. Note that the database won't
# actually be udpated until we commit all our transactions at the end, so we'll never see this value
# for a server/process/input that is in fact alive.
sqlList.append("UPDATE " + tables["wpsServers"] + " SET alive = false")
sqlList.append("UPDATE " + tables["processes"] + " SET alive = false")
# We'll delete the parameter list completely; There is no benefit of keeping older, but now disused module
# parameters around... it just confuses things.
# If the parameters reappaear in the process in the future, they will be added back, and any values
# associated with them will be retained.
sqlList.append("DELETE FROM " + tables["processParams"])
for row in serverCursor:
server_url = row[0]
# Run a GetCapabilities query on the WPS server -- could fail if URL is bogus
try:
wps = WebProcessingService(server_url, version = wpsVersion)
except:
print "Could not load WPS data from url " + server_url
# If URL is bogus, will raise a URLError... but whatever... no errors are reoverable at this point
continue
# Update the server title and abstract
sqlList.append(prepare_update_wps_server_info(server_url, wps))
# Iterate over the processes available on this server
for proc in wps.processes:
abstract = get_process_abstract(proc)
sqlList.append(prepare_update_wps_process(server_url, proc.identifier, proc.title, abstract))
# Need to do this here so that the SELECT below will find a record if the upsert inserts... a little messy
run_queries(db_conn, upsert_list, sqlList)
upsert_list = []
sqlList = []
# Get all processes associated with server that match the specified identifier -- could be more than one
update_cursor.execute(prepare_select_processes(server_url, proc.identifier))
for procrow in update_cursor:
procId = procrow[0]
try:
procDescr = wps.describeprocess(proc.identifier) # Call to OWSLib
except:
print "Could not describe process ", proc.identifier, " on server ", server_url
for input in procDescr.dataInputs:
upsert_list.append((tables["processParams"], "wps_process_id", procId, input.identifier))
sqlList.append(prepare_update_wps_param(procId, input, True))
for output in procDescr.processOutputs:
upsert_list.append((tables["processParams"], "wps_process_id", procId, output.identifier))
sqlList.append(prepare_update_wps_param(procId, output, False))
# Run and commit WPS transactions
run_queries(db_conn, upsert_list, sqlList)
upsert_list = []
def test_wps_describeprocess_usgs():
# Initialize WPS client
wps = WebProcessingService('http://cida.usgs.gov/gdp/process/WebProcessingService', skip_caps=True)
# Execute fake invocation of DescribeProcess operation by parsing cached response from
xml = open(resource_file('wps_USGSDescribeProcess.xml'), 'rb').read()
process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml)
# Check process description
assert process.identifier == 'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm'
assert process.title == 'Feature Weighted Grid Statistics'
assert process.abstract == 'This algorithm generates area weighted statistics of a gridded dataset for a set of vector polygon features. Using the bounding-box that encloses the feature data and the time range, if provided, a subset of the gridded dataset is requested from the remote gridded data server. Polygon representations are generated for cells in the retrieved grid. The polygon grid-cell representations are then projected to the feature data coordinate reference system. The grid-cells are used to calculate per grid-cell feature coverage fractions. Area-weighted statistics are then calculated for each feature using the grid values and fractions as weights. If the gridded dataset has a time range the last step is repeated for each time step within the time range or all time steps if a time range was not supplied.' # NOQA
# Check process inputs
# Expected Input:
# Process input:
# identifier=FEATURE_COLLECTION, title=Feature Collection, abstract=A feature collection encoded as a WFS request or one of the supported GML profiles., data type=ComplexData # NOQA
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.0.0/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.1.1/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.1.2/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.1.2.1/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.0.0/base/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.0.1/base/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.1.0/base/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.1.1/base/feature.xsd
# Supported Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/3.2.1/base/feature.xsd
# Default Value: mimeType=text/xml, encoding=UTF-8, schema=http://schemas.opengis.net/gml/2.0.0/feature.xsd
# minOccurs=1, maxOccurs=1
# Process input:
# identifier=DATASET_URI, title=Dataset URI, abstract=The base data web service URI for the dataset of interest., data type=anyURI # NOQA
# Any value allowed
# Default Value: None
# minOccurs=1, maxOccurs=1
# Process input:
# identifier=DATASET_ID, title=Dataset Identifier, abstract=The unique identifier for the data type or variable of interest., data type=string # NOQA
# Any value allowed
# Default Value: None
# minOccurs=1, maxOccurs=2147483647
# Process input:
# identifier=REQUIRE_FULL_COVERAGE, title=Require Full Coverage, abstract=If turned on, the service will require that the dataset of interest fully cover the polygon analysis zone data., data type=boolean # NOQA
# Any value allowed
# Default Value: True
# minOccurs=1, maxOccurs=1
# Process input:
# identifier=TIME_START, title=Time Start, abstract=The date to begin analysis., data type=dateTime
# Any value allowed
# Default Value: None
# minOccurs=0, maxOccurs=1
# Process input:
# identifier=TIME_END, title=Time End, abstract=The date to end analysis., data type=dateTime
# Any value allowed
# Default Value: None
# minOccurs=0, maxOccurs=1
# Process input:
# identifier=FEATURE_ATTRIBUTE_NAME, title=Feature Attribute Name, abstract=The attribute that will be used to label column headers in processing output., data type=string # NOQA
# Any value allowed
# Default Value: None
# minOccurs=1, maxOccurs=1
# Process input:
# identifier=DELIMITER, title=Delimiter, abstract=The delimiter that will be used to separate columns in the processing output., data type=string # NOQA
# Allowed Value: COMMA
# Allowed Value: TAB
# Allowed Value: SPACE
# Default Value: COMMA
# minOccurs=1, maxOccurs=1
# Process input:
# identifier=STATISTICS, title=Statistics, abstract=Statistics that will be returned for each feature in the processing output., data type=string # NOQA
# Allowed Value: MEAN
# Allowed Value: MINIMUM
# Allowed Value: MAXIMUM
# Allowed Value: VARIANCE
# Allowed Value: STD_DEV
# Allowed Value: WEIGHT_SUM
# Allowed Value: COUNT
# Default Value: None
# minOccurs=1, maxOccurs=7
# Process input:
# identifier=GROUP_BY, title=Group By, abstract=If multiple features and statistics are selected, this will change whether the processing output columns are sorted according to statistics or feature attributes., data type=string # NOQA
# Allowed Value: STATISTIC
# Allowed Value: FEATURE_ATTRIBUTE
# Default Value: None
# minOccurs=1, maxOccurs=1
# Process input:
# identifier=SUMMARIZE_TIMESTEP, title=Summarize Timestep, abstract=If selected, processing output will include columns with summarized statistics for all feature attribute values for each timestep, data type=boolean # NOQA
# Any value allowed
# Default Value: True
# minOccurs=0, maxOccurs=1
# Process input:
# identifier=SUMMARIZE_FEATURE_ATTRIBUTE, title=Summarize Feature Attribute, abstract=If selected, processing output will include a final row of statistics summarizing all timesteps for each feature attribute value, data type=boolean # NOQA
# Any value allowed
# Default Value: True
# minOccurs=0, maxOccurs=1
assert len(process.dataInputs) == 12
input = process.dataInputs[0]
assert input.identifier == 'FEATURE_COLLECTION'
assert input.dataType == 'ComplexData'
# Expected Output:
# identifier=OUTPUT, title=Output File, abstract=A delimited text file containing requested process output., data type=ComplexData # NOQA
# Supported Value: mimeType=text/csv, encoding=UTF-8, schema=None
# Default Value: mimeType=text/csv, encoding=UTF-8, schema=None
# reference=None, mimeType=None
# Check process outputs
assert len(process.processOutputs) == 1
output = process.processOutputs[0]
assert output.identifier == 'OUTPUT'
assert output.dataType == 'ComplexData'
def register_wps_processes_from_config(wps_processes_file_path, container):
# type: (Optional[FileSystemPathType], AnySettingsContainer) -> None
"""
Loads a `wps_processes.yml` file and registers `WPS-1` providers processes to the
current `Weaver` instance as equivalent `WPS-2` processes.
References listed under ``processes`` are registered.
When the reference is a service (provider), registration of each WPS process is done individually
for each of the specified providers with ID ``[service]_[process]`` per listed process by ``GetCapabilities``.
.. versionadded:: 1.14.0
When references are specified using ``providers`` section instead of ``processes``, the registration
only saves the remote WPS provider endpoint to dynamically populate WPS processes on demand.
.. seealso::
- `weaver.wps_processes.yml.example` for additional file format details
"""
if wps_processes_file_path is None:
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
wps_processes_file_path = get_weaver_config_file(
"", WEAVER_DEFAULT_WPS_PROCESSES_CONFIG)
elif wps_processes_file_path == "":
warnings.warn(
"Configuration file for WPS-1 providers registration explicitly defined as empty in settings. "
"Not loading anything.", RuntimeWarning)
return
# reprocess the path in case it is relative to default config directory
wps_processes_file_path = get_weaver_config_file(
wps_processes_file_path,
WEAVER_DEFAULT_WPS_PROCESSES_CONFIG,
generate_default_from_example=False)
if wps_processes_file_path == "":
warnings.warn("No file specified for WPS-1 providers registration.",
RuntimeWarning)
return
LOGGER.info("Using WPS-1 provider processes file: [%s]",
wps_processes_file_path)
try:
with open(wps_processes_file_path, "r") as f:
processes_config = yaml.safe_load(f)
processes = processes_config.get("processes") or []
providers = processes_config.get("providers") or []
if not processes and not providers:
LOGGER.warning("Nothing to process from file: [%s]",
wps_processes_file_path)
return
db = get_db(container)
process_store = db.get_store(StoreProcesses)
service_store = db.get_store(StoreServices)
# either 'service' references to register every underlying 'process' individually
# or explicit 'process' references to register by themselves
for cfg_service in processes:
svc_name, svc_url, svc_proc, svc_vis = parse_wps_process_config(
cfg_service)
# fetch data
LOGGER.info("Fetching WPS-1: [%s]", svc_url)
wps = WebProcessingService(url=svc_url)
if LooseVersion(wps.version) >= LooseVersion("2.0"):
LOGGER.warning("Invalid WPS-1 provider, version was [%s]",
wps.version)
continue
wps_processes = [wps.describeprocess(p)
for p in svc_proc] or wps.processes
for wps_process in wps_processes:
proc_id = "{}_{}".format(svc_name,
get_sane_name(wps_process.identifier))
try:
process_store.fetch_by_id(proc_id)
except ProcessNotFound:
pass
else:
LOGGER.warning(
"Process already registered: [%s]. Skipping...",
proc_id)
continue
proc_url = "{}?service=WPS&request=DescribeProcess&identifier={}&version={}" \
.format(svc_url, wps_process.identifier, wps.version)
svc_vis = VISIBILITY_PUBLIC if svc_vis else VISIBILITY_PRIVATE
payload = {
"processDescription": {
"process": {
"id": proc_id,
"visibility": svc_vis
}
},
"executionUnit": [{
"href": proc_url
}],
"deploymentProfileName":
"http://www.opengis.net/profiles/eoc/wpsApplication",
}
try:
resp = deploy_process_from_payload(payload, container)
if resp.status_code == HTTPOk.code:
LOGGER.info("Process registered: [%s]", proc_id)
else:
raise RuntimeError(
"Process registration failed: [{}]".format(
proc_id))
except Exception as ex:
LOGGER.exception(
"Exception during process registration: [%r]. Skipping...",
ex)
continue
# direct WPS providers to register
for cfg_service in providers:
svc_name, svc_url, _, svc_vis = parse_wps_process_config(
cfg_service)
LOGGER.info("Register WPS-1 provider: [%s]", svc_url)
WebProcessingService(
url=svc_url) # only attempt fetch to validate it exists
try:
service_store.fetch_by_name(svc_name)
except ServiceNotFound:
pass
else:
LOGGER.warning(
"Provider already registered: [%s]. Skipping...", svc_name)
continue
try:
service_store.save_service(
Service(name=svc_name, url=svc_url, public=svc_vis))
except Exception as ex:
LOGGER.exception(
"Exception during provider registration: [%r]. Skipping...",
ex)
continue
LOGGER.info("Finished processing configuration file [%s].",
wps_processes_file_path)
except Exception as exc:
msg = "Invalid WPS-1 providers configuration file [{!r}].".format(exc)
LOGGER.exception(msg)
raise RuntimeError(msg)
class WPSClient(object):
"""Returns a class where every public method is a WPS process available at
the given url.
Example:
>>> emu = WPSClient(url='<server url>')
>>> emu.hello('stranger')
'Hello stranger'
"""
def __init__(
self,
url,
processes=None,
converters=None,
username=None,
password=None,
headers=None,
verify=True,
cert=None,
verbose=False,
progress=False,
version=WPS_DEFAULT_VERSION,
):
"""
Args:
url (str): Link to WPS provider. config (Config): an instance
processes: Specify a subset of processes to bind. Defaults to all
processes.
converters (dict): Correspondence of {mimetype: class} to convert
this mimetype to a python object.
username (str): passed to :class:`owslib.wps.WebProcessingService`
password (str): passed to :class:`owslib.wps.WebProcessingService`
headers (str): passed to :class:`owslib.wps.WebProcessingService`
verify (bool): passed to :class:`owslib.wps.WebProcessingService`
cert (str): passed to :class:`owslib.wps.WebProcessingService`
verbose (str): passed to :class:`owslib.wps.WebProcessingService`
progress (bool): If True, enable interactive user mode.
version (str): WPS version to use.
"""
self._converters = converters
self._interactive = progress
self._mode = ASYNC if progress else SYNC
self._notebook = notebook.is_notebook()
self._inputs = {}
self._outputs = {}
if not verify:
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self._wps = WebProcessingService(
url,
version=version,
username=username,
password=password,
verbose=verbose,
headers=headers,
verify=verify,
cert=cert,
skip_caps=True,
)
try:
self._wps.getcapabilities()
except ServiceException as e:
if "AccessForbidden" in str(e):
raise UnauthorizedException(
"You are not authorized to do a request of type: GetCapabilities"
)
raise
self._processes = self._get_process_description(processes)
# Build the methods
for pid in self._processes:
setattr(self, sanitize(pid), types.MethodType(self._method_factory(pid), self))
self.logger = logging.getLogger('WPSClient')
if progress:
self._setup_logging()
self.__doc__ = utils.build_wps_client_doc(self._wps, self._processes)
def _get_process_description(self, processes):
"""Return the description for each process.
Sends the server a `describeProcess` request for each process.
Parameters
----------
processes: str, list, None
A process name, a list of process names or None (for all processes).
Returns
-------
OrderedDict
A dictionary keyed by the process identifier of process descriptions.
"""
all_wps_processes = [p.identifier for p in self._wps.processes]
if processes is None:
if owslib.__version__ > '0.17.0':
# Get the description for all processes in one request.
ps = self._wps.describeprocess('all')
return OrderedDict((p.identifier, p) for p in ps)
else:
processes = all_wps_processes
# Check for invalid process names, i.e. not matching the getCapabilities response.
process_names, missing = utils.filter_case_insensitive(
processes, all_wps_processes)
if missing:
message = "These process names were not found on the WPS server: {}"
raise ValueError(message.format(", ".join(missing)))
# Get the description for each process.
ps = [self._wps.describeprocess(pid) for pid in process_names]
return OrderedDict((p.identifier, p) for p in ps)
def _setup_logging(self):
self.logger.setLevel(logging.INFO)
import sys
fh = logging.StreamHandler(sys.stdout)
fh.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
self.logger.addHandler(fh)
def _method_factory(self, pid):
"""Create a custom function signature with docstring, instantiate it and
pass it to a wrapper which will actually call the process.
Parameters
----------
pid: str
Identifier of the WPS process.
Returns
-------
func
A Python function calling the remote process, complete with docstring and signature.
"""
process = self._processes[pid]
input_defaults = OrderedDict()
for inpt in process.dataInputs:
iid = sanitize(inpt.identifier)
default = getattr(inpt, "defaultValue", None) if inpt.dataType != 'ComplexData' else None
input_defaults[iid] = utils.from_owslib(default, inpt.dataType)
body = dedent("""
inputs = locals()
inputs.pop('self')
return self._execute('{pid}', **inputs)
""").format(pid=pid)
func_builder = FunctionBuilder(
name=sanitize(pid),
doc=utils.build_process_doc(process),
args=["self"] + list(input_defaults),
defaults=tuple(input_defaults.values()),
body=body,
filename=__file__,
module=self.__module__,
)
self._inputs[pid] = {}
if hasattr(process, "dataInputs"):
self._inputs[pid] = OrderedDict(
(i.identifier, i) for i in process.dataInputs
)
self._outputs[pid] = {}
if hasattr(process, "processOutputs"):
self._outputs[pid] = OrderedDict(
(o.identifier, o) for o in process.processOutputs
)
func = func_builder.get_func()
return func
def _execute(self, pid, **kwargs):
"""Execute the process."""
wps_inputs = []
for name, input_param in self._inputs[pid].items():
value = kwargs.get(sanitize(name))
if value is not None:
if isinstance(input_param.defaultValue, ComplexData):
encoding = input_param.defaultValue.encoding
mimetype = input_param.defaultValue.mimeType
if isinstance(value, ComplexData):
inp = value
else:
if utils.is_embedded_in_request(self._wps.url, value):
# If encoding is None, this will return the actual encoding used (utf-8 or base64).
value, encoding = embed(value, mimetype, encoding=encoding)
else:
value = fix_url(value)
inp = utils.to_owslib(value,
data_type=input_param.dataType,
encoding=encoding,
mimetype=mimetype)
else:
inp = utils.to_owslib(value, data_type=input_param.dataType)
wps_inputs.append((name, inp))
wps_outputs = [
(o.identifier, "ComplexData" in o.dataType)
for o in self._outputs[pid].values()
]
mode = self._mode if self._processes[pid].storeSupported else SYNC
try:
wps_response = self._wps.execute(
pid, inputs=wps_inputs, output=wps_outputs, mode=mode
)
if self._interactive and self._processes[pid].statusSupported:
if self._notebook:
notebook.monitor(wps_response, sleep=.2)
else:
self._console_monitor(wps_response)
except ServiceException as e:
if "AccessForbidden" in str(e):
raise UnauthorizedException(
"You are not authorized to do a request of type: Execute"
)
raise
# Add the convenience methods of WPSResult to the WPSExecution class. This adds a `get` method.
utils.extend_instance(wps_response, WPSResult)
wps_response.attach(wps_outputs=self._outputs[pid], converters=self._converters)
return wps_response
def _console_monitor(self, execution, sleep=3):
"""Monitor the execution of a process.
Parameters
----------
execution : WPSExecution instance
The execute response to monitor.
sleep: float
Number of seconds to wait before each status check.
"""
import signal
# Intercept CTRL-C
def sigint_handler(signum, frame):
self.cancel()
signal.signal(signal.SIGINT, sigint_handler)
while not execution.isComplete():
execution.checkStatus(sleepSecs=sleep)
self.logger.info("{} [{}/100] - {} ".format(
execution.process.identifier,
execution.percentCompleted,
execution.statusMessage[:50],))
if execution.isSucceded():
self.logger.info("{} done.".format(execution.process.identifier))
else:
self.logger.info("{} failed.".format(execution.process.identifier))
verbose=verbose,
skip_caps=True)
# 1) GetCapabilities
wps.getcapabilities()
print('WPS Identification type: %s' % wps.identification.type)
print('WPS Identification title: %s' % wps.identification.title)
print('WPS Identification abstract: %s' % wps.identification.abstract)
for operation in wps.operations:
print('WPS Operation: %s' % operation.name)
for process in wps.processes:
print('WPS Process: identifier=%s title=%s' %
(process.identifier, process.title))
# 2) DescribeProcess
process = wps.describeprocess('reprojectImage')
print('WPS Process: identifier=%s' % process.identifier)
print('WPS Process: title=%s' % process.title)
print('WPS Process: abstract=%s' % process.abstract)
for input in process.dataInputs:
print(
'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d'
% (input.identifier, input.dataType, input.minOccurs, input.maxOccurs))
for output in process.processOutputs:
print('Process output: identifier=%s, data type=%s' %
(output.identifier, output.dataType))
# 3a) Execute
# GET request: http://rsg.pml.ac.uk/wps/generic.cgi?request=Execute&service=wps&version=1.0.0&identifier=reprojectImage&datainputs=[inputImage=http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img;outputSRS=EPSG:4326]&responsedocument=outputImage=@asreference=true
processid = "reprojectImage"
inputs = [("inputImage",
class pyGDPwebProcessing():
"""
This class allows interactive calls to be made into the GDP.
"""
def _init_(self, wfsUrl=WFS_URL, wpsUrl=WPS_URL, version='1.1.0'):
self.wfsUrl = wfsUrl
self.wpsUrl = wpsUrl
self.version = version
self.wps = WebProcessingService(wpsUrl)
def WPSgetCapbilities(self, xml=None):
"""
Returns a list of capabilities.
"""
self.wps.getcapabilities(xml)
def WPSdescribeprocess(self, identifier, xml=None):
"""
Returns a list describing a specific identifier/process.
"""
self.wps.describeprocess(identifier, xml)
def _encodeZipFolder(self, filename):
"""
This function will encode a zipfile and return the filename.
"""
#check extension
if not filename.endswith('.zip'):
raise Exception('Wrong filetype.')
#encode the file
with open(filename, 'rb') as fin:
bytesRead = fin.read()
encode= base64.b64encode(bytesRead)
#renames the file and saves it onto local drive
filename = filename.split('.')
filename = str(filename[0]) + '_copy.' + str(filename[-1])
fout = open(filename, "w")
fout.write(encode)
fout.close()
return filename
def shapeToZip(self,inShape, outZip=None, allFiles=True):
"""Packs a shapefile to ZIP format.
arguments
-inShape - input shape file
-outZip - output ZIP file (optional)
default: <inShapeName>.zip in same folder as inShape
(If full path not specified, output is written to
to same folder as inShape)
-allFiles - Include all files? (optional)
True (default) - all shape file components
False - just .shp,.shx,.dbf,.prj,shp.xml files
reference: Esri, Inc, 1998, Esri Shapefile Technical Description
http://www.esri.com/library/whitepapers/pdfs/shapefile.pdf
author: Curtis Price, [email protected]"""
if not os.path.splitext(inShape)[1] == ".shp":
raise Exception, "inShape must be a *.shp"
if not os.path.exists(inShape):
raise Exception, "%s not found" % inShape
# get shapefile root name "path/file.shp" -> "file"
# and shapefile path
rootName = os.path.splitext(os.path.basename(inShape))[0]
inShape = os.path.realpath(inShape)
inDir = os.path.dirname(inShape)
# output zip file path
if outZip in [None,""]:
# default output: shapefilepath/shapefilename.zip
outDir = inDir
outZip = os.path.join(outDir,rootName) + ".zip"
else:
outDir = os.path.dirname(outZip)
if outDir.strip() in ["","."]:
# if full path not specified, use input shapefile folder
outDir = os.path.dirname(os.path.realpath(inShape))
else:
# if output path does exist, raise an exception
if not os.path.exists(outDir):
raise Exception, "Output folder %s not found" % outDir
outZip = os.path.join(outDir,outZip)
# enforce .zip extension
outZip = os.path.splitext(outZip)[0] + ".zip"
if not os.access(outDir, os.W_OK):
raise Exception, "Output directory %s not writeable" % outDir
if os.path.exists(outZip):
os.unlink(outZip)
try:
# open zipfile
zf = zipfile.ZipFile(outZip, 'w', zipfile.ZIP_DEFLATED)
# write shapefile parts to zipfile
ShapeExt = ["shp","shx","dbf","prj","shp.xml"]
if allFiles: ShapeExt += ["sbn","sbx","fbn","fbx",
"ain","aih","isx","mxs","atx","cpg"]
for f in ["%s.%s" % (os.path.join(inDir,rootName),ext)
for ext in ShapeExt]:
if os.path.exists(f):
zf.write(f)
##print f # debug print
return outZip
except Exception, msg:
raise Exception, \
"Could not write zipfile " + outZip + "\n" + str(msg)
finally:
# 1) GetCapabilities
# GET request: http://ceda-wps2.badc.rl.ac.uk/wps?Service=WPS&Request=GetCapabilities&Format=text/xml
wps.getcapabilities()
print('WPS Identification type: %s' % wps.identification.type)
print('WPS Identification title: %s' % wps.identification.title)
print('WPS Identification abstract: %s' % wps.identification.abstract)
for operation in wps.operations:
print('WPS Operation: %s' % operation.name)
for process in wps.processes:
print('WPS Process: identifier=%s title=%s' %
(process.identifier, process.title))
# 2) DescribeProcess
# GET request: http://ceda-wps2.badc.rl.ac.uk/wps?identifier=DoubleIt&version=1.0.0&request=DescribeProcess&service=WPS
process = wps.describeprocess('DoubleIt')
print('WPS Process: identifier=%s' % process.identifier)
print('WPS Process: title=%s' % process.title)
print('WPS Process: abstract=%s' % process.abstract)
for input in process.dataInputs:
print('Process input:')
printInputOutput(input, indent='\t')
for output in process.processOutputs:
print('Process output:')
printInputOutput(output, indent='\t')
# 3) Execute
# POST request:
# Note: not working, requires openid login ?
#processid = "DoubleIt"
#inputs = [ ("NumberToDouble","1") ]
# instantiate WPS client
verbose = False
wps = WebProcessingService('http://rsg.pml.ac.uk/wps/vector.cgi', verbose=verbose, skip_caps=True)
# 1) GetCapabilities
wps.getcapabilities()
print 'WPS Identification type: %s' % wps.identification.type
print 'WPS Identification title: %s' % wps.identification.title
print 'WPS Identification abstract: %s' % wps.identification.abstract
for operation in wps.operations:
print 'WPS Operation: %s' % operation.name
for process in wps.processes:
print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title)
# 2) DescribeProcess
process = wps.describeprocess('v.net.path')
# alternatively, read process description from XML file (no live request to WPS server)
#xml = open('../tests/USGSDescribeProcess.xml', 'r').read()
#process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml)
print 'WPS Process: identifier=%s' % process.identifier
print 'WPS Process: title=%s' % process.title
print 'WPS Process: abstract=%s' % process.abstract
for input in process.dataInputs:
print 'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d' % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs)
for output in process.processOutputs:
print 'Process output: identifier=%s, data type=%s' % (output.identifier, output.dataType)
# 3) Execute
# GET request: http://rsg.pml.ac.uk/wps/vector.cgi?request=execute&service=WPS&version=1.0.0&identifier=v.net.path&datainputs=[input=http://rsg.pml.ac.uk/wps/example/graph.gml;file=1%20-960123.1421801624%204665723.56559387%20-101288.65106088226%205108200.011823481]
processid = "v.net.path"
inputs = [ ("input","http://rsg.pml.ac.uk/wps/example/graph.gml"),
# instantiate WPS client
verbose = False
wps = WebProcessingService('http://rsg.pml.ac.uk/wps/generic.cgi', verbose=verbose)
# 1) GetCapabilities
wps.getcapabilities()
print 'WPS Identification type: %s' % wps.identification.type
print 'WPS Identification title: %s' % wps.identification.title
print 'WPS Identification abstract: %s' % wps.identification.abstract
for operation in wps.operations:
print 'WPS Operation: %s' % operation.name
for process in wps.processes:
print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title)
# 2) DescribeProcess
process = wps.describeprocess('reprojectImage')
print 'WPS Process: identifier=%s' % process.identifier
print 'WPS Process: title=%s' % process.title
print 'WPS Process: abstract=%s' % process.abstract
for input in process.dataInputs:
print 'Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d' % (input.identifier, input.dataType, input.minOccurs, input.maxOccurs)
for output in process.processOutputs:
print 'Process output: identifier=%s, data type=%s' % (output.identifier, output.dataType)
# 3a) Execute
# GET request: http://rsg.pml.ac.uk/wps/generic.cgi?request=Execute&service=wps&version=1.0.0&identifier=reprojectImage&datainputs=[inputImage=http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img;outputSRS=EPSG:4326]&responsedocument=outputImage=@asreference=true
processid = "reprojectImage"
inputs = [ ("inputImage","http://rsg.pml.ac.uk/wps/testdata/elev_srtm_30m.img"),
("outputSRS", "EPSG:4326") ]
output = "outputImage"
execution = wps.execute(processid, inputs, output)
if request == "GetCapabilities":
wps.getcapabilities()
print("WPS Identification type: %s" % wps.identification.type)
print("WPS Identification title: %s" % wps.identification.title)
print("WPS Identification abstract: %s" % wps.identification.abstract)
for operation in wps.operations:
print("WPS Operation: %s" % operation.name)
for process in wps.processes:
print("WPS Process: identifier=%s title=%s" % (process.identifier, process.title))
elif request == "DescribeProcess":
if identifier is None:
print('\nERROR: missing mandatory "-i (or --identifier)" argument')
usage()
sys.exit(4)
process = wps.describeprocess(identifier)
print("WPS Process: identifier=%s" % process.identifier)
print("WPS Process: title=%s" % process.title)
print("WPS Process: abstract=%s" % process.abstract)
for input in process.dataInputs:
print(
"Process input: identifier=%s, data type=%s, minOccurs=%d, maxOccurs=%d"
% (input.identifier, input.dataType, input.minOccurs, input.maxOccurs)
)
for output in process.processOutputs:
print("Process output: identifier=%s, data type=%s" % (output.identifier, output.dataType))
elif request == "Execute":
if xml is None:
print('\nERROR: missing mandatory "-x (or --xml)" argument')
usage()
return ref
# Hummingbird WPS url
wps_url = 'https://pavics.ouranos.ca/twitcher/ows/proxy/hummingbird/wps'
# connection
wps = WebProcessingService(url=wps_url)
# print wps title
print(wps.identification.title)
for process in wps.processes:
print('%s \t : %s \n' % (process.identifier, process.abstract))
# ncdump
proc_name = 'ncdump'
process = wps.describeprocess(proc_name) # get process info
print(process.abstract)
print("Inputs:")
for inputs in process.dataInputs:
print(' * ', inputs.identifier)
# Example netcdf url to NRCAN daily - tasmin 2013
nc_url = 'https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/nrcan/nrcan_canada_daily/tasmin/nrcan_canada_daily_tasmin_2013.nc'
print(nc_url)
myinputs = []
myinputs.append(
('dataset_opendap', nc_url)
) # inputs : use opendap link of a single netcdf file from catalogue search to erun ncdump
print(myinputs)
class BirdyCLI(click.MultiCommand):
"""BirdyCLI is an implementation of :class:`click.MultiCommand`. It
adds each process of a Web Processing Service as command to the
command-line interface.
:param url: URL of the Web Processing Service.
:param xml: A WPS GetCapabilities response for testing.
"""
def __init__(self, name=None, url=None, xml=None, **attrs):
click.MultiCommand.__init__(self, name, **attrs)
self.url = os.environ.get('WPS_SERVICE') or url
self.verify = get_ssl_verify()
self.xml = xml
self.wps = WebProcessingService(self.url, verify=self.verify, skip_caps=True)
self.commands = OrderedDict()
def _update_commands(self):
if not self.commands:
try:
self.wps.getcapabilities(xml=self.xml)
except SSLError:
raise ConnectionError('SSL verfication of server certificate failed. Set WPS_SSL_VERIFY=false.')
except Exception:
raise ConnectionError("Web Processing Service not available.")
for process in self.wps.processes:
self.commands[process.identifier] = dict(
name=process.identifier,
url=self.wps.url,
version=process.processVersion,
help=BirdyCLI.format_command_help(process),
options=[])
def list_commands(self, ctx):
ctx.obj = True
self._update_commands()
return self.commands.keys()
def get_command(self, ctx, name):
self._update_commands()
cmd_templ = template_env.get_template('cmd.py.j2')
rendered_cmd = cmd_templ.render(self._get_command_info(name, details=ctx.obj is None or False))
ns = {}
code = compile(rendered_cmd, filename='<string>', mode='exec')
eval(code, ns, ns)
return ns['cli']
def _get_command_info(self, name, details=False):
cmd = self.commands.get(name)
if details:
pp = self.wps.describeprocess(name)
for inp in pp.dataInputs:
help = inp.title or ''
default = BirdyCLI.get_param_default(inp)
if default:
help = "{}. Default: {}".format(help, default)
cmd['options'].append(dict(
name=inp.identifier,
# default=BirdyCLI.get_param_default(inp),
help=help,
type=BirdyCLI.get_param_type(inp),
multiple=inp.maxOccurs > 1))
return cmd
@staticmethod
def format_command_help(process):
return "{}: {}".format(process.title or process.identifier, process.abstract or '')
@staticmethod
def get_param_default(param):
if 'ComplexData' in param.dataType:
# TODO: get default value of complex type
default = None
elif 'BoundingBoxData' in param.dataType:
# TODO: get default value of bbox
default = None
else:
default = getattr(param, 'defaultValue', None)
return default
@staticmethod
def get_param_type(param):
if param.dataType is None:
param_type = click.STRING
elif 'boolean' in param.dataType:
param_type = click.BOOL
elif 'integer' in param.dataType:
param_type = click.INT
elif 'float' in param.dataType:
param_type = click.FLOAT
elif 'ComplexData' in param.dataType:
param_type = COMPLEX
else:
param_type = click.STRING
return param_type
def execute(self, workflow_inputs, out_dir, expected_outputs):
self.update_status("Preparing execute request for remote WPS1 provider.",
REMOTE_JOB_PROGRESS_REQ_PREP, status.STATUS_RUNNING)
LOGGER.debug("Execute process WPS request for %s", self.process)
try:
try:
wps = WebProcessingService(url=self.provider, headers=self.cookies, verify=self.verify)
raise_on_xml_exception(wps._capabilities) # noqa: W0212
except Exception as ex:
raise OWSNoApplicableCode("Failed to retrieve WPS capabilities. Error: [{}].".format(str(ex)))
try:
process = wps.describeprocess(self.process)
except Exception as ex:
raise OWSNoApplicableCode("Failed to retrieve WPS process description. Error: [{}].".format(str(ex)))
# prepare inputs
complex_inputs = []
for process_input in process.dataInputs:
if WPS_COMPLEX_DATA in process_input.dataType:
complex_inputs.append(process_input.identifier)
# remove any 'null' input, should employ the 'default' of the remote WPS process
inputs_provided_keys = filter(lambda i: workflow_inputs[i] != "null", workflow_inputs)
wps_inputs = []
for input_key in inputs_provided_keys:
input_val = workflow_inputs[input_key]
# in case of array inputs, must repeat (id,value)
# in case of complex input (File), obtain location, otherwise get data value
if not isinstance(input_val, list):
input_val = [input_val]
input_values = []
for val in input_val:
if isinstance(val, dict):
val = val["location"]
# owslib only accepts strings, not numbers directly
if isinstance(val, (int, float)):
val = str(val)
if val.startswith("file://"):
# we need to host file starting with file:// scheme
val = self.host_file(val)
input_values.append(val)
# need to use ComplexDataInput structure for complex input
# TODO: BoundingBox not supported
for input_value in input_values:
if input_key in complex_inputs:
input_value = ComplexDataInput(input_value)
wps_inputs.append((input_key, input_value))
# prepare outputs
outputs = [(o.identifier, o.dataType == WPS_COMPLEX_DATA) for o in process.processOutputs
if o.identifier in expected_outputs]
self.update_status("Executing job on remote WPS1 provider.",
REMOTE_JOB_PROGRESS_EXECUTION, status.STATUS_RUNNING)
mode = EXECUTE_MODE_ASYNC
execution = wps.execute(self.process, inputs=wps_inputs, output=outputs, mode=mode, lineage=True)
if not execution.process and execution.errors:
raise execution.errors[0]
self.update_status("Monitoring job on remote WPS1 provider : [{0}]".format(self.provider),
REMOTE_JOB_PROGRESS_MONITORING, status.STATUS_RUNNING)
max_retries = 5
num_retries = 0
run_step = 0
job_id = "<undefined>"
while execution.isNotComplete() or run_step == 0:
if num_retries >= max_retries:
raise Exception("Could not read status document after {} retries. Giving up.".format(max_retries))
try:
execution = check_wps_status(location=execution.statusLocation, verify=self.verify,
sleep_secs=wait_secs(run_step))
job_id = execution.statusLocation.replace(".xml", "").split("/")[-1]
LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()),
execution.percentCompleted, execution.statusMessage,
execution.statusLocation))
self.update_status(get_job_log_msg(status=status.map_status(execution.getStatus()),
message=execution.statusMessage,
progress=execution.percentCompleted,
duration=None), # get if available
map_progress(execution.percentCompleted,
REMOTE_JOB_PROGRESS_MONITORING, REMOTE_JOB_PROGRESS_FETCH_OUT),
status.STATUS_RUNNING)
except Exception as exc:
num_retries += 1
LOGGER.debug("Exception raised: %r", exc)
sleep(1)
else:
num_retries = 0
run_step += 1
if not execution.isSucceded():
exec_msg = execution.statusMessage or "Job failed."
LOGGER.debug(get_log_monitor_msg(job_id, status.map_status(execution.getStatus()),
execution.percentCompleted, exec_msg, execution.statusLocation))
raise Exception(execution.statusMessage or "Job failed.")
self.update_status("Fetching job outputs from remote WPS1 provider.",
REMOTE_JOB_PROGRESS_FETCH_OUT, status.STATUS_RUNNING)
results = [ows2json_output(output, process) for output in execution.processOutputs]
for result in results:
result_id = get_any_id(result)
result_val = get_any_value(result)
if result_id in expected_outputs:
# This is where cwl expect the output file to be written
# TODO We will probably need to handle multiple output value...
dst_fn = "/".join([out_dir.rstrip("/"), expected_outputs[result_id]])
# TODO Should we handle other type than File reference?
resp = request_extra("get", result_val, allow_redirects=True, settings=self.settings)
LOGGER.debug("Fetching result output from [%s] to cwl output destination: [%s]", result_val, dst_fn)
with open(dst_fn, mode="wb") as dst_fh:
dst_fh.write(resp.content)
except Exception as exc:
exception_class = "{}.{}".format(type(exc).__module__, type(exc).__name__)
errors = "{0}: {1!s}".format(exception_class, exc)
LOGGER.exception(exc)
raise Exception(errors)
self.update_status("Execution on remote WPS1 provider completed.",
REMOTE_JOB_PROGRESS_COMPLETED, status.STATUS_SUCCEEDED)
class WPSClient(object):
"""Returns a class where every public method is a WPS process available at
the given url.
Example:
>>> emu = WPSClient(url='<server url>')
>>> emu.hello('stranger')
'Hello stranger'
"""
def __init__(
self,
url,
processes=None,
converters=None,
username=None,
password=None,
headers=None,
auth=None,
verify=True,
cert=None,
verbose=False,
progress=False,
version=WPS_DEFAULT_VERSION,
caps_xml=None,
desc_xml=None,
language=None,
):
"""
Args:
url (str): Link to WPS provider. config (Config): an instance
processes: Specify a subset of processes to bind. Defaults to all
processes.
converters (dict): Correspondence of {mimetype: class} to convert
this mimetype to a python object.
username (str): passed to :class:`owslib.wps.WebProcessingService`
password (str): passed to :class:`owslib.wps.WebProcessingService`
headers (str): passed to :class:`owslib.wps.WebProcessingService`
auth (requests.auth.AuthBase): requests-style auth class to authenticate,
see https://2.python-requests.org/en/master/user/authentication/
verify (bool): passed to :class:`owslib.wps.WebProcessingService`
cert (str): passed to :class:`owslib.wps.WebProcessingService`
verbose (str): passed to :class:`owslib.wps.WebProcessingService`
progress (bool): If True, enable interactive user mode.
version (str): WPS version to use.
language (str): passed to :class:`owslib.wps.WebProcessingService`
ex: 'fr-CA', 'en_US'.
"""
self._converters = converters
self._interactive = progress
self._mode = ASYNC if progress else SYNC
self._notebook = notebook.is_notebook()
self._inputs = {}
self._outputs = {}
if not verify:
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
if headers is None:
headers = {}
if auth is not None:
if isinstance(auth, tuple) and len(auth) == 2:
# special-case basic HTTP auth
auth = requests.auth.HTTPBasicAuth(*auth)
# We only need some headers from the requests.auth.AuthBase implementation
# We prepare a dummy request, call the auth object with it, and get its headers
dummy_request = requests.Request("get", "http://localhost")
r = auth(dummy_request.prepare())
auth_headers = ["Authorization", "Proxy-Authorization", "Cookie"]
headers.update(
{h: r.headers[h]
for h in auth_headers if h in r.headers})
self._wps = WebProcessingService(url,
version=version,
username=username,
password=password,
verbose=verbose,
headers=headers,
verify=verify,
cert=cert,
skip_caps=True,
language=language)
try:
self._wps.getcapabilities(xml=caps_xml)
except ServiceException as e:
if "AccessForbidden" in str(e):
raise UnauthorizedException(
"You are not authorized to do a request of type: GetCapabilities"
)
raise
self._processes = self._get_process_description(processes,
xml=desc_xml)
# Build the methods
for pid in self._processes:
setattr(self, sanitize(pid),
types.MethodType(self._method_factory(pid), self))
self.logger = logging.getLogger('WPSClient')
if progress:
self._setup_logging()
self.__doc__ = utils.build_wps_client_doc(self._wps, self._processes)
@property
def language(self):
return self._wps.language
@language.setter
def language(self, value):
self._wps.language = value
@property
def languages(self):
return self._wps.languages
def _get_process_description(self, processes=None, xml=None):
"""Return the description for each process.
Sends the server a `describeProcess` request for each process.
Parameters
----------
processes: str, list, None
A process name, a list of process names or None (for all processes).
Returns
-------
OrderedDict
A dictionary keyed by the process identifier of process descriptions.
"""
all_wps_processes = [p.identifier for p in self._wps.processes]
if processes is None:
if owslib.__version__ > '0.17.0':
# Get the description for all processes in one request.
ps = self._wps.describeprocess('all', xml=xml)
return OrderedDict((p.identifier, p) for p in ps)
else:
processes = all_wps_processes
# Check for invalid process names, i.e. not matching the getCapabilities response.
process_names, missing = utils.filter_case_insensitive(
processes, all_wps_processes)
if missing:
message = "These process names were not found on the WPS server: {}"
raise ValueError(message.format(", ".join(missing)))
# Get the description for each process.
ps = [self._wps.describeprocess(pid, xml=xml) for pid in process_names]
return OrderedDict((p.identifier, p) for p in ps)
def _setup_logging(self):
self.logger.setLevel(logging.INFO)
import sys
fh = logging.StreamHandler(sys.stdout)
fh.setFormatter(logging.Formatter('%(asctime)s: %(message)s'))
self.logger.addHandler(fh)
def _method_factory(self, pid):
"""Create a custom function signature with docstring, instantiate it and
pass it to a wrapper which will actually call the process.
Parameters
----------
pid: str
Identifier of the WPS process.
Returns
-------
func
A Python function calling the remote process, complete with docstring and signature.
"""
process = self._processes[pid]
required_inputs_first = sorted(process.dataInputs, key=sort_inputs_key)
input_names = []
# defaults will be set to the function's __defaults__:
# A tuple containing default argument values for those arguments that have defaults,
# or None if no arguments have a default value.
defaults = []
for inpt in required_inputs_first:
input_names.append(sanitize(inpt.identifier))
if inpt.minOccurs == 0 or inpt.defaultValue is not None:
default = inpt.defaultValue if inpt.dataType != "ComplexData" else None
defaults.append(utils.from_owslib(default, inpt.dataType))
defaults = tuple(defaults) if defaults else None
body = dedent("""
inputs = locals()
inputs.pop('self')
return self._execute('{pid}', **inputs)
""").format(pid=pid)
func_builder = FunctionBuilder(
name=sanitize(pid),
doc=utils.build_process_doc(process),
args=["self"] + input_names,
defaults=defaults,
body=body,
filename=__file__,
module=self.__module__,
)
self._inputs[pid] = {}
if hasattr(process, "dataInputs"):
self._inputs[pid] = OrderedDict(
(i.identifier, i) for i in process.dataInputs)
self._outputs[pid] = {}
if hasattr(process, "processOutputs"):
self._outputs[pid] = OrderedDict(
(o.identifier, o) for o in process.processOutputs)
func = func_builder.get_func()
return func
def _build_inputs(self, pid, **kwargs):
"""Build the input sequence from the function arguments."""
wps_inputs = []
for name, input_param in list(self._inputs[pid].items()):
arg = kwargs.get(sanitize(name))
if arg is None:
continue
values = [
arg,
] if not isinstance(arg, (list, tuple)) else arg
supported_mimetypes = [
v.mimeType for v in input_param.supportedValues
]
for value in values:
# if input_param.dataType == "ComplexData": seems simpler
if isinstance(input_param.defaultValue, ComplexData):
# Guess the mimetype of the input value
mimetype, encoding = guess_type(value, supported_mimetypes)
if encoding is None:
encoding = input_param.defaultValue.encoding
if isinstance(value, ComplexData):
inp = value
# Either embed the file content or just the reference.
else:
if utils.is_embedded_in_request(self._wps.url, value):
# If encoding is None, this will return the actual encoding used (utf-8 or base64).
value, encoding = embed(value,
mimetype,
encoding=encoding)
else:
value = fix_url(str(value))
inp = utils.to_owslib(value,
data_type=input_param.dataType,
encoding=encoding,
mimetype=mimetype)
else:
inp = utils.to_owslib(value,
data_type=input_param.dataType)
wps_inputs.append((name, inp))
return wps_inputs
def _execute(self, pid, **kwargs):
"""Execute the process."""
wps_inputs = self._build_inputs(pid, **kwargs)
wps_outputs = [(o.identifier, "ComplexData" in o.dataType)
for o in list(self._outputs[pid].values())]
mode = self._mode if self._processes[pid].storeSupported else SYNC
try:
wps_response = self._wps.execute(pid,
inputs=wps_inputs,
output=wps_outputs,
mode=mode)
if self._interactive and self._processes[pid].statusSupported:
if self._notebook:
notebook.monitor(wps_response, sleep=.2)
else:
self._console_monitor(wps_response)
except ServiceException as e:
if "AccessForbidden" in str(e):
raise UnauthorizedException(
"You are not authorized to do a request of type: Execute")
raise
# Add the convenience methods of WPSResult to the WPSExecution class. This adds a `get` method.
utils.extend_instance(wps_response, WPSResult)
wps_response.attach(wps_outputs=self._outputs[pid],
converters=self._converters)
return wps_response
def _console_monitor(self, execution, sleep=3):
"""Monitor the execution of a process.
Parameters
----------
execution : WPSExecution instance
The execute response to monitor.
sleep: float
Number of seconds to wait before each status check.
"""
import signal
# Intercept CTRL-C
def sigint_handler(signum, frame):
self.cancel()
signal.signal(signal.SIGINT, sigint_handler)
while not execution.isComplete():
execution.checkStatus(sleepSecs=sleep)
self.logger.info("{} [{}/100] - {} ".format(
execution.process.identifier,
execution.percentCompleted,
execution.statusMessage[:50],
))
if execution.isSucceded():
self.logger.info("{} done.".format(execution.process.identifier))
else:
self.logger.info("{} failed.".format(execution.process.identifier))
def execute(self):
Outputs={}
sys.stdout = open(config.getConfigValue("server","logFile"),'w')
try:
wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True)
identifier = 'dummyprocess'
description = wps.describeprocess(identifier)
wps_outputs=[]
for item in description.processOutputs:
if item.dataType == 'ComplexOutput':
wps_outputs.append((item.identifier, True))
else: wps_outputs.append((item.identifier, False))
inputs = [('i1',str(self.Input1.getValue())),('i2',str(self.Input2.getValue())),('i3',str(self.Input3.getValue()))]
execution = wps.execute(identifier, inputs, output=wps_outputs)
monitorExecution(execution)
execution.checkStatus(sleepSecs=1)
Outputs['dummyprocess_1']={}
if "ComplexData" in description.processOutputs:
i=0
for output in execution.processOutputs:
if description.processOutputs[i].dataType == 'ComplexData':
Outputs['dummyprocess_1'][output.identifier]=output.reference
else: Outputs['dummyprocess_1'][output.identifier]=output.data[0]
i=i+1
else:
for output in execution.processOutputs:
Outputs['dummyprocess_1'][output.identifier]=output.data[0]
x= Outputs['dummyprocess_1' ]['output1']
except:
wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True)
identifier = 'dummyprocess'
description = wps.describeprocess(identifier)
wps_outputs=[]
for item in description.processOutputs:
if item.dataType == 'ComplexOutput':
wps_outputs.append((item.identifier, True))
else: wps_outputs.append((item.identifier, False))
inputs = [('i1',str(self.Input1.getValue())),('i2',str(self.Input2.getValue())),('i3',str(self.Input3.getValue()))]
execution = wps.execute(identifier, inputs, output=wps_outputs)
monitorExecution(execution)
execution.checkStatus(sleepSecs=1)
Outputs['dummyprocess_1']={}
if "ComplexData" in description.processOutputs:
i=0
for output in execution.processOutputs:
if description.processOutputs[i].dataType == 'ComplexData':
Outputs['dummyprocess_1'][output.identifier]=output.reference
else: Outputs['dummyprocess_1'][output.identifier]=output.data[0]
i=i+1
else:
for output in execution.processOutputs:
Outputs['dummyprocess_1'][output.identifier]=output.data[0]
x= Outputs['dummyprocess_1' ]['output1']
try:
wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True)
identifier = 'dummyprocess'
description = wps.describeprocess(identifier)
wps_outputs=[]
for item in description.processOutputs:
if item.dataType == 'ComplexOutput':
wps_outputs.append((item.identifier, True))
else: wps_outputs.append((item.identifier, False))
inputs = [('i1',Outputs['dummyprocess_1']['output1']),('i2',x),('i3',Outputs['dummyprocess_1']['output2'])]
execution = wps.execute(identifier, inputs)
Outputs['dummyprocess_2']={}
if "ComplexData" in description.processOutputs:
i=0
for output in execution.processOutputs:
if description.processOutputs[i].dataType == 'ComplexData':
Outputs['dummyprocess_2'][output.identifier]=output.reference
else: Outputs['dummyprocess_2'][output.identifier]=output.data[0]
i=i+1
else:
for output in execution.processOutputs:
Outputs['dummyprocess_2'][output.identifier]=output.data[0]
except:
wps = WebProcessingService('http://localhost/cgi-bin/pywps.cgi', verbose=True, skip_caps=True)
identifier = 'dummyprocess'
description = wps.describeprocess(identifier)
wps_outputs=[]
for item in description.processOutputs:
if item.dataType == 'ComplexOutput':
wps_outputs.append((item.identifier, True))
else: wps_outputs.append((item.identifier, False))
inputs = [('i1',Outputs['dummyprocess_1']['output1']),('i2',x),('i3',Outputs['dummyprocess_1']['output2'])]
execution = wps.execute(identifier, inputs)
Outputs['dummyprocess_2']={}
if "ComplexData" in description.processOutputs:
i=0
for output in execution.processOutputs:
if description.processOutputs[i].dataType == 'ComplexData':
Outputs['dummyprocess_2'][output.identifier]=output.reference
else: Outputs['dummyprocess_2'][output.identifier]=output.data[0]
i=i+1
else:
for output in execution.processOutputs:
Outputs['dummyprocess_2'][output.identifier]=output.data[0]
self.dummyprocess_1_output1.setValue(Outputs['dummyprocess_1']['output1'])
self.dummyprocess_2_output2.setValue(Outputs['dummyprocess_2']['output2'])
self.x.setValue(x)
sys.stdout.close()
sys.stdout = sys.__stdout__
return
class GenericWPS(MonitorPE):
STATUS_NAME = 'status'
STATUS_LOCATION_NAME = 'status_location'
def __init__(self, url, identifier, resource='resource',
inputs=[], output=None, headers=None):
MonitorPE.__init__(self)
self._add_output(self.STATUS_NAME)
self._add_output(self.STATUS_LOCATION_NAME)
self.wps = WebProcessingService(url=url, skip_caps=True, verify=False, headers=headers)
self.identifier = identifier
self.wps_resource = resource
self.wps_inputs = inputs
self.wps_output = output
def progress(self, execution):
return int(self._pstart + ((self._pend - self._pstart) / 100.0 * execution.percentCompleted))
def monitor_execution(self, execution):
progress = self.progress(execution)
self.monitor(
"status_location={0.statusLocation}".format(execution), progress)
while execution.isNotComplete():
try:
execution.checkStatus(sleepSecs=3)
except Exception:
LOGGER.exception("Could not read status xml document.")
else:
progress = self.progress(execution)
self.monitor(execution.statusMessage, progress)
if execution.isSucceded():
for output in execution.processOutputs:
self.monitor('ouput={0.identifier}'.format(output), progress)
else:
self.monitor('\n'.join(
['ERROR: {0.text} code={0.code} locator={0.locator})'.
format(ex) for ex in execution.errors]), progress)
def _build_wps_inputs(self):
process = self.wps.describeprocess(self.identifier)
complex_inpts = []
bbox_inpts = []
for inpt in process.dataInputs:
if 'ComplexData' in inpt.dataType:
complex_inpts.append(inpt.identifier)
elif 'BoundingBoxData' in inpt.dataType:
bbox_inpts.append(inpt.identifier)
inputs = []
for inpt in self.wps_inputs:
LOGGER.debug("input=%s", inpt)
if inpt[0] in complex_inpts:
inputs.append((inpt[0], ComplexDataInput(inpt[1])))
elif inpt[0] in bbox_inpts:
inputs.append((inpt[0], BoundingBoxDataInput(inpt[1])))
else:
inputs.append(inpt)
return inputs
def _build_wps_outputs(self):
outputs = []
if self.wps_output is not None:
outputs = [(self.wps_output, True)]
return outputs
def execute(self):
LOGGER.debug("execute inputs=%s", self.wps_inputs)
execution = self.wps.execute(
identifier=self.identifier,
inputs=self._build_wps_inputs(),
output=self._build_wps_outputs(),
lineage=True)
self.monitor_execution(execution)
result = {self.STATUS_NAME: execution.status,
self.STATUS_LOCATION_NAME: execution.statusLocation}
if execution.isSucceded():
# NOTE: only set workflow output if specific output was requested
if self.wps_output is not None:
for output in execution.processOutputs:
if self.wps_output == output.identifier:
result[self.OUTPUT_NAME] = output.reference
break
return result
else:
failure_msg = '\n'.join(['{0.text}'.
format(ex) for ex in execution.errors])
raise Exception(failure_msg)
def _set_inputs(self, inputs):
if self.INPUT_NAME in inputs:
for value in inputs[self.INPUT_NAME]:
self.wps_inputs.append((self.wps_resource, value))
def process(self, inputs):
try:
result = self._process(inputs)
if result is not None:
return result
except Exception:
LOGGER.exception("process failed!")
raise
def _process(self, inputs):
self._set_inputs(inputs)
return self.execute()
wps.getcapabilities()
# alternatively, read capabilities from XML file (no live request to WPS server)
#xml = open('../tests/USGSCapabilities.xml', 'r').read()
#wps.getcapabilities(xml=xml)
print 'WPS Identification type: %s' % wps.identification.type
print 'WPS Identification title: %s' % wps.identification.title
print 'WPS Identification abstract: %s' % wps.identification.abstract
for operation in wps.operations:
print 'WPS Operation: %s' % operation.name
for process in wps.processes:
print 'WPS Process: identifier=%s title=%s' % (process.identifier, process.title)
# 2) DescribeProcess
# Submits an HTTP GET "DescribeProcess" request to the WPS service and parses the HTTP response
process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm')
# alternatively, read process description from XML file (no live request to WPS server)
#xml = open('../tests/USGSDescribeProcess.xml', 'r').read()
#process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml)
print 'WPS Process: identifier=%s' % process.identifier
print 'WPS Process: title=%s' % process.title
print 'WPS Process: abstract=%s' % process.abstract
for input in process.dataInputs:
print 'Process input:'
printInputOutput(input, indent='\t')
for output in process.processOutputs:
print 'Process output:'
printInputOutput(output, indent='\t')
# 3a) Execute
# Submits an HTTP POST "Execute" process request to the WPS service, keeps checking the status of the request,
class ExecuteProcess(MyView):
def __init__(self, request):
self.request = request
self.execution = None
self.service_name = None
self.processid = None
self.process = None
if 'job_id' in request.params:
job = request.db.jobs.find_one(
{'identifier': request.params['job_id']})
self.service_name = job.get('service_name')
self.execution = check_status(url=job.get('status_location'),
response=job.get('response'),
verify=False,
sleep_secs=0)
self.processid = self.execution.process.identifier
elif 'wps' in request.params:
self.service_name = request.params.get('wps')
self.processid = request.params.get('process')
if self.service_name:
# TODO: avoid getcaps
self.wps = WebProcessingService(url=request.route_url(
'owsproxy', service_name=self.service_name),
verify=False)
# TODO: need to fix owslib to handle special identifiers
self.process = self.wps.describeprocess(self.processid)
super(ExecuteProcess, self).__init__(request,
name='processes_execute',
title='')
def breadcrumbs(self):
breadcrumbs = super(ExecuteProcess, self).breadcrumbs()
breadcrumbs.append(
dict(route_path=self.request.route_path('processes'),
title='Processes'))
breadcrumbs.append(
dict(route_path=self.request.route_path('processes_list',
_query=[('wps',
self.service_name)
]),
title=self.service_name))
breadcrumbs.append(
dict(route_path=self.request.route_path(self.name),
title=self.process.identifier))
return breadcrumbs
def appstruct(self):
# TODO: not a nice way to get inputs ... should be cleaned up in owslib
result = {}
if self.execution:
for inp in self.execution.dataInputs:
if inp.data or inp.reference:
if inp.identifier not in result:
# init result for param with empty list
result[inp.identifier] = []
if inp.data:
# add literal input, inp.data is a list
result[inp.identifier].extend(inp.data)
elif inp.reference:
# add reference to complex input
result[inp.identifier].append(inp.reference)
for inp in self.process.dataInputs:
# TODO: dupliate code in wizard.start
# convert boolean
if 'boolean' in inp.dataType and inp.identifier in result:
result[inp.identifier] = [
val.lower() == 'true' for val in result[inp.identifier]
]
elif 'dateTime' in inp.dataType and inp.identifier in result:
result[inp.identifier] = [
dateparser.parse(val) for val in result[inp.identifier]
]
elif 'date' in inp.dataType and inp.identifier in result:
result[inp.identifier] = [
dateparser.parse(val) for val in result[inp.identifier]
]
elif 'time' in inp.dataType and inp.identifier in result:
result[inp.identifier] = [
dateparser.parse(val) for val in result[inp.identifier]
]
elif inp.dataType == 'BoundingBoxData' and inp.identifier in result:
result[inp.identifier] = [
"{0.minx},{0.miny},{0.maxx},{0.maxy}".format(bbox)
for bbox in result[inp.identifier]
]
# TODO: very dirty ... if single value then take the first
if inp.maxOccurs < 2 and inp.identifier in result:
result[inp.identifier] = result[inp.identifier][0]
return result
def generate_form(self, formid='deform'):
schema = WPSSchema(request=self.request,
process=self.process,
use_async=self.request.has_permission('admin'),
user=self.request.user)
submit_button = Button(name='submit',
title='Submit',
css_class='btn btn-success btn-lg btn-block',
disabled=not has_execute_permission(
self.request, self.service_name))
return Form(
schema.bind(request=self.request),
buttons=(submit_button, ),
formid=formid,
)
def process_form(self, form):
controls = list(self.request.POST.items())
try:
# TODO: uploader puts qqfile in controls
controls = [
control for control in controls if 'qqfile' not in control[0]
]
LOGGER.debug("before validate %s", controls)
appstruct = form.validate(controls)
LOGGER.debug("before execute %s", appstruct)
job_id = self.execute(appstruct)
except ValidationFailure as e:
self.session.flash("Page validation failed.", queue='danger')
return dict(process=self.process,
url=wps_describe_url(self.wps.url, self.processid),
form=e.render())
else:
if not self.request.user: # not logged-in
return HTTPFound(location=self.request.route_url(
'job_status', job_id=job_id))
else:
return HTTPFound(location=self.request.route_url('monitor'))
def execute(self, appstruct):
inputs = appstruct_to_inputs(self.request, appstruct)
# need to use ComplexDataInput
complex_inpts = {}
bbox_inpts = []
for inpt in self.process.dataInputs:
if 'ComplexData' in inpt.dataType:
complex_inpts[inpt.identifier] = inpt
elif 'BoundingBoxData' in inpt.dataType:
bbox_inpts.append(inpt.identifier)
new_inputs = []
for inpt in inputs:
identifier = inpt[0]
value = inpt[1]
if identifier in complex_inpts:
new_inputs.append((identifier, ComplexDataInput(value)))
if is_reference(value):
if value not in self.request.cart:
if complex_inpts[identifier].supportedValues:
mime_type = complex_inpts[
identifier].supportedValues[0].mimeType
else:
mime_type = None
LOGGER.debug("add input to cart: %s %s", identifier,
mime_type)
self.request.cart.add_item(
value,
abstract=
"Automatically added in process execution.",
mime_type=mime_type)
elif identifier in bbox_inpts:
new_inputs.append((identifier, BoundingBoxDataInput(value)))
else:
new_inputs.append(inpt)
inputs = new_inputs
# prepare outputs
outputs = []
for output in self.process.processOutputs:
outputs.append(
(output.identifier, output.dataType == 'ComplexData'))
from phoenix.tasks.execute import execute_process
result = execute_process.delay(
userid=self.request.unauthenticated_userid,
url=self.wps.url,
service_name=self.service_name,
identifier=self.process.identifier,
inputs=inputs,
outputs=outputs,
async=appstruct.get('_async_check', True))
self.request.registry.notify(JobStarted(self.request, result.id))
return result.id
@view_config(route_name='processes_execute',
renderer='phoenix:processes/templates/processes/execute.pt',
accept='text/html')
def view(self):
form = self.generate_form()
if 'submit' in self.request.POST:
check_csrf_token(self.request)
return self.process_form(form)
if not has_execute_permission(self.request, self.service_name):
msg = """<strong>Warning:</strong> You are not allowed to run this process.
Please <a href="{0}" class="alert-link">sign in</a> and wait for account activation."""
msg = msg.format(self.request.route_path('sign_in'))
self.session.flash(msg, queue='warning')
return dict(process=self.process,
url=wps_describe_url(self.wps.url, self.processid),
form=form.render(self.appstruct()))
class pyGDPwebProcessing():
"""
This class allows interactive calls to be made into the GDP.
"""
def __init__(self, WFS_URL=None):
if WFS_URL == None:
from pygdp.namespaces import WFS_URL
wfsUrl = WFS_URL
self.wfsUrl = wfsUrl
self.wpsUrl = WPS_URL
self.version = '1.1.0'
self.wps = WebProcessingService(WPS_URL)
def WPSgetCapbilities(self, xml=None):
"""
Returns a list of capabilities.
"""
self.wps.getcapabilities(xml)
def WPSdescribeprocess(self, identifier, xml=None):
"""
Returns a list describing a specific identifier/process.
"""
self.wps.describeprocess(identifier, xml)
#pyGDP Submit Feature
def dodsReplace(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return _execute_request.dodsReplace(dataSetURI, verbose)
def submitFeatureCoverageOPenDAP(self,
geoType,
dataSetURI,
varID,
startTime,
endTime,
attribute='the_geom',
value=None,
gmlIDs=None,
verbose=False,
coverage='true',
outputfname=None,
sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCoverageOPenDAP(
geoType, dataSetURI, varID, startTime, endTime, attribute, value,
gmlIDs, verbose, coverage, self.wfsUrl, outputfname, sleepSecs)
def submitFeatureCoverageWCSIntersection(self,
geoType,
dataSetURI,
varID,
attribute='the_geom',
value=None,
gmlIDs=None,
verbose=False,
coverage='true',
outputfname=None,
sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCoverageWCSIntersection(
geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose,
coverage, self.wfsUrl, outputfname, sleepSecs)
def submitFeatureCategoricalGridCoverage(self,
geoType,
dataSetURI,
varID,
attribute='the_geom',
value=None,
gmlIDs=None,
verbose=False,
coverage='true',
delim='COMMA',
outputfname=None,
sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return feature_coverage.submitFeatureCategoricalGridCoverage(
geoType, dataSetURI, varID, attribute, value, gmlIDs, verbose,
coverage, delim, self.wfsUrl, outputfname, sleepSecs)
def submitFeatureWeightedGridStatistics(self,
geoType,
dataSetURI,
varID,
startTime,
endTime,
attribute='the_geom',
value=None,
gmlIDs=None,
verbose=None,
coverage=True,
delim='COMMA',
stat='MEAN',
grpby='STATISTIC',
timeStep=False,
summAttr=False,
weighted=True,
outputfname=None,
sleepSecs=10):
if verbose:
ch.setLevel(logging.INFO)
return fwgs.submitFeatureWeightedGridStatistics(
geoType, dataSetURI, varID, startTime, endTime, attribute, value,
gmlIDs, verbose, coverage, delim, stat, grpby, timeStep, summAttr,
weighted, self.wfsUrl, outputfname, sleepSecs)
#pyGDP File Utilities
def shapeToZip(self, inShape, outZip=None, allFiles=True):
return shape_to_zip.shapeToZip(inShape, outZip=None, allFiles=True)
def uploadShapeFile(self, filePath):
value = upload_shapefile.uploadShapeFile(filePath)
return value
#pyGDP WFS Utilities
def getTuples(self, shapefile, attribute):
return shapefile_id_handle.getTuples(shapefile, attribute)
def getShapefiles(self):
return shapefile_value_handle.getShapefiles(self.wfsUrl)
def getAttributes(self, shapefile):
return shapefile_value_handle.getAttributes(shapefile, self.wfsUrl)
def getValues(self,
shapefile,
attribute,
getTuples='false',
limitFeatures=None):
return shapefile_value_handle.getValues(shapefile, attribute,
getTuples, limitFeatures,
self.wfsUrl)
def getGMLIDs(self, shapefile, attribute, value):
return shapefile_id_handle.getGMLIDs(shapefile,
attribute,
value,
WFS_URL=self.wfsUrl)
def _getFilterID(self, tuples, value):
return shapefile_id_handle._getFilterID(tuples, value)
def _getFeatureCollectionGeoType(self,
geoType,
attribute='the_geom',
value=None,
gmlIDs=None):
return _get_geotype._getFeatureCollectionGeoType(
geoType, attribute, value, gmlIDs, self.wfsUrl)
def _generateRequest(self,
dataSetURI,
algorithm,
method,
varID=None,
verbose=False):
return _webdata_xml_generate._generateRequest(dataSetURI, algorithm,
method, varID, verbose)
#pyGDP WebData Utilities
def getDataLongName(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getDataLongName(dataSetURI, verbose)
def getDataType(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getDataType(dataSetURI, verbose)
def getDataUnits(self, dataSetURI, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getDataUnits(dataSetURI, verbose)
def getDataSetURI(self, anyText=None, CSWURL=CSWURL, BBox=None):
"""
Searches a given CSW server and returns metadata content for the datasets found.
:param anyText: keywords to be passed to CSW get records
:type anyText: list or None
"""
return webdata_handle.getDataSetURI(anyText, CSWURL, BBox)
def getTimeRange(self, dataSetURI, varID, verbose=False):
if verbose:
ch.setLevel(logging.INFO)
return webdata_handle.getTimeRange(dataSetURI, varID, verbose)
#Pull up docstrings.
#dodsReplace.__doc__ = _execute_request.dodsReplace.__doc__
getAttributes.__doc__ = shapefile_value_handle.getAttributes.__doc__
getDataLongName.__doc__ = webdata_handle.getDataLongName.__doc__
getDataSetURI.__doc__ = webdata_handle.getDataSetURI.__doc__
getDataType.__doc__ = webdata_handle.getDataType.__doc__
getDataUnits.__doc__ = webdata_handle.getDataUnits.__doc__
getGMLIDs.__doc__ = shapefile_id_handle.getGMLIDs.__doc__
getShapefiles.__doc__ = shapefile_value_handle.getShapefiles.__doc__
getTimeRange.__doc__ = webdata_handle.getTimeRange.__doc__
getTuples.__doc__ = shapefile_id_handle.getTuples.__doc__
getValues.__doc__ = shapefile_value_handle.getValues.__doc__
shapeToZip.__doc__ = shape_to_zip.shapeToZip.__doc__
submitFeatureCategoricalGridCoverage.__doc__ = feature_coverage.submitFeatureCategoricalGridCoverage.__doc__
submitFeatureCoverageOPenDAP.__doc__ = feature_coverage.submitFeatureCoverageOPenDAP.__doc__
submitFeatureCoverageWCSIntersection.__doc__ = feature_coverage.submitFeatureCoverageWCSIntersection.__doc__
submitFeatureWeightedGridStatistics.__doc__ = fwgs.submitFeatureWeightedGridStatistics.__doc__
uploadShapeFile.__doc__ = upload_shapefile.uploadShapeFile.__doc__
# alternatively, read capabilities from XML file (no live request to WPS server)
#xml = open('../tests/USGSCapabilities.xml', 'r').read()
#wps.getcapabilities(xml=xml)
print 'WPS Identification type: %s' % wps.identification.type
print 'WPS Identification title: %s' % wps.identification.title
print 'WPS Identification abstract: %s' % wps.identification.abstract
for operation in wps.operations:
print 'WPS Operation: %s' % operation.name
for process in wps.processes:
print 'WPS Process: identifier=%s title=%s' % (process.identifier,
process.title)
# 2) DescribeProcess
# Submits an HTTP GET "DescribeProcess" request to the WPS service and parses the HTTP response
process = wps.describeprocess(
'gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm')
# alternatively, read process description from XML file (no live request to WPS server)
#xml = open('../tests/USGSDescribeProcess.xml', 'r').read()
#process = wps.describeprocess('gov.usgs.cida.gdp.wps.algorithm.FeatureWeightedGridStatisticsAlgorithm', xml=xml)
print 'WPS Process: identifier=%s' % process.identifier
print 'WPS Process: title=%s' % process.title
print 'WPS Process: abstract=%s' % process.abstract
for input in process.dataInputs:
print 'Process input:'
printInputOutput(input, indent='\t')
for output in process.processOutputs:
print 'Process output:'
printInputOutput(output, indent='\t')
# 3a) Execute
# Submits an HTTP POST "Execute" process request to the WPS service, keeps checking the status of the request,
class ExecuteProcess(MyView):
def __init__(self, request):
self.request = request
self.execution = None
self.service_name = None
self.processid = None
self.process = None
if 'job_id' in request.params:
job = request.db.jobs.find_one(
{'identifier': request.params['job_id']})
self.service_name = job.get('service_name')
self.execution = check_status(url=job.get('status_location'),
response=job.get('response'),
verify=False,
sleep_secs=0)
self.processid = self.execution.process.identifier
elif 'wps' in request.params:
self.service_name = request.params.get('wps')
self.processid = request.params.get('process')
if self.service_name:
# TODO: avoid getcaps
self.wps = WebProcessingService(url=request.route_url(
'owsproxy', service_name=self.service_name),
verify=False)
# TODO: need to fix owslib to handle special identifiers
self.process = self.wps.describeprocess(self.processid)
super(ExecuteProcess, self).__init__(request,
name='namewps_execute',
title='')
def breadcrumbs(self):
breadcrumbs = super(ExecuteProcess, self).breadcrumbs()
breadcrumbs.append(
dict(route_path=self.request.route_path('namewps_list'),
title='NAME processes'))
breadcrumbs.append(
dict(route_path=self.request.route_path(self.name),
title=self.process.identifier))
return breadcrumbs
def appstruct(self):
# TODO: not a nice way to get inputs ... should be cleaned up in owslib
result = {}
if self.execution:
for inp in self.execution.dataInputs:
if inp.data or inp.reference:
if inp.identifier not in result:
# init result for param with empty list
result[inp.identifier] = []
if inp.data:
# add literal input, inp.data is a list
result[inp.identifier].extend(inp.data)
elif inp.reference:
# add reference to complex input
result[inp.identifier].append(inp.reference)
for inp in self.process.dataInputs:
# TODO: dupliate code in wizard.start
# convert boolean
if 'boolean' in inp.dataType and inp.identifier in result:
result[inp.identifier] = [
val.lower() == 'true' for val in result[inp.identifier]
]
elif inp.dataType in ['date', 'time', 'dateTime'
] and inp.identifier in result:
result[inp.identifier] = [
dateparser.parse(val) for val in result[inp.identifier]
]
elif inp.dataType == 'BoundingBoxData' and inp.identifier in result:
result[inp.identifier] = [
"{0.minx},{0.miny},{0.maxx},{0.maxy}".format(bbox)
for bbox in result[inp.identifier]
]
# TODO: very dirty ... if single value then take the first
if inp.maxOccurs < 2 and inp.identifier in result:
result[inp.identifier] = result[inp.identifier][0]
return result
def generate_form(self, formid='deform'):
schema = WPSSchema(request=self.request,
process=self.process,
use_async=self.request.has_permission('admin'),
user=self.request.user)
submit_button = Button(name='submit',
title='Submit',
css_class='btn btn-success btn-lg btn-block',
disabled=not has_execute_permission(
self.request, self.service_name))
return Form(
schema.bind(request=self.request),
buttons=(submit_button, ),
formid=formid,
)
def process_form(self, form):
controls = self.request.POST.items()
try:
# TODO: uploader puts qqfile in controls
controls = [
control for control in controls if 'qqfile' not in control[0]
]
LOGGER.debug("before validate %s", controls)
appstruct = form.validate(controls)
LOGGER.debug("before execute %s", appstruct)
job_id = self.execute(appstruct)
except ValidationFailure, e:
self.session.flash("Page validation failed.", queue='danger')
return dict(process=self.process,
url=wps_describe_url(self.wps.url, self.processid),
form=e.render())
else:
