def test_util_py34():
assert clean_ows_url(
'http://example.org/wms?map=/path/to/foo.map&SERVICE=WMS&version=1.3.0&request=GetCapabilities'
) == 'http://example.org/wms?map=%2Fpath%2Fto%2Ffoo.map' # noqa
assert clean_ows_url(
'http://example.org/wms?map=/path/to/foo.map&foo=bar&&SERVICE=WMS&version=1.3.0&request=GetCapabilities'
) == 'http://example.org/wms?map=%2Fpath%2Fto%2Ffoo.map&foo=bar' # noqa
def WebMapService(url,
version='1.3.0',
xml=None,
username=None,
password=None,
parse_remote_metadata=False,
timeout=30,
headers=None,
proxy_base=None):
"""
API for Web Map Service (WMS) methods and metadata.
"""
'''wms factory function, returns a version specific WebMapService object
@type url: string
@param url: url of WFS capabilities document
@type xml: string
@param xml: elementtree object
@type parse_remote_metadata: boolean
@param parse_remote_metadata: whether to fully process MetadataURL elements
@param timeout: time (in seconds) after which requests should timeout
@return: initialized WebFeatureService_2_0_0 object
'''
if not proxy_base:
clean_url = clean_ows_url(url)
base_ows_url = clean_url
else:
(clean_version, proxified_url,
base_ows_url) = base.get_proxified_ows_url(url,
version=version,
proxy_base=proxy_base)
version = clean_version
clean_url = proxified_url
if version in ['1.1.1']:
return (base_ows_url,
wms111.WebMapService_1_1_1(
clean_url,
version=version,
xml=xml,
parse_remote_metadata=parse_remote_metadata,
username=username,
password=password,
timeout=timeout,
headers=headers))
elif version in ['1.3.0']:
return (base_ows_url,
wms130.WebMapService_1_3_0(
clean_url,
version=version,
xml=xml,
parse_remote_metadata=parse_remote_metadata,
username=username,
password=password,
timeout=timeout,
headers=headers))
raise NotImplementedError(
f'The WMS version ({version}) you requested is not implemented. Please use 1.1.1 or 1.3.0.'
)
def WebCoverageService(url, version=None, xml=None, cookies=None, timeout=30):
''' wcs factory function, returns a version specific WebCoverageService object '''
if version is None:
if xml is None:
reader = wcsBase.WCSCapabilitiesReader()
request = reader.capabilities_url(url)
xml = openURL(request, cookies=cookies, timeout=timeout).read()
capabilities = etree.etree.fromstring(xml)
version = capabilities.get('version')
del capabilities
clean_url = clean_ows_url(url)
if version == '1.0.0':
return wcs100.WebCoverageService_1_0_0.__new__(
wcs100.WebCoverageService_1_0_0, clean_url, xml, cookies)
elif version == '1.1.0':
return wcs110.WebCoverageService_1_1_0.__new__(
wcs110.WebCoverageService_1_1_0, url, xml, cookies)
elif version == '1.1.1':
return wcs111.WebCoverageService_1_1_1.__new__(
wcs111.WebCoverageService_1_1_1, url, xml, cookies)
elif version == '2.0.0':
return wcs200.WebCoverageService_2_0_0.__new__(
wcs200.WebCoverageService_2_0_0, url, xml, cookies)
elif version == '2.0.1':
return wcs201.WebCoverageService_2_0_1.__new__(
wcs201.WebCoverageService_2_0_1, url, xml, cookies)
def WebCoverageService(url, version=None, xml=None, cookies=None, timeout=30):
''' wcs factory function, returns a version specific WebCoverageService object '''
if version is None:
if xml is None:
reader = wcsBase.WCSCapabilitiesReader()
request = reader.capabilities_url(url)
xml = openURL(request, cookies=cookies, timeout=timeout).read()
capabilities = etree.etree.fromstring(xml)
version = capabilities.get('version')
del capabilities
clean_url = clean_ows_url(url)
if version == '1.0.0':
return wcs100.WebCoverageService_1_0_0.__new__(wcs100.WebCoverageService_1_0_0, clean_url, xml, cookies)
elif version == '1.1.0':
return wcs110.WebCoverageService_1_1_0.__new__(wcs110.WebCoverageService_1_1_0, url, xml, cookies)
elif version == '1.1.1':
return wcs111.WebCoverageService_1_1_1.__new__(wcs111.WebCoverageService_1_1_1, url, xml, cookies)
elif version == '2.0.0':
return wcs200.WebCoverageService_2_0_0.__new__(wcs200.WebCoverageService_2_0_0, url, xml, cookies)
elif version == '2.0.1':
return wcs201.WebCoverageService_2_0_1.__new__(wcs201.WebCoverageService_2_0_1, url, xml, cookies)
def submit_job_handler(payload, # type: JSON
settings, # type: SettingsType
service_url, # type: str
provider_id=None, # type: Optional[str]
process_id=None, # type: str
is_workflow=False, # type: bool
is_local=True, # type: bool
visibility=None, # type: Optional[str]
language=None, # type: Optional[str]
auth=None, # type: Optional[HeaderCookiesType]
tags=None, # type: Optional[List[str]]
user=None, # type: Optional[int]
context=None, # type: Optional[str]
): # type: (...) -> JSON
"""
Submits the job to the Celery worker with provided parameters.
Assumes that parameters have been pre-fetched and validated, except for the input payload.
"""
try:
json_body = sd.Execute().deserialize(payload)
except colander.Invalid as ex:
raise HTTPBadRequest("Invalid schema: [{}]".format(str(ex)))
# TODO: remove when all parameter variations are supported
# FIXME:
# - support 'sync' and 'Prefer' header variants (https://github.com/crim-ca/weaver/issues/247)
# - support 'response: raw' (https://github.com/crim-ca/weaver/issues/376)
# - allow omitting 'outputs' (https://github.com/crim-ca/weaver/issues/375)
_validate_job_parameters(json_body)
is_execute_async = json_body["mode"] != EXECUTE_MODE_SYNC # convert auto to async
notification_email = json_body.get("notification_email")
encrypted_email = encrypt_email(notification_email, settings) if notification_email else None
store = get_db(settings).get_store(StoreJobs)
job = store.save_job(task_id=STATUS_ACCEPTED, process=process_id, service=provider_id,
inputs=json_body.get("inputs"), is_local=is_local, is_workflow=is_workflow,
access=visibility, user_id=user, execute_async=is_execute_async, custom_tags=tags,
notification_email=encrypted_email, accept_language=language, context=context)
job.save_log(logger=LOGGER, message="Job task submitted for execution.", status=STATUS_ACCEPTED, progress=0)
job = store.update_job(job)
result = execute_process.delay(job_id=job.id, wps_url=clean_ows_url(service_url), headers=auth)
LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id)
# local/provider process location
location_base = "/providers/{provider_id}".format(provider_id=provider_id) if provider_id else ""
location = "{base_url}{location_base}/processes/{process_id}/jobs/{job_id}".format(
base_url=get_wps_restapi_base_url(settings),
location_base=location_base,
process_id=process_id,
job_id=job.id)
body_data = {
"jobID": job.id,
"processID": job.process,
"providerID": provider_id, # dropped by validator if not applicable
"status": map_status(STATUS_ACCEPTED),
"location": location
}
return body_data
def test_clean_ows_url():
assert clean_ows_url('http//example.org/wms') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?service=WMS') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?SERVICE=WMS') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?SeRvIcE=WMS') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?SeRvIcE=WMS&version=1.3.0&request=GetCapabilities') == 'http//example.org/wms' # noqa
assert clean_ows_url('http//example.org/wms?foo=bar&SeRvIcE=WMS&version=1.3.0&request=GetCapabilities') == 'http//example.org/wms?foo=bar' # noqa
assert clean_ows_url('http://example.org/wms?map=/path/to/foo.map&SERVICE=WMS&version=1.3.0&request=GetCapabilities') == 'http://example.org/wms?map=%2Fpath%2Fto%2Ffoo.map' # noqa
clean_url = clean_ows_url('http://example.org/wms?map=/path/to/foo.map&foo=bar&&SERVICE=WMS&version=1.3.0&request=GetCapabilities') # noqa
assert 'http://example.org/wms?' in clean_url
assert 'map=%2Fpath%2Fto%2Ffoo.map' in clean_url
assert 'foo=bar' in clean_url
def test_clean_ows_url():
assert clean_ows_url('http//example.org/wms') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?service=WMS') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?SERVICE=WMS') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?SeRvIcE=WMS') == 'http//example.org/wms'
assert clean_ows_url('http//example.org/wms?SeRvIcE=WMS&version=1.3.0&request=GetCapabilities') == 'http//example.org/wms' # noqa
assert clean_ows_url('http//example.org/wms?foo=bar&SeRvIcE=WMS&version=1.3.0&request=GetCapabilities') == 'http//example.org/wms?foo=bar' # noqa
def WebMapService(url,
version='1.1.1',
xml=None,
username=None,
password=None,
parse_remote_metadata=False,
timeout=30,
headers=None,
proxy_base=None):
"""
API for Web Map Service (WMS) methods and metadata.
Currently supports only version 1.1.1 of the WMS protocol.
"""
'''wms factory function, returns a version specific WebMapService object
@type url: string
@param url: url of WFS capabilities document
@type xml: string
@param xml: elementtree object
@type parse_remote_metadata: boolean
@param parse_remote_metadata: whether to fully process MetadataURL elements
@param timeout: time (in seconds) after which requests should timeout
@return: initialized WebFeatureService_2_0_0 object
'''
if not proxy_base:
clean_url = clean_ows_url(url)
base_ows_url = clean_url
else:
(clean_version, proxified_url, base_ows_url) = base.get_proxified_ows_url(
url, version=version, proxy_base=proxy_base)
version = clean_version
clean_url = proxified_url
if version in ['1.1.1']:
return (base_ows_url, wms111.WebMapService_1_1_1(clean_url, version=version, xml=xml,
parse_remote_metadata=parse_remote_metadata,
username=username, password=password,
timeout=timeout, headers=headers))
elif version in ['1.3.0']:
return (base_ows_url, wms130.WebMapService_1_3_0(clean_url, version=version, xml=xml,
parse_remote_metadata=parse_remote_metadata,
username=username, password=password,
timeout=timeout, headers=headers))
raise NotImplementedError(
'The WMS version (%s) you requested is not implemented. Please use 1.1.1 or 1.3.0.' %
version)
def connection_sos(
url,
xml=None,
username=None,
password=None,
):
"""
SOS GetDataAvailability function
:param url: url of capabilities document
:param xml: elementtree object
:param username: username allowed to handle with SOS
:param password: password for the username
:return: a sos_2_0_0 object
"""
clean_url = clean_ows_url(
url) # Clean an OWS URL of basic service elements
version = "2.0.0"
return sos_2_0_0.__new__(sos_2_0_0, clean_url, version, xml, username,
password)
def __init__(self,
url,
lang='en-US',
version='3.0.0',
timeout=10,
skip_caps=False,
username=None,
password=None,
auth=None):
"""
Construct and process a GetCapabilities request
Parameters
----------
- url: the URL of the CSW
- lang: the language (default is 'en-US')
- version: version (default is '3.0.0')
- timeout: timeout in seconds
- skip_caps: whether to skip GetCapabilities processing on init (default is False)
- username: username for HTTP basic authentication
- password: password for HTTP basic authentication
- auth: instance of owslib.util.Authentication
"""
if auth:
if username:
auth.username = username
if password:
auth.password = password
self.url = util.clean_ows_url(url)
self.lang = lang
self.version = version
self.timeout = timeout
self.auth = auth or Authentication(username, password)
self.service = 'CSW'
self.exceptionreport = None
self.owscommon = ows.OwsCommon('2.0.0')
if not skip_caps: # process GetCapabilities
# construct request
data = {
'service': self.service,
'version': self.version,
'request': 'GetCapabilities'
}
self.request = urlencode(data)
self._invoke()
if self.exceptionreport is None:
self.updateSequence = self._exml.getroot().attrib.get(
'updateSequence')
# ServiceIdentification
val = self._exml.find(
util.nspath_eval('ows200:ServiceIdentification',
namespaces))
if val is not None:
self.identification = ows.ServiceIdentification(
val, self.owscommon.namespace)
else:
self.identification = None
# ServiceProvider
val = self._exml.find(
util.nspath_eval('ows200:ServiceProvider', namespaces))
if val is not None:
self.provider = ows.ServiceProvider(
val, self.owscommon.namespace)
else:
self.provider = None
# ServiceOperations metadata
self.operations = []
for elem in self._exml.findall(
util.nspath_eval(
'ows200:OperationsMetadata/ows200:Operation',
namespaces)): # noqa
self.operations.append(
ows.OperationsMetadata(elem, self.owscommon.namespace))
self.constraints = {}
for elem in self._exml.findall(
util.nspath_eval(
'ows200:OperationsMetadata/ows200:Constraint',
namespaces)): # noqa
self.constraints[elem.attrib['name']] = ows.Constraint(
elem, self.owscommon.namespace)
self.parameters = {}
for elem in self._exml.findall(
util.nspath_eval(
'ows200:OperationsMetadata/ows200:Parameter',
namespaces)): # noqa
self.parameters[elem.attrib['name']] = ows.Parameter(
elem, self.owscommon.namespace)
# FilterCapabilities
val = self._exml.find(
util.nspath_eval('fes:Filter_Capabilities', namespaces))
self.filters = fes2.FilterCapabilities(val)
def test_util_py34():
assert clean_ows_url('http://example.org/wms?map=/path/to/foo.map&SERVICE=WMS&version=1.3.0&request=GetCapabilities') == 'http://example.org/wms?map=%2Fpath%2Fto%2Ffoo.map' # noqa
assert clean_ows_url('http://example.org/wms?map=/path/to/foo.map&foo=bar&&SERVICE=WMS&version=1.3.0&request=GetCapabilities') == 'http://example.org/wms?map=%2Fpath%2Fto%2Ffoo.map&foo=bar' # noqa
def __init__(self, url, lang='en-US', version='2.0.2', timeout=10, skip_caps=False,
username=None, password=None):
"""
Construct and process a GetCapabilities request
Parameters
----------
- url: the URL of the CSW
- lang: the language (default is 'en-US')
- version: version (default is '2.0.2')
- timeout: timeout in seconds
- skip_caps: whether to skip GetCapabilities processing on init (default is False)
- username: username for HTTP basic authentication
- password: password for HTTP basic authentication
"""
self.url = util.clean_ows_url(url)
self.lang = lang
self.version = version
self.timeout = timeout
self.username = username
self.password = password
self.service = 'CSW'
self.exceptionreport = None
self.owscommon = ows.OwsCommon('1.0.0')
if not skip_caps: # process GetCapabilities
# construct request
data = {'service': self.service, 'version': self.version, 'request': 'GetCapabilities'}
self.request = urlencode(data)
self._invoke()
if self.exceptionreport is None:
self.updateSequence = self._exml.getroot().attrib.get('updateSequence')
# ServiceIdentification
val = self._exml.find(util.nspath_eval('ows:ServiceIdentification', namespaces))
if val is not None:
self.identification = ows.ServiceIdentification(val,self.owscommon.namespace)
else:
self.identification = None
# ServiceProvider
val = self._exml.find(util.nspath_eval('ows:ServiceProvider', namespaces))
if val is not None:
self.provider = ows.ServiceProvider(val,self.owscommon.namespace)
else:
self.provider = None
# ServiceOperations metadata
self.operations = []
for elem in self._exml.findall(util.nspath_eval('ows:OperationsMetadata/ows:Operation', namespaces)):
self.operations.append(ows.OperationsMetadata(elem, self.owscommon.namespace))
self.constraints = {}
for elem in self._exml.findall(util.nspath_eval('ows:OperationsMetadata/ows:Constraint', namespaces)):
self.constraints[elem.attrib['name']] = ows.Constraint(elem, self.owscommon.namespace)
self.parameters = {}
for elem in self._exml.findall(util.nspath_eval('ows:OperationsMetadata/ows:Parameter', namespaces)):
self.parameters[elem.attrib['name']] = ows.Parameter(elem, self.owscommon.namespace)
# FilterCapabilities
val = self._exml.find(util.nspath_eval('ogc:Filter_Capabilities', namespaces))
self.filters = fes.FilterCapabilities(val)
def submit_job_handler(
payload, # type: JSON
settings, # type: SettingsType
service_url, # type: str
provider_id=None, # type: Optional[str]
process_id=None, # type: str
is_workflow=False, # type: bool
is_local=True, # type: bool
visibility=None, # type: Optional[AnyVisibility]
language=None, # type: Optional[str]
headers=None, # type: Optional[HeaderCookiesType]
tags=None, # type: Optional[List[str]]
user=None, # type: Optional[int]
context=None, # type: Optional[str]
): # type: (...) -> AnyResponseType
"""
Submits the job to the Celery worker with provided parameters.
Assumes that parameters have been pre-fetched and validated, except for the input payload.
"""
try:
json_body = sd.Execute().deserialize(payload)
except colander.Invalid as ex:
raise HTTPBadRequest(f"Invalid schema: [{ex!s}]")
db = get_db(settings)
headers = headers or {}
if is_local:
proc_store = db.get_store(StoreProcesses)
process = proc_store.fetch_by_id(process_id)
job_ctl_opts = process.jobControlOptions
else:
job_ctl_opts = ExecuteControlOption.values()
max_wait = as_int(settings.get("weaver.exec_sync_max_wait"), default=20)
mode, wait, applied = parse_prefer_header_execute_mode(
headers, job_ctl_opts, max_wait)
get_header("prefer", headers, pop=True)
if not applied: # whatever returned is a default, consider 'mode' in body as alternative
is_execute_async = ExecuteMode.get(
json_body.get("mode")) != ExecuteMode.SYNC # convert auto to async
else:
# as per https://datatracker.ietf.org/doc/html/rfc7240#section-2
# Prefer header not resolve as valid still proces
is_execute_async = mode != ExecuteMode.SYNC
exec_resp = json_body.get("response")
notification_email = json_body.get("notification_email")
encrypted_email = encrypt_email(notification_email,
settings) if notification_email else None
store = db.get_store(StoreJobs) # type: StoreJobs
job = store.save_job(task_id=Status.ACCEPTED,
process=process_id,
service=provider_id,
inputs=json_body.get("inputs"),
outputs=json_body.get("outputs"),
is_local=is_local,
is_workflow=is_workflow,
access=visibility,
user_id=user,
context=context,
execute_async=is_execute_async,
execute_response=exec_resp,
custom_tags=tags,
notification_email=encrypted_email,
accept_language=language)
job.save_log(logger=LOGGER,
message="Job task submitted for execution.",
status=Status.ACCEPTED,
progress=0)
job = store.update_job(job)
location_url = job.status_url(settings)
resp_headers = {"Location": location_url}
resp_headers.update(applied)
wps_url = clean_ows_url(service_url)
result = execute_process.delay(job_id=job.id,
wps_url=wps_url,
headers=headers) # type: CeleryResult
LOGGER.debug("Celery pending task [%s] for job [%s].", result.id, job.id)
if not is_execute_async:
LOGGER.debug(
"Celery task requested as sync if it completes before (wait=%ss)",
wait)
try:
result.wait(timeout=wait)
except CeleryTaskTimeoutError:
pass
if result.ready():
job = store.fetch_by_id(job.id)
# when sync is successful, it must return the results direct instead of status info
# see: https://docs.ogc.org/is/18-062r2/18-062r2.html#sc_execute_response
if job.status == Status.SUCCEEDED:
return get_job_results_response(job,
settings,
headers=resp_headers)
# otherwise return the error status
body = job.json(container=settings, self_link="status")
body["location"] = location_url
resp = get_job_submission_response(body, resp_headers, error=True)
return resp
else:
LOGGER.debug(
"Celery task requested as sync took too long to complete (wait=%ss). Continue in async.",
wait)
# sync not respected, therefore must drop it
# since both could be provided as alternative preferences, drop only async with limited subset
prefer = get_header("Preference-Applied", headers, pop=True)
_, _, async_applied = parse_prefer_header_execute_mode(
{"Prefer": prefer}, [ExecuteMode.ASYNC])
if async_applied:
resp_headers.update(async_applied)
LOGGER.debug("Celery task submitted to run async.")
body = {
"jobID": job.id,
"processID": job.process,
"providerID": provider_id, # dropped by validator if not applicable
"status": map_status(Status.ACCEPTED),
"location": location_url
}
resp = get_job_submission_response(body, resp_headers)
return resp