def get_dov_xml(url, session=None):
"""Request the XML from the remote DOV webservices and return it.
Parameters
----------
url : str
URL of the DOV object to download.
session : requests.Session
Session to use to perform HTTP requests for data. Defaults to None,
which means a new session will be created for each request.
Returns
-------
xml : bytes
The raw XML data of this DOV object as bytes.
"""
response = HookRunner.execute_inject_xml_response(url)
if response is None:
response = get_remote_url(url, session)
HookRunner.execute_xml_received(url, response)
return response
def get(self, url):
datatype, key = self._get_type_key_from_url(url)
data = HookRunner.execute_inject_xml_response(url)
if data is not None:
HookRunner.execute_xml_received(url, data)
return data
if self._is_valid(datatype, key):
try:
self._emit_cache_hit(url)
data = self._load(datatype, key).encode('utf-8')
HookRunner.execute_xml_received(url, data)
return data
except Exception:
pass
data = self._get_remote(url)
try:
self._save(datatype, key, data)
except Exception:
pass
return data
def _emit_cache_hit(self, url):
"""Emit the XML cache hit event for all registered hooks.
Parameters
----------
url : str
Permanent URL to a DOV object.
"""
HookRunner.execute_xml_cache_hit(url.rstrip('.xml'))
def _get_remote_wfs_feature(wfs, typename, location, filter, sort_by,
propertyname, max_features, geometry_column):
"""Perform the WFS GetFeature call to get features from the remote
service.
Parameters
----------
typename : str
Layername to query.
location : pydov.util.location.AbstractLocationFilter
Location filter limiting the features to retrieve.
filter : str of owslib.fes.FilterRequest
Filter request to search on attribute values.
sort_by : str of owslib.fes.SortBy, optional
List of properties to sort by.
propertyname : list<str>
List of properties to return.
max_features : int
Limit the maximum number of features to request.
geometry_column : str
Name of the geometry column to use in the spatial filter.
Returns
-------
wfs_response, wfs_getfeature_request : bytes, etree.Element
Response of the WFS service.
"""
wfs_getfeature_xml = owsutil.wfs_build_getfeature_request(
version=wfs.version,
geometry_column=geometry_column,
typename=typename,
location=location,
filter=filter,
sort_by=sort_by,
max_features=max_features,
propertyname=propertyname)
HookRunner.execute_wfs_search_init(typename)
tree = HookRunner.execute_inject_wfs_getfeature_response(
wfs_getfeature_xml)
if tree is not None:
return tree, wfs_getfeature_xml
return owsutil.wfs_get_feature(
baseurl=wfs.url,
get_feature_request=wfs_getfeature_xml), wfs_getfeature_xml
def _get_remote(self, url):
"""Get the XML data by requesting it from the given URL.
Parameters
----------
url : str
Permanent URL to a DOV object.
Returns
-------
xml : bytes
The raw XML data of this DOV object as bytes.
"""
xml = get_dov_xml(url)
HookRunner.execute_xml_downloaded(url.rstrip('.xml'))
return xml
def get(self, url):
"""Get the XML data for the DOV object referenced by the given URL.
If a valid version exists in the cache, it will be loaded and
returned. If no valid version exists, the XML will be downloaded
from the DOV webservice, saved in the cache and returned.
Parameters
----------
url : str
Permanent URL to a DOV object.
Returns
-------
xml : bytes
The raw XML data of this DOV object as bytes.
"""
datatype, key = self._get_type_key_from_url(url)
data = HookRunner.execute_inject_xml_response(url)
if data is not None:
HookRunner.execute_xml_received(url, data)
return data
if self._is_valid(datatype, key):
try:
self._emit_cache_hit(url)
data = self._load(datatype, key).encode('utf-8')
HookRunner.execute_xml_received(url, data)
return data
except Exception:
pass
data = self._get_remote(url)
try:
self._save(datatype, key, data)
except Exception:
pass
return data
def _get_remote(self, url, session=None):
"""Get the XML data by requesting it from the given URL.
Parameters
----------
url : str
Permanent URL to a DOV object.
session : requests.Session
Session to use to perform HTTP requests for data. Defaults to None,
which means a new session will be created for each request.
Returns
-------
xml : bytes
The raw XML data of this DOV object as bytes.
"""
xml = get_dov_xml(url, session)
HookRunner.execute_xml_downloaded(url.rstrip('.xml'))
return xml
def get_dov_xml(url):
"""Request the XML from the remote DOV webservices and return it.
Parameters
----------
url : str
URL of the DOV object to download.
Returns
-------
xml : bytes
The raw XML data of this DOV object as bytes.
"""
response = HookRunner.execute_inject_xml_response(url)
if response is None:
response = get_remote_url(url)
HookRunner.execute_xml_received(url, response)
return response
def get_xsd_schema(url):
"""Request the XSD schema from DOV webservices and return it.
Parameters
----------
url : str
URL of the XSD schema to download.
Returns
-------
xml : bytes
The raw XML data of this XSD schema as bytes.
"""
response = HookRunner.execute_inject_meta_response(url)
if response is None:
response = get_remote_url(url)
HookRunner.execute_meta_received(url, response)
return response
def _init_wfs(self):
"""Initialise the WFS service. If the WFS service is not
instantiated yet, do so and save it in a static variable available
to all subclasses and instances.
"""
if AbstractSearch.__wfs is None:
capabilities = HookRunner.execute_inject_meta_response(
build_dov_url('geoserver/wfs') + '?version=1.1.0')
if capabilities is None:
AbstractSearch.__wfs = WebFeatureService(
url=build_dov_url('geoserver/wfs'), version="1.1.0")
else:
AbstractSearch.__wfs = WebFeatureService(
url=build_dov_url('geoserver/wfs'),
version="1.1.0",
xml=capabilities)
HookRunner.execute_meta_received(
build_dov_url('geoserver/wfs') + '?version=1.1.0',
etree.tostring(AbstractSearch.__wfs._capabilities,
encoding='utf8'))
def _search(self,
location=None,
query=None,
return_fields=None,
sort_by=None,
max_features=None,
extra_wfs_fields=[]):
"""Perform the WFS search by issuing a GetFeature request.
Parameters
----------
location : pydov.util.location.AbstractLocationFilter
Location filter limiting the features to retrieve.
query : owslib.fes.OgcExpression
OGC filter expression to use for searching. This can contain any
combination of filter elements defined in owslib.fes. The query
should use the fields provided in `get_fields()`. Note that not
all fields are currently supported as a search parameter.
return_fields : list<str>
A list of fields to be returned in the output data. This should
be a subset of the fields provided in `get_fields()`. Note that
not all fields are currently supported as return fields.
sort_by : owslib.fes.SortBy, optional
List of properties to sort by.
max_features : int
Limit the maximum number of features to request.
extra_wfs_fields: list<str>
A list of extra fields to be included in the WFS requests,
regardless whether they're needed as return field. Optional,
defaults to an empty list.
Returns
-------
etree.Element
XML tree of the WFS response containing the features matching
the location or the query.
Raises
------
pydov.util.errors.InvalidSearchParameterError
When not one of `location`, `query` or `max_features` is provided.
pydov.util.errors.InvalidFieldError
When at least one of the fields in `return_fields` is unknown.
When a field that is only accessible as return field is used as
a query parameter.
When a field that can only be used as a query parameter is used as
a return field.
pydov.util.errors.FeatureOverflowError
When the number of features to be returned is equal to the
maxFeatures limit of the WFS server.
"""
self._pre_search_validation(location, query, sort_by, return_fields,
max_features)
self._init_namespace()
self._init_wfs()
filter_request = None
if query is not None:
filter_request = FilterRequest()
filter_request = filter_request.setConstraint(query)
if filter_request is not None:
for property_name in filter_request.findall(
'.//{http://www.opengis.net/ogc}PropertyName'):
property_name.text = self._map_df_wfs_source.get(
property_name.text, property_name.text)
filter_request = etree.tostring(filter_request, encoding='unicode')
if return_fields is None:
wfs_property_names = [
f['sourcefield']
for f in self._type.get_fields(source=('wfs', )).values()
if not f.get('wfs_injected', False)
]
else:
wfs_property_names = [
self._map_df_wfs_source[i] for i in self._map_df_wfs_source
if i.startswith('pkey')
]
wfs_property_names.extend([
self._map_df_wfs_source[i] for i in self._map_df_wfs_source
if i in return_fields
])
wfs_property_names.extend(extra_wfs_fields)
wfs_property_names = list(set(wfs_property_names))
if sort_by is not None:
sort_by_xml = sort_by.toXML()
for property_name in sort_by_xml.findall(
'.//{http://www.opengis.net/ogc}PropertyName'):
property_name.text = self._map_df_wfs_source.get(
property_name.text, property_name.text)
sort_by = etree.tostring(sort_by_xml, encoding='unicode')
fts, getfeature = self._get_remote_wfs_feature(
wfs=self.__wfs,
typename=self._layer,
location=location,
filter=filter_request,
sort_by=sort_by,
max_features=max_features,
propertyname=wfs_property_names,
geometry_column=self._geometry_column)
tree = etree.fromstring(fts)
if tree.get('numberOfFeatures') is None:
raise WfsGetFeatureError(
'Error retrieving features from DOV WFS server:\n{}'.format(
etree.tostring(tree).decode('utf8')))
if int(tree.get('numberOfFeatures')) == 10000:
raise FeatureOverflowError(
'Reached the limit of {:d} returned features. Please split up '
'the query to ensure getting all results.'.format(10000))
HookRunner.execute_wfs_search_result(int(tree.get('numberOfFeatures')))
HookRunner.execute_wfs_search_result_received(getfeature, tree)
return tree