def get_dov_xml(url, session=None): """Request the XML from the remote DOV webservices and return it. Parameters ---------- url : str URL of the DOV object to download. session : requests.Session Session to use to perform HTTP requests for data. Defaults to None, which means a new session will be created for each request. Returns ------- xml : bytes The raw XML data of this DOV object as bytes. """ response = HookRunner.execute_inject_xml_response(url) if response is None: response = get_remote_url(url, session) HookRunner.execute_xml_received(url, response) return response
def get(self, url): datatype, key = self._get_type_key_from_url(url) data = HookRunner.execute_inject_xml_response(url) if data is not None: HookRunner.execute_xml_received(url, data) return data if self._is_valid(datatype, key): try: self._emit_cache_hit(url) data = self._load(datatype, key).encode('utf-8') HookRunner.execute_xml_received(url, data) return data except Exception: pass data = self._get_remote(url) try: self._save(datatype, key, data) except Exception: pass return data
def _emit_cache_hit(self, url): """Emit the XML cache hit event for all registered hooks. Parameters ---------- url : str Permanent URL to a DOV object. """ HookRunner.execute_xml_cache_hit(url.rstrip('.xml'))
def _get_remote_wfs_feature(wfs, typename, location, filter, sort_by, propertyname, max_features, geometry_column): """Perform the WFS GetFeature call to get features from the remote service. Parameters ---------- typename : str Layername to query. location : pydov.util.location.AbstractLocationFilter Location filter limiting the features to retrieve. filter : str of owslib.fes.FilterRequest Filter request to search on attribute values. sort_by : str of owslib.fes.SortBy, optional List of properties to sort by. propertyname : list<str> List of properties to return. max_features : int Limit the maximum number of features to request. geometry_column : str Name of the geometry column to use in the spatial filter. Returns ------- wfs_response, wfs_getfeature_request : bytes, etree.Element Response of the WFS service. """ wfs_getfeature_xml = owsutil.wfs_build_getfeature_request( version=wfs.version, geometry_column=geometry_column, typename=typename, location=location, filter=filter, sort_by=sort_by, max_features=max_features, propertyname=propertyname) HookRunner.execute_wfs_search_init(typename) tree = HookRunner.execute_inject_wfs_getfeature_response( wfs_getfeature_xml) if tree is not None: return tree, wfs_getfeature_xml return owsutil.wfs_get_feature( baseurl=wfs.url, get_feature_request=wfs_getfeature_xml), wfs_getfeature_xml
def _get_remote(self, url): """Get the XML data by requesting it from the given URL. Parameters ---------- url : str Permanent URL to a DOV object. Returns ------- xml : bytes The raw XML data of this DOV object as bytes. """ xml = get_dov_xml(url) HookRunner.execute_xml_downloaded(url.rstrip('.xml')) return xml
def get(self, url): """Get the XML data for the DOV object referenced by the given URL. If a valid version exists in the cache, it will be loaded and returned. If no valid version exists, the XML will be downloaded from the DOV webservice, saved in the cache and returned. Parameters ---------- url : str Permanent URL to a DOV object. Returns ------- xml : bytes The raw XML data of this DOV object as bytes. """ datatype, key = self._get_type_key_from_url(url) data = HookRunner.execute_inject_xml_response(url) if data is not None: HookRunner.execute_xml_received(url, data) return data if self._is_valid(datatype, key): try: self._emit_cache_hit(url) data = self._load(datatype, key).encode('utf-8') HookRunner.execute_xml_received(url, data) return data except Exception: pass data = self._get_remote(url) try: self._save(datatype, key, data) except Exception: pass return data
def _get_remote(self, url, session=None): """Get the XML data by requesting it from the given URL. Parameters ---------- url : str Permanent URL to a DOV object. session : requests.Session Session to use to perform HTTP requests for data. Defaults to None, which means a new session will be created for each request. Returns ------- xml : bytes The raw XML data of this DOV object as bytes. """ xml = get_dov_xml(url, session) HookRunner.execute_xml_downloaded(url.rstrip('.xml')) return xml
def get_dov_xml(url): """Request the XML from the remote DOV webservices and return it. Parameters ---------- url : str URL of the DOV object to download. Returns ------- xml : bytes The raw XML data of this DOV object as bytes. """ response = HookRunner.execute_inject_xml_response(url) if response is None: response = get_remote_url(url) HookRunner.execute_xml_received(url, response) return response
def get_xsd_schema(url): """Request the XSD schema from DOV webservices and return it. Parameters ---------- url : str URL of the XSD schema to download. Returns ------- xml : bytes The raw XML data of this XSD schema as bytes. """ response = HookRunner.execute_inject_meta_response(url) if response is None: response = get_remote_url(url) HookRunner.execute_meta_received(url, response) return response
def _init_wfs(self): """Initialise the WFS service. If the WFS service is not instantiated yet, do so and save it in a static variable available to all subclasses and instances. """ if AbstractSearch.__wfs is None: capabilities = HookRunner.execute_inject_meta_response( build_dov_url('geoserver/wfs') + '?version=1.1.0') if capabilities is None: AbstractSearch.__wfs = WebFeatureService( url=build_dov_url('geoserver/wfs'), version="1.1.0") else: AbstractSearch.__wfs = WebFeatureService( url=build_dov_url('geoserver/wfs'), version="1.1.0", xml=capabilities) HookRunner.execute_meta_received( build_dov_url('geoserver/wfs') + '?version=1.1.0', etree.tostring(AbstractSearch.__wfs._capabilities, encoding='utf8'))
def _search(self, location=None, query=None, return_fields=None, sort_by=None, max_features=None, extra_wfs_fields=[]): """Perform the WFS search by issuing a GetFeature request. Parameters ---------- location : pydov.util.location.AbstractLocationFilter Location filter limiting the features to retrieve. query : owslib.fes.OgcExpression OGC filter expression to use for searching. This can contain any combination of filter elements defined in owslib.fes. The query should use the fields provided in `get_fields()`. Note that not all fields are currently supported as a search parameter. return_fields : list<str> A list of fields to be returned in the output data. This should be a subset of the fields provided in `get_fields()`. Note that not all fields are currently supported as return fields. sort_by : owslib.fes.SortBy, optional List of properties to sort by. max_features : int Limit the maximum number of features to request. extra_wfs_fields: list<str> A list of extra fields to be included in the WFS requests, regardless whether they're needed as return field. Optional, defaults to an empty list. Returns ------- etree.Element XML tree of the WFS response containing the features matching the location or the query. Raises ------ pydov.util.errors.InvalidSearchParameterError When not one of `location`, `query` or `max_features` is provided. pydov.util.errors.InvalidFieldError When at least one of the fields in `return_fields` is unknown. When a field that is only accessible as return field is used as a query parameter. When a field that can only be used as a query parameter is used as a return field. pydov.util.errors.FeatureOverflowError When the number of features to be returned is equal to the maxFeatures limit of the WFS server. """ self._pre_search_validation(location, query, sort_by, return_fields, max_features) self._init_namespace() self._init_wfs() filter_request = None if query is not None: filter_request = FilterRequest() filter_request = filter_request.setConstraint(query) if filter_request is not None: for property_name in filter_request.findall( './/{http://www.opengis.net/ogc}PropertyName'): property_name.text = self._map_df_wfs_source.get( property_name.text, property_name.text) filter_request = etree.tostring(filter_request, encoding='unicode') if return_fields is None: wfs_property_names = [ f['sourcefield'] for f in self._type.get_fields(source=('wfs', )).values() if not f.get('wfs_injected', False) ] else: wfs_property_names = [ self._map_df_wfs_source[i] for i in self._map_df_wfs_source if i.startswith('pkey') ] wfs_property_names.extend([ self._map_df_wfs_source[i] for i in self._map_df_wfs_source if i in return_fields ]) wfs_property_names.extend(extra_wfs_fields) wfs_property_names = list(set(wfs_property_names)) if sort_by is not None: sort_by_xml = sort_by.toXML() for property_name in sort_by_xml.findall( './/{http://www.opengis.net/ogc}PropertyName'): property_name.text = self._map_df_wfs_source.get( property_name.text, property_name.text) sort_by = etree.tostring(sort_by_xml, encoding='unicode') fts, getfeature = self._get_remote_wfs_feature( wfs=self.__wfs, typename=self._layer, location=location, filter=filter_request, sort_by=sort_by, max_features=max_features, propertyname=wfs_property_names, geometry_column=self._geometry_column) tree = etree.fromstring(fts) if tree.get('numberOfFeatures') is None: raise WfsGetFeatureError( 'Error retrieving features from DOV WFS server:\n{}'.format( etree.tostring(tree).decode('utf8'))) if int(tree.get('numberOfFeatures')) == 10000: raise FeatureOverflowError( 'Reached the limit of {:d} returned features. Please split up ' 'the query to ensure getting all results.'.format(10000)) HookRunner.execute_wfs_search_result(int(tree.get('numberOfFeatures'))) HookRunner.execute_wfs_search_result_received(getfeature, tree) return tree