def compare_xml(a,b): if not isinstance(a, ElementType): a = etree.fromstring(a) if not isinstance(b, ElementType): b = etree.fromstring(b) return compare_elements(a,b)
def openURL(url_base, data=None, method='Get', cookies=None, username=None, password=None, timeout=30): """ Function to open URLs. Uses requests library but with additional checks for OGC service exceptions and url formatting. Also handles cookies and simple user password authentication. """ headers = {} rkwargs = {} rkwargs['timeout'] = timeout auth = None if username and password: auth = (username, password) rkwargs['auth'] = auth # FIXUP for WFS in particular, remove xml style namespace # @TODO does this belong here? method = method.split("}")[-1] if method.lower() == 'post': try: xml = etree.fromstring(data) headers['Content-Type'] = "text/xml" except (ParseError, UnicodeEncodeError): pass rkwargs['data'] = data elif method.lower() == 'get': rkwargs['params'] = data else: raise ValueError("Unknown method ('%s'), expected 'get' or 'post'" % method) if cookies is not None: rkwargs['cookies'] = cookies req = requests.request(method.upper(), url_base, **rkwargs) if req.status_code in [400, 401]: raise ServiceException(req.text) if req.status_code in [404]: # add more if needed req.raise_for_status() # check for service exceptions without the http header set if 'Content-Type' in req.headers and req.headers['Content-Type'] in ['text/xml', 'application/xml']: #just in case 400 headers were not set, going to have to read the xml to see if it's an exception report. se_tree = etree.fromstring(req.content) serviceException=se_tree.find('{http://www.opengis.net/ows}Exception') if serviceException is None: serviceException=se_tree.find('ServiceException') if serviceException is not None: raise ServiceException(str(serviceException.text).strip()) return ResponseWrapper(req)
def _getStoredQueries(self, timeout=None): ''' gets descriptions of the stored queries available on the server ''' if timeout: to = timeout else: to = self.timeout sqs=[] #This method makes two calls to the WFS - one ListStoredQueries, and one DescribeStoredQueries. The information is then #aggregated in 'StoredQuery' objects method=nspath('Get') #first make the ListStoredQueries response and save the results in a dictionary if form {storedqueryid:(title, returnfeaturetype)} try: base_url = next((m.get('url') for m in self.getOperationByName('ListStoredQueries').methods if m.get('type').lower() == method.lower())) except StopIteration: base_url = self.url request = {'service': 'WFS', 'version': self.version, 'request': 'ListStoredQueries'} encoded_request = urlencode(request) u = urlopen(base_url, data=encoded_request, timeout=to) tree=etree.fromstring(u.read()) tempdict={} for sqelem in tree[:]: title=rft=id=None id=sqelem.get('id') for elem in sqelem[:]: if elem.tag==nspath('Title', WFS_NAMESPACE): title=elem.text elif elem.tag==nspath('ReturnFeatureType', WFS_NAMESPACE): rft=elem.text tempdict[id]=(title,rft) #store in temporary dictionary #then make the DescribeStoredQueries request and get the rest of the information about the stored queries try: base_url = next((m.get('url') for m in self.getOperationByName('DescribeStoredQueries').methods if m.get('type').lower() == method.lower())) except StopIteration: base_url = self.url request = {'service': 'WFS', 'version': self.version, 'request': 'DescribeStoredQueries'} encoded_request = urlencode(request) u = urlopen(base_url, data=encoded_request, timeout=to) tree=etree.fromstring(u.read()) tempdict2={} for sqelem in tree[:]: params=[] #list to store parameters for the stored query description id =sqelem.get('id') for elem in sqelem[:]: if elem.tag==nspath('Abstract', WFS_NAMESPACE): abstract=elem.text elif elem.tag==nspath('Parameter', WFS_NAMESPACE): newparam=Parameter(elem.get('name'), elem.get('type')) params.append(newparam) tempdict2[id]=(abstract, params) #store in another temporary dictionary #now group the results into StoredQuery objects: for key in tempdict.keys(): abstract='blah' parameters=[] sqs.append(StoredQuery(key, tempdict[key][0], tempdict[key][1], tempdict2[key][0], tempdict2[key][1])) return sqs
def openURL(url_base, data, method='Get', cookies=None, username=None, password=None): ''' function to open urls - wrapper around urllib2.urlopen but with additional checks for OGC service exceptions and url formatting, also handles cookies and simple user password authentication''' url_base.strip() lastchar = url_base[-1] if lastchar not in ['?', '&']: if url_base.find('?') == -1: url_base = url_base + '?' else: url_base = url_base + '&' if username and password: # Provide login information in order to use the WMS server # Create an OpenerDirector with support for Basic HTTP # Authentication... passman = HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, url_base, username, password) auth_handler = HTTPBasicAuthHandler(passman) opener = urllib.request.build_opener(auth_handler) openit = opener.open else: # NOTE: optionally set debuglevel>0 to debug HTTP connection #opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=0)) #openit = opener.open openit = urlopen try: if method == 'Post': req = Request(url_base, data) # set appropriate header if posting XML try: xml = etree.fromstring(data) req.add_header('Content-Type', "text/xml") except: pass else: req=Request(url_base + data) if cookies is not None: req.add_header('Cookie', cookies) u = openit(req) except HTTPError as e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised. if e.code in [400, 401]: raise ServiceException(e.read()) else: raise e # check for service exceptions without the http header set if u.info()['Content-Type'] in ['text/xml', 'application/xml']: #just in case 400 headers were not set, going to have to read the xml to see if it's an exception report. #wrap the url stram in a extended StringIO object so it's re-readable u=RereadableURL(u) se_xml= u.read() se_tree = etree.fromstring(se_xml) serviceException=se_tree.find('{http://www.opengis.net/ows}Exception') if serviceException is None: serviceException=se_tree.find('ServiceException') if serviceException is not None: raise ServiceException(str(serviceException.text).strip()) u.seek(0) #return cursor to start of u return u
def _getStoredQueries(self): ''' gets descriptions of the stored queries available on the server ''' sqs=[] #This method makes two calls to the WFS - one ListStoredQueries, and one DescribeStoredQueries. The information is then #aggregated in 'StoredQuery' objects method=nspath('Get') #first make the ListStoredQueries response and save the results in a dictionary if form {storedqueryid:(title, returnfeaturetype)} base_url = self.get_operation_by_name('ListStoredQueries').methods[method]['url'] request = {'service': 'WFS', 'version': self.version, 'request': 'ListStoredQueries'} data = urlencode(request) u = urlopen(base_url + data) tree=etree.fromstring(u.read()) base_url = self.get_operation_by_name('ListStoredQueries').methods[method]['url'] tempdict={} for sqelem in tree[:]: title=rft=id=None id=sqelem.get('id') for elem in sqelem[:]: if elem.tag==nspath('Title',ns.get_namespace('wfs')): title=elem.text elif elem.tag==nspath('ReturnFeatureType',ns.get_namespace('wfs')): rft=elem.text tempdict[id]=(title,rft) #store in temporary dictionary #then make the DescribeStoredQueries request and get the rest of the information about the stored queries base_url = self.get_operation_by_name('DescribeStoredQueries').methods[method]['url'] request = {'service': 'WFS', 'version': self.version, 'request': 'DescribeStoredQueries'} data = urlencode(request) u = urlopen(base_url + data) tree=etree.fromstring(u.read()) tempdict2={} for sqelem in tree[:]: params=[] #list to store parameters for the stored query description id =sqelem.get('id') for elem in sqelem[:]: if elem.tag==nspath('Abstract',ns.get_namespace('wfs')): abstract=elem.text elif elem.tag==nspath('Parameter',ns.get_namespace('wfs')): newparam=Parameter(elem.get('name'), elem.get('type')) params.append(newparam) tempdict2[id]=(abstract, params) #store in another temporary dictionary #now group the results into StoredQuery objects: for key in tempdict.keys(): abstract='blah' parameters=[] sqs.append(StoredQuery(key, tempdict[key][0], tempdict[key][1], tempdict2[key][0], tempdict2[key][1])) return sqs
def _getStoredQueries(self): """ gets descriptions of the stored queries available on the server """ sqs = [] # This method makes two calls to the WFS - one ListStoredQueries, and one DescribeStoredQueries. The information is then # aggregated in 'StoredQuery' objects method = nspath("Get") # first make the ListStoredQueries response and save the results in a dictionary if form {storedqueryid:(title, returnfeaturetype)} base_url = self.getOperationByName("ListStoredQueries").methods[method]["url"] request = {"service": "WFS", "version": self.version, "request": "ListStoredQueries"} data = urlencode(request) u = urlopen(base_url + data) tree = etree.fromstring(u.read()) base_url = self.getOperationByName("ListStoredQueries").methods[method]["url"] tempdict = {} for sqelem in tree[:]: title = rft = id = None id = sqelem.get("id") for elem in sqelem[:]: if elem.tag == nspath("Title", WFS_NAMESPACE): title = elem.text elif elem.tag == nspath("ReturnFeatureType", WFS_NAMESPACE): rft = elem.text tempdict[id] = (title, rft) # store in temporary dictionary # then make the DescribeStoredQueries request and get the rest of the information about the stored queries base_url = self.getOperationByName("DescribeStoredQueries").methods[method]["url"] request = {"service": "WFS", "version": self.version, "request": "DescribeStoredQueries"} data = urlencode(request) u = urlopen(base_url + data) tree = etree.fromstring(u.read()) tempdict2 = {} for sqelem in tree[:]: params = [] # list to store parameters for the stored query description id = sqelem.get("id") for elem in sqelem[:]: if elem.tag == nspath("Abstract", WFS_NAMESPACE): abstract = elem.text elif elem.tag == nspath("Parameter", WFS_NAMESPACE): newparam = Parameter(elem.get("name"), elem.get("type")) params.append(newparam) tempdict2[id] = (abstract, params) # store in another temporary dictionary # now group the results into StoredQuery objects: for key in tempdict.keys(): abstract = "blah" parameters = [] sqs.append(StoredQuery(key, tempdict[key][0], tempdict[key][1], tempdict2[key][0], tempdict2[key][1])) return sqs
def describe_sensor(self, outputFormat=None, procedure=None, method=None, **kwargs): method = method or 'Get' try: base_url = next((m.get('url') for m in self.getOperationByName('DescribeSensor').methods if m.get('type').lower() == method.lower())) except StopIteration: base_url = self.url request = {'service': 'SOS', 'version': self.version, 'request': 'DescribeSensor'} # Required Fields assert isinstance(outputFormat, str) request['procedureDescriptionFormat'] = outputFormat assert isinstance(procedure, str) request['procedure'] = procedure url_kwargs = {} if 'timeout' in kwargs: url_kwargs['timeout'] = kwargs.pop('timeout') # Client specified timeout value # Optional Fields if kwargs: for kw in kwargs: request[kw] = kwargs[kw] response = openURL(base_url, request, method, username=self.username, password=self.password, **url_kwargs).read() tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) return response
def get_feature_of_interest(self, foi=None, method='Get', **kwargs): try: base_url = self.get_operation_by_name('GetFeatureOfInterest').methods[method]['url'] except: base_url = self.url request = {'service': 'SOS', 'version': self.version, 'request': 'GetFeatureOfInterest'} # Required Fields assert isinstance(foi, str) request['featureOfInterest'] = foi # Optional Fields if kwargs: for kw in kwargs: request[kw]=kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password).read() tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) return tr
def test_wps_literal_data_input_parsing_references(): xml = open(resource_file('wps_inout_parsing.xml'), 'r').read() inputs = etree.fromstring(xml) for i, i_elem in enumerate(inputs): wps_in = Input(i_elem) assert wps_in.identifier == 'input{}'.format(i + 1) assert wps_in.dataType == 'string'
def describe_sensor(self, outputFormat=None, procedure=None, method='Get', **kwargs): base_url = self.get_operation_by_name('DescribeSensor').methods[method]['url'] request = {'service': 'SOS', 'version': self.version, 'request': 'DescribeSensor'} # Required Fields assert isinstance(outputFormat, str) request['outputFormat'] = outputFormat assert isinstance(procedure, str) request['procedure'] = procedure # Optional Fields if kwargs: for kw in kwargs: request[kw]=kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password).read() tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) return response
def cleanup_namespaces(element): """ Remove unused namespaces from an element """ if etree.__name__ == 'lxml.etree': etree.cleanup_namespaces(element) return element else: return etree.fromstring(etree.tostring(element))
def read(*args, **kwargs): with open('tests/resources/wfs_dov_getcapabilities_110.xml', 'r') as f: data = f.read() if type(data) is not bytes: data = data.encode('utf-8') data = etree.fromstring(data) return data
def openURL(*args, **kwargs): with open('tests/resources/csw_dov_getrecordbyid.xml', 'r') as f: data = f.read() if type(data) is not bytes: data = data.encode('utf-8') data = etree.fromstring(data) return data
def getCoverages(self, unpackdir='./unpacked', returnRaw=False): """ returnRaw - boolean - default: False, don't return the raw possibly multipart mime message, rather the actual coverage data as a file-like object. When set to true, returns the raw file """ if self.urlType=='XML': paths=[] u_xml = self.u.read() u_tree = etree.fromstring(u_xml) for ref in u_tree.findall('{http://www.opengis.net/wcs/1.1}Coverage/{http://www.opengis.net/wcs/1.1}Reference'): path = ref.attrib['{http://www.w3.org/1999/xlink}href'] paths.append(path) for ref in u_tree.findall('{http://www.opengis.net/wcs/1.1.0/owcs}Coverage/{{http://www.opengis.net/wcs/1.1.0/owcs}Reference'): path = ref.attrib['{http://www.w3.org/1999/xlink}href'] paths.append(path) if len(paths) == 0: paths.append(u_xml) elif self.urlType=='Multipart': #Decode multipart mime and return fileobjects u_mpart=self.u.read() if returnRaw: mpart =MpartMime(u_mpart) paths= mpart.unpackToDir(unpackdir) else: mpart = MpartMime(str(self.u.info())+u_mpart) return mpart.unpackToStream() return paths
def describe_sensor(self, outputFormat=None, procedure=None, method="Get", **kwargs): base_url = self.get_operation_by_name("DescribeSensor").methods[method]["url"] request = {"service": "SOS", "version": self.version, "request": "DescribeSensor"} # Required Fields assert isinstance(outputFormat, str) request["outputFormat"] = outputFormat assert isinstance(procedure, str) request["procedure"] = procedure # Optional Fields if kwargs: for kw in kwargs: request[kw] = kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password).read() tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) return response
def get_grid_refs(shapefile_name): #Retrieve Shapefile Layer ID and GRID_REFs list from WFS to Geoserver #~ wfs_version="1.0.0" #~ username="******" #~ password="******" #~ method="Get" #~ base_url=OGC_SERVER['default']['PUBLIC_LOCATION'] + "wfs" #~ layer_name=shapefile_name request = { "version" : WFS_SETTINGS['VERSION'], "request" : "GetFeature", "format_options": "charset:UTF-8", "service" : "WFS", "typename" : shapefile_name, } data = urlencode(request) print("WFS URL: {0}/{1}".format(WFS_SETTINGS['URL'],data)) u = openURL(WFS_SETTINGS['URL'], data, 'Get', username = OGC_SERVER['default']['USER'], password = OGC_SERVER['default']['PASSWORD']) #Create lxml tree and namespace map wfs_xml = u.read() wfs_etree = etree.fromstring(wfs_xml) wfs_nsmap = wfs_etree.nsmap wfs_nsmap.pop(None,None) #Get a list of gridrefs using xpath return list(wfs_etree.xpath("//{0}GRID_REF/text()".format(WFS_SETTINGS['GEONODE_NS']), namespaces=wfs_nsmap))
def test_sos_20_bom_gov_au(): # Setup service = SensorObservationService(SERVICE_URL, version='2.0.0') id = service.identification # Check basic service metadata assert id.service == 'SOS' assert id.title == 'KISTERS KiWIS SOS2' assert id.keywords == [] assert service.provider.name == 'Provider Name' assert service.provider.contact.name == 'Name' assert service.provider.contact.position is None assert len(service.operations) == 5 assert service.get_operation_by_name('GetObservation').methods[0]['url'] == \ 'http://bom.gov.au/waterdata/services?datasource=0' response = service.get_observation( featureOfInterest='http://bom.gov.au/waterdata/services/stations/181.1', offerings=['http://bom.gov.au/waterdata/services/tstypes/Pat4_PC_1'], observedProperties=['http://bom.gov.au/waterdata/services/parameters/Water Course Discharge'], eventTime='om:phenomenonTime,2016-01-01T00:00:00+10/2016-03-05T00:00:00+10') # Process WaterML Response from owslib.etree import etree from owslib.swe.observation.sos200 import SOSGetObservationResponse from owslib.swe.observation.waterml2 import MeasurementTimeseriesObservation et = etree.fromstring(response) parsed_response = SOSGetObservationResponse(et) assert len(parsed_response.observations) > 0 for o in parsed_response.observations: assert isinstance(o, MeasurementTimeseriesObservation)
def get_observation(self, responseFormat=None, offerings=None, observedProperties=None, eventTime=None, method='Get', **kwargs): """ Parameters ---------- format : string Output format. Provide one that is available for all offerings method : string Optional. HTTP DCP method name: Get or Post. Must **kwargs : extra arguments anything else e.g. vendor specific parameters """ try: base_url = next((m.get('url') for m in self.getOperationByName('GetObservation').methods if m.get('type').lower() == method.lower())) except StopIteration: base_url = self.url request = {'service': 'SOS', 'version': self.version, 'request': 'GetObservation'} # Required Fields assert isinstance(offerings, list) and len(offerings) > 0 request['offering'] = ','.join(offerings) assert isinstance(observedProperties, list) and len(observedProperties) > 0 request['observedProperty'] = ','.join(observedProperties) assert isinstance(responseFormat, str) request['responseFormat'] = responseFormat # Optional Fields if eventTime is not None: request['eventTime'] = eventTime url_kwargs = {} if 'timeout' in kwargs: url_kwargs['timeout'] = kwargs.pop('timeout') # Client specified timeout value if kwargs: for kw in kwargs: request[kw]=kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password, **kwargs).read() try: tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) else: return response except ows.ExceptionReport: raise except BaseException: return response
def readString(self, st): """Parse a WCS capabilities document, returning an instance of WCSCapabilitiesInfoset string should be an XML capabilities document """ if not isinstance(st, str): raise ValueError("String must be of type string, not %s" % type(st)) return etree.fromstring(st)
def readString(self, st): """Parse a WMS capabilities document, returning an elementtree instance string should be an XML capabilities document """ if not isinstance(st, str) and not isinstance(st, bytes): raise ValueError("String must be of type string or bytes, not %s" % type(st)) return etree.fromstring(st)
def get_observation( self, responseFormat=None, offerings=None, observedProperties=None, eventTime=None, method="Get", **kwargs ): """ Parameters ---------- format : string Output format. Provide one that is available for all offerings method : string Optional. HTTP DCP method name: Get or Post. Must **kwargs : extra arguments anything else e.g. vendor specific parameters """ try: base_url = next( ( m.get("url") for m in self.getOperationByName("GetObservation").methods if m.get("type").lower() == method.lower() ) ) except StopIteration: base_url = self.url request = {"service": "SOS", "version": self.version, "request": "GetObservation"} # Required Fields assert isinstance(offerings, list) and len(offerings) > 0 request["offering"] = ",".join(offerings) assert isinstance(observedProperties, list) and len(observedProperties) > 0 request["observedProperty"] = ",".join(observedProperties) assert isinstance(responseFormat, str) request["responseFormat"] = responseFormat # Optional Fields if eventTime is not None: request["eventTime"] = eventTime if kwargs: for kw in kwargs: request[kw] = kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password).read() try: tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) else: return response except ows.ExceptionReport: raise except BaseException: return response
def _get_etree_for_record(self, uuid): """ Set self.record_etree to etree ElementTree of record with inputted uuid. """ xml = self.records[uuid].xml root = etree.fromstring(xml) return etree.ElementTree(root)
def _open_url(self, url_base, data=None, method='Get', cookies=None, username=None, password=None, timeout=30, headers=None): headers = headers if headers is not None else {} kwargs = dict(timeout=timeout) kwargs['auth'] = self.auth method = method.split("}")[-1] if method.lower() == 'post': try: _ = etree.fromstring(data) headers['Content-Type'] = 'text/xml' except (ParseError, UnicodeEncodeError): pass kwargs['data'] = data elif method.lower() == 'get': kwargs['params'] = data else: raise ValueError("Unknown method ('%s'), expected 'get' or 'post'" % method) if cookies is not None: kwargs['cookies'] = cookies req = requests.request(method.upper(), url_base, headers=headers, **kwargs) if req.status_code in [400, 401]: raise ServiceException(req.text) if req.status_code in [404]: # add more if needed req.raise_for_status() if 'Content-Type' in req.headers and req.headers['Content-Type'] in ['text/xml', 'application/xml']: se_tree = etree.fromstring(req.content) serviceException = se_tree.find('{http://www.opengis.net/ows}Exception') if serviceException is None: serviceException = se_tree.find('ServiceException') if serviceException is not None: raise ServiceException(str(serviceException.text).strip()) return ResponseWrapper(req)
def __init__(self, element): if isinstance(element, str): self._root = etree.fromstring(element) else: self._root = element if hasattr(self._root, 'getroot'): self._root = self._root.getroot() self.members = [Member(x) for x in self._root.findall(nsp('sml:member'))]
def parse_om_xml(response): # extract data and time from OM-XML response root = etree.fromstring(response) # root.findall(".//{%(om)s}Observation" % root.nsmap ) values = root.find(".//{%(swe)s}values" % root.nsmap ) date_value = array( [ (dt.datetime.strptime(d,"%Y-%m-%dT%H:%M:%SZ"),float(v)) for d,v in [l.split(',') for l in values.text.split()]] ) time = date_value[:,0] data = date_value[:,1] return data,time
def __init__(self, element): if isinstance(element, str) or isinstance(element, unicode): self._root = etree.fromstring(str(element)) else: self._root = element if hasattr(self._root, 'getroot'): self._root = self._root.getroot() self._ns = 'wml1.1'
def __init__(self, element): if isinstance(element, ElementType): self._root = element else: self._root = etree.fromstring(element) if hasattr(self._root, "getroot"): self._root = self._root.getroot() self._ns = "wml1.1"
def read(self, url): """Get and parse a WFS capabilities document, returning an instance of WFSCapabilitiesInfoset Parameters ---------- url : string The URL to the WFS capabilities document. """ request = self.capabilities_url(url) u = urlopen(request) return etree.fromstring(u.read())
def read(self, service_url): """ Get and parse a WMS capabilities document, returning an elementtree instance service_url is the base url, to which is appended the service, acceptVersions, and request parameters """ getcaprequest = self.capabilities_url(service_url) spliturl=getcaprequest.split('?') u = openURL(spliturl[0], spliturl[1], method='Get', username=self.username, password=self.password) return etree.fromstring(u.read())
def __init__(self, element): if isinstance(element, str): self._root = etree.fromstring(element) else: self._root = element if hasattr(self._root, 'getroot'): self._root = self._root.getroot() self.systems = [] for system in self._root.findall(nspath_eval('sml:member/sml:System', namespaces)): self.systems.append(SystemMetadata(system))
def read(self, service_url, timeout=30): """Get and parse a WMS capabilities document, returning an elementtree instance service_url is the base url, to which is appended the service, version, and request parameters """ self.request = self.capabilities_url(service_url) # now split it up again to use the generic openURL function... spliturl = self.request.split('?') u = openURL(spliturl[0], spliturl[1], method='Get', timeout=timeout, headers=self.headers, auth=self.auth) raw_text = strip_bom(u.read()) return etree.fromstring(raw_text)
def getcapabilities(self): """Request and return capabilities document from the WMS as a file-like object. NOTE: this is effectively redundant now""" reader = WMSCapabilitiesReader(self.version, url=self.url, auth=self.auth) u = self._open(reader.capabilities_url(self.url)) # check for service exceptions, and return if u.info()['Content-Type'] == 'application/vnd.ogc.se_xml': se_xml = u.read() se_tree = etree.fromstring(se_xml) err_message = str(se_tree.find('ServiceException').text).strip() raise ServiceException(err_message) return u
def test_wps_getcapabilities_parse_languages(): xml = """ <wps:Languages xmlns:wps="http://www.opengis.net/wps/1.0.0" xmlns:ows="http://www.opengis.net/ows"> <wps:Default> <ows:Language>en-US</ows:Language> </wps:Default> <wps:Supported> <ows:Language>en-US</ows:Language> <ows:Language>fr-CA</ows:Language> </wps:Supported> </wps:Languages> """ element = etree.fromstring(xml) languages = owslib.wps.Languages(element) assert languages.default == 'en-US' assert languages.supported == ['en-US', 'fr-CA']
def read(self, service_url): """Get and parse a Describe Coverage document, returning an elementtree tree @type service_url: string @param service_url: The base url, to which is appended the service, version, and request parameters @rtype: elementtree tree @return: An elementtree tree representation of the capabilities document """ request = self.descCov_url(service_url) req = Request(request) if self.cookies is not None: req.add_header('Cookie', self.cookies) u = urlopen(req) return etree.fromstring(u.read())
def read(self, service_url, timeout=30): """Get and parse a WCS capabilities document, returning an elementtree tree @type service_url: string @param service_url: The base url, to which is appended the service, version, and request parameters @rtype: elementtree tree @return: An elementtree tree representation of the capabilities document """ request = self.capabilities_url(service_url) u = openURL(request, timeout=timeout, cookies=self.cookies, auth=self.auth) return etree.fromstring(u.read())
def read(self, service_url): """ Get and parse a WMS capabilities document, returning an elementtree instance service_url is the base url, to which is appended the service, acceptVersions, and request parameters """ getcaprequest = self.capabilities_url(service_url) spliturl = getcaprequest.split('?') u = openURL(spliturl[0], spliturl[1], method='Get', username=self.username, password=self.password) return etree.fromstring(u.read())
def get_capabilities(self): json_out = { "max_features": None, "filter": None, "layers": {}, } all_epsg_code = None all_layer_property = None for layer_name in self.wfs.contents: wfs_response = self.wfs.getfeature(typename=layer_name, maxfeatures=1) tree = etree.fromstring(wfs_response.read()) nsmap = tree.nsmap layer_property = [] for feature in tree.getiterator("{%s}featureMember" % nsmap["gml"]): for layer in feature.iterfind('{%s}*' % nsmap["ms"]): for meta in layer.iterfind('{%s}*' % nsmap["ms"]): meta_name = meta.tag.split("{%s}" % nsmap[meta.prefix])[-1] layer_property.append(meta_name) epsg_code = [ my.code for my in self.wfs.contents[layer_name].crsOptions ] json_out['layers'][layer_name] = { "epsg_code": epsg_code, "layer_property": layer_property, "max_features": None, "filter": None, } if not all_layer_property: all_layer_property = layer_property else: all_layer_property = list( set(all_layer_property).intersection(set(layer_property))) if not all_epsg_code: all_epsg_code = epsg_code else: all_epsg_code = list( set(all_epsg_code).intersection(set(epsg_code))) json_out.update({ "epsg_code": all_epsg_code, "layer_property": all_layer_property, }) if self.debug: self.echo2json(json_out) return json_out
def describe_sensor(self, outputFormat=None, procedure=None, method='Get', raw=False, **kwargs): try: base_url = self.get_operation_by_name( 'DescribeSensor').methods[method]['url'] except: base_url = self.url request = { 'service': 'SOS', 'version': self.version, 'request': 'DescribeSensor' } # Required Fields assert isinstance(outputFormat, str) request['procedureDescriptionFormat'] = outputFormat assert isinstance(procedure, str) request['procedure'] = procedure # Optional Fields if kwargs: for kw in kwargs: request[kw] = kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password).read() tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) if raw: return response else: return SosDescribeSensorResponse(tr, namespaces)
def getCoverages(self, unpackdir='./unpacked'): if self.urlType=='XML': paths=[] u_xml = self.u.read() u_tree = etree.fromstring(u_xml) for ref in u_tree.findall('{http://www.opengis.net/wcs/1.1}Coverage/{http://www.opengis.net/wcs/1.1}Reference'): path = ref.attrib['{http://www.w3.org/1999/xlink}href'] paths.append(path) for ref in u_tree.findall('{http://www.opengis.net/wcs/1.1.0/owcs}Coverage/{{http://www.opengis.net/wcs/1.1.0/owcs}Reference'): path = ref.attrib['{http://www.w3.org/1999/xlink}href'] paths.append(path) elif self.urlType=='Multipart': #Decode multipart mime and return fileobjects u_mpart=self.u.read() mpart =MpartMime(u_mpart) paths= mpart.unpackToDir(unpackdir) return paths
def read(self, url, timeout=30): """Get and parse a WFS capabilities document, returning an instance of WFSCapabilitiesInfoset Parameters ---------- url : string The URL to the WFS capabilities document. timeout : number A timeout value (in seconds) for the request. """ request = self.capabilities_url(url) u = openURL(request, timeout=timeout, headers=self.headers, auth=self.auth) return etree.fromstring(u.read())
def test_ows_bbox_with_namespaces(): """ XML bounding box description as received from a wps request """ bbox_elem = etree.fromstring(""" <wps:BoundingBoxData xmlns:wps="{}" xmlns:ows="{}" ows:crs="EPSG:4326" ows:dimensions="2"> <ows:LowerCorner>0.0 -90.0</ows:LowerCorner> <ows:UpperCorner>180.0 90.0</ows:UpperCorner> </wps:BoundingBoxData>""".format(DEFAULT_WPS_NAMESPACE, DEFAULT_OWS_NAMESPACE)) bbox = BoundingBox(bbox_elem) assert bbox.crs == crs.Crs('EPSG:4326') assert bbox.crs.axisorder == 'yx' assert bbox.dimensions == 2 assert bbox.minx == '-90.0' assert bbox.miny == '0.0' assert bbox.maxx == '90.0' assert bbox.maxy == '180.0'
def from_xml(cls, xml_data): """Build instances of this subtype from XML data. Parameters ---------- xml_data : bytes Raw XML data of the DOV object that contains information about this subtype. Yields ------ An instance of this type for each occurence of the rootpath in the XML document. """ tree = etree.fromstring(xml_data) for element in tree.findall(cls._rootpath): yield cls.from_xml_element(element)
def describe_sensor(self, outputFormat=None, procedure=None, method='Get', **kwargs): try: base_url = next( (m.get('url') for m in self.getOperationByName('DescribeSensor').methods if m.get('type').lower() == method.lower())) except StopIteration: base_url = self.url request = { 'service': 'SOS', 'version': self.version, 'request': 'DescribeSensor' } # Required Fields assert isinstance(outputFormat, str) request['outputFormat'] = outputFormat assert isinstance(procedure, str) request['procedure'] = procedure # Optional Fields if kwargs: for kw in kwargs: request[kw] = kwargs[kw] data = urlencode(request) response = openURL(base_url, data, method, username=self.username, password=self.password).read() tr = etree.fromstring(response) if tr.tag == nspath_eval("ows:ExceptionReport", namespaces): raise ows.ExceptionReport(tr) return response
def openURL(url_base, data, method='Get', cookies=None, username=None, password=None): ''' function to open urls - wrapper around urllib2.urlopen but with additional checks for OGC service exceptions and url formatting, also handles cookies and simple user password authentication''' url_base.strip() lastchar = url_base[-1] if lastchar not in ['?', '&']: if url_base.find('?') == -1: url_base = url_base + '?' else: url_base = url_base + '&' if username and password: # Provide login information in order to use the WMS server # Create an OpenerDirector with support for Basic HTTP # Authentication... passman = HTTPPasswordMgrWithDefaultRealm() passman.add_password(None, url_base, username, password) auth_handler = HTTPBasicAuthHandler(passman) opener = urllib2.build_opener(auth_handler) openit = opener.open else: # NOTE: optionally set debuglevel>0 to debug HTTP connection #opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=0)) #openit = opener.open openit = urlopen try: if method == 'Post': req = Request(url_base, data) # set appropriate header if posting XML try: xml = etree.fromstring(data) req.add_header('Content-Type', "text/xml") except: pass else: req=Request(url_base + data) if cookies is not None: req.add_header('Cookie', cookies) u = openit(req) except HTTPError, e: #Some servers may set the http header to 400 if returning an OGC service exception or 401 if unauthorised. if e.code in [400, 401]: raise ServiceException, e.read() else: raise e
def test_from_wfs_list(self, wfs_getfeature): """Test the from_wfs method to construct objects from a WFS response, as list of elements. Parameters ---------- wfs_getfeature : pytest.fixture returing str Fixture providing a WFS GetFeature response. """ tree = etree.fromstring(wfs_getfeature.encode('utf8')) feature_members = tree.findall('.//{http://www.opengis.net/gml}' 'featureMembers') features = self.get_type().from_wfs(feature_members, self.get_namespace()) for feature in features: assert type(feature) is self.get_type()
def get_schema(url, typename, version="1.0.0", timeout=30, username=None, password=None, auth=None): """Parses DescribeFeatureType response and creates schema compatible with :class:`fiona` :param str url: url of the service :param str version: version of the service :param str typename: name of the layer :param int timeout: request timeout :param str username: service authentication username :param str password: service authentication password :param Authentication auth: instance of owslib.util.Authentication """ if auth: if username: auth.username = username if password: auth.password = password else: auth = Authentication(username, password) url = _get_describefeaturetype_url(url, version, typename) res = openURL(url, timeout=timeout, auth=auth) root = etree.fromstring(res.read()) if ":" in typename: typename = typename.split(":")[1] type_element = findall( root, "{%s}element" % XS_NAMESPACE, attribute_name="name", attribute_value=typename, )[0] complex_type = type_element.attrib["type"].split(":")[1] elements = _get_elements(complex_type, root) nsmap = None if hasattr(root, "nsmap"): nsmap = root.nsmap return _construct_schema(elements, nsmap)
def prefix_xsl_line(req, id): # if the layer is in the catalogue, try to get the distribution urls # that cannot be precalculated. resource = None try: resource = get_object_or_404(ResourceBase, pk=id) query = {'id': resource.get_real_instance().id} resource = resolve_object( req, resource.get_real_instance_class(), query, permission='base.view_resourcebase', permission_msg=_("You are not permitted to view this resource")) catalogue = get_catalogue() record = catalogue.get_record(resource.uuid) if record: logger.debug(record.xml) except PermissionDenied: return HttpResponse(_("Not allowed"), status=403) except Exception: logger.debug(traceback.format_exc()) msg = f'Could not connect to catalogue to save information for dataset "{str(resource)}"' return HttpResponse(msg, status=404) try: # generate an XML document (GeoNode's default is ISO) if resource.metadata_uploaded and resource.metadata_uploaded_preserve: md_doc = etree.tostring(dlxml.fromstring(resource.metadata_xml)) else: md_doc = catalogue.catalogue.csw_gen_xml( resource, settings.CATALOG_METADATA_TEMPLATE) xml = md_doc except Exception: logger.debug(traceback.format_exc()) return HttpResponse("Resource Metadata not available!") site_url = settings.SITEURL.rstrip('/') if settings.SITEURL.startswith( 'http') else settings.SITEURL xsl_static = getattr(settings, 'CATALOG_METADATA_XSL', '/static/metadataxsl/metadata.xsl') xsl_path = f'{site_url}{xsl_static}' xsl_line = f'<?xml-stylesheet type="text/xsl" href="{xsl_path}"?>' return HttpResponse(xsl_line + xml, content_type="text/xml")
def sensors(self): if self._sensors is None: sensors = {} get_obs = self.service.get_operation_by_name('GetObservation') describe_sensor = self.service.get_operation_by_name( 'DescribeSensor') sensor_ids = describe_sensor.parameters['procedure']['values'] for sensor_id in sensor_ids: response = self.service.describe_sensor( outputFormat='http://www.opengis.net/sensorML/1.0.1', procedure=sensor_id) tr = etree.fromstring(response) element = tr.find('.//' + nspath_eval('sml:SensorML', namespaces)) ds = SensorML(element) name = ds.members[0].name sensors[sensor_id] = name self._sensors = sensors return self._sensors
def custom_parsing(xml, vals, keywords): exml = etree.fromstring(xml.encode()) date = ( datetime.strftime(datetime.strptime(exml.find("date").text, "%Y"), "%Y-%m-%d") if not exml.find("date") else None ) vals = { "title": exml.find("title").text if exml.find("title") is not None else None, "date": date or datetime.now(), "edition": exml.find("edition").text if exml.find("edition") is not None else None, "abstract": exml.find("abstract").text if exml.find("abstract") is not None else None, "purpose": exml.find("purpose").text if exml.find("purpose") is not None else None, "supplemental_information": exml.find("supplemental_information").text if exml.find("supplemental_information") is not None else None, "data_quality_statement": exml.find("data_quality_statement").text if exml.find("data_quality_statement") is not None else None, "typename": exml.find("typename").text if exml.find("typename") is not None else None, "is_published": exml.find("is_published").text if exml.find("is_published") is not None else False, } kws = [] for kw in exml.findall("keywords"): kws = [k.text for k in kw.findall("item")] ''' Check if the keywords extracted from the XML are available in GeoNode. If the keyword is not available in GeoNode, is ignored ''' valid_keywords = [] for k in kws: is_available = HierarchicalKeyword.objects.filter(name=k).exists() if is_available: valid_keywords.append(k) if len(valid_keywords) == 0: valid_keywords.append('GeoTIFF') keywords = convert_keyword(valid_keywords or []) return vals, keywords, keywords
def execute(cls, instance, inputs=[]): """ e.g.: inputs = [("latitude","45.06"), ("longitude","6.34"), ("magnitude","8.5")] """ if not instance.is_initialized(): exception = Exception("Not initialized!") log.exception("Not initialized!") raise exception if instance.execution and isinstance( instance.execution, WebProcessingServiceExecution) and instance.execution.response: return instance.execution.status else: instance.execution = WebProcessingServiceExecution.objects.create( process=instance) _f = open(instance.request_template.path) _f_xml = _f.read() _req = etree.fromstring(_f_xml) _inputs = _req.xpath('.//wps:DataInputs/wps:Input', namespaces=namespaces) _subst_count = 0 if _inputs: for (key, val) in inputs: for _i in _inputs: try: identifier = _i.xpath('./ows:Identifier', namespaces=namespaces)[0].text if key in identifier: _i.xpath('./wps:Data/wps:LiteralData', namespaces=namespaces)[0].text = val _subst_count += 1 except: pass if _subst_count > 0: _req = etree.tostring(_req) else: _req = _f_xml _e = instance._wps.execute(None, [], request=_req) cls._update_instance_execution_status(instance, _e) return instance.execution.status
def from_wfs(cls, response, namespace): """Build instances of this type from a WFS response. Parameters ---------- response : str or bytes or etree.Element or iterable<etree.Element> WFS response containing GML features. Can either be a GML `str` or `byte` sequence, in which case it will be parsed and scanned for `gml:featureMembers`. Can also be a single instance of `etree.Element` containing the parsed GML response. It can also be an iterable (list, tuple or generator) of `etree.Element` in which case it will be looped over. namespace : str Namespace associated with this WFS featuretype. Yields ------ An instance of this type for each record in the WFS response. """ if type(response) is str: response = response.encode('utf-8') if type(response) is bytes: response = etree.fromstring(response) element_type = type(etree.Element(b'xml')) if isinstance(response, element_type): feature_members = response.find('.//{http://www.opengis.net/gml}' 'featureMembers') if feature_members is not None: for ft in feature_members: yield (cls.from_wfs_element(ft, namespace)) if type(response) in (list, tuple, set) \ or isinstance(response, types.GeneratorType): for el in response: yield (cls.from_wfs_element(el, namespace))
def wfs_feature(request): """PyTest fixture providing an XML of a WFS feature element. This monkeypatch requires a module variable ``location_wfs_feature`` with the path to the wfs_feature file on disk. Parameters ---------- request : pytest.fixtue PyTest fixture providing request context. Returns ------- etree.Element XML element representing a single record of the Boring WFS layer. """ file_path = getattr(request.module, "location_wfs_feature") with open(file_path, 'r') as f: return etree.fromstring(f.read())
def parse_remote_metadata(self, timeout=30): """Parse remote metadata for MetadataURL and add it as metadataUrl['metadata']""" for metadataUrl in self.metadataUrls: if metadataUrl['url'] is not None: try: content = openURL(metadataUrl['url'], timeout=timeout) doc = etree.fromstring(content.read()) mdelem = doc.find('.//metadata') if mdelem is not None: metadataUrl['metadata'] = Metadata(mdelem) continue mdelem = doc.find('.//' + util.nspath_eval('gmd:MD_Metadata', n.get_namespaces(['gmd']))) \ or doc.find('.//' + util.nspath_eval('gmi:MI_Metadata', n.get_namespaces(['gmi']))) if mdelem is not None: metadataUrl['metadata'] = MD_Metadata(mdelem) continue except: metadataUrl['metadata'] = None
def test_from_wfs_list(self, wfs_getfeature): """Test the from_wfs method to construct objects from a WFS response, as list of elements. Parameters ---------- wfs_getfeature : pytest.fixture returning str Fixture providing a WFS GetFeature response. """ tree = etree.fromstring(wfs_getfeature.encode('utf8')) feature_members = tree.find('.//{http://www.opengis.net/gml}' 'featureMembers') if feature_members is not None: fts = [ft for ft in feature_members] features = self.datatype_class.from_wfs(fts, self.namespace) for feature in features: assert isinstance(feature, self.datatype_class)
def parse_remote_metadata(self, timeout=30): """Parse remote metadata for MetadataURL and add it as metadataUrl['metadata']""" for metadataUrl in self.metadataUrls: if metadataUrl['url'] is not None \ and metadataUrl['format'].lower() in ['application/xml', 'text/xml']: # download URL try: content = openURL(metadataUrl['url'], timeout=timeout) doc = etree.fromstring(content.read()) mdelem = doc.find('.//metadata') if mdelem is not None: metadataUrl['metadata'] = Metadata(mdelem) continue mdelem = doc.find('.//' + nspath_eval('gmd:MD_Metadata', n.get_namespaces(['gmd']))) \ or doc.find('.//' + nspath_eval('gmi:MI_Metadata', n.get_namespaces(['gmi']))) if mdelem is not None: metadataUrl['metadata'] = MD_Metadata(mdelem) continue except Exception: metadataUrl['metadata'] = None
def test_get_featurecatalogue_uuid_nocontentinfo(self, md_metadata): """Test the owsutil.get_featurecatalogue_uuid method when the metadata is missing the gmd:contentInfo element. Test whether a FeatureCatalogueNotFoundError is raised. Parameters ---------- md_metadata : pytest.fixture providing owslib.iso.MD_Metadata Parsed metadata describing the Boringen WFS layer in more detail, in the ISO 19115/19139 format. """ tree = etree.fromstring(md_metadata.xml) for ci in tree.findall( './/{http://www.isotc211.org/2005/gmd}contentInfo'): tree.remove(ci) md_metadata = MD_Metadata(tree) with pytest.raises(FeatureCatalogueNotFoundError): owsutil.get_featurecatalogue_uuid(md_metadata)
def get_record(self, uuid): results = self._csw_local_dispatch(identifier=uuid) if len(results) < 1: return None result = dlxml.fromstring(results).find( '{http://www.isotc211.org/2005/gmd}MD_Metadata') if result is None: return None record = MD_Metadata(result) record.keywords = [] if hasattr(record, 'identification') and hasattr( record.identification, 'keywords'): for kw in record.identification.keywords: record.keywords.extend(kw['keywords']) record.links = {} record.links['metadata'] = self.catalogue.urls_for_uuid(uuid) record.links['download'] = self.catalogue.extract_links(record) return record
def parse_remote_metadata(self, timeout=30): """Parse remote metadata for MetadataURL and add it as metadataUrl['metadata']""" for metadataUrl in self.metadataUrls: if metadataUrl["url"] is not None: try: content = openURL(metadataUrl["url"], timeout=timeout) doc = etree.fromstring(content.read()) mdelem = doc.find(".//metadata") if mdelem is not None: metadataUrl["metadata"] = Metadata(mdelem) continue mdelem = doc.find(".//" + util.nspath_eval( "gmd:MD_Metadata", n.get_namespaces( ["gmd"]))) or doc.find(".//" + util.nspath_eval( "gmi:MI_Metadata", n.get_namespaces(["gmi"]))) if mdelem is not None: metadataUrl["metadata"] = MD_Metadata(mdelem) continue except Exception: metadataUrl["metadata"] = None
def test_map_to_wmc(self): """ /maps/1/wmc -> Test map WMC export Make some assertions about the data structure produced for serialization to a Web Map Context Document """ map_obj = Map.objects.all().first() map_obj.set_default_permissions() response = self.client.get(reverse('map_wmc', args=(map_obj.id, ))) self.assertEqual(response.status_code, 200) # check specific XPaths wmc = dlxml.fromstring(response.content) ns = '{http://www.opengis.net/context}' title = f'{ns}General/{ns}Title' abstract = f'{ns}General/{ns}Abstract' self.assertIsNotNone(wmc.attrib.get('id')) self.assertEqual(wmc.find(title).text, 'GeoNode Default Map') self.assertEqual( wmc.find(abstract).text, 'GeoNode default map abstract')
def get_schema(url, typename, version='1.0.0', timeout=30): """Parses DescribeFeatureType response and creates schema compatible with :class:`fiona` :param str url: url of the service :param str version: version of the service :param str typename: name of the layer :param int timeout: request timeout """ url = _get_describefeaturetype_url(url, version, typename) res = openURL(url, timeout=timeout) root = etree.fromstring(res.read()) type_element = findall(root, '{%s}element' % XS_NAMESPACE, attribute_name='name', attribute_value=typename)[0] complex_type = type_element.attrib['type'].split(":")[1] elements = _get_elements(complex_type, root) nsmap = None if hasattr(root, 'nsmap'): nsmap = root.nsmap return _construct_schema(elements, nsmap)