def post_xml(url, xml):
''' Execute HTTP XML POST request and print response '''
from owslib.util import http_post
try:
print http_post(url,open(xml).read())
except Exception, err:
print err
def wps_execute_layer_attribute_statistics(layer_name, field):
"""Derive aggregate statistics from WPS endpoint"""
# generate statistics using WPS
url = '%s/ows' % (settings.OGC_SERVER['default']['LOCATION'])
# TODO: use owslib.wps.WebProcessingService for WPS interaction
# this requires GeoServer's WPS gs:Aggregate function to
# return a proper wps:ExecuteResponse
request = render_to_string('layers/wps_execute_gs_aggregate.xml', {
'layer_name': 'geonode:%s' % layer_name,
'field': field
})
response = http_post(url, request)
exml = etree.fromstring(response)
result = {}
for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']:
fr = exml.find(f)
if fr is not None:
result[f] = fr.text
else:
result[f] = 'NA'
count = exml.find('Count')
if count is not None:
result['Count'] = int(count.text)
else:
result['Count'] = 0
result['unique_values'] = 'NA'
# TODO: find way of figuring out threshold better
if result['Count'] < 10000:
request = render_to_string('layers/wps_execute_gs_unique.xml', {
'layer_name': 'geonode:%s' % layer_name,
'field': field
})
response = http_post(url, request)
exml = etree.fromstring(response)
values = []
for value in exml.findall(
'{http://www.opengis.net/gml}featureMember/{http://www.opengis.net/gml}UniqueValue/{http://www.opengis.net/gml}value'
):
if value is not None:
values.append(value.text)
result['unique_values'] = ','.join(values)
return result
def wps_execute_layer_attribute_statistics(layer_name, field):
"""Derive aggregate statistics from WPS endpoint"""
# generate statistics using WPS
url = '%s/ows' % (ogc_server_settings.public_url)
# TODO: use owslib.wps.WebProcessingService for WPS interaction
# this requires GeoServer's WPS gs:Aggregate function to
# return a proper wps:ExecuteResponse
request = render_to_string('layers/wps_execute_gs_aggregate.xml', {
'layer_name': 'geonode:%s' % layer_name,
'field': field
})
response = http_post(url, request)
exml = etree.fromstring(response)
result = {}
for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']:
fr = exml.find(f)
if fr is not None:
result[f] = fr.text
else:
result[f] = 'NA'
count = exml.find('Count')
if count is not None:
result['Count'] = int(count.text)
else:
result['Count'] = 0
result['unique_values'] = 'NA'
# TODO: find way of figuring out threshold better
if result['Count'] < 10000:
request = render_to_string('layers/wps_execute_gs_unique.xml', {
'layer_name': 'geonode:%s' % layer_name,
'field': field
})
response = http_post(url, request)
exml = etree.fromstring(response)
values = []
for value in exml.findall('{http://www.opengis.net/gml}featureMember/{http://www.opengis.net/gml}UniqueValue/{http://www.opengis.net/gml}value'):
if value is not None:
values.append(value.text)
result['unique_values'] = ','.join(values)
return result
def wps_execute_layer_attribute_statistics(layer_name, field):
"""Derive aggregate statistics from WPS endpoint"""
# generate statistics using WPS
url = '%s/ows' % (ogc_server_settings.LOCATION)
# TODO: use owslib.wps.WebProcessingService for WPS interaction
# this requires GeoServer's WPS gs:Aggregate function to
# return a proper wps:ExecuteResponse
request = render_to_string('layers/wps_execute_gs_aggregate.xml', {
'layer_name': 'geonode:%s' % layer_name,
'field': field
})
response = http_post(url, request, timeout=ogc_server_settings.TIMEOUT)
exml = etree.fromstring(response)
result = {}
for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']:
fr = exml.find(f)
if fr is not None:
result[f] = fr.text
else:
result[f] = 'NA'
count = exml.find('Count')
if count is not None:
result['Count'] = int(count.text)
else:
result['Count'] = 0
result['unique_values'] = 'NA'
# TODO: find way of figuring out threshold better
if result['Count'] < 10000:
request = render_to_string('layers/wps_execute_gs_unique.xml', {
'layer_name': 'geonode:%s' % layer_name,
'field': field
})
response = http_post(url, request, timeout=ogc_server_settings.TIMEOUT)
exml = etree.fromstring(response)
values = []
def _invoke(self):
# do HTTP request
self.response = http_post(self.url, self.request, self.lang, self.timeout)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
nsp_ows('ows:ExceptionReport'),
nsp('csw:Capabilities'),
nsp('csw:DescribeRecordResponse'),
nsp('csw:GetDomainResponse'),
nsp('csw:GetRecordsResponse'),
nsp('csw:GetRecordByIdResponse'),
nsp('csw:HarvestResponse'),
nsp('csw:TransactionResponse')
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError, 'Document is XML, but not CSW-ish'
# check if it's an OGC Exception
val = self._exml.find(nsp_ows('ows:Exception'))
if val is not None:
raise ows.ExceptionReport(self._exml, ns.get_versioned_namespace('ows',_ows_version))
else:
self.exceptionreport = None
def _invoke(self):
# do HTTP request
self.response = util.http_post(self.url, self.request, self.lang,
self.timeout)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval('ows:ExceptionReport', namespaces),
util.nspath_eval('csw:Capabilities', namespaces),
util.nspath_eval('csw:DescribeRecordResponse', namespaces),
util.nspath_eval('csw:GetDomainResponse', namespaces),
util.nspath_eval('csw:GetRecordsResponse', namespaces),
util.nspath_eval('csw:GetRecordByIdResponse', namespaces),
util.nspath_eval('csw:HarvestResponse', namespaces),
util.nspath_eval('csw:TransactionResponse', namespaces)
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError, 'Document is XML, but not CSW-ish'
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval('ows:Exception', namespaces))
if val is not None:
self.exceptionreport = ExceptionReport(self._exml,
self.owscommon.namespace)
else:
self.exceptionreport = None
def _invoke(self):
# do HTTP request
self.response = util.http_post(self.url, self.request, self.lang, self.timeout)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath("ExceptionReport", namespaces["ows"]),
util.nspath("Capabilities", namespaces["csw"]),
util.nspath("DescribeRecordResponse", namespaces["csw"]),
util.nspath("GetDomainResponse", namespaces["csw"]),
util.nspath("GetRecordsResponse", namespaces["csw"]),
util.nspath("GetRecordByIdResponse", namespaces["csw"]),
util.nspath("HarvestResponse", namespaces["csw"]),
util.nspath("TransactionResponse", namespaces["csw"]),
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError, "Document is XML, but not CSW-ish"
# check if it's an OGC Exception
val = self._exml.find(util.nspath("Exception", namespaces["ows"]))
if val is not None:
self.exceptionreport = ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
def describerecord(self, typename='csw:Record', format=outputformat):
"""
Construct and process DescribeRecord request
Parameters
----------
- typename: the typename to describe (default is 'csw:Record')
- format: the outputFormat (default is 'application/xml')
"""
# construct request
node0 = etree.Element(util.nspath('DescribeRecord', namespaces['csw']))
node0.set('service', self.service)
node0.set('version', self.version)
node0.set('outputFormat', format)
node0.set('schemaLanguage', namespaces['xs2'])
node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location)
etree.SubElement(node0, util.nspath('TypeName', namespaces['csw'])).text = typename
self.request = util.xml2string(etree.tostring(node0))
# invoke
self.response = util.http_post(self.url, self.request, self.lang)
def _invoke(self):
# do HTTP request
if isinstance(self.request, basestring): # GET KVP
self.response = urlopen(self.request, timeout=self.timeout).read()
else:
self.request = cleanup_namespaces(self.request)
self.request = util.xml2string(etree.tostring(self.request))
self.response = util.http_post(self.url, self.request, self.lang, self.timeout)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval("ows:ExceptionReport", namespaces),
util.nspath_eval("csw:Capabilities", namespaces),
util.nspath_eval("csw:DescribeRecordResponse", namespaces),
util.nspath_eval("csw:GetDomainResponse", namespaces),
util.nspath_eval("csw:GetRecordsResponse", namespaces),
util.nspath_eval("csw:GetRecordByIdResponse", namespaces),
util.nspath_eval("csw:HarvestResponse", namespaces),
util.nspath_eval("csw:TransactionResponse", namespaces),
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError, "Document is XML, but not CSW-ish"
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval("ows:Exception", namespaces))
if val is not None:
raise ows.ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
def http_request(method, url, request=None, timeout=30):
"""Perform HTTP request"""
if method == 'POST':
return http_post(url, request, timeout=timeout)
else: # GET
request = urllib2.Request(url)
request.add_header('User-Agent', 'pycsw (http://pycsw.org/)')
return urllib2.urlopen(request, timeout=timeout).read()
def http_request(method, url, request=None, timeout=30):
"""Perform HTTP request"""
if method == 'POST':
return http_post(url, request, timeout=timeout)
else: # GET
request = urllib2.Request(url)
request.add_header('User-Agent', 'pycsw (http://pycsw.org/)')
return urllib2.urlopen(request, timeout=timeout).read()
def harvest(self, source, resourcetype, resourceformat=None, harvestinterval=None, responsehandler=None):
"""
Construct and process a Harvest request
Parameters
----------
- source: a URI to harvest
- resourcetype: namespace identifying the type of resource
- resourceformat: MIME type of the resource
- harvestinterval: frequency of harvesting, in ISO8601
- responsehandler: endpoint that CSW should responsd to with response
"""
# construct request
node0 = etree.Element(util.nspath('Harvest', namespaces['csw']))
node0.set('version', self.version)
node0.set('service', self.service)
node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location)
etree.SubElement(node0, util.nspath('Source', namespaces['csw'])).text = source
etree.SubElement(node0, util.nspath('ResourceType', namespaces['csw'])).text = resourcetype
if resourceformat is not None:
etree.SubElement(node0, util.nspath('ResourceFormat', namespaces['csw'])).text = resourceformat
if harvestinterval is not None:
etree.SubElement(node0, util.nspath('HarvestInterval', namespaces['csw'])).text = harvestinterval
if responsehandler is not None:
etree.SubElement(node0, util.nspath('ResponseHandler', namespaces['csw'])).text = responsehandler
self.request = util.xml2string(etree.tostring(node0))
self.response = util.http_post(self.url, self.request, self.lang)
# parse result
self._response = etree.parse(StringIO.StringIO(self.response))
# check for exceptions
self._isexception(self._response, self.owscommon.namespace)
self.results = {}
if self.exceptionreport is None:
val = self._response.find(util.nspath('Acknowledgement', namespaces['csw']))
if util.testXMLValue(val) is not None:
ts = val.attrib.get('timeStamp')
self.timestamp = util.testXMLValue(ts, True)
id = val.find(util.nspath('RequestId', namespaces['csw']))
self.id = util.testXMLValue(id)
else:
self._parsetransactionsummary()
self.results['inserted'] = []
for i in self._response.findall(util.nspath('TransactionResponse/InsertResult', namespaces['csw'])):
for j in i.findall(util.nspath('BriefRecord', namespaces['csw']) + '/' + util.nspath('identifier', namespaces['dc'])):
self.results['inserted'].append(util.testXMLValue(j))
def __xml_process_data(self):
# get the top of the check table, sensor with moth values
cur = self.__conn.cursor()
# TODO Update Database so sensor FOI and position are the same as on the service!
cur.execute("SELECT * FROM sensor WHERE MAC = '{0}'".format(
self.__check_table[0, 0]))
sensor = cur.fetchone()
# assign to the xml dict
self.__xml_dict['mac'] = sensor[
0] # 0 is Mac, we need it as internal PK
self.__xml_dict['id'] = sensor[3] # 3rd Col: UUID
self.__xml_dict['procedure'] = sensor[1] # 1st col: URN
self.__xml_dict['foi'] = sensor[2] # 2nd col: FOI URN
self.__xml_dict['begin'] = self.__check_table[
0, 2] # third cell is oldest
self.__xml_dict['end'] = self.__check_table[0,
3] # fourth cell is newest
# get properties
query = open('./resources/sql/get_observed_property.sql').read()
cur.execute(query.format(self.__xml_dict['mac']))
self.__observed_properties = np.array(cur.fetchall())
cur.close()
# Make xml strings
self.__make_composite_phenomenon()
self.__make_data_array_header()
self.__get_data()
self.__xml = open('./resources/xml/main.xml').read().format(
**self.__xml_dict)
# TODO handle response
response = http_post(self.__url, self.__xml)
print(response)
self.__last_insert = datetime.now()
# delete inserted data locally
cur = self.__conn.cursor()
notDelete = True
while notDelete:
try:
cur.execute(
open('./resources/sql/delete_inserted.sql').read().format(
**self.__xml_dict))
notDelete = False
except:
time.sleep(1)
self.__conn.commit()
cur.close()
# clear the dict and the xml
self.__xml_dict = {}
self.__xml = str()
self.__observed_properties = []
def post_xml(url, xml, timeout=30):
"""Execute HTTP XML POST request and print response"""
LOGGER.info('Executing HTTP POST request %s on server %s', xml, url)
from owslib.util import http_post
try:
return http_post(url=url, request=open(xml).read(), timeout=timeout)
except Exception as err:
raise RuntimeError(err)
def __init__(self, url, lang='en-US', version='2.0.2'):
"""
Construct and process a GetCapabilities request
Parameters
----------
- url: the URL of the CSW
- lang: the language (default is 'en-US')
- version: version (default is '2.0.2')
"""
self.url = url
self.lang = lang
self.version = version
self.service = 'CSW'
self.exceptionreport = None
self.owscommon = OwsCommon('1.0.0')
# construct request
node0 = etree.Element(util.nspath('GetCapabilities', namespaces['csw']))
node0.set('service', self.service)
node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location)
tmp = etree.SubElement(node0, util.nspath('AcceptVersions', namespaces['ows']))
etree.SubElement(tmp, util.nspath('Version', namespaces['ows'])).text = self.version
tmp2 = etree.SubElement(node0, util.nspath('AcceptFormats', namespaces['ows']))
etree.SubElement(tmp2, util.nspath('OutputFormat', namespaces['ows'])).text = outputformat
self.request = util.xml2string(etree.tostring(node0))
# invoke
self.response = util.http_post(self.url, self.request, self.lang)
# parse result
self._capabilities = etree.parse(StringIO.StringIO(self.response))
# check for exceptions
self._isexception(self._capabilities, self.owscommon.namespace)
if self.exceptionreport is None:
# ServiceIdentification
val = self._capabilities.find(util.nspath('ServiceIdentification', namespaces['ows']))
self.identification=ServiceIdentification(val,self.owscommon.namespace)
# ServiceProvider
val = self._capabilities.find(util.nspath('ServiceProvider', namespaces['ows']))
self.provider=ServiceProvider(val,self.owscommon.namespace)
# ServiceOperations metadata
self.operations=[]
for elem in self._capabilities.findall(util.nspath('OperationsMetadata/Operation', namespaces['ows'])):
self.operations.append(OperationsMetadata(elem, self.owscommon.namespace))
# FilterCapabilities
val = self._capabilities.find(util.nspath('Filter_Capabilities', namespaces['ogc']))
self.filters=FilterCapabilities(val)
def csw_request(self, layer, template):
md_doc = self.csw_gen_xml(layer, template).encode("utf-8")
if self.type == "geonetwork":
headers = {"Content-Type": "application/xml; charset=UTF-8", "Accept": "text/plain"}
request = urllib2.Request(self.url, md_doc, headers)
response = self.urlopen(request)
else:
response = http_post(self.url, md_doc, timeout=TIMEOUT)
return response
def csw_request(self, layer, template):
md_doc = self.csw_gen_xml(layer, template).encode('utf-8')
if self.type == 'geonetwork':
headers = {
"Content-Type": "application/xml; charset=UTF-8",
"Accept": "text/plain"
}
request = urllib2.Request(self.url, md_doc, headers)
response = self.urlopen(request)
else:
response = http_post(self.url, md_doc, timeout=TIMEOUT)
return response
def __xml_process_data(self):
# get the top of the check table, sensor with moth values
cur = self.__conn.cursor()
# TODO Update Database so sensor FOI and position are the same as on the service!
cur.execute("SELECT * FROM sensor WHERE MAC = '{0}'".format(self.__check_table[0,0]))
sensor = cur.fetchone()
# assign to the xml dict
self.__xml_dict['mac'] = sensor[0] # 0 is Mac, we need it as internal PK
self.__xml_dict['id'] = sensor[3] # 3rd Col: UUID
self.__xml_dict['procedure'] = sensor[1] # 1st col: URN
self.__xml_dict['foi'] = sensor[2] # 2nd col: FOI URN
self.__xml_dict['begin'] = self.__check_table[0,2] # third cell is oldest
self.__xml_dict['end'] = self.__check_table[0,3] # fourth cell is newest
# get properties
query = open('./resources/sql/get_observed_property.sql').read()
cur.execute(query.format(self.__xml_dict['mac']))
self.__observed_properties = np.array(cur.fetchall())
cur.close()
# Make xml strings
self.__make_composite_phenomenon()
self.__make_data_array_header()
self.__get_data()
self.__xml = open('./resources/xml/main.xml').read().format(**self.__xml_dict)
# TODO handle response
response = http_post(self.__url, self.__xml)
print(response)
self.__last_insert = datetime.now()
# delete inserted data locally
cur = self.__conn.cursor()
notDelete = True
while notDelete:
try:
cur.execute(open('./resources/sql/delete_inserted.sql').read().format(**self.__xml_dict))
notDelete = False
except:
time.sleep(1)
self.__conn.commit()
cur.close()
# clear the dict and the xml
self.__xml_dict = {}
self.__xml = str()
self.__observed_properties = []
def getdomain(self, dname, dtype='parameter'):
"""
Construct and process a GetDomain request
Parameters
----------
- dname: the value of the Parameter or Property to query
- dtype: whether to query a parameter (parameter) or property (property)
"""
# construct request
dtypename = 'ParameterName'
node0 = etree.Element(util.nspath('GetDomain', namespaces['csw']))
node0.set('service', self.service)
node0.set('version', self.version)
node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location)
if dtype == 'property':
dtypename = 'PropertyName'
etree.SubElement(node0, util.nspath(dtypename, namespaces['csw'])).text = dname
self.request = util.xml2string(etree.tostring(node0))
# invoke
self.response = util.http_post(self.url, self.request, self.lang)
# parse result
self._values = etree.parse(StringIO.StringIO(self.response))
# check for exceptions
self._isexception(self._values, self.owscommon.namespace)
if self.exceptionreport is None:
self.results = {}
val = self._values.find(util.nspath('DomainValues', namespaces['csw'])).attrib.get('type')
self.results['type'] = util.testXMLValue(val, True)
val = self._values.find(util.nspath('DomainValues/' + dtypename, namespaces['csw']))
self.results[dtype] = util.testXMLValue(val)
# get the list of values associated with the Domain
self.results['values'] = []
for f in self._values.findall(util.nspath('DomainValues/ListOfValues/Value', namespaces['csw'])):
self.results['values'].append(util.testXMLValue(f))
def _request(self,
path: str = None,
as_dict: bool = True,
kwargs: dict = {}) -> dict:
"""
helper function for request/response patterns against OGC API endpoints
@type path: string
@param path: path of request
@type as_dict: bool
@param as_dict: whether to return JSON dict (default True)
@type kwargs: string
@param kwargs: ``dict`` of keyword value pair request parameters
@returns: response as JSON ``dict``
"""
url = self._build_url(path)
self.request = url
LOGGER.debug(f'Request: {url}')
LOGGER.debug(f'Params: {kwargs}')
if 'cql' not in kwargs:
response = http_get(url,
headers=self.headers,
auth=self.auth,
params=kwargs)
else:
LOGGER.debug('CQL query detected')
kwargs2 = deepcopy(kwargs)
cql = kwargs2.pop('cql')
url2 = self._build_url(path, kwargs2)
response = http_post(url2, request=cql, auth=self.auth)
LOGGER.debug(f'URL: {response.url}')
if response.status_code != requests.codes.ok:
raise RuntimeError(response.text)
self.request = response.url
if as_dict:
return response.json()
else:
return response.content
def __xml_process_sensor(self):
self.__xml_dict['mac'] = self.__mac
self.__xml_dict['count'] = self.__observed_properties.shape[0]
# TODO set this to be something from GeoServer or what ever... hard coded
self.__xml_dict['foi_name'] = 'default'
self.__xml_dict['epsg'] = 4326 # set to WGS84
# make the other parameters
self.__make_data_array_header(True)
self.__make_composite_phenomenon()
self.__get_serial()
# all set! register this!
self.__xml = open('./resources/xml/insert_sensor_main.xml').read().format(**self.__xml_dict)
response = http_post(self.__url, self.__xml)
self.__parse_rs(response)
def __xml_process_sensor(self):
self.__xml_dict['mac'] = self.__mac
self.__xml_dict['count'] = self.__observed_properties.shape[0]
# TODO set this to be something from GeoServer or what ever... hard coded
self.__xml_dict['foi_name'] = 'default'
self.__xml_dict['epsg'] = 4326 # set to WGS84
# make the other parameters
self.__make_data_array_header(True)
self.__make_composite_phenomenon()
self.__get_serial()
# all set! register this!
self.__xml = open('./resources/xml/insert_sensor_main.xml').read(
).format(**self.__xml_dict)
response = http_post(self.__url, self.__xml)
self.__parse_rs(response)
def getrecordbyid(self, id=[], esn='full', outputschema=namespaces['csw'], format=outputformat):
"""
Construct and process a GetRecordById request
Parameters
----------
- id: the list of Ids
- esn: the ElementSetName 'full', 'brief' or 'summary' (default is 'full')
- outputschema: the outputSchema (default is 'http://www.opengis.net/cat/csw/2.0.2')
- format: the outputFormat (default is 'application/xml')
"""
# construct request
node0 = etree.Element(util.nspath('GetRecordById', namespaces['csw']))
node0.set('outputSchema', outputschema)
node0.set('outputFormat', format)
node0.set('version', self.version)
node0.set('service', self.service)
node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location)
for i in id:
etree.SubElement(node0, util.nspath('Id', namespaces['csw'])).text = i
etree.SubElement(node0, util.nspath('ElementSetName', namespaces['csw'])).text = esn
self.request = util.xml2string(etree.tostring(node0))
# invoke
self.response = util.http_post(self.url, self.request, self.lang)
# parse result
self._records = etree.parse(StringIO.StringIO(self.response))
# check for exceptions
self._isexception(self._records, self.owscommon.namespace)
if self.exceptionreport is None:
self.records = {}
self._parserecords(outputschema, esn)
def getrecords(self, qtype=None, keywords=[], typenames='csw:Record', propertyname='AnyText', bbox=None, esn='full', sortby=None, outputschema=namespaces['csw'], format=outputformat, startposition=0, maxrecords=10):
"""
Construct and process a GetRecords request
Parameters
----------
- qtype: type of resource to query (i.e. service, dataset)
- keywords: list of keywords
- typenames: the typeNames to query against (default is csw:Record)
- propertyname: the PropertyName to Filter against
- bbox: the bounding box of the spatial query in the form [minx,miny,maxx,maxy]
- esn: the ElementSetName 'full', 'brief' or 'summary' (default is 'full')
- sortby: property to sort results on (default is 'dc:title')
- outputschema: the outputSchema (default is 'http://www.opengis.net/cat/csw/2.0.2')
- format: the outputFormat (default is 'application/xml')
- startposition: requests a slice of the result set, starting at this position (default is 0)
- maxrecords: the maximum number of records to return. No records are returned if 0 (default is 10)
"""
# construct request
node0 = etree.Element(util.nspath('GetRecords', namespaces['csw']))
node0.set('outputSchema', outputschema)
node0.set('outputFormat', format)
node0.set('version', self.version)
node0.set('resultType', 'results')
node0.set('service', self.service)
if startposition > 0:
node0.set('startPosition', str(startposition))
node0.set('maxRecords', str(maxrecords))
node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location)
# decipher number of query parameters ( > 1 sets an 'And' Filter
pcount = 0
if qtype is not None:
pcount += 1
if keywords:
pcount += 1
if bbox is not None:
pcount += 1
node1 = etree.SubElement(node0, util.nspath('Query', namespaces['csw']))
node1.set('typeNames', typenames)
etree.SubElement(node1, util.nspath('ElementSetName', namespaces['csw'])).text = esn
# decipher if the query is for real
if keywords or bbox is not None or qtype is not None:
node2 = etree.SubElement(node1, util.nspath('Constraint', namespaces['csw']))
node2.set('version', '1.1.0')
node3 = etree.SubElement(node2, util.nspath('Filter', namespaces['ogc']))
node4 = None
# construct a Filter request
flt = FilterRequest()
if pcount > 1: # Filter should be And-ed
node4 = etree.SubElement(node3, util.nspath('And', namespaces['ogc']))
# set the query type if passed
# TODO: need a smarter way to figure these out
if qtype is not None:
if node4 is not None:
flt.setpropertyisequalto(node4, 'dc:type', qtype)
else:
flt.setpropertyisequalto(node3, 'dc:type', qtype)
# set a bbox query if passed
if bbox is not None:
if node4 is not None:
flt.setbbox(node4, bbox)
else:
flt.setbbox(node3, bbox)
# set a keyword query if passed
if len(keywords) > 0:
if len(keywords) > 1: # loop multiple keywords into an Or
if node4 is not None:
node5 = etree.SubElement(node4, util.nspath('Or', namespaces['ogc']))
else:
node5 = etree.SubElement(node3, util.nspath('Or', namespaces['ogc']))
for i in keywords:
flt.setpropertyislike(node5, propertyname, '%%%s%%' % i)
else: # one keyword
if node4 is not None:
flt.setpropertyislike(node4, propertyname, '%%%s%%' % keywords[0])
else:
flt.setpropertyislike(node3, propertyname, '%%%s%%' % keywords[0])
# set a sort if passed
if sortby is not None:
flt.setsortby(node1, sortby)
self.request = util.xml2string(etree.tostring(node0))
# invoke
self.response = util.http_post(self.url, self.request, self.lang)
# parse result
self._records = etree.parse(StringIO.StringIO(self.response))
# check for exceptions
self._isexception(self._records, self.owscommon.namespace)
if self.exceptionreport is None:
self.results = {}
# process search results attributes
val = self._records.find(util.nspath('SearchResults', namespaces['csw'])).attrib.get('numberOfRecordsMatched')
self.results['matches'] = int(util.testXMLValue(val, True))
val = self._records.find(util.nspath('SearchResults', namespaces['csw'])).attrib.get('numberOfRecordsReturned')
self.results['returned'] = int(util.testXMLValue(val, True))
val = self._records.find(util.nspath('SearchResults', namespaces['csw'])).attrib.get('nextRecord')
self.results['nextrecord'] = int(util.testXMLValue(val, True))
# process list of matching records
self.records = {}
self._parserecords(outputschema, esn)
def _invoke(self):
# do HTTP request
request_url = self.url
# Get correct URL based on Operation list.
# If skip_caps=True, then self.operations has not been set, so use
# default URL.
if hasattr(self, 'operations'):
caller = inspect.stack()[1][3]
if caller == 'getrecords2': caller = 'getrecords'
try:
op = self.get_operation_by_name(caller)
if isinstance(self.request, six.string_types): # GET KVP
get_verbs = [
x for x in op.methods if x.get('type').lower() == 'get'
]
request_url = get_verbs[0].get('url')
else:
post_verbs = [
x for x in op.methods
if x.get('type').lower() == 'post'
]
if len(post_verbs) > 1:
# Filter by constraints. We must match a PostEncoding of "XML"
for pv in post_verbs:
for const in pv.get('constraints'):
if const.name.lower() == 'postencoding':
values = [v.lower() for v in const.values]
if 'xml' in values:
request_url = pv.get('url')
break
else:
# Well, just use the first one.
request_url = post_verbs[0].get('url')
elif len(post_verbs) == 1:
request_post_url = post_verbs[0].get('url')
except: # no such luck, just go with request_url
pass
if isinstance(self.request, six.string_types): # GET KVP
self.request = '%s%s' % (bind_url(request_url), self.request)
self.response = openURL(self.request,
None,
'Get',
username=self.username,
password=self.password,
timeout=self.timeout).read()
else:
self.request = cleanup_namespaces(self.request)
# Add any namespaces used in the "typeNames" attribute of the
# csw:Query element to the query's xml namespaces.
for query in self.request.findall(
util.nspath_eval('csw:Query', namespaces)):
ns = query.get("typeNames", None)
if ns is not None:
# Pull out "gmd" from something like "gmd:MD_Metadata" from the list
# of typenames
ns_keys = [x.split(':')[0] for x in ns.split(' ')]
self.request = add_namespaces(self.request, ns_keys)
self.request = util.element_to_string(self.request,
encoding='utf-8')
self.response = util.http_post(request_url, self.request,
self.lang, self.timeout,
self.username, self.password)
# parse result see if it's XML
self._exml = etree.parse(BytesIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval('ows:ExceptionReport', namespaces),
util.nspath_eval('csw:Capabilities', namespaces),
util.nspath_eval('csw:DescribeRecordResponse', namespaces),
util.nspath_eval('csw:GetDomainResponse', namespaces),
util.nspath_eval('csw:GetRecordsResponse', namespaces),
util.nspath_eval('csw:GetRecordByIdResponse', namespaces),
util.nspath_eval('csw:HarvestResponse', namespaces),
util.nspath_eval('csw:TransactionResponse', namespaces)
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError('Document is XML, but not CSW-ish')
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval('ows:Exception', namespaces))
if val is not None:
raise ows.ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
def _invoke(self, mock_requests_post, mock_requests_request):
try:
if self.auth.token is not None:
self.auth.username = "******"
self.auth.password = self.auth.token
except AttributeError:
pass
# do HTTP request
request_url = self.url
# Get correct URL based on Operation list.
# If skip_caps=True, then self.operations has not been set, so use
# default URL.
if hasattr(self, "operations"):
caller = inspect.stack()[1][3]
if caller == "getrecords2":
caller = "getrecords"
# noinspection PyBroadException
try:
op = self.get_operation_by_name(caller)
if isinstance(self.request, str): # GET KVP
get_verbs = [
x for x in op.methods if x.get("type").lower() == "get"
]
request_url = get_verbs[0].get("url")
else:
post_verbs = [
x for x in op.methods
if x.get("type").lower() == "post"
]
if len(post_verbs) > 1:
# Filter by constraints. We must match a PostEncoding of "XML"
for pv in post_verbs:
for const in pv.get("constraints"):
if const.name.lower() == "postencoding":
values = [v.lower() for v in const.values]
if "xml" in values:
request_url = pv.get("url")
break
else:
# Well, just use the first one.
request_url = post_verbs[0].get("url")
elif len(post_verbs) == 1:
request_url = post_verbs[0].get("url")
except Exception: # nosec
# no such luck, just go with request_url
pass
# print("Echo")
if isinstance(self.request, str): # GET KVP
# print("Foxtrot")
self.request = "%s%s" % (bind_url(request_url), self.request)
self.response = openURL(self.request,
None,
"Get",
timeout=self.timeout,
auth=self.auth).read()
# debug
# print("invoke")
# print(self.response[0:100])
else:
# print("Golf")
self.request = cleanup_namespaces(self.request)
# Add any namespaces used in the "typeNames" attribute of the
# csw:Query element to the query's xml namespaces.
# noinspection PyUnresolvedReferences
for query in self.request.findall(
util.nspath_eval("csw:Query", csw_namespaces)):
ns = query.get("typeNames", None)
if ns is not None:
# Pull out "gmd" from something like "gmd:MD_Metadata" from the list
# of typenames
ns_keys = [x.split(":")[0] for x in ns.split(" ")]
self.request = add_namespaces(self.request, ns_keys)
self.request = add_namespaces(self.request, "ows")
self.request = util.element_to_string(self.request,
encoding="utf-8")
# print("Hotel")
self.response = http_post(request_url,
self.request,
self.lang,
self.timeout,
auth=self.auth)
# debug
# print("invoke 2")
# print(self.response[0:100])
# debug
# print("parse")
# print(self.response[0:100])
# print(self.response)
# parse result see if it's XML
self._exml = etree.parse(BytesIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval("ows:ExceptionReport", csw_namespaces),
util.nspath_eval("csw:Capabilities", csw_namespaces),
util.nspath_eval("csw:DescribeRecordResponse", csw_namespaces),
util.nspath_eval("csw:GetDomainResponse", csw_namespaces),
util.nspath_eval("csw:GetRecordsResponse", csw_namespaces),
util.nspath_eval("csw:GetRecordByIdResponse", csw_namespaces),
util.nspath_eval("csw:HarvestResponse", csw_namespaces),
util.nspath_eval("csw:TransactionResponse", csw_namespaces),
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError("Document is XML, but not CSW-ish")
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval("ows:Exception",
csw_namespaces))
if val is not None:
raise ows.ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
def csw_request(self, layer, template):
md_doc = self.csw_gen_xml(layer, template)
response = http_post(self.url, md_doc, timeout=TIMEOUT)
return response
def create_record(id):
response = http_post(settings.CSW_T_PATH,
request=open(settings.MEDIA_ROOT + 'csw/' + str(id) +
'_insert.xml').read())
print response
def delete_record(id):
if os.path.isfile(settings.MEDIA_ROOT + 'csw/' + str(id) + '_delete.xml'):
response = http_post(settings.CSW_T_PATH,
request=open(settings.MEDIA_ROOT + 'csw/' +
str(id) + '_delete.xml').read())
print response
def _invoke(self):
# do HTTP request
if isinstance(self.request, basestring): # GET KVP
req = Request(self.request)
if self.username is not None and self.password is not None:
base64string = base64.encodestring('%s:%s' % (self.username, self.password))[:-1]
req.add_header('Authorization', 'Basic %s' % base64string)
self.response = urlopen(req, timeout=self.timeout).read()
else:
xml_post_url = self.url
# Get correct POST URL based on Operation list.
# If skip_caps=True, then self.operations has not been set, so use
# default URL.
if hasattr(self, 'operations'):
caller = inspect.stack()[1][3]
if caller == 'getrecords2': caller = 'getrecords'
try:
op = self.get_operation_by_name(caller)
post_verbs = filter(lambda x: x.get('type').lower() == 'post', op.methods)
if len(post_verbs) > 1:
# Filter by constraints. We must match a PostEncoding of "XML"
try:
xml_post_url = next(x for x in filter(list, ([pv.get('url') for const in pv.get('constraints') if const.name.lower() == "postencoding" and 'xml' in map(lambda x: x.lower(), const.values)] for pv in post_verbs)))[0]
except StopIteration:
# Well, just use the first one.
xml_post_url = post_verbs[0].get('url')
elif len(post_verbs) == 1:
xml_post_url = post_verbs[0].get('url')
except: # no such luck, just go with xml_post_url
pass
self.request = cleanup_namespaces(self.request)
# Add any namespaces used in the "typeNames" attribute of the
# csw:Query element to the query's xml namespaces.
for query in self.request.findall(util.nspath_eval('csw:Query', namespaces)):
ns = query.get("typeNames", None)
if ns is not None:
# Pull out "gmd" from something like "gmd:MD_Metadata" from the list
# of typenames
ns_keys = [x.split(':')[0] for x in ns.split(' ')]
self.request = add_namespaces(self.request, ns_keys)
self.request = util.element_to_string(self.request, encoding='utf-8')
#Modified by Ross Thompson for use with FGP
#self.response = util.http_post(xml_post_url, self.request, self.lang, self.timeout, self.username, self.password)
self.response = util.http_post(self.url, self.request, self.lang, self.timeout, self.username, self.password)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval('ows:ExceptionReport', namespaces),
util.nspath_eval('csw:Capabilities', namespaces),
util.nspath_eval('csw:DescribeRecordResponse', namespaces),
util.nspath_eval('csw:GetDomainResponse', namespaces),
util.nspath_eval('csw:GetRecordsResponse', namespaces),
util.nspath_eval('csw:GetRecordByIdResponse', namespaces),
util.nspath_eval('csw:HarvestResponse', namespaces),
util.nspath_eval('csw:TransactionResponse', namespaces)
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError('Document is XML, but not CSW-ish')
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval('ows:Exception', namespaces))
if val is not None:
raise ows.ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
def _invoke(self):
# do HTTP request
if isinstance(self.request, basestring): # GET KVP
self.response = urlopen(self.request, timeout=self.timeout).read()
else:
xml_post_url = self.url
# Get correct POST URL based on Operation list.
# If skip_caps=True, then self.operations has not been set, so use
# default URL.
if hasattr(self, 'operations'):
for op in self.operations:
post_verbs = filter(lambda x: x.get('type').lower() == 'post', op.methods)
if len(post_verbs) > 1:
# Filter by constraints. We must match a PostEncoding of "XML"
try:
xml_post_url = next(x for x in filter(list, ([pv.get('url') for const in pv.get('constraints') if const.name.lower() == "postencoding" and 'xml' in map(lambda x: x.lower(), const.values)] for pv in post_verbs)))[0]
except StopIteration:
# Well, just use the first one.
xml_post_url = post_verbs[0].get('url')
elif len(post_verbs) == 1:
xml_post_url = post_verbs[0].get('url')
self.request = cleanup_namespaces(self.request)
# Add any namespaces used in the "typeNames" attribute of the
# csw:Query element to the query's xml namespaces.
for query in self.request.findall(util.nspath_eval('csw:Query', namespaces)):
ns = query.get("typeNames", None)
if ns is not None:
# Pull out "gmd" from something like "gmd:MD_Metadata" from the list
# of typenames
ns_keys = [x.split(':')[0] for x in ns.split(' ')]
self.request = add_namespaces(self.request, ns_keys)
self.request = util.element_to_string(self.request, encoding='utf-8')
self.response = util.http_post(xml_post_url, self.request, self.lang, self.timeout)
# parse result see if it's XML
self._exml = etree.parse(StringIO.StringIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval('ows:ExceptionReport', namespaces),
util.nspath_eval('csw:Capabilities', namespaces),
util.nspath_eval('csw:DescribeRecordResponse', namespaces),
util.nspath_eval('csw:GetDomainResponse', namespaces),
util.nspath_eval('csw:GetRecordsResponse', namespaces),
util.nspath_eval('csw:GetRecordByIdResponse', namespaces),
util.nspath_eval('csw:HarvestResponse', namespaces),
util.nspath_eval('csw:TransactionResponse', namespaces)
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError, 'Document is XML, but not CSW-ish'
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval('ows:Exception', namespaces))
if val is not None:
raise ows.ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
def _invoke(self):
# do HTTP request
request_url = self.url
# Get correct URL based on Operation list.
# If skip_caps=True, then self.operations has not been set, so use
# default URL.
if hasattr(self, 'operations'):
caller = inspect.stack()[1][3]
if caller == 'getrecords2': caller = 'getrecords'
try:
op = self.get_operation_by_name(caller)
if isinstance(self.request, six.string_types): # GET KVP
get_verbs = [x for x in op.methods if x.get('type').lower() == 'get']
request_url = get_verbs[0].get('url')
else:
post_verbs = [x for x in op.methods if x.get('type').lower() == 'post']
if len(post_verbs) > 1:
# Filter by constraints. We must match a PostEncoding of "XML"
for pv in post_verbs:
for const in pv.get('constraints'):
if const.name.lower() == 'postencoding':
values = [v.lower() for v in const.values]
if 'xml' in values:
request_url = pv.get('url')
break
else:
# Well, just use the first one.
request_url = post_verbs[0].get('url')
elif len(post_verbs) == 1:
request_url = post_verbs[0].get('url')
except: # no such luck, just go with request_url
pass
if isinstance(self.request, six.string_types): # GET KVP
self.request = '%s%s' % (bind_url(request_url), self.request)
self.response = openURL(self.request, None, 'Get', username=self.username, password=self.password, timeout=self.timeout).read()
else:
self.request = cleanup_namespaces(self.request)
# Add any namespaces used in the "typeNames" attribute of the
# csw:Query element to the query's xml namespaces.
for query in self.request.findall(util.nspath_eval('csw:Query', namespaces)):
ns = query.get("typeNames", None)
if ns is not None:
# Pull out "gmd" from something like "gmd:MD_Metadata" from the list
# of typenames
ns_keys = [x.split(':')[0] for x in ns.split(' ')]
self.request = add_namespaces(self.request, ns_keys)
self.request = util.element_to_string(self.request, encoding='utf-8')
self.response = util.http_post(request_url, self.request, self.lang, self.timeout, self.username, self.password)
# parse result see if it's XML
self._exml = etree.parse(BytesIO(self.response))
# it's XML. Attempt to decipher whether the XML response is CSW-ish """
valid_xpaths = [
util.nspath_eval('ows:ExceptionReport', namespaces),
util.nspath_eval('csw:Capabilities', namespaces),
util.nspath_eval('csw:DescribeRecordResponse', namespaces),
util.nspath_eval('csw:GetDomainResponse', namespaces),
util.nspath_eval('csw:GetRecordsResponse', namespaces),
util.nspath_eval('csw:GetRecordByIdResponse', namespaces),
util.nspath_eval('csw:HarvestResponse', namespaces),
util.nspath_eval('csw:TransactionResponse', namespaces)
]
if self._exml.getroot().tag not in valid_xpaths:
raise RuntimeError('Document is XML, but not CSW-ish')
# check if it's an OGC Exception
val = self._exml.find(util.nspath_eval('ows:Exception', namespaces))
if val is not None:
raise ows.ExceptionReport(self._exml, self.owscommon.namespace)
else:
self.exceptionreport = None
