def post_xml(url, xml): ''' Execute HTTP XML POST request and print response ''' from owslib.util import http_post try: print http_post(url,open(xml).read()) except Exception, err: print err
def wps_execute_layer_attribute_statistics(layer_name, field): """Derive aggregate statistics from WPS endpoint""" # generate statistics using WPS url = '%s/ows' % (settings.OGC_SERVER['default']['LOCATION']) # TODO: use owslib.wps.WebProcessingService for WPS interaction # this requires GeoServer's WPS gs:Aggregate function to # return a proper wps:ExecuteResponse request = render_to_string('layers/wps_execute_gs_aggregate.xml', { 'layer_name': 'geonode:%s' % layer_name, 'field': field }) response = http_post(url, request) exml = etree.fromstring(response) result = {} for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']: fr = exml.find(f) if fr is not None: result[f] = fr.text else: result[f] = 'NA' count = exml.find('Count') if count is not None: result['Count'] = int(count.text) else: result['Count'] = 0 result['unique_values'] = 'NA' # TODO: find way of figuring out threshold better if result['Count'] < 10000: request = render_to_string('layers/wps_execute_gs_unique.xml', { 'layer_name': 'geonode:%s' % layer_name, 'field': field }) response = http_post(url, request) exml = etree.fromstring(response) values = [] for value in exml.findall( '{http://www.opengis.net/gml}featureMember/{http://www.opengis.net/gml}UniqueValue/{http://www.opengis.net/gml}value' ): if value is not None: values.append(value.text) result['unique_values'] = ','.join(values) return result
def wps_execute_layer_attribute_statistics(layer_name, field): """Derive aggregate statistics from WPS endpoint""" # generate statistics using WPS url = '%s/ows' % (ogc_server_settings.public_url) # TODO: use owslib.wps.WebProcessingService for WPS interaction # this requires GeoServer's WPS gs:Aggregate function to # return a proper wps:ExecuteResponse request = render_to_string('layers/wps_execute_gs_aggregate.xml', { 'layer_name': 'geonode:%s' % layer_name, 'field': field }) response = http_post(url, request) exml = etree.fromstring(response) result = {} for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']: fr = exml.find(f) if fr is not None: result[f] = fr.text else: result[f] = 'NA' count = exml.find('Count') if count is not None: result['Count'] = int(count.text) else: result['Count'] = 0 result['unique_values'] = 'NA' # TODO: find way of figuring out threshold better if result['Count'] < 10000: request = render_to_string('layers/wps_execute_gs_unique.xml', { 'layer_name': 'geonode:%s' % layer_name, 'field': field }) response = http_post(url, request) exml = etree.fromstring(response) values = [] for value in exml.findall('{http://www.opengis.net/gml}featureMember/{http://www.opengis.net/gml}UniqueValue/{http://www.opengis.net/gml}value'): if value is not None: values.append(value.text) result['unique_values'] = ','.join(values) return result
def wps_execute_layer_attribute_statistics(layer_name, field): """Derive aggregate statistics from WPS endpoint""" # generate statistics using WPS url = '%s/ows' % (ogc_server_settings.LOCATION) # TODO: use owslib.wps.WebProcessingService for WPS interaction # this requires GeoServer's WPS gs:Aggregate function to # return a proper wps:ExecuteResponse request = render_to_string('layers/wps_execute_gs_aggregate.xml', { 'layer_name': 'geonode:%s' % layer_name, 'field': field }) response = http_post(url, request, timeout=ogc_server_settings.TIMEOUT) exml = etree.fromstring(response) result = {} for f in ['Min', 'Max', 'Average', 'Median', 'StandardDeviation', 'Sum']: fr = exml.find(f) if fr is not None: result[f] = fr.text else: result[f] = 'NA' count = exml.find('Count') if count is not None: result['Count'] = int(count.text) else: result['Count'] = 0 result['unique_values'] = 'NA' # TODO: find way of figuring out threshold better if result['Count'] < 10000: request = render_to_string('layers/wps_execute_gs_unique.xml', { 'layer_name': 'geonode:%s' % layer_name, 'field': field }) response = http_post(url, request, timeout=ogc_server_settings.TIMEOUT) exml = etree.fromstring(response) values = []
def _invoke(self): # do HTTP request self.response = http_post(self.url, self.request, self.lang, self.timeout) # parse result see if it's XML self._exml = etree.parse(StringIO.StringIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ nsp_ows('ows:ExceptionReport'), nsp('csw:Capabilities'), nsp('csw:DescribeRecordResponse'), nsp('csw:GetDomainResponse'), nsp('csw:GetRecordsResponse'), nsp('csw:GetRecordByIdResponse'), nsp('csw:HarvestResponse'), nsp('csw:TransactionResponse') ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError, 'Document is XML, but not CSW-ish' # check if it's an OGC Exception val = self._exml.find(nsp_ows('ows:Exception')) if val is not None: raise ows.ExceptionReport(self._exml, ns.get_versioned_namespace('ows',_ows_version)) else: self.exceptionreport = None
def _invoke(self): # do HTTP request self.response = util.http_post(self.url, self.request, self.lang, self.timeout) # parse result see if it's XML self._exml = etree.parse(StringIO.StringIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval('ows:ExceptionReport', namespaces), util.nspath_eval('csw:Capabilities', namespaces), util.nspath_eval('csw:DescribeRecordResponse', namespaces), util.nspath_eval('csw:GetDomainResponse', namespaces), util.nspath_eval('csw:GetRecordsResponse', namespaces), util.nspath_eval('csw:GetRecordByIdResponse', namespaces), util.nspath_eval('csw:HarvestResponse', namespaces), util.nspath_eval('csw:TransactionResponse', namespaces) ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError, 'Document is XML, but not CSW-ish' # check if it's an OGC Exception val = self._exml.find(util.nspath_eval('ows:Exception', namespaces)) if val is not None: self.exceptionreport = ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def _invoke(self): # do HTTP request self.response = util.http_post(self.url, self.request, self.lang, self.timeout) # parse result see if it's XML self._exml = etree.parse(StringIO.StringIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath("ExceptionReport", namespaces["ows"]), util.nspath("Capabilities", namespaces["csw"]), util.nspath("DescribeRecordResponse", namespaces["csw"]), util.nspath("GetDomainResponse", namespaces["csw"]), util.nspath("GetRecordsResponse", namespaces["csw"]), util.nspath("GetRecordByIdResponse", namespaces["csw"]), util.nspath("HarvestResponse", namespaces["csw"]), util.nspath("TransactionResponse", namespaces["csw"]), ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError, "Document is XML, but not CSW-ish" # check if it's an OGC Exception val = self._exml.find(util.nspath("Exception", namespaces["ows"])) if val is not None: self.exceptionreport = ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def describerecord(self, typename='csw:Record', format=outputformat): """ Construct and process DescribeRecord request Parameters ---------- - typename: the typename to describe (default is 'csw:Record') - format: the outputFormat (default is 'application/xml') """ # construct request node0 = etree.Element(util.nspath('DescribeRecord', namespaces['csw'])) node0.set('service', self.service) node0.set('version', self.version) node0.set('outputFormat', format) node0.set('schemaLanguage', namespaces['xs2']) node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location) etree.SubElement(node0, util.nspath('TypeName', namespaces['csw'])).text = typename self.request = util.xml2string(etree.tostring(node0)) # invoke self.response = util.http_post(self.url, self.request, self.lang)
def _invoke(self): # do HTTP request if isinstance(self.request, basestring): # GET KVP self.response = urlopen(self.request, timeout=self.timeout).read() else: self.request = cleanup_namespaces(self.request) self.request = util.xml2string(etree.tostring(self.request)) self.response = util.http_post(self.url, self.request, self.lang, self.timeout) # parse result see if it's XML self._exml = etree.parse(StringIO.StringIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval("ows:ExceptionReport", namespaces), util.nspath_eval("csw:Capabilities", namespaces), util.nspath_eval("csw:DescribeRecordResponse", namespaces), util.nspath_eval("csw:GetDomainResponse", namespaces), util.nspath_eval("csw:GetRecordsResponse", namespaces), util.nspath_eval("csw:GetRecordByIdResponse", namespaces), util.nspath_eval("csw:HarvestResponse", namespaces), util.nspath_eval("csw:TransactionResponse", namespaces), ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError, "Document is XML, but not CSW-ish" # check if it's an OGC Exception val = self._exml.find(util.nspath_eval("ows:Exception", namespaces)) if val is not None: raise ows.ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def http_request(method, url, request=None, timeout=30): """Perform HTTP request""" if method == 'POST': return http_post(url, request, timeout=timeout) else: # GET request = urllib2.Request(url) request.add_header('User-Agent', 'pycsw (http://pycsw.org/)') return urllib2.urlopen(request, timeout=timeout).read()
def harvest(self, source, resourcetype, resourceformat=None, harvestinterval=None, responsehandler=None): """ Construct and process a Harvest request Parameters ---------- - source: a URI to harvest - resourcetype: namespace identifying the type of resource - resourceformat: MIME type of the resource - harvestinterval: frequency of harvesting, in ISO8601 - responsehandler: endpoint that CSW should responsd to with response """ # construct request node0 = etree.Element(util.nspath('Harvest', namespaces['csw'])) node0.set('version', self.version) node0.set('service', self.service) node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location) etree.SubElement(node0, util.nspath('Source', namespaces['csw'])).text = source etree.SubElement(node0, util.nspath('ResourceType', namespaces['csw'])).text = resourcetype if resourceformat is not None: etree.SubElement(node0, util.nspath('ResourceFormat', namespaces['csw'])).text = resourceformat if harvestinterval is not None: etree.SubElement(node0, util.nspath('HarvestInterval', namespaces['csw'])).text = harvestinterval if responsehandler is not None: etree.SubElement(node0, util.nspath('ResponseHandler', namespaces['csw'])).text = responsehandler self.request = util.xml2string(etree.tostring(node0)) self.response = util.http_post(self.url, self.request, self.lang) # parse result self._response = etree.parse(StringIO.StringIO(self.response)) # check for exceptions self._isexception(self._response, self.owscommon.namespace) self.results = {} if self.exceptionreport is None: val = self._response.find(util.nspath('Acknowledgement', namespaces['csw'])) if util.testXMLValue(val) is not None: ts = val.attrib.get('timeStamp') self.timestamp = util.testXMLValue(ts, True) id = val.find(util.nspath('RequestId', namespaces['csw'])) self.id = util.testXMLValue(id) else: self._parsetransactionsummary() self.results['inserted'] = [] for i in self._response.findall(util.nspath('TransactionResponse/InsertResult', namespaces['csw'])): for j in i.findall(util.nspath('BriefRecord', namespaces['csw']) + '/' + util.nspath('identifier', namespaces['dc'])): self.results['inserted'].append(util.testXMLValue(j))
def __xml_process_data(self): # get the top of the check table, sensor with moth values cur = self.__conn.cursor() # TODO Update Database so sensor FOI and position are the same as on the service! cur.execute("SELECT * FROM sensor WHERE MAC = '{0}'".format( self.__check_table[0, 0])) sensor = cur.fetchone() # assign to the xml dict self.__xml_dict['mac'] = sensor[ 0] # 0 is Mac, we need it as internal PK self.__xml_dict['id'] = sensor[3] # 3rd Col: UUID self.__xml_dict['procedure'] = sensor[1] # 1st col: URN self.__xml_dict['foi'] = sensor[2] # 2nd col: FOI URN self.__xml_dict['begin'] = self.__check_table[ 0, 2] # third cell is oldest self.__xml_dict['end'] = self.__check_table[0, 3] # fourth cell is newest # get properties query = open('./resources/sql/get_observed_property.sql').read() cur.execute(query.format(self.__xml_dict['mac'])) self.__observed_properties = np.array(cur.fetchall()) cur.close() # Make xml strings self.__make_composite_phenomenon() self.__make_data_array_header() self.__get_data() self.__xml = open('./resources/xml/main.xml').read().format( **self.__xml_dict) # TODO handle response response = http_post(self.__url, self.__xml) print(response) self.__last_insert = datetime.now() # delete inserted data locally cur = self.__conn.cursor() notDelete = True while notDelete: try: cur.execute( open('./resources/sql/delete_inserted.sql').read().format( **self.__xml_dict)) notDelete = False except: time.sleep(1) self.__conn.commit() cur.close() # clear the dict and the xml self.__xml_dict = {} self.__xml = str() self.__observed_properties = []
def post_xml(url, xml, timeout=30): """Execute HTTP XML POST request and print response""" LOGGER.info('Executing HTTP POST request %s on server %s', xml, url) from owslib.util import http_post try: return http_post(url=url, request=open(xml).read(), timeout=timeout) except Exception as err: raise RuntimeError(err)
def __init__(self, url, lang='en-US', version='2.0.2'): """ Construct and process a GetCapabilities request Parameters ---------- - url: the URL of the CSW - lang: the language (default is 'en-US') - version: version (default is '2.0.2') """ self.url = url self.lang = lang self.version = version self.service = 'CSW' self.exceptionreport = None self.owscommon = OwsCommon('1.0.0') # construct request node0 = etree.Element(util.nspath('GetCapabilities', namespaces['csw'])) node0.set('service', self.service) node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location) tmp = etree.SubElement(node0, util.nspath('AcceptVersions', namespaces['ows'])) etree.SubElement(tmp, util.nspath('Version', namespaces['ows'])).text = self.version tmp2 = etree.SubElement(node0, util.nspath('AcceptFormats', namespaces['ows'])) etree.SubElement(tmp2, util.nspath('OutputFormat', namespaces['ows'])).text = outputformat self.request = util.xml2string(etree.tostring(node0)) # invoke self.response = util.http_post(self.url, self.request, self.lang) # parse result self._capabilities = etree.parse(StringIO.StringIO(self.response)) # check for exceptions self._isexception(self._capabilities, self.owscommon.namespace) if self.exceptionreport is None: # ServiceIdentification val = self._capabilities.find(util.nspath('ServiceIdentification', namespaces['ows'])) self.identification=ServiceIdentification(val,self.owscommon.namespace) # ServiceProvider val = self._capabilities.find(util.nspath('ServiceProvider', namespaces['ows'])) self.provider=ServiceProvider(val,self.owscommon.namespace) # ServiceOperations metadata self.operations=[] for elem in self._capabilities.findall(util.nspath('OperationsMetadata/Operation', namespaces['ows'])): self.operations.append(OperationsMetadata(elem, self.owscommon.namespace)) # FilterCapabilities val = self._capabilities.find(util.nspath('Filter_Capabilities', namespaces['ogc'])) self.filters=FilterCapabilities(val)
def csw_request(self, layer, template): md_doc = self.csw_gen_xml(layer, template).encode("utf-8") if self.type == "geonetwork": headers = {"Content-Type": "application/xml; charset=UTF-8", "Accept": "text/plain"} request = urllib2.Request(self.url, md_doc, headers) response = self.urlopen(request) else: response = http_post(self.url, md_doc, timeout=TIMEOUT) return response
def csw_request(self, layer, template): md_doc = self.csw_gen_xml(layer, template).encode('utf-8') if self.type == 'geonetwork': headers = { "Content-Type": "application/xml; charset=UTF-8", "Accept": "text/plain" } request = urllib2.Request(self.url, md_doc, headers) response = self.urlopen(request) else: response = http_post(self.url, md_doc, timeout=TIMEOUT) return response
def __xml_process_data(self): # get the top of the check table, sensor with moth values cur = self.__conn.cursor() # TODO Update Database so sensor FOI and position are the same as on the service! cur.execute("SELECT * FROM sensor WHERE MAC = '{0}'".format(self.__check_table[0,0])) sensor = cur.fetchone() # assign to the xml dict self.__xml_dict['mac'] = sensor[0] # 0 is Mac, we need it as internal PK self.__xml_dict['id'] = sensor[3] # 3rd Col: UUID self.__xml_dict['procedure'] = sensor[1] # 1st col: URN self.__xml_dict['foi'] = sensor[2] # 2nd col: FOI URN self.__xml_dict['begin'] = self.__check_table[0,2] # third cell is oldest self.__xml_dict['end'] = self.__check_table[0,3] # fourth cell is newest # get properties query = open('./resources/sql/get_observed_property.sql').read() cur.execute(query.format(self.__xml_dict['mac'])) self.__observed_properties = np.array(cur.fetchall()) cur.close() # Make xml strings self.__make_composite_phenomenon() self.__make_data_array_header() self.__get_data() self.__xml = open('./resources/xml/main.xml').read().format(**self.__xml_dict) # TODO handle response response = http_post(self.__url, self.__xml) print(response) self.__last_insert = datetime.now() # delete inserted data locally cur = self.__conn.cursor() notDelete = True while notDelete: try: cur.execute(open('./resources/sql/delete_inserted.sql').read().format(**self.__xml_dict)) notDelete = False except: time.sleep(1) self.__conn.commit() cur.close() # clear the dict and the xml self.__xml_dict = {} self.__xml = str() self.__observed_properties = []
def getdomain(self, dname, dtype='parameter'): """ Construct and process a GetDomain request Parameters ---------- - dname: the value of the Parameter or Property to query - dtype: whether to query a parameter (parameter) or property (property) """ # construct request dtypename = 'ParameterName' node0 = etree.Element(util.nspath('GetDomain', namespaces['csw'])) node0.set('service', self.service) node0.set('version', self.version) node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location) if dtype == 'property': dtypename = 'PropertyName' etree.SubElement(node0, util.nspath(dtypename, namespaces['csw'])).text = dname self.request = util.xml2string(etree.tostring(node0)) # invoke self.response = util.http_post(self.url, self.request, self.lang) # parse result self._values = etree.parse(StringIO.StringIO(self.response)) # check for exceptions self._isexception(self._values, self.owscommon.namespace) if self.exceptionreport is None: self.results = {} val = self._values.find(util.nspath('DomainValues', namespaces['csw'])).attrib.get('type') self.results['type'] = util.testXMLValue(val, True) val = self._values.find(util.nspath('DomainValues/' + dtypename, namespaces['csw'])) self.results[dtype] = util.testXMLValue(val) # get the list of values associated with the Domain self.results['values'] = [] for f in self._values.findall(util.nspath('DomainValues/ListOfValues/Value', namespaces['csw'])): self.results['values'].append(util.testXMLValue(f))
def _request(self, path: str = None, as_dict: bool = True, kwargs: dict = {}) -> dict: """ helper function for request/response patterns against OGC API endpoints @type path: string @param path: path of request @type as_dict: bool @param as_dict: whether to return JSON dict (default True) @type kwargs: string @param kwargs: ``dict`` of keyword value pair request parameters @returns: response as JSON ``dict`` """ url = self._build_url(path) self.request = url LOGGER.debug(f'Request: {url}') LOGGER.debug(f'Params: {kwargs}') if 'cql' not in kwargs: response = http_get(url, headers=self.headers, auth=self.auth, params=kwargs) else: LOGGER.debug('CQL query detected') kwargs2 = deepcopy(kwargs) cql = kwargs2.pop('cql') url2 = self._build_url(path, kwargs2) response = http_post(url2, request=cql, auth=self.auth) LOGGER.debug(f'URL: {response.url}') if response.status_code != requests.codes.ok: raise RuntimeError(response.text) self.request = response.url if as_dict: return response.json() else: return response.content
def __xml_process_sensor(self): self.__xml_dict['mac'] = self.__mac self.__xml_dict['count'] = self.__observed_properties.shape[0] # TODO set this to be something from GeoServer or what ever... hard coded self.__xml_dict['foi_name'] = 'default' self.__xml_dict['epsg'] = 4326 # set to WGS84 # make the other parameters self.__make_data_array_header(True) self.__make_composite_phenomenon() self.__get_serial() # all set! register this! self.__xml = open('./resources/xml/insert_sensor_main.xml').read().format(**self.__xml_dict) response = http_post(self.__url, self.__xml) self.__parse_rs(response)
def __xml_process_sensor(self): self.__xml_dict['mac'] = self.__mac self.__xml_dict['count'] = self.__observed_properties.shape[0] # TODO set this to be something from GeoServer or what ever... hard coded self.__xml_dict['foi_name'] = 'default' self.__xml_dict['epsg'] = 4326 # set to WGS84 # make the other parameters self.__make_data_array_header(True) self.__make_composite_phenomenon() self.__get_serial() # all set! register this! self.__xml = open('./resources/xml/insert_sensor_main.xml').read( ).format(**self.__xml_dict) response = http_post(self.__url, self.__xml) self.__parse_rs(response)
def getrecordbyid(self, id=[], esn='full', outputschema=namespaces['csw'], format=outputformat): """ Construct and process a GetRecordById request Parameters ---------- - id: the list of Ids - esn: the ElementSetName 'full', 'brief' or 'summary' (default is 'full') - outputschema: the outputSchema (default is 'http://www.opengis.net/cat/csw/2.0.2') - format: the outputFormat (default is 'application/xml') """ # construct request node0 = etree.Element(util.nspath('GetRecordById', namespaces['csw'])) node0.set('outputSchema', outputschema) node0.set('outputFormat', format) node0.set('version', self.version) node0.set('service', self.service) node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location) for i in id: etree.SubElement(node0, util.nspath('Id', namespaces['csw'])).text = i etree.SubElement(node0, util.nspath('ElementSetName', namespaces['csw'])).text = esn self.request = util.xml2string(etree.tostring(node0)) # invoke self.response = util.http_post(self.url, self.request, self.lang) # parse result self._records = etree.parse(StringIO.StringIO(self.response)) # check for exceptions self._isexception(self._records, self.owscommon.namespace) if self.exceptionreport is None: self.records = {} self._parserecords(outputschema, esn)
def getrecords(self, qtype=None, keywords=[], typenames='csw:Record', propertyname='AnyText', bbox=None, esn='full', sortby=None, outputschema=namespaces['csw'], format=outputformat, startposition=0, maxrecords=10): """ Construct and process a GetRecords request Parameters ---------- - qtype: type of resource to query (i.e. service, dataset) - keywords: list of keywords - typenames: the typeNames to query against (default is csw:Record) - propertyname: the PropertyName to Filter against - bbox: the bounding box of the spatial query in the form [minx,miny,maxx,maxy] - esn: the ElementSetName 'full', 'brief' or 'summary' (default is 'full') - sortby: property to sort results on (default is 'dc:title') - outputschema: the outputSchema (default is 'http://www.opengis.net/cat/csw/2.0.2') - format: the outputFormat (default is 'application/xml') - startposition: requests a slice of the result set, starting at this position (default is 0) - maxrecords: the maximum number of records to return. No records are returned if 0 (default is 10) """ # construct request node0 = etree.Element(util.nspath('GetRecords', namespaces['csw'])) node0.set('outputSchema', outputschema) node0.set('outputFormat', format) node0.set('version', self.version) node0.set('resultType', 'results') node0.set('service', self.service) if startposition > 0: node0.set('startPosition', str(startposition)) node0.set('maxRecords', str(maxrecords)) node0.set(util.nspath('schemaLocation', namespaces['xsi']), schema_location) # decipher number of query parameters ( > 1 sets an 'And' Filter pcount = 0 if qtype is not None: pcount += 1 if keywords: pcount += 1 if bbox is not None: pcount += 1 node1 = etree.SubElement(node0, util.nspath('Query', namespaces['csw'])) node1.set('typeNames', typenames) etree.SubElement(node1, util.nspath('ElementSetName', namespaces['csw'])).text = esn # decipher if the query is for real if keywords or bbox is not None or qtype is not None: node2 = etree.SubElement(node1, util.nspath('Constraint', namespaces['csw'])) node2.set('version', '1.1.0') node3 = etree.SubElement(node2, util.nspath('Filter', namespaces['ogc'])) node4 = None # construct a Filter request flt = FilterRequest() if pcount > 1: # Filter should be And-ed node4 = etree.SubElement(node3, util.nspath('And', namespaces['ogc'])) # set the query type if passed # TODO: need a smarter way to figure these out if qtype is not None: if node4 is not None: flt.setpropertyisequalto(node4, 'dc:type', qtype) else: flt.setpropertyisequalto(node3, 'dc:type', qtype) # set a bbox query if passed if bbox is not None: if node4 is not None: flt.setbbox(node4, bbox) else: flt.setbbox(node3, bbox) # set a keyword query if passed if len(keywords) > 0: if len(keywords) > 1: # loop multiple keywords into an Or if node4 is not None: node5 = etree.SubElement(node4, util.nspath('Or', namespaces['ogc'])) else: node5 = etree.SubElement(node3, util.nspath('Or', namespaces['ogc'])) for i in keywords: flt.setpropertyislike(node5, propertyname, '%%%s%%' % i) else: # one keyword if node4 is not None: flt.setpropertyislike(node4, propertyname, '%%%s%%' % keywords[0]) else: flt.setpropertyislike(node3, propertyname, '%%%s%%' % keywords[0]) # set a sort if passed if sortby is not None: flt.setsortby(node1, sortby) self.request = util.xml2string(etree.tostring(node0)) # invoke self.response = util.http_post(self.url, self.request, self.lang) # parse result self._records = etree.parse(StringIO.StringIO(self.response)) # check for exceptions self._isexception(self._records, self.owscommon.namespace) if self.exceptionreport is None: self.results = {} # process search results attributes val = self._records.find(util.nspath('SearchResults', namespaces['csw'])).attrib.get('numberOfRecordsMatched') self.results['matches'] = int(util.testXMLValue(val, True)) val = self._records.find(util.nspath('SearchResults', namespaces['csw'])).attrib.get('numberOfRecordsReturned') self.results['returned'] = int(util.testXMLValue(val, True)) val = self._records.find(util.nspath('SearchResults', namespaces['csw'])).attrib.get('nextRecord') self.results['nextrecord'] = int(util.testXMLValue(val, True)) # process list of matching records self.records = {} self._parserecords(outputschema, esn)
def _invoke(self): # do HTTP request request_url = self.url # Get correct URL based on Operation list. # If skip_caps=True, then self.operations has not been set, so use # default URL. if hasattr(self, 'operations'): caller = inspect.stack()[1][3] if caller == 'getrecords2': caller = 'getrecords' try: op = self.get_operation_by_name(caller) if isinstance(self.request, six.string_types): # GET KVP get_verbs = [ x for x in op.methods if x.get('type').lower() == 'get' ] request_url = get_verbs[0].get('url') else: post_verbs = [ x for x in op.methods if x.get('type').lower() == 'post' ] if len(post_verbs) > 1: # Filter by constraints. We must match a PostEncoding of "XML" for pv in post_verbs: for const in pv.get('constraints'): if const.name.lower() == 'postencoding': values = [v.lower() for v in const.values] if 'xml' in values: request_url = pv.get('url') break else: # Well, just use the first one. request_url = post_verbs[0].get('url') elif len(post_verbs) == 1: request_post_url = post_verbs[0].get('url') except: # no such luck, just go with request_url pass if isinstance(self.request, six.string_types): # GET KVP self.request = '%s%s' % (bind_url(request_url), self.request) self.response = openURL(self.request, None, 'Get', username=self.username, password=self.password, timeout=self.timeout).read() else: self.request = cleanup_namespaces(self.request) # Add any namespaces used in the "typeNames" attribute of the # csw:Query element to the query's xml namespaces. for query in self.request.findall( util.nspath_eval('csw:Query', namespaces)): ns = query.get("typeNames", None) if ns is not None: # Pull out "gmd" from something like "gmd:MD_Metadata" from the list # of typenames ns_keys = [x.split(':')[0] for x in ns.split(' ')] self.request = add_namespaces(self.request, ns_keys) self.request = util.element_to_string(self.request, encoding='utf-8') self.response = util.http_post(request_url, self.request, self.lang, self.timeout, self.username, self.password) # parse result see if it's XML self._exml = etree.parse(BytesIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval('ows:ExceptionReport', namespaces), util.nspath_eval('csw:Capabilities', namespaces), util.nspath_eval('csw:DescribeRecordResponse', namespaces), util.nspath_eval('csw:GetDomainResponse', namespaces), util.nspath_eval('csw:GetRecordsResponse', namespaces), util.nspath_eval('csw:GetRecordByIdResponse', namespaces), util.nspath_eval('csw:HarvestResponse', namespaces), util.nspath_eval('csw:TransactionResponse', namespaces) ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError('Document is XML, but not CSW-ish') # check if it's an OGC Exception val = self._exml.find(util.nspath_eval('ows:Exception', namespaces)) if val is not None: raise ows.ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def _invoke(self, mock_requests_post, mock_requests_request): try: if self.auth.token is not None: self.auth.username = "******" self.auth.password = self.auth.token except AttributeError: pass # do HTTP request request_url = self.url # Get correct URL based on Operation list. # If skip_caps=True, then self.operations has not been set, so use # default URL. if hasattr(self, "operations"): caller = inspect.stack()[1][3] if caller == "getrecords2": caller = "getrecords" # noinspection PyBroadException try: op = self.get_operation_by_name(caller) if isinstance(self.request, str): # GET KVP get_verbs = [ x for x in op.methods if x.get("type").lower() == "get" ] request_url = get_verbs[0].get("url") else: post_verbs = [ x for x in op.methods if x.get("type").lower() == "post" ] if len(post_verbs) > 1: # Filter by constraints. We must match a PostEncoding of "XML" for pv in post_verbs: for const in pv.get("constraints"): if const.name.lower() == "postencoding": values = [v.lower() for v in const.values] if "xml" in values: request_url = pv.get("url") break else: # Well, just use the first one. request_url = post_verbs[0].get("url") elif len(post_verbs) == 1: request_url = post_verbs[0].get("url") except Exception: # nosec # no such luck, just go with request_url pass # print("Echo") if isinstance(self.request, str): # GET KVP # print("Foxtrot") self.request = "%s%s" % (bind_url(request_url), self.request) self.response = openURL(self.request, None, "Get", timeout=self.timeout, auth=self.auth).read() # debug # print("invoke") # print(self.response[0:100]) else: # print("Golf") self.request = cleanup_namespaces(self.request) # Add any namespaces used in the "typeNames" attribute of the # csw:Query element to the query's xml namespaces. # noinspection PyUnresolvedReferences for query in self.request.findall( util.nspath_eval("csw:Query", csw_namespaces)): ns = query.get("typeNames", None) if ns is not None: # Pull out "gmd" from something like "gmd:MD_Metadata" from the list # of typenames ns_keys = [x.split(":")[0] for x in ns.split(" ")] self.request = add_namespaces(self.request, ns_keys) self.request = add_namespaces(self.request, "ows") self.request = util.element_to_string(self.request, encoding="utf-8") # print("Hotel") self.response = http_post(request_url, self.request, self.lang, self.timeout, auth=self.auth) # debug # print("invoke 2") # print(self.response[0:100]) # debug # print("parse") # print(self.response[0:100]) # print(self.response) # parse result see if it's XML self._exml = etree.parse(BytesIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval("ows:ExceptionReport", csw_namespaces), util.nspath_eval("csw:Capabilities", csw_namespaces), util.nspath_eval("csw:DescribeRecordResponse", csw_namespaces), util.nspath_eval("csw:GetDomainResponse", csw_namespaces), util.nspath_eval("csw:GetRecordsResponse", csw_namespaces), util.nspath_eval("csw:GetRecordByIdResponse", csw_namespaces), util.nspath_eval("csw:HarvestResponse", csw_namespaces), util.nspath_eval("csw:TransactionResponse", csw_namespaces), ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError("Document is XML, but not CSW-ish") # check if it's an OGC Exception val = self._exml.find(util.nspath_eval("ows:Exception", csw_namespaces)) if val is not None: raise ows.ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def csw_request(self, layer, template): md_doc = self.csw_gen_xml(layer, template) response = http_post(self.url, md_doc, timeout=TIMEOUT) return response
def create_record(id): response = http_post(settings.CSW_T_PATH, request=open(settings.MEDIA_ROOT + 'csw/' + str(id) + '_insert.xml').read()) print response
def delete_record(id): if os.path.isfile(settings.MEDIA_ROOT + 'csw/' + str(id) + '_delete.xml'): response = http_post(settings.CSW_T_PATH, request=open(settings.MEDIA_ROOT + 'csw/' + str(id) + '_delete.xml').read()) print response
def _invoke(self): # do HTTP request if isinstance(self.request, basestring): # GET KVP req = Request(self.request) if self.username is not None and self.password is not None: base64string = base64.encodestring('%s:%s' % (self.username, self.password))[:-1] req.add_header('Authorization', 'Basic %s' % base64string) self.response = urlopen(req, timeout=self.timeout).read() else: xml_post_url = self.url # Get correct POST URL based on Operation list. # If skip_caps=True, then self.operations has not been set, so use # default URL. if hasattr(self, 'operations'): caller = inspect.stack()[1][3] if caller == 'getrecords2': caller = 'getrecords' try: op = self.get_operation_by_name(caller) post_verbs = filter(lambda x: x.get('type').lower() == 'post', op.methods) if len(post_verbs) > 1: # Filter by constraints. We must match a PostEncoding of "XML" try: xml_post_url = next(x for x in filter(list, ([pv.get('url') for const in pv.get('constraints') if const.name.lower() == "postencoding" and 'xml' in map(lambda x: x.lower(), const.values)] for pv in post_verbs)))[0] except StopIteration: # Well, just use the first one. xml_post_url = post_verbs[0].get('url') elif len(post_verbs) == 1: xml_post_url = post_verbs[0].get('url') except: # no such luck, just go with xml_post_url pass self.request = cleanup_namespaces(self.request) # Add any namespaces used in the "typeNames" attribute of the # csw:Query element to the query's xml namespaces. for query in self.request.findall(util.nspath_eval('csw:Query', namespaces)): ns = query.get("typeNames", None) if ns is not None: # Pull out "gmd" from something like "gmd:MD_Metadata" from the list # of typenames ns_keys = [x.split(':')[0] for x in ns.split(' ')] self.request = add_namespaces(self.request, ns_keys) self.request = util.element_to_string(self.request, encoding='utf-8') #Modified by Ross Thompson for use with FGP #self.response = util.http_post(xml_post_url, self.request, self.lang, self.timeout, self.username, self.password) self.response = util.http_post(self.url, self.request, self.lang, self.timeout, self.username, self.password) # parse result see if it's XML self._exml = etree.parse(StringIO.StringIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval('ows:ExceptionReport', namespaces), util.nspath_eval('csw:Capabilities', namespaces), util.nspath_eval('csw:DescribeRecordResponse', namespaces), util.nspath_eval('csw:GetDomainResponse', namespaces), util.nspath_eval('csw:GetRecordsResponse', namespaces), util.nspath_eval('csw:GetRecordByIdResponse', namespaces), util.nspath_eval('csw:HarvestResponse', namespaces), util.nspath_eval('csw:TransactionResponse', namespaces) ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError('Document is XML, but not CSW-ish') # check if it's an OGC Exception val = self._exml.find(util.nspath_eval('ows:Exception', namespaces)) if val is not None: raise ows.ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def _invoke(self): # do HTTP request if isinstance(self.request, basestring): # GET KVP self.response = urlopen(self.request, timeout=self.timeout).read() else: xml_post_url = self.url # Get correct POST URL based on Operation list. # If skip_caps=True, then self.operations has not been set, so use # default URL. if hasattr(self, 'operations'): for op in self.operations: post_verbs = filter(lambda x: x.get('type').lower() == 'post', op.methods) if len(post_verbs) > 1: # Filter by constraints. We must match a PostEncoding of "XML" try: xml_post_url = next(x for x in filter(list, ([pv.get('url') for const in pv.get('constraints') if const.name.lower() == "postencoding" and 'xml' in map(lambda x: x.lower(), const.values)] for pv in post_verbs)))[0] except StopIteration: # Well, just use the first one. xml_post_url = post_verbs[0].get('url') elif len(post_verbs) == 1: xml_post_url = post_verbs[0].get('url') self.request = cleanup_namespaces(self.request) # Add any namespaces used in the "typeNames" attribute of the # csw:Query element to the query's xml namespaces. for query in self.request.findall(util.nspath_eval('csw:Query', namespaces)): ns = query.get("typeNames", None) if ns is not None: # Pull out "gmd" from something like "gmd:MD_Metadata" from the list # of typenames ns_keys = [x.split(':')[0] for x in ns.split(' ')] self.request = add_namespaces(self.request, ns_keys) self.request = util.element_to_string(self.request, encoding='utf-8') self.response = util.http_post(xml_post_url, self.request, self.lang, self.timeout) # parse result see if it's XML self._exml = etree.parse(StringIO.StringIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval('ows:ExceptionReport', namespaces), util.nspath_eval('csw:Capabilities', namespaces), util.nspath_eval('csw:DescribeRecordResponse', namespaces), util.nspath_eval('csw:GetDomainResponse', namespaces), util.nspath_eval('csw:GetRecordsResponse', namespaces), util.nspath_eval('csw:GetRecordByIdResponse', namespaces), util.nspath_eval('csw:HarvestResponse', namespaces), util.nspath_eval('csw:TransactionResponse', namespaces) ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError, 'Document is XML, but not CSW-ish' # check if it's an OGC Exception val = self._exml.find(util.nspath_eval('ows:Exception', namespaces)) if val is not None: raise ows.ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None
def _invoke(self): # do HTTP request request_url = self.url # Get correct URL based on Operation list. # If skip_caps=True, then self.operations has not been set, so use # default URL. if hasattr(self, 'operations'): caller = inspect.stack()[1][3] if caller == 'getrecords2': caller = 'getrecords' try: op = self.get_operation_by_name(caller) if isinstance(self.request, six.string_types): # GET KVP get_verbs = [x for x in op.methods if x.get('type').lower() == 'get'] request_url = get_verbs[0].get('url') else: post_verbs = [x for x in op.methods if x.get('type').lower() == 'post'] if len(post_verbs) > 1: # Filter by constraints. We must match a PostEncoding of "XML" for pv in post_verbs: for const in pv.get('constraints'): if const.name.lower() == 'postencoding': values = [v.lower() for v in const.values] if 'xml' in values: request_url = pv.get('url') break else: # Well, just use the first one. request_url = post_verbs[0].get('url') elif len(post_verbs) == 1: request_url = post_verbs[0].get('url') except: # no such luck, just go with request_url pass if isinstance(self.request, six.string_types): # GET KVP self.request = '%s%s' % (bind_url(request_url), self.request) self.response = openURL(self.request, None, 'Get', username=self.username, password=self.password, timeout=self.timeout).read() else: self.request = cleanup_namespaces(self.request) # Add any namespaces used in the "typeNames" attribute of the # csw:Query element to the query's xml namespaces. for query in self.request.findall(util.nspath_eval('csw:Query', namespaces)): ns = query.get("typeNames", None) if ns is not None: # Pull out "gmd" from something like "gmd:MD_Metadata" from the list # of typenames ns_keys = [x.split(':')[0] for x in ns.split(' ')] self.request = add_namespaces(self.request, ns_keys) self.request = util.element_to_string(self.request, encoding='utf-8') self.response = util.http_post(request_url, self.request, self.lang, self.timeout, self.username, self.password) # parse result see if it's XML self._exml = etree.parse(BytesIO(self.response)) # it's XML. Attempt to decipher whether the XML response is CSW-ish """ valid_xpaths = [ util.nspath_eval('ows:ExceptionReport', namespaces), util.nspath_eval('csw:Capabilities', namespaces), util.nspath_eval('csw:DescribeRecordResponse', namespaces), util.nspath_eval('csw:GetDomainResponse', namespaces), util.nspath_eval('csw:GetRecordsResponse', namespaces), util.nspath_eval('csw:GetRecordByIdResponse', namespaces), util.nspath_eval('csw:HarvestResponse', namespaces), util.nspath_eval('csw:TransactionResponse', namespaces) ] if self._exml.getroot().tag not in valid_xpaths: raise RuntimeError('Document is XML, but not CSW-ish') # check if it's an OGC Exception val = self._exml.find(util.nspath_eval('ows:Exception', namespaces)) if val is not None: raise ows.ExceptionReport(self._exml, self.owscommon.namespace) else: self.exceptionreport = None