def query_records(self, bbox=[], keywords=None, limit=10, offset=1): self.constraints = [] # only apply spatial filter if bbox is not global # even for a global bbox, if a spatial filter is applied, then # the CSW server will skip records without a bbox if bbox and bbox != ['-180', '-90', '180', '90']: minx, miny, maxx, maxy = bbox self.constraints.append( BBox([miny, minx, maxy, maxx], crs='urn:ogc:def:crs:EPSG::4326')) # keywords if keywords: # TODO: handle multiple word searches self.constraints.append(PropertyIsLike('csw:AnyText', keywords)) if len(self.constraints) > 1: # exclusive search (a && b) self.constraints = [self.constraints] self.conn.getrecords2(constraints=self.constraints, maxrecords=limit, startposition=offset, esn='full') self.matches = self.conn.results['matches'] self.returned = self.conn.results['returned'] self.request = self.conn.request self.response = self.conn.response
def filter_spat_bbox(self, propertyname=None, coord=None, epsg_code=None): if coord: tag = BBox( bbox=coord, crs="EPSG:{}".format(epsg_code), ) return etree.tostring(tag.toXML()).decode("utf-8") elif not propertyname and not coord: return { "coord": [ "Latitude down left", "Longitude down left", "Latitude up right", "Longitude up right", ], "epsg_code": "epsg code projection", } else: raise Exception("Filter bbox: error")
def get_filter_fes(filters, logical_operator=And): """Create filter encoding specification (OGC FES) object based on given filters :param Tupple.<dict> filters: tupple of filters :param logical_operator: owslib.fes.And or owslib.fes.Or :return: filter encoding specification :rtype: owslib.fes.AND """ conditions = [] filter_request = None for myfilter in filters: value = myfilter['value'] if myfilter['operator'] == '=': conditions.append(PropertyIsEqualTo(myfilter['attribute'], value)) elif myfilter['operator'] == '!=': conditions.append( PropertyIsNotEqualTo(myfilter['attribute'], value)) elif myfilter['operator'] == '~': conditions.append(PropertyIsLike(myfilter['attribute'], value)) elif myfilter['operator'] == '>': conditions.append( PropertyIsGreaterThan(myfilter['attribute'], value)) elif myfilter['operator'] == '>=': conditions.append( PropertyIsGreaterThanOrEqualTo(myfilter['attribute'], value)) elif myfilter['operator'] == '<': conditions.append(PropertyIsLessThan(myfilter['attribute'], value)) elif myfilter['operator'] == '<=': conditions.append( PropertyIsLessThanOrEqualTo(myfilter['attribute'], value)) elif myfilter['operator'] == 'BETWEEN': conditions.append( PropertyIsBetween(myfilter['attribute'], *value.split(','))) elif myfilter['operator'] == 'BBOX': bbox_filter = BBox(myfilter['value'], 'EPSG:3857') conditions.append(bbox_filter) elif myfilter['operator'] == 'IN': new_filters = [{ 'value': value, 'operator': '=', 'attribute': myfilter['attribute']}\ for value in value.split(',')] conditions.append(get_filter_fes(new_filters, logical_operator=Or)) if len(conditions) > 1: filter_request = logical_operator(conditions) else: filter_request = conditions[0] return filter_request
def __convert_query_params(self, product_type_def, product_type, params): """Translates eodag search to CSW constraints using owslib constraint classes""" constraints = [] # How the match should be performed (fuzzy, prefix, postfix or exact). # defaults to fuzzy pt_tag, matching = ( product_type_def["name"], product_type_def.get("matching", "fuzzy"), ) if matching == "prefix": constraints.append( PropertyIsLike(pt_tag, "{}%".format(product_type))) elif matching == "postfix": constraints.append( PropertyIsLike(pt_tag, "%{}".format(product_type))) elif matching == "exact": constraints.append(PropertyIsEqualTo(pt_tag, product_type)) else: # unknown matching is considered to be equal to 'fuzzy' constraints.append( PropertyIsLike(pt_tag, "%{}%".format(product_type))) # footprint fp = params.get("geometry") if fp: constraints.append( BBox([fp["lonmin"], fp["latmin"], fp["lonmax"], fp["latmax"]])) # dates start, end = ( params.get("startTimeFromAscendingNode"), params.get("completionTimeFromAscendingNode"), ) if start and "date_tags" in self.config.search_definition: constraints.append( PropertyIsGreaterThanOrEqualTo( self.config.search_definition["date_tags"]["start"], start)) if end and "date_tags" in self.config.search_definition: constraints.append( PropertyIsLessThanOrEqualTo( self.config.search_definition["date_tags"]["end"], end)) return [constraints] if len(constraints) > 1 else constraints
def search(self, keywords, startposition, maxrecords, bbox): """CSW search wrapper""" formats = [] for f in self.formats: formats.append(METADATA_FORMATS[f][0]) dataset_query_like = [] if keywords: for _kw in keywords: dataset_query_like.append(PropertyIsLike('csw:AnyText', _kw)) bbox_query = [] if bbox: bbox_query = BBox(bbox) return self.getrecords2( typenames=' '.join(formats), constraints=dataset_query_like + bbox_query, startposition=startposition, maxrecords=maxrecords, outputschema='http://www.isotc211.org/2005/gmd', esn='full')
def get_filepaths(self, collection_id: str, spatial_extent: List[float], temporal_extent: List[str]) -> List[str]: """Retrieve a file list from the a CSW server according to the specified parameters. Arguments: collecion_id {str} -- identifier of the collection spatial_extent {List[float]} -- bounding box [ymin, xmin, ymax, xmax] temporal_extent {List[str]} -- e.g. ["2018-06-04", "2018-06-23"] Returns: list -- list of filepaths """ csw = CatalogueServiceWeb(self.csw_server_uri, timeout=300) constraints = [] constraints.append(PropertyIsLike(self.group_property, collection_id)) # Spatial filter constraints.append(BBox(spatial_extent)) # Temporal filter constraints.append( PropertyIsGreaterThan('apiso:TempExtent_begin', temporal_extent[0])) constraints.append( PropertyIsLessThan('apiso:TempExtent_end', temporal_extent[1])) # Run the query csw.getrecords2(constraints=[constraints], maxrecords=100) # Put found records in a variable (dictionary) records0 = csw.records # Sort records records = [] for record in records0: records.append(records0[record].references[0]['url']) records = sorted(records) return records
def test_csw(self): # test 2.0.2 Basic Service Profile self.csw = CatalogueServiceWeb(self.url, version='2.0.2', username=self.username, password=self.password) self.assertEqual(self.csw.version, '2.0.2') self.assertIn('2.0.2', self.csw.parameters['version'].values) self.assertIn('3.0.0', self.csw.parameters['version'].values) for op in self.csw.operations: for method in op.methods: self.assertEqual(self.csw.url, method['url']) self.assertTrue('Transaction' in [o.name for o in self.csw.operations]) self.assertTrue('Harvest' in [o.name for o in self.csw.operations]) get_records_op = self.csw.get_operation_by_name('GetRecords') self.assertIn('application/json', get_records_op.parameters['outputFormat']['values']) # test basic search, no predicates self.csw.getrecords2() self.assertEqual(Layer.objects.all().count(), self.csw.results['matches']) # test csw:AnyText anytext = PropertyIsLike('csw:AnyText', 'Brasilia') self.csw.getrecords2(constraints=[anytext]) self.assertEqual(self.csw.results['matches'], 1) anytext = PropertyIsLike('csw:AnyText', 'roads') self.csw.getrecords2(constraints=[anytext]) self.assertEqual(self.csw.results['matches'], 4) # test ogc:BBOX bbox = BBox(['-13', '-80', '15', '-30']) self.csw.getrecords2(constraints=[bbox]) self.assertEqual(self.csw.results['matches'], 2) # test csw:AnyText OR ogc:BBOX self.csw.getrecords2(constraints=[anytext, bbox]) self.assertEqual(self.csw.results['matches'], 5) # test csw:AnyText && ogc:BBOX self.csw.getrecords2(constraints=[[anytext, bbox]]) self.assertEqual(self.csw.results['matches'], 1) # test that ElementSetName=full stores full metadata record as inserted self.csw.getrecords2(esn='full') self.assertIn( 'xmlns:registry="http://gis.harvard.edu/HHypermap/registry/0.1"', self.csw.response) # test JSON output # TODO: fix owslib.csw.CatalogueServiceWeb.getrecords2 to handle non-XML request/response with self.assertRaises(XMLSyntaxError): self.csw.getrecords2(constraints=[anytext, bbox], format='application/json') records_json = json.loads(self.csw.response) self.assertEqual( records_json['csw:GetRecordsResponse']['csw:SearchResults'] ['@numberOfRecordsMatched'], '5') # test 3.0.0 OpenSearch bsp = { 'mode': 'opensearch', 'service': 'CSW', 'version': '3.0.0', 'request': 'GetRecords', 'typenames': 'csw:Record', 'elementsetname': 'full', 'outputformat': 'application/json' } # test basic search, no predicates res = json.loads(self.client.get(self.script_name, bsp).content) self.assertEqual(res['atom:feed']['os:totalResults'], '10') # test q bsp['q'] = 'Brasilia' res = json.loads(self.client.get(self.script_name, bsp).content) self.assertEqual(res['atom:feed']['os:totalResults'], '1') bsp.pop('q') # test bbox bsp['bbox'] = '-80,-13,-30,15' res = json.loads(self.client.get(self.script_name, bsp).content) self.assertEqual(res['atom:feed']['os:totalResults'], '2') bsp.pop('bbox') # test time bsp['time'] = '2014-09-23T12:04:31.102243+00:00/' res = json.loads(self.client.get(self.script_name, bsp).content) self.assertEqual(res['atom:feed']['os:totalResults'], '10') bsp.pop('time') # test q and bbox bsp['q'] = 'roads' bsp['bbox'] = '-80,-13,-30,15' res = json.loads(self.client.get(self.script_name, bsp).content) self.assertEqual(res['atom:feed']['os:totalResults'], '1') # test q and bbox and time bsp['time'] = '2014-09-23T12:04:31.102243+00:00/' res = json.loads(self.client.get(self.script_name, bsp).content) self.assertEqual(res['atom:feed']['os:totalResults'], '1')
def search(self): """execute search""" self.catalog = None self.constraints = [] # clear all fields and disable buttons self.lblResults.clear() self.treeRecords.clear() self.reset_buttons() # save some settings self.settings.setValue('/MetaSearch/returnRecords', self.spnRecords.cleanText()) # set current catalog current_text = self.cmbConnectionsSearch.currentText() key = '/MetaSearch/%s' % current_text self.catalog_url = self.settings.value('%s/url' % key) self.catalog_username = self.settings.value('%s/username' % key) self.catalog_password = self.settings.value('%s/password' % key) # start position and number of records to return self.startfrom = 0 self.maxrecords = self.spnRecords.value() # set timeout self.timeout = self.spnTimeout.value() # bbox # CRS is WGS84 with axis order longitude, latitude # defined by 'urn:ogc:def:crs:OGC:1.3:CRS84' minx = self.leWest.text() miny = self.leSouth.text() maxx = self.leEast.text() maxy = self.leNorth.text() bbox = [minx, miny, maxx, maxy] # only apply spatial filter if bbox is not global # even for a global bbox, if a spatial filter is applied, then # the CSW server will skip records without a bbox if bbox != ['-180', '-90', '180', '90']: self.constraints.append( BBox(bbox, crs='urn:ogc:def:crs:OGC:1.3:CRS84')) # keywords if self.leKeywords.text(): # TODO: handle multiple word searches keywords = self.leKeywords.text() self.constraints.append(PropertyIsLike('csw:AnyText', keywords)) if len(self.constraints) > 1: # exclusive search (a && b) self.constraints = [self.constraints] # build request if not self._get_csw(): return # TODO: allow users to select resources types # to find ('service', 'dataset', etc.) try: with OverrideCursor(Qt.WaitCursor): self.catalog.getrecords2(constraints=self.constraints, maxrecords=self.maxrecords, esn='full') except ExceptionReport as err: QMessageBox.warning(self, self.tr('Search error'), self.tr('Search error: {0}').format(err)) return except Exception as err: QMessageBox.warning(self, self.tr('Connection error'), self.tr('Connection error: {0}').format(err)) return if self.catalog.results['matches'] == 0: self.lblResults.setText(self.tr('0 results')) return self.display_results()
def search(self, text, bbox=None, start=0): constraints = [] try: csw = CatalogueServiceWeb(self.url) except: return { 'error': 'Error in establishing connection to server ' + self.url } query = PropertyIsLike('csw:AnyText', '%' + text + '%') constraints.append(query) if bbox != None: bbox_query = BBox(bbox) constraints.append(bbox_query) # execute search # esn = elementsetname (full, summary, brief) # other options may not work with all csw instances! csw.getrecords2(constraints=constraints, startposition=start, maxrecords=10, typenames='gmd:MD_Metadata', outputschema='http://www.isotc211.org/2005/gmd', esn='full') totalCount = csw.results['matches'] returned = csw.results['returned'] results = [] error = [] for identifier in csw.records: record = csw.records[identifier] try: ogc_link = ogc_type = ogc_layer = '' for online in record.distribution.online: # only allow WMS, WFS and SOS services for visualization if online.protocol == None: continue if 'TiledMapService' in online.protocol: ogc_link = online.url.replace( 'http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver') ogc_type = 'TMS' break elif 'WebMapService' in online.protocol: ogc_link = online.url.replace( 'http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver') ogc_layer = online.name ogc_type = 'WMS' break #elif 'WebMapTileService' in online.protocol: # ogc_link = online.url.replace('http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver') # ogc_layer = online.name # ogc_type = 'WMTS' # break elif 'WebFeatureService' in online.protocol: ogc_link = online.url.replace( 'http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver') ogc_layer = online.name ogc_type = 'WFS' break elif 'SensorObservationService' in online.protocol: ogc_link = online.url.replace( 'http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver') ogc_layer = online.name ogc_type = 'SOS' break # check spatial resolution info distance = '' if len(record.identification.distance) > 0: distance = record.identification.distance[0] # convert OWSlib metadata object to django layers metadata object structure results.append({ 'identifier': record.identifier, 'title': record.identification.title, 'abstract': record.identification.abstract, 'ogc_link': ogc_link, 'ogc_layer': ogc_layer, 'ogc_type': ogc_type, 'topicCategory': ', '.join(record.identification.topiccategory), 'dataset_contact_new': { 'first_name': '', 'last_name': record.identification.contact[0].name, 'position': record.identification.contact[0].position, 'address': record.identification.contact[0].address, 'postcode': record.identification.contact[0].postcode, 'city': record.identification.contact[0].city, 'country': record.identification.contact[0].country, 'state': record.identification.contact[0].region, 'email': record.identification.contact[0].email, 'organisation': record.identification.contact[0].organization, 'telephone': record.identification.contact[0].phone, 'fax': record.identification.contact[0].fax, 'mobile': '', 'website': '', #record.identification.contact[0].onlineresource.url, 'role': record.identification.contact[0].role }, 'meta_contact': { 'first_name': '', 'last_name': record.contact[0].name, 'position': record.contact[0].position, 'address': record.contact[0].address, 'postcode': record.contact[0].postcode, 'city': record.contact[0].city, 'country': record.contact[0].country, 'state': record.contact[0].region, 'email': record.contact[0].email, 'organisation': record.contact[0].organization, 'telephone': record.contact[0].phone, 'fax': record.contact[0].fax, 'mobile': '', 'website': '', #record.contact[0].onlineresource.url, 'role': record.contact[0].role }, 'date_create': record.identification.date[0].date, 'date_type': record.identification.date[0].type, 'language': record.identification.resourcelanguage, 'characterset': '', 'format': record.distribution.format, 'west': float(record.identification.bbox.minx), 'east': float(record.identification.bbox.maxx), 'north': float(record.identification.bbox.maxy), 'south': float(record.identification.bbox.miny), 'alternatetitle': record.identification.alternatetitle, 'geo_description': '', 'representation_type': '', 'equi_scale': distance, #'epsg': record.referencesystem.code, 'meta_language': record.language, 'meta_characterset': record.charset, 'meta_date': record.datestamp }) except Exception as e: error.append(str(e)) return { 'records': results, 'totalCount': totalCount, 'count': returned, 'error': error }
from owslib.csw import CatalogueServiceWeb cenia = CatalogueServiceWeb('http://geoportal.gov.cz/php/micka/csw/index.php') print (cenia.service) cenia.getrecords2() print (cenia.results) for rec in cenia.records: print (cenia.records[rec].title) from owslib.fes import PropertyIsLike, BBox, And, PropertyIsEqualTo wms_query = PropertyIsEqualTo('csw:AnyText', 'WMS') praha_query = BBox([14.22,49.94,14.71,50.18]) praha_and_wms = And([praha_query, wms_query]) cenia.getrecords2([praha_and_wms], esn='full') print (cenia.results) for recid in cenia.records: record = cenia.records[recid] print (u'{}: {} {} {} {}'.format(record.title, record.bbox.minx, record.bbox.miny, record.bbox.maxx, record.bbox.maxy)) zm_query = PropertyIsEqualTo('csw:AnyText', 'ZM10') cenia.getrecords2([zm_query], esn='full') zm10 = cenia.records['CZ-CUZK-WMS-ZM10-P'] print (zm10.type) print (u'{}\n{}'.format(zm10.title, zm10.abstract)) url = zm10.references[0]['url']
[op.name for op in csw.operations] # Get supported resultType’s: csw.getdomain('GetRecords.resultType') csw.results # Search for bird data: from owslib.fes import PropertyIsEqualTo, PropertyIsLike, BBox birds_query = PropertyIsEqualTo('csw:AnyText', 'birds') csw.getrecords2(constraints=[birds_query], maxrecords=20) csw.results for rec in csw.records: print(csw.records[rec].title) # Search for bird data in Canada bbox_query = BBox([-141, 42, -52, 84]) csw.getrecords2(constraints=[birds_query, bbox_query]) csw.results # Search for keywords like ‘birds’ or ‘fowl’ birds_query_like = PropertyIsLike('dc:subject', '%birds%') fowl_query_like = PropertyIsLike('dc:subject', '%fowl%') csw.getrecords2(constraints=[birds_query_like, fowl_query_like]) csw.results # Search for a specific record: csw.getrecordbyid(id=['9250AA67-F3AC-6C12-0CB9-0662231AA181']) c.records['9250AA67-F3AC-6C12-0CB9-0662231AA181'].title
#print "*" * 30 #print "Metadata" #print "*" * 30 #for lst in [my for my in out.read().split('\n')]: #print lst #gml2json(out.read(), echo=True) filter1 = BBox( bbox=[ 59.97111801186481728, 30.21720754623224181, 59.97569926211409097, 30.22404557000332304, ], #bbox=[ #59.94617, #30.23334, #59.94618, #30.23335, #], #crs="urn:ogc:def:crs:EPSG::4326", crs="EPSG:4326") #filter1 = BBox( #bbox=[ #3364107.934602736961, #8393636.548086917028, #3364263.219452924561, #8393740.583811631426 #], ##crs="urn:ogc:def:crs:EPSG::3857",