def query_records(self, bbox=[], keywords=None, limit=10, offset=1):
self.constraints = []
# only apply spatial filter if bbox is not global
# even for a global bbox, if a spatial filter is applied, then
# the CSW server will skip records without a bbox
if bbox and bbox != ['-180', '-90', '180', '90']:
minx, miny, maxx, maxy = bbox
self.constraints.append(
BBox([miny, minx, maxy, maxx],
crs='urn:ogc:def:crs:EPSG::4326'))
# keywords
if keywords:
# TODO: handle multiple word searches
self.constraints.append(PropertyIsLike('csw:AnyText', keywords))
if len(self.constraints) > 1: # exclusive search (a && b)
self.constraints = [self.constraints]
self.conn.getrecords2(constraints=self.constraints,
maxrecords=limit,
startposition=offset,
esn='full')
self.matches = self.conn.results['matches']
self.returned = self.conn.results['returned']
self.request = self.conn.request
self.response = self.conn.response
def filter_spat_bbox(self, propertyname=None, coord=None, epsg_code=None):
if coord:
tag = BBox(
bbox=coord,
crs="EPSG:{}".format(epsg_code),
)
return etree.tostring(tag.toXML()).decode("utf-8")
elif not propertyname and not coord:
return {
"coord": [
"Latitude down left",
"Longitude down left",
"Latitude up right",
"Longitude up right",
],
"epsg_code": "epsg code projection",
}
else:
raise Exception("Filter bbox: error")
def get_filter_fes(filters, logical_operator=And):
"""Create filter encoding specification (OGC FES) object based on given
filters
:param Tupple.<dict> filters: tupple of filters
:param logical_operator: owslib.fes.And or owslib.fes.Or
:return: filter encoding specification
:rtype: owslib.fes.AND
"""
conditions = []
filter_request = None
for myfilter in filters:
value = myfilter['value']
if myfilter['operator'] == '=':
conditions.append(PropertyIsEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == '!=':
conditions.append(
PropertyIsNotEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == '~':
conditions.append(PropertyIsLike(myfilter['attribute'], value))
elif myfilter['operator'] == '>':
conditions.append(
PropertyIsGreaterThan(myfilter['attribute'], value))
elif myfilter['operator'] == '>=':
conditions.append(
PropertyIsGreaterThanOrEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == '<':
conditions.append(PropertyIsLessThan(myfilter['attribute'], value))
elif myfilter['operator'] == '<=':
conditions.append(
PropertyIsLessThanOrEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == 'BETWEEN':
conditions.append(
PropertyIsBetween(myfilter['attribute'], *value.split(',')))
elif myfilter['operator'] == 'BBOX':
bbox_filter = BBox(myfilter['value'], 'EPSG:3857')
conditions.append(bbox_filter)
elif myfilter['operator'] == 'IN':
new_filters = [{
'value': value,
'operator': '=',
'attribute': myfilter['attribute']}\
for value in value.split(',')]
conditions.append(get_filter_fes(new_filters, logical_operator=Or))
if len(conditions) > 1:
filter_request = logical_operator(conditions)
else:
filter_request = conditions[0]
return filter_request
def __convert_query_params(self, product_type_def, product_type, params):
"""Translates eodag search to CSW constraints using owslib constraint classes"""
constraints = []
# How the match should be performed (fuzzy, prefix, postfix or exact).
# defaults to fuzzy
pt_tag, matching = (
product_type_def["name"],
product_type_def.get("matching", "fuzzy"),
)
if matching == "prefix":
constraints.append(
PropertyIsLike(pt_tag, "{}%".format(product_type)))
elif matching == "postfix":
constraints.append(
PropertyIsLike(pt_tag, "%{}".format(product_type)))
elif matching == "exact":
constraints.append(PropertyIsEqualTo(pt_tag, product_type))
else: # unknown matching is considered to be equal to 'fuzzy'
constraints.append(
PropertyIsLike(pt_tag, "%{}%".format(product_type)))
# footprint
fp = params.get("geometry")
if fp:
constraints.append(
BBox([fp["lonmin"], fp["latmin"], fp["lonmax"], fp["latmax"]]))
# dates
start, end = (
params.get("startTimeFromAscendingNode"),
params.get("completionTimeFromAscendingNode"),
)
if start and "date_tags" in self.config.search_definition:
constraints.append(
PropertyIsGreaterThanOrEqualTo(
self.config.search_definition["date_tags"]["start"],
start))
if end and "date_tags" in self.config.search_definition:
constraints.append(
PropertyIsLessThanOrEqualTo(
self.config.search_definition["date_tags"]["end"], end))
return [constraints] if len(constraints) > 1 else constraints
def search(self, keywords, startposition, maxrecords, bbox):
"""CSW search wrapper"""
formats = []
for f in self.formats:
formats.append(METADATA_FORMATS[f][0])
dataset_query_like = []
if keywords:
for _kw in keywords:
dataset_query_like.append(PropertyIsLike('csw:AnyText', _kw))
bbox_query = []
if bbox:
bbox_query = BBox(bbox)
return self.getrecords2(
typenames=' '.join(formats),
constraints=dataset_query_like + bbox_query,
startposition=startposition,
maxrecords=maxrecords,
outputschema='http://www.isotc211.org/2005/gmd',
esn='full')
def get_filepaths(self, collection_id: str, spatial_extent: List[float],
temporal_extent: List[str]) -> List[str]:
"""Retrieve a file list from the a CSW server according to the specified parameters.
Arguments:
collecion_id {str} -- identifier of the collection
spatial_extent {List[float]} -- bounding box [ymin, xmin, ymax, xmax]
temporal_extent {List[str]} -- e.g. ["2018-06-04", "2018-06-23"]
Returns:
list -- list of filepaths
"""
csw = CatalogueServiceWeb(self.csw_server_uri, timeout=300)
constraints = []
constraints.append(PropertyIsLike(self.group_property, collection_id))
# Spatial filter
constraints.append(BBox(spatial_extent))
# Temporal filter
constraints.append(
PropertyIsGreaterThan('apiso:TempExtent_begin',
temporal_extent[0]))
constraints.append(
PropertyIsLessThan('apiso:TempExtent_end', temporal_extent[1]))
# Run the query
csw.getrecords2(constraints=[constraints], maxrecords=100)
# Put found records in a variable (dictionary)
records0 = csw.records
# Sort records
records = []
for record in records0:
records.append(records0[record].references[0]['url'])
records = sorted(records)
return records
def test_csw(self):
# test 2.0.2 Basic Service Profile
self.csw = CatalogueServiceWeb(self.url,
version='2.0.2',
username=self.username,
password=self.password)
self.assertEqual(self.csw.version, '2.0.2')
self.assertIn('2.0.2', self.csw.parameters['version'].values)
self.assertIn('3.0.0', self.csw.parameters['version'].values)
for op in self.csw.operations:
for method in op.methods:
self.assertEqual(self.csw.url, method['url'])
self.assertTrue('Transaction' in [o.name for o in self.csw.operations])
self.assertTrue('Harvest' in [o.name for o in self.csw.operations])
get_records_op = self.csw.get_operation_by_name('GetRecords')
self.assertIn('application/json',
get_records_op.parameters['outputFormat']['values'])
# test basic search, no predicates
self.csw.getrecords2()
self.assertEqual(Layer.objects.all().count(),
self.csw.results['matches'])
# test csw:AnyText
anytext = PropertyIsLike('csw:AnyText', 'Brasilia')
self.csw.getrecords2(constraints=[anytext])
self.assertEqual(self.csw.results['matches'], 1)
anytext = PropertyIsLike('csw:AnyText', 'roads')
self.csw.getrecords2(constraints=[anytext])
self.assertEqual(self.csw.results['matches'], 4)
# test ogc:BBOX
bbox = BBox(['-13', '-80', '15', '-30'])
self.csw.getrecords2(constraints=[bbox])
self.assertEqual(self.csw.results['matches'], 2)
# test csw:AnyText OR ogc:BBOX
self.csw.getrecords2(constraints=[anytext, bbox])
self.assertEqual(self.csw.results['matches'], 5)
# test csw:AnyText && ogc:BBOX
self.csw.getrecords2(constraints=[[anytext, bbox]])
self.assertEqual(self.csw.results['matches'], 1)
# test that ElementSetName=full stores full metadata record as inserted
self.csw.getrecords2(esn='full')
self.assertIn(
'xmlns:registry="http://gis.harvard.edu/HHypermap/registry/0.1"',
self.csw.response)
# test JSON output
# TODO: fix owslib.csw.CatalogueServiceWeb.getrecords2 to handle non-XML request/response
with self.assertRaises(XMLSyntaxError):
self.csw.getrecords2(constraints=[anytext, bbox],
format='application/json')
records_json = json.loads(self.csw.response)
self.assertEqual(
records_json['csw:GetRecordsResponse']['csw:SearchResults']
['@numberOfRecordsMatched'], '5')
# test 3.0.0 OpenSearch
bsp = {
'mode': 'opensearch',
'service': 'CSW',
'version': '3.0.0',
'request': 'GetRecords',
'typenames': 'csw:Record',
'elementsetname': 'full',
'outputformat': 'application/json'
}
# test basic search, no predicates
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '10')
# test q
bsp['q'] = 'Brasilia'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '1')
bsp.pop('q')
# test bbox
bsp['bbox'] = '-80,-13,-30,15'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '2')
bsp.pop('bbox')
# test time
bsp['time'] = '2014-09-23T12:04:31.102243+00:00/'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '10')
bsp.pop('time')
# test q and bbox
bsp['q'] = 'roads'
bsp['bbox'] = '-80,-13,-30,15'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '1')
# test q and bbox and time
bsp['time'] = '2014-09-23T12:04:31.102243+00:00/'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '1')
def search(self):
"""execute search"""
self.catalog = None
self.constraints = []
# clear all fields and disable buttons
self.lblResults.clear()
self.treeRecords.clear()
self.reset_buttons()
# save some settings
self.settings.setValue('/MetaSearch/returnRecords',
self.spnRecords.cleanText())
# set current catalog
current_text = self.cmbConnectionsSearch.currentText()
key = '/MetaSearch/%s' % current_text
self.catalog_url = self.settings.value('%s/url' % key)
self.catalog_username = self.settings.value('%s/username' % key)
self.catalog_password = self.settings.value('%s/password' % key)
# start position and number of records to return
self.startfrom = 0
self.maxrecords = self.spnRecords.value()
# set timeout
self.timeout = self.spnTimeout.value()
# bbox
# CRS is WGS84 with axis order longitude, latitude
# defined by 'urn:ogc:def:crs:OGC:1.3:CRS84'
minx = self.leWest.text()
miny = self.leSouth.text()
maxx = self.leEast.text()
maxy = self.leNorth.text()
bbox = [minx, miny, maxx, maxy]
# only apply spatial filter if bbox is not global
# even for a global bbox, if a spatial filter is applied, then
# the CSW server will skip records without a bbox
if bbox != ['-180', '-90', '180', '90']:
self.constraints.append(
BBox(bbox, crs='urn:ogc:def:crs:OGC:1.3:CRS84'))
# keywords
if self.leKeywords.text():
# TODO: handle multiple word searches
keywords = self.leKeywords.text()
self.constraints.append(PropertyIsLike('csw:AnyText', keywords))
if len(self.constraints) > 1: # exclusive search (a && b)
self.constraints = [self.constraints]
# build request
if not self._get_csw():
return
# TODO: allow users to select resources types
# to find ('service', 'dataset', etc.)
try:
with OverrideCursor(Qt.WaitCursor):
self.catalog.getrecords2(constraints=self.constraints,
maxrecords=self.maxrecords,
esn='full')
except ExceptionReport as err:
QMessageBox.warning(self, self.tr('Search error'),
self.tr('Search error: {0}').format(err))
return
except Exception as err:
QMessageBox.warning(self, self.tr('Connection error'),
self.tr('Connection error: {0}').format(err))
return
if self.catalog.results['matches'] == 0:
self.lblResults.setText(self.tr('0 results'))
return
self.display_results()
def search(self, text, bbox=None, start=0):
constraints = []
try:
csw = CatalogueServiceWeb(self.url)
except:
return {
'error':
'Error in establishing connection to server ' + self.url
}
query = PropertyIsLike('csw:AnyText', '%' + text + '%')
constraints.append(query)
if bbox != None:
bbox_query = BBox(bbox)
constraints.append(bbox_query)
# execute search
# esn = elementsetname (full, summary, brief)
# other options may not work with all csw instances!
csw.getrecords2(constraints=constraints,
startposition=start,
maxrecords=10,
typenames='gmd:MD_Metadata',
outputschema='http://www.isotc211.org/2005/gmd',
esn='full')
totalCount = csw.results['matches']
returned = csw.results['returned']
results = []
error = []
for identifier in csw.records:
record = csw.records[identifier]
try:
ogc_link = ogc_type = ogc_layer = ''
for online in record.distribution.online:
# only allow WMS, WFS and SOS services for visualization
if online.protocol == None:
continue
if 'TiledMapService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_type = 'TMS'
break
elif 'WebMapService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_layer = online.name
ogc_type = 'WMS'
break
#elif 'WebMapTileService' in online.protocol:
# ogc_link = online.url.replace('http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver')
# ogc_layer = online.name
# ogc_type = 'WMTS'
# break
elif 'WebFeatureService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_layer = online.name
ogc_type = 'WFS'
break
elif 'SensorObservationService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_layer = online.name
ogc_type = 'SOS'
break
# check spatial resolution info
distance = ''
if len(record.identification.distance) > 0:
distance = record.identification.distance[0]
# convert OWSlib metadata object to django layers metadata object structure
results.append({
'identifier':
record.identifier,
'title':
record.identification.title,
'abstract':
record.identification.abstract,
'ogc_link':
ogc_link,
'ogc_layer':
ogc_layer,
'ogc_type':
ogc_type,
'topicCategory':
', '.join(record.identification.topiccategory),
'dataset_contact_new': {
'first_name': '',
'last_name': record.identification.contact[0].name,
'position': record.identification.contact[0].position,
'address': record.identification.contact[0].address,
'postcode': record.identification.contact[0].postcode,
'city': record.identification.contact[0].city,
'country': record.identification.contact[0].country,
'state': record.identification.contact[0].region,
'email': record.identification.contact[0].email,
'organisation':
record.identification.contact[0].organization,
'telephone': record.identification.contact[0].phone,
'fax': record.identification.contact[0].fax,
'mobile': '',
'website':
'', #record.identification.contact[0].onlineresource.url,
'role': record.identification.contact[0].role
},
'meta_contact': {
'first_name': '',
'last_name': record.contact[0].name,
'position': record.contact[0].position,
'address': record.contact[0].address,
'postcode': record.contact[0].postcode,
'city': record.contact[0].city,
'country': record.contact[0].country,
'state': record.contact[0].region,
'email': record.contact[0].email,
'organisation': record.contact[0].organization,
'telephone': record.contact[0].phone,
'fax': record.contact[0].fax,
'mobile': '',
'website': '', #record.contact[0].onlineresource.url,
'role': record.contact[0].role
},
'date_create':
record.identification.date[0].date,
'date_type':
record.identification.date[0].type,
'language':
record.identification.resourcelanguage,
'characterset':
'',
'format':
record.distribution.format,
'west':
float(record.identification.bbox.minx),
'east':
float(record.identification.bbox.maxx),
'north':
float(record.identification.bbox.maxy),
'south':
float(record.identification.bbox.miny),
'alternatetitle':
record.identification.alternatetitle,
'geo_description':
'',
'representation_type':
'',
'equi_scale':
distance,
#'epsg': record.referencesystem.code,
'meta_language':
record.language,
'meta_characterset':
record.charset,
'meta_date':
record.datestamp
})
except Exception as e:
error.append(str(e))
return {
'records': results,
'totalCount': totalCount,
'count': returned,
'error': error
}
from owslib.csw import CatalogueServiceWeb
cenia = CatalogueServiceWeb('http://geoportal.gov.cz/php/micka/csw/index.php')
print (cenia.service)
cenia.getrecords2()
print (cenia.results)
for rec in cenia.records:
print (cenia.records[rec].title)
from owslib.fes import PropertyIsLike, BBox, And, PropertyIsEqualTo
wms_query = PropertyIsEqualTo('csw:AnyText', 'WMS')
praha_query = BBox([14.22,49.94,14.71,50.18])
praha_and_wms = And([praha_query, wms_query])
cenia.getrecords2([praha_and_wms], esn='full')
print (cenia.results)
for recid in cenia.records:
record = cenia.records[recid]
print (u'{}: {} {} {} {}'.format(record.title, record.bbox.minx, record.bbox.miny,
record.bbox.maxx, record.bbox.maxy))
zm_query = PropertyIsEqualTo('csw:AnyText', 'ZM10')
cenia.getrecords2([zm_query], esn='full')
zm10 = cenia.records['CZ-CUZK-WMS-ZM10-P']
print (zm10.type)
print (u'{}\n{}'.format(zm10.title, zm10.abstract))
url = zm10.references[0]['url']
[op.name for op in csw.operations]
# Get supported resultType’s:
csw.getdomain('GetRecords.resultType')
csw.results
# Search for bird data:
from owslib.fes import PropertyIsEqualTo, PropertyIsLike, BBox
birds_query = PropertyIsEqualTo('csw:AnyText', 'birds')
csw.getrecords2(constraints=[birds_query], maxrecords=20)
csw.results
for rec in csw.records:
print(csw.records[rec].title)
# Search for bird data in Canada
bbox_query = BBox([-141, 42, -52, 84])
csw.getrecords2(constraints=[birds_query, bbox_query])
csw.results
# Search for keywords like ‘birds’ or ‘fowl’
birds_query_like = PropertyIsLike('dc:subject', '%birds%')
fowl_query_like = PropertyIsLike('dc:subject', '%fowl%')
csw.getrecords2(constraints=[birds_query_like, fowl_query_like])
csw.results
# Search for a specific record:
csw.getrecordbyid(id=['9250AA67-F3AC-6C12-0CB9-0662231AA181'])
c.records['9250AA67-F3AC-6C12-0CB9-0662231AA181'].title
#print "*" * 30
#print "Metadata"
#print "*" * 30
#for lst in [my for my in out.read().split('\n')]:
#print lst
#gml2json(out.read(), echo=True)
filter1 = BBox(
bbox=[
59.97111801186481728,
30.21720754623224181,
59.97569926211409097,
30.22404557000332304,
],
#bbox=[
#59.94617,
#30.23334,
#59.94618,
#30.23335,
#],
#crs="urn:ogc:def:crs:EPSG::4326",
crs="EPSG:4326")
#filter1 = BBox(
#bbox=[
#3364107.934602736961,
#8393636.548086917028,
#3364263.219452924561,
#8393740.583811631426
#],
##crs="urn:ogc:def:crs:EPSG::3857",
