def test():
wfs = WebFeatureService("https://wfs.geonorge.no/skwms1/wfs.inspire-lcv",
version="2.0.0")
filter = PropertyIsLike(propertyname="lcv:LandCoverObservation",
literal="21")
filterxml = etree.tostring(filter.toXML()).decode("utf-8")
response = wfs.getfeature(typename="lcv:LandCoverUnit", filter=filterxml)
def wfs_request_matching_file_pattern(
imos_layer_name,
filename_wfs_filter,
url_column='url',
geoserver_url='http://geoserver-123.aodn.org.au/geoserver/wfs',
s3_bucket_url=False):
"""
returns a list of url matching a file pattern defined by filename_wfs_filter
* if s3_bucket_url is False, returns the url as stored in WFS layer
* if s3_bucket_url is True, append to its start the s3 IMOS bucket link used to
download the file
Examples:
wfs_request_matching_file_pattern('srs_oc_ljco_wws_hourly_wqm_fv01_timeseries_map', '%')
wfs_request_matching_file_pattern('srs_oc_ljco_wws_hourly_wqm_fv01_timeseries_map', '%', s3_bucket_url=True)
wfs_request_matching_file_pattern('srs_oc_ljco_wws_hourly_wqm_fv01_timeseries_map', '%2014/06/%')
wfs_request_matching_file_pattern('anmn_nrs_rt_meteo_timeseries_map', '%IMOS_ANMN-NRS_MT_%', url_column='file_url', s3_bucket_url=True)
WARNING: Please exec $DATA_SERVICES_DIR/lib/test/python/manual_test_wfs_query.py to run unittests before modifying function
"""
from owslib.etree import etree
from owslib.fes import PropertyIsLike
from owslib.wfs import WebFeatureService
import os
import xml.etree.ElementTree as ET
imos_layer_name = 'imos:%s' % imos_layer_name
data_aodn_http_prefix = 'http://data.aodn.org.au'
wfs11 = WebFeatureService(url=geoserver_url, version='1.1.0')
wfs_filter = PropertyIsLike(propertyname=url_column,
literal=filename_wfs_filter,
wildCard='%')
filterxml = etree.tostring(wfs_filter.toXML()).decode("utf-8")
response = wfs11.getfeature(typename=imos_layer_name,
filter=filterxml,
propertyname=[url_column])
# parse XML to get list of URLS
xml_wfs_output = response.read()
root = ET.fromstring(xml_wfs_output)
list_url = []
# parse xml
if len(root) > 0:
for item in root[0]:
for subitem in item:
file_url = subitem.text
if s3_bucket_url:
list_url.append(
os.path.join(data_aodn_http_prefix, file_url))
else:
list_url.append(file_url)
return list_url
def _get_feature_attributes_wfs(
attribute: str = None,
value: Union[str, float, int] = None,
layer: str = None,
geoserver: str = GEO_URL,
) -> str:
"""Return a URL that formats and returns remote GetFeatures request from a hosted WFS dataset.
For geographic rasters, subsetting is based on WGS84 (Long, Lat) boundaries. If not geographic, subsetting based
on projected coordinate system (Easting, Northing) boundaries.
Parameters
----------
attribute : str
Attribute/field to be queried.
value: Union[str, float, int]
Value for attribute queried.
geoserver: str
The address of the geoserver housing the layer to be queried. Default: http://pavics.ouranos.ca/geoserver/.
Returns
-------
str
URL to the GeoJSON-encoded WFS response.
"""
if attribute is None or value is None:
raise NotImplementedError()
try:
attribute = str(attribute)
value = str(value)
except ValueError:
raise Exception("Unable to cast attribute/filter to string")
filter_request = PropertyIsLike(propertyname=attribute,
literal=value,
wildCard="*")
filterxml = etree.tostring(filter_request.toXML()).decode("utf-8")
params = dict(
service="WFS",
version="1.1.0",
request="GetFeature",
typename=layer,
outputFormat="json",
filter=filterxml,
)
q = Request("GET", url=urljoin(geoserver, "wfs"),
params=params).prepare().url
return q
def test_xmlfilter_wfs_200():
wfs = WebFeatureService(
'https://services.ga.gov.au/gis/stratunits/ows',
version='2.0.0')
filter_prop = PropertyIsLike(propertyname='stratunit:geologichistory', literal='Cisuralian - Guadalupian',
matchCase=True)
filterxml = etree.tostring(filter_prop.toXML()).decode("utf-8")
getfeat_params = {'typename': 'stratunit:StratigraphicUnit', 'filter': filterxml}
response = wfs.getfeature(**getfeat_params).read()
assert b'<stratunit:geologichistory>Cisuralian - Guadalupian</stratunit:geologichistory>' in response
def filter_comp_like(self, propertyname=None, literal=None):
if propertyname and literal:
tag = PropertyIsLike(
propertyname=self.data2literal(propertyname),
literal=self.data2literal(literal),
wildCard="*",
singleChar=".",
escapeChar="!",
)
return etree.tostring(tag.toXML()).decode("utf-8")
elif not propertyname and not literal:
return "value"
else:
raise Exception("Filter error")
def test_xmlfilter_wfs_110():
wfs = WebFeatureService('https://services.ga.gov.au/gis/stratunits/ows',
version='1.1.0')
filter_prop = PropertyIsLike(propertyname='stratunit:GEOLOGICHISTORY',
literal='Cisuralian - Guadalupian',
matchCase=True)
filterxml = etree.tostring(filter_prop.toXML()).decode("utf-8")
getfeat_params = {
'typename': 'stratunit:StratigraphicUnit',
'filter': filterxml
}
response = wfs.getfeature(**getfeat_params).read()
assert b'<stratunit:NAME>Ellen Harkins carbonaceous shale</stratunit:NAME>' in response
def query_records(self, bbox=[], keywords=None, limit=10, offset=1):
self.constraints = []
# only apply spatial filter if bbox is not global
# even for a global bbox, if a spatial filter is applied, then
# the CSW server will skip records without a bbox
if bbox and bbox != ['-180', '-90', '180', '90']:
minx, miny, maxx, maxy = bbox
self.constraints.append(
BBox([miny, minx, maxy, maxx],
crs='urn:ogc:def:crs:EPSG::4326'))
# keywords
if keywords:
# TODO: handle multiple word searches
self.constraints.append(PropertyIsLike('csw:AnyText', keywords))
if len(self.constraints) > 1: # exclusive search (a && b)
self.constraints = [self.constraints]
self.conn.getrecords2(constraints=self.constraints,
maxrecords=limit,
startposition=offset,
esn='full')
self.matches = self.conn.results['matches']
self.returned = self.conn.results['returned']
self.request = self.conn.request
self.response = self.conn.response
def test_csw_outputschema_dc(self):
"""Verify that GeoNode CSW can handle ISO metadata with Dublin Core outputSchema"""
csw = get_catalogue()
# search for 'san_andres_y_providencia_location', output as Dublin Core
dataset_query_like = PropertyIsLike('csw:AnyText', '%san_andres_y_providencia_location%')
csw.catalogue.getrecords2(
typenames='gmd:MD_Metadata',
constraints=[dataset_query_like],
outputschema='http://www.opengis.net/cat/csw/2.0.2',
esn='full')
record = list(csw.catalogue.records.values())[0]
# test that the ISO title maps correctly in Dublin Core
self.assertTrue(record.title in "san_andres_y_providencia_location.shp")
# test that the ISO abstract maps correctly in Dublin Core
self.assertEqual(record.abstract, 'No abstract provided')
# test for correct service link articulation
for link in record.references:
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
if link['scheme'] == 'OGC:WMS':
self.assertEqual(link['url'], f"{settings.GEOSERVER_PUBLIC_LOCATION}ows")
elif link['scheme'] == 'OGC:WFS':
self.assertEqual(link['url'], f"{settings.GEOSERVER_PUBLIC_LOCATION}ows")
elif link['scheme'] == 'OGC:WCS':
self.assertEqual(link['url'], f"{settings.GEOSERVER_PUBLIC_LOCATION}ows")
def _filter_feature_attributes_wfs(
attribute: str,
value: Union[str, float, int],
layer: str,
geoserver: str = GEO_URL,
) -> str:
"""Return WFS GetFeature URL request filtering geographic features based on a property's value.
Parameters
----------
attribute : str
Attribute/field name.
value: Union[str, float, int]
Value for attribute queried.
layer : str
Name of geographic layer queried.
geoserver: str
The address of the geoserver housing the layer to be queried. Default: https://pavics.ouranos.ca/geoserver/.
Returns
-------
str
WFS request URL.
"""
try:
attribute = str(attribute)
value = str(value)
except ValueError:
raise Exception("Unable to cast attribute/filter to string.")
filter_request = PropertyIsLike(propertyname=attribute,
literal=value,
wildCard="*")
filterxml = etree.tostring(filter_request.toXML()).decode("utf-8")
params = dict(
service="WFS",
version="1.1.0",
request="GetFeature",
typename=layer,
outputFormat="application/json",
filter=filterxml,
)
return Request("GET", url=urljoin(geoserver, "wfs"),
params=params).prepare().url
def __convert_query_params(self, product_type_def, product_type, params):
"""Translates eodag search to CSW constraints using owslib constraint classes"""
constraints = []
# How the match should be performed (fuzzy, prefix, postfix or exact).
# defaults to fuzzy
pt_tag, matching = (
product_type_def["name"],
product_type_def.get("matching", "fuzzy"),
)
if matching == "prefix":
constraints.append(
PropertyIsLike(pt_tag, "{}%".format(product_type)))
elif matching == "postfix":
constraints.append(
PropertyIsLike(pt_tag, "%{}".format(product_type)))
elif matching == "exact":
constraints.append(PropertyIsEqualTo(pt_tag, product_type))
else: # unknown matching is considered to be equal to 'fuzzy'
constraints.append(
PropertyIsLike(pt_tag, "%{}%".format(product_type)))
# footprint
fp = params.get("geometry")
if fp:
constraints.append(
BBox([fp["lonmin"], fp["latmin"], fp["lonmax"], fp["latmax"]]))
# dates
start, end = (
params.get("startTimeFromAscendingNode"),
params.get("completionTimeFromAscendingNode"),
)
if start and "date_tags" in self.config.search_definition:
constraints.append(
PropertyIsGreaterThanOrEqualTo(
self.config.search_definition["date_tags"]["start"],
start))
if end and "date_tags" in self.config.search_definition:
constraints.append(
PropertyIsLessThanOrEqualTo(
self.config.search_definition["date_tags"]["end"], end))
return [constraints] if len(constraints) > 1 else constraints
def get_filter_fes(filters, logical_operator=And):
"""Create filter encoding specification (OGC FES) object based on given
filters
:param Tupple.<dict> filters: tupple of filters
:param logical_operator: owslib.fes.And or owslib.fes.Or
:return: filter encoding specification
:rtype: owslib.fes.AND
"""
conditions = []
filter_request = None
for myfilter in filters:
value = myfilter['value']
if myfilter['operator'] == '=':
conditions.append(PropertyIsEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == '!=':
conditions.append(
PropertyIsNotEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == '~':
conditions.append(PropertyIsLike(myfilter['attribute'], value))
elif myfilter['operator'] == '>':
conditions.append(
PropertyIsGreaterThan(myfilter['attribute'], value))
elif myfilter['operator'] == '>=':
conditions.append(
PropertyIsGreaterThanOrEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == '<':
conditions.append(PropertyIsLessThan(myfilter['attribute'], value))
elif myfilter['operator'] == '<=':
conditions.append(
PropertyIsLessThanOrEqualTo(myfilter['attribute'], value))
elif myfilter['operator'] == 'BETWEEN':
conditions.append(
PropertyIsBetween(myfilter['attribute'], *value.split(',')))
elif myfilter['operator'] == 'BBOX':
bbox_filter = BBox(myfilter['value'], 'EPSG:3857')
conditions.append(bbox_filter)
elif myfilter['operator'] == 'IN':
new_filters = [{
'value': value,
'operator': '=',
'attribute': myfilter['attribute']}\
for value in value.split(',')]
conditions.append(get_filter_fes(new_filters, logical_operator=Or))
if len(conditions) > 1:
filter_request = logical_operator(conditions)
else:
filter_request = conditions[0]
return filter_request
def test_num_records():
publs_by_type = uuid.check_redis_consistency()
num_publications = sum([len(publs) for publs in publs_by_type.values()])
csw = common_util.create_csw()
assert csw is not None, f"{settings.CSW_URL}, {settings.CSW_BASIC_AUTHN}"
from owslib.fes import PropertyIsLike
any_query = PropertyIsLike('apiso:Identifier', '*', wildCard='*')
csw.getrecords2(constraints=[any_query], maxrecords=100, outputschema="http://www.isotc211.org/2005/gmd")
assert csw.exceptionreport is None
url_part = f"://{settings.LAYMAN_PROXY_SERVER_NAME}/rest/"
records = {
k: r for k, r in csw.records.items()
if any((url_part in u for u in [ol.url for ol in r.distribution.online]))
}
import json
assert len(
records) == num_publications, f"md_record_ids={json.dumps(list(records.keys()), indent=5)}\npubls={json.dumps(publs_by_type, indent=2)}"
def test_csw_outputschema_fgdc(self):
"""Verify that GeoNode CSW can handle ISO metadata with FGDC outputSchema"""
csw = get_catalogue()
if csw.catalogue.type in {'pycsw_http', 'pycsw_local'}:
# get all ISO records in FGDC schema
dataset_query_like = PropertyIsLike('csw:AnyText', '%san_andres_y_providencia_location%')
csw.catalogue.getrecords2(
typenames='gmd:MD_Metadata',
constraints=[dataset_query_like],
outputschema='http://www.opengis.net/cat/csw/csdgm')
record = list(csw.catalogue.records.values())[0]
# test that the ISO title maps correctly in FGDC
self.assertTrue(record.idinfo.citation.citeinfo['title'] in "san_andres_y_providencia_location.shp")
# test that the ISO abstract maps correctly in FGDC
self.assertEqual(record.idinfo.descript.abstract, 'No abstract provided')
def test_csw_outputschema_iso(self):
"""Verify that GeoNode CSW can handle ISO metadata with ISO outputSchema"""
csw = get_catalogue()
# search for 'san_andres_y_providencia_location', output as Dublin Core
dataset_query_like = PropertyIsLike('csw:AnyText', '%san_andres_y_providencia_location%')
csw.catalogue.getrecords2(
typenames='gmd:MD_Metadata',
constraints=[dataset_query_like],
maxrecords=20,
outputschema='http://www.isotc211.org/2005/gmd',
esn='full')
record = list(csw.catalogue.records.values())[0]
# test that the ISO title maps correctly in Dublin Core
self.assertTrue(record.identification.title in "san_andres_y_providencia_location.shp")
# test that the ISO abstract maps correctly in Dublin Core
self.assertEqual(record.identification.abstract, 'No abstract provided')
# test BBOX properties in Dublin Core
from decimal import Decimal
self.assertAlmostEqual(Decimal(record.identification.bbox.minx), Decimal('-81.8593555'), places=3)
self.assertAlmostEqual(Decimal(record.identification.bbox.miny), Decimal('12.1665322'), places=3)
self.assertAlmostEqual(Decimal(record.identification.bbox.maxx), Decimal('-81.356409'), places=3)
self.assertAlmostEqual(Decimal(record.identification.bbox.maxy), Decimal('13.396306'), places=3)
# test for correct link articulation
for link in record.distribution.online:
if check_ogc_backend(geoserver.BACKEND_PACKAGE):
if link.protocol == 'OGC:WMS':
self.assertEqual(
link.url,
f'{settings.GEOSERVER_PUBLIC_LOCATION}ows',
'Expected a specific OGC:WMS URL')
elif link.protocol == 'OGC:WFS':
self.assertEqual(
link.url,
f'{settings.GEOSERVER_PUBLIC_LOCATION}ows',
'Expected a specific OGC:WFS URL')
def search(self, keywords, startposition, maxrecords, bbox):
"""CSW search wrapper"""
formats = []
for f in self.formats:
formats.append(METADATA_FORMATS[f][0])
dataset_query_like = []
if keywords:
for _kw in keywords:
dataset_query_like.append(PropertyIsLike('csw:AnyText', _kw))
bbox_query = []
if bbox:
bbox_query = BBox(bbox)
return self.getrecords2(
typenames=' '.join(formats),
constraints=dataset_query_like + bbox_query,
startposition=startposition,
maxrecords=maxrecords,
outputschema='http://www.isotc211.org/2005/gmd',
esn='full')
def get_filepaths(self, collection_id: str, spatial_extent: List[float],
temporal_extent: List[str]) -> List[str]:
"""Retrieve a file list from the a CSW server according to the specified parameters.
Arguments:
collecion_id {str} -- identifier of the collection
spatial_extent {List[float]} -- bounding box [ymin, xmin, ymax, xmax]
temporal_extent {List[str]} -- e.g. ["2018-06-04", "2018-06-23"]
Returns:
list -- list of filepaths
"""
csw = CatalogueServiceWeb(self.csw_server_uri, timeout=300)
constraints = []
constraints.append(PropertyIsLike(self.group_property, collection_id))
# Spatial filter
constraints.append(BBox(spatial_extent))
# Temporal filter
constraints.append(
PropertyIsGreaterThan('apiso:TempExtent_begin',
temporal_extent[0]))
constraints.append(
PropertyIsLessThan('apiso:TempExtent_end', temporal_extent[1]))
# Run the query
csw.getrecords2(constraints=[constraints], maxrecords=100)
# Put found records in a variable (dictionary)
records0 = csw.records
# Sort records
records = []
for record in records0:
records.append(records0[record].references[0]['url'])
records = sorted(records)
return records
def test_csw_outputschema_dc_bbox(self):
"""Verify that GeoNode CSW can handle ISO metadata BBOX model with Dublin Core outputSchema"""
csw = get_catalogue()
# search for 'san_andres_y_providencia_location', output as DublinCore
dataset_query_like = PropertyIsLike('csw:AnyText', '%san_andres_y_providencia_location%')
csw.catalogue.getrecords2(
typenames='gmd:MD_Metadata',
constraints=[dataset_query_like],
outputschema='http://www.opengis.net/cat/csw/2.0.2',
esn='full')
record = list(csw.catalogue.records.values())[0]
# test CRS constructs in Dublin Core
self.assertEqual(record.bbox.crs.code, 4326)
# test BBOX properties in Dublin Core
from decimal import Decimal
logger.debug([Decimal(record.bbox.minx), Decimal(record.bbox.miny),
Decimal(record.bbox.maxx), Decimal(record.bbox.maxy)])
self.assertAlmostEqual(Decimal(record.bbox.minx), Decimal('-81.859356'), places=3)
self.assertAlmostEqual(Decimal(record.bbox.miny), Decimal('12.166532'), places=3)
self.assertAlmostEqual(Decimal(record.bbox.maxx), Decimal('-81.356409'), places=3)
self.assertAlmostEqual(Decimal(record.bbox.maxy), Decimal('13.396306'), places=3)
def search(text, catalog_set=None):
'''
Search for 'text' across a given set of catalogs (or the default set)
text - Free text to search for in each catalog
catalog_set - Set of catalogs to use.
Defaults to the user's default catalog set if not provided or None
User's default catalog set can be configured with `set_default_catalogs` function.
'''
if not catalog_set:
catalog_set = _default_catalog_set
from .catalogs import neii, ga
from .results import CSWQueryResult
from owslib.fes import PropertyIsLike
q = PropertyIsLike('csw:AnyText', text)
catalogs = [c() for c in catalog_set]
for cat in catalogs:
cat.getrecords2([q], esn='full')
results = {k: v for cat in catalogs for k, v in cat.records.items()}
logger.info('Found %d results for search "%s"' % (len(results), text))
return CSWQueryResult(results, text)
def search(self, text, bbox=None, start=0):
constraints = []
try:
csw = CatalogueServiceWeb(self.url)
except:
return {
'error':
'Error in establishing connection to server ' + self.url
}
query = PropertyIsLike('csw:AnyText', '%' + text + '%')
constraints.append(query)
if bbox != None:
bbox_query = BBox(bbox)
constraints.append(bbox_query)
# execute search
# esn = elementsetname (full, summary, brief)
# other options may not work with all csw instances!
csw.getrecords2(constraints=constraints,
startposition=start,
maxrecords=10,
typenames='gmd:MD_Metadata',
outputschema='http://www.isotc211.org/2005/gmd',
esn='full')
totalCount = csw.results['matches']
returned = csw.results['returned']
results = []
error = []
for identifier in csw.records:
record = csw.records[identifier]
try:
ogc_link = ogc_type = ogc_layer = ''
for online in record.distribution.online:
# only allow WMS, WFS and SOS services for visualization
if online.protocol == None:
continue
if 'TiledMapService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_type = 'TMS'
break
elif 'WebMapService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_layer = online.name
ogc_type = 'WMS'
break
#elif 'WebMapTileService' in online.protocol:
# ogc_link = online.url.replace('http://earthcare.ads.uni-jena.de:8080/geoserver', 'http://artemis.geogr.uni-jena.de/geoserver')
# ogc_layer = online.name
# ogc_type = 'WMTS'
# break
elif 'WebFeatureService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_layer = online.name
ogc_type = 'WFS'
break
elif 'SensorObservationService' in online.protocol:
ogc_link = online.url.replace(
'http://earthcare.ads.uni-jena.de:8080/geoserver',
'http://artemis.geogr.uni-jena.de/geoserver')
ogc_layer = online.name
ogc_type = 'SOS'
break
# check spatial resolution info
distance = ''
if len(record.identification.distance) > 0:
distance = record.identification.distance[0]
# convert OWSlib metadata object to django layers metadata object structure
results.append({
'identifier':
record.identifier,
'title':
record.identification.title,
'abstract':
record.identification.abstract,
'ogc_link':
ogc_link,
'ogc_layer':
ogc_layer,
'ogc_type':
ogc_type,
'topicCategory':
', '.join(record.identification.topiccategory),
'dataset_contact_new': {
'first_name': '',
'last_name': record.identification.contact[0].name,
'position': record.identification.contact[0].position,
'address': record.identification.contact[0].address,
'postcode': record.identification.contact[0].postcode,
'city': record.identification.contact[0].city,
'country': record.identification.contact[0].country,
'state': record.identification.contact[0].region,
'email': record.identification.contact[0].email,
'organisation':
record.identification.contact[0].organization,
'telephone': record.identification.contact[0].phone,
'fax': record.identification.contact[0].fax,
'mobile': '',
'website':
'', #record.identification.contact[0].onlineresource.url,
'role': record.identification.contact[0].role
},
'meta_contact': {
'first_name': '',
'last_name': record.contact[0].name,
'position': record.contact[0].position,
'address': record.contact[0].address,
'postcode': record.contact[0].postcode,
'city': record.contact[0].city,
'country': record.contact[0].country,
'state': record.contact[0].region,
'email': record.contact[0].email,
'organisation': record.contact[0].organization,
'telephone': record.contact[0].phone,
'fax': record.contact[0].fax,
'mobile': '',
'website': '', #record.contact[0].onlineresource.url,
'role': record.contact[0].role
},
'date_create':
record.identification.date[0].date,
'date_type':
record.identification.date[0].type,
'language':
record.identification.resourcelanguage,
'characterset':
'',
'format':
record.distribution.format,
'west':
float(record.identification.bbox.minx),
'east':
float(record.identification.bbox.maxx),
'north':
float(record.identification.bbox.maxy),
'south':
float(record.identification.bbox.miny),
'alternatetitle':
record.identification.alternatetitle,
'geo_description':
'',
'representation_type':
'',
'equi_scale':
distance,
#'epsg': record.referencesystem.code,
'meta_language':
record.language,
'meta_characterset':
record.charset,
'meta_date':
record.datestamp
})
except Exception as e:
error.append(str(e))
return {
'records': results,
'totalCount': totalCount,
'count': returned,
'error': error
}
from owslib.fes import PropertyIsEqualTo, PropertyIsLike, BBox
birds_query = PropertyIsEqualTo('csw:AnyText', 'birds')
csw.getrecords2(constraints=[birds_query], maxrecords=20)
csw.results
for rec in csw.records:
print(csw.records[rec].title)
# Search for bird data in Canada
bbox_query = BBox([-141, 42, -52, 84])
csw.getrecords2(constraints=[birds_query, bbox_query])
csw.results
# Search for keywords like ‘birds’ or ‘fowl’
birds_query_like = PropertyIsLike('dc:subject', '%birds%')
fowl_query_like = PropertyIsLike('dc:subject', '%fowl%')
csw.getrecords2(constraints=[birds_query_like, fowl_query_like])
csw.results
# Search for a specific record:
csw.getrecordbyid(id=['9250AA67-F3AC-6C12-0CB9-0662231AA181'])
c.records['9250AA67-F3AC-6C12-0CB9-0662231AA181'].title
# Search with a CQL query
csw.getrecords(cql='csw:AnyText like "%birds%"')
csw.transaction(ttype='insert',
typename='gmd:MD_Metadata',
record=open("file.xml").read())
print(cenia.records[rec].title)
from owslib.fes import PropertyIsLike, BBox, And, PropertyIsEqualTo
wms_query = PropertyIsEqualTo('csw:AnyText', 'WMS')
praha_query = BBox([14.22, 49.94, 14.71, 50.18])
praha_and_wms = And([wms_query, praha_query])
cenia.getrecords2([praha_and_wms], esn='full')
print(cenia.results)
for recid in cenia.records:
record = cenia.records[recid]
print('{}: {} {} {} {}'.format(record.title,
record.bbox.minx, record.bbox.miny,
record.bbox.maxx, record.bbox.maxy))
zm_query = PropertyIsLike('csw:AnyText', '%WMS - ZM 10%')
cenia.getrecords2([zm_query], esn='full')
zm10 = cenia.records['CZ-CUZK-WMS-ZM10-P']
print(zm10.type)
print('{}\n{}'.format(zm10.title, zm10.abstract))
url = zm10.references[0]['url']
wfs_query = PropertyIsLike('csw:AnyText', 'WFS%')
aopk_query = PropertyIsLike('csw:AnyText', 'Agentura%')
service_query = PropertyIsLike('apiso:type', 'service')
aopk_and_wfs = And([aopk_query, wfs_query, service_query])
cenia.getrecords2([aopk_and_wfs], esn='full')
print(cenia.results)
cenia.getrecords2([praha_and_wms], esn='full')
print (cenia.results)
for recid in cenia.records:
record = cenia.records[recid]
print (u'{}: {} {} {} {}'.format(record.title, record.bbox.minx, record.bbox.miny,
record.bbox.maxx, record.bbox.maxy))
zm_query = PropertyIsEqualTo('csw:AnyText', 'ZM10')
cenia.getrecords2([zm_query], esn='full')
zm10 = cenia.records['CZ-CUZK-WMS-ZM10-P']
print (zm10.type)
print (u'{}\n{}'.format(zm10.title, zm10.abstract))
url = zm10.references[0]['url']
wfs_query = PropertyIsLike('csw:AnyText', 'WFS')
aopk_query = PropertyIsLike('csw:AnyText', 'AOPK')
service_query = PropertyIsLike('apiso:type', 'service')
aopk_and_wfs = And([aopk_query, wfs_query, service_query])
cenia.getrecords2([aopk_and_wfs], esn='full')
print (cenia.results)
for recid in cenia.records:
record = cenia.records[recid]
print (u'{}: {}'.format(recid, record.title))
natura = cenia.records['53e37222-89a0-472b-9781-5bfc0a02080a']
print (u'{}\n{}'.format(natura.abstract, natura.identifiers[1]['identifier']))
def search(self):
"""execute search"""
self.catalog = None
self.constraints = []
# clear all fields and disable buttons
self.lblResults.clear()
self.treeRecords.clear()
self.reset_buttons()
# save some settings
self.settings.setValue('/MetaSearch/returnRecords',
self.spnRecords.cleanText())
# set current catalog
current_text = self.cmbConnectionsSearch.currentText()
key = '/MetaSearch/%s' % current_text
self.catalog_url = self.settings.value('%s/url' % key)
self.catalog_username = self.settings.value('%s/username' % key)
self.catalog_password = self.settings.value('%s/password' % key)
# start position and number of records to return
self.startfrom = 0
self.maxrecords = self.spnRecords.value()
# set timeout
self.timeout = self.spnTimeout.value()
# bbox
# CRS is WGS84 with axis order longitude, latitude
# defined by 'urn:ogc:def:crs:OGC:1.3:CRS84'
minx = self.leWest.text()
miny = self.leSouth.text()
maxx = self.leEast.text()
maxy = self.leNorth.text()
bbox = [minx, miny, maxx, maxy]
# only apply spatial filter if bbox is not global
# even for a global bbox, if a spatial filter is applied, then
# the CSW server will skip records without a bbox
if bbox != ['-180', '-90', '180', '90']:
self.constraints.append(
BBox(bbox, crs='urn:ogc:def:crs:OGC:1.3:CRS84'))
# keywords
if self.leKeywords.text():
# TODO: handle multiple word searches
keywords = self.leKeywords.text()
self.constraints.append(PropertyIsLike('csw:AnyText', keywords))
if len(self.constraints) > 1: # exclusive search (a && b)
self.constraints = [self.constraints]
# build request
if not self._get_csw():
return
# TODO: allow users to select resources types
# to find ('service', 'dataset', etc.)
try:
with OverrideCursor(Qt.WaitCursor):
self.catalog.getrecords2(constraints=self.constraints,
maxrecords=self.maxrecords,
esn='full')
except ExceptionReport as err:
QMessageBox.warning(self, self.tr('Search error'),
self.tr('Search error: {0}').format(err))
return
except Exception as err:
QMessageBox.warning(self, self.tr('Connection error'),
self.tr('Connection error: {0}').format(err))
return
if self.catalog.results['matches'] == 0:
self.lblResults.setText(self.tr('0 results'))
return
self.display_results()
def get_hydrobasins_attributes_wfs(attribute=None,
value=None,
level=12,
lakes=True):
"""Return features from the USGS HydroBASINS data set using attribute value selection and WFS 1.1.0 protocol.
For geographic rasters, subsetting is based on WGS84 (Long, Lat) boundaries. If not geographic, subsetting based
on projected coordinate system (Easting, Northing) boundaries.
Parameters
----------
attribute : str
Attribute/field to be queried.
value: str or float or int
Value for attribute queried.
level : int
Level of granularity requested for the lakes vector (1:12). Default: 12.
lakes : bool
Whether or not the vector should include the delimitation of lakes.
Returns
-------
str
URL to the GeoJSON-encoded WFS response.
"""
from requests import Request
from owslib.fes import PropertyIsLike
from lxml import etree
url = 'http://boreas.ouranos.ca/geoserver/wfs'
layer = 'public:USGS_HydroBASINS_{}na_lev{}'.format(
'lake_' if lakes else '', level)
if attribute is not None and value is not None:
try:
attribute = str(attribute)
value = str(value)
except ValueError:
raise Exception('Unable to cast attribute/filter to string')
try:
filter_request = PropertyIsLike(propertyname=attribute,
literal=value,
wildCard='*')
filterxml = etree.tostring(filter_request.toXML()).decode('utf-8')
params = dict(service='WFS',
version='1.1.0',
request='GetFeature',
typename=layer,
outputFormat='json',
filter=filterxml)
q = Request('GET', url, params=params).prepare().url
except Exception as e:
raise Exception(e)
else:
raise NotImplementedError
return q
def test_csw(self):
# test 2.0.2 Basic Service Profile
self.csw = CatalogueServiceWeb(self.url,
version='2.0.2',
username=self.username,
password=self.password)
self.assertEqual(self.csw.version, '2.0.2')
self.assertIn('2.0.2', self.csw.parameters['version'].values)
self.assertIn('3.0.0', self.csw.parameters['version'].values)
for op in self.csw.operations:
for method in op.methods:
self.assertEqual(self.csw.url, method['url'])
self.assertTrue('Transaction' in [o.name for o in self.csw.operations])
self.assertTrue('Harvest' in [o.name for o in self.csw.operations])
get_records_op = self.csw.get_operation_by_name('GetRecords')
self.assertIn('application/json',
get_records_op.parameters['outputFormat']['values'])
# test basic search, no predicates
self.csw.getrecords2()
self.assertEqual(Layer.objects.all().count(),
self.csw.results['matches'])
# test csw:AnyText
anytext = PropertyIsLike('csw:AnyText', 'Brasilia')
self.csw.getrecords2(constraints=[anytext])
self.assertEqual(self.csw.results['matches'], 1)
anytext = PropertyIsLike('csw:AnyText', 'roads')
self.csw.getrecords2(constraints=[anytext])
self.assertEqual(self.csw.results['matches'], 4)
# test ogc:BBOX
bbox = BBox(['-13', '-80', '15', '-30'])
self.csw.getrecords2(constraints=[bbox])
self.assertEqual(self.csw.results['matches'], 2)
# test csw:AnyText OR ogc:BBOX
self.csw.getrecords2(constraints=[anytext, bbox])
self.assertEqual(self.csw.results['matches'], 5)
# test csw:AnyText && ogc:BBOX
self.csw.getrecords2(constraints=[[anytext, bbox]])
self.assertEqual(self.csw.results['matches'], 1)
# test that ElementSetName=full stores full metadata record as inserted
self.csw.getrecords2(esn='full')
self.assertIn(
'xmlns:registry="http://gis.harvard.edu/HHypermap/registry/0.1"',
self.csw.response)
# test JSON output
# TODO: fix owslib.csw.CatalogueServiceWeb.getrecords2 to handle non-XML request/response
with self.assertRaises(XMLSyntaxError):
self.csw.getrecords2(constraints=[anytext, bbox],
format='application/json')
records_json = json.loads(self.csw.response)
self.assertEqual(
records_json['csw:GetRecordsResponse']['csw:SearchResults']
['@numberOfRecordsMatched'], '5')
# test 3.0.0 OpenSearch
bsp = {
'mode': 'opensearch',
'service': 'CSW',
'version': '3.0.0',
'request': 'GetRecords',
'typenames': 'csw:Record',
'elementsetname': 'full',
'outputformat': 'application/json'
}
# test basic search, no predicates
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '10')
# test q
bsp['q'] = 'Brasilia'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '1')
bsp.pop('q')
# test bbox
bsp['bbox'] = '-80,-13,-30,15'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '2')
bsp.pop('bbox')
# test time
bsp['time'] = '2014-09-23T12:04:31.102243+00:00/'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '10')
bsp.pop('time')
# test q and bbox
bsp['q'] = 'roads'
bsp['bbox'] = '-80,-13,-30,15'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '1')
# test q and bbox and time
bsp['time'] = '2014-09-23T12:04:31.102243+00:00/'
res = json.loads(self.client.get(self.script_name, bsp).content)
self.assertEqual(res['atom:feed']['os:totalResults'], '1')
if echo:
print json.dumps(
json_out,
sort_keys=True,
indent=4,
separators=(',', ': '),
ensure_ascii=False,
)
return json_out
filter1 = PropertyIsEqualTo(propertyname="type", literal=u"hotel")
filter2 = PropertyIsLike(propertyname="name",
literal=u"*Пет*",
wildCard="*",
singleChar=".",
escapeChar="!")
filter3 = PropertyIsLike(propertyname="name",
literal=u"Бал*",
wildCard="*",
singleChar=".",
escapeChar="!")
filterxml = u"<Filter><OR><AND>{0}{1}</AND><AND>{0}{2}</AND></OR></Filter>".format(
etree.tostring(filter1.toXML()).decode("utf-8"),
etree.tostring(filter2.toXML()).decode("utf-8"),
etree.tostring(filter3.toXML()).decode("utf-8"),
)
#out = wfs.getfeature(
#typename='buildings',