def make_bbox(self):
'''
Helper function to make the filter bounding box
'''
crs = 'urn:ogc:def:crs:OGC:1.3:CRS84'
self.bbox_crs = fes.BBox(
[self.roi[0], self.roi[2], self.roi[1], self.roi[3]], crs=crs)
def make_filter(config, ):
kw = {
"wildCard": "*",
"escapeChar": "\\",
"singleChar": "?",
"propertyname": "apiso:Subject",
}
if len(config["cf_names"]) > 1:
or_filt = fes.Or([
fes.PropertyIsLike(literal=("*%s*" % val), **kw)
for val in config["cf_names"]
])
else:
or_filt = fes.PropertyIsLike(literal=("*%s*" % config["cf_names"][0]),
**kw)
kw.update({"propertyname": "apiso:AnyText"})
not_filt = fes.Not([fes.PropertyIsLike(literal="*cdip*", **kw)])
begin, end = fes_date_filter(config["date"]["start"],
config["date"]["stop"])
bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"])
filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])]
return filter_list
def test_csw_query_bbox(self):
"""Verify that GeoNode CSW can handle bbox queries"""
csw = get_catalogue()
bbox = fes.BBox([-140, -70, 80, 70])
try:
csw.catalogue.getrecords2([bbox, ])
logger.debug(csw.catalogue.results)
self.assertEqual(csw.catalogue.results, {'matches': 7, 'nextrecord': 0, 'returned': 7})
except Exception:
# This test seems to borken actually on pycsw
pass
def csw_query(endpoint,
bbox=None,
start=None,
stop=None,
kw_names=None,
crs="urn:ogc:def:crs:OGC:1.3:CRS84"):
crs = "urn:ogc:def:crs:::EPSG:4326" #https://github.com/qgis/QGIS/issues/40778
constraints = []
csw = None
while csw is None:
try:
csw = CatalogueServiceWeb(endpoint, timeout=60)
#csw.getrecords2(maxrecords=10)
#for rec in csw.records:
# print(vars(csw.records[rec]))
# print(csw.records[rec].title)
except:
pass
if kw_names:
kw = dict(wildCard="*",
escapeChar="\\",
singleChar="?",
propertyname="apiso:AnyText")
or_filt = fes.Or([
fes.PropertyIsLike(literal=("*%s*" % val), **kw)
for val in kw_names
])
constraints.append(or_filt)
if all(v is not None for v in [start, stop]):
begin, end = fes_date_filter(start, stop)
constraints.append(begin)
constraints.append(end)
if bbox:
bbox_crs = fes.BBox(bbox, crs=crs)
constraints.append(bbox_crs)
if len(constraints) >= 2:
filter_list = [fes.And(constraints)]
else:
filter_list = constraints
get_csw_records(csw, filter_list, pagesize=10, maxrecords=10)
print("Found {} records.\n".format(len(csw.records.keys())))
for key, value in list(csw.records.items()):
print(u"Title: [{}]\nID: {}\n".format(value.title, key))
msg = "geolink: {geolink}\nscheme: {scheme}\nURL: {url}\n".format
for ref in value.references:
print(msg(geolink=sniff_link(ref["url"]), **ref))
print("#########################################################",
'\n')
def test_bbox(endpoints, bbox):
for title, url in endpoints.iteritems():
try:
csw = CatalogueServiceWeb(url, timeout=40)
if "BBOX" in csw.filters.spatial_operators:
filter_list = [fes.BBox(bbox)]
try:
csw.getrecords2(constraints=filter_list, maxrecords=1000)
print("%s : Datasets = %d" %
(title, len(csw.records.keys())))
except Exception:
print "%s : BBOX Query FAILS" % title
else:
print "%s - BBOX Query NOT supported" % title
except Exception:
print "%s - Timed out" % title
def make_filter(config):
from owslib import fes
from ioos_tools.ioos import fes_date_filter
kw = dict(
wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText"
)
or_filt = fes.Or(
[fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"]]
)
not_filt = fes.Not([fes.PropertyIsLike(literal="GRIB-2", **kw)])
begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"])
bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"])
filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])]
return filter_list
# <markdowncell>
# ### Can we discover other datasets in this polygon area?
# <markdowncell>
# ##### Setup BCSW Filters to find models in the area of the Important Bird Polygon
# <codecell>
from owslib import fes
# Polygon filters
polygon_filters = []
for s in shapes:
f = fes.BBox(bbox=list(reversed(s.bounds)))
polygon_filters.append(f)
# If we have more than one polygon filter, OR them together
if len(polygon_filters) > 1:
polygon_filters = fes.Or(polygon_filters)
elif len(polygon_filters) == 1:
polygon_filters = polygon_filters[0]
# Name filters
name_filters = []
model_strings = [
'roms', 'selfe', 'adcirc', 'ncom', 'hycom', 'fvcom', 'wrf', 'wrams'
]
for model in model_strings:
title_filter = fes.PropertyIsLike(propertyname='apiso:Title',
literal='*%s*' % model,
def compose_query(query_string, bbox, wfs_filters):
"""Compose a wfs filter query from a string
The query string should be composed as: "property_name operator
literal". The property names and operators are
initialized with the DovBoringen class.
Multiple filters can be added by comma separation e.g.:
"property_name1 operator1 literal1, property_name2 operator2 literal2"
The PropertyIsBetween operator requires a lower and upper boundary,
it is given by a tuple in the string, e.g.:
"diepte_tot_m << (20,100)"
Parameters
----------
query_string : str
A string containing the query that will be used as constrained in
the WFS call. See also: get_boringen()
bbox : tuple of floats, or empty tuple
The X, Y coordinates of the bounding box as (xmin, ymin, xmax,
ymax)
wfs_filters : dict
A dictionary mapping the operator in the query string to the
comparison operator of the wfs call
Returns
-------
filterxml : str
A string of the xml constraint for a wfs call using owslib
"""
filters = []
# extract criteria
if query_string:
query_raw = [x.strip(' ,') for x in query_string.split(' ')]
if len(query_raw) % 3 != 0:
raise ValueError('The query string is not correct. '
'It should be composed of "property operator '
'literal"')
idx = 1
for fltr in query_raw[1::3]:
if fltr != '<<':
filters.append(wfs_filters[fltr](
propertyname=query_raw[idx - 1],
literal=query_raw[idx + 1]))
else:
lb, ub = [
x.strip(['(', ')'])
for x in query_raw[idx + 1].split(',')
]
filters.append(wfs_filters[fltr](
propertyname=query_raw[idx - 1],
lowerboundary=lb,
upperboundary=ub))
idx += 3
if bbox:
filters.append(fes.BBox(bbox))
if len(filters) == 1:
filter = fes.FilterRequest().setConstraint(filters[0])
elif len(filters) > 1:
# only logical AND is evaluated (constraint = [[a, b]])
filter = fes.FilterRequest().setConstraintList([filters])
else:
return ''
filterxml = fes.etree.tostring(filter, encoding="utf-8", method='xml')
return filterxml
csw.records[choice].references
# In[8]:
csw.request
# In[9]:
csw.records[choice].xml
# Add bounding box constraint. To specify lon,lat order for bbox (which we want to do so that we can use the same bbox with either geoportal server or pycsw requests), we need to request the bounding box specifying the CRS84 coordinate reference system. The CRS84 option is available in `pycsw 1.1.10`+. The ability to specify the `crs` in the bounding box request is available in `owslib 0.8.12`+. For more info on the bounding box problem and how it was solved, see this [pycsw issue](https://github.com/geopython/pycsw/issues/287), this [geoportal server issue](https://github.com/Esri/geoportal-server/issues/124), and this [owslib issue](https://github.com/geopython/OWSLib/issues/201)
# In[10]:
bbox = [-87.40, 34.25, -63.70, 66.70] # [lon_min, lat_min, lon_max, lat_max]
bbox_filter = fes.BBox(bbox, crs='urn:ogc:def:crs:OGC:1.3:CRS84')
filter_list = [fes.And([filter1, bbox_filter])]
csw.getrecords2(constraints=filter_list, maxrecords=1000)
# In[11]:
print(len(csw.records.keys()))
for rec in list(csw.records.keys()):
print('title:' + csw.records[rec].title)
print('identifier:' + csw.records[rec].identifier)
print('modified:' + csw.records[rec].modified)
print(' ')
# In[12]:
val = 'WMS'
constraint='overlaps'):
if constraint == 'overlaps':
start = fes.PropertyIsLessThanOrEqualTo(
propertyname='apiso:TempExtent_begin', literal=stop_date)
stop = fes.PropertyIsGreaterThanOrEqualTo(
propertyname='apiso:TempExtent_end', literal=start_date)
elif constraint == 'within':
start = fes.PropertyIsGreaterThanOrEqualTo(
propertyname='apiso:TempExtent_begin', literal=start_date)
stop = fes.PropertyIsLessThanOrEqualTo(
propertyname='apiso:TempExtent_end', literal=stop_date)
return start, stop
#Establish bounding box filter for Geographic Range of IBAs
bbox = fes.BBox([-130.5, 47.9, 167.6, 74.7])
# <codecell>
sparql = SPARQLWrapper("http://mmisw.org/sparql")
queryString = """
PREFIX ioos: <http://mmisw.org/ont/ioos/parameter/>
SELECT DISTINCT ?parameter ?definition ?unit ?property ?value
WHERE {?parameter a ioos:Parameter .
?parameter ?property ?value .
?parameter ioos:Term ?term .
?parameter ioos:Definition ?definition .
?parameter ioos:Units ?unit .
FILTER (regex(str(?property), "(exactMatch|closeMatch)", "i") && regex(str(?value), "temperature", "i") )
}
ORDER BY ?parameter
'currents', 'surface_eastward_sea_water_velocity',
'*surface_eastward_sea_water_velocity*'
],
"sos_name": ['currents']
}
# <codecell>
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal
csw = CatalogueServiceWeb(endpoint, timeout=60)
# <codecell>
# convert User Input into FES filters
start, stop = date_range(start_date, stop_date)
bbox = fes.BBox(bounding_box)
#use the search name to create search filter
or_filt = fes.Or([
fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?')
for val in data_dict['currents']['names']
])
val = 'Averages'
not_filt = fes.Not([
fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
def get_data(self, typename, **kwargs):
"""
Download WOUDC observations
:param bbox: a list representing a bounding box spatial
filter (`minx, miny, maxx, maxy`)
:param temporal: a list of two elements representing a time period
(start, end) which accepts the following types:
- :py:class:`datetime.date`
- :py:class:`datetime.datetime`
- string date (e.g. ``2012-10-30``)
- string datetime (e.g. ``2012-10-30 11:11:11``)
:param filters: `dict` of key-value pairs of property names and
values. Constructs exclusive search
:param variables: a list of variables to return
as part of the response (default returns all)
:param sort_property: a string representing the property on which
to sort results (default ``instance_datetime``)
:param sort_order: a string representing sort order of response
(``asc`` or ``desc``). Default is ``asc``.
Applied if `sort_property` is specified
:returns: list of WOUDC observations GeoJSON payload
"""
constraints = []
filters = []
variables = '*'
filter_string = None
bbox = None
temporal = None
sort_property = None
sort_order = 'asc'
startindex = 0
features = None
feature_collection = None
sort_descending = False
LOGGER.info('Downloading dataset %s', typename)
LOGGER.debug('Assembling query parameters')
for key, value in kwargs.items():
if key == 'bbox':
bbox = value
if key == 'temporal':
temporal = value
if key == 'filters':
filters = value
if key == 'variables':
variables = value
if key == 'sortby':
sort_property = value
if key == 'sort_order':
sort_order = value
LOGGER.debug('Assembling constraints')
if filters:
for key, value in filters.items():
constraints.append(fes.PropertyIsEqualTo(key, value))
if bbox is not None:
if not isinstance(bbox, list) or len(bbox) != 4:
raise ValueError('bbox must be list of minx, miny, maxx, maxy')
LOGGER.debug('Setting spatial constraint')
constraints.append(fes.BBox(bbox))
if temporal is not None:
if not isinstance(temporal, list) or len(temporal) != 2:
msg = 'temporal must be list of start date, end date'
raise ValueError(msg)
LOGGER.info('Setting temporal constraint')
temporal_start = date2string(temporal[0], 'begin')
temporal_end = date2string(temporal[1], 'end')
constraints.append(fes.PropertyIsBetween(
'instance_datetime', temporal_start, temporal_end))
if sort_order not in ['asc', 'desc']:
raise ValueError('sort_order must be asc or desc')
else:
if sort_order == 'desc':
sort_descending = True
if variables != '*':
if not isinstance(variables, list):
raise ValueError('variables must be list')
if constraints:
LOGGER.debug('Combining constraints')
flt = fes.FilterRequest()
if len(constraints) == 1:
LOGGER.debug('Single constraint')
filter_string = flt.setConstraint(constraints[0],
tostring=True)
if len(constraints) > 1:
LOGGER.debug('Multiple constraints')
filter_string = flt.setConstraintList([constraints],
tostring=True)
LOGGER.info('Fetching observations')
LOGGER.info('Filters:')
LOGGER.info('bbox: %r', bbox)
LOGGER.info('temporal: %r', temporal)
LOGGER.info('attribute queries: %r', filters)
# page download and assemble single list of JSON features
while True:
LOGGER.debug('Fetching features %d - %d',
startindex, startindex + self.maxfeatures)
payload = self.server.getfeature(
typename=typename,
startindex=startindex,
propertyname=variables,
maxfeatures=self.maxfeatures,
filter=filter_string,
outputFormat=self.outputformat).read()
LOGGER.debug('Processing response')
if payload.isspace():
LOGGER.debug('Empty response. Exiting')
break
try:
features = json.loads(payload)
except ValueError:
msg = 'Query produced no results'
LOGGER.info(msg)
return None
len_features = len(features['features'])
LOGGER.debug('Found %d features', len_features)
if feature_collection is None:
feature_collection = features
else:
feature_collection['features'].extend(features['features'])
if len_features < self.maxfeatures:
break
startindex = startindex + self.maxfeatures
len_feature_collection = len(feature_collection['features'])
LOGGER.info('Found %d total features', len_feature_collection)
if sort_property is not None:
LOGGER.info('Sorting response by %s', sort_property)
feature_collection['features'].sort(
key=lambda e: e['properties'][sort_property],
reverse=sort_descending)
return feature_collection
from owslib import fes
kw = dict(wildCard='*',
escapeChar='\\',
singleChar='?',
propertyname='apiso:AnyText')
or_filt = fes.Or(
[fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in name_list])
# Exculde ROMS Averages and History files.
not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)])
begin, end = fes_date_filter(start, stop)
filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])]
# In[6]:
from owslib.csw import CatalogueServiceWeb
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw'
csw = CatalogueServiceWeb(endpoint, timeout=60)
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
log.info(fmt(' Catalog information '))
log.info("URL: {}".format(endpoint))
log.info("CSW version: {}".format(csw.version))
log.info("Number of datasets available: {}".format(len(csw.records.keys())))
# In[7]:
def query_csw(self,
keyword_list=None,
bounding_box=None,
bounding_box_crs=None,
anytext_list=None,
titleword_list=None,
start_datetime=None,
stop_datetime=None,
max_total_records=None):
'''
Function to query CSW using AND combination of provided search parameters and return generator object
yielding nested dicts containing information about each record including distributions
@param keyword_list: List of strings or comma-separated string containing keyword search terms
@param bounding_box: Bounding box to search as a list of ordinates [bbox.minx, bbox.minx, bbox.maxx, bbox.maxy]
@param bounding_box_crs: Coordinate reference system for bounding box. Defaults to value of CSWUtils.DEFAULT_CRS
@param anytext_list: List of strings or comma-separated string containing any text search terms
@param titleword: List of strings or comma-separated string containing title search terms
@param start_datetime: Datetime object defining start of temporal search period
@param stop_datetime: Datetime object defining end of temporal search period
@param max_total_records: Maximum total number of records to return. Defaults to value of CSWUtils.DEFAULT_MAXTOTALRECORDS
@return: generator object yielding nested dicts containing information about each record including distributions
'''
bounding_box_crs = bounding_box_crs or CSWUtils.DEFAULT_CRS
# Convert strings to lists if required
if type(keyword_list) == str:
keyword_list = self.list_from_comma_separated_string(keyword_list)
if type(anytext_list) == str:
anytext_list = self.list_from_comma_separated_string(anytext_list)
if type(titleword_list) == str:
titleword_list = self.list_from_comma_separated_string(
titleword_list)
# Build filter list
fes_filter_list = []
# Check for unchanged, upper-case, lower-case and capitalised keywords
if keyword_list:
fes_filter_list += self.any_case_match_filters(
'Subject', keyword_list)
if anytext_list:
fes_filter_list += [
fes.PropertyIsLike(propertyname='anyText',
literal=phrase,
matchCase=False) for phrase in anytext_list
]
if start_datetime or stop_datetime:
fes_filter_list += self.get_date_filter(start_datetime,
stop_datetime)
if titleword_list:
fes_filter_list += [
fes.PropertyIsLike(propertyname='title',
literal=titleword,
matchCase=False)
for titleword in titleword_list
]
if bounding_box:
fes_filter_list += [fes.BBox(bounding_box, crs=bounding_box_crs)]
assert fes_filter_list, 'No search criteria defined'
# Use single filter if no "and" required
if len(fes_filter_list) == 1:
fes_filter_list = fes_filter_list[0]
# Return generator object
return self.get_csw_records(fes_filter_list,
max_total_records=max_total_records)
# <codecell>
from owslib import fes
def fes_date_filter(start_date='1900-01-01',stop_date='2100-01-01',constraint='overlaps'):
if constraint == 'overlaps':
start = fes.PropertyIsGreaterThanOrEqualTo(propertyname='apiso:TempExtent_end', literal=start_date)
stop = fes.PropertyIsLessThanOrEqualTo(propertyname='apiso:TempExtent_begin', literal=stop_date)
elif constraint == 'within':
start = fes.PropertyIsGreaterThanOrEqualTo(propertyname='apiso:TempExtent_begin', literal=start_date)
stop = fes.PropertyIsLessThanOrEqualTo(propertyname='apiso:TempExtent_end', literal=stop_date)
return fes.And([start, stop])
# <codecell>
# Geographic filters
geographic_filter = fes.BBox(bbox=bounding_box)
# Temporal filters
temporal_filter = fes_date_filter(start_date_string, end_date_string)
filters = fes.And([geographic_filter, temporal_filter])
# <markdowncell>
# ##### The actual CSW filter POST envelope looks like this
# <codecell>
from owslib.etree import etree
print etree.tostring(filters.toXML(), pretty_print=True)
start_date = jd_start.strftime('%Y-%m-%d %H:00')
stop_date = jd_stop.strftime('%Y-%m-%d %H:00')
jd_start = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
jd_stop = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M')
print start_date,'to',stop_date
sos_name = 'water_surface_height_above_reference_datum'
# <codecell>
# convert User Input into FES filters
start,stop = dateRange(start_date,stop_date)
bbox = fes.BBox(box)
or_filt = fes.Or([fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
escapeChar='\\',wildCard='*',singleChar='?') for val in model_name_list])
val = 'Averages'
not_filt = fes.Not([fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
escapeChar='\\',wildCard='*',singleChar='?')])
filter_list = [fes.And([ bbox, start, stop, or_filt, not_filt]) ]
# <markdowncell>
# ##Find model results at NODC
def query_csw(self,
identifier_list=None,
alt_identifier_list=None,
keyword_list=None,
bounding_box=None,
bounding_box_crs=None,
anytext_list=None,
titleword_list=None,
start_datetime=None,
stop_datetime=None,
record_type_list=None,
max_total_records=None,
get_layers=None):
'''
Function to query CSW using AND combination of provided search parameters and return generator object
yielding nested dicts containing information about each record including distributions
@param identifier_list: List of strings or comma-separated string containing metadata identifiers (UUID)
@param alt_identifier: List of strings or comma-separated string containing metadata alternate identifiers (eCat ID)
@param keyword_list: List of strings or comma-separated string containing keyword search terms
@param bounding_box: Bounding box to search as a list of ordinates [bbox.minx, bbox.minx, bbox.maxx, bbox.maxy]
@param bounding_box_crs: Coordinate reference system for bounding box. Defaults to value of self.settings['DEFAULT_CRS']
@param anytext_list: List of strings or comma-separated string containing any text search terms
@param titleword: List of strings or comma-separated string containing title search terms
@param start_datetime: Datetime object defining start of temporal search period
@param stop_datetime: Datetime object defining end of temporal search period
@param record_type_list: List of strings or comma-separated string containing record type(s) to return
@param max_total_records: Maximum total number of records to return. Defaults to value of self.settings['DEFAULT_MAXTOTALRECORDS']
@param get_layers: Boolean flag indicating whether to get WMS/WCS layer names. Defaults to value of self.settings['DEFAULT_GET_LAYERS']
@return: generator object yielding nested dicts containing information about each record including distributions
'''
bounding_box_crs = bounding_box_crs or self.settings['DEFAULT_CRS']
get_layers = self.settings[
'DEFAULT_GET_LAYERS'] if get_layers is None else get_layers
# Convert strings to lists if required
if type(identifier_list) == str:
identifier_list = self.list_from_comma_separated_string(
identifier_list)
if type(alt_identifier_list) == str:
alt_identifier_list = self.list_from_comma_separated_string(
alt_identifier_list)
if type(keyword_list) == str:
keyword_list = self.list_from_comma_separated_string(keyword_list)
if type(anytext_list) == str:
anytext_list = self.list_from_comma_separated_string(anytext_list)
if type(titleword_list) == str:
titleword_list = self.list_from_comma_separated_string(
titleword_list)
record_type_list = record_type_list or self.settings[
'DEFAULT_RECORD_TYPES']
if type(record_type_list) == str:
record_type_list = self.list_from_comma_separated_string(
record_type_list)
# Build filter list
fes_filter_list = []
if identifier_list:
if len(identifier_list) == 1:
fes_filter_list += [
fes.PropertyIsLike(propertyname='Identifier',
literal=identifier_list[0],
matchCase=False)
]
else:
fes_filter_list.append(
fes.Or([
fes.PropertyIsLike(propertyname='Identifier',
literal=identifier,
matchCase=False)
for identifier in identifier_list
]))
if alt_identifier_list:
if len(alt_identifier_list) == 1:
fes_filter_list += [
fes.PropertyIsLike(propertyname='AlternateIdentifier',
literal=alt_identifier_list[0],
matchCase=False)
]
else:
fes_filter_list.append(
fes.Or([
fes.PropertyIsLike(propertyname='AlternateIdentifier',
literal=alt_identifier,
matchCase=False)
for alt_identifier in alt_identifier_list
]))
# Check for unchanged, upper-case, lower-case and capitalised keywords
# with single-character wildcards substituted for whitespace characters
# GeoNetwork keyword search is always case sensitive
if keyword_list:
fes_filter_list += self.any_case_match_filters(
'Subject', keyword_list)
if anytext_list:
fes_filter_list += [
fes.PropertyIsLike(propertyname='anyText',
literal=phrase,
matchCase=False) for phrase in anytext_list
]
if start_datetime or stop_datetime:
fes_filter_list += self.get_date_filter(start_datetime,
stop_datetime)
if titleword_list:
fes_filter_list += [
fes.PropertyIsLike(propertyname='title',
literal=titleword,
matchCase=False)
for titleword in titleword_list
]
# Check for unchanged, upper-case, lower-case and capitalised keywords
# with single-character wildcards substituted for whitespace characters
# GeoNetwork type search is always case sensitive
if record_type_list:
fes_filter_list += self.any_case_match_filters(
'type', record_type_list)
if bounding_box:
# N.B: Bounding box ordinate ordering must match CRS. Default CRS84 supports lon-lat ordering, not lat-lon
# See https://gis.stackexchange.com/questions/124050/how-do-i-specify-the-lon-lat-ordering-in-csw-bounding-box-request
fes_filter_list += [fes.BBox(bounding_box, crs=bounding_box_crs)]
assert fes_filter_list, 'No search criteria defined'
# Use single filter if no "and" required
if len(fes_filter_list) == 1:
fes_filter_list = fes_filter_list[0]
# Return generator object
return self.get_csw_records(fes_filter_list,
max_total_records=max_total_records,
get_layers=get_layers)
def make_bbox(self):
crs = 'urn:ogc:def:crs:OGC:1.3:CRS84'
self.bbox_crs = fes.BBox([self.roi[0], self.roi[2], self.roi[1], self.roi[3]], crs=crs)
#jd_stop = jd_now + dt.timedelta(days=3)
start_date = jd_start.strftime('%Y-%m-%d %H:00')
stop_date = jd_stop.strftime('%Y-%m-%d %H:00')
jd_start = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M')
jd_stop = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M')
print start_date,'to',stop_date
# <codecell>
start,stop = dateRange(start_date,stop_date)
filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val),
escapeChar='\\',wildCard='*',singleChar='?')
bbox = fes.BBox(box,crs='urn:ogc:def:crs:OGC:1.3:CRS84')
#filter_list = [fes.And([ bbox, filter1, start,stop]) ]
# <codecell>
filter_list = [fes.And([ bbox, filter1]) ]
csw.getrecords2(constraints=filter_list)
csw.results['matches']
# <codecell>
filter_list = [fes.And([ bbox, filter1, start,stop]) ]
csw.getrecords2(constraints=filter_list)
csw.results['matches']
# <codecell>
escapeChar='\\',
singleChar='?',
propertyname='apiso:AnyText')
or_filt = fes.Or(
[fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in model_names])
kw = dict(wildCard='*',
escapeChar='\\',
singleChar='?',
propertyname='apiso:ServiceType')
serviceType = fes.PropertyIsLike(literal=('*%s*' % service_type), **kw)
begin, end = fes_date_filter(start, stop)
bbox_crs = fes.BBox(bbox, crs=crs)
filter_list = [
fes.And([
bbox_crs, # bounding box
begin,
end, # start and end date
or_filt, # or conditions (CF variable names)
serviceType # search only for datasets that have WMS services
])
]
# In[9]:
from owslib.csw import CatalogueServiceWeb
def get_data(self, typename, **kwargs):
"""
Download WOUDC observations
:param bbox: a list representing a bounding box spatial
filter (`minx, miny, maxx, maxy`)
:param temporal: a list of two elements representing a time period
(start, end) which accepts the following types:
- :py:class:`datetime.date`
- :py:class:`datetime.datetime`
- string date (e.g. ``2012-10-30``)
- string datetime (e.g. ``2012-10-30 11:11:11``)
:param property_name: a string representing the property name to apply
as filter against
:param property_value: a string representing the value which filters
against `property_name`
:param sort_property: a string representing the property on which
to sort results (default ``instance_datetime``)
:param sort_descending: a boolean of whether to sort descending
(default is ``False``).
Applied if `sort_property` is specified
:returns: list of WOUDC observations GeoJSON payload
"""
constraints = []
variables = []
filter_string = None
bbox = None
temporal = None
property_name = None
property_value = None
sort_property = None
sort_descending = False
startindex = 0
output = []
LOGGER.info('Downloading dataset %s', typename)
LOGGER.debug('Assembling query parameters')
for key, value in kwargs.iteritems():
if key == 'bbox':
bbox = value
if key == 'temporal':
temporal = value
if key == 'property_name':
property_name = value
if key == 'property_value':
property_value = str(value)
if key == 'variables':
variables = value
if key == 'sortby':
sort_property = value
if key == 'sort_descending':
sort_descending = value
LOGGER.debug('Assembling constraints')
if property_name is not None and property_value is not None:
constraints.append(fes.PropertyIsEqualTo(property_name,
property_value))
if bbox is not None:
if not isinstance(bbox, list) or len(bbox) != 4:
raise ValueError('bbox must be list of minx, miny, maxx, maxy')
LOGGER.debug('Setting spatial constraint')
constraints.append(fes.BBox(bbox))
if temporal is not None:
if not isinstance(temporal, list) or len(temporal) != 2:
msg = 'temporal must be list of start date, end date'
raise ValueError(msg)
LOGGER.info('Setting temporal constraint')
temporal_start = date2string(temporal[0], 'begin')
temporal_end = date2string(temporal[1], 'end')
constraints.append(fes.PropertyIsBetween(
'instance_datetime', temporal_start, temporal_end))
if sort_descending is not None:
if not isinstance(sort_descending, bool):
raise ValueError('sort_descending must be boolean')
if constraints:
LOGGER.debug('Combining constraints')
flt = fes.FilterRequest()
if len(constraints) == 1:
LOGGER.debug('Single constraint')
filter_string = flt.setConstraint(constraints[0],
tostring=True)
if len(constraints) > 1:
LOGGER.debug('Multiple constraints')
filter_string = flt.setConstraintList([constraints],
tostring=True)
LOGGER.info('Fetching observations')
LOGGER.info('Filters:')
LOGGER.info('bbox: %r', bbox)
LOGGER.info('temporal: %r', temporal)
LOGGER.info('attribute query: %r = %r', property_name, property_value)
# page download and assemble single list of JSON features
while True:
LOGGER.debug('Fetching features %d - %d',
startindex, startindex + self.maxfeatures)
payload = self.server.getfeature(
typename=typename,
startindex=startindex,
propertyname=variables,
maxfeatures=self.maxfeatures,
filter=filter_string,
outputFormat=self.outputformat).read()
LOGGER.debug('Processing response')
if payload.isspace():
LOGGER.debug('Empty response. Exiting')
break
try:
features = json.loads(payload)['features']
except ValueError:
msg = 'Query produced no results'
LOGGER.info(msg)
return None
len_features = len(features)
LOGGER.debug('Found %d features', len_features)
output.extend(features)
if len_features < self.maxfeatures:
break
startindex = startindex + self.maxfeatures
LOGGER.info('Found %d features', len(output))
if sort_property is not None:
LOGGER.info('Sorting response by %s', sort_property)
output.sort(key=lambda e: e['properties'][sort_property],
reverse=sort_descending)
return output
