def make_fes_filter(self):
'''
Generates the filter for querying the IOOS database
'''
begin, end = fes_date_filter(self.start, self.stop)
kw = dict(wildCard='*',
escapeChar='\\',
singleChar='?',
propertyname='apiso:AnyText')
if len(self.target) > 1:
prop_filt = fes.Or([
fes.PropertyIsLike(literal=('*%s*' % val), **kw)
for val in self.target
])
else:
prop_filt = fes.PropertyIsLike(literal=(self.target[0]), **kw)
if self.models == True:
self.filter_list = [
fes.And([
self.bbox_crs, begin, end, prop_filt,
fes.PropertyIsLike(literal=('*%s*' % 'forecast'), **kw),
fes.Not([fes.PropertyIsLike(literal='*cdip', **kw)]),
fes.Not([fes.PropertyIsLike(literal='*grib*', **kw)])
])
]
else:
self.filter_list = [
fes.And([
self.bbox_crs, begin, end, prop_filt,
fes.Not([fes.PropertyIsLike(literal='*cdip', **kw)]),
fes.Not([fes.PropertyIsLike(literal='*grib*', **kw)])
])
]
def make_filter(config, ):
kw = {
"wildCard": "*",
"escapeChar": "\\",
"singleChar": "?",
"propertyname": "apiso:Subject",
}
if len(config["cf_names"]) > 1:
or_filt = fes.Or([
fes.PropertyIsLike(literal=("*%s*" % val), **kw)
for val in config["cf_names"]
])
else:
or_filt = fes.PropertyIsLike(literal=("*%s*" % config["cf_names"][0]),
**kw)
kw.update({"propertyname": "apiso:AnyText"})
not_filt = fes.Not([fes.PropertyIsLike(literal="*cdip*", **kw)])
begin, end = fes_date_filter(config["date"]["start"],
config["date"]["stop"])
bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"])
filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])]
return filter_list
def make_filter(config):
from owslib import fes
from ioos_tools.ioos import fes_date_filter
kw = dict(
wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText"
)
or_filt = fes.Or(
[fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"]]
)
not_filt = fes.Not([fes.PropertyIsLike(literal="GRIB-2", **kw)])
begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"])
bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"])
filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])]
return filter_list
#use the search name to create search filter
or_filt = fes.Or([
fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?')
for val in data_dict['currents']['names']
])
val = 'Averages'
not_filt = fes.Not([
fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?')
])
filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])]
# connect to CSW, explore it's properties
# try request using multiple filters "and" syntax: [[filter1,filter2]]
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
print str(len(csw.records)) + " csw records found"
for rec, item in csw.records.items():
print(item.title)
# <markdowncell>
# #### List end points available
That search returned a lot of records!
What if we are not interested in those model results nor global dataset?
We can those be excluded from the search with a `fes.Not` filter.
kw = dict(wildCard="*", escapeChar="\\\\", singleChar="?", propertyname="apiso:AnyText")
filter_list = [
fes.And(
[
bbox_crs, # Bounding box
begin,
end, # start and end date
or_filt, # or conditions (CF variable names).
fes.Not([fes.PropertyIsLike(literal="*NAM*", **kw)]), # no NAM results
fes.Not([fes.PropertyIsLike(literal="*CONUS*", **kw)]), # no NAM results
fes.Not([fes.PropertyIsLike(literal="*GLOBAL*", **kw)]), # no NAM results
fes.Not([fes.PropertyIsLike(literal="*ROMS*", **kw)]), # no NAM results
]
)
]
get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000)
records = "\n".join(csw.records.keys())
print("Found {} records.\n".format(len(csw.records.keys())))
for key, value in list(csw.records.items()):
print(u"[{}]\n{}\n".format(value.title, key))
Now we got fewer records to deal with. That's better. But if the user is interested in only some specific service, it is better to filter by a string, like [`CO-OPS`](https://tidesandcurrents.noaa.gov/).
filter1 = fes.Or(filt)
val = 'ocean'
filter2 = fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?')
val = 'regridded'
filt = fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?')
filter3 = fes.Not([filt])
val = 'espresso'
filt = fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?')
filter4 = fes.Not([filt])
filter_list = [fes.And([filter1, filter2, filter3, filter4])]
# In[ ]:
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
len(csw.records.keys())
start, stop = fes_date_filter(start_date, stop_date)
bbox = fes.BBox(bounding_box)
# Use the search name to create search filter.
kw = dict(propertyname='apiso:AnyText',
escapeChar='\\',
wildCard='*',
singleChar='?')
or_filt = fes.Or([
fes.PropertyIsLike(literal='*%s*' % val, **kw)
for val in data_dict['winds']['u_names']
])
val = 'Averages'
not_filt = fes.Not([fes.PropertyIsLike(literal=('*%s*' % val), **kw)])
filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])]
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
print("%s csw records found" % len(csw.records))
# <markdowncell>
# #### DAP endpoints
# <codecell>
dap_urls = service_urls(csw.records)
# Remove duplicates and organize.
dap_urls = sorted(set(dap_urls))
print("Total DAP: %s" % len(dap_urls))
# In[5]:
from owslib import fes
kw = dict(wildCard='*',
escapeChar='\\',
singleChar='?',
propertyname='apiso:AnyText')
or_filt = fes.Or(
[fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in name_list])
# Exculde ROMS Averages and History files.
not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)])
begin, end = fes_date_filter(start, stop)
filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])]
# In[6]:
from owslib.csw import CatalogueServiceWeb
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw'
csw = CatalogueServiceWeb(endpoint, timeout=60)
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
log.info(fmt(' Catalog information '))
log.info("URL: {}".format(endpoint))
log.info("CSW version: {}".format(csw.version))
propertyname = "apiso:TempExtent_begin"
begin = fes.PropertyIsGreaterThanOrEqualTo(
propertyname=propertyname, literal=start
)
propertyname = "apiso:TempExtent_end"
end = fes.PropertyIsLessThanOrEqualTo(propertyname=propertyname, literal=stop)
else:
raise NameError("Unrecognized constraint {}".format(constraint))
return begin, end
kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText")
or_filt = fes.Or([fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in cf_names])
# Exclude GNOME returns.
not_filt = fes.Not([fes.PropertyIsLike(literal="*GNOME*", **kw)])
begin, end = fes_date_filter(start, stop)
bbox_crs = fes.BBox(bbox, crs=crs)
filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])]
It is possible to use the same filter to search multiple catalogs. The cell below loops over 3 catalogs hoping to find which one is more up-to-date and returns the near real time data.
def get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000):
"""Iterate `maxrecords`/`pagesize` times until the requested value in
`maxrecords` is reached.
"""
from owslib.fes import SortBy, SortProperty
# Iterate over sorted results.
sortby = SortBy([SortProperty("dc:title", "ASC")])
