def make_fes_filter(self): ''' Generates the filter for querying the IOOS database ''' begin, end = fes_date_filter(self.start, self.stop) kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') if len(self.target) > 1: prop_filt = fes.Or([ fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in self.target ]) else: prop_filt = fes.PropertyIsLike(literal=(self.target[0]), **kw) if self.models == True: self.filter_list = [ fes.And([ self.bbox_crs, begin, end, prop_filt, fes.PropertyIsLike(literal=('*%s*' % 'forecast'), **kw), fes.Not([fes.PropertyIsLike(literal='*cdip', **kw)]), fes.Not([fes.PropertyIsLike(literal='*grib*', **kw)]) ]) ] else: self.filter_list = [ fes.And([ self.bbox_crs, begin, end, prop_filt, fes.Not([fes.PropertyIsLike(literal='*cdip', **kw)]), fes.Not([fes.PropertyIsLike(literal='*grib*', **kw)]) ]) ]
def make_filter(config, ): kw = { "wildCard": "*", "escapeChar": "\\", "singleChar": "?", "propertyname": "apiso:Subject", } if len(config["cf_names"]) > 1: or_filt = fes.Or([ fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"] ]) else: or_filt = fes.PropertyIsLike(literal=("*%s*" % config["cf_names"][0]), **kw) kw.update({"propertyname": "apiso:AnyText"}) not_filt = fes.Not([fes.PropertyIsLike(literal="*cdip*", **kw)]) begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"]) bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"]) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] return filter_list
def make_filter(config): from owslib import fes from ioos_tools.ioos import fes_date_filter kw = dict( wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText" ) or_filt = fes.Or( [fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"]] ) not_filt = fes.Not([fes.PropertyIsLike(literal="GRIB-2", **kw)]) begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"]) bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"]) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] return filter_list
#use the search name to create search filter or_filt = fes.Or([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') for val in data_dict['currents']['names'] ]) val = 'Averages' not_filt = fes.Not([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') ]) filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])] # connect to CSW, explore it's properties # try request using multiple filters "and" syntax: [[filter1,filter2]] csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') print str(len(csw.records)) + " csw records found" for rec, item in csw.records.items(): print(item.title) # <markdowncell> # #### List end points available
That search returned a lot of records! What if we are not interested in those model results nor global dataset? We can those be excluded from the search with a `fes.Not` filter. kw = dict(wildCard="*", escapeChar="\\\\", singleChar="?", propertyname="apiso:AnyText") filter_list = [ fes.And( [ bbox_crs, # Bounding box begin, end, # start and end date or_filt, # or conditions (CF variable names). fes.Not([fes.PropertyIsLike(literal="*NAM*", **kw)]), # no NAM results fes.Not([fes.PropertyIsLike(literal="*CONUS*", **kw)]), # no NAM results fes.Not([fes.PropertyIsLike(literal="*GLOBAL*", **kw)]), # no NAM results fes.Not([fes.PropertyIsLike(literal="*ROMS*", **kw)]), # no NAM results ] ) ] get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000) records = "\n".join(csw.records.keys()) print("Found {} records.\n".format(len(csw.records.keys()))) for key, value in list(csw.records.items()): print(u"[{}]\n{}\n".format(value.title, key)) Now we got fewer records to deal with. That's better. But if the user is interested in only some specific service, it is better to filter by a string, like [`CO-OPS`](https://tidesandcurrents.noaa.gov/).
filter1 = fes.Or(filt) val = 'ocean' filter2 = fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') val = 'regridded' filt = fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') filter3 = fes.Not([filt]) val = 'espresso' filt = fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') filter4 = fes.Not([filt]) filter_list = [fes.And([filter1, filter2, filter3, filter4])] # In[ ]: csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') len(csw.records.keys())
start, stop = fes_date_filter(start_date, stop_date) bbox = fes.BBox(bounding_box) # Use the search name to create search filter. kw = dict(propertyname='apiso:AnyText', escapeChar='\\', wildCard='*', singleChar='?') or_filt = fes.Or([ fes.PropertyIsLike(literal='*%s*' % val, **kw) for val in data_dict['winds']['u_names'] ]) val = 'Averages' not_filt = fes.Not([fes.PropertyIsLike(literal=('*%s*' % val), **kw)]) filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])] csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') print("%s csw records found" % len(csw.records)) # <markdowncell> # #### DAP endpoints # <codecell> dap_urls = service_urls(csw.records) # Remove duplicates and organize. dap_urls = sorted(set(dap_urls)) print("Total DAP: %s" % len(dap_urls))
# In[5]: from owslib import fes kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') or_filt = fes.Or( [fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in name_list]) # Exculde ROMS Averages and History files. not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)]) begin, end = fes_date_filter(start, stop) filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])] # In[6]: from owslib.csw import CatalogueServiceWeb endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' csw = CatalogueServiceWeb(endpoint, timeout=60) csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') log.info(fmt(' Catalog information ')) log.info("URL: {}".format(endpoint)) log.info("CSW version: {}".format(csw.version))
propertyname = "apiso:TempExtent_begin" begin = fes.PropertyIsGreaterThanOrEqualTo( propertyname=propertyname, literal=start ) propertyname = "apiso:TempExtent_end" end = fes.PropertyIsLessThanOrEqualTo(propertyname=propertyname, literal=stop) else: raise NameError("Unrecognized constraint {}".format(constraint)) return begin, end kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") or_filt = fes.Or([fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in cf_names]) # Exclude GNOME returns. not_filt = fes.Not([fes.PropertyIsLike(literal="*GNOME*", **kw)]) begin, end = fes_date_filter(start, stop) bbox_crs = fes.BBox(bbox, crs=crs) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] It is possible to use the same filter to search multiple catalogs. The cell below loops over 3 catalogs hoping to find which one is more up-to-date and returns the near real time data. def get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000): """Iterate `maxrecords`/`pagesize` times until the requested value in `maxrecords` is reached. """ from owslib.fes import SortBy, SortProperty # Iterate over sorted results. sortby = SortBy([SortProperty("dc:title", "ASC")])