def make_filter(config, ): kw = { "wildCard": "*", "escapeChar": "\\", "singleChar": "?", "propertyname": "apiso:Subject", } if len(config["cf_names"]) > 1: or_filt = fes.Or([ fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"] ]) else: or_filt = fes.PropertyIsLike(literal=("*%s*" % config["cf_names"][0]), **kw) kw.update({"propertyname": "apiso:AnyText"}) not_filt = fes.Not([fes.PropertyIsLike(literal="*cdip*", **kw)]) begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"]) bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"]) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] return filter_list
def any_case_match_filters(self, propertyname, word_list): ''' Helper function to return and/or filter from a word list for partial case insensitivity Work-around resolves issues where "matchCase=False" is ignored and keywords with whitespace characters are not found @param propertyname: String denoting CSW property to search @param word_list: List of strings for (partially) case-insensitive "and" search @return: List of FES filters expressing "and" query ''' filter_list = [] for word in word_list: word_variant_set = set( [word, word.upper(), word.lower(), word.title()]) if len(word_variant_set ) == 1: # Use single filter if no "or" required filter_list.append( fes.PropertyIsLike(propertyname=propertyname, literal=word_variant_set.pop(), matchCase=False)) else: filter_list.append( fes.Or([ fes.PropertyIsLike(propertyname=propertyname, literal=re.sub( '(\s)', '_', word_variant), matchCase=False) for word_variant in set([word, word.upper(), word.lower(), word.title()]) ])) return filter_list
def make_fes_filter(self): begin, end = fes_date_filter(self.start, self.stop) kw = dict(wildCard='*',escapeChar='\\',singleChar='?',propertyname='apiso:AnyText') if len(self.target) > 1: prop_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in self.target]) else: prop_filt = fes.PropertyIsLike(literal=(self.target[0]), **kw) if self.models == True: self.filter_list = [fes.And([self.bbox_crs, begin, end, prop_filt, fes.PropertyIsLike(literal=('*%s*' % 'forecast'), **kw), fes.Not([fes.PropertyIsLike(literal='*cdip',**kw)]),fes.Not([fes.PropertyIsLike(literal='*grib*', **kw)])])] else: self.filter_list = [fes.And([self.bbox_crs, begin, end, prop_filt, fes.Not([fes.PropertyIsLike(literal='*cdip',**kw)]),fes.Not([fes.PropertyIsLike(literal='*grib*', **kw)])])]
def req_alpha(self, ident, schema, attribut, valeur, mods, maxi=0, ordre=None): """recupere les elements d'une requete alpha""" niveau, classe = ident requete = "" data = "" schema.resolve() attlist = schema.get_liste_attributs() self.get_attr_of_classe params = {"typename": niveau + ":" + classe} if attribut: filter = F.PropertyIsLike(propertyname=attribut, literal=valeur, wildCard="*") filterxml = etree.tostring(filter.toXML(), encoding="unicode") params["filter"] = filterxml print("envoi requete", params) # reponse = self.connection.getfeature(**params) reponse = self.connection.getfeature(typename=niveau + ":" + classe) print("wfs apres reponse", type(reponse)) return reponse
def make_filter(config): from owslib import fes from ioos_tools.ioos import fes_date_filter kw = dict( wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText" ) or_filt = fes.Or( [fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"]] ) not_filt = fes.Not([fes.PropertyIsLike(literal="GRIB-2", **kw)]) begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"]) bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"]) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] return filter_list
def csw_ajax(request, *args, **kwargs): if request.method == 'GET': csw_url = request.session['csw_url'] user = request.session['user'] password = request.session['password'] keywords = request.session['keywords'] keywords_query = [fes.PropertyIsLike( 'csw:AnyText', '%%%s%%' % keywords)] if not csw_url: return HttpResponseServerError() try: csw = CatalogueServiceWeb( csw_url, username=user, password=password) result = csw.identification.type if result == 'CSW': offset = int(request.GET['offset']) per_page = int(request.GET['perPage']) csw.getrecords2( typenames='gmd:MD_Metadata', esn='full', outputschema='http://www.isotc211.org/2005/gmd', constraints=keywords_query, startposition=offset, maxrecords=per_page) result = [] for key in csw.records: rec = csw.records[key] res = {} if isinstance(rec, MD_Metadata): res['id'] = rec.identifier res['title'] = rec.identification.title res['inasafe_keywords'] = rec.identification.\ supplementalinformation if res['inasafe_keywords']: res['inasafe_layer'] = ( '<inasafe_keywords/>' in res['inasafe_keywords']) result.append(res) json_result = { 'records': result, 'queryRecordCount': csw.results['matches'], 'totalRecordCount': csw.results['matches'] } return JsonResponse(json_result, safe=False) except Exception as e: LOGGER.exception(e) return HttpResponseServerError() return HttpResponseServerError()
def csw_query(endpoint, bbox=None, start=None, stop=None, kw_names=None, crs="urn:ogc:def:crs:OGC:1.3:CRS84"): crs = "urn:ogc:def:crs:::EPSG:4326" #https://github.com/qgis/QGIS/issues/40778 constraints = [] csw = None while csw is None: try: csw = CatalogueServiceWeb(endpoint, timeout=60) #csw.getrecords2(maxrecords=10) #for rec in csw.records: # print(vars(csw.records[rec])) # print(csw.records[rec].title) except: pass if kw_names: kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") or_filt = fes.Or([ fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in kw_names ]) constraints.append(or_filt) if all(v is not None for v in [start, stop]): begin, end = fes_date_filter(start, stop) constraints.append(begin) constraints.append(end) if bbox: bbox_crs = fes.BBox(bbox, crs=crs) constraints.append(bbox_crs) if len(constraints) >= 2: filter_list = [fes.And(constraints)] else: filter_list = constraints get_csw_records(csw, filter_list, pagesize=10, maxrecords=10) print("Found {} records.\n".format(len(csw.records.keys()))) for key, value in list(csw.records.items()): print(u"Title: [{}]\nID: {}\n".format(value.title, key)) msg = "geolink: {geolink}\nscheme: {scheme}\nURL: {url}\n".format for ref in value.references: print(msg(geolink=sniff_link(ref["url"]), **ref)) print("#########################################################", '\n')
def parseFilter(self, s): """translates inspirobot filter syntax into fes for example: 'OrganisationName = DREAL Bretagne && Type = dataset || OrganisationName ~ DDTM 29 && Type = dataset' """ filters = [] for f_or in [x.split('&&') for x in s.split('||')]: andgroup = [] for f_and in f_or: if '=' in f_and: a = [s.strip() for s in f_and.split('=')] andgroup.append(fes.PropertyIsEqualTo(propertyname=a[0], literal=a[1])) elif '~' in f_and: a = [s.strip() for s in f_and.split('~')] andgroup.append(fes.PropertyIsLike(propertyname=a[0], literal=a[1])) filters.append(andgroup) return filters
f = fes.BBox(bbox=list(reversed(s.bounds))) polygon_filters.append(f) # If we have more than one polygon filter, OR them together if len(polygon_filters) > 1: polygon_filters = fes.Or(polygon_filters) elif len(polygon_filters) == 1: polygon_filters = polygon_filters[0] # Name filters name_filters = [] model_strings = [ 'roms', 'selfe', 'adcirc', 'ncom', 'hycom', 'fvcom', 'wrf', 'wrams' ] for model in model_strings: title_filter = fes.PropertyIsLike(propertyname='apiso:Title', literal='*%s*' % model, wildCard='*') name_filters.append(title_filter) subject_filter = fes.PropertyIsLike(propertyname='apiso:Subject', literal='*%s*' % model, wildCard='*') name_filters.append(subject_filter) # Or all of the name filters together name_filters = fes.Or(name_filters) # Final filters filters = fes.And([polygon_filters, name_filters]) # <markdowncell> # ##### The actual CSW filters look like this
# In[2]: endpoint = 'http://www.nodc.noaa.gov/geoportal/csw' # NODC/UAF Geoportal: granule level csw = CatalogueServiceWeb(endpoint, timeout=60) print csw.version # In[3]: csw.get_operation_by_name('GetRecords').constraints # In[4]: val = 'level' filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') filter_list = [filter1] # In[5]: csw.getrecords2(constraints=filter_list, maxrecords=100, esn='full') len(csw.records.keys()) # In[6]: choice = np.random.choice(list(csw.records.keys())) print(csw.records[choice].title) csw.records[choice].references
propertyname=propertyname, literal=start ) elif constraint == "within": propertyname = "apiso:TempExtent_begin" begin = fes.PropertyIsGreaterThanOrEqualTo( propertyname=propertyname, literal=start ) propertyname = "apiso:TempExtent_end" end = fes.PropertyIsLessThanOrEqualTo(propertyname=propertyname, literal=stop) else: raise NameError("Unrecognized constraint {}".format(constraint)) return begin, end kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") or_filt = fes.Or([fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in cf_names]) # Exclude GNOME returns. not_filt = fes.Not([fes.PropertyIsLike(literal="*GNOME*", **kw)]) begin, end = fes_date_filter(start, stop) bbox_crs = fes.BBox(bbox, crs=crs) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] It is possible to use the same filter to search multiple catalogs. The cell below loops over 3 catalogs hoping to find which one is more up-to-date and returns the near real time data. def get_csw_records(csw, filter_list, pagesize=10, maxrecords=1000): """Iterate `maxrecords`/`pagesize` times until the requested value in `maxrecords` is reached. """ from owslib.fes import SortBy, SortProperty
# <codecell> endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal csw = CatalogueServiceWeb(endpoint,timeout=60) # <codecell> # convert User Input into FES filters start, stop = fes_date_filter(start_date,end_date) bbox = fes.BBox(bounding_box) #use the search name to create search filter or_filt = fes.Or([fes.PropertyIsLike(propertyname='apiso:AnyText', literal='*%s*' % val, escapeChar='\\', wildCard='*', singleChar='?') for val in data_dict["temp"]["names"]]) # try request using multiple filters "and" syntax: [[filter1,filter2]] filter_list = [fes.And([ bbox, start, stop, or_filt]) ] csw.getrecords2(constraints=filter_list,maxrecords=1000,esn='full') print str(len(csw.records)) + " csw records found" # <markdowncell> # #### Dap URLs # <codecell>
for d in rec.references if d['scheme'] == service_string), None) if url is not None: urls.append(url) return urls # <codecell> filt = [] search_text = ['roms', 'selfe', 'adcirc', 'ncom', 'hycom', 'fvcom'] for val in search_text: filt.append( fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?')) val = 'sea_water_temperature' filt.append( fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?')) # <markdowncell> # ##Find model results at NODC
# <markdowncell> # <div class="error"><strong>Missing CF Standard Names</strong> - "sea_surface_height" and "sea_surface_elevation" are valid CF Aliases but are not returned by MMI when running the SPARQL query. We added them here manually. See: https://github.com/ioos/system-test/issues/129</div> # <markdowncell> # #### Construct CSW Filters # <codecell> from owslib import fes cf_name_filters = [] for cf_name in variables_to_query: text_filter = fes.PropertyIsLike(propertyname='apiso:AnyText', literal="*%s*" % cf_name, wildCard='*') cf_name_filters.append(text_filter) # <markdowncell> # #### Query each CSW catalog for the cf_name_filters constructed above # <codecell> from owslib.csw import CatalogueServiceWeb from utilities import normalize_service_urn var_results = [] for x in range(len(cf_name_filters)): var_name = variables_to_query[x]
propertyname=propertyname, literal=start ) elif constraint == "within": propertyname = "apiso:TempExtent_begin" begin = fes.PropertyIsGreaterThanOrEqualTo( propertyname=propertyname, literal=start ) propertyname = "apiso:TempExtent_end" end = fes.PropertyIsLessThanOrEqualTo(propertyname=propertyname, literal=stop) else: raise NameError("Unrecognized constraint {}".format(constraint)) return begin, end kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") or_filt = fes.Or([fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in cf_names]) begin, end = fes_date_filter(start, stop) bbox_crs = fes.BBox(bbox, crs=crs) filter_list = [ fes.And( [ bbox_crs, # bounding box begin, end, # start and end date or_filt, # or conditions (CF variable names) ] ) ]
def retrieve(self, variable=None, dataframe=False, count=False, bbox=None, or_filter=False, *searchArgs, **kwargs): ''' Retrieve data from WFS service ''' if not self._valid: raise Exception(self._status) if not variable: variable = self._client.items()[0][0] query_params = dict(typename=variable, propertyname=None) # construct query from searchArgs if count: query_params['resulttype'] = 'hits' if bbox: query_params['bbox'] = bbox #query_params.update(kwargs) if len(kwargs) > 0: import owslib.fes as fes from owslib.etree import etree filter_components = [ fes.PropertyIsLike(propertyname=k, literal='%s' % (str(v))) for k, v in kwargs.items() ] if len(filter_components) == 1: query_filter = filter_components[0] elif or_filter: query_filter = fes.Or(filter_components) else: query_filter = fes.And(filter_components) filterxml = etree.tostring(query_filter.toXML()).decode('utf-8') filterxml = '<Filter>%s</Filter>' % filterxml query_params['filter'] = filterxml filter_result = False try: results = self._client.getfeature(**query_params) except: if 'filter' in query_params: query_params.pop('filter') filter_result = True results = self._client.getfeature(**query_params) else: raise feature_text = results.read() if not isinstance(feature_text, str): feature_text = feature_text.decode('utf-8') tmp_fn = 'tmp.gml' open(tmp_fn, 'w').write(feature_text) from osgeo import ogr ds = ogr.Open(tmp_fn) # remove tmp file layer = ds.GetLayer() features = [ layer.GetNextFeature() for i in range(layer.GetFeatureCount()) ] if filter_result: features = self.filter_features(features, or_filter, kwargs) from shapely.wkt import loads for f in features: geom = f.geometry() wkt = geom.ExportToWkt() try: f.geom = loads(wkt) except: f.geom = loads(wkt.replace('MULTISURFACE', 'MULTIPOLYGON')) if dataframe: from geopandas import GeoDataFrame feature_dicts = [ dict([('geometry', f.geom)] + [(p, f[p]) for p in f.keys()]) for f in features ] return GeoDataFrame(feature_dicts) return features
# #### Search the catologue using the FES filters # <codecell> # Convert User Input into FES filters. start, stop = fes_date_filter(start_date, stop_date) bbox = fes.BBox(bounding_box) # Use the search name to create search filter. kw = dict(propertyname='apiso:AnyText', escapeChar='\\', wildCard='*', singleChar='?') or_filt = fes.Or([ fes.PropertyIsLike(literal='*%s*' % val, **kw) for val in data_dict['winds']['u_names'] ]) val = 'Averages' not_filt = fes.Not([fes.PropertyIsLike(literal=('*%s*' % val), **kw)]) filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])] csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') print("%s csw records found" % len(csw.records)) # <markdowncell> # #### DAP endpoints # <codecell>
else: raise NameError('Unrecognized constraint {}'.format(constraint)) return begin, end # In[5]: from owslib import fes kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') or_filt = fes.Or( [fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in name_list]) # Exculde ROMS Averages and History files. not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)]) begin, end = fes_date_filter(start, stop) filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])] # In[6]: from owslib.csw import CatalogueServiceWeb endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' csw = CatalogueServiceWeb(endpoint, timeout=60) csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
def query_csw(self, keyword_list=None, bounding_box=None, bounding_box_crs=None, anytext_list=None, titleword_list=None, start_datetime=None, stop_datetime=None, max_total_records=None): ''' Function to query CSW using AND combination of provided search parameters and return generator object yielding nested dicts containing information about each record including distributions @param keyword_list: List of strings or comma-separated string containing keyword search terms @param bounding_box: Bounding box to search as a list of ordinates [bbox.minx, bbox.minx, bbox.maxx, bbox.maxy] @param bounding_box_crs: Coordinate reference system for bounding box. Defaults to value of CSWUtils.DEFAULT_CRS @param anytext_list: List of strings or comma-separated string containing any text search terms @param titleword: List of strings or comma-separated string containing title search terms @param start_datetime: Datetime object defining start of temporal search period @param stop_datetime: Datetime object defining end of temporal search period @param max_total_records: Maximum total number of records to return. Defaults to value of CSWUtils.DEFAULT_MAXTOTALRECORDS @return: generator object yielding nested dicts containing information about each record including distributions ''' bounding_box_crs = bounding_box_crs or CSWUtils.DEFAULT_CRS # Convert strings to lists if required if type(keyword_list) == str: keyword_list = self.list_from_comma_separated_string(keyword_list) if type(anytext_list) == str: anytext_list = self.list_from_comma_separated_string(anytext_list) if type(titleword_list) == str: titleword_list = self.list_from_comma_separated_string( titleword_list) # Build filter list fes_filter_list = [] # Check for unchanged, upper-case, lower-case and capitalised keywords if keyword_list: fes_filter_list += self.any_case_match_filters( 'Subject', keyword_list) if anytext_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='anyText', literal=phrase, matchCase=False) for phrase in anytext_list ] if start_datetime or stop_datetime: fes_filter_list += self.get_date_filter(start_datetime, stop_datetime) if titleword_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='title', literal=titleword, matchCase=False) for titleword in titleword_list ] if bounding_box: fes_filter_list += [fes.BBox(bounding_box, crs=bounding_box_crs)] assert fes_filter_list, 'No search criteria defined' # Use single filter if no "and" required if len(fes_filter_list) == 1: fes_filter_list = fes_filter_list[0] # Return generator object return self.get_csw_records(fes_filter_list, max_total_records=max_total_records)
def getDataSetURI(anyText, CSWURL, BBox): """ Searches a given CSW server and returns metadata content for the datasets found. Arguments --------- - anyText - A string that will be submitted to the CSW search. (Optional, default is empty which will return all records.) - CSWURL - A base URL for the CSW server to be searched. (Optional, defaults to the CDIA/GDP CSW server.) - BBox - A lat/lon bounding box in [minx,miny,maxx,maxy] that will be used to limit results to datasets that atleast partially intersect. (Optional) """ csw = CatalogueServiceWeb(CSWURL, skip_caps=True) # FIXME: we should allow for "real" multiple keywords, # or change the API of anyText if that if that does not make sense in pygdp. # If the former we need `fes.And`, if the latter we need to not listfy `anyText`. if not anyText: constraints = [] else: constraints = [ fes.PropertyIsLike(propertyname='csw:AnyText', literal=literal) for literal in anyText ] csw.getrecords2(constraints=constraints, outputschema='http://www.isotc211.org/2005/gmd', esn='full', maxrecords=100) dataset_uris = [['title', 'abstract', ['urls']]] for rec in csw.records: title = csw.records[rec].identification.title abstract = csw.records[rec].identification.abstract urls = [] try: for onlineresource in range( len(csw.records[rec].distribution.online)): urls.append( csw.records[rec].distribution.online[onlineresource].url) except AttributeError: pass for ident in range(len(csw.records[rec].identificationinfo)): try: for operation in range( len(csw.records[rec].identificationinfo[ident]. operations)): urls.append(csw.records[rec].identificationinfo[ident]. operations[0]['connectpoint'][0].url) except AttributeError: pass entry = [title, abstract, urls] dataset_uris.append(entry) for i, dataset in enumerate(dataset_uris): dataset_uris[i][2] = [ uri.replace("https", "dods").replace("http", "dods") if "/dodsC/" in uri else uri for uri in dataset[2] ] return dataset_uris
def query_csw(self, identifier_list=None, alt_identifier_list=None, keyword_list=None, bounding_box=None, bounding_box_crs=None, anytext_list=None, titleword_list=None, start_datetime=None, stop_datetime=None, record_type_list=None, max_total_records=None, get_layers=None): ''' Function to query CSW using AND combination of provided search parameters and return generator object yielding nested dicts containing information about each record including distributions @param identifier_list: List of strings or comma-separated string containing metadata identifiers (UUID) @param alt_identifier: List of strings or comma-separated string containing metadata alternate identifiers (eCat ID) @param keyword_list: List of strings or comma-separated string containing keyword search terms @param bounding_box: Bounding box to search as a list of ordinates [bbox.minx, bbox.minx, bbox.maxx, bbox.maxy] @param bounding_box_crs: Coordinate reference system for bounding box. Defaults to value of self.settings['DEFAULT_CRS'] @param anytext_list: List of strings or comma-separated string containing any text search terms @param titleword: List of strings or comma-separated string containing title search terms @param start_datetime: Datetime object defining start of temporal search period @param stop_datetime: Datetime object defining end of temporal search period @param record_type_list: List of strings or comma-separated string containing record type(s) to return @param max_total_records: Maximum total number of records to return. Defaults to value of self.settings['DEFAULT_MAXTOTALRECORDS'] @param get_layers: Boolean flag indicating whether to get WMS/WCS layer names. Defaults to value of self.settings['DEFAULT_GET_LAYERS'] @return: generator object yielding nested dicts containing information about each record including distributions ''' bounding_box_crs = bounding_box_crs or self.settings['DEFAULT_CRS'] get_layers = self.settings[ 'DEFAULT_GET_LAYERS'] if get_layers is None else get_layers # Convert strings to lists if required if type(identifier_list) == str: identifier_list = self.list_from_comma_separated_string( identifier_list) if type(alt_identifier_list) == str: alt_identifier_list = self.list_from_comma_separated_string( alt_identifier_list) if type(keyword_list) == str: keyword_list = self.list_from_comma_separated_string(keyword_list) if type(anytext_list) == str: anytext_list = self.list_from_comma_separated_string(anytext_list) if type(titleword_list) == str: titleword_list = self.list_from_comma_separated_string( titleword_list) record_type_list = record_type_list or self.settings[ 'DEFAULT_RECORD_TYPES'] if type(record_type_list) == str: record_type_list = self.list_from_comma_separated_string( record_type_list) # Build filter list fes_filter_list = [] if identifier_list: if len(identifier_list) == 1: fes_filter_list += [ fes.PropertyIsLike(propertyname='Identifier', literal=identifier_list[0], matchCase=False) ] else: fes_filter_list.append( fes.Or([ fes.PropertyIsLike(propertyname='Identifier', literal=identifier, matchCase=False) for identifier in identifier_list ])) if alt_identifier_list: if len(alt_identifier_list) == 1: fes_filter_list += [ fes.PropertyIsLike(propertyname='AlternateIdentifier', literal=alt_identifier_list[0], matchCase=False) ] else: fes_filter_list.append( fes.Or([ fes.PropertyIsLike(propertyname='AlternateIdentifier', literal=alt_identifier, matchCase=False) for alt_identifier in alt_identifier_list ])) # Check for unchanged, upper-case, lower-case and capitalised keywords # with single-character wildcards substituted for whitespace characters # GeoNetwork keyword search is always case sensitive if keyword_list: fes_filter_list += self.any_case_match_filters( 'Subject', keyword_list) if anytext_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='anyText', literal=phrase, matchCase=False) for phrase in anytext_list ] if start_datetime or stop_datetime: fes_filter_list += self.get_date_filter(start_datetime, stop_datetime) if titleword_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='title', literal=titleword, matchCase=False) for titleword in titleword_list ] # Check for unchanged, upper-case, lower-case and capitalised keywords # with single-character wildcards substituted for whitespace characters # GeoNetwork type search is always case sensitive if record_type_list: fes_filter_list += self.any_case_match_filters( 'type', record_type_list) if bounding_box: # N.B: Bounding box ordinate ordering must match CRS. Default CRS84 supports lon-lat ordering, not lat-lon # See https://gis.stackexchange.com/questions/124050/how-do-i-specify-the-lon-lat-ordering-in-csw-bounding-box-request fes_filter_list += [fes.BBox(bounding_box, crs=bounding_box_crs)] assert fes_filter_list, 'No search criteria defined' # Use single filter if no "and" required if len(fes_filter_list) == 1: fes_filter_list = fes_filter_list[0] # Return generator object return self.get_csw_records(fes_filter_list, max_total_records=max_total_records, get_layers=get_layers)
jd_start = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M') jd_stop = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M') print start_date,'to',stop_date sos_name = 'water_surface_height_above_reference_datum' # <codecell> # convert User Input into FES filters start,stop = dateRange(start_date,stop_date) bbox = fes.BBox(box) or_filt = fes.Or([fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val), escapeChar='\\',wildCard='*',singleChar='?') for val in model_name_list]) val = 'Averages' not_filt = fes.Not([fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val), escapeChar='\\',wildCard='*',singleChar='?')]) filter_list = [fes.And([ bbox, start, stop, or_filt, not_filt]) ] # <markdowncell> # ##Find model results at NODC # <codecell>
# -*- coding: utf-8 -*- # <nbformat>3.0</nbformat> # <codecell> from owslib import fes from owslib.csw import CatalogueServiceWeb schemes = set() c = CatalogueServiceWeb("https://data.noaa.gov/csw", timeout=20) fil = fes.PropertyIsLike(propertyname='apiso:AnyText', literal="*sea_surface_height_above_sea_level*", wildCard='*') c.getrecords2(constraints=[fil], maxrecords=1000, esn='full') # <codecell> for record, item in c.records.items(): for d in item.references: schemes.add(d['scheme']) # <codecell> for scheme in schemes: print scheme
try: csw.get_operation_by_name('GetDomain') csw.getdomain('apiso:ServiceType', 'property') print(csw.results['values']) except: print('GetDomain not supported') # ## Query for all COAWST datasets # In[5]: val = 'COAWST' filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') filter_list = [filter1] # In[6]: csw.getrecords2(constraints=filter_list, maxrecords=100, esn='full') print len(csw.records.keys()) for rec in list(csw.records.keys()): print csw.records[rec].title # In[7]: choice = np.random.choice(list(csw.records.keys())) print(csw.records[choice].title)
# <codecell> # convert User Input into FES filters start, stop = date_range(start_date, end_date) box = [] box.append(bounding_box[0][0]) box.append(bounding_box[0][1]) box.append(bounding_box[1][0]) box.append(bounding_box[1][1]) bbox = fes.BBox(box) #use the search name to create search filter or_filt = fes.Or([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') for val in data_dict["waves"]["names"] ]) val = 'Averages' not_filt = fes.Not([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') ]) filter_list = [fes.And([bbox, start, stop, or_filt, not_filt])] # connect to CSW, explore it's properties # try request using multiple filters "and" syntax: [[filter1,filter2]] csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
[op.name for op in csw.operations] # <codecell> for oper in csw.operations: if oper.name == 'GetRecords': print oper.constraints # <markdowncell> # Since the supported ISO queryables contain `apiso:ServiceType`, we can use CSW to find all datasets with services that contain the string "dap" # <codecell> val = 'dap' service_type = fes.PropertyIsLike(propertyname='apiso:ServiceType',literal=('*%s*' % val), escapeChar='\\',wildCard='*',singleChar='?') filter_list = [ service_type] # <codecell> csw.getrecords2(constraints=filter_list,maxrecords=10000,esn='full') len(csw.records.keys()) # <markdowncell> # By printing out the references from a random record, we see that for this CSW the DAP URL is identified by # `urn:x-esri:specification:ServiceType:odp:url` # <codecell> choice=random.choice(list(csw.records.keys()))
# <codecell> for endpoint in bbox_endpoints: print endpoint csw = CatalogueServiceWeb(endpoint, timeout=60) # convert User Input into FES filters start, stop = fes_date_filter(start_date, stop_date) bbox = fes.BBox(bounding_box) #use the search name to create search filter or_filt = fes.Or([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal='*%s*' % val, escapeChar='\\', wildCard='*', singleChar='?') for val in data_dict['winds']['u_names'] ]) filter_list = [fes.And([bbox, start, stop, or_filt])] # filter_list = [fes.And([ bbox, or_filt]) ] # connect to CSW, explore it's properties # try request using multiple filters "and" syntax: [[filter1,filter2]] try: csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') except Exception as e: print 'ERROR - ' + str(e) else: print str(len(csw.records)) + " csw records found"
# With these 3 elements it is possible to assemble a [OGC Filter Encoding (FE)](http://www.opengeospatial.org/standards/filter) using the `owslib.fes`\* module. # # \* OWSLib is a Python package for client programming with Open Geospatial Consortium (OGC) web service (hence OWS) interface standards, and their related content models. # In[8]: from owslib import fes from ioos_tools.ioos import fes_date_filter kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') or_filt = fes.Or( [fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in model_names]) kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:ServiceType') serviceType = fes.PropertyIsLike(literal=('*%s*' % service_type), **kw) begin, end = fes_date_filter(start, stop) bbox_crs = fes.BBox(bbox, crs=crs) filter_list = [ fes.And([ bbox_crs, # bounding box begin,
#endpoint = 'http://data.nodc.noaa.gov/geoportal/csw' csw = CatalogueServiceWeb(endpoint,timeout=60) print csw.version # In[ ]: from owslib import fes from utilities import fes_date_filter kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') or_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in name_list]) # Exclude ROMS Averages and History files. not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)]) bbox_filter = fes.BBox(bbox,crs='urn:ogc:def:crs:OGC:1.3:CRS84') begin, end = fes_date_filter(start, stop) filter_list = [fes.And([bbox_filter, begin, end, or_filt, not_filt])] # In[ ]: bbox = [-87.40, 34.25, -63.70, 66.70] # [lon_min, lat_min, lon_max, lat_max]