def make_bbox(self): ''' Helper function to make the filter bounding box ''' crs = 'urn:ogc:def:crs:OGC:1.3:CRS84' self.bbox_crs = fes.BBox( [self.roi[0], self.roi[2], self.roi[1], self.roi[3]], crs=crs)
def make_filter(config, ): kw = { "wildCard": "*", "escapeChar": "\\", "singleChar": "?", "propertyname": "apiso:Subject", } if len(config["cf_names"]) > 1: or_filt = fes.Or([ fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"] ]) else: or_filt = fes.PropertyIsLike(literal=("*%s*" % config["cf_names"][0]), **kw) kw.update({"propertyname": "apiso:AnyText"}) not_filt = fes.Not([fes.PropertyIsLike(literal="*cdip*", **kw)]) begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"]) bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"]) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] return filter_list
def test_csw_query_bbox(self): """Verify that GeoNode CSW can handle bbox queries""" csw = get_catalogue() bbox = fes.BBox([-140, -70, 80, 70]) try: csw.catalogue.getrecords2([bbox, ]) logger.debug(csw.catalogue.results) self.assertEqual(csw.catalogue.results, {'matches': 7, 'nextrecord': 0, 'returned': 7}) except Exception: # This test seems to borken actually on pycsw pass
def csw_query(endpoint, bbox=None, start=None, stop=None, kw_names=None, crs="urn:ogc:def:crs:OGC:1.3:CRS84"): crs = "urn:ogc:def:crs:::EPSG:4326" #https://github.com/qgis/QGIS/issues/40778 constraints = [] csw = None while csw is None: try: csw = CatalogueServiceWeb(endpoint, timeout=60) #csw.getrecords2(maxrecords=10) #for rec in csw.records: # print(vars(csw.records[rec])) # print(csw.records[rec].title) except: pass if kw_names: kw = dict(wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText") or_filt = fes.Or([ fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in kw_names ]) constraints.append(or_filt) if all(v is not None for v in [start, stop]): begin, end = fes_date_filter(start, stop) constraints.append(begin) constraints.append(end) if bbox: bbox_crs = fes.BBox(bbox, crs=crs) constraints.append(bbox_crs) if len(constraints) >= 2: filter_list = [fes.And(constraints)] else: filter_list = constraints get_csw_records(csw, filter_list, pagesize=10, maxrecords=10) print("Found {} records.\n".format(len(csw.records.keys()))) for key, value in list(csw.records.items()): print(u"Title: [{}]\nID: {}\n".format(value.title, key)) msg = "geolink: {geolink}\nscheme: {scheme}\nURL: {url}\n".format for ref in value.references: print(msg(geolink=sniff_link(ref["url"]), **ref)) print("#########################################################", '\n')
def test_bbox(endpoints, bbox): for title, url in endpoints.iteritems(): try: csw = CatalogueServiceWeb(url, timeout=40) if "BBOX" in csw.filters.spatial_operators: filter_list = [fes.BBox(bbox)] try: csw.getrecords2(constraints=filter_list, maxrecords=1000) print("%s : Datasets = %d" % (title, len(csw.records.keys()))) except Exception: print "%s : BBOX Query FAILS" % title else: print "%s - BBOX Query NOT supported" % title except Exception: print "%s - Timed out" % title
def make_filter(config): from owslib import fes from ioos_tools.ioos import fes_date_filter kw = dict( wildCard="*", escapeChar="\\", singleChar="?", propertyname="apiso:AnyText" ) or_filt = fes.Or( [fes.PropertyIsLike(literal=("*%s*" % val), **kw) for val in config["cf_names"]] ) not_filt = fes.Not([fes.PropertyIsLike(literal="GRIB-2", **kw)]) begin, end = fes_date_filter(config["date"]["start"], config["date"]["stop"]) bbox_crs = fes.BBox(config["region"]["bbox"], crs=config["region"]["crs"]) filter_list = [fes.And([bbox_crs, begin, end, or_filt, not_filt])] return filter_list
# <markdowncell> # ### Can we discover other datasets in this polygon area? # <markdowncell> # ##### Setup BCSW Filters to find models in the area of the Important Bird Polygon # <codecell> from owslib import fes # Polygon filters polygon_filters = [] for s in shapes: f = fes.BBox(bbox=list(reversed(s.bounds))) polygon_filters.append(f) # If we have more than one polygon filter, OR them together if len(polygon_filters) > 1: polygon_filters = fes.Or(polygon_filters) elif len(polygon_filters) == 1: polygon_filters = polygon_filters[0] # Name filters name_filters = [] model_strings = [ 'roms', 'selfe', 'adcirc', 'ncom', 'hycom', 'fvcom', 'wrf', 'wrams' ] for model in model_strings: title_filter = fes.PropertyIsLike(propertyname='apiso:Title', literal='*%s*' % model,
def compose_query(query_string, bbox, wfs_filters): """Compose a wfs filter query from a string The query string should be composed as: "property_name operator literal". The property names and operators are initialized with the DovBoringen class. Multiple filters can be added by comma separation e.g.: "property_name1 operator1 literal1, property_name2 operator2 literal2" The PropertyIsBetween operator requires a lower and upper boundary, it is given by a tuple in the string, e.g.: "diepte_tot_m << (20,100)" Parameters ---------- query_string : str A string containing the query that will be used as constrained in the WFS call. See also: get_boringen() bbox : tuple of floats, or empty tuple The X, Y coordinates of the bounding box as (xmin, ymin, xmax, ymax) wfs_filters : dict A dictionary mapping the operator in the query string to the comparison operator of the wfs call Returns ------- filterxml : str A string of the xml constraint for a wfs call using owslib """ filters = [] # extract criteria if query_string: query_raw = [x.strip(' ,') for x in query_string.split(' ')] if len(query_raw) % 3 != 0: raise ValueError('The query string is not correct. ' 'It should be composed of "property operator ' 'literal"') idx = 1 for fltr in query_raw[1::3]: if fltr != '<<': filters.append(wfs_filters[fltr]( propertyname=query_raw[idx - 1], literal=query_raw[idx + 1])) else: lb, ub = [ x.strip(['(', ')']) for x in query_raw[idx + 1].split(',') ] filters.append(wfs_filters[fltr]( propertyname=query_raw[idx - 1], lowerboundary=lb, upperboundary=ub)) idx += 3 if bbox: filters.append(fes.BBox(bbox)) if len(filters) == 1: filter = fes.FilterRequest().setConstraint(filters[0]) elif len(filters) > 1: # only logical AND is evaluated (constraint = [[a, b]]) filter = fes.FilterRequest().setConstraintList([filters]) else: return '' filterxml = fes.etree.tostring(filter, encoding="utf-8", method='xml') return filterxml
csw.records[choice].references # In[8]: csw.request # In[9]: csw.records[choice].xml # Add bounding box constraint. To specify lon,lat order for bbox (which we want to do so that we can use the same bbox with either geoportal server or pycsw requests), we need to request the bounding box specifying the CRS84 coordinate reference system. The CRS84 option is available in `pycsw 1.1.10`+. The ability to specify the `crs` in the bounding box request is available in `owslib 0.8.12`+. For more info on the bounding box problem and how it was solved, see this [pycsw issue](https://github.com/geopython/pycsw/issues/287), this [geoportal server issue](https://github.com/Esri/geoportal-server/issues/124), and this [owslib issue](https://github.com/geopython/OWSLib/issues/201) # In[10]: bbox = [-87.40, 34.25, -63.70, 66.70] # [lon_min, lat_min, lon_max, lat_max] bbox_filter = fes.BBox(bbox, crs='urn:ogc:def:crs:OGC:1.3:CRS84') filter_list = [fes.And([filter1, bbox_filter])] csw.getrecords2(constraints=filter_list, maxrecords=1000) # In[11]: print(len(csw.records.keys())) for rec in list(csw.records.keys()): print('title:' + csw.records[rec].title) print('identifier:' + csw.records[rec].identifier) print('modified:' + csw.records[rec].modified) print(' ') # In[12]: val = 'WMS'
constraint='overlaps'): if constraint == 'overlaps': start = fes.PropertyIsLessThanOrEqualTo( propertyname='apiso:TempExtent_begin', literal=stop_date) stop = fes.PropertyIsGreaterThanOrEqualTo( propertyname='apiso:TempExtent_end', literal=start_date) elif constraint == 'within': start = fes.PropertyIsGreaterThanOrEqualTo( propertyname='apiso:TempExtent_begin', literal=start_date) stop = fes.PropertyIsLessThanOrEqualTo( propertyname='apiso:TempExtent_end', literal=stop_date) return start, stop #Establish bounding box filter for Geographic Range of IBAs bbox = fes.BBox([-130.5, 47.9, 167.6, 74.7]) # <codecell> sparql = SPARQLWrapper("http://mmisw.org/sparql") queryString = """ PREFIX ioos: <http://mmisw.org/ont/ioos/parameter/> SELECT DISTINCT ?parameter ?definition ?unit ?property ?value WHERE {?parameter a ioos:Parameter . ?parameter ?property ?value . ?parameter ioos:Term ?term . ?parameter ioos:Definition ?definition . ?parameter ioos:Units ?unit . FILTER (regex(str(?property), "(exactMatch|closeMatch)", "i") && regex(str(?value), "temperature", "i") ) } ORDER BY ?parameter
'currents', 'surface_eastward_sea_water_velocity', '*surface_eastward_sea_water_velocity*' ], "sos_name": ['currents'] } # <codecell> endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal csw = CatalogueServiceWeb(endpoint, timeout=60) # <codecell> # convert User Input into FES filters start, stop = date_range(start_date, stop_date) bbox = fes.BBox(bounding_box) #use the search name to create search filter or_filt = fes.Or([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val), escapeChar='\\', wildCard='*', singleChar='?') for val in data_dict['currents']['names'] ]) val = 'Averages' not_filt = fes.Not([ fes.PropertyIsLike(propertyname='apiso:AnyText', literal=('*%s*' % val),
def get_data(self, typename, **kwargs): """ Download WOUDC observations :param bbox: a list representing a bounding box spatial filter (`minx, miny, maxx, maxy`) :param temporal: a list of two elements representing a time period (start, end) which accepts the following types: - :py:class:`datetime.date` - :py:class:`datetime.datetime` - string date (e.g. ``2012-10-30``) - string datetime (e.g. ``2012-10-30 11:11:11``) :param filters: `dict` of key-value pairs of property names and values. Constructs exclusive search :param variables: a list of variables to return as part of the response (default returns all) :param sort_property: a string representing the property on which to sort results (default ``instance_datetime``) :param sort_order: a string representing sort order of response (``asc`` or ``desc``). Default is ``asc``. Applied if `sort_property` is specified :returns: list of WOUDC observations GeoJSON payload """ constraints = [] filters = [] variables = '*' filter_string = None bbox = None temporal = None sort_property = None sort_order = 'asc' startindex = 0 features = None feature_collection = None sort_descending = False LOGGER.info('Downloading dataset %s', typename) LOGGER.debug('Assembling query parameters') for key, value in kwargs.items(): if key == 'bbox': bbox = value if key == 'temporal': temporal = value if key == 'filters': filters = value if key == 'variables': variables = value if key == 'sortby': sort_property = value if key == 'sort_order': sort_order = value LOGGER.debug('Assembling constraints') if filters: for key, value in filters.items(): constraints.append(fes.PropertyIsEqualTo(key, value)) if bbox is not None: if not isinstance(bbox, list) or len(bbox) != 4: raise ValueError('bbox must be list of minx, miny, maxx, maxy') LOGGER.debug('Setting spatial constraint') constraints.append(fes.BBox(bbox)) if temporal is not None: if not isinstance(temporal, list) or len(temporal) != 2: msg = 'temporal must be list of start date, end date' raise ValueError(msg) LOGGER.info('Setting temporal constraint') temporal_start = date2string(temporal[0], 'begin') temporal_end = date2string(temporal[1], 'end') constraints.append(fes.PropertyIsBetween( 'instance_datetime', temporal_start, temporal_end)) if sort_order not in ['asc', 'desc']: raise ValueError('sort_order must be asc or desc') else: if sort_order == 'desc': sort_descending = True if variables != '*': if not isinstance(variables, list): raise ValueError('variables must be list') if constraints: LOGGER.debug('Combining constraints') flt = fes.FilterRequest() if len(constraints) == 1: LOGGER.debug('Single constraint') filter_string = flt.setConstraint(constraints[0], tostring=True) if len(constraints) > 1: LOGGER.debug('Multiple constraints') filter_string = flt.setConstraintList([constraints], tostring=True) LOGGER.info('Fetching observations') LOGGER.info('Filters:') LOGGER.info('bbox: %r', bbox) LOGGER.info('temporal: %r', temporal) LOGGER.info('attribute queries: %r', filters) # page download and assemble single list of JSON features while True: LOGGER.debug('Fetching features %d - %d', startindex, startindex + self.maxfeatures) payload = self.server.getfeature( typename=typename, startindex=startindex, propertyname=variables, maxfeatures=self.maxfeatures, filter=filter_string, outputFormat=self.outputformat).read() LOGGER.debug('Processing response') if payload.isspace(): LOGGER.debug('Empty response. Exiting') break try: features = json.loads(payload) except ValueError: msg = 'Query produced no results' LOGGER.info(msg) return None len_features = len(features['features']) LOGGER.debug('Found %d features', len_features) if feature_collection is None: feature_collection = features else: feature_collection['features'].extend(features['features']) if len_features < self.maxfeatures: break startindex = startindex + self.maxfeatures len_feature_collection = len(feature_collection['features']) LOGGER.info('Found %d total features', len_feature_collection) if sort_property is not None: LOGGER.info('Sorting response by %s', sort_property) feature_collection['features'].sort( key=lambda e: e['properties'][sort_property], reverse=sort_descending) return feature_collection
from owslib import fes kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') or_filt = fes.Or( [fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in name_list]) # Exculde ROMS Averages and History files. not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)]) begin, end = fes_date_filter(start, stop) filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])] # In[6]: from owslib.csw import CatalogueServiceWeb endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' csw = CatalogueServiceWeb(endpoint, timeout=60) csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full') log.info(fmt(' Catalog information ')) log.info("URL: {}".format(endpoint)) log.info("CSW version: {}".format(csw.version)) log.info("Number of datasets available: {}".format(len(csw.records.keys()))) # In[7]:
def query_csw(self, keyword_list=None, bounding_box=None, bounding_box_crs=None, anytext_list=None, titleword_list=None, start_datetime=None, stop_datetime=None, max_total_records=None): ''' Function to query CSW using AND combination of provided search parameters and return generator object yielding nested dicts containing information about each record including distributions @param keyword_list: List of strings or comma-separated string containing keyword search terms @param bounding_box: Bounding box to search as a list of ordinates [bbox.minx, bbox.minx, bbox.maxx, bbox.maxy] @param bounding_box_crs: Coordinate reference system for bounding box. Defaults to value of CSWUtils.DEFAULT_CRS @param anytext_list: List of strings or comma-separated string containing any text search terms @param titleword: List of strings or comma-separated string containing title search terms @param start_datetime: Datetime object defining start of temporal search period @param stop_datetime: Datetime object defining end of temporal search period @param max_total_records: Maximum total number of records to return. Defaults to value of CSWUtils.DEFAULT_MAXTOTALRECORDS @return: generator object yielding nested dicts containing information about each record including distributions ''' bounding_box_crs = bounding_box_crs or CSWUtils.DEFAULT_CRS # Convert strings to lists if required if type(keyword_list) == str: keyword_list = self.list_from_comma_separated_string(keyword_list) if type(anytext_list) == str: anytext_list = self.list_from_comma_separated_string(anytext_list) if type(titleword_list) == str: titleword_list = self.list_from_comma_separated_string( titleword_list) # Build filter list fes_filter_list = [] # Check for unchanged, upper-case, lower-case and capitalised keywords if keyword_list: fes_filter_list += self.any_case_match_filters( 'Subject', keyword_list) if anytext_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='anyText', literal=phrase, matchCase=False) for phrase in anytext_list ] if start_datetime or stop_datetime: fes_filter_list += self.get_date_filter(start_datetime, stop_datetime) if titleword_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='title', literal=titleword, matchCase=False) for titleword in titleword_list ] if bounding_box: fes_filter_list += [fes.BBox(bounding_box, crs=bounding_box_crs)] assert fes_filter_list, 'No search criteria defined' # Use single filter if no "and" required if len(fes_filter_list) == 1: fes_filter_list = fes_filter_list[0] # Return generator object return self.get_csw_records(fes_filter_list, max_total_records=max_total_records)
# <codecell> from owslib import fes def fes_date_filter(start_date='1900-01-01',stop_date='2100-01-01',constraint='overlaps'): if constraint == 'overlaps': start = fes.PropertyIsGreaterThanOrEqualTo(propertyname='apiso:TempExtent_end', literal=start_date) stop = fes.PropertyIsLessThanOrEqualTo(propertyname='apiso:TempExtent_begin', literal=stop_date) elif constraint == 'within': start = fes.PropertyIsGreaterThanOrEqualTo(propertyname='apiso:TempExtent_begin', literal=start_date) stop = fes.PropertyIsLessThanOrEqualTo(propertyname='apiso:TempExtent_end', literal=stop_date) return fes.And([start, stop]) # <codecell> # Geographic filters geographic_filter = fes.BBox(bbox=bounding_box) # Temporal filters temporal_filter = fes_date_filter(start_date_string, end_date_string) filters = fes.And([geographic_filter, temporal_filter]) # <markdowncell> # ##### The actual CSW filter POST envelope looks like this # <codecell> from owslib.etree import etree print etree.tostring(filters.toXML(), pretty_print=True)
start_date = jd_start.strftime('%Y-%m-%d %H:00') stop_date = jd_stop.strftime('%Y-%m-%d %H:00') jd_start = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M') jd_stop = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M') print start_date,'to',stop_date sos_name = 'water_surface_height_above_reference_datum' # <codecell> # convert User Input into FES filters start,stop = dateRange(start_date,stop_date) bbox = fes.BBox(box) or_filt = fes.Or([fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val), escapeChar='\\',wildCard='*',singleChar='?') for val in model_name_list]) val = 'Averages' not_filt = fes.Not([fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val), escapeChar='\\',wildCard='*',singleChar='?')]) filter_list = [fes.And([ bbox, start, stop, or_filt, not_filt]) ] # <markdowncell> # ##Find model results at NODC
def query_csw(self, identifier_list=None, alt_identifier_list=None, keyword_list=None, bounding_box=None, bounding_box_crs=None, anytext_list=None, titleword_list=None, start_datetime=None, stop_datetime=None, record_type_list=None, max_total_records=None, get_layers=None): ''' Function to query CSW using AND combination of provided search parameters and return generator object yielding nested dicts containing information about each record including distributions @param identifier_list: List of strings or comma-separated string containing metadata identifiers (UUID) @param alt_identifier: List of strings or comma-separated string containing metadata alternate identifiers (eCat ID) @param keyword_list: List of strings or comma-separated string containing keyword search terms @param bounding_box: Bounding box to search as a list of ordinates [bbox.minx, bbox.minx, bbox.maxx, bbox.maxy] @param bounding_box_crs: Coordinate reference system for bounding box. Defaults to value of self.settings['DEFAULT_CRS'] @param anytext_list: List of strings or comma-separated string containing any text search terms @param titleword: List of strings or comma-separated string containing title search terms @param start_datetime: Datetime object defining start of temporal search period @param stop_datetime: Datetime object defining end of temporal search period @param record_type_list: List of strings or comma-separated string containing record type(s) to return @param max_total_records: Maximum total number of records to return. Defaults to value of self.settings['DEFAULT_MAXTOTALRECORDS'] @param get_layers: Boolean flag indicating whether to get WMS/WCS layer names. Defaults to value of self.settings['DEFAULT_GET_LAYERS'] @return: generator object yielding nested dicts containing information about each record including distributions ''' bounding_box_crs = bounding_box_crs or self.settings['DEFAULT_CRS'] get_layers = self.settings[ 'DEFAULT_GET_LAYERS'] if get_layers is None else get_layers # Convert strings to lists if required if type(identifier_list) == str: identifier_list = self.list_from_comma_separated_string( identifier_list) if type(alt_identifier_list) == str: alt_identifier_list = self.list_from_comma_separated_string( alt_identifier_list) if type(keyword_list) == str: keyword_list = self.list_from_comma_separated_string(keyword_list) if type(anytext_list) == str: anytext_list = self.list_from_comma_separated_string(anytext_list) if type(titleword_list) == str: titleword_list = self.list_from_comma_separated_string( titleword_list) record_type_list = record_type_list or self.settings[ 'DEFAULT_RECORD_TYPES'] if type(record_type_list) == str: record_type_list = self.list_from_comma_separated_string( record_type_list) # Build filter list fes_filter_list = [] if identifier_list: if len(identifier_list) == 1: fes_filter_list += [ fes.PropertyIsLike(propertyname='Identifier', literal=identifier_list[0], matchCase=False) ] else: fes_filter_list.append( fes.Or([ fes.PropertyIsLike(propertyname='Identifier', literal=identifier, matchCase=False) for identifier in identifier_list ])) if alt_identifier_list: if len(alt_identifier_list) == 1: fes_filter_list += [ fes.PropertyIsLike(propertyname='AlternateIdentifier', literal=alt_identifier_list[0], matchCase=False) ] else: fes_filter_list.append( fes.Or([ fes.PropertyIsLike(propertyname='AlternateIdentifier', literal=alt_identifier, matchCase=False) for alt_identifier in alt_identifier_list ])) # Check for unchanged, upper-case, lower-case and capitalised keywords # with single-character wildcards substituted for whitespace characters # GeoNetwork keyword search is always case sensitive if keyword_list: fes_filter_list += self.any_case_match_filters( 'Subject', keyword_list) if anytext_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='anyText', literal=phrase, matchCase=False) for phrase in anytext_list ] if start_datetime or stop_datetime: fes_filter_list += self.get_date_filter(start_datetime, stop_datetime) if titleword_list: fes_filter_list += [ fes.PropertyIsLike(propertyname='title', literal=titleword, matchCase=False) for titleword in titleword_list ] # Check for unchanged, upper-case, lower-case and capitalised keywords # with single-character wildcards substituted for whitespace characters # GeoNetwork type search is always case sensitive if record_type_list: fes_filter_list += self.any_case_match_filters( 'type', record_type_list) if bounding_box: # N.B: Bounding box ordinate ordering must match CRS. Default CRS84 supports lon-lat ordering, not lat-lon # See https://gis.stackexchange.com/questions/124050/how-do-i-specify-the-lon-lat-ordering-in-csw-bounding-box-request fes_filter_list += [fes.BBox(bounding_box, crs=bounding_box_crs)] assert fes_filter_list, 'No search criteria defined' # Use single filter if no "and" required if len(fes_filter_list) == 1: fes_filter_list = fes_filter_list[0] # Return generator object return self.get_csw_records(fes_filter_list, max_total_records=max_total_records, get_layers=get_layers)
def make_bbox(self): crs = 'urn:ogc:def:crs:OGC:1.3:CRS84' self.bbox_crs = fes.BBox([self.roi[0], self.roi[2], self.roi[1], self.roi[3]], crs=crs)
#jd_stop = jd_now + dt.timedelta(days=3) start_date = jd_start.strftime('%Y-%m-%d %H:00') stop_date = jd_stop.strftime('%Y-%m-%d %H:00') jd_start = dt.datetime.strptime(start_date,'%Y-%m-%d %H:%M') jd_stop = dt.datetime.strptime(stop_date,'%Y-%m-%d %H:%M') print start_date,'to',stop_date # <codecell> start,stop = dateRange(start_date,stop_date) filter1 = fes.PropertyIsLike(propertyname='apiso:AnyText',literal=('*%s*' % val), escapeChar='\\',wildCard='*',singleChar='?') bbox = fes.BBox(box,crs='urn:ogc:def:crs:OGC:1.3:CRS84') #filter_list = [fes.And([ bbox, filter1, start,stop]) ] # <codecell> filter_list = [fes.And([ bbox, filter1]) ] csw.getrecords2(constraints=filter_list) csw.results['matches'] # <codecell> filter_list = [fes.And([ bbox, filter1, start,stop]) ] csw.getrecords2(constraints=filter_list) csw.results['matches'] # <codecell>
escapeChar='\\', singleChar='?', propertyname='apiso:AnyText') or_filt = fes.Or( [fes.PropertyIsLike(literal=('*%s*' % val), **kw) for val in model_names]) kw = dict(wildCard='*', escapeChar='\\', singleChar='?', propertyname='apiso:ServiceType') serviceType = fes.PropertyIsLike(literal=('*%s*' % service_type), **kw) begin, end = fes_date_filter(start, stop) bbox_crs = fes.BBox(bbox, crs=crs) filter_list = [ fes.And([ bbox_crs, # bounding box begin, end, # start and end date or_filt, # or conditions (CF variable names) serviceType # search only for datasets that have WMS services ]) ] # In[9]: from owslib.csw import CatalogueServiceWeb
def get_data(self, typename, **kwargs): """ Download WOUDC observations :param bbox: a list representing a bounding box spatial filter (`minx, miny, maxx, maxy`) :param temporal: a list of two elements representing a time period (start, end) which accepts the following types: - :py:class:`datetime.date` - :py:class:`datetime.datetime` - string date (e.g. ``2012-10-30``) - string datetime (e.g. ``2012-10-30 11:11:11``) :param property_name: a string representing the property name to apply as filter against :param property_value: a string representing the value which filters against `property_name` :param sort_property: a string representing the property on which to sort results (default ``instance_datetime``) :param sort_descending: a boolean of whether to sort descending (default is ``False``). Applied if `sort_property` is specified :returns: list of WOUDC observations GeoJSON payload """ constraints = [] variables = [] filter_string = None bbox = None temporal = None property_name = None property_value = None sort_property = None sort_descending = False startindex = 0 output = [] LOGGER.info('Downloading dataset %s', typename) LOGGER.debug('Assembling query parameters') for key, value in kwargs.iteritems(): if key == 'bbox': bbox = value if key == 'temporal': temporal = value if key == 'property_name': property_name = value if key == 'property_value': property_value = str(value) if key == 'variables': variables = value if key == 'sortby': sort_property = value if key == 'sort_descending': sort_descending = value LOGGER.debug('Assembling constraints') if property_name is not None and property_value is not None: constraints.append(fes.PropertyIsEqualTo(property_name, property_value)) if bbox is not None: if not isinstance(bbox, list) or len(bbox) != 4: raise ValueError('bbox must be list of minx, miny, maxx, maxy') LOGGER.debug('Setting spatial constraint') constraints.append(fes.BBox(bbox)) if temporal is not None: if not isinstance(temporal, list) or len(temporal) != 2: msg = 'temporal must be list of start date, end date' raise ValueError(msg) LOGGER.info('Setting temporal constraint') temporal_start = date2string(temporal[0], 'begin') temporal_end = date2string(temporal[1], 'end') constraints.append(fes.PropertyIsBetween( 'instance_datetime', temporal_start, temporal_end)) if sort_descending is not None: if not isinstance(sort_descending, bool): raise ValueError('sort_descending must be boolean') if constraints: LOGGER.debug('Combining constraints') flt = fes.FilterRequest() if len(constraints) == 1: LOGGER.debug('Single constraint') filter_string = flt.setConstraint(constraints[0], tostring=True) if len(constraints) > 1: LOGGER.debug('Multiple constraints') filter_string = flt.setConstraintList([constraints], tostring=True) LOGGER.info('Fetching observations') LOGGER.info('Filters:') LOGGER.info('bbox: %r', bbox) LOGGER.info('temporal: %r', temporal) LOGGER.info('attribute query: %r = %r', property_name, property_value) # page download and assemble single list of JSON features while True: LOGGER.debug('Fetching features %d - %d', startindex, startindex + self.maxfeatures) payload = self.server.getfeature( typename=typename, startindex=startindex, propertyname=variables, maxfeatures=self.maxfeatures, filter=filter_string, outputFormat=self.outputformat).read() LOGGER.debug('Processing response') if payload.isspace(): LOGGER.debug('Empty response. Exiting') break try: features = json.loads(payload)['features'] except ValueError: msg = 'Query produced no results' LOGGER.info(msg) return None len_features = len(features) LOGGER.debug('Found %d features', len_features) output.extend(features) if len_features < self.maxfeatures: break startindex = startindex + self.maxfeatures LOGGER.info('Found %d features', len(output)) if sort_property is not None: LOGGER.info('Sorting response by %s', sort_property) output.sort(key=lambda e: e['properties'][sort_property], reverse=sort_descending) return output