# endpoint = 'http://geo.gov.ckan.org/csw' # data.gov
# endpoint = 'https://data.noaa.gov/csw' # data.noaa.gov
# endpoint = 'http://www.nodc.noaa.gov/geoportal/csw' # nodc
endpoint = "http://www.ngdc.noaa.gov/geoportal/csw" # NGDC Geoportal
csw = CatalogueServiceWeb(endpoint, timeout=60)
for oper in csw.operations:
if oper.name == "GetRecords":
cnstr = oper.constraints["SupportedISOQueryables"]["values"]
print ("\nISO Queryables:%s\n" % "\n".join(cnstr))
# <codecell>
# convert User Input into FES filters
start, stop = fes_date_filter(start_date, stop_date)
bbox = fes.BBox(bounding_box)
# use the search name to create search filter
or_filt = fes.Or(
[
fes.PropertyIsLike(
propertyname="apiso:AnyText", literal=("*%s*" % val), escapeChar="\\", wildCard="*", singleChar="?"
)
for val in data_dict["currents"]["u_names"]
]
)
val = "Averages"
not_filt = fes.Not(
[
# In[ ]:
from owslib import fes
from utilities import fes_date_filter
kw = dict(wildCard='*',
escapeChar='\\',
singleChar='?',
propertyname='apiso:AnyText')
or_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw)
for val in name_list])
not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)])
begin, end = fes_date_filter(start, stop)
filter_list = [fes.And([fes.BBox(bbox), begin, end, or_filt, not_filt])]
# In[ ]:
from owslib.csw import CatalogueServiceWeb
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw'
csw = CatalogueServiceWeb(endpoint, timeout=60)
csw.getrecords2(constraints=filter_list, maxrecords=1000, esn='full')
log.info(fmt(' Catalog information '))
log.info("URL: {}".format(endpoint))
log.info("CSW version: {}".format(csw.version))
log.info("Number of datasets available: {}".format(len(csw.records.keys())))
for oper in csw.operations:
if oper.name == 'GetRecords':
print('\nISO Queryables:\n%s' %
'\n'.join(oper.constraints['SupportedISOQueryables']['values']))
# Put the names in a dict for ease of access.
data_dict = {}
data_dict["water"] = {"names": name_list, "sos_name": sos_name}
# <markdowncell>
# #### Convert User Input into FES filters.
# <codecell>
start, stop = fes_date_filter(start_date, end_date)
box = []
box.append(bounding_box[0][0])
box.append(bounding_box[0][1])
box.append(bounding_box[1][0])
box.append(bounding_box[1][1])
bbox = fes.BBox(box)
or_filt = fes.Or([
fes.PropertyIsLike(propertyname='apiso:AnyText',
literal=('*%s*' % val),
escapeChar='\\',
wildCard='*',
singleChar='?') for val in name_list
])
val = 'Averages'
'SST'],
"sos_name":["sea_water_temperature"]}
# <headingcell level=3>
# Search CSW for datasets of interest
# <codecell>
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal
csw = CatalogueServiceWeb(endpoint,timeout=60)
# <codecell>
# convert User Input into FES filters
start, stop = fes_date_filter(start_date,end_date)
bbox = fes.BBox(bounding_box)
#use the search name to create search filter
or_filt = fes.Or([fes.PropertyIsLike(propertyname='apiso:AnyText',
literal='*%s*' % val,
escapeChar='\\',
wildCard='*',
singleChar='?') for val in data_dict["temp"]["names"]])
# <markdowncell>
# <div class="warning"><strong>ROMS model output often has Averages and History files. </strong> The Averages files are usually averaged over a tidal cycle or more, while the History files are snapshots at that time instant. We are not interested in averaged data for this test, so in the cell below we remove any Averages files here by removing any datasets that have the term "Averages" in the metadata text. A better approach would be to look at the `cell_methods` attributes propagated through to some term in the ISO metadata, but this is not implemented yet, as far as I know </div>
# <codecell>
# ### Check the CSW endpoints for wind data in the date range specified
# <markdowncell>
# <div class="warning"><strong>Data discovery is limited</strong> - Most of the CSW endpoints don't have recent wind data available.</div>
# <codecell>
for endpoint in bbox_endpoints:
print endpoint
csw = CatalogueServiceWeb(endpoint, timeout=60)
# convert User Input into FES filters
start, stop = fes_date_filter(start_date, stop_date)
bbox = fes.BBox(bounding_box)
#use the search name to create search filter
or_filt = fes.Or([
fes.PropertyIsLike(propertyname='apiso:AnyText',
literal='*%s*' % val,
escapeChar='\\',
wildCard='*',
singleChar='?')
for val in data_dict['winds']['u_names']
])
filter_list = [fes.And([bbox, start, stop, or_filt])]
# filter_list = [fes.And([ bbox, or_filt]) ]
# connect to CSW, explore it's properties
kw = dict(wildCard='*',
escapeChar='\\',
singleChar='?',
propertyname='apiso:AnyText')
or_filt = fes.Or([fes.PropertyIsLike(literal=('*%s*' % val), **kw)
for val in name_list])
# Exclude ROMS Averages and History files.
not_filt = fes.Not([fes.PropertyIsLike(literal='*Averages*', **kw)])
bbox_filter = fes.BBox(bbox,crs='urn:ogc:def:crs:OGC:1.3:CRS84')
begin, end = fes_date_filter(start, stop)
filter_list = [fes.And([bbox_filter, begin, end, or_filt, not_filt])]
# In[ ]:
bbox = [-87.40, 34.25, -63.70, 66.70] # [lon_min, lat_min, lon_max, lat_max]
bbox_filter = fes.BBox(bbox,crs='urn:ogc:def:crs:OGC:1.3:CRS84')
filter_list = [fes.And([filter1, bbox_filter])]
csw.getrecords2(constraints=filter_list, maxrecords=1000)
# In[141]:
csw.get_operation_by_name('GetRecords').constraints
