class NwisLoader(WebserviceLoader):
"""
Base class for loading data from the USGS NWIS REST services.
"""
# Map WebserviceLoader options to NWIS equivalents
start_date = DateOpt(url_param='startDT')
end_date = DateOpt(url_param='endDT')
state = FilterOpt(url_param='stateCd')
county = FilterOpt(url_param='countyCd', multi=True)
basin = FilterOpt(url_param='huc', multi=True)
station = FilterOpt(url_param='site', multi=True)
parameter = FilterOpt(url_param='parameterCd', multi=True)
# Additional options unique to NWIS
sitetype = ChoiceOpt(
url_param='siteType',
multi=True,
choices=list(SITE_TYPES.keys()),
)
# Each NWIS webservice uses the same base URL, with a service path
service = None
@property
def url(self):
return "http://waterservices.usgs.gov/nwis/%s/" % self.service
class AcisIO(WebserviceLoader, JsonParser, TupleMapper, BaseIO):
"""
Base class for loading data from ACIS web services
See http://data.rcc-acis.org/doc/
"""
path = None # ACIS web service path
# (Re-)define some default WebserviceLoader options
state = FilterOpt(multi=True)
county = FilterOpt(multi=True)
basin = FilterOpt(multi=True)
station = FilterOpt(ignored=True)
# Additional ACIS-specific option
meta = ChoiceOpt(
multi=True,
choices=ALL_META_FIELDS,
default=DEFAULT_META_FIELDS,
)
@property
def url(self):
"""
URL for wq.io.loaders.NetLoader
"""
return "http://data.rcc-acis.org/%s" % self.path
def serialize_params(self, params, complex):
if complex:
# ACIS web service supports JSON object as "params" parameter
nparams = {}
for key, val in params.items():
url_param = self.get_url_param(key)
if len(val) == 1 and isinstance(val[0], basestring):
val = val[0]
nparams[url_param] = val
return {'params': json.dumps(nparams)}
else:
# Simpler queries can use traditional URL parameters
return super(AcisIO, self).serialize_params(params)
class CocorahsIO(WebserviceLoader, XmlParser, TimeSeriesMapper, BaseIO):
"""
Retrieves CoCoRaHS observations from data.cocorahs.org
Usage:
data = CocorahsIO(state='MN', county='HN')
for row in data:
print row.stationname, row.observationdate.date(), row.totalprecipamt
"""
# Customize date parameters
start_date = DateOpt(
required=True,
date_only=False,
url_param="StartDate",
)
end_date = DateOpt(
date_only=False,
url_param="EndDate",
)
# These region filters are supported
state = FilterOpt(required=True)
county = FilterOpt()
# Other filters are ignored
basin = FilterOpt(ignored=True)
station = FilterOpt(ignored=True)
parameter = FilterOpt(ignored=True)
# CoCoRaHS-specific options
datetype = ChoiceOpt(
url_param="ReportDateType",
default="reportdate",
choices=["reportdate", "timestamp"],
)
reporttype = ChoiceOpt(
url_param="ReportType",
default="Daily",
choices=["Daily", "MultiDay"],
)
# Configuration for wq.io base classes
url = "http://data.cocorahs.org/cocorahs/export/exportreports.aspx"
root_tag = 'Cocorahs'
date_formats = [
'%Y-%m-%d %I:%M %p', '%Y-%m-%d', '%I:%M %p', '%m/%d/%Y %I:%M %p'
]
key_fields = [
"stationnumber",
"stationname",
"latitude",
"longitude",
"datetimestamp",
"observationdate",
"observationtime",
"entrydatetime",
]
# These params apply to every request
default_params = {
'dtf': "1",
'Format': "XML",
'TimesInGMT': "False",
'responsefields': "all"
}
@property
def item_tag(self):
if self.getvalue('reporttype') == "Daily":
return 'DailyPrecipReports/DailyPrecipReport'
else:
# i.e. self.getvalue('reporttype') == "MultiDay"
return 'MultiDayPrecipReports/MultiDayPrecipReport'
def serialize_params(self, params, complex):
params = super(CocorahsIO, self).serialize_params(params, complex)
fmt = '%m/%d/%Y'
# Different date parameters and formats depending on use case
if 'EndDate' in params:
# Date range (usually used with datetype=reportdate)
params['StartDate'] = self.getvalue('start_date').strftime(fmt)
params['EndDate'] = self.getvalue('end_date').strftime(fmt)
else:
# Only start date (usually used with datetype=timestamp)
params['Date'] = self.getvalue('start_date').strftime(fmt +
" %I:%M %p")
del params['StartDate']
return params
def map_value(self, field, value):
value = super(CocorahsIO, self).map_value(field, value)
# CoCoRaHS empty dates are represented as 1/1/0001
if isinstance(value, datetime) and value.year == 1:
return None
return value
class EnsembleForecastIO(ZipWebserviceLoader, EnsembleCsvParser, TupleMapper,
BaseIO):
"""
Load ensemble forecast zip files from the CNRFC website.
- start_date and basin are required to specify the zip file;
- station and end_date can be used to filter the downloaded data.
"""
nested = True
start_date = DateOpt(required=True)
end_date = DateOpt()
# Region filters
state = FilterOpt(ignored=True)
county = FilterOpt(ignored=True)
# FIXME: this isn't actually a HUC8 basin
basin = FilterOpt(required=True)
station = FilterOpt(multi=True)
parameter = FilterOpt(ignored=True)
region = ChoiceOpt(default="cnrfc", choices=["cnrfc"])
urls = {
"cnrfc": ("http://www.cnrfc.noaa.gov/csv/" +
"{date}12_{basin}_hefs_csv_daily.zip")
}
@property
def params(self):
# Don't actually need params, but ensure validation logic is called
params = super(EnsembleForecastIO, self).params
return None
@property
def url(self):
url = self.urls[self.getvalue("region")]
return url.format(
date=self.getvalue("start_date").strftime("%Y%m%d"),
basin=self.getvalue("basin"),
)
def parse(self):
super(EnsembleForecastIO, self).parse()
# Optionally filter by station id
site_filter = self.getvalue('station')
date_filter = self.getvalue('end_date')
if not site_filter:
return
self.data = [item for item in self.data if item['site'] in site_filter]
if not date_filter:
return
date_filter = date_filter.strftime('%Y-%m-%d') + " 23:59:59"
for item in self.data:
item['data'] = [
row for row in item['data'] if row['date'] <= date_filter
]
def usable_item(self, item):
item = item.copy()
item['data'] = TimeSeriesIO(data=item['data'])
return super(EnsembleForecastIO, self).usable_item(item)
class StationDataIO(StationMetaIO):
"""
Retrieve daily time series data from the climate stations in a region.
See http://data.rcc-acis.org/doc/#title19
"""
nested = True
namespace = "data" # For wq.io.parsers.text.JsonParser
path = "MultiStnData"
# Specify ACIS-defined URL parameters for start/end date
start_date = DateOpt(required=True, url_param='sdate')
end_date = DateOpt(required=True, url_param='edate')
parameter = ParameterOpt(required=True)
# Additional information for daily results
add = ChoiceOpt(multi=True, choices=ADD_IDS)
def get_field_names(self):
"""
ACIS web service returns "meta" and "data" for each station;
Use meta attributes as field names
"""
field_names = super(StationDataIO, self).get_field_names()
if field_names == ['meta', 'data']:
meta_fields = self.data[0]['meta'].keys()
if set(meta_fields) < set(self.getvalue('meta')):
meta_fields = self.getvalue('meta')
field_names = list(meta_fields) + ['data']
return field_names
def serialize_params(self, params, complex):
# If set, apply "add" option to each requested element / parameter
# (Rather than as a top-level URL param)
if 'add' in params:
complex = True
elems = []
for elem in params.get('parameter', []):
if not isinstance(elem, dict):
elem = {'name': elem}
elem['add'] = ",".join(params['add'])
elems.append(elem)
params['parameter'] = elems
del params['add']
return super(StationDataIO, self).serialize_params(params, complex)
def usable_item(self, data):
"""
ACIS web service returns "meta" and "data" for each station; use meta
attributes as item values, and add an IO for iterating over "data"
"""
# Use metadata as item
item = data['meta']
# Add nested IO for data
elems, elems_is_complex = self.getlist('parameter')
if elems_is_complex:
elems = [elem['name'] for elem in elems]
add, add_is_complex = self.getlist('add')
item['data'] = DataIO(
data=data['data'],
parameter=elems,
add=add,
start_date=self.getvalue('start_date'),
end_date=self.getvalue('end_date'),
)
# TupleMapper will convert item to namedtuple
return super(StationDataIO, self).usable_item(item)