class NwisLoader(WebserviceLoader):
"""
Base class for loading data from the USGS NWIS REST services.
"""
# Map WebserviceLoader options to NWIS equivalents
start_date = DateOpt(url_param='startDT')
end_date = DateOpt(url_param='endDT')
state = FilterOpt(url_param='stateCd')
county = FilterOpt(url_param='countyCd', multi=True)
basin = FilterOpt(url_param='huc', multi=True)
station = FilterOpt(url_param='site', multi=True)
parameter = FilterOpt(url_param='parameterCd', multi=True)
# Additional options unique to NWIS
sitetype = ChoiceOpt(
url_param='siteType',
multi=True,
choices=list(SITE_TYPES.keys()),
)
# Each NWIS webservice uses the same base URL, with a service path
service = None
@property
def url(self):
return "http://waterservices.usgs.gov/nwis/%s/" % self.service
class HydroForecastIO(WebserviceLoader, XmlParser, TimeSeriesMapper, BaseIO):
"""
Loads hydrograph forecast data (next 3 days) from weather.gov
"""
###########################################
# valid is the time of the forecast in UTC
# primary is the stage in ft
# secondary is the flow in kcfs
# It appears to return 3 days forecast in the future
###########################################
start_date = DateOpt(ignored=True)
end_date = DateOpt(ignored=True)
state = FilterOpt(ignored=True)
county = FilterOpt(ignored=True)
station = FilterOpt(url_param='gage')
parameter = FilterOpt(ignored=True)
basin = FilterOpt(ignored=True)
root_tag = "forecast"
date_formats = ['%Y-%m-%dT%H:%M:%S']
url = 'http://water.weather.gov/ahps2/hydrograph_to_xml.php'
def parse_item(self, elem):
valid = elem.find('valid')
primary = elem.find('primary')
secondary = elem.find('secondary')
return {
'date': valid.text.replace('-00:00', ''),
primary.attrib['name']: primary.text,
secondary.attrib['name']: secondary.text,
}
class ForecastIO(SnotelIO):
data_function = 'getForecasts'
station = FilterOpt(required=True, url_param='stationTriplet')
parameter = FilterOpt(required=True, url_param='elementCd')
forecast_period = FilterOpt(required=True, url_param='forecastPeriod')
class HourlyDataIO(TimeSeriesMapper, SnotelIO):
"""
Wrapper for getHourlyData(), used internally by StationHourlyDataIO
"""
data_function = 'getHourlyData'
# TimeSeriesMapper configuration
date_formats = ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M']
# Applicable WebserviceLoader default options
station = FilterOpt(required=True, url_param='stationTriplets')
parameter = FilterOpt(required=True, url_param='elementCd')
start_date = DateOpt(required=True, url_param='beginDate')
end_date = DateOpt(required=True, url_param='endDate')
# Additional options
begin_hour = FilterOpt(url_param='beginHour')
end_hour = FilterOpt(url_param='endHour')
# HeightDepth parameters don't seem to be necessary.
default_params = {
'ordinal': 1,
}
def load(self):
super(HourlyDataIO, self).load()
if self.data and 'values' in self.data[0]:
self.data = as_list(self.data[0]['values'])
else:
raise NoData
class ForecastDataIO(SnotelIO):
data_function = 'getForecast'
station = FilterOpt(required=True, url_param='stationTriplets')
parameter = FilterOpt(required=True, url_param='elementCd')
forecast_period = FilterOpt(required=True, url_param='forecastPeriod')
publication_date = DateOpt(required=True, url_param='publicationDate')
class MultiStationDailyIO(HydrometLoader, TupleMapper, BaseIO):
"""
Retrieves daily values for one or more USBR Hydromet/Agrimet sites.
(Internally calls DailyDataIO for each site.)
Usage:
data = MultiStationDailyIO(station=['ACAO'], parameter=['GD','QD'])
for s in data:
print s.station
for row in s.data:
print row.date, row.gd, row.qd
"""
nested = True
station = FilterOpt(required=True, multi=True)
# Customize load function with nested IOs
def load(self):
self.data = [{
'station': station,
'data': DailyDataIO(
station=station,
parameter=self.getvalue('parameter'),
start_date=self.getvalue('start_date'),
end_date=self.getvalue('end_date'),
debug=self.debug,
)
} for station in self.getvalue('station')]
def parse(self):
pass
class SnotelIO(WebserviceLoader, BaseParser, TupleMapper, BaseIO):
"""
Base class for accessing SNOTEL AWDB SOAP web services.
"""
webservice_name = "awdbWebService"
data_function = None
# Override Default WebserviceLoader options
start_date = DateOpt(ignored=True)
end_date = DateOpt(ignored=True)
state = FilterOpt(ignored=True)
basin = FilterOpt(ignored=True)
county = FilterOpt(ignored=True)
station = FilterOpt(ignored=True)
parameter = FilterOpt(ignored=True)
def load(self):
if self.debug:
self.print_debug()
params = self.params
fn = getattr(get_server(), self.data_function)
self.data = fn(**params)
if len(self.data) == 0:
self.data = []
else:
self.data = as_list(self.data)
if isinstance(self.data[0], SudsObject):
parse = asdict
else:
parse = str
self.data = [parse(row) for row in self.data]
# Some records may have additional fields; loop through entire
# array to ensure all field names are accounted for. (Otherwise BaseIO
# will guess field names using only the first record.)
scan_fields = True
def print_debug(self):
print('%s.%s(%s)' % (
self.webservice_name,
self.data_function,
','.join(
'%s=%s' % (key, val)
for key, val in self.params.items()
)
))
class HydrometLoader(WebserviceLoader):
"""
Shared options for Hydromet IO classes.
"""
# start_date and end_date are the same as WebserviceLoader defaults
# None of the default region filters will work
state = FilterOpt(ignored=True)
county = FilterOpt(ignored=True)
basin = FilterOpt(ignored=True)
# Instead, specify a four-letter Hydromet station code (one per request)
station = FilterOpt(required=True)
# Hydromet parameter codes are also required (multiple allowed)
parameter = FilterOpt(required=True, multi=True)
class StationMetaIO(SnotelIO):
"""
Wrapper for getStationMetadata() - used internally by StationIO.
"""
data_function = 'getStationMetadata'
station = FilterOpt(required=True, url_param='stationTriplet')
class RegionDailyDataIO(StationIO):
"""
All-in-one IO for loading site metadata and daily data for a region (i.e. a
state, county, or basin). Internally calls:
- getStations()
- getStationMetadata()
- getStationElements()
- getData()
The outer IO is a list of sites in the region - derived from StationIO, but
with an extra "data" property on each station pointing to an inner time
series IO for each site. The inner IO is based on StationDailyDataIO but
flattened to avoid multiple levels of nesting. parameter is optional but
recommended (otherwise all available data for all sites will be returned).
Usage:
sites = RegionDailyDataIO(
basin='17060105',
start_date='2014-07-01',
end_date='2014-07-31',
parameter='TAVG',
)
for site in sites:
print site.name
for row in site.data:
print " ", row.date, row.value, row.storedunitcd
"""
nested = True
# Applicable WebserviceLoader default options
start_date = DateOpt(required=True)
end_date = DateOpt(required=True)
parameter = FilterOpt(url_param='elementCds')
@property
def params(self):
params = super(RegionDailyDataIO, self).params
# Start and end date are actually only used by inner io.
del params['start_date']
del params['end_date']
return params
def load(self):
super(RegionDailyDataIO, self).load()
for station in self.data:
station['data'] = flattened(
StationDailyDataIO,
station=station['stationTriplet'],
start_date=self.getvalue('start_date'),
end_date=self.getvalue('end_date'),
parameter=self.getvalue('parameter'),
debug=self.debug,
)
class StationMetaMultipleIO(SnotelIO):
"""
Wrapper for getStationMetadataMultiple() - used internally by StationIO.
"""
data_function = 'getStationMetadataMultiple'
stations = FilterOpt(required=True,
url_param='stationTriplets',
multi=True)
class AcisIO(WebserviceLoader, JsonParser, TupleMapper, BaseIO):
"""
Base class for loading data from ACIS web services
See http://data.rcc-acis.org/doc/
"""
path = None # ACIS web service path
# (Re-)define some default WebserviceLoader options
state = FilterOpt(multi=True)
county = FilterOpt(multi=True)
basin = FilterOpt(multi=True)
station = FilterOpt(ignored=True)
# Additional ACIS-specific option
meta = ChoiceOpt(
multi=True,
choices=ALL_META_FIELDS,
default=DEFAULT_META_FIELDS,
)
@property
def url(self):
"""
URL for wq.io.loaders.NetLoader
"""
return "http://data.rcc-acis.org/%s" % self.path
def serialize_params(self, params, complex):
if complex:
# ACIS web service supports JSON object as "params" parameter
nparams = {}
for key, val in params.items():
url_param = self.get_url_param(key)
if len(val) == 1 and isinstance(val[0], basestring):
val = val[0]
nparams[url_param] = val
return {'params': json.dumps(nparams)}
else:
# Simpler queries can use traditional URL parameters
return super(AcisIO, self).serialize_params(params)
class StationDataIO(StationElementIO):
"""
Base class for StationDailyDataIO and StationHourlyDataIO. Retrieves all
data for a station that matches the specified duration by calling the
specified inner_io_class.
"""
nested = True
# Applicable WebserviceLoader default options
start_date = DateOpt(url_param='beginDate', required=True)
end_date = DateOpt(url_param='endDate', required=True)
parameter = FilterOpt()
inner_io_class = None
duration = None
@property
def params(self):
params = super(StationDataIO, self).params
# Parameter filter (if any) is applied *after* the initial request
params.pop('parameter', None)
return params
def load(self):
super(StationDataIO, self).load()
data = []
for row in self.data:
# Only include records matching the specified duration
# and parameter
if row['duration'] != self.duration:
continue
elem = self.getvalue('parameter')
if elem and row['elementCd'] != elem:
continue
# getStationElements() sometimes returns parameters that don't
# actually have data for the requested timeframe - silently catch
# the exception and remove parameter from results.
try:
row['data'] = self.inner_io_class(
station=row['stationTriplet'],
parameter=row['elementCd'],
start_date=self.getvalue('start_date'),
end_date=self.getvalue('end_date'),
debug=self.debug,
)
except NoData:
continue
data.append(row)
self.data = data
class AgrimetRecentIO(InstantDataIO):
"""
Load recent Agrimet data (all available parameters)
Usage:
data = AgrimetRecentIO(station='ABEI')
for row in data:
print row.datetime, row.ob, row.wd
"""
script = "agrimet.pl"
start_date = FilterOpt(ignored=True)
end_date = FilterOpt(ignored=True)
parameter = FilterOpt(ignored=True)
@property
def params(self):
return urlencode(
OrderedDict([('cbtt', self.getvalue('station')),
('interval', 'instant'), ('format', 2),
('back', 360)]))
class StationIO(SnotelIO):
"""
Retrieve metadata for all stations in a region. Leverages both
getStations() and getStationMetadata().
"""
data_function = 'getStations'
# Applicable WebserviceLoader default options
state = FilterOpt(url_param='stateCds', multi=True)
county = FilterOpt(url_param='countyNames', multi=True)
basin = FilterOpt(url_param='hucs', multi=True)
parameter = FilterOpt(url_param='elementCds', multi=True)
# Additional options
min_latitude = FilterOpt(url_param='minLatitude')
max_latitude = FilterOpt(url_param='maxLatitude')
min_elevation = FilterOpt(url_param='minElevation')
max_elevation = FilterOpt(url_param='maxElevation')
ordinals = FilterOpt(url_param='ordinals')
# This is not the same as station (stationTriplet)
station_ids = FilterOpt(url_param='stationIds', multi=True)
# heightDepths = FilterOpt(url_param='heightDepths')
# This parameter is submitted as
# <heightDepths><value>value</value><unitCd>unit</unitCd></heightDepths>
# Left out since it doesn't seem important and isn't well-documented
default_params = {
'logicalAnd': 'true',
}
def load(self):
super(StationIO, self).load()
self.data = [
StationMetaIO(station=station, debug=self.debug).data[0]
for station in self.data
]
class HourlyDataIO(TimeSeriesMapper, SnotelIO):
"""
Wrapper for getHourlyData(), used internally by StationHourlyDataIO
"""
data_function = 'getHourlyData'
# TimeSeriesMapper configuration
date_formats = ['%Y-%m-%d %H:%M:%S', '%Y-%m-%d %H:%M']
# Applicable WebserviceLoader default options
station = FilterOpt(required=True, url_param='stationTriplets', multi=True)
parameter = FilterOpt(required=True, url_param='elementCd')
start_date = DateOpt(required=True, url_param='beginDate')
end_date = DateOpt(required=True, url_param='endDate')
# Additional options
begin_hour = FilterOpt(url_param='beginHour')
end_hour = FilterOpt(url_param='endHour')
# HeightDepth parameters don't seem to be necessary.
default_params = {
'ordinal': 1,
}
def load(self):
super(HourlyDataIO, self).load()
d = {
} # since mutiple triplets can be used, then go through each station
for data in self.data:
stationTriplet = data['stationTriplet']
if data and 'values' in data:
data = [asdict(row) for row in as_list(data['values'])]
else:
data = []
d[stationTriplet] = data
self.data = d
class DailyDataIO(SnotelIO):
"""
Wrapper for getData(), used internally by StationDailyDataIO
"""
data_function = 'getData'
# Applicable WebserviceLoader default options
station = FilterOpt(required=True, url_param='stationTriplets')
parameter = FilterOpt(required=True, url_param='elementCd')
start_date = DateOpt(required=True, url_param='beginDate')
end_date = DateOpt(required=True, url_param='endDate')
# HeightDepth parameters don't seem to be necessary.
default_params = {
'ordinal': 1,
'duration': 'DAILY',
'getFlags': 'true',
'alwaysReturnDailyFeb29': 'false',
}
def parse(self):
data = self.data[0]
if not data or 'values' not in data:
raise NoData
bd = data['beginDate']
ed = data['endDate']
dates = fill_date_range(bd, ed, date_format='%Y-%m-%d %H:%M:%S')
vals = as_list(data['values'])
flags = as_list(data['flags'])
self.data = [{
'date': date,
'value': val,
'flag': flag
} for date, val, flag in zip(dates, vals, flags)]
class StationElementIO(SnotelIO):
"""
Wrapper for getStationElements(), incorporating element names from
getElements()
"""
data_function = 'getStationElements'
# Applicable WebserviceLoader default options
start_date = DateOpt(url_param='beginDate')
end_date = DateOpt(url_param='endDate')
station = FilterOpt(required=True, url_param='stationTriplet')
def load(self):
super(StationElementIO, self).load()
names = ElementIO.get_names()
for elem in self.data:
elem['element_name'] = names[elem['elementCd']]
class CocorahsIO(WebserviceLoader, XmlParser, TimeSeriesMapper, BaseIO):
"""
Retrieves CoCoRaHS observations from data.cocorahs.org
Usage:
data = CocorahsIO(state='MN', county='HN')
for row in data:
print row.stationname, row.observationdate.date(), row.totalprecipamt
"""
# Customize date parameters
start_date = DateOpt(
required=True,
date_only=False,
url_param="StartDate",
)
end_date = DateOpt(
date_only=False,
url_param="EndDate",
)
# These region filters are supported
state = FilterOpt(required=True)
county = FilterOpt()
# Other filters are ignored
basin = FilterOpt(ignored=True)
station = FilterOpt(ignored=True)
parameter = FilterOpt(ignored=True)
# CoCoRaHS-specific options
datetype = ChoiceOpt(
url_param="ReportDateType",
default="reportdate",
choices=["reportdate", "timestamp"],
)
reporttype = ChoiceOpt(
url_param="ReportType",
default="Daily",
choices=["Daily", "MultiDay"],
)
# Configuration for wq.io base classes
url = "http://data.cocorahs.org/cocorahs/export/exportreports.aspx"
root_tag = 'Cocorahs'
date_formats = [
'%Y-%m-%d %I:%M %p', '%Y-%m-%d', '%I:%M %p', '%m/%d/%Y %I:%M %p'
]
key_fields = [
"stationnumber",
"stationname",
"latitude",
"longitude",
"datetimestamp",
"observationdate",
"observationtime",
"entrydatetime",
]
# These params apply to every request
default_params = {
'dtf': "1",
'Format': "XML",
'TimesInGMT': "False",
'responsefields': "all"
}
@property
def item_tag(self):
if self.getvalue('reporttype') == "Daily":
return 'DailyPrecipReports/DailyPrecipReport'
else:
# i.e. self.getvalue('reporttype') == "MultiDay"
return 'MultiDayPrecipReports/MultiDayPrecipReport'
def serialize_params(self, params, complex):
params = super(CocorahsIO, self).serialize_params(params, complex)
fmt = '%m/%d/%Y'
# Different date parameters and formats depending on use case
if 'EndDate' in params:
# Date range (usually used with datetype=reportdate)
params['StartDate'] = self.getvalue('start_date').strftime(fmt)
params['EndDate'] = self.getvalue('end_date').strftime(fmt)
else:
# Only start date (usually used with datetype=timestamp)
params['Date'] = self.getvalue('start_date').strftime(fmt +
" %I:%M %p")
del params['StartDate']
return params
def map_value(self, field, value):
value = super(CocorahsIO, self).map_value(field, value)
# CoCoRaHS empty dates are represented as 1/1/0001
if isinstance(value, datetime) and value.year == 1:
return None
return value
class EnsembleForecastIO(ZipWebserviceLoader, EnsembleCsvParser, TupleMapper,
BaseIO):
"""
Load ensemble forecast zip files from the CNRFC website.
- start_date and basin are required to specify the zip file;
- station and end_date can be used to filter the downloaded data.
"""
nested = True
start_date = DateOpt(required=True)
end_date = DateOpt()
# Region filters
state = FilterOpt(ignored=True)
county = FilterOpt(ignored=True)
# FIXME: this isn't actually a HUC8 basin
basin = FilterOpt(required=True)
station = FilterOpt(multi=True)
parameter = FilterOpt(ignored=True)
region = ChoiceOpt(default="cnrfc", choices=["cnrfc"])
urls = {
"cnrfc": ("http://www.cnrfc.noaa.gov/csv/" +
"{date}12_{basin}_hefs_csv_daily.zip")
}
@property
def params(self):
# Don't actually need params, but ensure validation logic is called
params = super(EnsembleForecastIO, self).params
return None
@property
def url(self):
url = self.urls[self.getvalue("region")]
return url.format(
date=self.getvalue("start_date").strftime("%Y%m%d"),
basin=self.getvalue("basin"),
)
def parse(self):
super(EnsembleForecastIO, self).parse()
# Optionally filter by station id
site_filter = self.getvalue('station')
date_filter = self.getvalue('end_date')
if not site_filter:
return
self.data = [item for item in self.data if item['site'] in site_filter]
if not date_filter:
return
date_filter = date_filter.strftime('%Y-%m-%d') + " 23:59:59"
for item in self.data:
item['data'] = [
row for row in item['data'] if row['date'] <= date_filter
]
def usable_item(self, item):
item = item.copy()
item['data'] = TimeSeriesIO(data=item['data'])
return super(EnsembleForecastIO, self).usable_item(item)
class SnotelIO(WebserviceLoader, BaseParser, TupleMapper, BaseIO):
"""
Base class for accessing SNOTEL AWDB SOAP web services.
"""
webservice_name = "awdbWebService"
data_function = None
# Override Default WebserviceLoader options
start_date = DateOpt(ignored=True)
end_date = DateOpt(ignored=True)
state = FilterOpt(ignored=True)
basin = FilterOpt(ignored=True)
county = FilterOpt(ignored=True)
station = FilterOpt(ignored=True)
parameter = FilterOpt(ignored=True)
def load(self):
if self.debug:
self.print_debug()
params = self.params
fn = getattr(get_server(), self.data_function)
self.data = fn(**params)
if len(self.data) == 0:
self.data = []
else:
self.data = as_list(self.data)
if isinstance(self.data[0], SudsObject):
parse = asdict
else:
parse = str
self.data = [parse(row) for row in self.data]
def serialize_params(self, params, complex_type):
"""
The AWDB NRCS webservice allows for multiple parameters, need
to have the ability to query multiple stations at a time for
effeciency. If one of the parameters is a list, the client will
be able to make multiple types for the list.
Therefore, overwrite the `serialize_params` to not join vals if
it's a list
"""
if complex_type:
raise NotImplementedError("Cannot serialize %s!" % params)
else:
return {
self.get_url_param(key): val
for key, val in params.items()
}
# Some records may have additional fields; loop through entire
# array to ensure all field names are accounted for. (Otherwise BaseIO
# will guess field names using only the first record.)
scan_fields = True
def print_debug(self):
print('%s.%s(%s)' % (self.webservice_name, self.data_function,
','.join('%s=%s' % (key, val)
for key, val in self.params.items())))