def get_site_info(wsdl_url, site_code, suds_cache=("default",), timeout=None, user_cache=False):
"""
Retrieves detailed site information from a WaterOneFlow service using a
GetSiteInfo request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
site_code : str
Site code of the site you'd like to get more information for. Site codes
MUST contain the network and be of the form <network>:<site_code>, as is
required by WaterOneFlow.
suds_cache : ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
timeout : int or float
suds SOAP URL open timeout (seconds).
If unspecified, the suds default (90 seconds) will be used.
user_cache : bool
If False (default), use the system temp location to store cache WSDL and
other files. Use the default user ulmo directory if True.
Returns
-------
site_info : dict
a python dict containing site information
"""
suds_client = _get_client(wsdl_url, suds_cache, timeout, user_cache)
waterml_version = _waterml_version(suds_client)
if waterml_version == '1.0':
response = suds_client.service.GetSiteInfo(site_code)
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_0.parse_sites(response_buffer)
elif waterml_version == '1.1':
response = suds_client.service.GetSiteInfo(site_code)
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_1.parse_sites(response_buffer)
if len(sites) == 0:
return {}
site_info = list(sites.values())[0]
series_dict = dict([
(series['variable']['vocabulary'] + ':' + series['variable']['code'],
series)
for series in site_info['series']
])
site_info['series'] = series_dict
return site_info
def get_site_info(wsdl_url, site_code, suds_cache=("default",)):
"""
Retrieves detailed site information from a WaterOneFlow service using a
GetSiteInfo request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
site_code : str
Site code of the site you'd like to get more information for. Site codes
MUST contain the network and be of the form <network>:<site_code>, as is
required by WaterOneFlow.
suds_cache: ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
Returns
-------
site_info : dict
a python dict containing site information
"""
suds_client = _get_client(wsdl_url, suds_cache)
waterml_version = _waterml_version(suds_client)
if waterml_version == '1.0':
response = suds_client.service.GetSiteInfo(site_code)
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_0.parse_sites(response_buffer)
elif waterml_version == '1.1':
response = suds_client.service.GetSiteInfo(site_code)
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_1.parse_sites(response_buffer)
if len(sites) == 0:
return {}
site_info = list(sites.values())[0]
series_dict = dict([
(series['variable']['vocabulary'] + ':' + series['variable']['code'],
series)
for series in site_info['series']
])
site_info['series'] = series_dict
return site_info
def _get_site_values(service, url_params, input_file=None, methods=None):
"""downloads and parses values for a site
returns a values dict containing variable and data values
"""
if input_file is None:
query_isodate = isodate.datetime_isoformat(datetime.datetime.now())
service_url = _get_service_url(service)
try:
req = requests.get(service_url, params=url_params)
except requests.exceptions.ConnectionError:
log.info("There was a connection error with query:\n\t%s\n\t%s" % (service_url, url_params))
return {}
log.info("processing data from request: %s" % req.request.url)
if req.status_code != 200:
return {}
input_file = io.BytesIO(util.to_bytes(req.content))
else:
query_isodate = None
with _open_input_file(input_file) as content_io:
data_dict = wml.parse_site_values(content_io, query_isodate,
methods=methods)
for variable_dict in list(data_dict.values()):
variable_dict['site'] = _extract_site_properties(variable_dict['site'])
return data_dict
def _get_site_values(service, url_params, input_file=None, methods=None):
"""downloads and parses values for a site
returns a values dict containing variable and data values
"""
if input_file is None:
query_isodate = isodate.datetime_isoformat(datetime.datetime.now())
service_url = _get_service_url(service)
try:
req = requests.get(service_url, params=url_params)
except requests.exceptions.ConnectionError:
log.info("There was a connection error with query:\n\t%s\n\t%s" %
(service_url, url_params))
return {}
log.info("processing data from request: %s" % req.request.url)
if req.status_code != 200:
return {}
input_file = io.BytesIO(util.to_bytes(req.content))
else:
query_isodate = None
with _open_input_file(input_file) as content_io:
data_dict = wml.parse_site_values(content_io,
query_isodate,
methods=methods)
for variable_dict in list(data_dict.values()):
variable_dict['site'] = _extract_site_properties(
variable_dict['site'])
return data_dict
def get_sites(wsdl_url, suds_cache=("default",), timeout=None, user_cache=False):
"""
Retrieves information on the sites that are available from a WaterOneFlow
service using a GetSites request. For more detailed information including
which variables and time periods are available for a given site, use
``get_site_info()``.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
suds_cache : `None` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
timeout : int or float
suds SOAP URL open timeout (seconds).
If unspecified, the suds default (90 seconds) will be used.
user_cache : bool
If False (default), use the system temp location to store cache WSDL and
other files. Use the default user ulmo directory if True.
Returns
-------
sites_dict : dict
a python dict with site codes mapped to site information
"""
suds_client = _get_client(wsdl_url, suds_cache, timeout, user_cache)
waterml_version = _waterml_version(suds_client)
if waterml_version == '1.0':
response = suds_client.service.GetSitesXml('')
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_0.parse_site_infos(response_buffer)
elif waterml_version == '1.1':
response = suds_client.service.GetSites('')
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_1.parse_site_infos(response_buffer)
return dict([
(site['network'] + ':' + site['code'], site)
for site in list(sites.values())
])
def get_variable_info(wsdl_url, variable_code=None,
suds_cache=("default",), timeout=None, user_cache=False):
"""
Retrieves site values from a WaterOneFlow service using a GetVariableInfo
request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
variable_code : `None` or str
If `None` (default) then information on all variables will be returned,
otherwise, this should be set to the variable code of the variable you'd
like to get more information on. Variable codes MUST contain the
network and be of the form <vocabulary>:<variable_code>, as is required
by WaterOneFlow.
suds_cache : ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
timeout : int or float
suds SOAP URL open timeout (seconds).
If unspecified, the suds default (90 seconds) will be used.
user_cache : bool
If False (default), use the system temp location to store cache WSDL and
other files. Use the default user ulmo directory if True.
Returns
-------
variable_info : dict
a python dict containing variable information. If no variable code is
`None` (default) then this will be a nested set of dicts keyed by
<vocabulary>:<variable_code>
"""
suds_client = _get_client(wsdl_url, suds_cache, timeout, user_cache)
waterml_version = _waterml_version(suds_client)
response = suds_client.service.GetVariableInfo(variable_code)
response_buffer = io.BytesIO(util.to_bytes(response))
if waterml_version == '1.0':
variable_info = waterml.v1_0.parse_variables(response_buffer)
elif waterml_version == '1.1':
variable_info = waterml.v1_1.parse_variables(response_buffer)
if not variable_code is None and len(variable_info) == 1:
return list(variable_info.values())[0]
else:
return dict([
('%s:%s' % (var['vocabulary'], var['code']), var)
for var in list(variable_info.values())
])
def get_sites(wsdl_url, suds_cache=("default",)):
"""
Retrieves information on the sites that are available from a WaterOneFlow
service using a GetSites request. For more detailed information including
which variables and time periods are available for a given site, use
``get_site_info()``.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
suds_cache: ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
Returns
-------
sites_dict : dict
a python dict with site codes mapped to site information
"""
suds_client = _get_client(wsdl_url, suds_cache)
waterml_version = _waterml_version(suds_client)
if waterml_version == '1.0':
response = suds_client.service.GetSitesXml('')
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_0.parse_site_infos(response_buffer)
elif waterml_version == '1.1':
response = suds_client.service.GetSites('')
response_buffer = io.BytesIO(util.to_bytes(response))
sites = waterml.v1_1.parse_site_infos(response_buffer)
return dict([
(site['network'] + ':' + site['code'], site)
for site in list(sites.values())
])
def get_variable_info(wsdl_url, variable_code=None, suds_cache=("default",)):
"""
Retrieves site values from a WaterOneFlow service using a GetVariableInfo
request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
variable_code : `None` or str
If `None` (default) then information on all variables will be returned,
otherwise, this should be set to the variable code of the variable you'd
like to get more information on. Variable codes MUST contain the
network and be of the form <vocabulary>:<variable_code>, as is required
by WaterOneFlow.
suds_cache: ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
Returns
-------
variable_info : dict
a python dict containing variable information. If no variable code is
`None` (default) then this will be a nested set of dicts keyed by
<vocabulary>:<variable_code>
"""
suds_client = _get_client(wsdl_url, suds_cache)
waterml_version = _waterml_version(suds_client)
response = suds_client.service.GetVariableInfo(variable_code)
response_buffer = io.BytesIO(util.to_bytes(response))
if waterml_version == '1.0':
variable_info = waterml.v1_0.parse_variables(response_buffer)
elif waterml_version == '1.1':
variable_info = waterml.v1_1.parse_variables(response_buffer)
if not variable_code is None and len(variable_info) == 1:
return list(variable_info.values())[0]
else:
return dict([
('%s:%s' % (var['vocabulary'], var['code']), var)
for var in list(variable_info.values())
])
def get_values(wsdl_url, site_code, variable_code, start=None, end=None,
suds_cache=("default",), timeout=None):
"""
Retrieves site values from a WaterOneFlow service using a GetValues request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
site_code : str
Site code of the site you'd like to get values for. Site codes MUST
contain the network and be of the form <network>:<site_code>, as is
required by WaterOneFlow.
variable_code : str
Variable code of the variable you'd like to get values for. Variable
codes MUST contain the network and be of the form
<vocabulary>:<variable_code>, as is required by WaterOneFlow.
start : ``None`` or datetime (see :ref:`dates-and-times`)
Start of a date range for a query. If both start and end parameters are
omitted, the entire time series available will be returned.
end : ``None`` or datetime (see :ref:`dates-and-times`)
End of a date range for a query. If both start and end parameters are
omitted, the entire time series available will be returned.
suds_cache : ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
timeout : int or float
suds SOAP URL open timeout (seconds).
If unspecified, the suds default (90 seconds) will be used.
Returns
-------
site_values : dict
a python dict containing values
"""
suds_client = _get_client(wsdl_url, suds_cache, timeout)
# Note from Emilio:
# Not clear if WOF servers really do handle time zones (time offsets or
# "Z" in the iso8601 datetime strings. In the past, I (Emilio) have
# passed naive strings to GetValues(). if a datetime object is passed to
# this ulmo function, the isodate code above will include it in the
# resulting iso8601 string; if not, no. Test effect of dt_isostr having
# a timezone code or offset, vs not having it (the latter, naive dt
# strings, is what I've been using all along)
# the interpretation of start and end time zone is server-dependent
start_dt_isostr = None
end_dt_isostr = None
if start is not None:
start_datetime = util.convert_datetime(start)
start_dt_isostr = isodate.datetime_isoformat(start_datetime)
if end is not None:
end_datetime = util.convert_datetime(end)
end_dt_isostr = isodate.datetime_isoformat(end_datetime)
waterml_version = _waterml_version(suds_client)
response = suds_client.service.GetValues(
site_code, variable_code, startDate=start_dt_isostr,
endDate=end_dt_isostr)
response_buffer = io.BytesIO(util.to_bytes(response))
if waterml_version == '1.0':
values = waterml.v1_0.parse_site_values(response_buffer)
elif waterml_version == '1.1':
values = waterml.v1_1.parse_site_values(response_buffer)
if not variable_code is None:
return list(values.values())[0]
else:
return values
def get_values(wsdl_url, site_code, variable_code, start=None, end=None,
suds_cache=("default",), timeout=None, user_cache=False):
"""
Retrieves site values from a WaterOneFlow service using a GetValues request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
site_code : str
Site code of the site you'd like to get values for. Site codes MUST
contain the network and be of the form <network>:<site_code>, as is
required by WaterOneFlow.
variable_code : str
Variable code of the variable you'd like to get values for. Variable
codes MUST contain the network and be of the form
<vocabulary>:<variable_code>, as is required by WaterOneFlow.
start : ``None`` or datetime (see :ref:`dates-and-times`)
Start of the query datetime range. If omitted, data from the start of
the time series to the ``end`` timestamp will be returned (but see caveat,
in note below).
end : ``None`` or datetime (see :ref:`dates-and-times`)
End of the query datetime range. If omitted, data from the ``start``
timestamp to end of the time series will be returned (but see caveat,
in note below).
suds_cache : ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
timeout : int or float
suds SOAP URL open timeout (seconds).
If unspecified, the suds default (90 seconds) will be used.
user_cache : bool
If False (default), use the system temp location to store cache WSDL and
other files. Use the default user ulmo directory if True.
Returns
-------
site_values : dict
a python dict containing values
Notes
-----
If both ``start`` and ``end`` parameters are omitted, the entire time series
available will typically be returned. However, some service providers will return
an error if either start or end are omitted; this is specially true for services
hosted or redirected by CUAHSI via the CUAHSI HydroPortal, which have a 'WSDL' url
using the domain https://hydroportal.cuahsi.org. For HydroPortal, a start datetime
of '1753-01-01' has been known to return valid results while catching the oldest
start times, though the response may be broken up into chunks ('paged').
"""
suds_client = _get_client(wsdl_url, suds_cache, timeout, user_cache)
# Note from Emilio:
# Not clear if WOF servers really do handle time zones (time offsets or
# "Z" in the iso8601 datetime strings. In the past, I (Emilio) have
# passed naive strings to GetValues(). if a datetime object is passed to
# this ulmo function, the isodate code above will include it in the
# resulting iso8601 string; if not, no. Test effect of dt_isostr having
# a timezone code or offset, vs not having it (the latter, naive dt
# strings, is what I've been using all along)
# the interpretation of start and end time zone is server-dependent
start_dt_isostr = None
end_dt_isostr = None
if start is not None:
start_datetime = util.convert_datetime(start)
start_dt_isostr = isodate.datetime_isoformat(start_datetime)
if end is not None:
end_datetime = util.convert_datetime(end)
end_dt_isostr = isodate.datetime_isoformat(end_datetime)
waterml_version = _waterml_version(suds_client)
response = suds_client.service.GetValues(
site_code, variable_code, startDate=start_dt_isostr,
endDate=end_dt_isostr)
response_buffer = io.BytesIO(util.to_bytes(response))
if waterml_version == '1.0':
values = waterml.v1_0.parse_site_values(response_buffer)
elif waterml_version == '1.1':
values = waterml.v1_1.parse_site_values(response_buffer)
if not variable_code is None:
return list(values.values())[0]
else:
return values
def get_values(wsdl_url, site_code, variable_code, start=None, end=None, suds_cache=("default",)):
"""
Retrieves site values from a WaterOneFlow service using a GetValues request.
Parameters
----------
wsdl_url : str
URL of a service's web service definition language (WSDL) description.
All WaterOneFlow services publish a WSDL description and this url is the
entry point to the service.
site_code : str
Site code of the site you'd like to get values for. Site codes MUST
contain the network and be of the form <network>:<site_code>, as is
required by WaterOneFlow.
variable_code : str
Variable code of the variable you'd like to get values for. Variable
codes MUST contain the network and be of the form
<vocabulary>:<variable_code>, as is required by WaterOneFlow.
start : ``None`` or datetime (see :ref:`dates-and-times`)
Start of a date range for a query. If both start and end parameters are
omitted, the entire time series available will be returned.
end : ``None`` or datetime (see :ref:`dates-and-times`)
End of a date range for a query. If both start and end parameters are
omitted, the entire time series available will be returned.
suds_cache: ``None`` or tuple
SOAP local cache duration for WSDL description and client object.
Pass a cache duration tuple like ('days', 3) to set a custom duration.
Duration may be in months, weeks, days, hours, or seconds.
If unspecified, the default duration (1 day) will be used.
Use ``None`` to turn off caching.
Returns
-------
site_values : dict
a python dict containing values
"""
suds_client = _get_client(wsdl_url, suds_cache)
# Note from Emilio:
# Not clear if WOF servers really do handle time zones (time offsets or
# "Z" in the iso8601 datetime strings. In the past, I (Emilio) have
# passed naive strings to GetValues(). if a datetime object is passed to
# this ulmo function, the isodate code above will include it in the
# resulting iso8601 string; if not, no. Test effect of dt_isostr having
# a timezone code or offset, vs not having it (the latter, naive dt
# strings, is what I've been using all along)
# the interpretation of start and end time zone is server-dependent
start_dt_isostr = None
end_dt_isostr = None
if start is not None:
start_datetime = util.convert_datetime(start)
start_dt_isostr = isodate.datetime_isoformat(start_datetime)
if end is not None:
end_datetime = util.convert_datetime(end)
end_dt_isostr = isodate.datetime_isoformat(end_datetime)
waterml_version = _waterml_version(suds_client)
response = suds_client.service.GetValues(
site_code, variable_code, startDate=start_dt_isostr,
endDate=end_dt_isostr)
response_buffer = io.BytesIO(util.to_bytes(response))
if waterml_version == '1.0':
values = waterml.v1_0.parse_site_values(response_buffer)
elif waterml_version == '1.1':
values = waterml.v1_1.parse_site_values(response_buffer)
if not variable_code is None:
return list(values.values())[0]
else:
return values
def get_sites(sites=None, state_code=None, huc=None, bounding_box=None,
county_code=None, parameter_code=None, site_type=None, service=None,
input_file=None, **kwargs):
"""Fetches site information from USGS services. See the `USGS Site Service`_
documentation for a detailed description of options. For convenience, major
options have been included with pythonic names. Options that are not listed
below may be provided as extra kwargs (i.e. keyword='argument') and will be
passed along with the web services request. These extra keywords must match
the USGS names exactly. The `USGS Site Service`_ website describes available
keyword names and argument formats.
.. USGS Site Service:http://waterservices.usgs.gov/rest/Site-Service.html
.. note::
Only the options listed below have been tested and you may have mixed
results retrieving data with extra options specified. Currently ulmo
requests and parses data in the waterml format. Some options are not
available in this format.
Parameters
==========
service : {``None``, 'instantaneous', 'iv', 'daily', 'dv'}
The service to use, either "instantaneous", "daily", or ``None``
(default). If set to ``None``, then both services are used. The
abbreviations "iv" and "dv" can be used for "instantaneous" and "daily",
respectively.
input_file: ``None``, file path or file object
If ``None`` (default), then the NWIS web services will be queried, but
if a file is passed then this file will be used instead of requesting
data from the NWIS web services.
Major Filters (At least one filter must be specified)
-----------------------------------------------------
sites : str, iterable of strings or ``None``
The site(s) to use; lists will be joined by a ','.
state_code : str or ``None``
Two-letter state code used in stateCd parameter.
county_code : str, iterable of strings or ``None``
The 5 digit FIPS county code(s) used in the countyCd parameter; lists
will be joined by a ','.
huc : str, iterable of strings or ``None``
The hydrologic unit code(s) to use; lists will be joined by a ','.
bounding_box : str, iterable of strings or ``None``
This bounding box used in the bBox parameter. The format is westernmost
longitude, southernmost latitude, easternmost longitude, northernmost
latitude; lists will be joined by a ','.
Optional Filters Provided
-------------------------
parameter_code : str, iterable of strings or ``None``
Parameter code(s) that will be passed as the parameterCd parameter; lists will be joined by a ','.
This parameter represents the following usgs website input: Sites serving parameter codes
site_types : str, iterable of strings or ``None``
The type(s) of site used in siteType parameter; lists will be joined by a ','.
Returns
-------
sites_dict : dict
a python dict with site codes mapped to site information
"""
if input_file is None:
# Checking to see if the correct amount of major filters are being used.
# The NWIS site requires only one major filter to be used at a time.
major_filters = [sites, state_code, huc, bounding_box, county_code]
if not any(major_filters):
error_msg = (
'*At least one* of the following major filters must be supplied: '
'sites, state_code, huc, bounding_box, country_code.'
)
raise ValueError(error_msg)
if len([_f for _f in major_filters if _f]) > 1:
error_msg = (
'*Only one* of the following major filters can be supplied:'
'sites, state_code, huc, bounding_box, country_code.'
)
raise ValueError(error_msg)
url_params = {'format': 'waterml'}
if state_code:
url_params['stateCd'] = state_code
if sites:
url_params['sites'] = _as_str(sites)
if huc:
url_params['hucs'] = _as_str(huc)
if bounding_box:
url_params['bBox'] = _as_str(bounding_box)
if county_code:
url_params['countyCd'] = _as_str(county_code)
if site_type:
url_params['siteType'] = _as_str(site_type)
if parameter_code:
url_params['parameterCd'] = _as_str(parameter_code)
url_params.update(kwargs)
if not service:
return_sites = {}
for service in ['daily', 'instantaneous']:
new_sites = get_sites(sites=sites, state_code=state_code, huc=huc,
bounding_box=bounding_box, county_code=county_code, parameter_code=parameter_code,
site_type=site_type, service=service, input_file=input_file, **kwargs)
return_sites.update(new_sites)
return return_sites
url = _get_service_url(service)
log.info('making request for sites: %s' % url)
req = requests.get(url, params=url_params)
log.info("processing data from request: %s" % req.request.url)
req.raise_for_status()
input_file = io.BytesIO(util.to_bytes(req.content))
with _open_input_file(input_file) as content_io:
return_sites = wml.parse_site_infos(content_io)
return_sites = dict([
(code, _extract_site_properties(site))
for code, site in return_sites.items()
])
return return_sites
def get_daymet_singlepixel(latitude, longitude,
variables=['tmax', 'tmin', 'prcp'], years=None,
as_dataframe=True):
"""Fetches a time series of climate variables from the DAYMET single pixel extraction
Parameters
----------
latitude: float
The latitude (WGS84), value between 52.0 and 14.5.
longitude: float
The longitude (WGS84), value between -131.0 and -53.0.
variables : List of str
Daymet parameters to fetch.
Available options:
``tmax`` - maximum temperature
``tmin`` - minimum temperature
``srad`` - shortwave radiation
``vp`` - vapor pressure
``swe`` - snow-water equivalent
``prcp`` - precipitation
``dayl`` - daylength
default = ['tmax', 'tmin', 'prcp']
years: list of int
List of years to return.
Daymet version 2 available 1980 to the latest full calendar year.
If ``None`` (default), all years will be returned
as_dataframe : ``True`` (default) or ``False``
if ``True`` return pandas dataframe
if ``False`` return open file with contents in csv format
Returns
-------
single_pixel_timeseries : pandas dataframe or csv filename
"""
_check_coordinates(latitude, longitude)
_check_variables(variables)
if not years is None:
_check_years(years)
url_params = {'lat': latitude,
'lon': longitude,
'vars': _as_str(variables)}
if years:
url_params['years'] = _as_str(years)
url = _get_service_url(url_params)
log.info("making request for latitude, longitude: {}, {}".format(latitude, longitude))
req = requests.get(url, params=url_params)
log.info("processing data from request: %s" % req.request.url)
req.raise_for_status()
input_file = io.BytesIO(util.to_bytes(req.content))
df = pd.read_csv(input_file, header=6)
df.index = pd.to_datetime(df.year.astype('str') + '-' + df.yday.astype('str'), format="%Y-%j")
df.columns = [c[:c.index('(')].strip() if '(' in c else c for c in df.columns ]
if as_dataframe:
return df
else:
results = {}
for key in df.columns:
if key not in ['yday', 'year']:
results[key] = dict(zip(df[key].index.format(), df[key]))
return results
def get_sites(sites=None,
state_code=None,
huc=None,
bounding_box=None,
county_code=None,
parameter_code=None,
site_type=None,
service=None,
input_file=None,
**kwargs):
"""Fetches site information from USGS services. See the `USGS Site Service`_
documentation for a detailed description of options. For convenience, major
options have been included with pythonic names. Options that are not listed
below may be provided as extra kwargs (i.e. keyword='argument') and will be
passed along with the web services request. These extra keywords must match
the USGS names exactly. The `USGS Site Service`_ website describes available
keyword names and argument formats.
.. USGS Site Service:http://waterservices.usgs.gov/rest/Site-Service.html
.. note::
Only the options listed below have been tested and you may have mixed
results retrieving data with extra options specified. Currently ulmo
requests and parses data in the waterml format. Some options are not
available in this format.
Parameters
==========
service : {``None``, 'instantaneous', 'iv', 'daily', 'dv'}
The service to use, either "instantaneous", "daily", or ``None``
(default). If set to ``None``, then both services are used. The
abbreviations "iv" and "dv" can be used for "instantaneous" and "daily",
respectively.
input_file: ``None``, file path or file object
If ``None`` (default), then the NWIS web services will be queried, but
if a file is passed then this file will be used instead of requesting
data from the NWIS web services.
Major Filters (At least one filter must be specified)
-----------------------------------------------------
sites : str, iterable of strings or ``None``
The site(s) to use; lists will be joined by a ','.
state_code : str or ``None``
Two-letter state code used in stateCd parameter.
county_code : str, iterable of strings or ``None``
The 5 digit FIPS county code(s) used in the countyCd parameter; lists
will be joined by a ','.
huc : str, iterable of strings or ``None``
The hydrologic unit code(s) to use; lists will be joined by a ','.
bounding_box : str, iterable of strings or ``None``
This bounding box used in the bBox parameter. The format is westernmost
longitude, southernmost latitude, easternmost longitude, northernmost
latitude; lists will be joined by a ','.
Optional Filters Provided
-------------------------
parameter_code : str, iterable of strings or ``None``
Parameter code(s) that will be passed as the parameterCd parameter; lists will be joined by a ','.
This parameter represents the following usgs website input: Sites serving parameter codes
site_types : str, iterable of strings or ``None``
The type(s) of site used in siteType parameter; lists will be joined by a ','.
Returns
-------
sites_dict : dict
a python dict with site codes mapped to site information
"""
if input_file is None:
# Checking to see if the correct amount of major filters are being used.
# The NWIS site requires only one major filter to be used at a time.
major_filters = [sites, state_code, huc, bounding_box, county_code]
if not any(major_filters):
error_msg = (
'*At least one* of the following major filters must be supplied: '
'sites, state_code, huc, bounding_box, country_code.')
raise ValueError(error_msg)
if len([_f for _f in major_filters if _f]) > 1:
error_msg = (
'*Only one* of the following major filters can be supplied:'
'sites, state_code, huc, bounding_box, country_code.')
raise ValueError(error_msg)
url_params = {'format': 'waterml'}
if state_code:
url_params['stateCd'] = state_code
if sites:
url_params['sites'] = _as_str(sites)
if huc:
url_params['hucs'] = _as_str(huc)
if bounding_box:
url_params['bBox'] = _as_str(bounding_box)
if county_code:
url_params['countyCd'] = _as_str(county_code)
if site_type:
url_params['siteType'] = _as_str(site_type)
if parameter_code:
url_params['parameterCd'] = _as_str(parameter_code)
url_params.update(kwargs)
if not service:
return_sites = {}
for service in ['daily', 'instantaneous']:
new_sites = get_sites(sites=sites,
state_code=state_code,
huc=huc,
bounding_box=bounding_box,
county_code=county_code,
parameter_code=parameter_code,
site_type=site_type,
service=service,
input_file=input_file,
**kwargs)
return_sites.update(new_sites)
return return_sites
url = _get_service_url(service)
log.info('making request for sites: %s' % url)
req = requests.get(url, params=url_params)
log.info("processing data from request: %s" % req.request.url)
req.raise_for_status()
input_file = io.BytesIO(util.to_bytes(req.content))
with _open_input_file(input_file) as content_io:
return_sites = wml.parse_site_infos(content_io)
return_sites = dict([(code, _extract_site_properties(site))
for code, site in return_sites.items()])
return return_sites
def get_daymet_singlepixel(latitude,
longitude,
variables=['tmax', 'tmin', 'prcp'],
years=None,
as_dataframe=True):
"""Fetches a time series of climate variables from the DAYMET single pixel extraction
Parameters
----------
latitude: float
The latitude (WGS84), value between 52.0 and 14.5.
longitude: float
The longitude (WGS84), value between -131.0 and -53.0.
variables : List of str
Daymet parameters to fetch.
Available options:
``tmax`` - maximum temperature
``tmin`` - minimum temperature
``srad`` - shortwave radiation
``vp`` - vapor pressure
``swe`` - snow-water equivalent
``prcp`` - precipitation
``dayl`` - daylength
default = ['tmax', 'tmin', 'prcp']
years: list of int
List of years to return.
Daymet version 2 available 1980 to the latest full calendar year.
If ``None`` (default), all years will be returned
as_dataframe : ``True`` (default) or ``False``
if ``True`` return pandas dataframe
if ``False`` return open file with contents in csv format
Returns
-------
single_pixel_timeseries : pandas dataframe or csv filename
"""
_check_coordinates(latitude, longitude)
_check_variables(variables)
if not years is None:
_check_years(years)
url_params = {
'lat': latitude,
'lon': longitude,
'vars': _as_str(variables)
}
if years:
url_params['years'] = _as_str(years)
url = _get_service_url(url_params)
log.info("making request for latitude, longitude: {}, {}".format(
latitude, longitude))
req = requests.get(url, params=url_params)
log.info("processing data from request: %s" % req.request.url)
req.raise_for_status()
input_file = io.BytesIO(util.to_bytes(req.content))
df = pd.read_csv(input_file, header=6)
df.index = pd.to_datetime(df.year.astype('str') + '-' +
df.yday.astype('str'),
format="%Y-%j")
df.columns = [
c[:c.index('(')].strip() if '(' in c else c for c in df.columns
]
if as_dataframe:
return df
else:
results = {}
for key in df.columns:
if key not in ['yday', 'year']:
results[key] = dict(zip(df[key].index.format(), df[key]))
return results
