def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of DOE RTC Observations
from start to end.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
doe_rtc_api_key = os.getenv('DOE_RTC_API_KEY')
if doe_rtc_api_key is None:
raise KeyError('"DOE_RTC_API_KEY" environment variable must be '
'set to update DOE RTC observation data.')
doe_rtc_sites = common.filter_by_networks(sites, 'DOE RTC')
for site in doe_rtc_sites:
common.update_site_observations(
api, partial(fetch, doe_rtc_api_key=doe_rtc_api_key), site,
observations, start, end)
def update_observation_data(api,
sites,
observations,
start,
end,
*,
gaps_only=False):
"""Retrieve data from the network, and then format and post it to each
observation at the site.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites.
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
Raises
------
KeyError
If EIA_API_KEY environmental variable is not set.
"""
eia_api_key = os.getenv("EIA_API_KEY")
if eia_api_key is None:
raise KeyError('"EIA_API_KEY" environment variable must be '
'set to update EIA observation data.')
eia_sites = common.filter_by_networks(sites, ['EIA'])
for site in eia_sites:
common.update_site_observations(api,
partial(fetch,
eia_api_key=eia_api_key),
site,
observations,
start,
end,
gaps_only=gaps_only)
def update_observation_data(api,
sites,
observations,
start,
end,
*,
gaps_only=False):
"""Post new observation data to all PVDAQ observations from
start to end.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
Raises
------
KeyError
If NREL_PVDAQ_API_KEY environmental variable is not set.
Abuse of KeyError - should probably be ValueError - but kept for
consistency with other reference_observations modules.
"""
nrel_pvdaq_api_key = os.getenv('NREL_PVDAQ_API_KEY')
if nrel_pvdaq_api_key is None:
raise KeyError('"NREL_PVDAQ_API_KEY" environment variable must be '
'set to update PVDAQ observation data.')
pvdaq_sites = common.filter_by_networks(sites, ['NREL PVDAQ'])
for site in pvdaq_sites:
common.update_site_observations(
api,
partial(fetch, nrel_pvdaq_api_key=nrel_pvdaq_api_key),
site,
observations,
start,
end,
gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of SRML Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list
List of all reference sites as Objects
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
srml_sites = common.filter_by_networks(sites, 'UO SRML')
for site in srml_sites:
common.update_site_observations(api, fetch, site, observations, start,
end)
def update_observation_data(api,
sites,
observations,
start,
end,
*,
gaps_only=False):
"""Post new observation data to a list of DOE ARM Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
"""
doe_arm_api_key = os.getenv('DOE_ARM_API_KEY')
if doe_arm_api_key is None:
raise KeyError('"DOE_ARM_API_KEY" environment variable must be '
'set to update DOE ARM observation data.')
doe_arm_user_id = os.getenv('DOE_ARM_USER_ID')
if doe_arm_user_id is None:
raise KeyError('"DOE_ARM_USER_ID" environment variable must be '
'set to update DOE ARM observation data.')
doe_arm_sites = common.filter_by_networks(sites, 'DOE ARM')
for site in doe_arm_sites:
common.update_site_observations(api,
partial(
fetch,
doe_arm_user_id=doe_arm_user_id,
doe_arm_api_key=doe_arm_api_key),
site,
observations,
start,
end,
gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of Surfrad Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
surfrad_sites = common.filter_by_networks(sites, 'NOAA SURFRAD')
for site in surfrad_sites:
common.update_site_observations(api, fetch, site, observations, start,
end)
def update_observation_data(api, sites, observations, start, end):
""" Post new observation data to all reference observations at each
USCRN site between start and end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
crn_sites = common.filter_by_networks(sites, 'NOAA USCRN')
for site in crn_sites:
common.update_site_observations(api, fetch, site, observations, start,
end)
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of Surfrad Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
sandia_api_key = os.getenv('SANDIA_API_KEY')
if sandia_api_key is None:
raise KeyError('"SANDIA_API_KEY" environment variable must be '
'set to update SANDIA observation data.')
sandia_sites = common.filter_by_networks(sites, 'SANDIA')
for site in sandia_sites:
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
continue
sandia_site_id = site_extra_params['network_api_id']
obs_df = sandia.fetch_sandia(
sandia_site_id, sandia_api_key,
start.tz_convert(site.timezone), end.tz_convert(site.timezone))
obs_df = obs_df.rename(columns=SANDIA_VARIABLE_MAP).tz_localize(
site.timezone)
data_in_range = obs_df[start:end]
if data_in_range.empty:
logger.warning(f'Data for site {site.name} contained no '
f'entries from {start} to {end}.')
continue
site_observations = [obs for obs in observations if obs.site == site]
for obs in site_observations:
common.post_observation_data(api, obs, data_in_range, start, end)
def test_filter_by_network(networks, expected):
assert common.filter_by_networks(site_objects, networks) == expected
