def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to all PVDAQ observations from
start to end.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
Raises
------
KeyError
If NREL_PVDAQ_API_KEY environmental variable is not set.
Abuse of KeyError - should probably be ValueError - but kept for
consistency with other reference_observations modules.
"""
nrel_pvdaq_api_key = os.getenv('NREL_PVDAQ_API_KEY')
if nrel_pvdaq_api_key is None:
raise KeyError('"NREL_PVDAQ_API_KEY" environment variable must be '
'set to update PVDAQ observation data.')
pvdaq_sites = common.filter_by_networks(sites, ['NREL PVDAQ'])
for site in pvdaq_sites:
common.update_site_observations(
api, partial(fetch, nrel_pvdaq_api_key=nrel_pvdaq_api_key), site,
observations, start, end)
def test_update_site_observations_gaps(mock_api, mock_fetch,
observation_objects_param,
fake_ghi_data, mocker):
mocker.patch.object(mock_api,
'get_observation_value_gaps',
return_value=[
(pd.Timestamp('2019-01-01T12:10Z'),
pd.Timestamp('2019-01-01T12:13Z')),
(pd.Timestamp('2019-01-01T12:19Z'),
pd.Timestamp('2019-01-01T12:20Z')),
])
start = pd.Timestamp('20190101T1205Z')
end = pd.Timestamp('20190101T1225Z')
site = site_objects[0]
common.update_site_observations(mock_api,
mock_fetch,
site,
observation_objects_param,
start,
end,
gaps_only=True)
assert mock_api.post_observation_values.call_count == 2
kargs = mock_api.post_observation_values.call_args_list
assert kargs[0][0][0] == ''
pd.testing.assert_frame_equal(
kargs[0][0][1],
fake_ghi_data.rename(columns={'ghi': 'value'})
['2019-01-01T12:10Z':'2019-01-01T12:13Z'].resample(
kargs[0][0][1].index.freq).first())
pd.testing.assert_frame_equal(
kargs[1][0][1],
fake_ghi_data.rename(columns={'ghi': 'value'})
['2019-01-01T12:19Z':'2019-01-01T12:20Z'].resample(
kargs[1][0][1].index.freq).first())
def update_observation_data(api, sites, observations, start, end):
"""Retrieve data from the network, and then format and post it to each
observation at the site.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites.
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
Raises
------
KeyError
If EIA_API_KEY environmental variable is not set.
"""
eia_api_key = os.getenv("EIA_API_KEY")
if eia_api_key is None:
raise KeyError('"EIA_API_KEY" environment variable must be '
'set to update EIA observation data.')
eia_sites = common.filter_by_networks(sites, ['EIA'])
for site in eia_sites:
common.update_site_observations(
api, partial(fetch, eia_api_key=eia_api_key),
site, observations, start, end)
def update_observation_data(api,
sites,
observations,
start,
end,
*,
gaps_only=False):
"""Post new observation data to all reference observations
at each SOLRAD site between start and end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites : list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations : list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
"""
solrad_sites = common.filter_by_networks(sites, 'NOAA SOLRAD')
for site in solrad_sites:
common.update_site_observations(api,
fetch,
site,
observations,
start,
end,
gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end, *,
gaps_only=False):
"""Post new observation data to a list of DOE RTC Observations
from start to end.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
"""
doe_rtc_api_key = os.getenv('DOE_RTC_API_KEY')
if doe_rtc_api_key is None:
raise KeyError('"DOE_RTC_API_KEY" environment variable must be '
'set to update DOE RTC observation data.')
doe_rtc_sites = common.filter_by_networks(sites, 'DOE RTC')
for site in doe_rtc_sites:
common.update_site_observations(
api, partial(fetch, doe_rtc_api_key=doe_rtc_api_key), site,
observations, start, end, gaps_only=gaps_only)
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of DOE ARM Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
doe_arm_api_key = os.getenv('DOE_ARM_API_KEY')
if doe_arm_api_key is None:
raise KeyError('"DOE_ARM_API_KEY" environment variable must be '
'set to update DOE ARM observation data.')
doe_arm_user_id = os.getenv('DOE_ARM_USER_ID')
if doe_arm_user_id is None:
raise KeyError('"DOE_ARM_USER_ID" environment variable must be '
'set to update DOE ARM observation data.')
doe_arm_sites = common.filter_by_networks(sites, 'DOE ARM')
for site in doe_arm_sites:
common.update_site_observations(
api,
partial(fetch,
doe_arm_user_id=doe_arm_user_id,
doe_arm_api_key=doe_arm_api_key), site, observations,
start, end)
def update_observation_data(api, sites, observations, start, end, *,
gaps_only=False):
"""Post new observation data to all MIDC observations from
start to end.
Parameters
----------
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
"""
midc_sites = common.filter_by_networks(sites, ['NREL MIDC'])
for site in midc_sites:
common.update_site_observations(
api, fetch, site, observations, start, end, gaps_only=gaps_only)
def update_observation_data(api,
sites,
observations,
start,
end,
*,
gaps_only=False):
"""Post new observation data to a list of SRML Observations
from start to end.
api : :py:class:`solarforecastarbiter.io.api.APISession`
An active Reference user session.
sites: list of :py:class:`solarforecastarbiter.datamodel.Site`
List of all reference sites as Objects
observations: list of :py:class:`solarforecastarbiter.datamodel.Observation`
List of all reference observations as Objects
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
gaps_only : bool, default False
If True, only update periods between start and end where there
are data gaps.
""" # noqa
srml_sites = common.filter_by_networks(sites, 'UO SRML')
for site in srml_sites:
common.update_site_observations(api,
fetch,
site,
observations,
start,
end,
gaps_only=gaps_only)
def test_update_site_observations_no_data(mock_api, mocker, site_objects_param,
observation_objects_param, log,
start, end):
fetch = mocker.MagicMock()
fetch.return_value = pd.DataFrame()
common.update_site_observations(mock_api, fetch, site_objects[1],
observation_objects_param, start, end)
mock_api.assert_not_called()
def test_update_site_observations_uptodate(mock_api, mock_fetch,
observation_objects_param,
fake_ghi_data, now):
site = site_objects[0]
mock_api.get_observation_time_range.return_value = (0, now)
common.update_site_observations(mock_api, mock_fetch, site,
observation_objects_param, None, None)
mock_api.post_observation_values.assert_not_called
def test_update_site_observations(mock_api, mock_fetch, site,
observation_objects_param, fake_ghi_data):
start = pd.Timestamp('20190101T1205Z')
end = pd.Timestamp('20190101T1225Z')
common.update_site_observations(mock_api, mock_fetch, site,
observation_objects_param, start, end)
args, _ = mock_api.post_observation_values.call_args
assert args[0] == ''
pd.testing.assert_frame_equal(
args[1],
fake_ghi_data.rename(columns={'ghi': 'value'})[start:end])
def test_update_site_observations_no_end(mock_api, mock_fetch,
observation_objects_param,
fake_ghi_data, now):
site = site_objects[0]
start = pd.Timestamp('20190101T1210Z')
end = None
common.update_site_observations(mock_api, mock_fetch, site,
observation_objects_param, start, end)
args, _ = mock_api.post_observation_values.call_args
assert args[0] == ''
pd.testing.assert_frame_equal(
args[1],
fake_ghi_data.rename(columns={'ghi': 'value'})[start:].resample(
args[1].index.freq).first())
def test_update_site_observations_out_of_order(mock_api, site_objects_param,
mocker,
observation_objects_param,
fake_ghi_data):
start = pd.Timestamp('20190101T1200Z')
end = pd.Timestamp('20190101T1230Z')
fetch = mocker.MagicMock()
fetch.return_value = fake_ghi_data.sample(frac=1)
common.update_site_observations(mock_api, fetch, site_objects[1],
observation_objects_param, start, end)
args, _ = mock_api.post_observation_values.call_args
assert args[0] == ''
pd.testing.assert_frame_equal(
args[1],
fake_ghi_data.rename(columns={'ghi': 'value'})[start:end].resample(
args[1].index.freq).first())
def test_update_site_observations_no_start(mock_api, mock_fetch,
observation_objects_param,
fake_ghi_data, now):
site = site_objects[0]
start = None
slimit = pd.Timestamp('20190101T1219Z')
end = pd.Timestamp('20190108T1219Z')
mock_api.get_observation_time_range.return_value = (0, slimit)
common.update_site_observations(mock_api, mock_fetch, site,
observation_objects_param, start, end)
args, _ = mock_api.post_observation_values.call_args
assert args[0] == ''
pd.testing.assert_frame_equal(
args[1],
fake_ghi_data.rename(columns={'ghi': 'value'})[slimit:end].resample(
args[1].index.freq).first())
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of SRML Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list
List of all reference sites as Objects
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
srml_sites = common.filter_by_networks(sites, 'UO SRML')
for site in srml_sites:
common.update_site_observations(api, fetch, site, observations, start,
end)
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of Surfrad Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
surfrad_sites = common.filter_by_networks(sites, 'NOAA SURFRAD')
for site in surfrad_sites:
common.update_site_observations(api, fetch, site, observations, start,
end)
def update_observation_data(api, sites, observations, start, end):
""" Post new observation data to all reference observations at each
USCRN site between start and end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
crn_sites = common.filter_by_networks(sites, 'NOAA USCRN')
for site in crn_sites:
common.update_site_observations(api, fetch, site, observations, start,
end)
