def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to all MIDC observations from
start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list
List of all reference sites as Objects
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
midc_sites = common.filter_by_networks(sites, ['NREL MIDC'])
for site in midc_sites:
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
continue
try:
obs_df = iotools.read_midc_raw_data_from_nrel(
site_extra_params['network_api_id'], start, end)
except IndexError:
# The MIDC api returns a text file on 404 that is parsed as a
# 1-column csv and causes an index error.
logger.warning(f'Could not retrieve data for site {site.name}'
f' between {start} and {end}.')
continue
site_observations = [obs for obs in observations if obs.site == site]
for obs in site_observations:
common.post_observation_data(api, obs, obs_df, start, end)
def test_post_observation_data_HTTPError(mock_api, log, fake_ghi_data, start,
end):
# mocking this error means that when the debugging logging call
# fires, an AttributeError is thrown while looking for a response
# but this also tests that we've followed the correct logic.
mock_api.post_observation_values.side_effect = HTTPError
with pytest.raises(AttributeError):
common.post_observation_data(mock_api, site_test_observation,
fake_ghi_data, start, end)
log.error.assert_called()
def test_post_observation_data(mock_api, log, fake_ghi_data, start, end):
common.post_observation_data(mock_api, site_test_observation,
fake_ghi_data, start, end)
args, _ = mock_api.post_observation_values.call_args
# test observations never get assigned an id so the observation_id
# argument should be an empty string
assert args[0] == ''
pd.testing.assert_frame_equal(
args[1], fake_ghi_data.rename(columns={'ghi': 'value'}))
def test_post_observation_data_no_data(
mock_api,
log,
start,
end,
):
common.post_observation_data(
mock_api, test_kwarg_observation,
pd.DataFrame({
'a': [1, 2, 3],
'b': ['a', 'b', 'c']
}), start, end)
log.error.assert_called()
def test_post_observation_data_all_nans(mock_api, log, fake_ghi_data, start,
end):
nan_data = fake_ghi_data.copy()
nan_data['ghi'] = np.NaN
ret = common.post_observation_data(mock_api, site_test_observation,
nan_data, start, end)
log.warning.assert_called()
assert ret is None
def update_observation_data(api, sites, observations, start, end):
"""Post new observation data to a list of Surfrad Observations
from start to end.
api : solarforecastarbiter.io.api.APISession
An active Reference user session.
sites: list of solarforecastarbiter.datamodel.Site
List of all reference sites as Objects
observations: list of solarforecastarbiter.datamodel.Observation
List of all reference observations.
start : datetime
The beginning of the period to request data for.
end : datetime
The end of the period to request data for.
"""
sandia_api_key = os.getenv('SANDIA_API_KEY')
if sandia_api_key is None:
raise KeyError('"SANDIA_API_KEY" environment variable must be '
'set to update SANDIA observation data.')
sandia_sites = common.filter_by_networks(sites, 'SANDIA')
for site in sandia_sites:
try:
site_extra_params = common.decode_extra_parameters(site)
except ValueError:
continue
sandia_site_id = site_extra_params['network_api_id']
obs_df = sandia.fetch_sandia(
sandia_site_id, sandia_api_key,
start.tz_convert(site.timezone), end.tz_convert(site.timezone))
obs_df = obs_df.rename(columns=SANDIA_VARIABLE_MAP).tz_localize(
site.timezone)
data_in_range = obs_df[start:end]
if data_in_range.empty:
logger.warning(f'Data for site {site.name} contained no '
f'entries from {start} to {end}.')
continue
site_observations = [obs for obs in observations if obs.site == site]
for obs in site_observations:
common.post_observation_data(api, obs, data_in_range, start, end)
def test_post_observation_data_AttributeError(mock_api, log, fake_ghi_data,
start, end):
common.post_observation_data(mock_api, test_observation_fail,
fake_ghi_data, start, end)
log.error.assert_called()
