def test_persistence_interval_missing_data(site_metadata):
# interval beginning obs
observation = default_observation(site_metadata,
interval_length='5min',
interval_label='ending')
tz = 'America/Phoenix'
data_index = pd.date_range(start='20190404T1200',
end='20190406',
freq='5min',
tz=tz)
# each element of data is equal to the hour value of its label
end = '20190406 0000'
data = pd.Series(data_index.hour, index=data_index, dtype=float)
data = data.shift(1)
data_start = pd.Timestamp('20190404 0000', tz=tz)
data_end = pd.Timestamp(end, tz=tz) - pd.Timedelta('1d')
forecast_start = pd.Timestamp('20190405 0000', tz=tz)
interval_length = pd.Timedelta('60min')
load_data = partial(load_data_base, data)
expected_index = pd.date_range(start='20190405 0000',
end=end,
freq='60min',
tz=tz,
closed='right')
expected_vals = [None] * 12 + list(range(12, 24))
expected = pd.Series(expected_vals, index=expected_index, dtype=float)
fx = persistence.persistence_interval(observation, data_start, data_end,
forecast_start, interval_length,
'ending', load_data)
assert_series_equal(fx, expected)
def test_persistence_interval(site_metadata, obs_interval_label,
interval_label, closed, end):
# interval beginning obs
observation = default_observation(site_metadata,
interval_length='5min',
interval_label=obs_interval_label)
tz = 'America/Phoenix'
data_index = pd.date_range(start='20190404',
end='20190406',
freq='5min',
tz=tz)
# each element of data is equal to the hour value of its label
data = pd.Series(data_index.hour, index=data_index, dtype=float)
if obs_interval_label == 'ending':
# e.g. timestamp 12:00:00 should be equal to 11
data = data.shift(1).fillna(0)
data_start = pd.Timestamp('20190404 0000', tz=tz)
data_end = pd.Timestamp(end, tz=tz) - pd.Timedelta('1d')
forecast_start = pd.Timestamp('20190405 0000', tz=tz)
interval_length = pd.Timedelta('60min')
load_data = partial(load_data_base, data)
expected_index = pd.date_range(start='20190405 0000',
end=end,
freq='60min',
tz=tz,
closed=closed)
expected_vals = list(range(0, 24))
expected = pd.Series(expected_vals, index=expected_index, dtype=float)
# handle permutations of parameters that should fail
if data_end.minute == 59 and obs_interval_label != 'instant':
expectation = pytest.raises(ValueError)
elif data_end.minute == 0 and obs_interval_label == 'instant':
expectation = pytest.raises(ValueError)
else:
expectation = does_not_raise()
with expectation:
fx = persistence.persistence_interval(observation, data_start,
data_end, forecast_start,
interval_length, interval_label,
load_data)
assert_series_equal(fx, expected)
def run_persistence(session,
observation,
forecast,
run_time,
issue_time,
index=False):
"""
Run a persistence *forecast* for an *observation*.
For intraday forecasts, the *index* argument controls if the
forecast is constructed using persistence of the measured values
(*index = False*) or persistence using clear sky index or AC power
index.
For day ahead forecasts, only persistence of measured values
(*index = False*) is supported.
Forecasts may be run operationally or retrospectively. For
operational forecasts, *run_time* is typically set to now. For
retrospective forecasts, *run_time* is the time by which the
forecast should be run so that it could have been be delivered for
the *issue_time*. Forecasts will only use data with timestamps
before *run_time*.
The persistence *window* is the time over which the persistence
quantity (irradiance, power, clear sky index, or power index) is
averaged. The persistence window is automatically determined
from the *forecast* attributes:
* Intraday persistence forecasts:
*window = forecast.run_length*.
No longer than 1 hour.
* Day ahead forecasts (all but net load) and week ahead forecasts (net
load only):
*window = forecast.interval_length*.
Users that would like more flexibility may use the lower-level
functions in
:py:mod:`solarforecastarbiter.reference_forecasts.persistence`.
Parameters
----------
session : api.Session
The session object to use to request data from the
SolarForecastArbiter API.
observation : datamodel.Observation
The metadata of the observation to be used to create the
forecast.
forecast : datamodel.Forecast
The metadata of the desired forecast.
run_time : pd.Timestamp
Run time of the forecast.
issue_time : pd.Timestamp
Issue time of the forecast run.
index : bool, default False
If False, use persistence of observed value. If True, use
persistence of clear sky or AC power index.
Returns
-------
forecast : pd.Series
Forecast conforms to the metadata specified by the *forecast*
argument.
Raises
------
ValueError
If forecast and issue_time are incompatible.
ValueError
If persistence window < observation.interval_length.
ValueError
If forecast.run_length = 1 day and forecast period is not
midnight to midnight.
ValueError
If forecast.run_length = 1 day and index=True.
ValueError
If instantaneous forecast and instantaneous observation interval
lengths do not match.
ValueError
If average observations are used to make instantaneous forecast.
Notes
-----
For non-intraday net load forecasts, this function will use a weekahead
persistence due to the fact that net load exhibits stronger correlation
week-to-week than day-to-day. For example, the net load on a Monday tends
to look more similar to the previous Monday that it does to the previous
day (Sunday).
"""
utils.check_persistence_compatibility(observation, forecast, index)
forecast_start, forecast_end = utils.get_forecast_start_end(
forecast, issue_time, False)
intraday = utils._is_intraday(forecast)
if not intraday:
# raise ValueError if not intraday and not midnight to midnight
utils._check_midnight_to_midnight(forecast_start, forecast_end)
data_start, data_end = utils.get_data_start_end(observation, forecast,
run_time)
def load_data(observation, data_start, data_end):
df = session.get_observation_values(observation.observation_id,
data_start, data_end,
observation.interval_label)
df = df.tz_convert(observation.site.timezone)
return df['value']
if intraday and index:
fx = persistence.persistence_scalar_index(
observation, data_start, data_end, forecast_start, forecast_end,
forecast.interval_length, forecast.interval_label, load_data)
elif intraday and not index:
fx = persistence.persistence_scalar(observation, data_start, data_end,
forecast_start, forecast_end,
forecast.interval_length,
forecast.interval_label, load_data)
elif not intraday and not index:
fx = persistence.persistence_interval(observation, data_start,
data_end, forecast_start,
forecast.interval_length,
forecast.interval_label,
load_data)
else: # pragma: no cover
raise ValueError(
'index=True not supported for forecasts with run_length >= 1day')
return fx
def run_persistence(session,
observation,
forecast,
run_time,
issue_time,
index=False,
load_data=None):
"""
Run a persistence *forecast* for an *observation*.
For intraday forecasts, the *index* argument controls if the
forecast is constructed using persistence of the measured values
(*index = False*) or persistence using clear sky index or AC power
index.
For day ahead forecasts, only persistence of measured values
(*index = False*) is supported.
Forecasts may be run operationally or retrospectively. For
operational forecasts, *run_time* is typically set to now. For
retrospective forecasts, *run_time* is the time by which the
forecast should be run so that it could have been be delivered for
the *issue_time*. Forecasts will only use data with timestamps
before *run_time*.
The persistence *window* is the time over which the persistence
quantity (irradiance, power, clear sky index, or power index) is
averaged. The persistence window is automatically determined
from the *forecast* attributes:
- Intraday persistence forecasts:
+ ``window = forecast.run_length``. No longer than 1 hour.
- Day ahead forecasts (all but net load) and week ahead forecasts (net
load only):
+ ``window = forecast.interval_length``.
Users that would like more flexibility may use the lower-level
functions in
:py:mod:`solarforecastarbiter.reference_forecasts.persistence`.
Parameters
----------
session : api.Session
The session object to use to request data from the
SolarForecastArbiter API.
observation : datamodel.Observation
The metadata of the observation to be used to create the
forecast.
forecast : datamodel.Forecast
The metadata of the desired forecast.
run_time : pd.Timestamp
Run time of the forecast.
issue_time : pd.Timestamp
Issue time of the forecast run.
index : bool, default False
If False, use persistence of observed value. If True, use
persistence of clear sky or AC power index.
load_data : function
Function to load the observation data 'value' series given
(observation, data_start, data_end) arguments. Typically,
calls `session.get_observation_values` and selects the 'value'
column. May also have data preloaded to then slice from
data_start to data_end.
Returns
-------
forecast : pd.Series
Forecast conforms to the metadata specified by the *forecast*
argument.
Raises
------
ValueError
If forecast and issue_time are incompatible.
ValueError
If data is required from after run_time.
ValueError
If persistence window < observation.interval_length.
ValueError
If forecast.run_length => 1 day and index=True.
ValueError
If instantaneous forecast and instantaneous observation interval
lengths do not match.
ValueError
If average observations are used to make instantaneous forecast.
Notes
-----
For non-intraday net load forecasts, this function will use a weekahead
persistence due to the fact that net load exhibits stronger correlation
week-to-week than day-to-day. For example, the net load on a Monday tends
to look more similar to the previous Monday that it does to the previous
day (Sunday).
"""
utils.check_persistence_compatibility(observation, forecast, index)
forecast_start, forecast_end = utils.get_forecast_start_end(
forecast, issue_time, False)
intraday = utils._is_intraday(forecast)
if load_data is None:
load_data = _default_load_data(session)
data_start, data_end = utils.get_data_start_end(observation, forecast,
run_time, issue_time)
if data_end > run_time:
raise ValueError(
'Persistence forecast requires data from after run_time')
if isinstance(forecast, datamodel.ProbabilisticForecast):
cvs = [f.constant_value for f in forecast.constant_values]
fx = persistence.persistence_probabilistic(
observation, data_start, data_end, forecast_start, forecast_end,
forecast.interval_length, forecast.interval_label, load_data,
forecast.axis, cvs)
elif intraday and index:
fx = persistence.persistence_scalar_index(
observation, data_start, data_end, forecast_start, forecast_end,
forecast.interval_length, forecast.interval_label, load_data)
elif intraday and not index:
fx = persistence.persistence_scalar(observation, data_start, data_end,
forecast_start, forecast_end,
forecast.interval_length,
forecast.interval_label, load_data)
elif not intraday and not index:
fx = persistence.persistence_interval(observation, data_start,
data_end, forecast_start,
forecast.interval_length,
forecast.interval_label,
load_data)
else: # pragma: no cover
raise ValueError(
'index=True not supported for forecasts with run_length >= 1day')
return fx