def test_get_default_origin_for_time_vars(hourly_data):
"""Tests get_default_origin_for_time_vars"""
train_df = hourly_data["train_df"]
conti_year_origin = get_default_origin_for_time_vars(train_df, TIME_COL)
assert round(conti_year_origin, 3) == 2018.496
df = pd.DataFrame({"time": ["2018-07-01", "2018-08-01"]})
conti_year_origin = get_default_origin_for_time_vars(df, "time")
assert round(conti_year_origin, 3) == 2018.496
def test_get_changepoint_values_from_config(hourly_data):
"""Tests get_changepoint_values_from_config"""
train_df = hourly_data["train_df"]
conti_year_origin = get_default_origin_for_time_vars(train_df, TIME_COL)
time_features_df = build_time_features_df(
dt=train_df[TIME_COL], conti_year_origin=conti_year_origin)
with pytest.raises(Exception,
match="changepoint method must be specified"):
get_changepoint_values_from_config(
changepoints_dict={"n_changepoints": 2},
time_features_df=time_features_df,
time_col="datetime")
with pytest.raises(NotImplementedError,
match="changepoint method.*not recognized"):
get_changepoint_values_from_config(
changepoints_dict={"method": "not implemented"},
time_features_df=time_features_df,
time_col="datetime")
# tests uniform method
changepoint_values = get_changepoint_values_from_config(
changepoints_dict={
"method": "uniform",
"n_changepoints": 20
},
time_features_df=time_features_df,
time_col="datetime")
expected_changepoint_values = get_evenly_spaced_changepoints_values(
df=time_features_df, n_changepoints=20)
assert np.array_equal(changepoint_values, expected_changepoint_values)
changepoint_values = get_changepoint_values_from_config(
changepoints_dict={
"method": "uniform",
"n_changepoints": 20,
"continuous_time_col": "ct2"
},
time_features_df=time_features_df,
time_col="datetime")
expected_changepoint_values = get_evenly_spaced_changepoints_values(
df=time_features_df, n_changepoints=20, continuous_time_col="ct2")
assert np.array_equal(changepoint_values, expected_changepoint_values)
# tests custom method
dates = ["2018-01-01", "2019-01-02-16", "2019-01-03", "2019-02-01"]
changepoint_values = get_changepoint_values_from_config(
changepoints_dict={
"method": "custom",
"dates": dates
},
time_features_df=time_features_df,
time_col="datetime")
expected_changepoint_values = get_custom_changepoints_values(
df=time_features_df, changepoint_dates=dates, time_col="datetime")
assert np.array_equal(changepoint_values, expected_changepoint_values)
changepoint_values = get_changepoint_values_from_config(
changepoints_dict={
"method": "custom",
"dates": dates,
"continuous_time_col": "ct2"
},
time_features_df=time_features_df,
time_col="datetime")
expected_changepoint_values = get_custom_changepoints_values(
df=time_features_df,
changepoint_dates=dates,
time_col="datetime",
continuous_time_col="ct2")
assert np.array_equal(changepoint_values, expected_changepoint_values)
def get_forecast_time_properties(
df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq=None,
regressor_cols=None,
lagged_regressor_cols=None,
train_end_date=None,
forecast_horizon=None):
"""Returns the number of training points in `df`, the start year, and prediction end year
Parameters
----------
df : `pandas.DataFrame` with columns [``time_col``, ``value_col``]
Univariate timeseries data to forecast
time_col : `str`, default ``TIME_COL`` in constants.py
Name of timestamp column in df
value_col : `str`, default ``VALUE_COL`` in constants.py
Name of value column in df (the values to forecast)
freq : `str` or None, default None
Frequency of input data. Used to generate future dates for prediction.
Frequency strings can have multiples, e.g. '5H'.
See https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
for a list of frequency aliases.
If None, inferred by pd.infer_freq.
Provide this parameter if ``df`` has missing timepoints.
regressor_cols : `list` [`str`] or None, optional, default None
A list of regressor columns used in the training and prediction DataFrames.
If None, no regressor columns are used.
Regressor columns that are unavailable in ``df`` are dropped.
lagged_regressor_cols : `list` [`str`] or None, optional, default None
A list of lagged regressor columns used in the training and prediction DataFrames.
If None, no lagged regressor columns are used.
Lagged regressor columns that are unavailable in ``df`` are dropped.
train_end_date : `datetime.datetime`, optional, default None
Last date to use for fitting the model. Forecasts are generated after this date.
If None, it is set to the last date with a non-null value in
``value_col`` of ``df``.
forecast_horizon : `int` or None, default None
Number of periods to forecast into the future. Must be > 0
If None, default is determined from input data frequency
Returns
-------
time_properties : `dict` [`str`, `any`]
Time properties dictionary with keys:
``"period"`` : `int`
Period of each observation (i.e. minimum time between observations, in seconds).
``"simple_freq"`` : `SimpleTimeFrequencyEnum`
``SimpleTimeFrequencyEnum`` member corresponding to data frequency.
``"num_training_points"`` : `int`
Number of observations for training.
``"num_training_days"`` : `int`
Number of days for training.
``"days_per_observation"``: `float`
The time frequency in day units.
``"forecast_horizon"``: `int`
The number of time intervals for which forecast is needed.
``"forecast_horizon_in_timedelta"``: `datetime.timedelta`
The forecast horizon length in timedelta units.
``"forecast_horizon_in_days"``: `float`
The forecast horizon length in day units.
``"start_year"`` : `int`
Start year of the training period.
``"end_year"`` : `int`
End year of the forecast period.
``"origin_for_time_vars"`` : `float`
Continuous time representation of the first date in ``df``.
"""
if regressor_cols is None:
regressor_cols = []
# Defines ``fit_df``, the data available for fitting the model
# and its time column (in `datetime.datetime` format)
canonical_data_dict = get_canonical_data(
df=df,
time_col=time_col,
value_col=value_col,
freq=freq,
train_end_date=train_end_date,
regressor_cols=regressor_cols,
lagged_regressor_cols=lagged_regressor_cols)
fit_df = canonical_data_dict["fit_df"]
# Calculates basic time properties
train_start = fit_df[TIME_COL].min()
start_year = int(train_start.strftime("%Y"))
origin_for_time_vars = get_default_origin_for_time_vars(fit_df, TIME_COL)
period = min_gap_in_seconds(df=fit_df, time_col=TIME_COL)
simple_freq = get_simple_time_frequency_from_period(period)
num_training_points = fit_df.shape[0]
# Calculates number of (fractional) days in the training set
time_delta = fit_df[TIME_COL].max() - train_start
num_training_days = (
time_delta.days
+ (time_delta.seconds + period) / TimeEnum.ONE_DAY_IN_SECONDS.value)
# Calculates forecast horizon (as a number of periods)
if forecast_horizon is None:
# expected to be kept in sync with default value set in ``get_default_time_parameters``
forecast_horizon = get_default_horizon_from_period(
period=period,
num_observations=num_training_points)
days_per_observation = period / TimeEnum.ONE_DAY_IN_SECONDS.value
forecast_horizon_in_days = forecast_horizon * days_per_observation
forecast_horizon_in_timedelta = timedelta(days=forecast_horizon_in_days)
# Calculates forecast end year
train_end = fit_df[TIME_COL].max()
days_to_forecast = math.ceil(forecast_horizon * days_per_observation)
future_end = train_end + timedelta(days=days_to_forecast)
end_year = int(future_end.strftime("%Y"))
return {
"period": period,
"simple_freq": simple_freq,
"num_training_points": num_training_points,
"num_training_days": num_training_days,
"days_per_observation": days_per_observation,
"forecast_horizon": forecast_horizon,
"forecast_horizon_in_timedelta": forecast_horizon_in_timedelta,
"forecast_horizon_in_days": forecast_horizon_in_days,
"start_year": start_year,
"end_year": end_year,
"origin_for_time_vars": origin_for_time_vars
}
def generate_df_for_tests(freq,
periods,
train_start_date=datetime.datetime(2018, 7, 1),
train_end_date=None,
train_frac=0.8,
conti_year_origin=None,
noise_std=2.0,
remove_extra_cols=True,
autoreg_coefs=None,
fs_coefs=[-1, 3, 4],
growth_coef=3.0,
growth_pow=1.1,
intercept=0.0):
"""Generates dataset for unit tests.
:param freq: str
pd.date_range freq parameter, e.g. H or D
:param periods: int
number of periods to generate
:param train_start_date: datetime.datetime
train start date
:param train_end_date: Optional[datetime.datetime]
train end date
:param train_frac: Optional[float]
fraction of data to use for training
only used if train_end_date isn't provided
:param noise_std: float
standard deviation of gaussian noise
:param conti_year_origin: float
the time origin for continuous time variables
:param remove_extra_cols: bool
whether to remove extra columns besides TIME_COL, VALUE_COL
:param autoreg_coefs: Optional[List[int]]
The coefficients for the autoregressive terms.
If provided the generated series denoted mathematically by Y(t) will be
converted as follows:
Y(t) -> Y(t) + c1 Y(t-1) + c2 Y(t-2) + c3 Y(t-3) + ...
where autoreg_coefs = [c1, c2, c3, ...]
In this fashion, the obtained series will have autoregressive
properties not explained by seasonality and growth.
:param fs_coefs: List[float]
The fourier series coefficients used.
:param growth_coef: float
Multiplier for growth
:param growth_pow: float
Power for growth, as function of continuous time
:param intercept: float
Constant term added to Y(t)
:return: Dict[str, any]
contains full dataframe, train dataframe, test dataframe,
and nrows in test dataframe
"""
np.random.seed(123)
date_list = pd.date_range(start=train_start_date,
periods=periods,
freq=freq).tolist()
df0 = pd.DataFrame({TIME_COL: date_list})
if conti_year_origin is None:
conti_year_origin = get_default_origin_for_time_vars(df0, TIME_COL)
time_df = build_time_features_df(dt=df0[TIME_COL],
conti_year_origin=conti_year_origin)
df = pd.concat([df0, time_df], axis=1)
df["growth"] = growth_coef * (df["ct1"]**growth_pow)
func = fourier_series_multi_fcn(col_names=["toy", "tow", "tod"],
periods=[1.0, 7.0, 24.0],
orders=[1, 1, 1],
seas_names=None)
res = func(df)
df_seas = res["df"]
df = pd.concat([df, df_seas], axis=1)
df[VALUE_COL] = (
intercept + df["growth"] +
fs_coefs[0] * df[get_fourier_col_name(1, "tod", function_name="sin")] +
fs_coefs[1] * df[get_fourier_col_name(1, "tow", function_name="sin")] +
fs_coefs[2] * df[get_fourier_col_name(1, "toy", function_name="sin")] +
noise_std * np.random.normal(size=df.shape[0]))
if autoreg_coefs is not None:
df["temporary_new_value"] = df[VALUE_COL]
k = len(autoreg_coefs)
for i in range(k):
df["temporary_new_value"] = (
df["temporary_new_value"] +
autoreg_coefs[i] * df[VALUE_COL].shift(-i)).bfill()
df[VALUE_COL] = df["temporary_new_value"]
del df["temporary_new_value"]
if train_end_date is None:
train_rows = np.floor(train_frac * df.shape[0]).astype(int)
train_end_date = df[TIME_COL][train_rows]
if remove_extra_cols:
df = df[[TIME_COL, VALUE_COL]]
train_df = df.loc[df[TIME_COL] <= train_end_date]
test_df = df.loc[df[TIME_COL] > train_end_date]
fut_time_num = test_df.shape[0]
return {
"df": df,
"train_df": train_df.reset_index(drop=True),
"test_df": test_df.reset_index(drop=True),
"fut_time_num": fut_time_num,
}
def test_get_forecast_time_properties():
"""Tests get_forecast_time_properties"""
num_training_points = 365 # one year of daily data
data = generate_df_for_tests(freq="D", periods=num_training_points)
df = data["df"]
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="D",
forecast_horizon=0)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_DAY_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.DAY,
"num_training_points": num_training_points,
"num_training_days": num_training_points,
"days_per_observation": 1,
"forecast_horizon": 0,
"forecast_horizon_in_timedelta": timedelta(days=0),
"forecast_horizon_in_days": 0,
"start_year": 2018,
"end_year": 2019,
"origin_for_time_vars": default_origin
}
# longer forecast_horizon
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="D",
forecast_horizon=365)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_DAY_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.DAY,
"num_training_points": num_training_points,
"num_training_days": num_training_points,
"days_per_observation": 1,
"forecast_horizon": 365,
"forecast_horizon_in_timedelta": timedelta(days=365),
"forecast_horizon_in_days": 365,
"start_year": 2018,
"end_year": 2020,
"origin_for_time_vars": default_origin
}
# two years of hourly data
num_training_points = 2 * 365 * 24
data = generate_df_for_tests(freq="H", periods=num_training_points)
df = data["df"]
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
forecast_horizon=0)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_HOUR_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.HOUR,
"num_training_points": num_training_points,
"num_training_days": num_training_points / 24,
"days_per_observation": 1 / 24,
"forecast_horizon": 0,
"forecast_horizon_in_timedelta": timedelta(days=0),
"forecast_horizon_in_days": 0,
"start_year": 2018,
"end_year": 2020,
"origin_for_time_vars": default_origin
}
# longer forecast_horizon
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
forecast_horizon=365 * 24)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_HOUR_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.HOUR,
"num_training_points": num_training_points,
"num_training_days": num_training_points / 24,
"days_per_observation": 1 / 24,
"forecast_horizon": 365 * 24,
"forecast_horizon_in_timedelta": timedelta(days=365),
"forecast_horizon_in_days": 365,
"start_year": 2018,
"end_year": 2021,
"origin_for_time_vars": default_origin
}
# ``forecast_horizon=None``
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
forecast_horizon=None)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_HOUR_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.HOUR,
"num_training_points": num_training_points,
"num_training_days": num_training_points / 24,
"days_per_observation": 1 / 24,
"forecast_horizon": 24,
"forecast_horizon_in_timedelta": timedelta(days=1),
"forecast_horizon_in_days": 1,
"start_year": 2018,
"end_year": 2020,
"origin_for_time_vars": default_origin
}
# weekly df with regressors
num_training_points = 50
data = generate_df_with_reg_for_tests(freq="W-SUN",
periods=num_training_points,
train_start_date=datetime.datetime(
2018, 11, 30),
remove_extra_cols=True,
mask_test_actuals=True)
df = data["df"]
train_df = data["train_df"]
forecast_horizon = data["fut_time_num"]
regressor_cols = [
col for col in df.columns if col not in [TIME_COL, VALUE_COL]
]
result = get_forecast_time_properties(df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="W-SUN",
regressor_cols=regressor_cols,
forecast_horizon=forecast_horizon)
default_origin = get_default_origin_for_time_vars(df, TIME_COL)
assert result == {
"period": TimeEnum.ONE_WEEK_IN_SECONDS.value,
"simple_freq": SimpleTimeFrequencyEnum.WEEK,
"num_training_points": train_df.shape[0], # size of training set
"num_training_days": train_df.shape[0] * 7,
"days_per_observation": 7,
"forecast_horizon": 9,
"forecast_horizon_in_timedelta": timedelta(days=63),
"forecast_horizon_in_days": 63.0,
"start_year": 2018,
"end_year": 2019,
"origin_for_time_vars": default_origin
}
# checks `num_training_days` with `train_end_date`
data = generate_df_with_reg_for_tests(freq="H",
periods=300 * 24,
train_start_date=datetime.datetime(
2018, 7, 1),
remove_extra_cols=True,
mask_test_actuals=True)
df = data["df"]
train_end_date = datetime.datetime(2019, 2, 1)
result = get_forecast_time_properties(
df=df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
regressor_cols=data["regressor_cols"],
train_end_date=train_end_date,
forecast_horizon=forecast_horizon)
period = 3600 # seconds between observations
time_delta = (train_end_date - df[TIME_COL].min()
) # train end - train start
num_training_days = (
time_delta.days +
(time_delta.seconds + period) / TimeEnum.ONE_DAY_IN_SECONDS.value)
assert result["num_training_days"] == num_training_days
# checks `num_training_days` without `train_end_date`
result = get_forecast_time_properties(
df=df,
time_col=TIME_COL,
value_col=VALUE_COL,
freq="H",
regressor_cols=data["regressor_cols"],
train_end_date=None,
forecast_horizon=forecast_horizon)
time_delta = (
datetime.datetime(2019, 2, 26) - df[TIME_COL].min()
) # by default, train end is the last date with nonnull value_col
num_training_days = (
time_delta.days +
(time_delta.seconds + period) / TimeEnum.ONE_DAY_IN_SECONDS.value)
assert result["num_training_days"] == num_training_days