def test(self):
df_cv = cross_validation(self.model, horizon='365 days')
df = performance_metrics(df_cv)
df.drop(['horizon'], axis=1, inplace=True)
df.apply(np.sum, inplace=True)
print(df)
return df
def _eval_cross_validation(self, expected_horizon):
df_cv = cross_validation(self.model, horizon=expected_horizon)
df_p = performance_metrics(df_cv,
metrics=[self.metric],
rolling_window=1)
return {
self.metric: np.mean(df_p[self.metric].values)
} # here we use the mean metrics
def prophet_model(Train, Days):
# make sure to get the input here, then uncomment the rest
###
# HERE, convert the json that looks like this
# {"income":[{income fields},{},{}], "expense":spend_dict}
# to a dataframe
# the you can just do this: jsonify({"predictions": yourlist})
###
k_trans_pro = Train
n = len(k_trans_pro)
d = Days
pro_train_df = k_trans_pro[0:n - d]
pro_test_df_y = k_trans_pro[n - d:]
pro_test_df = k_trans_pro[n - d:].drop(['y'], axis=1)
param_grid = {
'changepoint_prior_scale': [0.001, 0.01, 0.1, 0.5],
'seasonality_prior_scale': [0.01, 0.1, 1.0, 10.0],
}
# Generate all combinations of parameters
all_params = [
dict(zip(param_grid.keys(), v))
for v in itertools.product(*param_grid.values())
]
rmses = [] # Store the RMSEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(pro_train_df) # Fit model with given params
df_cv = cross_validation(m,
period='30 days',
horizon='30 days',
parallel="processes")
df_p = performance_metrics(df_cv, rolling_window=1)
rmses.append(df_p['rmse'].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results['rmse'] = rmses
best_params = all_params[np.argmin(rmses)]
pro_model_tuned = Prophet(**best_params).fit(pro_train_df)
with open('serialized_model.json', 'w') as fout:
json.dump(model_to_json(pro_model_tuned), fout) # Save model
## Here you find the predictions
###
# HERE, convert the precitions to a json doc that looks like this
# { "predictions": [ {"ds": value, "yhat":value}, {"ds": value, "yhat":value}, {"ds": value, "yhat":value}, ]}
# just get a list from your dataframe.
# the you can just do this: jsonify({"predictions": yourlist})
return None
def test_cross_validation_uncertainty_disabled(self):
df = self.__df.copy()
for uncertainty in [0, False]:
m = Prophet(uncertainty_samples=uncertainty)
m.fit(df, algorithm='Newton')
df_cv = diagnostics.cross_validation(m,
horizon='4 days',
period='4 days',
initial='115 days')
expected_cols = ['ds', 'yhat', 'y', 'cutoff']
self.assertTrue(
all(col in expected_cols for col in df_cv.columns.tolist()))
df_p = diagnostics.performance_metrics(df_cv)
self.assertTrue('coverage' not in df_p.columns)
def _single_model_performance_evaluation(cv_df, metrics, rolling_window, monthly):
return performance_metrics(cv_df, metrics, rolling_window, monthly)
def _cross_validate_and_score_model(
model,
horizon,
period=None,
initial=None,
parallel=None,
cutoffs=None,
metrics=None,
**kwargs,
):
"""
Wrapper around Prophet's `cross_validation` and `performance_metrics` functions within
the `prophet.diagnostics` module.
Provides backtesting metric evaluation based on the configurations specified for
initial, horizon, and period (optionally, a specified 'cutoffs' list of DateTime or string
date-time entries can override the backtesting split boundaries for training and validation).
:param model: Prophet model instance that has been fit
:param horizon: String pd.Timedelta format that defines the length of forecasting values
to generate in order to acquire error metrics.
examples: '30 days', '1 year'
:param period: the periodicity of how often a windowed validation will occur. Default is
0.5 * horizon value.
:param initial: The minimum amount of training data to include in the first cross validation
window.
:param parallel: mode of computing cross validation statistics. (None, processes, or threads)
:param cutoffs: List of pd.Timestamp values that specify cutoff overrides to be used in
conducting cross validation.
:param metrics: List of metrics to evaluate and return for the provided model
note: see supported metrics in Prophet documentation:
https://facebook.github.io/prophet/docs/diagnostics.html#cross-validation
:param kwargs: cross validation overrides for Prophet's implementation of metric evaluation.
:return: Dict[str, float] of each metric and its value averaged over each time horizon.
"""
if metrics:
metrics = prophet_config_utils._remove_coverage_metric_if_necessary(
metrics, model.uncertainty_samples
)
# extract `performance_metrics` *args if present
performance_metrics_defaults = signature(performance_metrics).parameters
performance_metrics_args = {}
for param, value in performance_metrics_defaults.items():
if value.default != value.empty and value.name != "metrics":
performance_metrics_args[param] = kwargs.pop(param, value.default)
model_cv = cross_validation(
model=model,
horizon=horizon,
period=period,
initial=initial,
parallel=parallel,
cutoffs=cutoffs,
disable_tqdm=kwargs.pop("disable_tqdm", True),
)
horizon_metrics = performance_metrics(
model_cv, metrics=metrics, **performance_metrics_args
)
return {
metric: horizon_metrics[metric].mean()
for metric in list(horizon_metrics.columns)
if metric != "horizon"
}
with mlflow.start_run():
model = Prophet().fit(sales_data)
params = extract_params(model)
metric_keys = ["mse", "rmse", "mae", "mape", "mdape", "smape", "coverage"]
metrics_raw = cross_validation(
model=model,
horizon="365 days",
period="180 days",
initial="710 days",
parallel="threads",
disable_tqdm=True,
)
cv_metrics = performance_metrics(metrics_raw)
metrics = {k: cv_metrics[k].mean() for k in metric_keys}
print(f"Logged Metrics: \n{json.dumps(metrics, indent=2)}")
print(f"Logged Params: \n{json.dumps(params, indent=2)}")
mlflow.prophet.log_model(model, artifact_path=ARTIFACT_PATH)
mlflow.log_params(params)
mlflow.log_metrics(metrics)
model_uri = mlflow.get_artifact_uri(ARTIFACT_PATH)
print(f"Model artifact logged to: {model_uri}")
loaded_model = mlflow.prophet.load_model(model_uri)
forecast = loaded_model.predict(loaded_model.make_future_dataframe(60))
def plot_cross_validation_metric(df_cv,
metric,
rolling_window=0.1,
ax=None,
figsize=(10, 6),
color='b'):
"""Plot a performance metric vs. forecast horizon from cross validation.
Cross validation produces a collection of out-of-sample model predictions
that can be compared to actual values, at a range of different horizons
(distance from the cutoff). This computes a specified performance metric
for each prediction, and aggregated over a rolling window with horizon.
This uses prophet.diagnostics.performance_metrics to compute the metrics.
Valid values of metric are 'mse', 'rmse', 'mae', 'mape', and 'coverage'.
rolling_window is the proportion of data included in the rolling window of
aggregation. The default value of 0.1 means 10% of data are included in the
aggregation for computing the metric.
As a concrete example, if metric='mse', then this plot will show the
squared error for each cross validation prediction, along with the MSE
averaged over rolling windows of 10% of the data.
Parameters
----------
df_cv: The output from prophet.diagnostics.cross_validation.
metric: Metric name, one of ['mse', 'rmse', 'mae', 'mape', 'coverage'].
rolling_window: Proportion of data to use for rolling average of metric.
In [0, 1]. Defaults to 0.1.
ax: Optional matplotlib axis on which to plot. If not given, a new figure
will be created.
figsize: Optional tuple width, height in inches.
color: Optional color for plot and error points, useful when plotting
multiple model performances on one axis for comparison.
Returns
-------
a matplotlib figure.
"""
if ax is None:
fig = plt.figure(facecolor='w', figsize=figsize)
ax = fig.add_subplot(111)
else:
fig = ax.get_figure()
# Get the metric at the level of individual predictions, and with the rolling window.
df_none = performance_metrics(df_cv, metrics=[metric], rolling_window=-1)
df_h = performance_metrics(df_cv,
metrics=[metric],
rolling_window=rolling_window)
# Some work because matplotlib does not handle timedelta
# Target ~10 ticks.
tick_w = max(df_none['horizon'].astype('timedelta64[ns]')) / 10.
# Find the largest time resolution that has <1 unit per bin.
dts = ['D', 'h', 'm', 's', 'ms', 'us', 'ns']
dt_names = [
'days', 'hours', 'minutes', 'seconds', 'milliseconds', 'microseconds',
'nanoseconds'
]
dt_conversions = [
24 * 60 * 60 * 10**9,
60 * 60 * 10**9,
60 * 10**9,
10**9,
10**6,
10**3,
1.,
]
for i, dt in enumerate(dts):
if np.timedelta64(1, dt) < np.timedelta64(tick_w, 'ns'):
break
x_plt = df_none['horizon'].astype('timedelta64[ns]').astype(
np.int64) / float(dt_conversions[i])
x_plt_h = df_h['horizon'].astype('timedelta64[ns]').astype(
np.int64) / float(dt_conversions[i])
ax.plot(x_plt, df_none[metric], '.', alpha=0.1, c=color)
ax.plot(x_plt_h, df_h[metric], '-', c=color)
ax.grid(True)
ax.set_xlabel('Horizon ({})'.format(dt_names[i]))
ax.set_ylabel(metric)
return fig
def test_performance_metrics(self):
m = Prophet()
m.fit(self.__df)
df_cv = diagnostics.cross_validation(m,
horizon='4 days',
period='10 days',
initial='90 days')
# Aggregation level none
df_none = diagnostics.performance_metrics(df_cv, rolling_window=-1)
self.assertEqual(
set(df_none.columns),
{
'horizon', 'coverage', 'mae', 'mape', 'mdape', 'mse', 'rmse',
'smape'
},
)
self.assertEqual(df_none.shape[0], 16)
# Aggregation level 0
df_0 = diagnostics.performance_metrics(df_cv, rolling_window=0)
self.assertEqual(len(df_0), 4)
self.assertEqual(len(df_0['horizon'].unique()), 4)
# Aggregation level 0.2
df_horizon = diagnostics.performance_metrics(df_cv, rolling_window=0.2)
self.assertEqual(len(df_horizon), 4)
self.assertEqual(len(df_horizon['horizon'].unique()), 4)
# Aggregation level all
df_all = diagnostics.performance_metrics(df_cv, rolling_window=1)
self.assertEqual(df_all.shape[0], 1)
for metric in ['mse', 'mape', 'mae', 'coverage']:
self.assertAlmostEqual(df_all[metric].values[0],
df_none[metric].mean())
self.assertAlmostEqual(df_all['mdape'].values[0],
df_none['mdape'].median())
# Custom list of metrics
df_horizon = diagnostics.performance_metrics(
df_cv,
metrics=['coverage', 'mse'],
)
self.assertEqual(
set(df_horizon.columns),
{'coverage', 'mse', 'horizon'},
)
# Skip MAPE
df_cv.loc[0, 'y'] = 0.
df_horizon = diagnostics.performance_metrics(
df_cv,
metrics=['coverage', 'mape'],
)
self.assertEqual(
set(df_horizon.columns),
{'coverage', 'horizon'},
)
df_horizon = diagnostics.performance_metrics(
df_cv,
metrics=['mape'],
)
self.assertIsNone(df_horizon)
# List of metrics containing non-valid metrics
with self.assertRaises(ValueError):
diagnostics.performance_metrics(
df_cv,
metrics=['mse', 'error_metric'],
)
from prophet.diagnostics import cross_validation, performance_metrics
# float_precision='high' required for pd.read_csv to match precision of Rover.read_csv
df = pd.read_csv('examples/example_wp_log_peyton_manning.csv',
float_precision='high')
cutoffs = pd.to_datetime(['2013-02-15', '2013-08-15', '2014-02-15'])
param_grid = {
'changepoint_prior_scale': [0.001, 0.01, 0.1, 0.5],
'seasonality_prior_scale': [0.01, 0.1, 1.0, 10.0],
}
# Generate all combinations of parameters
all_params = [
dict(zip(param_grid.keys(), v))
for v in itertools.product(*param_grid.values())
]
rmses = [] # Store the RMSEs for each params here
# Use cross validation to evaluate all parameters
for params in all_params:
m = Prophet(**params).fit(df) # Fit model with given params
df_cv = cross_validation(m, cutoffs=cutoffs, horizon='30 days')
df_p = performance_metrics(df_cv, rolling_window=1)
rmses.append(df_p['rmse'].values[0])
# Find the best parameters
tuning_results = pd.DataFrame(all_params)
tuning_results['rmse'] = rmses
print(tuning_results)
import pandas as pd
from prophet import Prophet
from prophet.diagnostics import cross_validation, performance_metrics
# float_precision='high' required for pd.read_csv to match precision of Rover.read_csv
df = pd.read_csv('examples/example_wp_log_peyton_manning.csv',
float_precision='high')
m = Prophet()
m.fit(df, seed=123)
df_cv = cross_validation(m,
initial='730 days',
period='180 days',
horizon='365 days')
print(len(df_cv))
print(df_cv.head())
print(df_cv.tail())
df_p = performance_metrics(df_cv)
print(len(df_p))
print(df_p.head())
print(df_p.tail())
