def test_performance_metrics(self): m = Prophet() m.fit(self.__df) df_cv = diagnostics.cross_validation( m, horizon='4 days', period='10 days', initial='90 days') # Aggregation level none df_none = diagnostics.performance_metrics(df_cv, rolling_window=0) self.assertEqual( set(df_none.columns), {'horizon', 'coverage', 'mae', 'mape', 'mse', 'rmse'}, ) self.assertEqual(df_none.shape[0], 16) # Aggregation level 0.2 df_horizon = diagnostics.performance_metrics(df_cv, rolling_window=0.2) self.assertEqual(len(df_horizon['horizon'].unique()), 4) self.assertEqual(df_horizon.shape[0], 14) # Aggregation level all df_all = diagnostics.performance_metrics(df_cv, rolling_window=1) self.assertEqual(df_all.shape[0], 1) for metric in ['mse', 'mape', 'mae', 'coverage']: self.assertEqual(df_all[metric].values[0], df_none[metric].mean()) # Custom list of metrics df_horizon = diagnostics.performance_metrics( df_cv, metrics=['coverage', 'mse'], ) self.assertEqual( set(df_horizon.columns), {'coverage', 'mse', 'horizon'}, )
def test_performance_metrics(self): m = Prophet() m.fit(self.__df) df_cv = diagnostics.cross_validation(m, horizon='4 days', period='10 days', initial='90 days') # Aggregation level none df_none = diagnostics.performance_metrics(df_cv, rolling_window=0) self.assertEqual( set(df_none.columns), {'horizon', 'coverage', 'mae', 'mape', 'mse', 'rmse'}, ) self.assertEqual(df_none.shape[0], 16) # Aggregation level 0.2 df_horizon = diagnostics.performance_metrics(df_cv, rolling_window=0.2) self.assertEqual(len(df_horizon['horizon'].unique()), 4) self.assertEqual(df_horizon.shape[0], 14) # Aggregation level all df_all = diagnostics.performance_metrics(df_cv, rolling_window=1) self.assertEqual(df_all.shape[0], 1) for metric in ['mse', 'mape', 'mae', 'coverage']: self.assertEqual(df_all[metric].values[0], df_none[metric].mean()) # Custom list of metrics df_horizon = diagnostics.performance_metrics( df_cv, metrics=['coverage', 'mse'], ) self.assertEqual( set(df_horizon.columns), {'coverage', 'mse', 'horizon'}, )
def test_performance_metrics(self): m = Prophet() m.fit(self.__df) df_cv = diagnostics.cross_validation(m, horizon='4 days', period='10 days', initial='90 days') # Aggregation level none df_none = diagnostics.performance_metrics(df_cv, rolling_window=-1) self.assertEqual( set(df_none.columns), {'horizon', 'coverage', 'mae', 'mape', 'mdape', 'mse', 'rmse'}, ) self.assertEqual(df_none.shape[0], 16) # Aggregation level 0 df_0 = diagnostics.performance_metrics(df_cv, rolling_window=0) self.assertEqual(len(df_0), 4) self.assertEqual(len(df_0['horizon'].unique()), 4) # Aggregation level 0.2 df_horizon = diagnostics.performance_metrics(df_cv, rolling_window=0.2) self.assertEqual(len(df_horizon), 4) self.assertEqual(len(df_horizon['horizon'].unique()), 4) # Aggregation level all df_all = diagnostics.performance_metrics(df_cv, rolling_window=1) self.assertEqual(df_all.shape[0], 1) for metric in ['mse', 'mape', 'mae', 'coverage']: self.assertAlmostEqual(df_all[metric].values[0], df_none[metric].mean()) self.assertAlmostEqual(df_all['mdape'].values[0], df_none['mdape'].median()) # Custom list of metrics df_horizon = diagnostics.performance_metrics( df_cv, metrics=['coverage', 'mse'], ) self.assertEqual( set(df_horizon.columns), {'coverage', 'mse', 'horizon'}, ) # Skip MAPE df_cv.loc[0, 'y'] = 0. df_horizon = diagnostics.performance_metrics( df_cv, metrics=['coverage', 'mape'], ) self.assertEqual( set(df_horizon.columns), {'coverage', 'horizon'}, ) df_horizon = diagnostics.performance_metrics( df_cv, metrics=['mape'], ) self.assertIsNone(df_horizon) # List of metrics containing non-valid metrics with self.assertRaises(ValueError): diagnostics.performance_metrics( df_cv, metrics=['mse', 'error_metric'], )
def FB_Model(train_x, train_y,train_ratio,predict_period,changepoint_prior_scale,analysisornot,holiday): time_split = int((train_x.shape[0]) * train_ratio) df = pd.DataFrame(columns=['ds', 'y']) df['ds'] = train_x[:time_split];df['y'] = train_y[:time_split] df['y'] = np.log((np.asarray(df['y'], dtype=float))) #log能让预测效果更好 Hoday = pd.DataFrame({ 'holiday': 'weekend', 'ds': pd.to_datetime(holiday), 'lower_window': 0, 'upper_window': 1, }) #训练 model = Prophet(changepoint_prior_scale=changepoint_prior_scale,mcmc_samples=0,holidays=Hoday)#默认为0,增大后将最大后验估计取代为马尔科夫蒙特卡洛取样,但是会极大地延长训练时间 model.fit(df) future = model.make_future_dataframe(freq='D', periods=predict_period) # 建立数据预测框架,数据粒度为天,预测步长为20天 forecast = model.predict(future) model.plot(forecast).show() # 绘制预测效果图 model.plot_components(forecast).show() #分析结果 if analysisornot: df_cv = cross_validation(model, initial='1100 days', period='20 days', horizon='20 days') df_p = performance_metrics(df_cv) print(df_cv.head()) print(df_p.head()) fig = plot_cross_validation_metric(df_cv, metric='mape') fig.show()
def fit_cv( self, param_grid=None, initial="80 days", horizon="14 days", period="14 day", rolling_window=1, ): if param_grid is None: param_grid = { "changepoint_prior_scale": [0.001, 0.01, 0.1, 0.5], "seasonality_prior_scale": [0.01, 0.1, 1.0, 10.0], "holidays_prior_scale": [0.01, 0.1, 1.0, 10.0], } all_params = [ dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values()) ] # We manually set this cap otherwise are not able # to see the plots clearly. If we were to set the correct cap, # we would have used the total pop: # self.df['cap'] = get_region_pop(province, pop_prov_df, prov_list_df) if self.growth == "logistic": if self.cap is None: self.cap = self.df["y"].max() + 20000 self.df["cap"] = self.cap rmses, mae, mse = [], [], [] for params in all_params: m = Prophet(self.growth, **params) if self.holidays: m.add_country_holidays(country_name="IT") m.fit(self.df) df_cv = cross_validation(m, initial=initial, horizon=horizon, period=period, parallel="processes") df_p = performance_metrics(df_cv, rolling_window=rolling_window) rmses.append(df_p["rmse"].values[0]) mae.append(df_p["mae"].values[0]) mse.append(df_p["mse"].values[0]) tuning_results = pd.DataFrame(all_params) tuning_results["rmse"] = rmses tuning_results["mae"] = mae tuning_results["mse"] = mse self.tuning_results = tuning_results self.best_params = all_params[np.argmin(rmses)] self.fit(**self.best_params)
def validate_model(model,dates): """ Background: This model validation function is still under construction and will be updated during a future release. """ count_of_time_units = len(dates) #print(count_of_time_units) initial_size = str(int(count_of_time_units * 0.20)) + " days" horizon_size = str(int(count_of_time_units * 0.10)) + " days" period_size = str(int(count_of_time_units * 0.05)) + " days" df_cv = cross_validation(model, initial=initial_size, horizon=horizon_size, period=period_size) #df_cv = cross_validation(model,initial='730 days', period='180 days', horizon = '365 days') df_p = performance_metrics(df_cv) #print(df_cv.head(100)) #print(df_p.head(100)) mape_score_avg = str(round(df_p['mape'].mean()*100,2)) + "%" return mape_score_avg
def getparameters_4ph( self): #return best_params and overide default parameters param_grid = { 'changepoint_prior_scale': [0.001, 0.01, 0.1, 0.3, 0.5, 1], 'seasonality_prior_scale': [0.01, 0.1, 1, 3, 6, 10, 20], } all_params = [ dict(zip(param_grid.keys(), v)) for v in itertools.product(*param_grid.values()) ] rmses = [] # Use cross validation to evaluate all parameters for params in all_params: m = Prophet(**params, daily_seasonality=True, weekly_seasonality=True, yearly_seasonality=True, growth='linear') m.add_seasonality(name='yearly', period=365.25, fourier_order=20) m = m.fit(self.ph_train) df_cv = cross_validation(m, initial='730 days', period='90 days', horizon='180 days') df_p = performance_metrics(df_cv, rolling_window=1) rmses.append(df_p['rmse'].values[0]) # Find the best parameters tuning_results = pd.DataFrame(all_params) tuning_results['rmse'] = rmses if disp: print(tuning_results) best_params = all_params[np.argmin(rmses)] if disp: print(best_params) self.ph_bestparameters = best_params return best_params
def train(self, metric_dict, oldest_data_datetime): """Train the Prophet model and store the predictions in predicted_df.""" prediction_freq = "1MIN" # convert incoming metric to Metric Object metric = Metric(metric_dict, oldest_data_datetime) self._model = Prophet(daily_seasonality=True, weekly_seasonality=True, yearly_seasonality=True) _LOGGER.info("training data range: %s - %s", metric.start_time, metric.end_time) _LOGGER.debug("begin training") df_fit = self._model.fit(metric.metric_values) if True: df_cv = cross_validation(self._model, horizon="1 day", period="8 hours", initial="4 days") df_p = performance_metrics(df_cv) _LOGGER.info("Performance data: %s %s", metric.metric_name, df_p)
def cross_validation(self): if self.cv is None: LOGGER.info("Cross validation not configured, skipping") return None try: horizon = self._get_timedelta(self.cv["horizon"]) period = self._get_timedelta(self.cv.get("period")) initial = self._get_timedelta(self.cv.get("initial")) except KeyError: raise ValueError( "Horizon is the required config for cross validation") self.cv_data = cross_validation(self.model, horizon=horizon, period=period, initial=initial) if self.cv_output_uri is not None: self.save(self.cv_data, "cv_output_uri") LOGGER.info(f"Cross validation data saved to {self.cv_output_uri}") rolling_window = self.cv.get("rolling_window") or 0.1 metrics = self.cv.get("metrics") self.cv_metrics = performance_metrics(self.cv_data, rolling_window=rolling_window, metrics=metrics) if self.cv_metrics_uri is not None: self.save(self.cv_metrics, "cv_metrics_uri") LOGGER.info( f"Cross validation metrics saved to {self.cv_metrics_uri}")
def prophet_cv_performance(model, train, minmax_pipe): #fit the model model.fit(train) #run cross validation #cv_results = cross_validation(model, initial='365 days', period='24 hours', horizon='24 hours') cv_results = cross_validation(model, initial='366 days', period='24 hours', horizon='24 hours') #inverse transform the target and prediction columns cv_results[['yhat', 'y']] = minmax_pipe.inverse_transform(cv_results[['yhat', 'y']]) #get performance results on the cv output cv_performance = performance_metrics(cv_results) #display the results print('Model Mean Hourly MAE {0:.2f}'.format(np.mean( cv_performance['mae']))) print('Model Mean Hourly MAPE {0:.2f}'.format( np.mean(cv_performance['mape']))) return cv_results, cv_performance
def evaluate_model(model): df_cv = cross_validation(model, initial="700 days", period="92 days", horizon="8 days") df_p = performance_metrics(df_cv) return df_cv, df_p
def fbp(df, p, freq): model = fbprophet.Prophet() model.fit(df) future = model.make_future_dataframe(periods=p, freq=freq, include_history=True) # future.tail() forecast = model.predict(future) # model.plot(forecast) # model.plot_components(forecast) # print(forecast) if freq == 'Y': time_format = '%Y' elif freq == 'M': time_format = '%Y-%m' elif freq == 'D': time_format = '%Y-%m-%d' df_cv = cross_validation(model, horizon='30 days') df_pe = performance_metrics(df_cv) df_cv.to_csv('C:/Users/47135/Desktop/df_cv.csv', encoding='UTF-8') df_pe.to_csv('C:/Users/47135/Desktop/df_pe.csv', encoding='UTF-8') forecast['ds'] = forecast['ds'].dt.strftime(time_format) result = forecast.to_dict(orient='list') # print(result) return result
def random_search(df, param_grid, max_evals=MAX_EVALS): """Random search for hyperparameter optimization""" results = pd.DataFrame(columns=['mape', 'params', 'iteration', 'model'], index=list(range(MAX_EVALS))) for i in range(max_evals): # Choose random hyperparameters hyperparameters = {k: sample(v, 1)[0] for k, v in param_grid.items()} # Evaluate randomly selected hyperparameters eval_results = objective(df, hyperparameters, i) results.loc[i, :] = eval_results # Model without param tuning m = Prophet() m.fit(df) df_cv = cross_validation(m, initial='{} days'.format(round(len(df) * 0.75)), horizon='{} days'.format(round(len(df) * 0.1))) df_p = performance_metrics(df_cv) mape = df_p.mape.mean() results.loc[max_evals, :] = [mape, None, None, m] # Sort with best score on top results.sort_values('mape', inplace=True) results.reset_index(inplace=True) return results
def model_prophet(series, holidays, weekly=0, monthly=0, dev_length=729, period='1 days', horizon='1 days', cvf=True, perf=True, val_dates=None): #define model modelf = Prophet(interval_width=0.95, daily_seasonality=False, weekly_seasonality=False, yearly_seasonality=False, holidays=holidays) if weekly != 0: modelf.add_seasonality(name='weeklyx', period=7, fourier_order=weekly) if monthly != 0: modelf.add_seasonality(name='monthlyx', period=30.5, fourier_order=monthly) modelf.fit(series) #test for horizon if cvf: cvf = cross_validation(modelf, initial=str(dev_length) + ' days', period=period, horizon=horizon) if perf: perf = performance_metrics(cvf) return modelf, cvf, perf
def evaluate_metrics(model): df_cv = cross_validation(model, initial='730 days', period='90 days', horizon='180 days') df_p = performance_metrics(df_cv) return df_p
def plot_residuals(self): print("CROSS VALIDATION RESULTS") df_cv = cross_validation(self.m, initial='365.25 days', period='365.25 days', horizon='365.25 days') # df_cv = cross_validation(self.m, initial='180 days', period='180 days', horizon='180 days') # self.out_table = df_cv print(pdtabulate(df_cv)) print("PERFORMANCE METRICS") df_p = performance_metrics(df_cv) print(pdtabulate(df_p)) # Mean absolute percentage error fig3 = plot_cross_validation_metric(df_cv, metric='mape', figsize=(11, 6)) ax3 = fig3.gca() blue_patch = mpatches.Patch(color='#5F86BC', label='Accuracy ~= ((1 - MAPE) * 100) %') plt.legend(handles=[blue_patch]) # ax3.legend("Accuracy ~= ((1 - MAPE) * 100) %") ax3.set_title( label="Prophet Forecast - Mean Absolute Percentage Error - " + self.name, fontsize=24) ax3.set_xlabel(xlabel="Forecast Horizon (days)", fontsize=16) ax3.set_ylabel(ylabel="MAPE", fontsize=16) ax3.set_ylim([0.0, 0.8]) plt.show()
def cross_validation_worker(model, initial, period, horizon, metric): df_cv = cross_validation(model, initial=initial, period=period, horizon=horizon) df_p = performance_metrics(df_cv) average_metric = df_p[metric].mean() return average_metric
def test_model(model): # Making 90 forecasts with cutoffs between 2008-03-11 00:00:00 and 2010-08-18 00:00:00 df_cv = cross_validation(model, initial='450 days', period='10 days', horizon = '100 days') df_cv.head() df_p = performance_metrics(df_cv) df_p.head()
def __cv_run(self, params): #交叉验证过程 model = Prophet(**params, holidays=self.holidays) model.fit(self.data) future = model.make_future_dataframe(freq=self.freq, periods=1) forecast = model.predict(future) data_merage = pd.concat([self.data, forecast[['yhat']]], axis=1, join='inner') mape_train = np.abs(data_merage['yhat'] / data_merage['y'] - 1).mean() initial_value = (self.data['ds'].max() - self.data['ds'].min()).days / 1.9 if 1.5 < initial_value < 5 or 7 < initial_value < 194 or initial_value > 368: initial_value = (self.data['ds'].max() - self.data['ds'].min()) / 1.9 horizon_weight = 1 - 1 / 1.9 df_cv = cross_validation(model, initial=(self.data['ds'].max()-sef.data['ds'].min())/1.9, \ period=(self.data['ds'].max()-self.data['ds'].min())/4.2, \ horizon =(self.data['ds'].max()-self.data['ds'].min())/2.2) elif initial_value <= 1.5: initial_value = '25 hours' period_value = '25 hours' horizon_value = self.data['ds'].max() - self.data['ds'].min( ) - datetime.timedelta(hours=25) horizon_weight = horizon_value / (self.data['ds'].max() - self.data['ds'].min()) df_cv = cross_validation(model, initial=initial_value, period=period_value, horizon=horizon_value) elif initial_value <= 7: initial_value = '8 days' period_value = '8 days' horizon_value = self.data['ds'].max() - self.data['ds'].min( ) - datetime.timedelta(days=8) horizon_weight = horizon_value / (self.data['ds'].max() - self.data['ds'].min()) df_cv = cross_validation(model, initial=initial_value, period=period_value, horizon=horizon_value) else: initial_value = '367 days' period_value = '367 days' horizon_value = self.data['ds'].max() - self.data['ds'].min( ) - datetime.timedelta(days=367) horizon_weight = horizon_value / (self.data['ds'].max() - self.data['ds'].min()) df_cv = cross_validation(model, initial=initial_value, period=period_value, horizon=horizon_value) mape = horizon_weight * performance_metrics(df_cv, metrics=[ 'mape' ])['mape'].mean() + (1 - horizon_weight) * mape_train return mape
def ts_diagnose(self): """Diagnoses the fitted model""" try: assert self.model_fit is not None except AssertionError: self._prophet_logger.exception( "Model has to be fitted first! Please call ts_fit(...)") sys.exit("STOP") self.plot_residuals() if self._diagnose: if input( "Run cross validation y/n? Note, depending on parameters provided " "this can take some time...").strip().lower() == 'y': start = time() self._prophet_logger.info( "Running cross validation using parameters provided....") if self._history is not None: try: self._prophet_cv = cross_validation( self.model_fit, initial=self._history, period=self._step, horizon=self._horizon) except Exception: self._prophet_logger.exception( "Prophet cross validation error: check your " "parameters 'history', 'horizon', 'step'!") else: try: self._prophet_cv = cross_validation( self.model_fit, period=self._step, horizon=self._horizon) except Exception: self._prophet_logger.exception( "Prophet cross validation error: " "check your parameters 'horizon', 'step'!") self._prophet_logger.info("Time elapsed: {}".format(time() - start)) simu_intervals = self._prophet_cv.groupby('cutoff')['ds'].agg([ ('forecast_start', 'min'), ('forecast_till', 'max') ]) self._prophet_logger.info( "Following time windows and cutoffs have been set-up:\n") print(simu_intervals) # plot_cross_validation_metric(self._prophet_cv, metric='mape') # self._prophet_logger.info("Running performance metrics...") self._prophet_p = performance_metrics(self._prophet_cv) else: self._prophet_logger.info("OK") return
def cross_val(self): df_cv = cross_validation(self.m, initial='62 days', period='1 days', horizon='7 days') # for col in ['yhat', 'yhat_lower', 'yhat_upper', 'y']: # df_cv[col] = inv_boxcox(df_cv[col], lmbda) print(df_cv.sort_values('ds').tail()) df_p = performance_metrics(df_cv) print(df_p)
def cross_validate(fitted_model, training_range, forecast_range, cv_interval): ''' Input --> 1 already defined model Output --> avg MAPE ''' # nee dot make sure that each initial, horizon covers full week?? df_cv = cross_validation(fitted_model, initial=training_range, horizon=forecast_range, period=cv_interval) df_p = performance_metrics(df_cv, rolling_window = 1/7) return df_p['mape'].mean()
def test_cross_validation_uncertainty_disabled(self): df = self.__df.copy() for uncertainty in [0, False]: m = Prophet(uncertainty_samples=uncertainty) m.fit(df, algorithm='Newton') df_cv = diagnostics.cross_validation( m, horizon='4 days', period='4 days', initial='115 days') expected_cols = ['ds', 'yhat', 'y', 'cutoff'] self.assertTrue(all(col in expected_cols for col in df_cv.columns.tolist())) df_p = diagnostics.performance_metrics(df_cv) self.assertTrue('coverage' not in df_p.columns)
def validator(self): self.__cv_metrics.df_cv = cross_validation(self.__model, initial="1 days", period="120 days", horizon="15 days") self.__cv_metrics.df_perf = performance_metrics( self.__cv_metrics.df_cv) self.__cv_metrics.manual_mape = self.mean_absolute_percentage_error( self.__cv_metrics.df_cv.y, self.__cv_metrics.df_cv.yhat)
def test_prophecy(ticker): df = get_daily_data(ticker, 90) df.rename(columns={'time': 'ds', 'close': 'y'}, inplace=True) m.fit(df) df_cv = cross_validation(m, horizon='10 days') df_p = performance_metrics(df_cv) df_p.head(5) plot_cross_validation_metric(df_cv, metric='mape') plt.show()
def cross_validate(df): prophet = Prophet() prophet.fit(df) df_cv = cross_validation(prophet, initial='30 days', period='4 days', horizon='7 days') df_performance = performance_metrics(df_cv) fig_performance = plot_cross_validation_metric(df_cv, metric='mape') return df_performance #print(predict(df_cases_fb))
def cross_validate(df): prophet = Prophet() prophet.fit(df) df_cv = cross_validation( prophet, initial="30 days", period="4 days", horizon="7 days" ) df_performance = performance_metrics(df_cv) fig_performance = plot_cross_validation_metric(df_cv, metric="mape") return df_performance
def fit_hex_model(hex_data: pd.DataFrame) -> Prophet: fit_data = hex_data[['timestamp', 'y']].copy() fit_data.rename(columns={'timestamp': 'ds'}, inplace=True) fit_data['cap'] = 1 fit_data['floor'] = 0 model = Prophet(n_changepoints=0, growth='logistic') forecaster = model.fit(fit_data, algorithm='Newton') df_cv = cross_validation(model, initial='40 days', period='7 days', horizon='7 days') df_metrics = performance_metrics(df_cv) print(df_metrics.mdape.mean())
def process_job(conn, job): assert job['type'].lower() in ['cases', 'deaths', 'tests'] print(f"{time.strftime('%H:%M:%S')} Starting job={job}") data = query_data(conn, job) df = prepare_data(job, data) m = create_model(job) m.fit(df) # predict a third into the future of what we looked back future_days = round(job['days_to_look_back'] / 3) future = m.make_future_dataframe(periods=future_days) future['cap'] = df['cap'][0] forecast = m.predict(future) # region debug if os.getenv('DOCKER_ACTIVE') is None: fig = m.plot(forecast) add_changepoints_to_plot(fig.gca(), m, forecast) fig.savefig(f"../job/prediction-{job['id']}.png") # endregion change_points = m.changepoints.dt.date.tolist() store_prediction(conn, job, forecast, change_points) # cross validate and create score if job['with_score']: # compute period to have 5-6 simulated forecasts horizon = pd.Timedelta("14 days") initial = horizon * 3 period = (df.iloc[-1]['ds'] - df.iloc[0]['ds'] - horizon - initial) / 5 df_cv = cross_validation(m, initial=initial, horizon=horizon, period=period, parallel='processes') df_p = performance_metrics(df_cv) # region debug if os.getenv('DOCKER_ACTIVE') is None: fig = plot_cross_validation_metric(df_cv, metric='mape') fig.savefig(f"../job/score-{job['id']}.png") # endregion score = df_p.iloc[-1]['mape'] store_score(conn, job, score)
def cv_cases(state): df = prepdata_cases(state) prophet = Prophet() prophet.fit(df) df_cv = cross_validation( prophet, initial="50 days", period="4 days", horizon="7 days" ) df_performance = performance_metrics(df_cv) fig_performance = plot_cross_validation_metric(df_cv, metric="mape") return plt.show()
def single_cv_run(history_df, metrics, param_dict, parallel): m = Prophet(**param_dict) m.add_country_holidays(country_name='BR') history_df['cap'] = 2 * history_df["y"].max() m.fit(history_df) df_cv = cross_validation(m, initial='3600 days', horizon='1200 days', parallel=parallel) df_p = performance_metrics(df_cv, rolling_window=1) df_p['params'] = str(param_dict) print(df_p.head()) df_p = df_p.loc[:, metrics] return df_p
def plot_cross_validation_metric(df_cv, metric, rolling_window=0.1, ax=None): """Plot a performance metric vs. forecast horizon from cross validation. Cross validation produces a collection of out-of-sample model predictions that can be compared to actual values, at a range of different horizons (distance from the cutoff). This computes a specified performance metric for each prediction, and aggregated over a rolling window with horizon. This uses fbprophet.diagnostics.performance_metrics to compute the metrics. Valid values of metric are 'mse', 'rmse', 'mae', 'mape', and 'coverage'. rolling_window is the proportion of data included in the rolling window of aggregation. The default value of 0.1 means 10% of data are included in the aggregation for computing the metric. As a concrete example, if metric='mse', then this plot will show the squared error for each cross validation prediction, along with the MSE averaged over rolling windows of 10% of the data. Parameters ---------- df_cv: The output from fbprophet.diagnostics.cross_validation. metric: Metric name, one of ['mse', 'rmse', 'mae', 'mape', 'coverage']. rolling_window: Proportion of data to use for rolling average of metric. In [0, 1]. Defaults to 0.1. ax: Optional matplotlib axis on which to plot. If not given, a new figure will be created. Returns ------- a matplotlib figure. """ if ax is None: fig = plt.figure(facecolor='w', figsize=(10, 6)) ax = fig.add_subplot(111) else: fig = ax.get_figure() # Get the metric at the level of individual predictions, and with the rolling window. df_none = performance_metrics(df_cv, metrics=[metric], rolling_window=0) df_h = performance_metrics(df_cv, metrics=[metric], rolling_window=rolling_window) # Some work because matplotlib does not handle timedelta # Target ~10 ticks. tick_w = max(df_none['horizon'].astype('timedelta64[ns]')) / 10. # Find the largest time resolution that has <1 unit per bin. dts = ['D', 'h', 'm', 's', 'ms', 'us', 'ns'] dt_names = [ 'days', 'hours', 'minutes', 'seconds', 'milliseconds', 'microseconds', 'nanoseconds' ] dt_conversions = [ 24 * 60 * 60 * 10 ** 9, 60 * 60 * 10 ** 9, 60 * 10 ** 9, 10 ** 9, 10 ** 6, 10 ** 3, 1., ] for i, dt in enumerate(dts): if np.timedelta64(1, dt) < np.timedelta64(tick_w, 'ns'): break x_plt = df_none['horizon'].astype('timedelta64[ns]').astype(np.int64) / float(dt_conversions[i]) x_plt_h = df_h['horizon'].astype('timedelta64[ns]').astype(np.int64) / float(dt_conversions[i]) ax.plot(x_plt, df_none[metric], '.', alpha=0.5, c='gray') ax.plot(x_plt_h, df_h[metric], '-', c='b') ax.grid(True) ax.set_xlabel('Horizon ({})'.format(dt_names[i])) ax.set_ylabel(metric) return fig