def main():
path_example = "../examples"
"""example 1"""
df = read_dataframe(path_examples=path_example, data_idx=4)
model, future, forecasted = forecast(
df, periods=1096,
showflag=True) # uncertainty intervals seem way too wide
df.loc[(df['ds'] > '2010-01-01') & (df['ds'] < '2011-01-01'), 'y'] = None
model_removed = Prophet().fit(df)
fig = model_removed.plot(
model_removed.predict(future)
) # model with missing data. prediction of whole data with future.
fig.set_figheight(18)
fig.set_figwidth(9)
plt.title('prediction (model with missing data)')
plt.show()
"""example 2"""
df2 = read_dataframe(path_examples=path_example, data_idx=5)
model2, future2, forecasted2 = forecast(
df2, periods=1096,
showflag=True) # extreme outlieres in June 2015 mess up estimate.
df2.loc[(df2['ds'] > '2015-06-01') & (df2['ds'] < '2015-06-30'),
'y'] = None
model2_removed = Prophet().fit(df2) # Same approach as previous example
fig = model2_removed.plot(model2_removed.predict(future2))
fig.set_figheight(18)
fig.set_figwidth(9)
plt.title('prediction2 (model with missing data)')
plt.show()
def test_fit_predict_no_changepoints_mcmc(self):
N = DATA.shape[0]
train = DATA.head(N // 2)
future = DATA.tail(N // 2)
forecaster = Prophet(n_changepoints=0, mcmc_samples=100)
forecaster.fit(train)
forecaster.predict(future)
def test_fit_predict_no_seasons(self):
N = DATA.shape[0]
train = DATA.head(N // 2)
future = DATA.tail(N // 2)
forecaster = Prophet(weekly_seasonality=False,
yearly_seasonality=False)
forecaster.fit(train)
forecaster.predict(future)
def test_fit_predict_duplicates(self):
N = DATA.shape[0]
train1 = DATA.head(N // 2).copy()
train2 = DATA.head(N // 2).copy()
train2['y'] += 10
train = train1.append(train2)
future = pd.DataFrame({'ds': DATA['ds'].tail(N // 2)})
forecaster = Prophet()
forecaster.fit(train)
forecaster.predict(future)
def test_fit_predict_constant_history(self):
N = DATA.shape[0]
train = DATA.head(N // 2).copy()
train['y'] = 20
future = pd.DataFrame({'ds': DATA['ds'].tail(N // 2)})
m = Prophet()
m.fit(train)
fcst = m.predict(future)
self.assertEqual(fcst['yhat'].values[-1], 20)
train['y'] = 0
future = pd.DataFrame({'ds': DATA['ds'].tail(N // 2)})
m = Prophet()
m.fit(train)
fcst = m.predict(future)
self.assertEqual(fcst['yhat'].values[-1], 0)
def predict_in_france(data,
country,
vaccination_metric,
future_days,
plot=True):
df = data[(data['country'] == country)]
df = df[['date', vaccination_metric]]
df.columns = ['ds', 'y']
model = Prophet(interval_width=0.95)
model.fit(df)
layout = dict(
title=
'Prediction : Nombre de personnes vaccinées par cent dans les 60 prochains jours (jusqu'
"'"
'au 28/06/21)',
xaxis=dict(title='Dates'),
yaxis=dict(title='Pourcentage'))
future = model.make_future_dataframe(periods=future_days)
forecast = model.predict(future)
if plot:
fig = plot_plotly(model, forecast)
fig.layout = layout
fig.write_html("analyse/Prediction_France.html")
else:
return forecast
def get_forecast(df, month, country="ALL"):
df_ts = df.copy() if country=="ALL" else df[df['country']==country].copy()
if len(df_ts) <= 180:
return None
actual = df_ts[df_ts['inv_month'] == month.isoformat()]['value'].sum()
df_ts = df_ts[['inv_month', 'inv_date', 'value']].groupby(['inv_month', 'inv_date']).sum().reset_index()
df_train = df_ts[df_ts['inv_month'] < month.isoformat()]
df_train['inv_date'] = pd.to_datetime(df_train['inv_date'])
df_train.rename(columns={'inv_date':'ds', 'value':'y'}, inplace=True)
m = Prophet(yearly_seasonality=20)
m.fit(df_train)
future = m.make_future_dataframe(periods=60, freq='D')
df_forecast = m.predict(future)
df_forecast['inv_month'] = df_forecast['ds'].apply(lambda v: date(v.year, v.month, 1).isoformat())
forecast = df_forecast[df_forecast['inv_month'] == month.isoformat()]['yhat'].sum()
exp_model = SimpleExpSmoothing(df_ts[['inv_month', 'inv_date', 'value']].set_index(['inv_month', 'inv_date'])).fit(smoothing_level=0.2, optimized=False)
exp_forecast = forecast_df = sum(exp_model.forecast(30))
return actual, forecast, exp_forecast
def make_forecast(df, len_forecast: int, time_series_label: str):
"""
Function for making time series forecasting with Prophet library
:param df: dataframe to process
:param len_forecast: forecast length
:param time_series_label: name of time series to process
:return predicted_values: forecast
:return model_name: name of the model (always 'AutoTS')
"""
df['ds'] = df['datetime']
df['y'] = df[time_series_label]
prophet_model = Prophet()
prophet_model.fit(df)
future = prophet_model.make_future_dataframe(periods=len_forecast,
include_history=False)
forecast = prophet_model.predict(future)
predicted_values = np.array(forecast['yhat'])
model_name = 'Prophet'
return predicted_values, model_name
def _estimate_weekly_pattern(self, time, energy_per_day, daily_masks):
epd = energy_per_day.copy()
for i in range(daily_masks.shape[0]):
if daily_masks[i] < 1:
epd[i] = np.nan
daily_times = [i*self.vpd for i in range(daily_masks.shape[0])]
try:
time_index = pd.to_datetime(
time.take(daily_times), format='%Y-%m-%d %H:%M:%S')
except ValueError:
time_index = pd.to_datetime(
time.take(daily_times), format='%d-%b-%Y %H:%M:%S')
time_index.reset_index(drop=True, inplace=True)
daily_data = pd.concat([time_index, pd.Series(
epd)], keys=['ds', 'y'], axis=1)
prophet_model = Prophet(weekly_seasonality=True,
yearly_seasonality=True)
if daily_data.count()['y'] > 1:
prophet_model.fit(daily_data)
# `future` starts with the first day of the time series.
# This allows us to select the corresponding pattern value
# for day `i` by accessing `weekly[i % 7]`.
future = prophet_model.make_future_dataframe(periods=0)
forecast = prophet_model.predict(future)
return forecast.get('weekly').head(7)
return pd.Series(np.zeros(7))
def _prophet_fit_and_predict( # pylint: disable=too-many-arguments
df: DataFrame,
confidence_interval: float,
yearly_seasonality: Union[bool, str, int],
weekly_seasonality: Union[bool, str, int],
daily_seasonality: Union[bool, str, int],
periods: int,
freq: str,
) -> DataFrame:
"""
Fit a prophet model and return a DataFrame with predicted results.
"""
try:
prophet_logger = logging.getLogger("prophet.plot")
prophet_logger.setLevel(logging.CRITICAL)
from prophet import Prophet # pylint: disable=import-error
prophet_logger.setLevel(logging.NOTSET)
except ModuleNotFoundError:
raise QueryObjectValidationError(_("`prophet` package not installed"))
model = Prophet(
interval_width=confidence_interval,
yearly_seasonality=yearly_seasonality,
weekly_seasonality=weekly_seasonality,
daily_seasonality=daily_seasonality,
)
if df["ds"].dt.tz:
df["ds"] = df["ds"].dt.tz_convert(None)
model.fit(df)
future = model.make_future_dataframe(periods=periods, freq=freq)
forecast = model.predict(future)[[
"ds", "yhat", "yhat_lower", "yhat_upper"
]]
return forecast.join(df.set_index("ds"), on="ds").set_index(["ds"])
def forecast(self, forecast_horizon: int = 96):
super().forecast(forecast_horizon)
print("Running Prophet forecast for Currency-pair: {} using forecast horizon: {}", self.currency_pair.upper(),
forecast_horizon)
print("Dataset: ", self.currency_pair.upper())
print(self.training_data.head(5))
print(".....\t.........\t...")
print(self.training_data.tail(5))
# model = Prophet(interval_width=0.99, mcmc_samples=60)
model = Prophet(interval_width=0.99)
model.fit(self.training_data)
future = model.make_future_dataframe(periods=forecast_horizon)
future.tail()
_forecast = model.predict(future)
last_n = _forecast.tail(forecast_horizon)
last_n.to_csv(f"output/{self.currency_pair}__{self.model_name.lower()}__{forecast_horizon}__forecasts.csv")
# last_n = _forecast[["ds", "yhat", "yhat_lower", "yhat_upper"]].tail(n)
print(last_n)
self.forecasts = last_n["yhat"]
self.forecasts_upper = last_n["yhat_upper"]
self.forecasts_lower = last_n["yhat_lower"]
self.errors = [(abs(upper - lower) / 2) for upper, lower in zip(self.forecasts_upper, self.forecasts_lower)]
self.forecasts_raw = last_n
def predict_trend(self):
object_type = 'PyTrends.trend_ratio'
# Pull and prep data
conn = sqlite3.connect("db.sqlite3")
df = pd.read_sql_query(
"select date as ds, btc_usd, buy_bitcoin, trend_ratio from crypto_track_pytrends;",
conn)
df['y'] = df['trend_ratio']
df['ds'] = pd.to_datetime(df['ds'], errors='coerce')
# Get prior 'y' value to fill in nan
df['y-1'] = df.y.shift(1)
df['y_nan'] = df.y.isna()
df.y.fillna(df['y-1'], inplace=True)
df.drop(['y-1'], axis=1, inplace=True)
# Instantiate a Prophet object
future_trend = Prophet(df, 'Trend Ratio')
# Change default attributes as analyzed in Trend Prediction.ipynb notebook.
future_trend.weekly_seasonality = True
future_trend.training_years = 6
future_trend.changepoint_prior_scale = 0.05
future, train, model = future_trend.predict(self.prediction_days)
# Create predictions and load table.
return_message = self.dbload_prophet(df=future,
object_type=object_type)
return return_message
def test_logistic_floor(self):
m = Prophet(growth='logistic')
N = DATA.shape[0]
history = DATA.head(N // 2).copy()
history['floor'] = 10.
history['cap'] = 80.
future = DATA.tail(N // 2).copy()
future['cap'] = 80.
future['floor'] = 10.
m.fit(history, algorithm='Newton')
self.assertTrue(m.logistic_floor)
self.assertTrue('floor' in m.history)
self.assertAlmostEqual(m.history['y_scaled'][0], 1.)
self.assertEqual(m.fit_kwargs, {'algorithm': 'Newton'})
fcst1 = m.predict(future)
m2 = Prophet(growth='logistic')
history2 = history.copy()
history2['y'] += 10.
history2['floor'] += 10.
history2['cap'] += 10.
future['cap'] += 10.
future['floor'] += 10.
m2.fit(history2, algorithm='Newton')
self.assertAlmostEqual(m2.history['y_scaled'][0], 1.)
def fit_prophet(dtf_train, dtf_test, lst_exog=None, model=None, freq="D", conf=0.95, figsize=(15,10)):
## setup prophet
if model is None:
model = Prophet(growth="linear", changepoints=None, n_changepoints=25, seasonality_mode="multiplicative",
yearly_seasonality="auto", weekly_seasonality="auto", daily_seasonality="auto",
holidays=None, interval_width=conf)
if lst_exog != None:
for regressor in lst_exog:
model.add_regressor(regressor)
## train
model.fit(dtf_train)
## test
dtf_prophet = model.make_future_dataframe(periods=len(dtf_test)+10, freq=freq, include_history=True)
if model.growth == "logistic":
dtf_prophet["cap"] = dtf_train["cap"].unique()[0]
if lst_exog != None:
dtf_prophet = dtf_prophet.merge(dtf_train[["ds"]+lst_exog], how="left")
dtf_prophet.iloc[-len(dtf_test):][lst_exog] = dtf_test[lst_exog].values
dtf_prophet = model.predict(dtf_prophet)
dtf_train = dtf_train.merge(dtf_prophet[["ds","yhat"]], how="left").rename(
columns={'yhat':'model', 'y':'ts'}).set_index("ds")
dtf_test = dtf_test.merge(dtf_prophet[["ds","yhat","yhat_lower","yhat_upper"]], how="left").rename(
columns={'yhat':'forecast', 'y':'ts', 'yhat_lower':'lower', 'yhat_upper':'upper'}).set_index("ds")
## evaluate
dtf = dtf_train.append(dtf_test)
dtf = utils_evaluate_ts_model(dtf, conf=conf, figsize=figsize, title="Prophet")
return dtf, model
def test_prophet_trend_onnx(self):
df = self._get_data()
m = Prophet()
m.fit(df)
future = m.make_future_dataframe(periods=365)
future_np = (future.values -
np.datetime64("1970-01-01T00:00:00.000000000")).astype(
np.int64) / 1000000000
# Convert with Hummingbird.
hb_model = hummingbird.ml.convert(m, "onnx", future_np)
# Predictions.
prophet_trend = m.predict(future)["trend"]
hb_trend = hb_model.predict(future_np)
import onnx
onnx.save(hb_model.model, "prophet.onnx")
np.testing.assert_allclose(prophet_trend,
hb_trend,
rtol=1e-06,
atol=1e-06)
def get_prophet_forecast(df, country="ALL", save_model=True):
df_ts = df.copy() if country == "ALL" else df[df['country'] ==
country].copy()
if len(df_ts) <= 180:
return None
df_month = df_ts.groupby('inv_month').sum().reset_index()
df_ts['inv_date'] = pd.to_datetime(df_ts['inv_date'])
df_ts.rename(columns={'inv_date': 'ds', 'value': 'y'}, inplace=True)
m = Prophet(yearly_seasonality=20)
m.fit(df_ts)
if save_model:
with open(fr'models\{country}_model.json', 'w') as fout:
json.dump(model_to_json(m), fout) # Save model
future = m.make_future_dataframe(periods=180, freq='D')
forecast = m.predict(future)
forecast['inv_month'] = forecast['ds'].apply(
lambda v: date(v.year, v.month, 1).isoformat())
monthly_forecast = forecast[['inv_month', 'yhat'
]].groupby('inv_month').sum().reset_index()
df_forecast = pd.merge(monthly_forecast,
df_month,
on='inv_month',
how='left')
return df_forecast
def predict(ticker, start_date):
today = datetime.date.today()
end_date = today.strftime("%Y-%m-%d")
data = yf.download(ticker, start_date, end_date)
df_forecast = data.copy()
df_forecast.reset_index(inplace=True)
df_forecast["ds"] = df_forecast["Date"]
df_forecast["y"] = df_forecast["Adj Close"]
df_forecast = df_forecast[["ds", "y"]]
df_forecast
model = Prophet()
model.fit(df_forecast)
future = pd.to_datetime(end_date) + pd.DateOffset(days=7)
future_date = future.strftime("%Y-%m-%d")
dates = pd.date_range(start=end_date, end=future_date)
df_pred = pd.DataFrame({"ds": dates})
forecast = model.predict(df_pred)
prediction_list = forecast.tail(7).to_dict("records")
output = {}
for data in prediction_list:
date = data["ds"].strftime("%Y-%m-%d")
output[date] = data["trend"]
return output
def generate_forecast(df: pd.DataFrame, length: int) -> pd.DataFrame:
"""Takes in a training set (data - value) and a length to
return a forecast DataFrame"""
model = Prophet(interval_width=0.50)
model.fit(df)
return model.predict(model.make_future_dataframe(periods=length))
def test_subdaily_holidays(self):
holidays = pd.DataFrame({
'ds': pd.to_datetime(['2017-01-02']),
'holiday': ['special_day'],
})
m = Prophet(holidays=holidays)
m.fit(DATA2)
fcst = m.predict()
self.assertEqual(sum(fcst['special_day'] == 0), 575)
def prophet_forecast(self, train_start, train_end, test_start, test_end):
from sys import platform
if platform == "linux" or platform == "linux2":
from prophet import Prophet # linux
elif platform == "darwin":
from fbprophet import Prophet # OS X
elif platform == "win32":
from fbprophet import Prophet # Windows...
data = self.data.reset_index()
data.rename(columns={'Date': 'ds', self.column: 'y'}, inplace=True)
# FIXME and take user input
size = len(data)
df_train = data.iloc[int(-size * .10):, :]
df_test = data.iloc[int(-size * .20):, :]
model_prophet = Prophet(seasonality_mode='additive')
model_prophet.add_seasonality(name='monthly',
period=30.5,
fourier_order=5)
model_prophet.fit(df_train)
df_future = model_prophet.make_future_dataframe(periods=365)
df_pred = model_prophet.predict(df_future)
model_prophet.plot(df_pred)
plt.tight_layout()
plt.title('Prophet Forecast')
plt.savefig(os.path.join(img_dirp, f'img/prophet_forecast.png'))
model_prophet.plot_components(df_pred)
plt.tight_layout()
plt.savefig(os.path.join(img_dirp, 'img/components.png'))
# merge test set with predicted data and plot accuracy of model's predictions
selected_columns = ['ds', 'yhat_lower', 'yhat_upper', 'yhat']
df_pred = df_pred.loc[:, selected_columns].reset_index(drop=True)
df_test = df_test.merge(df_pred, on=['ds'], how='left')
df_test.ds = pd.to_datetime(df_test.ds)
df_test.set_index('ds', inplace=True)
fig, ax = plt.subplots(1, 1)
ax = sns.lineplot(
data=df_test[['y', 'yhat_lower', 'yhat_upper', 'yhat']])
ax.fill_between(df_test.index,
df_test.yhat_lower,
df_test.yhat_upper,
alpha=0.3)
ax.set(title=f'{self.column} - actual vs. predicted',
xlabel='Date',
ylabel='{self.column}')
plt.tight_layout()
plt.savefig(os.path.join(img_dirp, 'img/actual_v_predicted.png'))
def fit_predict_model(dataframe, interval_width=0.99, changepoint_range=0.8):
m = Prophet(daily_seasonality=False,
yearly_seasonality=False,
weekly_seasonality=False,
seasonality_mode='multiplicative',
interval_width=interval_width,
changepoint_range=changepoint_range)
m = m.fit(dataframe)
forecast = m.predict(dataframe)
forecast['fact'] = dataframe['y'].reset_index(drop=True)
return forecast
def getTickerForcast(ticker, period, dataPeriod):
stockTicker = yf.Ticker(ticker)
data = stockTicker.history(period=dataPeriod, interval="1d")
df = pd.DataFrame({'ds': data.index.values, 'y': data['Close'].values})
m = Prophet(daily_seasonality=True)
m.fit(df)
future = m.make_future_dataframe(periods=period)
prediction = m.predict(future)
return pd.DataFrame({
'date': prediction['ds'].values,
'close': prediction['yhat'].values
}).to_dict('records')
def run_prophet(data, predict_period=40):
data = data.reset_index()
print("data columns", data.columns)
data = data[["Date", "Adj Close"]]
data.columns = ["ds", "y"]
m = Prophet()
m.fit(data)
future = m.make_future_dataframe(periods=predict_period)
future.tail()
forecast = m.predict(future)
# forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()
forecast = forecast[['ds', 'trend', 'yhat', 'yhat_lower', 'yhat_upper']]
return forecast, m
def forecast(df, periods, showflag=True):
m = Prophet()
m.fit(df)
future = m.make_future_dataframe(periods=periods)
predict = m.predict(future)
if showflag:
fig = m.plot(predict)
fig.set_figheight(18)
fig.set_figwidth(9)
plt.title("forcasted")
plt.show()
return m, future, predict
def prophet_model(df, fh, seasonality):
df = df.reset_index(drop=False)
df.columns = ["ds", "y"]
model = Prophet(
daily_seasonality=False,
yearly_seasonality=True,
weekly_seasonality=False,
interval_width=0.95,
)
model = model.add_seasonality(
name="custom", period=seasonality, fourier_order=5, prior_scale=0.02
)
model.fit(df.iloc[:-fh])
return model.predict(df[-fh:][["ds"]])["yhat"].values
class MetricPredictor:
"""docstring for Predictor."""
model_name = "prophet"
model_description = "Forecasted value from Prophet model"
model = None
predicted_df = None
metric = None
def __init__(self, metric, rolling_data_window_size="10d"):
"""Initialize the Metric object."""
self.metric = Metric(metric, rolling_data_window_size)
def train(self, metric_data=None, prediction_duration=15):
"""Train the Prophet model and store the predictions in predicted_df."""
prediction_freq = "1MIN"
# convert incoming metric to Metric Object
if metric_data:
# because the rolling_data_window_size is set, this df should not bloat
self.metric += Metric(metric_data)
# Don't really need to store the model, as prophet models are not retrainable
# But storing it as an example for other models that can be retrained
self.model = Prophet(daily_seasonality=True,
weekly_seasonality=True,
yearly_seasonality=True)
_LOGGER.info("training data range: %s - %s", self.metric.start_time,
self.metric.end_time)
# _LOGGER.info("training data end time: %s", self.metric.end_time)
_LOGGER.debug("begin training")
self.model.fit(self.metric.metric_values)
future = self.model.make_future_dataframe(
periods=int(prediction_duration),
freq=prediction_freq,
include_history=False,
)
forecast = self.model.predict(future)
forecast["timestamp"] = forecast["ds"]
forecast = forecast[["timestamp", "yhat", "yhat_lower", "yhat_upper"]]
forecast = forecast.set_index("timestamp")
self.predicted_df = forecast
_LOGGER.debug(forecast)
def predict_value(self, prediction_datetime):
"""Return the predicted value of the metric for the prediction_datetime."""
nearest_index = self.predicted_df.index.get_loc(prediction_datetime,
method="nearest")
return self.predicted_df.iloc[[nearest_index]]
def test_fit_predict_uncertainty_disabled(self):
N = DATA.shape[0]
train = DATA.head(N // 2)
future = DATA.tail(N // 2)
for uncertainty in [0, False]:
m = Prophet(uncertainty_samples=uncertainty)
m.fit(train)
fcst = m.predict(future)
expected_cols = [
'ds', 'trend', 'additive_terms', 'multiplicative_terms',
'weekly', 'yhat'
]
self.assertTrue(
all(col in expected_cols for col in fcst.columns.tolist()))
def distributed_model_prediction(history_pd):
history_pd['ds'] = pd.to_datetime(history_pd['ds'])
# make the model
prophet_model = Prophet()
prophet_model.fit(history_pd)
future_pd = prophet_model.make_future_dataframe(periods=90,
freq='d',
include_history=True)
# make predictions
results_pd = prophet_model.predict(future_pd)
results_pd["country"] = history_pd["country"].iloc[0]
return pd.DataFrame(results_pd, columns=result_schema.fieldNames())
def test_fit_predict(self):
days = 30
N = DATA.shape[0]
train = DATA.head(N - days)
test = DATA.tail(days)
test.reset_index(inplace=True)
forecaster = Prophet()
forecaster.fit(train, seed=1237861298)
np.random.seed(876543987)
future = forecaster.make_future_dataframe(days, include_history=False)
future = forecaster.predict(future)
# this gives ~ 10.64
res = self.rmse(future['yhat'], test['y'])
self.assertTrue(15 > res > 5,
msg="backend: {}".format(forecaster.stan_backend))
def test_flat_growth(self):
m = Prophet(growth='flat')
x = np.linspace(0, 2 * np.pi, 8 * 7)
history = pd.DataFrame({
'ds':
pd.date_range(start='2020-01-01', periods=8 * 7, freq='d'),
'y':
30 + np.sin(x * 8.),
})
m.fit(history)
future = m.make_future_dataframe(10, include_history=True)
fcst = m.predict(future)
m_ = m.params['m'][0, 0]
k = m.params['k'][0, 0]
self.assertAlmostEqual(k, 0)
self.assertAlmostEqual(fcst['trend'].unique()[0], m_ * m.y_scale)
self.assertEqual(np.round(m_ * m.y_scale), 30.0)
