def real_forecast_one_signal(self, df, signal, last_date, H):
import pyaf.ForecastEngine as autof
lEngine = autof.cForecastEngine()
lEngine.mOptions.mAddPredictionIntervals = False
lEngine.mOptions.mParallelMode = False
lEngine.mOptions.set_active_transformations(
['None', 'Difference', 'Anscombe'])
lEngine.mOptions.mMaxAROrder = 16
# lEngine
df1 = df[['Date', signal]].fillna(0.0)
lEngine.train(df1, 'Date', signal, 1)
lEngine.getModelInfo()
# lEngine.standrdPlots()
df_forecast = lEngine.forecast(iInputDS=df1, iHorizon=H)
dates = df_forecast['Date'].tail(H).values
predictions = df_forecast[str(signal) + '_Forecast'].tail(H).values
# logger.info(dates)
# logger.info(predictions)
fcst_dict = {}
for i in range(H):
ts = pd.to_datetime(str(dates[i]))
date_str = ts.strftime('%Y-%m-%d')
fcst_dict[date_str] = int(predictions[i])
logger.info("SIGNAL_FORECAST " + str(signal) + " " + str(fcst_dict))
return fcst_dict
def buildModel(df, ar_order, H):
lEngine = autof.cForecastEngine()
lEngine.mOptions.mMaxAROrder = ar_order
lEngine.train(df, b1.mTimeVar, b1.mSignalVar, H)
lEngine.getModelInfo()
def process_dataset_with_noise(idataset, sigma):
import warnings
with warnings.catch_warnings():
warnings.simplefilter("error")
N = idataset.mFullDataset.shape[0]
idataset.mFullDataset["orig_" +
idataset.mSignalVar] = idataset.mFullDataset[
idataset.mSignalVar]
lSignalVar = idataset.mSignalVar + "_" + str(sigma)
lNoise = np.random.randn(N) * sigma
H = idataset.mHorizon[idataset.mSignalVar]
idataset.mFullDataset[lSignalVar] = idataset.mFullDataset[
"orig_" + idataset.mSignalVar] + lNoise
idataset.mPastData = idataset.mFullDataset[:-H]
idataset.mFutureData = idataset.mFullDataset.tail(H)
training_ds = idataset.mPastData
# #df.to_csv("outputs/rand_exogenous.csv")
# N = df.shape[0];
# df1 = df;
lEngine = autof.cForecastEngine()
# lEngine.mOptions.mEnableSeasonals = False;
# lEngine.mOptions.mDebugCycles = True;
# lEngine.mOptions.enable_slow_mode();
# mDebugProfile = True;
# lEngine
lEngine.mOptions.set_active_transformations(['None'])
lEngine.mOptions.set_active_trends(['LinearTrend'])
lEngine.mOptions.set_active_periodics(['Seasonal_DayOfWeek'])
lEngine.mOptions.set_active_autoregressions(['ARX'])
lExogenousData = (idataset.mExogenousDataFrame,
idataset.mExogenousVariables)
lEngine.train(training_ds, idataset.mTimeVar, lSignalVar, H,
lExogenousData)
lEngine.getModelInfo()
# lEngine.standardPlots(name = "outputs/my_exog_" + str(nbex) + "_" + str(n));
# lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
print(lEngine.mSignalDecomposition.mTrPerfDetails.head())
dfapp_in = training_ds.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
idataset.mTimeVar, lSignalVar, lSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n",
Forecast_DF.tail(H).values)
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def trainModel(self, iSignal, iHorizon):
df = self.mTrainDataset[iSignal + "_" + str(iHorizon)]
lAutoF1 = autof.cForecastEngine()
lAutoF1.mOptions.mParallelMode = self.mParallelMode
self.mAutoForecastBySignal[iSignal + "_" + str(iHorizon)] = lAutoF1
lAutoF1.train(df, self.mTSSpec.mTimeVar, iSignal, iHorizon)
self.reportModelInfo(lAutoF1)
print(lAutoF1.mSignalDecomposition.mTrPerfDetails.head())
def process_dataset_with_noise(idataset, sigma, debug=False):
import warnings
with warnings.catch_warnings():
warnings.simplefilter("error")
N = idataset.mFullDataset.shape[0]
lSignalVar = idataset.mSignalVar + "_" + str(sigma)
lHorizon = idataset.mHorizon[idataset.mSignalVar]
lNoise = np.random.randn(N) * sigma
idataset.mFullDataset[lSignalVar] = idataset.mFullDataset[
"orig_" + idataset.mSignalVar] + lNoise
idataset.mPastData = idataset.mFullDataset[:-lHorizon]
idataset.mFutureData = idataset.mFullDataset.tail(lHorizon)
training_ds = idataset.mPastData
# #df.to_csv("outputs/rand_exogenous.csv")
H = lHorizon
# N = df.shape[0];
# df1 = df;
lEngine = autof.cForecastEngine()
# lEngine.mOptions.mEnableSeasonals = False;
lEngine.mOptions.mDebug = debug
# lEngine.mOptions.enable_slow_mode();
# mDebugProfile = True;
# lEngine
lExogenousData = (idataset.mExogenousDataFrame,
idataset.mExogenousVariables)
lEngine.train(training_ds, idataset.mTimeVar, lSignalVar, H,
lExogenousData)
lEngine.getModelInfo()
# lEngine.standrdPlots(name = "outputs/my_artificial_" + idataset.mName + "_" + str(sigma));
# lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine2 = pickleModel(lEngine)
dfapp_in = training_ds.copy()
dfapp_in.tail()
dfapp_out = lEngine2.forecast(dfapp_in, H)
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
lForecastName = lSignalVar + '_Forecast'
Forecast_DF = dfapp_out[[
idataset.mTimeVar, lSignalVar, lForecastName,
lForecastName + "_Lower_Bound", lForecastName + "_Upper_Bound"
]]
Forecast_DF.info()
print("Forecasts\n",
Forecast_DF.tail(H).values)
print("\n\n<ModelInfo>")
print(lEngine2.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def trainModel(self, iSignal, iHorizon):
df = self.mTrainDataset[iSignal + "_" + str(iHorizon)];
lAutoF1 = autof.cForecastEngine();
lAutoF1.mOptions.mParallelMode = self.mParallelMode;
# lAutoF1.mOptions.mCycleLengths = range(2, df.shape[0]//10);
self.mAutoForecastBySignal[iSignal + "_" + str(iHorizon)] = lAutoF1
lAutoF1.train(df , self.mTSSpec.mTimeVar , iSignal, iHorizon)
self.reportModelInfo(lAutoF1);
print(lAutoF1.mSignalDecomposition.mTrPerfDetails.head());
def test_random_exogenous(n, nbex):
with warnings.catch_warnings():
warnings.simplefilter("error")
b1 = tsds.generate_random_TS(N=600,
FREQ='D',
seed=0,
trendtype="constant",
cycle_length=12,
transform="",
sigma=0.0,
exog_count=2000)
df = b1.mPastData
# this script works on mysql with N = 600, exog_count = 20 when thread_stack = 1920K in
# /etc/mysql/mysql.conf.d/mysqld.cnf
# #df.to_csv("outputs/rand_exogenous.csv")
print(b1)
H = b1.mHorizon[b1.mSignalVar]
N = df.shape[0]
df1 = df.head(n).copy()
lEngine = autof.cForecastEngine()
# lEngine.mOptions.mEnableSeasonals = False;
# lEngine.mOptions.mDebugCycles = False;
lEngine.mOptions.mDebugProfile = True
lEngine
lExogenousData = (b1.mExogenousDataFrame,
b1.mExogenousVariables[0:nbex])
lEngine.train(df1, b1.mTimeVar, b1.mSignalVar, H, lExogenousData)
lEngine.getModelInfo()
lEngine.standardPlots(name="outputs/my_exog_" + str(nbex) + "_" +
str(n))
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
dfapp_in = df1.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
b1.mTimeVar, b1.mSignalVar, b1.mSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n",
Forecast_DF.tail(H).values)
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def create_model(N=365, croston_type=None, iTrend=False):
# N = 365
np.random.seed(seed=1960)
signal = None
if (iTrend):
signal = create_intermittent_signal_linear_trend(N)
else:
signal = create_intermittent_signal(N)
df_train = pd.DataFrame({
"Date":
pd.date_range(start="2016-01-25", periods=N, freq='D'),
"Signal":
signal
})
# print(df_train.head(N))
import pyaf.ForecastEngine as autof
lEngine = autof.cForecastEngine()
lEngine.mOptions.set_active_trends(['ConstantTrend', 'LinearTrend'])
lEngine.mOptions.set_active_periodics(['None'])
lEngine.mOptions.set_active_transformations(['None'])
lEngine.mOptions.set_active_autoregressions(['CROSTON'])
lEngine.mOptions.mModelSelection_Criterion = "L2"
lEngine.mOptions.mCrostonOptions.mMethod = croston_type
lEngine.mOptions.mCrostonOptions.mZeroRate = 0.0
# get the best time series model for predicting one week
lEngine.train(iInputDS=df_train,
iTime='Date',
iSignal='Signal',
iHorizon=7)
lEngine.getModelInfo()
lName = "outputs/croston_" + str(croston_type) + "_"
lName = lName + ("linear_trend" if iTrend else "no_trend")
lEngine.standardPlots(lName)
# predict one week
df_forecast = lEngine.forecast(iInputDS=df_train, iHorizon=7)
# list the columns of the forecast dataset
print(df_forecast.columns) #
cols = [
'Date', 'Signal', '_Signal', '_Signal_TransformedForecast',
'Signal_Forecast'
]
# print the real forecasts
print(df_forecast[cols].tail(12))
print(df_forecast['Signal'].describe())
print(df_forecast['Signal_Forecast'].describe())
def process_dataset(idataset):
import warnings
with warnings.catch_warnings():
warnings.simplefilter("error")
N = idataset.mFullDataset.shape[0]
lSignalVar = idataset.mSignalVar
H = 2
idataset.mPastData = idataset.mFullDataset[:-H]
idataset.mFutureData = idataset.mFullDataset.tail(H)
training_ds = idataset.mPastData
# #df.to_csv("outputs/rand_exogenous.csv")
# N = df.shape[0];
# df1 = df;
lEngine = autof.cForecastEngine()
# lEngine.mOptions.mEnableSeasonals = False;
# lEngine.mOptions.mDebugCycles = True;
# lEngine.mOptions.enable_slow_mode();
# mDebugProfile = True;
# lEngine
lExogenousData = (idataset.mExogenousDataFrame,
idataset.mExogenousVariables)
lEngine.train(training_ds, idataset.mTimeVar, lSignalVar, H,
lExogenousData)
lEngine.getModelInfo()
# lEngine.standardPlots(name = "outputs/my_exog_" + str(nbex) + "_" + str(n));
# lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
dfapp_in = training_ds.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
idataset.mTimeVar, lSignalVar, lSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n",
Forecast_DF.tail(H).values)
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def buildModel(arg):
(cyc , freq, nbrows) = arg
print("TEST_CYCLES_START", nbrows, freq, cyc)
b1 = tsds.generate_random_TS(N = nbrows , FREQ = freq, seed = 0, trendtype = "constant", cycle_length = cyc, transform = "", sigma = 0.1, exog_count = 0);
df = b1.mPastData[[b1.mTimeVar , b1.mSignalVar]].copy()
lSignal = 'Signal_Cycle_' + str(nbrows) + "_" + str(freq) + "_" + str(cyc)
df.columns = [b1.mTimeVar, lSignal]
print(df.head())
print(df.tail())
print(df.describe())
lEngine = autof.cForecastEngine()
lEngine.mOptions.disable_all_transformations();
lEngine.mOptions.set_active_transformations(['None']);
lEngine.mOptions.mCycleLengths = [ k for k in range(2,128) ];
lEngine.mOptions.mDebugCycles = True;
lEngine
H = 12;
lEngine.train(df , b1.mTimeVar , lSignal, H);
lEngine.getModelInfo();
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
dfapp_in = df.copy();
dfapp_in.tail()
# H = 12
dfapp_out = lEngine.forecast(dfapp_in, H);
dfapp_out.tail(2 * H)
print("Forecast Columns " , dfapp_out.columns);
Forecast_DF = dfapp_out[[b1.mTimeVar , lSignal, lSignal + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(H).values);
print("\n\n<ModelInfo>")
print(lEngine.to_json());
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(H).to_json(date_format='iso'))
print("</Forecast>\n\n")
print("TEST_CYCLES_END", cyc)
del lEngine
del df
del b1
del dfapp_in
del dfapp_out
del Forecast_DF
def analyze_dataset(name, horizon):
signal = name.replace(".csv", "")
url = "https://raw.githubusercontent.com/antoinecarme/TimeSeriesData/master/R_TSData/expsmooth/" + name
df = pd.read_csv(url)
df = df[[df.columns[0], df.columns[-1]]].dropna()
df.columns = ['Date', signal]
lEngine = autof.cForecastEngine()
# lEngine.mOptions.enable_slow_mode();
N = df.shape[0]
lEngine.train(df, 'Date', signal, horizon)
lEngine.getModelInfo()
print("PERFORMANCE MAPE_FORECAST", signal,
lEngine.mSignalDecomposition.mBestModel.mForecastPerf.mMAPE)
return lEngine
def test_fake_model_2_rows(iHorizon_train , iHorizon_apply):
# one row dataset => always constant forecast
df = pd.DataFrame([[0 , 0.54543] , [1 , 0.43]], columns = ['date' , 'signal'])
lEngine = autof.cForecastEngine()
lEngine.train(df , 'date' , 'signal', iHorizon_train);
# print(lEngine.mSignalDecomposition.mBestModel.mTimeInfo.info())
print(lEngine.mSignalDecomposition.mBestModel.getFormula())
print("PERFS_MAPE_MASE", lEngine.mSignalDecomposition.mBestModel.mForecastPerf.mMAPE,
lEngine.mSignalDecomposition.mBestModel.mForecastPerf.mMASE, )
# print(df.head())
df1 = lEngine.forecast(df , iHorizon_apply)
# print(df1.columns)
Forecast_DF = df1[['date' , 'signal', 'signal' + '_Forecast', 'signal_Residue', 'signal_Forecast_Lower_Bound',
'signal_Forecast_Upper_Bound']]
# print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(iHorizon_apply));
def createModelForAllLevels(df, hier, iStrcture, H, iDateColumn):
lModels = {}
lLevelCount = len(hier['Levels'])
for level in range(lLevelCount):
lModels[level] = {}
for signal in iStrcture[level].keys():
lEngine = autof.cForecastEngine()
lEngine
# lEngine.mOptions.enable_slow_mode();
# lEngine.mOptions.mDebugPerformance = True;
lEngine.mOptions.set_active_autoregressions([])
lEngine.train(df1, lDateColumn, signal, H)
lEngine.getModelInfo()
lModels[level][signal] = lEngine
return lModels
def test_ozone_debug_perf():
b1 = tsds.load_ozone()
df = b1.mPastData
# df.tail(10)
# df[:-10].tail()
# df[:-10:-1]
# df.describe()
lEngine = autof.cForecastEngine()
lEngine
H = b1.mHorizon
lEngine.mOptions.mDebugPerformance = True
lEngine.mOptions.mEnableCycles = False
lEngine.mOptions.mEnableTimeBasedTrends = False
lEngine.mOptions.mEnableARModels = False
lEngine.train(df, b1.mTimeVar, b1.mSignalVar, H)
lEngine.getModelInfo()
print(lEngine.mSignalDecomposition.mTrPerfDetails.head())
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine.standardPlots("outputs/my_ozone")
dfapp_in = df.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
#dfapp_out.to_csv("outputs/ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
b1.mTimeVar, b1.mSignalVar, b1.mSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n",
Forecast_DF.tail(H).values)
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def build_model(transformations, trends, periodics, autoregs):
b1 = tsds.load_ozone_exogenous()
df = b1.mPastData
lEngine = autof.cForecastEngine()
lEngine
H = b1.mHorizon;
# lEngine.mOptions.enable_slow_mode();
# lEngine.mOptions.mDebugPerformance = True;
lEngine.mOptions.set_active_transformations(transformations);
lEngine.mOptions.set_active_trends(trends);
lEngine.mOptions.set_active_periodics(periodics);
lEngine.mOptions.set_active_autoregressions(autoregs);
lExogenousData = (b1.mExogenousDataFrame , b1.mExogenousVariables)
lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
lEngine.getModelInfo();
print(lEngine.mSignalDecomposition.mTrPerfDetails.head());
lEngine2 = pickleModel(lEngine)
lEngine2.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine2.standrdPlots("outputs/my_ozone");
dfapp_in = df.copy();
dfapp_in.tail()
#H = 12
dfapp_out = lEngine2.forecast(dfapp_in, H);
#dfapp_out.to_csv("outputs/ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns " , dfapp_out.columns);
Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(H));
print("\n\n<ModelInfo>")
print(lEngine2.to_json());
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2*H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def test_transformation(itransformation):
b1 = tsds.load_ozone()
df = b1.mPastData
lEngine = autof.cForecastEngine()
lEngine
H = b1.mHorizon
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mDebugPerformance = True
lEngine.mOptions.disable_all_transformations()
lEngine.mOptions.mActiveTransformations[itransformation] = True
lEngine.mOptions.mBoxCoxOrders = lEngine.mOptions.mExtensiveBoxCoxOrders
lEngine.train(df, b1.mTimeVar, b1.mSignalVar, H)
lEngine.getModelInfo()
print(lEngine.mSignalDecomposition.mTrPerfDetails.head())
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine.standrdPlots("outputs/my_ozone_" + itransformation)
dfapp_in = df.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
#dfapp_out.to_csv("outputs/ozone_apply_out_" + itransformation + ".csv")
dfapp_out.tail(H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
b1.mTimeVar, b1.mSignalVar, b1.mSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n", Forecast_DF.tail(H))
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def trainModel(self):
self.mTrainDataFrame = self.mFullDataFrame
if (self.mPresentTime is not None and self.mPresentTime != ""):
self.mPresent = self.convert_string_to_date(self.mPresentTime)
self.mTrainDataFrame = self.mFullDataFrame[
self.mFullDataFrame[self.mTimeVar] <= self.mPresent]
self.mForecastEngine = autof.cForecastEngine()
# heroku does not have a lot of memory!!! issue #25
# self.mForecastEngine.mOptions.enable_low_memory_mode();
print("TRAIN_PARAMS", self.mTrainDataFrame.shape,
self.mTrainDataFrame.columns, self.mTimeVar, self.mSignalVar,
self.mHorizon)
lExogenousData = None
if (self.mExogenousDataFrame is not None):
lExogenousData = (self.mExogenousDataFrame,
self.mExogenousVariables)
self.mForecastEngine.train(self.mTrainDataFrame, self.mTimeVar,
self.mSignalVar, self.mHorizon,
lExogenousData)
def test_transformation(itransformation):
df = create_df()
df.to_csv('a.csv')
lEngine = autof.cForecastEngine()
lEngine
H = 12
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mDebugPerformance = True
if (itransformation is not None):
lEngine.mOptions.disable_all_transformations()
lEngine.mOptions.set_active_transformations([itransformation])
lEngine.mOptions.mBoxCoxOrders = lEngine.mOptions.mExtensiveBoxCoxOrders
lSignalVar = 'signal2'
lTimeVar = 'time'
lEngine.train(df, lTimeVar, lSignalVar, H)
lEngine.getModelInfo()
N = lEngine.mSignalDecomposition.mTrPerfDetails.shape[0]
print(lEngine.mSignalDecomposition.mTrPerfDetails.head(N))
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine.standrdPlots("outputs/my_airline_" + str(itransformation))
dfapp_in = df.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
#dfapp_out.to_csv("outputs/ozone_apply_out_" + itransformation + ".csv")
dfapp_out.tail(H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[lTimeVar, lSignalVar, lSignalVar + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n", Forecast_DF.tail(H))
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def test_nbrows_cycle(nbrows , cyc):
# lValues = [ k for k in range(2,24, 4)];
# lValues = lValues + [ k for k in range(24, 128, 8)];
# for cyc in lValues:
print("TEST_CYCLES_START", nbrows, cyc)
b1 = tsds.generate_random_TS(N = nbrows , FREQ = 'H', seed = 0, trendtype = "constant", cycle_length = cyc, transform = "None", sigma = 0.1, exog_count = 0, ar_order=0);
df = b1.mPastData
# df.tail(10)
# df[:-10].tail()
# df[:-10:-1]
# df.describe()
lEngine = autof.cForecastEngine()
lEngine.mOptions.mCycleLengths = [ k for k in range(2, cyc * 4) ];
lEngine
H = cyc * 2;
lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
lEngine.getModelInfo();
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
dfapp_in = df.copy();
dfapp_in.tail()
# H = 12
dfapp_out = lEngine.forecast(dfapp_in, H);
dfapp_out.tail(2 * H)
print("Forecast Columns " , dfapp_out.columns);
Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(H).values);
print("\n\n<ModelInfo>")
print(lEngine.to_json());
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(H).to_json(date_format='iso'))
print("</Forecast>\n\n")
print("TEST_CYCLES_END", cyc)
def create_model(croston_type):
df_train = create_dataset()
# print(df_train.head(N))
import pyaf.ForecastEngine as autof
lEngine = autof.cForecastEngine()
lSlowMode = False
if (lSlowMode):
lEngine.mOptions.set_active_trends(['ConstantTrend', 'LinearTrend'])
lEngine.mOptions.set_active_periodics(['None'])
lEngine.mOptions.set_active_transformations(['None'])
lEngine.mOptions.set_active_autoregressions(['CROSTON'])
lEngine.mOptions.mModelSelection_Criterion = "L2"
lEngine.mOptions.mCrostonOptions.mMethod = croston_type
lEngine.mOptions.mCrostonOptions.mZeroRate = 0.0
# get the best time series model for predicting one week
lEngine.train(iInputDS=df_train,
iTime='Date',
iSignal='Signal',
iHorizon=7)
lEngine.getModelInfo()
lName = "outputs/fpp2_slow_mode_" + str(croston_type) + "_"
lEngine.standardPlots(lName)
# predict one week
df_forecast = lEngine.forecast(iInputDS=df_train, iHorizon=7)
# list the columns of the forecast dataset
print(df_forecast.columns) #
cols = [
'Date', 'Signal', '_Signal', '_Signal_TransformedForecast',
'Signal_Forecast'
]
# print the real forecasts
print(df_forecast[cols].tail(12))
print(df_forecast['Signal'].describe())
print(df_forecast['Signal_Forecast'].describe())
def testSignalIdempotency(self, iSignal, iHorizon, tr, cy, ar):
lAutoF1 = self.mAutoForecastBySignal[iSignal + "_" + str(iHorizon)];
lApplyOut = self.mApplyOut.head(self.mApplyIn.shape[0]);
print(lApplyOut.columns);
lNewSignal = iSignal + "_" + str(tr) + "_" + str(cy) + "_" + str(ar);
lTransformedSignal = lAutoF1.mSignalDecomposition.mBestModel.mSignal;
lSignal = 0.0 * lApplyOut[iSignal];
if(tr is not None):
lSignal = lSignal + lApplyOut[lTransformedSignal + "_Trend"];
if(cy is not None ):
lSignal = lSignal + lApplyOut[lTransformedSignal + "_Cycle"];
if(ar is not None ):
lSignal = lSignal + lApplyOut[lTransformedSignal + "_AR"];
df= pd.DataFrame();
df[self.mTSSpec.mTimeVar] = lApplyOut[self.mTSSpec.mTimeVar];
df[lNewSignal] = lSignal;
lAutoF1 = autof.cForecastEngine();
lAutoF1.mOptions.mParallelMode = self.mParallelMode;
lAutoF1.train(df , self.mTSSpec.mTimeVar , lNewSignal, iHorizon)
self.reportModelInfo(lAutoF1);
def test_air_passengers_missing_data(iTimeMissingDataImputation,
iSignalMissingDataImputation):
b1 = tsds.load_airline_passengers()
df = b1.mPastData
if (iTimeMissingDataImputation is not None):
df = add_some_missing_data_in_time(df, b1.mTimeVar)
if (iSignalMissingDataImputation is not None):
df = add_some_missing_data_in_signal(df, b1.mSignalVar)
lEngine = autof.cForecastEngine()
H = b1.mHorizon
lEngine.mOptions.mMissingDataOptions.mTimeMissingDataImputation = iTimeMissingDataImputation
lEngine.mOptions.mMissingDataOptions.mSignalMissingDataImputation = iSignalMissingDataImputation
lEngine.train(df, b1.mTimeVar, b1.mSignalVar, H)
lEngine.getModelInfo()
print(lEngine.mSignalDecomposition.mTrPerfDetails.head())
dfapp_in = df.copy()
dfapp_in.tail()
dfapp_out = lEngine.forecast(dfapp_in, H)
#dfapp_out.to_csv("outputs/ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
b1.mTimeVar, b1.mSignalVar, b1.mSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n", Forecast_DF.tail(H))
print("\n\n<ModelInfo>")
print(lEngine.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def build_model(iPerf):
b1 = tsds.load_ozone()
df = b1.mPastData
lEngine = autof.cForecastEngine()
lEngine
H = b1.mHorizon;
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mDebugPerformance = True;
lEngine.mOptions.mModelSelection_Criterion = iPerf;
lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
lEngine.getModelInfo();
print(lEngine.mSignalDecomposition.mTrPerfDetails.head());
lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine.standardPlots("outputs/my_ozone_perfs_" + iPerf);
dfapp_in = df.copy();
dfapp_in.tail()
# H = 12
dfapp_out = lEngine.forecast(dfapp_in, H);
# dfapp_out.to_csv("outputs/ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns " , dfapp_out.columns);
Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(H));
print("\n\n<ModelInfo>")
print(lEngine.to_json());
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2*H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def buildModel(iParallel=True):
import pandas as pd
import numpy as np
import pyaf.ForecastEngine as autof
import pyaf.Bench.TS_datasets as tsds
import logging
import logging.config
# logging.config.fileConfig('logging.conf')
logging.basicConfig(level=logging.INFO)
b1 = tsds.load_airline_passengers()
df = b1.mPastData
df.head()
lEngine = autof.cForecastEngine()
lEngine
H = b1.mHorizon
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mEnableSeasonals = True
lEngine.mOptions.mEnableCycles = True
lEngine.mOptions.mDebugPerformance = True
lEngine.mOptions.mParallelMode = iParallel
lEngine.mOptions.set_active_autoregressions(['MLP', 'LSTM'])
# lEngine.mOptions.mMaxAROrder = 2;
lEngine.train(df, b1.mTimeVar, b1.mSignalVar, H)
lEngine2 = pickleModel(lModel)
lEngine2.getModelInfo()
print(lEngine2.mSignalDecomposition.mTrPerfDetails.head())
lEngine2.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine2.standrdPlots(name="outputs/rnn_my_airline_passengers")
dfapp_in = df.copy()
dfapp_in.tail()
# H = 12
dfapp_out = lEngine2.forecast(dfapp_in, H)
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
lForecastColumnName = b1.mSignalVar + '_Forecast'
Forecast_DF = dfapp_out[[
b1.mTimeVar, b1.mSignalVar, lForecastColumnName,
lForecastColumnName + '_Lower_Bound',
lForecastColumnName + '_Upper_Bound'
]]
print(Forecast_DF.info())
print("Forecasts\n", Forecast_DF.tail(2 * H))
print("\n\n<ModelInfo>")
print(lEngine2.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
def buildModel(iParallel=True):
import pandas as pd
import numpy as np
import pyaf.ForecastEngine as autof
import pyaf.Bench.TS_datasets as tsds
import logging
import logging.config
# logging.config.fileConfig('logging.conf')
logging.basicConfig(level=logging.INFO)
# get_ipython().magic('matplotlib inline')
b1 = tsds.load_ozone()
df = b1.mPastData
# df.tail(10)
# df[:-10].tail()
# df[:-10:-1]
# df.describe()
lEngine = autof.cForecastEngine()
lEngine
H = b1.mHorizon
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mDebugPerformance = True
lEngine.mOptions.mParallelMode = iParallel
lEngine.mOptions.set_active_autoregressions(['MLP', 'LSTM'])
lEngine.train(df, b1.mTimeVar, b1.mSignalVar, H)
lEngine2 = pickleModel(lEngine)
lEngine2.getModelInfo()
print(lEngine2.mSignalDecomposition.mTrPerfDetails.head())
lEngine2.mSignalDecomposition.mBestModel.mTimeInfo.mResolution
lEngine2.standardPlots("outputs/my_rnn_ozone")
dfapp_in = df.copy()
dfapp_in.tail()
# H = 12
dfapp_out = lEngine2.forecast(dfapp_in, H)
# dfapp_out.to_csv("outputs/rnn_ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns ", dfapp_out.columns)
Forecast_DF = dfapp_out[[
b1.mTimeVar, b1.mSignalVar, b1.mSignalVar + '_Forecast'
]]
print(Forecast_DF.info())
print("Forecasts\n", Forecast_DF.tail(H))
print("\n\n<ModelInfo>")
print(lEngine2.to_json())
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(2 * H).to_json(date_format='iso'))
print("</Forecast>\n\n")
import numpy as np
import pandas as pd
N = 360
df_train = pd.DataFrame({"Date" : pd.date_range(start="2016-01-25", periods=N, freq='D'),
"Signal" : (np.arange(N)//40 + np.arange(N) % 21 + np.random.randn(N))})
import pyaf.ForecastEngine as autof
# create a forecast engine. This is the main object handling all the operations
lEngine = autof.cForecastEngine()
# get the best time series model for predicting one week
lEngine.train(iInputDS = df_train, iTime = 'Date', iSignal = 'Signal', iHorizon = 7);
lEngine.getModelInfo() # => relative error 7% (MAPE)
# In[6]:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("error")
lEngine.standardPlots("outputs/issue_94")
# In[7]:
df2 = df.copy()
df2=df2.rename(columns={"sdttm": "Date"})
columns = '''Date,pefmax,so2,co,o3,no2,temperaturec,windspeedms,precipitationpercent,vaporpressurehpa,
dewpointtemperaturec,airpressurehpa,sealevelpressurehpa,groundtemperaturec,tmax,amax,tmin'''
#amin,pmin,tmaxlesstmin,amaxlessamin,pmaxlesspmin'''
cs = [c.strip() for c in columns.split(',')]
df2=df2[cs]
#df2.dtypes
# In[6]:
import pyaf.ForecastEngine as autof
lEngine = autof.cForecastEngine()
lExogenousData = (df2 , cs[2:6])
df3=df2["Date,pefmax".split(',')].copy()
# In[ ]:
lEngine.train(df3, 'Date' , 'pefmax', 12 , lExogenousData);
# In[7]:
import pyaf.ForecastEngine as autof
lEngine_Without_Exogenous = autof.cForecastEngine()
lEngine_Without_Exogenous.train(df3 , 'Date' , 'pefmax', 7);