def plot_training(self,model,df_a):
'''
Print one step ahead preidctidn during the training period , not working yet
Parameters
----------
model : Sequential object
model
df_a : dataframe
dataframe of historical data before any processing.
Returns
-------
None.
'''
if model=="trend":
model=self.model_trend
x_train,y_train= decoupe_dataframe(self.trend, self.look_back)
elif model=="residual":
model=self.model_residual
x_train,y_train =decoupe_dataframe(self.residual, self.look_back)
elif model=="seasonal":
model=self.model_seasonal
x_train,y_train =decoupe_dataframe(self.seasonal, self.look_back)
x_train = np.reshape(x_train,(int(len(x_train)/self.look_back), 1,self.look_back))
train_predict=model.predict(x_train)
plt.plot(train_predict)
plt.plot(df_a[: -self.look_back]) #plot tout df_a si on souhaite le mettre en prod
plt.show()
def prepare_data(self, df, look_back, freq_period):
'''
Parameters
----------
df : DataFrame
datafrmae contening historical data .
look_back : int
length entry of the model .
Decompose the signal in three sub signals, trend,seasonal and residual in order to work separetly on each signal
Returns
-------
trend_x : array
values of the trend of the signal, matrix of dimention X array of dimension (1,length entry of model) X= length(dataframe)/look_back.
trend_y : array
vaklues to be predicted during the training
seasonal_x : array
same as trend_x but with the seasonal part of the signal.
seasonal_y : array
same as trend_y but with the seasonal part of the signal.
residual_x : array
same as trend_x but with the residual part of the signal.
residual_y : array
same as trend_y but with the residual part of the signal.
'''
self.look_back = look_back
df = df.dropna()
decomposition = seasonal_decompose(df["y"],
period=freq_period) #5*12*7
df.loc[:, 'trend'] = decomposition.trend
df.loc[:, 'seasonal'] = decomposition.seasonal
df.loc[:, 'residual'] = decomposition.resid
df_a = df
df = df.dropna()
self.shift = len(df_a) - len(df)
df = df.reset_index(drop=True)
#df["trend"] = savgol_filter(df["trend"], 12*24*7+1, 2) ########*7 lors du premier trainning
df["trend"] = df["trend"].fillna(method="bfill")
self.trend = np.asarray(df.loc[:, 'trend'])
self.seasonal = np.asarray(df.loc[:, 'seasonal'])
self.residual = np.asarray(df.loc[:, 'residual'])
trend_x, trend_y = decoupe_dataframe(df["trend"], look_back)
print(trend_x.shape)
seasonal_x, seasonal_y = decoupe_dataframe(df["seasonal"], look_back)
residual_x, residual_y = decoupe_dataframe(df["residual"], look_back)
print("prepared")
return trend_x, trend_y, seasonal_x, seasonal_y, residual_x, residual_y
def fit_predict_without_dec(self, df, look_back, freq_period, len_prediction, verbose, nb_features=1, nb_epochs=50,
nb_batch=100, nb_layers=50, attention=True, loss="mape", metric="mse",
optimizer="Adamax"):
df = df.reset_index()
i = Input(shape=(nb_features, look_back))
x = LSTM(nb_layers, return_sequences=True, activation='relu', input_shape=(nb_features, look_back))(i)
x = Activation("relu")(x)
x = Dropout(0.2)(x)
x = LSTM(nb_layers, return_sequences=True)(x)
x = Dropout(0.2)(x)
x = Dense(int(nb_layers / 2), activation='relu')(x)
output = Dense(1)(x)
model = Model(inputs=[i], outputs=[output])
model.compile(loss=loss, optimizer=optimizer, metrics=[metric])
x_train, y_train = decoupe_dataframe(df["y"], look_back)
nb_epochs = nb_epochs * 7
try:
x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1])
except Exception as e:
if e == IndexError:
print("Not enought data in the input compared to the look back size. Put more data as input")
hist = model.fit(x_train, y_train, epochs=nb_epochs, batch_size=nb_batch, verbose=verbose)
data_residual = np.array(df["y"][-1 * look_back:])
prediction = np.zeros((1, len_prediction))
for i in progressbar(range(len_prediction), "Computing: ", 40):
dataset = np.reshape(data_residual, (1, 1, look_back))
prediction_residual = (model.predict(dataset))
data_residual = np.append(data_residual[1:], [prediction_residual]).reshape(-1, 1)
prediction[0, i] = prediction_residual
return prediction, prediction, prediction
def train_model(self, y, look_back, model,len_pred):
scaler = MinMaxScaler()
#Y = scaler.fit_transform(np.reshape(np.array(y),(-1,1)))
x_train, y_train = decoupe_dataframe(y, look_back)
print(np.shape(x_train))
x_train = x_train.reshape(x_train.shape[0], 1, x_train.shape[1])
history = model.fit(
x_train, y_train,
epochs=1000,
batch_size=32,
validation_split=0.1,
shuffle=False
)
return scaler, history
def prepare_data(self, df, look_back, freq_period, first=0,seq2seq=False):
'''
Parameters
----------
df : DataFrame
datafrmae contening historical data .
look_back : int
length entry of the model .
Decompose the signal in three sub signals, trend,seasonal and residual in order to work separetly on each signal
Returns
-------
trend_x : array
values of the trend of the signal, matrix of dimention X array of dimension (1,length entry of model) X= length(dataframe)/look_back.
trend_y : array
vaklues to be predicted during the training
seasonal_x : array
same as trend_x but with the seasonal part of the signal.
seasonal_y : array
same as trend_y but with the seasonal part of the signal.
residual_x : array
same as trend_x but with the residual part of the signal.
residual_y : array
same as trend_y but with the residual part of the signal.
'''
self.seq2seq=seq2seq
imputer = KNNImputer(n_neighbors=2, weights="uniform")
df.loc[:,"y"]=imputer.fit_transform(np.array(df["y"]).reshape(-1, 1))
if look_back%2==0:
window=freq_period+1
else:
window=freq_period
scalerfile = self.directory + '/scaler_pred.sav'
if not os.path.isfile(scalerfile) or os.path.isfile(scalerfile) and first == 1:
if (df["y"].max() - df["y"].min()) > 100:
if self.verbose == 1:
print("PowerTransformation scaler used")
scaler = PowerTransformer()
else:
if self.verbose == 1:
print("Identity scaler used")
scaler = IdentityTransformer()
self.scaler2 = scaler.fit(np.reshape(np.array(df["y"]), (-1, 1)))
Y = self.scaler2.transform(np.reshape(np.array(df["y"]), (-1, 1)))
pickle.dump(self.scaler2, open(scalerfile, 'wb'))
elif os.path.isfile(scalerfile) and first == 0:
self.scaler2 = pickle.load(open(scalerfile, "rb"))
Y = self.scaler2.transform(np.reshape(np.array(df["y"]), (-1, 1)))
if freq_period % 2 == 0:
freq_period = freq_period + 1
decomposition = STL(Y, period=freq_period)
decomposition = decomposition.fit()
df.loc[:, 'trend'] = decomposition.trend
df.loc[:, 'seasonal'] = decomposition.seasonal
df.loc[:, 'residual'] = decomposition.resid
self.trend = np.asarray(df.loc[:, 'trend'])
self.seasonal = np.asarray(df.loc[:, 'seasonal'])
self.residual = np.asarray(df.loc[:, 'residual'])
if not self.seq2seq :
trend_x, trend_y = decoupe_dataframe(df["trend"], look_back)
seasonal_x, seasonal_y = decoupe_dataframe(df["seasonal"], look_back)
residual_x, residual_y = decoupe_dataframe(df["residual"], look_back)
else :
trend_x, trend_y = sequence_dataframe(df["trend"], look_back,self.len_pred)
seasonal_x, seasonal_y = sequence_dataframe(df["seasonal"], look_back,self.len_pred)
residual_x, residual_y = sequence_dataframe(df["residual"], look_back,self.len_pred)
if self.verbose == 1:
print("prepared")
return trend_x, trend_y, seasonal_x, seasonal_y, residual_x, residual_y