def save_to_GAF_img(df, file, step):
OHLC = ["Open", "High", "Low", "Close"]
high = max(df["High"])
low = min(df["Low"])
for col in OHLC:
Path("/content/GASF/" + col + "/").mkdir(parents=True, exist_ok=True)
Path("/content/GADF/" + col + "/").mkdir(parents=True, exist_ok=True)
Path("/content/MTF/" + col + "/").mkdir(parents=True, exist_ok=True)
gasf = GramianAngularField(image_size=step, method="summation")
gadf = GramianAngularField(image_size=step, method="difference")
mtf = MarkovTransitionField(image_size=step)
ts_norm = [(i - low) / (high - low) for i in list(df[col])]
X_mtf = mtf.fit_transform([ts_norm])
X_gasf = gasf.fit_transform([ts_norm])
X_gadf = gadf.fit_transform([ts_norm])
plt.imsave("/content/other_n/GASF/" + col + "/" + file,
X_gasf[0],
cmap="gray")
plt.imsave("/content/other_n/GADF/" + col + "/" + file,
X_gadf[0],
cmap="gray")
plt.imsave("/content/other_n/MTF/" + col + "/" + file,
X_mtf[0],
cmap="gray")
class TSToMTF(Transform):
r"""Transforms a time series batch to a 4d TSImage (bs, n_vars, size, size) by applying Markov Transition Field"""
order = 98
def __init__(self, size=224, cmap=None, n_bins=5, **kwargs):
self.size, self.cmap = size, cmap
self.encoder = MarkovTransitionField(n_bins=n_bins, **kwargs)
def encodes(self, o: TSTensor):
bs, *_, seq_len = o.shape
size = ifnone(self.size, seq_len)
if size != seq_len:
o = F.interpolate(o.reshape(-1, 1, seq_len),
size=size,
mode='linear',
align_corners=False)[:, 0]
else:
o = o.reshape(-1, seq_len)
output = self.encoder.fit_transform(o.cpu().numpy()).reshape(
bs, -1, size, size)
if self.cmap and output.shape[1] == 1:
output = TSImage(plt.get_cmap(
self.cmap)(output)[..., :3]).squeeze(1).permute(0, 3, 1, 2)
else:
output = TSImage(output)
return output.to(device=o.device)
def prep_seriesConvLSTM(seq_len, out_window, in_window, img_size, channels,
series_test, h):
print("Preparing data: ")
sample_range = (-1, 1)
signal_test = series_test
signal_test = signal_test.reshape(-1, 1)
signal_test_scaled = signal_test.flatten()
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, in_window, out_window, h)
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_test = np.expand_dims(gadf.fit_transform(window_input_test), axis=3)
gasf_test = np.expand_dims(gasf.fit_transform(window_input_test), axis=3)
mtf_test = np.expand_dims(mtf.fit_transform(window_input_test), axis=3)
y_test = window_output_test.reshape(-1)
if (channels == 2):
X_test_windowed = np.concatenate((gadf_test, gasf_test), axis=3)
else:
X_test_windowed = np.concatenate((gadf_test, gasf_test, mtf_test),
axis=3)
X_test_Conv_LSTM = np.zeros((X_test_windowed.shape[0] - seq_len + 1,
seq_len, img_size, img_size, channels))
y_test_Conv_LSTM = np.zeros(
(X_test_windowed.shape[0] - seq_len + 1, out_window))
print("Test data:")
for i in tqdm(range(0, X_test_windowed.shape[0] - seq_len + 1)):
current_seq_X = np.zeros((seq_len, img_size, img_size, channels))
for l in range(seq_len):
current_seq_X[l] = X_test_windowed[i + l]
current_seq_X = current_seq_X.reshape(1, seq_len, img_size, img_size,
channels)
X_test_Conv_LSTM[i] = current_seq_X
y_test_Conv_LSTM[i] = y_test[i + seq_len - 1]
X_test_Conv_LSTM = X_test_Conv_LSTM.reshape(-1, seq_len, img_size,
img_size, channels)
y_test_Conv_LSTM = y_test_Conv_LSTM.reshape(-1, out_window)
return (X_test_Conv_LSTM, y_test_Conv_LSTM)
def toMTFdata(tsdatas,
image_size=1.,
n_bins=5,
strategy='quantile',
overlapping=False,
flatten=False):
X = []
mtf = MarkovTransitionField(image_size=image_size,
n_bins=n_bins,
strategy=strategy,
overlapping=overlapping,
flatten=flatten)
for data in tsdatas:
data_mtf = mtf.fit_transform(data)
X.append(data_mtf[0])
return np.array(X)
def MTF_encoder(ts,
size=None,
n_bins=8,
strategy='quantile',
overlapping=False,
**kwargs):
ts = To2dArray(ts)
assert ts.ndim == 2, 'ts ndim must be 2!'
if size is None: size = ts.shape[-1]
else: size = min(size, ts.shape[-1])
ts = PAA(window_size=None, output_size=size).fit_transform(ts)
encoder = MTF(size,
n_bins=n_bins,
strategy=strategy,
overlapping=overlapping)
output = np.squeeze(encoder.fit_transform(ts), 0)
return output
def prep_seriesConvMLP(window_size_x, window_size_y, img_size, signal_test, h):
signal_test = signal_test.reshape(-1, 1)
sample_range = (-1, 1)
signal_test_scaled = signal_test.flatten()
# Split Sequence
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, window_size_x, window_size_y, h)
# %%---------------------------------------------------------------------------
'''
Field transformations
'''
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_transformed_test = np.expand_dims(
gadf.fit_transform(window_input_test), axis=3)
gasf_transformed_test = np.expand_dims(
gasf.fit_transform(window_input_test), axis=3)
mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test),
axis=3)
X_test_windowed = np.concatenate(
(gadf_transformed_test, gasf_transformed_test, mtf_transformed_test),
axis=3)
# Data reshaping
X_test_Conv_MLP = X_test_windowed
y_test_Conv_MLP = window_output_test
return (X_test_Conv_MLP, y_test_Conv_MLP)
"""
=======================
Markov Transition Field
=======================
This example shows how to transform a time series into a Markov
Transition Field using :class:`pyts.image.MarkovTransitionField`.
"""
import matplotlib.pyplot as plt
from pyts.image import MarkovTransitionField
from pyts.datasets import load_gunpoint
X, _, _, _ = load_gunpoint(return_X_y=True)
# MTF transformation
mtf = MarkovTransitionField(image_size=24)
X_mtf = mtf.fit_transform(X)
# Show the image for the first time series
plt.figure(figsize=(6, 6))
plt.imshow(X_mtf[0], cmap='rainbow', origin='lower')
plt.title('Markov Transition Field', fontsize=18)
plt.colorbar(fraction=0.0457, pad=0.04)
plt.show()
def get_multivariate_mtf(timeseries_list,
tags_list=None,
resample_rate=None,
image_size=IMAGE_SIZE):
"""
This function computes the MTF for each of the timeseries passed as
argument. It perform the appropriate data preprocessing to allow the
MTF to be computed (NaN removal, identifying constant signals...).
PARAMS
======
timeseries_list: list of pandas.dataframe
A list of dataframes (one per time series)
tags_list: list of strings (default to None)
List of all the tag names if available.
resample_rate: string (default to None)
A resampling rate to be used before applying the MTF computation.
image_size: integer (default to 48)
Resolution of the MTF
RETURNS
=======
tags_mtf: numpy.ndarray
An array of shape (num_timeseries, image_size, image_size) with
the MTF computed for each signal.
constant_signals: list of string
A list of all the constant signals removed from the final result
selected_signals: list of string
A list of all the signals selected for the final result
"""
# Building a single tags dataframe: timestamps MUST be aligned:
tags_df = pd.concat(timeseries_list, axis='columns')
# Resampling before taking MTF to reduce computational load:
if resample_rate is not None:
tags_df = tags_df.resample(resample_rate).mean()
# Cleaning NaN as they are not allowed to build the MTF:
tags_df.replace(np.nan, 0.0, inplace=True)
num_timeseries = len(timeseries_list)
# Adjust the column names to reflect the tags list:
if tags_list is not None:
tags_df.columns = tags_list
# Check for constant signals and remove them:
tags_stats = tags_df.describe().T
constant_signals = tags_stats[(tags_stats['max'] -
tags_stats['min']) == 0].index
tags_df = tags_df[tags_df.columns.difference(constant_signals)]
selected_signals = tags_df.columns.tolist()
# Get the MTF for all the signals:
mtf = MarkovTransitionField(image_size=image_size)
X = tags_df.values.T.reshape(tags_df.shape[1], -1)
tags_mtf = mtf.fit_transform(X)
return tags_mtf, constant_signals.tolist(), selected_signals
def __init__(self,serie,serie_test,m,h,windowsize=12,stride=1,alpha=0.25,beta=0,gamma=0.35,pr=3,compute_mtf=True):
"""Parameters:
-- serie: the training serie
-- serie_test: the test serie (to simplify the implementation it must be of the same length of serie)
-- m: seasonality of the serie
-- h: horizon of prediction
-- alpha,beta,gamma: intial guess of parameters of HW. The optimal
parameters are computed by the method parameter refinment given the training serie
-- windowsize: the size of the window
--stride: every how many steps a prediction is done e.g. stride=2 a predition is done a time t,an other a time t+2, predicting t+h, and t+h+2
-- compute_mtf wheter computing the mtf field: the library pyts does not manage to compute this field for Lorenz series
Requires: if m!=1, m>h (i.e. prediction are possible only within a season)"""
super(Holt_Winters_NN,self).__init__(serie,serie_test,m,h,alpha,0,gamma)
self._b[self._m-1]=0
self._b_test[self._m-1]=0
self.compute_states()
self.parameter_refinment(pr)
self.compute_states_test()
#the vector to give to the NN for training (i.e. the time series scaled and desonalised)
self._training_vector=(self._serie[self._m:self._length-self._h]/ \
self._l[self._m:(self._length-self._h)])/ \
self._s[0:(self._length-self._h-self._m)]
self._test_vector=(self._serie_test[self._m:self._length-self._h]/ \
self._l_test[self._m:(self._length-self._h)])/ \
self._s_test[0:(self._length-self._h-self._m)]
self._windowsize=windowsize
self._stride=stride
#n_windows=length of the list of images,lag: the first lag element of the serie are not used so that the windowsize fit the length of the serie
[n_windows,lag] = windomize_size(self._training_vector.size,self._windowsize,self._stride)
#serie deseasonalised and scaled, from which obtaining the imgs to give to the NN
self._training_output=self._serie[self._m+lag+self._windowsize-1+self._h:self._length]/ \
(self._l[(self._m+lag+self._windowsize-1):(self._length-self._h)]* \
self._s[(lag+self._windowsize-1+self._h):(self._length-self._m)])
#value for which the prediction of the NN must be multiplied
self.forecast_multiplier=self._l_test[(self._m+lag+self._windowsize-1):(self._length-self._h)]* \
self._s_test[(lag+self._windowsize-1+self._h):(self._length-self._m)]
#contains the value of the test serie aligned with the prediction
self.test_output=self._serie_test[self._m+lag+self._windowsize-1+self._h:self._length]
self.test_output_val=self._serie_test[self._m+lag+self._windowsize-1+self._h:self._length]/ \
(self._l_test[(self._m+lag+self._windowsize-1):(self._length-self._h)]* \
self._s_test[(lag+self._windowsize-1+self._h):(self._length-self._m)])
#self._training_output_multiple=np.zeros([m,self._training_output.size])
#check end of the vector it may
#for hh in range(1,self._m+1):
#self._training_output_multiple[hh-1,:]=self._serie[self._m+lag+self._windowsize-1+hh:self._length]/ \
#(self._l[(self._m+lag+self._windowsize-1):(self._length-hh)]* \
#self._s[(lag+self._windowsize-1):(self._length-hh-self._m)])
print(self._training_vector.mean())
#computation of the list of images for training and test
b=max(self._training_vector)
a=min(self._training_vector)
self._scale=b-a
self._training_vector=2*(self._training_vector-a)/(b-a)-1
b=max(self._test_vector)
a=min(self._test_vector)
self._scale_test=b-a
self._test_vector=2*(self._test_vector-a)/(b-a)-1
self._training_matrix=windowmize(self._training_vector,self._windowsize,self._stride)
gasf = GramianAngularField(image_size=1., method='summation',sample_range=None)
self.gasf = gasf.fit_transform(self._training_matrix)
gadf = GramianAngularField(image_size=1., method='difference',sample_range=None)
self.gadf = gadf.fit_transform(self._training_matrix)
if(compute_mtf):
mtf=MarkovTransitionField(image_size=1.)
self.mtf= mtf.fit_transform(self._training_matrix)
#in case of a first dense layer they could be usefull
#self.concatenated_images=np.concatenate((self.gadf,self.gasf), axis=1)
#self.concatenated_images=np.concatenate((self.gadf,self.gasf,self.mtf), axis=1)
self._test_matrix=windowmize(self._test_vector,self._windowsize,self._stride)
gasf_test = GramianAngularField(image_size=1., method='summation',sample_range=None)
self.gasf_test = gasf_test.fit_transform(self._test_matrix)
gadf_test= GramianAngularField(image_size=1., method='difference',sample_range=None)
self.gadf_test= gadf_test.fit_transform(self._test_matrix)
#check if it is correct
if(compute_mtf):
mtf_test=MarkovTransitionField(image_size=1.)
self.mtf_test= mtf_test.fit_transform(self._test_matrix)
from pyts.image import GramianAngularField
from pyts.image import MarkovTransitionField
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size, n_bins=8, strategy='quantile')
gadf_transformed_train = np.expand_dims(gadf.fit_transform(window_input_train),
axis=3)
gasf_transformed_train = np.expand_dims(gasf.fit_transform(window_input_train),
axis=3)
mtf_transformed_train = np.expand_dims(mtf.fit_transform(window_input_train),
axis=3)
X_train_windowed = np.concatenate(
(gadf_transformed_train, gasf_transformed_train, mtf_transformed_train),
axis=3)
gadf_transformed_test = np.expand_dims(gadf.fit_transform(window_input_test),
axis=3)
gasf_transformed_test = np.expand_dims(gasf.fit_transform(window_input_test),
axis=3)
mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test),
axis=3)
X_test_windowed = np.concatenate(
(gadf_transformed_test, gasf_transformed_test, mtf_transformed_test),
def train_series(self, signal_train, signal_test, window_size_x=100, window_size_y=1, epochs=5, bsize=32, p_filepath="predictions", l_filepath="tensorboard_logs", w_filepath="weights_conv_mlp", h=12, callbacks=True):
Conv_MLP_model = self.Conv_MLP_series(self.img_size, window_size_y, self.N_Channel)
# N_Channel = self.N_Channel
img_size = self.img_size
signal_train = signal_train.reshape(-1, 1)
signal_test = signal_test.reshape(-1, 1)
sample_range = (-1, 1)
# Scaling
# from sklearn.preprocessing import MinMaxScaler
#
# MMscaler = MinMaxScaler(feature_range=(-1,1))
# MMscaler_test = MinMaxScaler(feature_range=(-1, 1))
# signal_train_scaled = MMscaler.fit_transform(signal_train).flatten()
# signal_test_scaled = MMscaler_test.fit_transform(signal_test).flatten()
signal_train_scaled = signal_train.flatten()
signal_test_scaled = signal_test.flatten()
# Split Sequence
window_input_train, window_output_train = sequence_splitter(signal_train_scaled, window_size_x, window_size_y, h)
window_input_test, window_output_test = sequence_splitter(signal_test_scaled, window_size_x, window_size_y, h)
# %%---------------------------------------------------------------------------
'''
Field transformations
'''
from pyts.image import GramianAngularField
from pyts.image import MarkovTransitionField
gadf = GramianAngularField(image_size=img_size, method='difference', sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size, method='summation', sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size, n_bins=8, strategy='quantile')
gadf_transformed_train = np.expand_dims(gadf.fit_transform(window_input_train), axis=3)
gasf_transformed_train = np.expand_dims(gasf.fit_transform(window_input_train), axis=3)
mtf_transformed_train = np.expand_dims(mtf.fit_transform(window_input_train), axis=3)
X_train_windowed = np.concatenate((gadf_transformed_train, gasf_transformed_train, mtf_transformed_train),
axis=3)
gadf_transformed_test = np.expand_dims(gadf.fit_transform(window_input_test), axis=3)
gasf_transformed_test = np.expand_dims(gasf.fit_transform(window_input_test), axis=3)
mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test), axis=3)
X_test_windowed = np.concatenate((gadf_transformed_test, gasf_transformed_test, mtf_transformed_test), axis=3)
# Data reshaping
X_train_Conv_MLP = X_train_windowed
y_train_Conv_MLP = window_output_train
X_test_Conv_MLP = X_test_windowed
y_test_Conv_MLP = window_output_test
# *** Callbacks *** #
name = "Conv-MLP_raw_" + datetime.now().strftime("%Y%m%d-%H%M")
logdir = os.path.join(l_filepath, name)
tensorboard_callback = TensorBoard(log_dir=logdir)
path_w = w_filepath
filepath = os.path.join(path_w, "Conv_MLP_raw-weights-improvement-{epoch:02d}-{loss:.4f}-bigger.hdf5")
checkpoint = tf.keras.callbacks.ModelCheckpoint(
filepath,
monitor='loss',
verbose=0,
save_best_only=True,
mode='min'
)
if callbacks:
callback_list = [checkpoint, tensorboard_callback]
else:
callback_list = []
# *** Fit Model *** #
history_Conv_MLP = Conv_MLP_model.fit(x=X_train_Conv_MLP, y=y_train_Conv_MLP, batch_size=bsize, epochs=epochs,
callbacks=callback_list, shuffle=False,
validation_data=(X_test_Conv_MLP, y_test_Conv_MLP))
# *** Predict *** #
print("Average test loss: ", np.average(history_Conv_MLP.history['val_loss']))
preds_prep = Conv_MLP_model.predict(X_test_Conv_MLP)
preds_prep = preds_prep.reshape(-1, 1)
y_test_prep = y_test_Conv_MLP.reshape(-1, 1)
# preds_unscaled = MMscaler_test.inverse_transform(preds_prep)
# y_test_unscaled = MMscaler_test.inverse_transform(y_test_prep)
preds_unscaled = preds_prep
y_test_unscaled = y_test_prep
MSE_test_no_HW = ((y_test_unscaled - preds_unscaled) ** 2).mean()
print("Test loss: ", MSE_test_no_HW)
# show_result(y_test_prep, preds_prep)
df = pd.DataFrame({'True value': y_test_unscaled.flatten(), 'Predictions': preds_unscaled.flatten()})
fname = "Conv_MLP_raw-" + datetime.now().strftime("%Y%m%d-%H%M")
fpath_p = os.path.join(p_filepath, fname + ".csv")
df.to_csv(fpath_p)
return (history_Conv_MLP, preds_unscaled, y_test_unscaled, MSE_test_no_HW)
def _method(self, X, **kwargs):
mtk = MarkovTransitionField(**kwargs)
return mtk.fit_transform(X)
m = len(data[k])
for j in range(m):
new_data1 += [float(data[k][j].split(',')[2][1:-1])]
new_data2 += [float(data[k][j].split(',')[3][1:-1])]
new_data3 += [float(data[k][j].split(',')[4][1:-1])]
X = []
X.append(new_data1)
X.append(new_data2)
X.append(new_data3)
Z[k] = X
for i in range(N):
# MTF transformation
mtf = MarkovTransitionField(image_size=24)
X_mtf = mtf.fit_transform(Z[i])
if (i == 0):
title = 'NORMAL CASE'
if (i == 1):
title = 'ABNORMAL FREQUENCY'
if (i == 2):
title = 'VOLTAGE SAG'
if (i == 3):
title = 'VOLTAGE SWELL'
if (i == 4):
title = 'WAVEFORM DISTORTION'
fig, ax = plt.subplots(1)
fig = plt.figure(figsize=(6, 6))
plt.gca().axes.get_yaxis().set_visible(False)
plt.gca().axes.get_xaxis().set_visible(False)
def get_diff(self, series_train, series_test):
img_size = self.img_size
Conv_LSTM_model = self.model
out_window = self.out_window
in_window = self.in_window
sample_range = (-1, 1)
seq_len = self.seq_length
channels = self.channels
signal_train = series_train
signal_test = series_test
signal_train = signal_train.reshape(-1, 1)
signal_test = signal_test.reshape(-1, 1)
from sklearn.preprocessing import MinMaxScaler
MMscaler = MinMaxScaler(feature_range=sample_range)
signal_train_scaled = MMscaler.fit_transform(signal_train).flatten()
signal_test_scaled = MMscaler.fit_transform(signal_test).flatten()
window_input_train, window_output_train = sequence_splitter(
signal_train_scaled, in_window, out_window)
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, in_window, out_window)
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_transformed_train = np.expand_dims(
gadf.fit_transform(window_input_train), axis=3)
gasf_transformed_train = np.expand_dims(
gasf.fit_transform(window_input_train), axis=3)
mtf_transformed_train = np.expand_dims(
mtf.fit_transform(window_input_train), axis=3)
gadf_transformed_test = np.expand_dims(
gadf.fit_transform(window_input_test), axis=3)
gasf_transformed_test = np.expand_dims(
gasf.fit_transform(window_input_test), axis=3)
mtf_transformed_test = np.expand_dims(
mtf.fit_transform(window_input_test), axis=3)
X_train_windowed = np.concatenate(
(gadf_transformed_train, gasf_transformed_train,
mtf_transformed_train),
axis=3)
X_test_windowed = np.concatenate(
(gadf_transformed_test, gasf_transformed_test,
mtf_transformed_test),
axis=3)
X_train_Conv_LSTM = []
y_train_Conv_LSTM = []
X_test_Conv_LSTM = []
y_test_Conv_LSTM = []
print("Getting Train original")
for i in tqdm(range(0, X_train_windowed.shape[0] - seq_len + 1)):
current_seq_X = []
for l in range(seq_len):
current_seq_X.append(X_train_windowed[i + l])
X_train_Conv_LSTM.append(current_seq_X)
y_train_Conv_LSTM.append(window_output_train[i + seq_len - 1])
X_train_Conv_LSTM = np.array(X_train_Conv_LSTM)
X_train_Conv_LSTM = X_train_Conv_LSTM.reshape(-1, seq_len, img_size,
img_size, channels)
y_train_Conv_LSTM = np.array(y_train_Conv_LSTM)
y_train_Conv_LSTM = y_train_Conv_LSTM.reshape(-1, out_window)
print("Getting test original")
for i in tqdm(range(0, X_test_windowed.shape[0] - seq_len + 1)):
current_seq_X = []
for l in range(seq_len):
current_seq_X.append(X_test_windowed[i + l])
X_test_Conv_LSTM.append(current_seq_X)
y_test_Conv_LSTM.append(window_output_test[i + seq_len - 1])
X_test_Conv_LSTM = np.array(X_test_Conv_LSTM)
X_test_Conv_LSTM = X_test_Conv_LSTM.reshape(-1, seq_len, img_size,
img_size, channels)
y_test_Conv_LSTM = np.array(y_test_Conv_LSTM)
y_test_Conv_LSTM = y_test_Conv_LSTM.reshape(-1, out_window)
X_train_prep, y_train_prep, X_test_prep, y_test_prep = self.prep_series(
gadf_transformed_train, gasf_transformed_train,
mtf_transformed_train, gadf_transformed_test,
gasf_transformed_test, mtf_transformed_test, window_output_train,
window_output_test)
# df = pd.DataFrame({'x_test_prep_c3': X_train_prep[:,0,15,15,2].flatten(), 'x_test_c3': X_train_Conv_LSTM[:,0,15,15,2].flatten(), 'x_test_prep_c2': X_train_prep[:,0,15,15,1].flatten(), 'x_test_c2': X_train_Conv_LSTM[:,0,15,15,1].flatten()})
# df.to_csv("test.csv")
history_Conv_LSTM_prep = Conv_LSTM_model.fit(
x=X_train_prep,
y=y_train_prep,
batch_size=16,
epochs=5,
callbacks=None,
shuffle=True,
validation_data=(X_test_prep, y_test_prep))
print("Average test loss prep: ",
np.average(history_Conv_LSTM_prep.history['val_loss']))
preds_prep = Conv_LSTM_model.predict(X_train_prep)
preds_prep = preds_prep.reshape(-1)
y_test_prep = y_test_prep.reshape(-1)
MSE_test_no_HW_prep = ((y_test_prep - preds_prep)**2).mean()
print("Test loss prep: ", MSE_test_no_HW_prep)
df = pd.DataFrame({
'True value': y_test_prep.flatten(),
'Predictions': preds_prep.flatten()
})
fpath_p = os.path.join(
"predictions", "predictions_raw_prep_" +
datetime.now().strftime("%Y%m%d-%H%M%S") + ".csv")
df.to_csv(fpath_p)
model2 = self.get_model_lstm(out_window,
img_size,
3,
0,
1,
bidirectional=True,
dropout=0.4,
channels=3)
model2.compile(optimizer="adam", loss="mse", metrics=['mse'])
history_Conv_LSTM = model2.fit(x=X_train_Conv_LSTM,
y=y_train_Conv_LSTM,
batch_size=16,
epochs=5,
callbacks=None,
shuffle=True,
validation_data=(X_test_Conv_LSTM,
y_test_Conv_LSTM))
print("Average test loss: ",
np.average(history_Conv_LSTM.history['val_loss']))
preds = model2.predict(X_test_Conv_LSTM)
preds = preds.reshape(-1)
y_test_Conv_LSTM = y_test_Conv_LSTM.reshape(-1)
MSE_test_no_HW = ((y_test_Conv_LSTM - preds)**2).mean()
print("Test loss: ", MSE_test_no_HW)
df = pd.DataFrame({
'True value': y_test_Conv_LSTM.flatten(),
'Predictions': preds.flatten()
})
fpath_p = os.path.join(
"predictions", "predictions_raw_fresh_" +
datetime.now().strftime("%Y%m%d-%H%M%S") + ".csv")
df.to_csv(fpath_p)
return y_test_Conv_LSTM, preds, y_test_prep, preds_prep
def train_series(self,
series_train,
series_test,
p_filepath="predictions",
l_filepath="logs",
w_filepath="weights",
epochs=5,
bsize=16,
h=12,
callbacks=True):
img_size = self.img_size
Conv_LSTM_model = self.model
out_window = self.out_window
in_window = self.in_window
sample_range = (-1, 1)
name = "Conv-LSTM_raw_" + datetime.now().strftime("%Y%m%d-%H%M")
logdir = os.path.join(l_filepath, name)
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)
path_w = w_filepath
filepath = os.path.join(
path_w,
"Conv_LSTM_raw_weights-improvement-{epoch:02d}-{loss:.4f}-bigger.hdf5"
)
checkpoint = keras.callbacks.ModelCheckpoint(filepath,
monitor='loss',
verbose=0,
save_best_only=True,
mode='min')
if callbacks:
callbacks_list = [checkpoint, tensorboard_callback]
else:
callbacks_list = []
signal_train = series_train
signal_test = series_test
signal_train = signal_train.reshape(-1, 1)
signal_test = signal_test.reshape(-1, 1)
# MMscaler = MinMaxScaler(feature_range=sample_range)
#
# signal_train_scaled = MMscaler.fit_transform(signal_train).flatten()
# signal_test_scaled = MMscaler.fit_transform(signal_test).flatten()
signal_train_scaled = signal_train.flatten()
signal_test_scaled = signal_test.flatten()
window_input_train, window_output_train = sequence_splitter(
signal_train_scaled, in_window, out_window, h)
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, in_window, out_window, h)
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_transformed_train = np.expand_dims(
gadf.fit_transform(window_input_train), axis=3)
gasf_transformed_train = np.expand_dims(
gasf.fit_transform(window_input_train), axis=3)
mtf_transformed_train = np.expand_dims(
mtf.fit_transform(window_input_train), axis=3)
gadf_transformed_test = np.expand_dims(
gadf.fit_transform(window_input_test), axis=3)
gasf_transformed_test = np.expand_dims(
gasf.fit_transform(window_input_test), axis=3)
mtf_transformed_test = np.expand_dims(
mtf.fit_transform(window_input_test), axis=3)
X_train_prep, y_train_prep, X_test_prep, y_test_prep = self.prep_series(
gadf_transformed_train, gasf_transformed_train,
mtf_transformed_train, gadf_transformed_test,
gasf_transformed_test, mtf_transformed_test, window_output_train,
window_output_test)
history_Conv_LSTM_prep = Conv_LSTM_model.fit(
x=X_train_prep,
y=y_train_prep,
batch_size=bsize,
epochs=epochs,
callbacks=callbacks_list,
shuffle=True,
validation_data=(X_test_prep, y_test_prep))
print("Average test loss: ",
np.average(history_Conv_LSTM_prep.history['val_loss']))
preds_prep = Conv_LSTM_model.predict(X_test_prep)
preds_prep = preds_prep.reshape(-1, 1)
y_test_prep = y_test_prep.reshape(-1, 1)
# preps_unscaled = MMscaler.inverse_transform(preds_prep)
# y_test_unscaled = MMscaler.inverse_transform(y_test_prep)
preps_unscaled = preds_prep
y_test_unscaled = y_test_prep
MSE_test_no_HW = ((y_test_unscaled - preps_unscaled)**2).mean()
print("Test loss: ", MSE_test_no_HW)
# show_result(y_test_prep, preds_prep)
df = pd.DataFrame({
'True value': y_test_unscaled.flatten(),
'Predictions': preps_unscaled.flatten()
})
fpath_p = os.path.join(
p_filepath,
"Conv-LSTM_raw_" + datetime.now().strftime("%Y%m%d-%H%M") + ".csv")
df.to_csv(fpath_p)
return Conv_LSTM_model, y_test_prep, preds_prep
def mtf_encode_2_to_3(arr_2d, dim_multiple=3):
dim = arr_2d[0].shape[1] * dim_multiple
mtf = MarkovTransitionField(image_size=dim)
mtf_fdd_3d = (mtf.fit_transform(array.T) for array in arr_2d)
return mtf_fdd_3d
def GAF_data_2(df, step):
col = ["Open", "High", "Close", "Low"]
gasf = GramianAngularField(image_size=step, method="summation")
gadf = GramianAngularField(image_size=step, method="difference")
mtf = MarkovTransitionField(image_size=step)
X_mtf = []
X_gasf = []
X_gadf = []
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
ope = np.round(mtf.fit_transform([ts_1])[0] * 255)
high = np.round(mtf.fit_transform([ts_2])[0] * 255)
close = np.round(mtf.fit_transform([ts_3])[0] * 255)
low = np.round(mtf.fit_transform([ts_4])[0] * 255)
mtf_oh = np.hstack((ope, high))
mtf_cl = np.hstack((close, low))
mtf_ohcl = np.vstack((mtf_oh, mtf_cl))
X_mtf.append(mtf_ohcl.reshape(step * 2, step * 2, 1))
X_mtf = np.stack(X_mtf)
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
gadf_oh = np.hstack((np.round(gadf.fit_transform([ts_1])[0] * 255),
np.round(gadf.fit_transform([ts_2])[0] * 255)))
gadf_cl = np.hstack((np.round(gadf.fit_transform([ts_3])[0] * 255),
np.round(gadf.fit_transform([ts_4])[0] * 255)))
gadf_ohcl = np.vstack((gadf_oh, gadf_cl))
X_gadf.append(gadf_ohcl.reshape(step * 2, step * 2, 1))
X_gadf = np.stack(X_gadf)
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
gasf_oh = np.hstack((np.round(gasf.fit_transform([ts_1])[0] * 255),
np.round(gasf.fit_transform([ts_2])[0] * 255)))
gasf_cl = np.hstack((np.round(gasf.fit_transform([ts_3])[0] * 255),
np.round(gasf.fit_transform([ts_4])[0] * 255)))
gasf_ohcl = np.vstack((gasf_oh, gasf_cl))
X_gasf.append(gasf_ohcl.reshape(step * 2, step * 2, 1))
X_gasf = np.stack(X_gasf)
return (X_gasf, X_gadf, X_mtf)
def markov_transition(data, img_size=24):
X = data
mtf = MarkovTransitionField(image_size=img_size)
X_mtf = mtf.fit_transform(X)
return X_mtf