def save_to_GAF_img(df, file, step):
OHLC = ["Open", "High", "Low", "Close"]
high = max(df["High"])
low = min(df["Low"])
for col in OHLC:
Path("/content/GASF/" + col + "/").mkdir(parents=True, exist_ok=True)
Path("/content/GADF/" + col + "/").mkdir(parents=True, exist_ok=True)
Path("/content/MTF/" + col + "/").mkdir(parents=True, exist_ok=True)
gasf = GramianAngularField(image_size=step, method="summation")
gadf = GramianAngularField(image_size=step, method="difference")
mtf = MarkovTransitionField(image_size=step)
ts_norm = [(i - low) / (high - low) for i in list(df[col])]
X_mtf = mtf.fit_transform([ts_norm])
X_gasf = gasf.fit_transform([ts_norm])
X_gadf = gadf.fit_transform([ts_norm])
plt.imsave("/content/other_n/GASF/" + col + "/" + file,
X_gasf[0],
cmap="gray")
plt.imsave("/content/other_n/GADF/" + col + "/" + file,
X_gadf[0],
cmap="gray")
plt.imsave("/content/other_n/MTF/" + col + "/" + file,
X_mtf[0],
cmap="gray")
def preprocess_audio(opt, y, sr):
if opt == 'spect':
D = librosa.stft(y, n_fft=480, hop_length=160, win_length=480, window='hamming')
spect,phase = librosa.magphase(D)
graph_data = log_spect = np.log(spect)
elif opt == 'mp_spect':
S = librosa.feature.melspectrogram(y, sr=sr, n_mels=128)
graph_data = log_S = librosa.power_to_db(S, ref=np.max)
elif opt == 'cqt':
graph_data = C = librosa.cqt(y,sr)
elif opt == 'chrom':
C = np.abs(librosa.cqt(y,sr))
graph_data = chroma = librosa.feature.chroma_cqt(C=C,sr=sr)
elif opt == 'mfcc':
raw_mfcc = librosa.feature.mfcc(y=y,sr=sr)
graph_data = scaled_mfcc = scale(raw_mfcc, axis=1)
elif opt == 'gasf':
gasf = GramianAngularField(image_size=256, method='summation')
graph_data = np.squeeze(gasf.fit_transform(y.reshape(1,-1)))
elif opt == 'gadf':
gadf = GramianAngularField(image_size=256, method='difference')
graph_data = np.squeeze(gadf.fit_transform(y.reshape(1,-1)))
else:
raise NotImplementedError('Graph data not known for {}'.format(opt))
return graph_data
def prep_seriesConvLSTM(seq_len, out_window, in_window, img_size, channels,
series_test, h):
print("Preparing data: ")
sample_range = (-1, 1)
signal_test = series_test
signal_test = signal_test.reshape(-1, 1)
signal_test_scaled = signal_test.flatten()
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, in_window, out_window, h)
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_test = np.expand_dims(gadf.fit_transform(window_input_test), axis=3)
gasf_test = np.expand_dims(gasf.fit_transform(window_input_test), axis=3)
mtf_test = np.expand_dims(mtf.fit_transform(window_input_test), axis=3)
y_test = window_output_test.reshape(-1)
if (channels == 2):
X_test_windowed = np.concatenate((gadf_test, gasf_test), axis=3)
else:
X_test_windowed = np.concatenate((gadf_test, gasf_test, mtf_test),
axis=3)
X_test_Conv_LSTM = np.zeros((X_test_windowed.shape[0] - seq_len + 1,
seq_len, img_size, img_size, channels))
y_test_Conv_LSTM = np.zeros(
(X_test_windowed.shape[0] - seq_len + 1, out_window))
print("Test data:")
for i in tqdm(range(0, X_test_windowed.shape[0] - seq_len + 1)):
current_seq_X = np.zeros((seq_len, img_size, img_size, channels))
for l in range(seq_len):
current_seq_X[l] = X_test_windowed[i + l]
current_seq_X = current_seq_X.reshape(1, seq_len, img_size, img_size,
channels)
X_test_Conv_LSTM[i] = current_seq_X
y_test_Conv_LSTM[i] = y_test[i + seq_len - 1]
X_test_Conv_LSTM = X_test_Conv_LSTM.reshape(-1, seq_len, img_size,
img_size, channels)
y_test_Conv_LSTM = y_test_Conv_LSTM.reshape(-1, out_window)
return (X_test_Conv_LSTM, y_test_Conv_LSTM)
def getMidpointFields(samples, size):
fields = []
g = GramianAngularField(image_size=size, method='summation')
S = np.array(samples).reshape(1, size)
T = g.fit_transform(S)
fields.append(T[0])
g = GramianAngularField(image_size=size, method='difference')
S = np.array(samples).reshape(1, size)
T = g.fit_transform(S)
fields.append(T[0])
return fields
def ts_imaging(self, data):
"""
Calcultes the image representation for each batch
Args:
data (tf.tensor(batch_size, sequence_length)): a batch of the aggregate power sequences
Raises:
ImagingMethodError: Error raised in case a wrong image transform is provided
Returns:
tensor(batch_size, img_size, img_size, 1): the image representation of the of the input data
"""
if self.img_method == 'gasf':
transformer = GramianAngularField(image_size=self.img_size,
method='summation')
tsi = transformer.fit_transform(data)
elif self.img_method == 'gadf':
transformer = GramianAngularField(image_size=self.img_size,
method='difference')
tsi = transformer.fit_transform(data)
elif self.img_method == 'mtf':
transformer = MarkovTransitionField(image_size=self.img_size)
tsi = transformer.fit_transform(data)
elif self.img_method == 'rp':
transformer = RecurrencePlot(threshold='point', percentage=20)
tsi = transformer.fit_transform(data)
elif self.img_size == 'all':
print("""
To use the three images at once
the input layer need to be adapted in (line 197 in IM2Seq.py).
The new input layer should be :
model.add(Conv2D(filters=8, kernel_size=4, strides=2, activation='linear',
input_shape=(self.img_size, self.img_size, 3)))
""")
RP = RecurrencePlot(threshold='point',
percentage=20).fit_transform(data)
GASF = GramianAngularField(image_size=self.img_size,
method='summation').fit_transform(data)
MTF = MarkovTransitionField(
image_size=self.img_size).fit_transform(data)
tsi = np.stack([RP, GASF, MTF], axis=3)
else:
raise ImagingMethodError()
return tsi
def generateGaf(self):
values = self.to_numpy()
# Function Specific parameters
index_val = self.df.index.max()
outfile = self.output_dir + str(index_val) + '.jpeg'
if self.fieldType == 'gasf':
gaf = GramianAngularField(image_size=self.window_size, method='summation')
elif self.fieldType == 'gadf':
gaf = GramianAngularField(image_size=self.window_size, method='difference')
gaf = gaf.fit_transform(X=values)
# Generate Image
fig = plt.figure(figsize=(8, 8))
grid = ImageGrid(fig, 111,
nrows_ncols=(2, 2),
axes_pad=0,
share_all=True
)
images = [gaf[0], gaf[1], gaf[2], gaf[3]]
for image, ax in zip(images, grid):
ax.imshow(image, cmap='rainbow', origin='lower')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
ax.cax.toggle_label(False)
plt.axis('off')
if self.debug_level == 2:
plt.show()
if self.debug_level == 1:
print(outfile)
plt.savefig(outfile, pil_kwargs={'optimize': True, 'quality': self.quality})
plt.close('all')
def GAF(sin_data, image_size):
sin_data = np.array(sin_data)
sin_data = sin_data.reshape(1, -1)
print(sin_data)
gasf = GramianAngularField(image_size=image_size,
method='summation',
overlapping='False')
sin_gasf = gasf.fit_transform(sin_data)
print(sin_gasf)
gadf = GramianAngularField(image_size=image_size, method='difference')
sin_gadf = gadf.fit_transform(sin_data)
images = [sin_gasf[0], sin_gadf[0]]
with open('GAF1.csv', 'w') as csvfile:
writer = csv.writer(csvfile)
writer.writerows(images[0])
return images
def create_gaf(ts) -> Dict[str, Any]:
"""
:param ts:
:return:
"""
data = dict()
gadf = GramianAngularField(method='difference', image_size=ts.shape[0])
data['gadf'] = gadf.fit_transform(pd.DataFrame(ts).T)[0]
return data
def prep_seriesConvMLP(window_size_x, window_size_y, img_size, signal_test, h):
signal_test = signal_test.reshape(-1, 1)
sample_range = (-1, 1)
signal_test_scaled = signal_test.flatten()
# Split Sequence
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, window_size_x, window_size_y, h)
# %%---------------------------------------------------------------------------
'''
Field transformations
'''
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_transformed_test = np.expand_dims(
gadf.fit_transform(window_input_test), axis=3)
gasf_transformed_test = np.expand_dims(
gasf.fit_transform(window_input_test), axis=3)
mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test),
axis=3)
X_test_windowed = np.concatenate(
(gadf_transformed_test, gasf_transformed_test, mtf_transformed_test),
axis=3)
# Data reshaping
X_test_Conv_MLP = X_test_windowed
y_test_Conv_MLP = window_output_test
return (X_test_Conv_MLP, y_test_Conv_MLP)
def ts_imaging(self, data):
"""
Calcultes the image representation for each batch
Args:
data (tf.tensor(batch_size, sequence_length)): a batch of the aggregate power sequences
Raises:
ImagingMethodError: Error raised in case a wrong image transform is provided
Returns:
tensor(batch_size, img_size, img_size, 1): the image representation of the of the input data
"""
if self.img_method == 'gasf':
transformer = GramianAngularField(image_size=self.img_size,
method='summation')
tsi = transformer.fit_transform(data)
elif self.img_method == 'gadf':
transformer = GramianAngularField(image_size=self.img_size,
method='difference')
tsi = transformer.fit_transform(data)
elif self.img_method == 'mtf':
transformer = MarkovTransitionField(image_size=self.img_size)
tsi = transformer.fit_transform(data)
elif self.img_method == 'rp':
transformer = RecurrencePlot(threshold='point', percentage=20)
tsi = transformer.fit_transform(data)
elif self.img_size == 'all':
RP = RecurrencePlot(threshold='point',
percentage=20).fit_transform(data)
GASF = GramianAngularField(image_size=self.img_size,
method='summation').fit_transform(data)
MTF = MarkovTransitionField(
image_size=self.img_size).fit_transform(data)
tsi = np.stack([RP, GASF, MTF], axis=3)
else:
raise ImagingMethodError()
return tsi
def getOrderbookField(askPriceSamples, askSizeSamples, bidPriceSamples,
bidSizeSamples, size):
numSamples = len(askPriceSamples)
depth = len(askPriceSamples[0])
samples = []
for i in range(numSamples):
samples.append([])
for j in range(depth):
samples[i].append((askPriceSamples[i][j] * askSizeSamples[i][j] -
bidPriceSamples[i][j] * bidSizeSamples[i][j]) /
(askSizeSamples[i][j] + bidSizeSamples[i][j]))
G = GramianAngularField(image_size=size, method='summation')
S = np.transpose(np.array(samples))
T = G.fit_transform(S)
return T
def GASF_encoder(ts,
size=None,
sample_range=None,
overlapping=False,
**kwargs):
ts = To2dArray(ts)
assert ts.ndim == 2, 'ts ndim must be 2!'
if size is None: size = ts.shape[-1]
else: size = min(size, ts.shape[-1])
encoder = GAF(image_size=size,
sample_range=sample_range,
method='s',
overlapping=overlapping)
output = np.squeeze(encoder.fit_transform(ts), 0)
return (output + 1) / 2
def toGAFdata(tsdatas,
image_size=1.,
sample_range=(-1, 1),
method='summation',
overlapping=False,
flatten=False):
X = []
gaf = GramianAngularField(image_size=image_size,
sample_range=sample_range,
method=method,
overlapping=overlapping,
flatten=flatten)
for data in tsdatas:
data_gaf = gaf.fit_transform(data)
X.append(data_gaf[0])
return np.array(X)
def gen_GAF_exec(data: list,
sample_range: None or tuple = (-1, 1),
method: str = 'summation',
null_value: str = '0'):
"""
**Generate a Gramian angular Field**
this is the actual function when it comes to generating a Gramian Angular Field
out of the data of a numpy array. This function takes different variables to determine
how the Field should be scaled, what its size should be
and if it is either a summation or difference Field
param data: as the content of a npy file , type list
param size: this is the size of the square output image, type int or float
param sample_range: as the range the data should be scaled to, type None or tuple
param method: as the type of field it should be, type 'summation' or 'difference'
param **null_value**: as the number to use instead of NULL, type str
"""
gaf = GAF(sample_range=sample_range, method=method)
data = np.where(data == 'NULL', null_value, data)
data = data[:, 3:].astype(dtype=float)
data_gaf = gaf.fit_transform(data)
plt.imshow(data_gaf[0], cmap='rainbow', origin='lower')
plt.show()
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid
from pyts.image import GramianAngularField
# Parameters
n_samples, n_timestamps = 100, 144
# Toy dataset
rng = np.random.RandomState(41)
X = rng.randn(n_samples, n_timestamps)
# Transform the time series into Gramian Angular Fields
gasf = GramianAngularField(image_size=24, method='summation')
X_gasf = gasf.fit_transform(X)
gadf = GramianAngularField(image_size=24, method='difference')
X_gadf = gadf.fit_transform(X)
# Show the images for the first time series
fig = plt.figure(figsize=(12, 7))
grid = ImageGrid(fig, 111,
nrows_ncols=(1, 2),
axes_pad=0.15,
share_all=True,
cbar_location="right",
cbar_mode="single",
cbar_size="7%",
cbar_pad=0.3,
)
images = [X_gasf[0], X_gadf[0]]
def generate_data(
rates,
r=0.7,
test=True,
save_img=True,
save_gasf=True,
gasf_imsize=16,
save_npy=True,
tp=0.00500,
sl=0.00250,
sl_h=0.00250,
window_range_back=72,
window_range_front=30,
keys=[],
dpi=60,
rcparams={
'axes.spines.bottom': False,
'axes.spines.left': False,
'axes.spines.right': False,
'axes.spines.top': False,
}):
gasf = GramianAngularField(image_size=gasf_imsize, method='summation')
X_buy = list() #N OHLC candles window
X_buy_chart = list()
X_buy_gasf = list()
Y_reg_buy = list() #Next M values
X_sell = list()
X_sell_chart = list()
X_sell_gasf = list()
Y_reg_sell = list()
X_hold = list()
X_hold_chart = list()
X_hold_gasf = list()
Y_reg_hold = list()
save_img = save_img
save_gasf = save_gasf
TP = tp
SL_hold = sl_h
SL = sl
if test == True:
start = window_range_back + int(r * len(rates))
end = len(rates)
folder = 'data/chart/test'
else:
start = window_range_back
end = int(len(rates) * r)
folder = 'data/chart/train'
#Clean old data (Just in case acquisition window is changed or something, else you end up with unwanted data.)
if save_img == True:
for path in paths:
files = glob.glob(path + '/*.jpg')
for f in files:
os.remove(f)
i = start
while (i < end): #hold ops
window_back = rates[i - window_range_back:i]
window_front = rates[i:i + window_range_front]
if len(window_front) < window_range_front:
i = end
continue
lastclose = window_back.Close.iloc[-1]
hold_up = window_front[window_front.High > lastclose +
(SL_hold + 0.00100)]
hold_down = window_front[window_front.Low < lastclose -
(SL_hold + 0.00100)]
if len(hold_up) == 0 and len(
hold_down
) == 0: #Hold (strong consolidation or possible swing back)
X_hold.append(np.array(window_back.values))
Y_reg_hold.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_hold_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_hold_gasf.append(X_gasf)
else:
i += 1
i = start
while (i < end): #buy ops
window_back = rates[i - window_range_back:i]
window_front = rates[i:i + window_range_front]
if len(window_front) < window_range_front:
i = end
continue
lastclose = window_back.Close.iloc[-1]
TP_hit = window_front[window_front.High >= lastclose + TP]
SL_hit = window_front[window_front.Low <= lastclose - SL]
hold_up = window_front[window_front.High > lastclose +
(SL_hold + 0.00100)]
hold_down = window_front[window_front.Low <= lastclose -
(SL_hold + 0.00100)]
if (len(TP_hit) == 0 and len(SL_hit) == 0) and (
len(hold_up) == 0 and len(hold_down)
== 0): #Hold (strong consolidation or possible swing back)
i += window_range_front
continue
elif len(TP_hit) > 0 and len(
SL_hit) == 0: #buy (no SL hit in period but TP is hit)
X_buy.append(np.array(window_back.values))
Y_reg_buy.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_buy_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_buy_gasf.append(X_gasf)
elif len(TP_hit) > 0 and len(
SL_hit
) > 0: #buy (both tp and sl hit but SL is hit after TP so trade won)
TP_hit = TP_hit.iloc[0]
SL_hit = SL_hit.iloc[0]
if TP_hit.timestamp < SL_hit.timestamp:
X_buy.append(np.array(window_back.values))
Y_reg_buy.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_buy_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_buy_gasf.append(X_gasf)
else:
i += 1
else:
i += 1
#look for sells and holds
i = start
while (i < end): #sell_ops
window_back = rates[i - window_range_back:i]
window_front = rates[i:i + window_range_front]
if len(window_front) < window_range_front:
i = end
continue
lastclose = window_back.Close.iloc[-1]
SL_hit = window_front[window_front.High >= lastclose + SL]
TP_hit = window_front[window_front.Low <= lastclose - TP]
hold_up = window_front[window_front.High > lastclose +
(SL_hold + 0.00100)]
hold_down = window_front[window_front.Low <= lastclose -
(SL_hold + 0.00100)]
if (len(TP_hit) == 0
and len(SL_hit) == 0) and (len(hold_up) == 0 and len(hold_down)
== 0): #Hold (strong consolidation)
i += window_range_front
continue
elif len(TP_hit) > 0 and len(
SL_hit) == 0: #buy (no SL hit in period but TP is hit)
X_sell.append(np.array(window_back.values))
Y_reg_sell.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_sell_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_sell_gasf.append(X_gasf)
elif len(TP_hit) > 0 and len(
SL_hit
) > 0: #buy (both tp and sl hit but SL is hit after TP so trade won)
TP_hit = TP_hit.iloc[0]
SL_hit = SL_hit.iloc[0]
if TP_hit.timestamp < SL_hit.timestamp:
X_sell.append(np.array(window_back.values))
Y_reg_sell.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_sell_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_sell_gasf.append(X_gasf)
else:
i += 1
else:
i += 1
X_buy = np.array(X_buy) #72 OHLC candles window
X_buy_chart = np.array(X_buy_chart)
X_buy_gasf = np.array(X_buy_gasf)
Y_reg_buy = np.array(Y_reg_buy) #Next 72 values
X_sell = np.array(X_sell)
X_sell_chart = np.array(X_sell_chart)
X_sell_gasf = np.array(X_sell_gasf)
Y_reg_sell = np.array(Y_reg_sell)
X_hold = np.array(X_hold)
X_hold_chart = np.array(X_hold_chart)
X_hold_gasf = np.array(X_hold_gasf)
Y_reg_hold = np.array(Y_reg_hold)
print(X_buy.shape)
print(Y_reg_buy.shape)
print(X_buy_gasf.shape)
print(X_buy_chart.shape)
print(X_sell.shape)
print(Y_reg_sell.shape)
print(X_sell_gasf.shape)
print(X_sell_chart.shape)
print(X_hold.shape)
print(Y_reg_hold.shape)
print(X_hold_gasf.shape)
print(X_hold_chart.shape)
if save_npy == True:
print('Saving npy candle data [to be added]')
return X_buy, X_buy_chart, X_buy_gasf, Y_reg_buy, X_sell, X_sell_chart, X_sell_gasf, Y_reg_sell, X_hold, X_hold_chart, X_sell_gasf, Y_reg_hold
def to_gaf(df):
gaf = GramianAngularField(image_size=image_size, sample_range=sample_range, method=method,
overlapping=overlapping, flatten=flatten)
return gaf.fit_transform(df.values)
keys = create_zigzag(rates, pct=0.3)
addp3 = mpf.make_addplot(rates[keys])
mpf.plot(rates,
type='candle',
volume=True,
addplot=[addp3],
style='yahoo',
show_nontrading=False,
block=False)
price = rates.Close
err_allowed = 0.1 / 100
# Find peaks
for i in range(100, len(price)):
current_idx, current_pat, start, end = peak_detect(price.values[:i])
plt.plot(np.arange(start, i), price.values[start:i])
plt.scatter(current_idx, current_pat, c='r')
plt.show()
gasf = GramianAngularField(image_size=16, method='summation')
X_gasf = gasf.fit_transform(X_buy[0].transpose([1, 0]))
print(X_gasf[3].shape)
plt.imshow(X_gasf[3, :])
plt.show()
while (True):
msg = input('Close? [N],y\n')
if msg == 'y':
break
def _method(self, X, **kwargs):
gadf = GramianAngularField(method='difference', **kwargs)
return gadf.fit_transform(X)
def _method(self, X, **kwargs):
gasf = GramianAngularField(method='summation', **kwargs)
return gasf.fit_transform(X)
def getAskPriceFields(samples, size):
G = GramianAngularField(image_size=size, method='summation')
S = np.transpose(np.array(samples))
T = G.fit_transform(S)
return T
def __init__(self,serie,serie_test,m,h,windowsize=12,stride=1,alpha=0.25,beta=0,gamma=0.35,pr=3,compute_mtf=True):
"""Parameters:
-- serie: the training serie
-- serie_test: the test serie (to simplify the implementation it must be of the same length of serie)
-- m: seasonality of the serie
-- h: horizon of prediction
-- alpha,beta,gamma: intial guess of parameters of HW. The optimal
parameters are computed by the method parameter refinment given the training serie
-- windowsize: the size of the window
--stride: every how many steps a prediction is done e.g. stride=2 a predition is done a time t,an other a time t+2, predicting t+h, and t+h+2
-- compute_mtf wheter computing the mtf field: the library pyts does not manage to compute this field for Lorenz series
Requires: if m!=1, m>h (i.e. prediction are possible only within a season)"""
super(Holt_Winters_NN,self).__init__(serie,serie_test,m,h,alpha,0,gamma)
self._b[self._m-1]=0
self._b_test[self._m-1]=0
self.compute_states()
self.parameter_refinment(pr)
self.compute_states_test()
#the vector to give to the NN for training (i.e. the time series scaled and desonalised)
self._training_vector=(self._serie[self._m:self._length-self._h]/ \
self._l[self._m:(self._length-self._h)])/ \
self._s[0:(self._length-self._h-self._m)]
self._test_vector=(self._serie_test[self._m:self._length-self._h]/ \
self._l_test[self._m:(self._length-self._h)])/ \
self._s_test[0:(self._length-self._h-self._m)]
self._windowsize=windowsize
self._stride=stride
#n_windows=length of the list of images,lag: the first lag element of the serie are not used so that the windowsize fit the length of the serie
[n_windows,lag] = windomize_size(self._training_vector.size,self._windowsize,self._stride)
#serie deseasonalised and scaled, from which obtaining the imgs to give to the NN
self._training_output=self._serie[self._m+lag+self._windowsize-1+self._h:self._length]/ \
(self._l[(self._m+lag+self._windowsize-1):(self._length-self._h)]* \
self._s[(lag+self._windowsize-1+self._h):(self._length-self._m)])
#value for which the prediction of the NN must be multiplied
self.forecast_multiplier=self._l_test[(self._m+lag+self._windowsize-1):(self._length-self._h)]* \
self._s_test[(lag+self._windowsize-1+self._h):(self._length-self._m)]
#contains the value of the test serie aligned with the prediction
self.test_output=self._serie_test[self._m+lag+self._windowsize-1+self._h:self._length]
self.test_output_val=self._serie_test[self._m+lag+self._windowsize-1+self._h:self._length]/ \
(self._l_test[(self._m+lag+self._windowsize-1):(self._length-self._h)]* \
self._s_test[(lag+self._windowsize-1+self._h):(self._length-self._m)])
#self._training_output_multiple=np.zeros([m,self._training_output.size])
#check end of the vector it may
#for hh in range(1,self._m+1):
#self._training_output_multiple[hh-1,:]=self._serie[self._m+lag+self._windowsize-1+hh:self._length]/ \
#(self._l[(self._m+lag+self._windowsize-1):(self._length-hh)]* \
#self._s[(lag+self._windowsize-1):(self._length-hh-self._m)])
print(self._training_vector.mean())
#computation of the list of images for training and test
b=max(self._training_vector)
a=min(self._training_vector)
self._scale=b-a
self._training_vector=2*(self._training_vector-a)/(b-a)-1
b=max(self._test_vector)
a=min(self._test_vector)
self._scale_test=b-a
self._test_vector=2*(self._test_vector-a)/(b-a)-1
self._training_matrix=windowmize(self._training_vector,self._windowsize,self._stride)
gasf = GramianAngularField(image_size=1., method='summation',sample_range=None)
self.gasf = gasf.fit_transform(self._training_matrix)
gadf = GramianAngularField(image_size=1., method='difference',sample_range=None)
self.gadf = gadf.fit_transform(self._training_matrix)
if(compute_mtf):
mtf=MarkovTransitionField(image_size=1.)
self.mtf= mtf.fit_transform(self._training_matrix)
#in case of a first dense layer they could be usefull
#self.concatenated_images=np.concatenate((self.gadf,self.gasf), axis=1)
#self.concatenated_images=np.concatenate((self.gadf,self.gasf,self.mtf), axis=1)
self._test_matrix=windowmize(self._test_vector,self._windowsize,self._stride)
gasf_test = GramianAngularField(image_size=1., method='summation',sample_range=None)
self.gasf_test = gasf_test.fit_transform(self._test_matrix)
gadf_test= GramianAngularField(image_size=1., method='difference',sample_range=None)
self.gadf_test= gadf_test.fit_transform(self._test_matrix)
#check if it is correct
if(compute_mtf):
mtf_test=MarkovTransitionField(image_size=1.)
self.mtf_test= mtf_test.fit_transform(self._test_matrix)
#%%
import os
import numpy as np
import matplotlib.pyplot as plt
from pyts.image import GramianAngularField, MarkovTransitionField
path = "D:\\研究所\\雷射切割\\0529\\power data\\"
list_data = os.listdir(path)
list_data.sort(key=lambda x: tuple(map(int, x[:-4].split("-"))))
print(list_data)
#%%
for x in list_data:
path_data = path + x
name = x.split(".")
print(x, name[0])
#%%
a = np.random.randn(20, 20)
gasf = GramianAngularField(image_size=20, method='summation')
a_gasf = gasf.fit_transform(a)
plt.imshow(a)
plt.show()
def CWRU_data_2d_gaf(self, type='DE'):
'''
把CWRU数据集做成2d图像，使用Gramian Angular Field (GAF)，保存为png图片
因为GAF将n的时序信号转换为n*n，这样导致数据量过大，采用分段聚合近似（PAA）转换压缩时序数据的长度
97：243938
:type: DE还是FE
:return: 保存为2d图像
'''
# CWRU原始数据
CWRU_data_path = opt.CWRU_data_root
# 维度
dim = opt.CWRU_dim
# 读取文件列表
frame_name = os.path.join(CWRU_data_path, 'annotations.txt')
frame = pd.read_table(frame_name)
# 保存路径
save_path = os.path.join(opt.CWRU_data_2d_root, type)
if not os.path.exists(save_path):
os.makedirs(save_path)
# gasf文件目录
gasf_path = os.path.join(save_path, 'gasf')
if not os.path.exists(gasf_path):
os.makedirs(gasf_path)
# gadf文件目录
gadf_path = os.path.join(save_path, 'gadf')
if not os.path.exists(gadf_path):
os.makedirs(gadf_path)
for idx in tqdm(range(len(frame))):
# mat文件名
mat_name = os.path.join(CWRU_data_path, frame['file_name'][idx])
# 读取mat文件中的原始数据
raw_data = scio.loadmat(mat_name)
# raw_data.items() X097_DE_time 所以选取5:7为DE的
for key, value in raw_data.items():
if key[5:7] == type:
# dim个数据点一个划分，计算数据块的数量
sample_num = value.shape[0] // dim
# 数据取整，把列向量转换成行向量
signal = value[0:dim * sample_num].reshape(1, -1)
# PAA 分段聚合近似（PAA）转换
# paa = PiecewiseAggregateApproximation(n_segments=100)
# paa_signal = paa.fit_transform(signal)
# 按sample_num切分，每个dim大小
signals = np.split(signal, sample_num, axis=1)
for i in tqdm(range(len(signals))):
# 将每个dim的数据转换为2d图像
gasf = GramianAngularField(image_size=dim,
method='summation')
signals_gasf = gasf.fit_transform(signals[i])
gadf = GramianAngularField(image_size=dim,
method='difference')
signals_gadf = gadf.fit_transform(signals[i])
# 保存图像
filename_gasf = os.path.join(gasf_path,
str(idx) + '.%d.png' % i)
image.imsave(filename_gasf, signals_gasf[0])
filename_gadf = os.path.join(gadf_path,
str(idx) + '.%d.png' % i)
image.imsave(filename_gadf, signals_gadf[0])
#%%---------------------------------------------------------------------------
'''
Field transformations
'''
from pyts.image import GramianAngularField
from pyts.image import MarkovTransitionField
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size, n_bins=8, strategy='quantile')
gadf_transformed_train = np.expand_dims(gadf.fit_transform(window_input_train),
axis=3)
gasf_transformed_train = np.expand_dims(gasf.fit_transform(window_input_train),
axis=3)
mtf_transformed_train = np.expand_dims(mtf.fit_transform(window_input_train),
axis=3)
X_train_windowed = np.concatenate(
(gadf_transformed_train, gasf_transformed_train, mtf_transformed_train),
axis=3)
gadf_transformed_test = np.expand_dims(gadf.fit_transform(window_input_test),
axis=3)
gasf_transformed_test = np.expand_dims(gasf.fit_transform(window_input_test),
axis=3)
mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test),
import numpy as np
import matplotlib.pyplot as plt
from pyts.image import GramianAngularField
sin_data = np.loadtxt('sinx.csv', delimiter=",", skiprows=0).reshape(1, -1)
image_size = 28
gasf = GramianAngularField(image_size=image_size, method='summation')
sin_gasf = gasf.fit_transform(sin_data)
gadf = GramianAngularField(image_size=image_size, method='difference')
sin_gadf = gadf.fit_transform(sin_data)
images = [sin_gasf[0], sin_gadf[0]]
titles = ['Summation', 'Difference']
fig, axs = plt.subplots(1, 2, constrained_layout=True)
for image, title, ax in zip(images, titles, axs):
ax.imshow(image)
ax.set_title(title)
fig.suptitle('GramianAngularField', y=0.94, fontsize=16)
plt.margins(0, 0)
plt.savefig("GramianAngularField.pdf", pad_inches=0)
plt.show()
def plot_skeleton_data(skeleton_data,
size=5,
offset=False,
approach_id=1,
alpha=0.2,
attention=False,
filter_percentage=0.2):
fig = plt.figure(figsize=(5, 5))
# fig.title(filename)
ax = fig.add_subplot(frameon=False)
# ax = fig.add_axes([0, 0, 1, 1])
ax.axis("off")
# ax = plt.figure(frameon=False)
# ax.spines['top'].set_visible(False)
# ax.spines['right'].set_visible(False)
# ax.spines['bottom'].set_linewidth(0.0)
# ax.spines['left'].set_linewidth(0.0)
# plt.subplots_adjust(left=0.1, right=0.9, top=0.9, bottom=0.1)
if approach_id == 3:
skeleton_data = np.sort(skeleton_data, axis=0)
if approach_id == 8:
skeleton_data = filter_skeleton(skeleton_data)
if approach_id == 9:
print("before filter, ", skeleton_data.shape)
skeleton_data = filter_attention(skeleton_data)
print("after filter, ", skeleton_data.shape)
if attention:
print("before filter, ", skeleton_data.shape)
skeleton_data = filter_attention(skeleton_data, filter_percentage)
print("after filter, ", skeleton_data.shape)
for joint_index, joint_data in enumerate(skeleton_data):
# line currently the best results
if approach_id == 1:
of1 = 0
of2 = 2
of3 = 3
if approach_id == 2:
of1 = joint_index / 2 + offset
of2 = joint_index / 2 + offset
of3 = joint_index / 2 + offset
if approach_id == 3:
of1 = joint_index / 2 + offset
of2 = joint_index / 2 + offset
of3 = joint_index / 2 + offset
if joint_index > len(skeleton_data) / 6:
continue
if approach_id == 4:
of1 = joint_index / 2
of2 = joint_index / 2
of3 = joint_index / 2
if joint_data.var() > 1:
continue
if approach_id == 5:
import pyts
from pyts.image import GramianAngularField
transformer = GramianAngularField(image_size=len(joint_data))
image_data = transformer.fit_transform(joint_data)
plt.figure(figsize=(10, 10))
plt.imshow(image_data)
plt.savefig("gaf.png")
continue
if approach_id == 6:
of1 = -np.average(joint_data[0])
of2 = -np.average(joint_data[1])
of3 = -np.average(joint_data[2])
if approach_id == 7:
of1 = -np.max(joint_data[0])
of2 = -np.max(joint_data[1])
of3 = -np.max(joint_data[2])
if approach_id == 8:
of1 = -np.max(joint_data[0])
of2 = -np.max(joint_data[1])
of3 = -np.max(joint_data[2])
if approach_id == 9:
of1 = -np.max(joint_data[0])
of2 = -np.max(joint_data[1])
of3 = -np.max(joint_data[2])
if approach_id == 10:
of1 = -np.average(joint_data[0])
of2 = -np.average(joint_data[1]) + 0.5
of3 = -np.average(joint_data[2]) + 1
if approach_id == 11: # like 10 but use alpha for temporal
of1 = -np.average(joint_data[0])
of2 = -np.average(joint_data[1]) + 0.5
of3 = -np.average(joint_data[2]) + 1
n = len(joint_data[0])
s = 5 # Segment length
alpha = 0.1
for i in range(0, n - s, s):
alpha = min(alpha + (s / n), 1.0)
# print(alpha)
ax.plot(range(i, i + s + 1),
of1 + joint_data[0][i:i + s + 1],
"-",
linewidth=size,
c=color_map[joint_index],
alpha=alpha)
ax.plot(range(i, i + s + 1),
of2 + joint_data[1][i:i + s + 1],
"-",
linewidth=size,
c=color_map[25 + joint_index],
alpha=alpha)
ax.plot(range(i, i + s + 1),
of3 + joint_data[2][i:i + s + 1],
"-",
linewidth=size,
c=color_map[50 + joint_index],
alpha=alpha)
else:
ax.plot(range(len(joint_data[0])),
of1 + joint_data[0],
"-",
linewidth=size,
c=color_map[joint_index],
alpha=alpha)
ax.plot(range(len(joint_data[1])),
of2 + joint_data[1],
"-",
linewidth=size,
c=color_map[25 + joint_index],
alpha=alpha)
ax.plot(range(len(joint_data[2])),
of3 + joint_data[2],
"-",
linewidth=size,
c=color_map[50 + joint_index],
alpha=alpha)
# ax.view_init(0, 0)
return fig
def gaf(audio):
gasf = GramianAngularField(image_size=100, method='summation')
X_gasf = gasf.fit_transform(audio.reshape((-1, len(audio))))
_, x, y = X_gasf.shape
return np.float32(X_gasf.reshape((x, y)))
def GAF_data_2(df, step):
col = ["Open", "High", "Close", "Low"]
gasf = GramianAngularField(image_size=step, method="summation")
gadf = GramianAngularField(image_size=step, method="difference")
mtf = MarkovTransitionField(image_size=step)
X_mtf = []
X_gasf = []
X_gadf = []
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
ope = np.round(mtf.fit_transform([ts_1])[0] * 255)
high = np.round(mtf.fit_transform([ts_2])[0] * 255)
close = np.round(mtf.fit_transform([ts_3])[0] * 255)
low = np.round(mtf.fit_transform([ts_4])[0] * 255)
mtf_oh = np.hstack((ope, high))
mtf_cl = np.hstack((close, low))
mtf_ohcl = np.vstack((mtf_oh, mtf_cl))
X_mtf.append(mtf_ohcl.reshape(step * 2, step * 2, 1))
X_mtf = np.stack(X_mtf)
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
gadf_oh = np.hstack((np.round(gadf.fit_transform([ts_1])[0] * 255),
np.round(gadf.fit_transform([ts_2])[0] * 255)))
gadf_cl = np.hstack((np.round(gadf.fit_transform([ts_3])[0] * 255),
np.round(gadf.fit_transform([ts_4])[0] * 255)))
gadf_ohcl = np.vstack((gadf_oh, gadf_cl))
X_gadf.append(gadf_ohcl.reshape(step * 2, step * 2, 1))
X_gadf = np.stack(X_gadf)
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
gasf_oh = np.hstack((np.round(gasf.fit_transform([ts_1])[0] * 255),
np.round(gasf.fit_transform([ts_2])[0] * 255)))
gasf_cl = np.hstack((np.round(gasf.fit_transform([ts_3])[0] * 255),
np.round(gasf.fit_transform([ts_4])[0] * 255)))
gasf_ohcl = np.vstack((gasf_oh, gasf_cl))
X_gasf.append(gasf_ohcl.reshape(step * 2, step * 2, 1))
X_gasf = np.stack(X_gasf)
return (X_gasf, X_gadf, X_mtf)
def plot_skeleton_data(skeleton_data,
size=5,
offset=False,
approach_id=1,
alpha=0.2,
attention=False,
filter_percentage=0.2):
fig = plt.figure(figsize=(5, 5))
# fig.title(filename)
ax = fig.add_subplot(frameon=False)
# ax = fig.add_axes([0, 0, 1, 1])
ax.axis("off")
fig.tight_layout(pad=0)
# To remove the huge white borders
ax.margins(0)
if approach_id == 3:
skeleton_data = np.sort(skeleton_data, axis=0)
if approach_id == 8:
skeleton_data = filter_skeleton(skeleton_data)
if approach_id == 9:
print("before filter, ", skeleton_data.shape)
skeleton_data = filter_attention(skeleton_data)
print("after filter, ", skeleton_data.shape)
if attention:
print("before filter, ", skeleton_data.shape)
skeleton_data = filter_attention(skeleton_data, filter_percentage)
print("after filter, ", skeleton_data.shape)
for joint_index, joint_data in enumerate(skeleton_data):
# line currently the best results
color_offset = len(joint_data)
if approach_id == 1:
of1 = 0
of2 = 2
of3 = 3
if approach_id == 2:
of1 = joint_index / 2 + offset
of2 = joint_index / 2 + offset
of3 = joint_index / 2 + offset
if approach_id == 3:
of1 = joint_index / 2 + offset
of2 = joint_index / 2 + offset
of3 = joint_index / 2 + offset
if joint_index > len(skeleton_data) / 6:
continue
if approach_id == 4:
of1 = joint_index / 2
of2 = joint_index / 2
of3 = joint_index / 2
if joint_data.var() > 1:
continue
if approach_id == 5:
import pyts
from pyts.image import GramianAngularField
transformer = GramianAngularField(image_size=len(joint_data))
image_data = transformer.fit_transform(joint_data)
plt.figure(figsize=(10, 10))
plt.imshow(image_data)
plt.savefig("gaf.png")
continue
if approach_id == 6:
of1 = -np.average(joint_data[0])
of2 = -np.average(joint_data[1])
of3 = -np.average(joint_data[2])
if approach_id == 7:
of1 = -np.max(joint_data[0])
of2 = -np.max(joint_data[1])
of3 = -np.max(joint_data[2])
if approach_id == 8:
of1 = -np.max(joint_data[0])
of2 = -np.max(joint_data[1])
of3 = -np.max(joint_data[2])
if approach_id == 9:
of1 = -np.max(joint_data[0])
of2 = -np.max(joint_data[1])
of3 = -np.max(joint_data[2])
if approach_id == 10:
of1 = -np.average(joint_data[0])
of2 = -np.average(joint_data[1]) + 0.5
of3 = -np.average(joint_data[2]) + 1
if approach_id == 11: # like 10 but use alpha for temporal
of1 = -np.average(joint_data[0])
of2 = -np.average(joint_data[1]) + 0.5
of3 = -np.average(joint_data[2]) + 1
n = len(joint_data[0])
s = 5 # Segment length
alpha = 0.1
for i in range(0, n - s, s):
alpha = min(alpha + (s / n), 1.0)
ax.plot(range(i, i + s + 1),
of1 + joint_data[0][i:i + s + 1],
"-",
linewidth=size,
c=color_map[joint_index],
alpha=alpha)
ax.plot(range(i, i + s + 1),
of2 + joint_data[1][i:i + s + 1],
"-",
linewidth=size,
c=color_map[color_offset + joint_index],
alpha=alpha)
ax.plot(range(i, i + s + 1),
of3 + joint_data[2][i:i + s + 1],
"-",
linewidth=size,
c=color_map[2 * color_offset + joint_index],
alpha=alpha)
else:
try:
ax.plot(range(len(joint_data[0])),
of1 + joint_data[0],
"-",
linewidth=size,
c=color_map[joint_index],
alpha=alpha)
ax.plot(range(len(joint_data[1])),
of2 + joint_data[1],
"-",
linewidth=size,
c=color_map[color_offset + joint_index],
alpha=alpha)
ax.plot(range(len(joint_data[2])),
of3 + joint_data[2],
"-",
linewidth=size,
c=color_map[2 * color_offset + joint_index],
alpha=alpha)
except Exception as e:
print(e)
return fig