def create_image(data,
date=None,
image_size=10,
method='summation',
field='gaf',
strategy='uniform'):
"""Creates an image of the specified size and using the specified field
Arguments:
data {list} -- [the list of data to transform]
Keyword Arguments:
image_size {int} -- the size of the output image (default: {30})
method {str} -- the method for the gramian angular field. Def. is summation (default: {'s'})
field {sr} -- The specified field to use. GAF or MTF (default: {'g'})
strategy {str} -- the strategy for MTF
Returns dataframe if successful or None if not
"""
if field is 'gaf':
field = GramianAngularField(image_size=image_size, method=method)
elif field is 'mtf':
field = MarkovTransitionField(image_size=image_size, strategy=strategy)
else:
return None
data = pd.DataFrame(data).transpose()
img = pd.DataFrame(field.transform(data).flatten()).transpose()
img['Date'] = date
img = img.set_index('Date')
return img
def generateGaf(self):
values = self.to_numpy()
# Function Specific parameters
index_val = self.df.index.max()
outfile = self.output_dir + str(index_val) + '.jpeg'
if self.fieldType == 'gasf':
gaf = GramianAngularField(image_size=self.window_size, method='summation')
elif self.fieldType == 'gadf':
gaf = GramianAngularField(image_size=self.window_size, method='difference')
gaf = gaf.fit_transform(X=values)
# Generate Image
fig = plt.figure(figsize=(8, 8))
grid = ImageGrid(fig, 111,
nrows_ncols=(2, 2),
axes_pad=0,
share_all=True
)
images = [gaf[0], gaf[1], gaf[2], gaf[3]]
for image, ax in zip(images, grid):
ax.imshow(image, cmap='rainbow', origin='lower')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
ax.cax.toggle_label(False)
plt.axis('off')
if self.debug_level == 2:
plt.show()
if self.debug_level == 1:
print(outfile)
plt.savefig(outfile, pil_kwargs={'optimize': True, 'quality': self.quality})
plt.close('all')
def create_gaf(ts) -> Dict[str, Any]:
"""
:param ts:
:return:
"""
data = dict()
gadf = GramianAngularField(method='difference', image_size=ts.shape[0])
data['gadf'] = gadf.fit_transform(pd.DataFrame(ts).T)[0]
return data
def apply_model(self, df, time_idx):
data = np.array(df.iloc[time_idx - (self.window - 1):time_idx + 1]).T
transformer = GramianAngularField()
image = np.expand_dims(transformer.transform(data), axis=0)
pred = F.softmax(self.model(torch.tensor(image)), dim=1).detach().numpy()
signal = np.argmax(pred)
return signal
def _build_images_one_stock(df_one_permno, window_len, retrain_freq,
encoding_method, image_size):
"""
Encodes images as timeseries for one stock
:param df_one_permno: dataframe of the timeseries of all data for one particular stock
:param window_len: number of observations to consider (42 for 2 months)
:param retrain_freq: lag to consider between making two samples
:param encoding_method: method to encode the images
:param image_size: final size of the image (using window_len*window_len will avoid any averaging)
:return: np.ndarray of the samples of shape (N,window_len,window_len,M) where:
- M is the number of features
- N is the number of final samples ~ len(df_one_permno)/retrain_freq
"""
n_days = df_one_permno.T.shape[-1]
samples_list, dates_list, prc_list = [], [], []
for i in range(window_len, n_days, retrain_freq):
window_data = df_one_permno.T.iloc[:, i - window_len:i]
# Use GADF algorithm to transform data
if encoding_method == 'GADF':
try:
from pyts.image import GADF
gadf = GADF(image_size)
except:
from pyts.image import GramianAngularField
gadf = GramianAngularField(image_size, method='difference')
samples_list.append(gadf.fit_transform(window_data).T)
# Use GASF algorithm to transform data
elif encoding_method == 'GASF':
try:
from pyts.image import GASF
gasf = GASF(image_size)
except:
from pyts.image import GramianAngularField
gasf = GramianAngularField(image_size, method='summation')
samples_list.append(gasf.fit_transform(window_data).T)
# Use MTF algorithm to transform data
elif encoding_method == 'MTF':
try:
from pyts.image import MTF
mtf = MTF(image_size)
except:
from pyts.image import MarkovTransitionField
mtf = MarkovTransitionField(image_size)
samples_list.append(mtf.fit_transform(window_data).T)
else:
raise BaseException(
'Method must be either GADF, GASF or MTF not {}'.format(
encoding_method))
samples_list = np.asarray(samples_list)
return samples_list
def apply_gaf(self, data_mat):
data_mat = np.transpose(data_mat, (0, 2, 1))
transformer = GramianAngularField()
n_samples = data_mat.shape[0]
images = np.zeros((n_samples, 5, self.train_window, self.train_window))
for i, item in enumerate(data_mat):
images[i, :, :, :] = transformer.transform(item)
return images
def _show_images(self, df_window_data):
"""
Plots a multi dimensional timeseries encoded as an image
:param df_window_data: timeseries we want to encode as an image
"""
data = df_window_data.reset_index().set_index('date').drop('PERMNO',
axis=1).T
channels = list(data.index)
if self._encoding_method == 'GADF':
try:
from pyts.image import GADF
gadf = GADF(self._image_size)
except:
from pyts.image import GramianAngularField
gadf = GramianAngularField(self._image_size,
method='difference')
image_data = (gadf.fit_transform(data).T)
elif self._encoding_method == 'GASF':
try:
from pyts.image import GASF
gasf = GASF(self._image_size)
except:
from pyts.image import GramianAngularField
gasf = GramianAngularField(self._image_size,
method='summation')
image_data = (gasf.fit_transform(data).T)
elif self._encoding_method == 'MTF':
try:
from pyts.image import MTF
mtf = MTF(self._image_size)
except:
from pyts.image import MarkovTransitionField
mtf = MarkovTransitionField(self._image_size)
image_data = (mtf.fit_transform(data).T)
else:
raise BaseException(
'Method must be either GADF, GASF or MTF not {}'.format(
self._encoding_method))
num_channels = image_data.shape[-1]
plt.figure(figsize=(12, 14))
for j in range(1, num_channels + 1):
channel = image_data[:, :, j - 1]
plt.subplot(int(num_channels / 2) + 1, 2, j)
plt.imshow(channel, cmap='rainbow', origin='lower')
plt.xlabel('$time$')
plt.ylabel('$time$')
plt.title(channels[j - 1])
plt.tight_layout()
plt.show()
def apply_gaf(self, data):
if os.path.exists('data/gaf_images.npy'):
return np.load('data/gaf_images.npy')
transformer = GramianAngularField()
n_samples = data.shape[0]
images = np.zeros((n_samples, 5, self.train_window, self.train_window))
for i, item in enumerate(data):
images[i, :, :, :] = transformer.transform(item)
np.save('data/gaf_images.npy', arr=images)
return images
def GASF_encoder(ts,
size=None,
sample_range=None,
overlapping=False,
**kwargs):
ts = To2dArray(ts)
assert ts.ndim == 2, 'ts ndim must be 2!'
if size is None: size = ts.shape[-1]
else: size = min(size, ts.shape[-1])
encoder = GAF(image_size=size,
sample_range=sample_range,
method='s',
overlapping=overlapping)
output = np.squeeze(encoder.fit_transform(ts), 0)
return (output + 1) / 2
def getOrderbookField(askPriceSamples, askSizeSamples, bidPriceSamples,
bidSizeSamples, size):
numSamples = len(askPriceSamples)
depth = len(askPriceSamples[0])
samples = []
for i in range(numSamples):
samples.append([])
for j in range(depth):
samples[i].append((askPriceSamples[i][j] * askSizeSamples[i][j] -
bidPriceSamples[i][j] * bidSizeSamples[i][j]) /
(askSizeSamples[i][j] + bidSizeSamples[i][j]))
G = GramianAngularField(image_size=size, method='summation')
S = np.transpose(np.array(samples))
T = G.fit_transform(S)
return T
def toGAFdata(tsdatas,
image_size=1.,
sample_range=(-1, 1),
method='summation',
overlapping=False,
flatten=False):
X = []
gaf = GramianAngularField(image_size=image_size,
sample_range=sample_range,
method=method,
overlapping=overlapping,
flatten=flatten)
for data in tsdatas:
data_gaf = gaf.fit_transform(data)
X.append(data_gaf[0])
return np.array(X)
def evaluate_classifiers(dst):
print("[%s] Processing dataset %s" % (datetime.now().strftime("%F %T"), dst))
train_x, train_y = load_from_tsfile_to_dataframe(os.path.join(UCR_DATASET_PATH, dst, dst + "_TRAIN.ts"))
test_x, test_y = load_from_tsfile_to_dataframe(os.path.join(UCR_DATASET_PATH, dst, dst + "_TEST.ts"))
data_train = [train_x.iloc[i][0] for i in range(train_x.shape[0])]
data_test = [test_x.iloc[i][0] for i in range(test_x.shape[0])]
enc = LabelEncoder().fit(train_y)
ohe = OneHotEncoder(sparse=False)
labels_encoded = enc.transform(train_y)
integer_encoded = labels_encoded.reshape(len(labels_encoded), 1)
labels_train = ohe.fit_transform(integer_encoded)
ts_plotters = [RecurrencePlot(threshold='point', percentage=20),
MarkovTransitionField(),
GramianAngularField()]
def evaluate_classifier(plot_obj):
try:
classifier = classif_class(input_dim, num_classes=len(set(train_y)),
batch_size=batch_size, series_plot_obj=plot_obj)
classifier.train(data_train, labels_train, n_epochs=n_epochs)
y_pred = [classifier.predict(series) for series in data_test]
y_pred = enc.inverse_transform(y_pred)
accuracy = accuracy_score(test_y, y_pred)
f1 = f1_score(test_y, y_pred, average='macro')
with open("tsplot_results.csv", "a") as f:
f.write("{};{};{};{};{}\n".format(classif_class.__class__.__name__, plot_obj.__class__.__name__, dst, accuracy, f1))
return accuracy, f1
except Exception as e:
print("Exception while evaluating classifier:", e.__str__())
return float('nan'), float('nan')
return list(itertools.chain(*[evaluate_classifier(plot_obj) for plot_obj in ts_plotters]))
def encode_dataset(batch_size,downscale_factor,dataset, pooling_function):
""" Computation of encodings has to be done in batches due to the large size of the dataset.
Otherwise the kernel will die!
For downscaling pick np.mean (average pooling) or np.max (max pooling) respectively.
If downscaling is not required choose downscale_factor=1.
Keep in mind the network expects an input image size of 64x64.
The function returns a 3D matrix.
The new 3D matrix contains several 2D matrices, which correspond to the time series encodings/images.
The order of the objects does not change, which means for example that the 23rd slice of the
input dataset corresponds to the 23rd encoding in the 3D Matrix."""
n,l=np.shape(dataset)
f=downscale_factor
n_batches=n//batch_size
batches=np.linspace(1,n_batches,n_batches, dtype=int) * batch_size
gaf = GramianAngularField(image_size=1., method='summation')
print('Encoding started...')
for p in range(n_batches):
if p==0:
X_gaf = gaf.transform(dataset[0:batches[p],:])
sample=block_reduce(X_gaf[0], block_size=(f, f), func=pooling_function)
l_red = sample.shape[0]
X_gaf_red = np.zeros((n,l_red,l_red))
print('output 3D Matrix shape: ', np.shape(X_gaf_red))
j=0
for i in range(0,batches[p]):
X_gaf_red[i] = block_reduce(X_gaf[j], block_size=(f, f) , func=pooling_function)
j+=1
else:
X_gaf = gaf.transform(X[batches[p-1]:batches[p],:])
j=0
for i in range(batches[p-1],batches[p]):
X_gaf_red[i] = block_reduce(X_gaf[j], block_size=(f, f) , func=pooling_function)
j+=1
print('Encoding successful!')
print('#####################################')
return X_gaf_red
def gadf_transform(data, image_size=500, show=False, img_index=0):
# GAF transformation
transform = GramianAngularField(image_size, method='difference')
return (pyts_transform(transform,
data,
image_size=image_size,
show=show,
cmap='rainbow',
img_index=img_index))
def preprocess_audio(opt, y, sr):
if opt == 'spect':
D = librosa.stft(y, n_fft=480, hop_length=160, win_length=480, window='hamming')
spect,phase = librosa.magphase(D)
graph_data = log_spect = np.log(spect)
elif opt == 'mp_spect':
S = librosa.feature.melspectrogram(y, sr=sr, n_mels=128)
graph_data = log_S = librosa.power_to_db(S, ref=np.max)
elif opt == 'cqt':
graph_data = C = librosa.cqt(y,sr)
elif opt == 'chrom':
C = np.abs(librosa.cqt(y,sr))
graph_data = chroma = librosa.feature.chroma_cqt(C=C,sr=sr)
elif opt == 'mfcc':
raw_mfcc = librosa.feature.mfcc(y=y,sr=sr)
graph_data = scaled_mfcc = scale(raw_mfcc, axis=1)
elif opt == 'gasf':
gasf = GramianAngularField(image_size=256, method='summation')
graph_data = np.squeeze(gasf.fit_transform(y.reshape(1,-1)))
elif opt == 'gadf':
gadf = GramianAngularField(image_size=256, method='difference')
graph_data = np.squeeze(gadf.fit_transform(y.reshape(1,-1)))
else:
raise NotImplementedError('Graph data not known for {}'.format(opt))
return graph_data
def save_to_GAF_img(df, file, step):
OHLC = ["Open", "High", "Low", "Close"]
high = max(df["High"])
low = min(df["Low"])
for col in OHLC:
Path("/content/GASF/" + col + "/").mkdir(parents=True, exist_ok=True)
Path("/content/GADF/" + col + "/").mkdir(parents=True, exist_ok=True)
Path("/content/MTF/" + col + "/").mkdir(parents=True, exist_ok=True)
gasf = GramianAngularField(image_size=step, method="summation")
gadf = GramianAngularField(image_size=step, method="difference")
mtf = MarkovTransitionField(image_size=step)
ts_norm = [(i - low) / (high - low) for i in list(df[col])]
X_mtf = mtf.fit_transform([ts_norm])
X_gasf = gasf.fit_transform([ts_norm])
X_gadf = gadf.fit_transform([ts_norm])
plt.imsave("/content/other_n/GASF/" + col + "/" + file,
X_gasf[0],
cmap="gray")
plt.imsave("/content/other_n/GADF/" + col + "/" + file,
X_gadf[0],
cmap="gray")
plt.imsave("/content/other_n/MTF/" + col + "/" + file,
X_mtf[0],
cmap="gray")
def save_gaf(periodData, simpleFileName):
# Clear any previous graphs
plt.clf()
# Need to reshape to numpy for plotting in gasf
periodnp = periodData.to_numpy().reshape(1, -1)
# Create GAF from data
gaf = GramianAngularField(image_size=len(periodData), method='summation')
gafData = gaf.transform(periodnp)
# Plot the gaf to an image
plt.imshow(gafData[0],
cmap='rainbow',
origin='lower',
interpolation='nearest')
plt.axis('off')
# Now we save this gaf as a file
dest = "gafs/" + simpleFileName
plt.savefig(dest, bbox_inches='tight', pad_inches=0, transparent=True)
plt.close()
def gen_GAF_exec(data: list,
sample_range: None or tuple = (-1, 1),
method: str = 'summation',
null_value: str = '0'):
"""
**Generate a Gramian angular Field**
this is the actual function when it comes to generating a Gramian Angular Field
out of the data of a numpy array. This function takes different variables to determine
how the Field should be scaled, what its size should be
and if it is either a summation or difference Field
param data: as the content of a npy file , type list
param size: this is the size of the square output image, type int or float
param sample_range: as the range the data should be scaled to, type None or tuple
param method: as the type of field it should be, type 'summation' or 'difference'
param **null_value**: as the number to use instead of NULL, type str
"""
gaf = GAF(sample_range=sample_range, method=method)
data = np.where(data == 'NULL', null_value, data)
data = data[:, 3:].astype(dtype=float)
data_gaf = gaf.fit_transform(data)
plt.imshow(data_gaf[0], cmap='rainbow', origin='lower')
plt.show()
def prep_seriesConvLSTM(seq_len, out_window, in_window, img_size, channels,
series_test, h):
print("Preparing data: ")
sample_range = (-1, 1)
signal_test = series_test
signal_test = signal_test.reshape(-1, 1)
signal_test_scaled = signal_test.flatten()
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, in_window, out_window, h)
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_test = np.expand_dims(gadf.fit_transform(window_input_test), axis=3)
gasf_test = np.expand_dims(gasf.fit_transform(window_input_test), axis=3)
mtf_test = np.expand_dims(mtf.fit_transform(window_input_test), axis=3)
y_test = window_output_test.reshape(-1)
if (channels == 2):
X_test_windowed = np.concatenate((gadf_test, gasf_test), axis=3)
else:
X_test_windowed = np.concatenate((gadf_test, gasf_test, mtf_test),
axis=3)
X_test_Conv_LSTM = np.zeros((X_test_windowed.shape[0] - seq_len + 1,
seq_len, img_size, img_size, channels))
y_test_Conv_LSTM = np.zeros(
(X_test_windowed.shape[0] - seq_len + 1, out_window))
print("Test data:")
for i in tqdm(range(0, X_test_windowed.shape[0] - seq_len + 1)):
current_seq_X = np.zeros((seq_len, img_size, img_size, channels))
for l in range(seq_len):
current_seq_X[l] = X_test_windowed[i + l]
current_seq_X = current_seq_X.reshape(1, seq_len, img_size, img_size,
channels)
X_test_Conv_LSTM[i] = current_seq_X
y_test_Conv_LSTM[i] = y_test[i + seq_len - 1]
X_test_Conv_LSTM = X_test_Conv_LSTM.reshape(-1, seq_len, img_size,
img_size, channels)
y_test_Conv_LSTM = y_test_Conv_LSTM.reshape(-1, out_window)
return (X_test_Conv_LSTM, y_test_Conv_LSTM)
def __init__(self,
source_columns,
image_size=1.,
sample_range=(-1, 1),
method='summation',
overlapping=False,
flatten=False):
super().__init__()
self.labels_source = source_columns if isinstance(
source_columns, List) else [source_columns]
self.encoded_labels = ["+".join(self.labels_source_columns)]
self.gaf = GramianAngularField(image_size=image_size,
sample_range=sample_range,
method=method,
overlapping=overlapping,
flatten=flatten)
def getMidpointFields(samples, size):
fields = []
g = GramianAngularField(image_size=size, method='summation')
S = np.array(samples).reshape(1, size)
T = g.fit_transform(S)
fields.append(T[0])
g = GramianAngularField(image_size=size, method='difference')
S = np.array(samples).reshape(1, size)
T = g.fit_transform(S)
fields.append(T[0])
return fields
def __init__(self,
data,
window=100,
next_steps=60,
skip=20,
instrument=None,
load=False,
save=True):
self.signal2idx = {v: k for k, v in idx2signal.items()}
self.samples = []
self.signals = []
self.gasf_dict = {
ins: GramianAngularField(image_size=window, method='summation')
for ins in instrument
}
for ins, c in zip(instrument, data):
prices = [p[1] for p in c] #close price
# x_min = data["min"]
# x_max = data["max"]
self.gasf_dict[ins].fit(np.array(prices))
len_data = len(prices)
# prices_normalized = normalize(prices).tolist()#, x_min=x_min, x_max=x_max)
for i in range(0, len_data - window - next_steps, skip):
self.samples.append((
#norm_by_latest_close_triplet(prices[i:i+window]),
self.gasf_dict[ins].transform(
np.array([prices[i:i + window]])),
self.gasf_dict[ins].transform(
np.array([prices[i + skip:i + window + skip]])),
*compute_trend([
p for p in prices[i + window:i + window + next_steps]
]) #midclose - midclose
# prices[i+skip:i+window+skip],
))
def GAF(sin_data, image_size):
sin_data = np.array(sin_data)
sin_data = sin_data.reshape(1, -1)
print(sin_data)
gasf = GramianAngularField(image_size=image_size,
method='summation',
overlapping='False')
sin_gasf = gasf.fit_transform(sin_data)
print(sin_gasf)
gadf = GramianAngularField(image_size=image_size, method='difference')
sin_gadf = gadf.fit_transform(sin_data)
images = [sin_gasf[0], sin_gadf[0]]
with open('GAF1.csv', 'w') as csvfile:
writer = csv.writer(csvfile)
writer.writerows(images[0])
return images
def prep_seriesConvMLP(window_size_x, window_size_y, img_size, signal_test, h):
signal_test = signal_test.reshape(-1, 1)
sample_range = (-1, 1)
signal_test_scaled = signal_test.flatten()
# Split Sequence
window_input_test, window_output_test = sequence_splitter(
signal_test_scaled, window_size_x, window_size_y, h)
# %%---------------------------------------------------------------------------
'''
Field transformations
'''
gadf = GramianAngularField(image_size=img_size,
method='difference',
sample_range=sample_range)
gasf = GramianAngularField(image_size=img_size,
method='summation',
sample_range=sample_range)
mtf = MarkovTransitionField(image_size=img_size,
n_bins=8,
strategy='quantile')
gadf_transformed_test = np.expand_dims(
gadf.fit_transform(window_input_test), axis=3)
gasf_transformed_test = np.expand_dims(
gasf.fit_transform(window_input_test), axis=3)
mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test),
axis=3)
X_test_windowed = np.concatenate(
(gadf_transformed_test, gasf_transformed_test, mtf_transformed_test),
axis=3)
# Data reshaping
X_test_Conv_MLP = X_test_windowed
y_test_Conv_MLP = window_output_test
return (X_test_Conv_MLP, y_test_Conv_MLP)
def GAF_data_2(df, step):
col = ["Open", "High", "Close", "Low"]
gasf = GramianAngularField(image_size=step, method="summation")
gadf = GramianAngularField(image_size=step, method="difference")
mtf = MarkovTransitionField(image_size=step)
X_mtf = []
X_gasf = []
X_gadf = []
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
ope = np.round(mtf.fit_transform([ts_1])[0] * 255)
high = np.round(mtf.fit_transform([ts_2])[0] * 255)
close = np.round(mtf.fit_transform([ts_3])[0] * 255)
low = np.round(mtf.fit_transform([ts_4])[0] * 255)
mtf_oh = np.hstack((ope, high))
mtf_cl = np.hstack((close, low))
mtf_ohcl = np.vstack((mtf_oh, mtf_cl))
X_mtf.append(mtf_ohcl.reshape(step * 2, step * 2, 1))
X_mtf = np.stack(X_mtf)
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
gadf_oh = np.hstack((np.round(gadf.fit_transform([ts_1])[0] * 255),
np.round(gadf.fit_transform([ts_2])[0] * 255)))
gadf_cl = np.hstack((np.round(gadf.fit_transform([ts_3])[0] * 255),
np.round(gadf.fit_transform([ts_4])[0] * 255)))
gadf_ohcl = np.vstack((gadf_oh, gadf_cl))
X_gadf.append(gadf_ohcl.reshape(step * 2, step * 2, 1))
X_gadf = np.stack(X_gadf)
for i in range((step - 1), len(df[col[0]])):
high = max(df["High"][i - (step - 1):i + 1])
low = min(df["Low"][i - (step - 1):i + 1])
ts_1 = [(x - low) / (high - low)
for x in list(df[col[0]][i - (step - 1):i + 1])]
ts_2 = [(x - low) / (high - low)
for x in list(df[col[1]][i - (step - 1):i + 1])]
ts_3 = [(x - low) / (high - low)
for x in list(df[col[2]][i - (step - 1):i + 1])]
ts_4 = [(x - low) / (high - low)
for x in list(df[col[3]][i - (step - 1):i + 1])]
gasf_oh = np.hstack((np.round(gasf.fit_transform([ts_1])[0] * 255),
np.round(gasf.fit_transform([ts_2])[0] * 255)))
gasf_cl = np.hstack((np.round(gasf.fit_transform([ts_3])[0] * 255),
np.round(gasf.fit_transform([ts_4])[0] * 255)))
gasf_ohcl = np.vstack((gasf_oh, gasf_cl))
X_gasf.append(gasf_ohcl.reshape(step * 2, step * 2, 1))
X_gasf = np.stack(X_gasf)
return (X_gasf, X_gadf, X_mtf)
"""
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid
from pyts.image import GramianAngularField
# Parameters
n_samples, n_timestamps = 100, 144
# Toy dataset
rng = np.random.RandomState(41)
X = rng.randn(n_samples, n_timestamps)
# Transform the time series into Gramian Angular Fields
gasf = GramianAngularField(image_size=24, method='summation')
X_gasf = gasf.fit_transform(X)
gadf = GramianAngularField(image_size=24, method='difference')
X_gadf = gadf.fit_transform(X)
# Show the images for the first time series
fig = plt.figure(figsize=(12, 7))
grid = ImageGrid(fig, 111,
nrows_ncols=(1, 2),
axes_pad=0.15,
share_all=True,
cbar_location="right",
cbar_mode="single",
cbar_size="7%",
cbar_pad=0.3,
)
def generate_data(
rates,
r=0.7,
test=True,
save_img=True,
save_gasf=True,
gasf_imsize=16,
save_npy=True,
tp=0.00500,
sl=0.00250,
sl_h=0.00250,
window_range_back=72,
window_range_front=30,
keys=[],
dpi=60,
rcparams={
'axes.spines.bottom': False,
'axes.spines.left': False,
'axes.spines.right': False,
'axes.spines.top': False,
}):
gasf = GramianAngularField(image_size=gasf_imsize, method='summation')
X_buy = list() #N OHLC candles window
X_buy_chart = list()
X_buy_gasf = list()
Y_reg_buy = list() #Next M values
X_sell = list()
X_sell_chart = list()
X_sell_gasf = list()
Y_reg_sell = list()
X_hold = list()
X_hold_chart = list()
X_hold_gasf = list()
Y_reg_hold = list()
save_img = save_img
save_gasf = save_gasf
TP = tp
SL_hold = sl_h
SL = sl
if test == True:
start = window_range_back + int(r * len(rates))
end = len(rates)
folder = 'data/chart/test'
else:
start = window_range_back
end = int(len(rates) * r)
folder = 'data/chart/train'
#Clean old data (Just in case acquisition window is changed or something, else you end up with unwanted data.)
if save_img == True:
for path in paths:
files = glob.glob(path + '/*.jpg')
for f in files:
os.remove(f)
i = start
while (i < end): #hold ops
window_back = rates[i - window_range_back:i]
window_front = rates[i:i + window_range_front]
if len(window_front) < window_range_front:
i = end
continue
lastclose = window_back.Close.iloc[-1]
hold_up = window_front[window_front.High > lastclose +
(SL_hold + 0.00100)]
hold_down = window_front[window_front.Low < lastclose -
(SL_hold + 0.00100)]
if len(hold_up) == 0 and len(
hold_down
) == 0: #Hold (strong consolidation or possible swing back)
X_hold.append(np.array(window_back.values))
Y_reg_hold.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_hold_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_hold_gasf.append(X_gasf)
else:
i += 1
i = start
while (i < end): #buy ops
window_back = rates[i - window_range_back:i]
window_front = rates[i:i + window_range_front]
if len(window_front) < window_range_front:
i = end
continue
lastclose = window_back.Close.iloc[-1]
TP_hit = window_front[window_front.High >= lastclose + TP]
SL_hit = window_front[window_front.Low <= lastclose - SL]
hold_up = window_front[window_front.High > lastclose +
(SL_hold + 0.00100)]
hold_down = window_front[window_front.Low <= lastclose -
(SL_hold + 0.00100)]
if (len(TP_hit) == 0 and len(SL_hit) == 0) and (
len(hold_up) == 0 and len(hold_down)
== 0): #Hold (strong consolidation or possible swing back)
i += window_range_front
continue
elif len(TP_hit) > 0 and len(
SL_hit) == 0: #buy (no SL hit in period but TP is hit)
X_buy.append(np.array(window_back.values))
Y_reg_buy.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_buy_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_buy_gasf.append(X_gasf)
elif len(TP_hit) > 0 and len(
SL_hit
) > 0: #buy (both tp and sl hit but SL is hit after TP so trade won)
TP_hit = TP_hit.iloc[0]
SL_hit = SL_hit.iloc[0]
if TP_hit.timestamp < SL_hit.timestamp:
X_buy.append(np.array(window_back.values))
Y_reg_buy.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_buy_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_buy_gasf.append(X_gasf)
else:
i += 1
else:
i += 1
#look for sells and holds
i = start
while (i < end): #sell_ops
window_back = rates[i - window_range_back:i]
window_front = rates[i:i + window_range_front]
if len(window_front) < window_range_front:
i = end
continue
lastclose = window_back.Close.iloc[-1]
SL_hit = window_front[window_front.High >= lastclose + SL]
TP_hit = window_front[window_front.Low <= lastclose - TP]
hold_up = window_front[window_front.High > lastclose +
(SL_hold + 0.00100)]
hold_down = window_front[window_front.Low <= lastclose -
(SL_hold + 0.00100)]
if (len(TP_hit) == 0
and len(SL_hit) == 0) and (len(hold_up) == 0 and len(hold_down)
== 0): #Hold (strong consolidation)
i += window_range_front
continue
elif len(TP_hit) > 0 and len(
SL_hit) == 0: #buy (no SL hit in period but TP is hit)
X_sell.append(np.array(window_back.values))
Y_reg_sell.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_sell_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_sell_gasf.append(X_gasf)
elif len(TP_hit) > 0 and len(
SL_hit
) > 0: #buy (both tp and sl hit but SL is hit after TP so trade won)
TP_hit = TP_hit.iloc[0]
SL_hit = SL_hit.iloc[0]
if TP_hit.timestamp < SL_hit.timestamp:
X_sell.append(np.array(window_back.values))
Y_reg_sell.append(np.array(window_front.values))
i += window_range_front
if save_img == True:
img = return_img(window_back, dpi=dpi, rcparams=rcparams)
X_sell_chart.append(img)
if save_gasf == True:
X_gasf = gasf.fit_transform(
window_back.values.transpose([1, 0]))
X_sell_gasf.append(X_gasf)
else:
i += 1
else:
i += 1
X_buy = np.array(X_buy) #72 OHLC candles window
X_buy_chart = np.array(X_buy_chart)
X_buy_gasf = np.array(X_buy_gasf)
Y_reg_buy = np.array(Y_reg_buy) #Next 72 values
X_sell = np.array(X_sell)
X_sell_chart = np.array(X_sell_chart)
X_sell_gasf = np.array(X_sell_gasf)
Y_reg_sell = np.array(Y_reg_sell)
X_hold = np.array(X_hold)
X_hold_chart = np.array(X_hold_chart)
X_hold_gasf = np.array(X_hold_gasf)
Y_reg_hold = np.array(Y_reg_hold)
print(X_buy.shape)
print(Y_reg_buy.shape)
print(X_buy_gasf.shape)
print(X_buy_chart.shape)
print(X_sell.shape)
print(Y_reg_sell.shape)
print(X_sell_gasf.shape)
print(X_sell_chart.shape)
print(X_hold.shape)
print(Y_reg_hold.shape)
print(X_hold_gasf.shape)
print(X_hold_chart.shape)
if save_npy == True:
print('Saving npy candle data [to be added]')
return X_buy, X_buy_chart, X_buy_gasf, Y_reg_buy, X_sell, X_sell_chart, X_sell_gasf, Y_reg_sell, X_hold, X_hold_chart, X_sell_gasf, Y_reg_hold
check = 0
serial_port.write("STOP".encode())
print(app.bcolors.FAIL + '#' + app.bcolors.ENDC)
sys.stdout.flush()
current_state = state_values[j]
X[0][:] = []
X[1][:] = []
X[0].extend(np.interp(data_amplitude[0], (0, 4096), (0, 1)))
X[1].extend(np.interp(data_deltatime[0], (0, 100), (0, 1)))
csv_write_row(np.interp(data_amplitude[0], (0, 4096), (0, 1)))
gasf = GramianAngularField(image_size=sample_count, method='summation')
X_gasf = gasf.fit_transform(X)
plt.imsave("img/sample_" + str(i) + "_state_" + states[j] + ".jpg",
X_gasf[0],
format='jpg')
file.write("img/sample_" + str(i) + "_state_" + states[j] + ".jpg," +
str(current_state) + "\n")
del gasf
del X_gasf
data_amplitude[:] = []
data_deltatime[:] = []
data_electrodes_num = 0
print("")
file.close()
def to_gaf(df):
gaf = GramianAngularField(image_size=image_size, sample_range=sample_range, method=method,
overlapping=overlapping, flatten=flatten)
return gaf.fit_transform(df.values)
keys = create_zigzag(rates, pct=0.3)
addp3 = mpf.make_addplot(rates[keys])
mpf.plot(rates,
type='candle',
volume=True,
addplot=[addp3],
style='yahoo',
show_nontrading=False,
block=False)
price = rates.Close
err_allowed = 0.1 / 100
# Find peaks
for i in range(100, len(price)):
current_idx, current_pat, start, end = peak_detect(price.values[:i])
plt.plot(np.arange(start, i), price.values[start:i])
plt.scatter(current_idx, current_pat, c='r')
plt.show()
gasf = GramianAngularField(image_size=16, method='summation')
X_gasf = gasf.fit_transform(X_buy[0].transpose([1, 0]))
print(X_gasf[3].shape)
plt.imshow(X_gasf[3, :])
plt.show()
while (True):
msg = input('Close? [N],y\n')
if msg == 'y':
break
