def create_image(data, date=None, image_size=10, method='summation', field='gaf', strategy='uniform'): """Creates an image of the specified size and using the specified field Arguments: data {list} -- [the list of data to transform] Keyword Arguments: image_size {int} -- the size of the output image (default: {30}) method {str} -- the method for the gramian angular field. Def. is summation (default: {'s'}) field {sr} -- The specified field to use. GAF or MTF (default: {'g'}) strategy {str} -- the strategy for MTF Returns dataframe if successful or None if not """ if field is 'gaf': field = GramianAngularField(image_size=image_size, method=method) elif field is 'mtf': field = MarkovTransitionField(image_size=image_size, strategy=strategy) else: return None data = pd.DataFrame(data).transpose() img = pd.DataFrame(field.transform(data).flatten()).transpose() img['Date'] = date img = img.set_index('Date') return img
def generateGaf(self): values = self.to_numpy() # Function Specific parameters index_val = self.df.index.max() outfile = self.output_dir + str(index_val) + '.jpeg' if self.fieldType == 'gasf': gaf = GramianAngularField(image_size=self.window_size, method='summation') elif self.fieldType == 'gadf': gaf = GramianAngularField(image_size=self.window_size, method='difference') gaf = gaf.fit_transform(X=values) # Generate Image fig = plt.figure(figsize=(8, 8)) grid = ImageGrid(fig, 111, nrows_ncols=(2, 2), axes_pad=0, share_all=True ) images = [gaf[0], gaf[1], gaf[2], gaf[3]] for image, ax in zip(images, grid): ax.imshow(image, cmap='rainbow', origin='lower') ax.xaxis.set_visible(False) ax.yaxis.set_visible(False) ax.cax.toggle_label(False) plt.axis('off') if self.debug_level == 2: plt.show() if self.debug_level == 1: print(outfile) plt.savefig(outfile, pil_kwargs={'optimize': True, 'quality': self.quality}) plt.close('all')
def create_gaf(ts) -> Dict[str, Any]: """ :param ts: :return: """ data = dict() gadf = GramianAngularField(method='difference', image_size=ts.shape[0]) data['gadf'] = gadf.fit_transform(pd.DataFrame(ts).T)[0] return data
def apply_model(self, df, time_idx): data = np.array(df.iloc[time_idx - (self.window - 1):time_idx + 1]).T transformer = GramianAngularField() image = np.expand_dims(transformer.transform(data), axis=0) pred = F.softmax(self.model(torch.tensor(image)), dim=1).detach().numpy() signal = np.argmax(pred) return signal
def _build_images_one_stock(df_one_permno, window_len, retrain_freq, encoding_method, image_size): """ Encodes images as timeseries for one stock :param df_one_permno: dataframe of the timeseries of all data for one particular stock :param window_len: number of observations to consider (42 for 2 months) :param retrain_freq: lag to consider between making two samples :param encoding_method: method to encode the images :param image_size: final size of the image (using window_len*window_len will avoid any averaging) :return: np.ndarray of the samples of shape (N,window_len,window_len,M) where: - M is the number of features - N is the number of final samples ~ len(df_one_permno)/retrain_freq """ n_days = df_one_permno.T.shape[-1] samples_list, dates_list, prc_list = [], [], [] for i in range(window_len, n_days, retrain_freq): window_data = df_one_permno.T.iloc[:, i - window_len:i] # Use GADF algorithm to transform data if encoding_method == 'GADF': try: from pyts.image import GADF gadf = GADF(image_size) except: from pyts.image import GramianAngularField gadf = GramianAngularField(image_size, method='difference') samples_list.append(gadf.fit_transform(window_data).T) # Use GASF algorithm to transform data elif encoding_method == 'GASF': try: from pyts.image import GASF gasf = GASF(image_size) except: from pyts.image import GramianAngularField gasf = GramianAngularField(image_size, method='summation') samples_list.append(gasf.fit_transform(window_data).T) # Use MTF algorithm to transform data elif encoding_method == 'MTF': try: from pyts.image import MTF mtf = MTF(image_size) except: from pyts.image import MarkovTransitionField mtf = MarkovTransitionField(image_size) samples_list.append(mtf.fit_transform(window_data).T) else: raise BaseException( 'Method must be either GADF, GASF or MTF not {}'.format( encoding_method)) samples_list = np.asarray(samples_list) return samples_list
def apply_gaf(self, data_mat): data_mat = np.transpose(data_mat, (0, 2, 1)) transformer = GramianAngularField() n_samples = data_mat.shape[0] images = np.zeros((n_samples, 5, self.train_window, self.train_window)) for i, item in enumerate(data_mat): images[i, :, :, :] = transformer.transform(item) return images
def _show_images(self, df_window_data): """ Plots a multi dimensional timeseries encoded as an image :param df_window_data: timeseries we want to encode as an image """ data = df_window_data.reset_index().set_index('date').drop('PERMNO', axis=1).T channels = list(data.index) if self._encoding_method == 'GADF': try: from pyts.image import GADF gadf = GADF(self._image_size) except: from pyts.image import GramianAngularField gadf = GramianAngularField(self._image_size, method='difference') image_data = (gadf.fit_transform(data).T) elif self._encoding_method == 'GASF': try: from pyts.image import GASF gasf = GASF(self._image_size) except: from pyts.image import GramianAngularField gasf = GramianAngularField(self._image_size, method='summation') image_data = (gasf.fit_transform(data).T) elif self._encoding_method == 'MTF': try: from pyts.image import MTF mtf = MTF(self._image_size) except: from pyts.image import MarkovTransitionField mtf = MarkovTransitionField(self._image_size) image_data = (mtf.fit_transform(data).T) else: raise BaseException( 'Method must be either GADF, GASF or MTF not {}'.format( self._encoding_method)) num_channels = image_data.shape[-1] plt.figure(figsize=(12, 14)) for j in range(1, num_channels + 1): channel = image_data[:, :, j - 1] plt.subplot(int(num_channels / 2) + 1, 2, j) plt.imshow(channel, cmap='rainbow', origin='lower') plt.xlabel('$time$') plt.ylabel('$time$') plt.title(channels[j - 1]) plt.tight_layout() plt.show()
def apply_gaf(self, data): if os.path.exists('data/gaf_images.npy'): return np.load('data/gaf_images.npy') transformer = GramianAngularField() n_samples = data.shape[0] images = np.zeros((n_samples, 5, self.train_window, self.train_window)) for i, item in enumerate(data): images[i, :, :, :] = transformer.transform(item) np.save('data/gaf_images.npy', arr=images) return images
def GASF_encoder(ts, size=None, sample_range=None, overlapping=False, **kwargs): ts = To2dArray(ts) assert ts.ndim == 2, 'ts ndim must be 2!' if size is None: size = ts.shape[-1] else: size = min(size, ts.shape[-1]) encoder = GAF(image_size=size, sample_range=sample_range, method='s', overlapping=overlapping) output = np.squeeze(encoder.fit_transform(ts), 0) return (output + 1) / 2
def getOrderbookField(askPriceSamples, askSizeSamples, bidPriceSamples, bidSizeSamples, size): numSamples = len(askPriceSamples) depth = len(askPriceSamples[0]) samples = [] for i in range(numSamples): samples.append([]) for j in range(depth): samples[i].append((askPriceSamples[i][j] * askSizeSamples[i][j] - bidPriceSamples[i][j] * bidSizeSamples[i][j]) / (askSizeSamples[i][j] + bidSizeSamples[i][j])) G = GramianAngularField(image_size=size, method='summation') S = np.transpose(np.array(samples)) T = G.fit_transform(S) return T
def toGAFdata(tsdatas, image_size=1., sample_range=(-1, 1), method='summation', overlapping=False, flatten=False): X = [] gaf = GramianAngularField(image_size=image_size, sample_range=sample_range, method=method, overlapping=overlapping, flatten=flatten) for data in tsdatas: data_gaf = gaf.fit_transform(data) X.append(data_gaf[0]) return np.array(X)
def evaluate_classifiers(dst): print("[%s] Processing dataset %s" % (datetime.now().strftime("%F %T"), dst)) train_x, train_y = load_from_tsfile_to_dataframe(os.path.join(UCR_DATASET_PATH, dst, dst + "_TRAIN.ts")) test_x, test_y = load_from_tsfile_to_dataframe(os.path.join(UCR_DATASET_PATH, dst, dst + "_TEST.ts")) data_train = [train_x.iloc[i][0] for i in range(train_x.shape[0])] data_test = [test_x.iloc[i][0] for i in range(test_x.shape[0])] enc = LabelEncoder().fit(train_y) ohe = OneHotEncoder(sparse=False) labels_encoded = enc.transform(train_y) integer_encoded = labels_encoded.reshape(len(labels_encoded), 1) labels_train = ohe.fit_transform(integer_encoded) ts_plotters = [RecurrencePlot(threshold='point', percentage=20), MarkovTransitionField(), GramianAngularField()] def evaluate_classifier(plot_obj): try: classifier = classif_class(input_dim, num_classes=len(set(train_y)), batch_size=batch_size, series_plot_obj=plot_obj) classifier.train(data_train, labels_train, n_epochs=n_epochs) y_pred = [classifier.predict(series) for series in data_test] y_pred = enc.inverse_transform(y_pred) accuracy = accuracy_score(test_y, y_pred) f1 = f1_score(test_y, y_pred, average='macro') with open("tsplot_results.csv", "a") as f: f.write("{};{};{};{};{}\n".format(classif_class.__class__.__name__, plot_obj.__class__.__name__, dst, accuracy, f1)) return accuracy, f1 except Exception as e: print("Exception while evaluating classifier:", e.__str__()) return float('nan'), float('nan') return list(itertools.chain(*[evaluate_classifier(plot_obj) for plot_obj in ts_plotters]))
def encode_dataset(batch_size,downscale_factor,dataset, pooling_function): """ Computation of encodings has to be done in batches due to the large size of the dataset. Otherwise the kernel will die! For downscaling pick np.mean (average pooling) or np.max (max pooling) respectively. If downscaling is not required choose downscale_factor=1. Keep in mind the network expects an input image size of 64x64. The function returns a 3D matrix. The new 3D matrix contains several 2D matrices, which correspond to the time series encodings/images. The order of the objects does not change, which means for example that the 23rd slice of the input dataset corresponds to the 23rd encoding in the 3D Matrix.""" n,l=np.shape(dataset) f=downscale_factor n_batches=n//batch_size batches=np.linspace(1,n_batches,n_batches, dtype=int) * batch_size gaf = GramianAngularField(image_size=1., method='summation') print('Encoding started...') for p in range(n_batches): if p==0: X_gaf = gaf.transform(dataset[0:batches[p],:]) sample=block_reduce(X_gaf[0], block_size=(f, f), func=pooling_function) l_red = sample.shape[0] X_gaf_red = np.zeros((n,l_red,l_red)) print('output 3D Matrix shape: ', np.shape(X_gaf_red)) j=0 for i in range(0,batches[p]): X_gaf_red[i] = block_reduce(X_gaf[j], block_size=(f, f) , func=pooling_function) j+=1 else: X_gaf = gaf.transform(X[batches[p-1]:batches[p],:]) j=0 for i in range(batches[p-1],batches[p]): X_gaf_red[i] = block_reduce(X_gaf[j], block_size=(f, f) , func=pooling_function) j+=1 print('Encoding successful!') print('#####################################') return X_gaf_red
def gadf_transform(data, image_size=500, show=False, img_index=0): # GAF transformation transform = GramianAngularField(image_size, method='difference') return (pyts_transform(transform, data, image_size=image_size, show=show, cmap='rainbow', img_index=img_index))
def preprocess_audio(opt, y, sr): if opt == 'spect': D = librosa.stft(y, n_fft=480, hop_length=160, win_length=480, window='hamming') spect,phase = librosa.magphase(D) graph_data = log_spect = np.log(spect) elif opt == 'mp_spect': S = librosa.feature.melspectrogram(y, sr=sr, n_mels=128) graph_data = log_S = librosa.power_to_db(S, ref=np.max) elif opt == 'cqt': graph_data = C = librosa.cqt(y,sr) elif opt == 'chrom': C = np.abs(librosa.cqt(y,sr)) graph_data = chroma = librosa.feature.chroma_cqt(C=C,sr=sr) elif opt == 'mfcc': raw_mfcc = librosa.feature.mfcc(y=y,sr=sr) graph_data = scaled_mfcc = scale(raw_mfcc, axis=1) elif opt == 'gasf': gasf = GramianAngularField(image_size=256, method='summation') graph_data = np.squeeze(gasf.fit_transform(y.reshape(1,-1))) elif opt == 'gadf': gadf = GramianAngularField(image_size=256, method='difference') graph_data = np.squeeze(gadf.fit_transform(y.reshape(1,-1))) else: raise NotImplementedError('Graph data not known for {}'.format(opt)) return graph_data
def save_to_GAF_img(df, file, step): OHLC = ["Open", "High", "Low", "Close"] high = max(df["High"]) low = min(df["Low"]) for col in OHLC: Path("/content/GASF/" + col + "/").mkdir(parents=True, exist_ok=True) Path("/content/GADF/" + col + "/").mkdir(parents=True, exist_ok=True) Path("/content/MTF/" + col + "/").mkdir(parents=True, exist_ok=True) gasf = GramianAngularField(image_size=step, method="summation") gadf = GramianAngularField(image_size=step, method="difference") mtf = MarkovTransitionField(image_size=step) ts_norm = [(i - low) / (high - low) for i in list(df[col])] X_mtf = mtf.fit_transform([ts_norm]) X_gasf = gasf.fit_transform([ts_norm]) X_gadf = gadf.fit_transform([ts_norm]) plt.imsave("/content/other_n/GASF/" + col + "/" + file, X_gasf[0], cmap="gray") plt.imsave("/content/other_n/GADF/" + col + "/" + file, X_gadf[0], cmap="gray") plt.imsave("/content/other_n/MTF/" + col + "/" + file, X_mtf[0], cmap="gray")
def save_gaf(periodData, simpleFileName): # Clear any previous graphs plt.clf() # Need to reshape to numpy for plotting in gasf periodnp = periodData.to_numpy().reshape(1, -1) # Create GAF from data gaf = GramianAngularField(image_size=len(periodData), method='summation') gafData = gaf.transform(periodnp) # Plot the gaf to an image plt.imshow(gafData[0], cmap='rainbow', origin='lower', interpolation='nearest') plt.axis('off') # Now we save this gaf as a file dest = "gafs/" + simpleFileName plt.savefig(dest, bbox_inches='tight', pad_inches=0, transparent=True) plt.close()
def gen_GAF_exec(data: list, sample_range: None or tuple = (-1, 1), method: str = 'summation', null_value: str = '0'): """ **Generate a Gramian angular Field** this is the actual function when it comes to generating a Gramian Angular Field out of the data of a numpy array. This function takes different variables to determine how the Field should be scaled, what its size should be and if it is either a summation or difference Field param data: as the content of a npy file , type list param size: this is the size of the square output image, type int or float param sample_range: as the range the data should be scaled to, type None or tuple param method: as the type of field it should be, type 'summation' or 'difference' param **null_value**: as the number to use instead of NULL, type str """ gaf = GAF(sample_range=sample_range, method=method) data = np.where(data == 'NULL', null_value, data) data = data[:, 3:].astype(dtype=float) data_gaf = gaf.fit_transform(data) plt.imshow(data_gaf[0], cmap='rainbow', origin='lower') plt.show()
def prep_seriesConvLSTM(seq_len, out_window, in_window, img_size, channels, series_test, h): print("Preparing data: ") sample_range = (-1, 1) signal_test = series_test signal_test = signal_test.reshape(-1, 1) signal_test_scaled = signal_test.flatten() window_input_test, window_output_test = sequence_splitter( signal_test_scaled, in_window, out_window, h) gadf = GramianAngularField(image_size=img_size, method='difference', sample_range=sample_range) gasf = GramianAngularField(image_size=img_size, method='summation', sample_range=sample_range) mtf = MarkovTransitionField(image_size=img_size, n_bins=8, strategy='quantile') gadf_test = np.expand_dims(gadf.fit_transform(window_input_test), axis=3) gasf_test = np.expand_dims(gasf.fit_transform(window_input_test), axis=3) mtf_test = np.expand_dims(mtf.fit_transform(window_input_test), axis=3) y_test = window_output_test.reshape(-1) if (channels == 2): X_test_windowed = np.concatenate((gadf_test, gasf_test), axis=3) else: X_test_windowed = np.concatenate((gadf_test, gasf_test, mtf_test), axis=3) X_test_Conv_LSTM = np.zeros((X_test_windowed.shape[0] - seq_len + 1, seq_len, img_size, img_size, channels)) y_test_Conv_LSTM = np.zeros( (X_test_windowed.shape[0] - seq_len + 1, out_window)) print("Test data:") for i in tqdm(range(0, X_test_windowed.shape[0] - seq_len + 1)): current_seq_X = np.zeros((seq_len, img_size, img_size, channels)) for l in range(seq_len): current_seq_X[l] = X_test_windowed[i + l] current_seq_X = current_seq_X.reshape(1, seq_len, img_size, img_size, channels) X_test_Conv_LSTM[i] = current_seq_X y_test_Conv_LSTM[i] = y_test[i + seq_len - 1] X_test_Conv_LSTM = X_test_Conv_LSTM.reshape(-1, seq_len, img_size, img_size, channels) y_test_Conv_LSTM = y_test_Conv_LSTM.reshape(-1, out_window) return (X_test_Conv_LSTM, y_test_Conv_LSTM)
def __init__(self, source_columns, image_size=1., sample_range=(-1, 1), method='summation', overlapping=False, flatten=False): super().__init__() self.labels_source = source_columns if isinstance( source_columns, List) else [source_columns] self.encoded_labels = ["+".join(self.labels_source_columns)] self.gaf = GramianAngularField(image_size=image_size, sample_range=sample_range, method=method, overlapping=overlapping, flatten=flatten)
def getMidpointFields(samples, size): fields = [] g = GramianAngularField(image_size=size, method='summation') S = np.array(samples).reshape(1, size) T = g.fit_transform(S) fields.append(T[0]) g = GramianAngularField(image_size=size, method='difference') S = np.array(samples).reshape(1, size) T = g.fit_transform(S) fields.append(T[0]) return fields
def __init__(self, data, window=100, next_steps=60, skip=20, instrument=None, load=False, save=True): self.signal2idx = {v: k for k, v in idx2signal.items()} self.samples = [] self.signals = [] self.gasf_dict = { ins: GramianAngularField(image_size=window, method='summation') for ins in instrument } for ins, c in zip(instrument, data): prices = [p[1] for p in c] #close price # x_min = data["min"] # x_max = data["max"] self.gasf_dict[ins].fit(np.array(prices)) len_data = len(prices) # prices_normalized = normalize(prices).tolist()#, x_min=x_min, x_max=x_max) for i in range(0, len_data - window - next_steps, skip): self.samples.append(( #norm_by_latest_close_triplet(prices[i:i+window]), self.gasf_dict[ins].transform( np.array([prices[i:i + window]])), self.gasf_dict[ins].transform( np.array([prices[i + skip:i + window + skip]])), *compute_trend([ p for p in prices[i + window:i + window + next_steps] ]) #midclose - midclose # prices[i+skip:i+window+skip], ))
def GAF(sin_data, image_size): sin_data = np.array(sin_data) sin_data = sin_data.reshape(1, -1) print(sin_data) gasf = GramianAngularField(image_size=image_size, method='summation', overlapping='False') sin_gasf = gasf.fit_transform(sin_data) print(sin_gasf) gadf = GramianAngularField(image_size=image_size, method='difference') sin_gadf = gadf.fit_transform(sin_data) images = [sin_gasf[0], sin_gadf[0]] with open('GAF1.csv', 'w') as csvfile: writer = csv.writer(csvfile) writer.writerows(images[0]) return images
def prep_seriesConvMLP(window_size_x, window_size_y, img_size, signal_test, h): signal_test = signal_test.reshape(-1, 1) sample_range = (-1, 1) signal_test_scaled = signal_test.flatten() # Split Sequence window_input_test, window_output_test = sequence_splitter( signal_test_scaled, window_size_x, window_size_y, h) # %%--------------------------------------------------------------------------- ''' Field transformations ''' gadf = GramianAngularField(image_size=img_size, method='difference', sample_range=sample_range) gasf = GramianAngularField(image_size=img_size, method='summation', sample_range=sample_range) mtf = MarkovTransitionField(image_size=img_size, n_bins=8, strategy='quantile') gadf_transformed_test = np.expand_dims( gadf.fit_transform(window_input_test), axis=3) gasf_transformed_test = np.expand_dims( gasf.fit_transform(window_input_test), axis=3) mtf_transformed_test = np.expand_dims(mtf.fit_transform(window_input_test), axis=3) X_test_windowed = np.concatenate( (gadf_transformed_test, gasf_transformed_test, mtf_transformed_test), axis=3) # Data reshaping X_test_Conv_MLP = X_test_windowed y_test_Conv_MLP = window_output_test return (X_test_Conv_MLP, y_test_Conv_MLP)
def GAF_data_2(df, step): col = ["Open", "High", "Close", "Low"] gasf = GramianAngularField(image_size=step, method="summation") gadf = GramianAngularField(image_size=step, method="difference") mtf = MarkovTransitionField(image_size=step) X_mtf = [] X_gasf = [] X_gadf = [] for i in range((step - 1), len(df[col[0]])): high = max(df["High"][i - (step - 1):i + 1]) low = min(df["Low"][i - (step - 1):i + 1]) ts_1 = [(x - low) / (high - low) for x in list(df[col[0]][i - (step - 1):i + 1])] ts_2 = [(x - low) / (high - low) for x in list(df[col[1]][i - (step - 1):i + 1])] ts_3 = [(x - low) / (high - low) for x in list(df[col[2]][i - (step - 1):i + 1])] ts_4 = [(x - low) / (high - low) for x in list(df[col[3]][i - (step - 1):i + 1])] ope = np.round(mtf.fit_transform([ts_1])[0] * 255) high = np.round(mtf.fit_transform([ts_2])[0] * 255) close = np.round(mtf.fit_transform([ts_3])[0] * 255) low = np.round(mtf.fit_transform([ts_4])[0] * 255) mtf_oh = np.hstack((ope, high)) mtf_cl = np.hstack((close, low)) mtf_ohcl = np.vstack((mtf_oh, mtf_cl)) X_mtf.append(mtf_ohcl.reshape(step * 2, step * 2, 1)) X_mtf = np.stack(X_mtf) for i in range((step - 1), len(df[col[0]])): high = max(df["High"][i - (step - 1):i + 1]) low = min(df["Low"][i - (step - 1):i + 1]) ts_1 = [(x - low) / (high - low) for x in list(df[col[0]][i - (step - 1):i + 1])] ts_2 = [(x - low) / (high - low) for x in list(df[col[1]][i - (step - 1):i + 1])] ts_3 = [(x - low) / (high - low) for x in list(df[col[2]][i - (step - 1):i + 1])] ts_4 = [(x - low) / (high - low) for x in list(df[col[3]][i - (step - 1):i + 1])] gadf_oh = np.hstack((np.round(gadf.fit_transform([ts_1])[0] * 255), np.round(gadf.fit_transform([ts_2])[0] * 255))) gadf_cl = np.hstack((np.round(gadf.fit_transform([ts_3])[0] * 255), np.round(gadf.fit_transform([ts_4])[0] * 255))) gadf_ohcl = np.vstack((gadf_oh, gadf_cl)) X_gadf.append(gadf_ohcl.reshape(step * 2, step * 2, 1)) X_gadf = np.stack(X_gadf) for i in range((step - 1), len(df[col[0]])): high = max(df["High"][i - (step - 1):i + 1]) low = min(df["Low"][i - (step - 1):i + 1]) ts_1 = [(x - low) / (high - low) for x in list(df[col[0]][i - (step - 1):i + 1])] ts_2 = [(x - low) / (high - low) for x in list(df[col[1]][i - (step - 1):i + 1])] ts_3 = [(x - low) / (high - low) for x in list(df[col[2]][i - (step - 1):i + 1])] ts_4 = [(x - low) / (high - low) for x in list(df[col[3]][i - (step - 1):i + 1])] gasf_oh = np.hstack((np.round(gasf.fit_transform([ts_1])[0] * 255), np.round(gasf.fit_transform([ts_2])[0] * 255))) gasf_cl = np.hstack((np.round(gasf.fit_transform([ts_3])[0] * 255), np.round(gasf.fit_transform([ts_4])[0] * 255))) gasf_ohcl = np.vstack((gasf_oh, gasf_cl)) X_gasf.append(gasf_ohcl.reshape(step * 2, step * 2, 1)) X_gasf = np.stack(X_gasf) return (X_gasf, X_gadf, X_mtf)
""" import numpy as np import matplotlib.pyplot as plt from mpl_toolkits.axes_grid1 import ImageGrid from pyts.image import GramianAngularField # Parameters n_samples, n_timestamps = 100, 144 # Toy dataset rng = np.random.RandomState(41) X = rng.randn(n_samples, n_timestamps) # Transform the time series into Gramian Angular Fields gasf = GramianAngularField(image_size=24, method='summation') X_gasf = gasf.fit_transform(X) gadf = GramianAngularField(image_size=24, method='difference') X_gadf = gadf.fit_transform(X) # Show the images for the first time series fig = plt.figure(figsize=(12, 7)) grid = ImageGrid(fig, 111, nrows_ncols=(1, 2), axes_pad=0.15, share_all=True, cbar_location="right", cbar_mode="single", cbar_size="7%", cbar_pad=0.3, )
def generate_data( rates, r=0.7, test=True, save_img=True, save_gasf=True, gasf_imsize=16, save_npy=True, tp=0.00500, sl=0.00250, sl_h=0.00250, window_range_back=72, window_range_front=30, keys=[], dpi=60, rcparams={ 'axes.spines.bottom': False, 'axes.spines.left': False, 'axes.spines.right': False, 'axes.spines.top': False, }): gasf = GramianAngularField(image_size=gasf_imsize, method='summation') X_buy = list() #N OHLC candles window X_buy_chart = list() X_buy_gasf = list() Y_reg_buy = list() #Next M values X_sell = list() X_sell_chart = list() X_sell_gasf = list() Y_reg_sell = list() X_hold = list() X_hold_chart = list() X_hold_gasf = list() Y_reg_hold = list() save_img = save_img save_gasf = save_gasf TP = tp SL_hold = sl_h SL = sl if test == True: start = window_range_back + int(r * len(rates)) end = len(rates) folder = 'data/chart/test' else: start = window_range_back end = int(len(rates) * r) folder = 'data/chart/train' #Clean old data (Just in case acquisition window is changed or something, else you end up with unwanted data.) if save_img == True: for path in paths: files = glob.glob(path + '/*.jpg') for f in files: os.remove(f) i = start while (i < end): #hold ops window_back = rates[i - window_range_back:i] window_front = rates[i:i + window_range_front] if len(window_front) < window_range_front: i = end continue lastclose = window_back.Close.iloc[-1] hold_up = window_front[window_front.High > lastclose + (SL_hold + 0.00100)] hold_down = window_front[window_front.Low < lastclose - (SL_hold + 0.00100)] if len(hold_up) == 0 and len( hold_down ) == 0: #Hold (strong consolidation or possible swing back) X_hold.append(np.array(window_back.values)) Y_reg_hold.append(np.array(window_front.values)) i += window_range_front if save_img == True: img = return_img(window_back, dpi=dpi, rcparams=rcparams) X_hold_chart.append(img) if save_gasf == True: X_gasf = gasf.fit_transform( window_back.values.transpose([1, 0])) X_hold_gasf.append(X_gasf) else: i += 1 i = start while (i < end): #buy ops window_back = rates[i - window_range_back:i] window_front = rates[i:i + window_range_front] if len(window_front) < window_range_front: i = end continue lastclose = window_back.Close.iloc[-1] TP_hit = window_front[window_front.High >= lastclose + TP] SL_hit = window_front[window_front.Low <= lastclose - SL] hold_up = window_front[window_front.High > lastclose + (SL_hold + 0.00100)] hold_down = window_front[window_front.Low <= lastclose - (SL_hold + 0.00100)] if (len(TP_hit) == 0 and len(SL_hit) == 0) and ( len(hold_up) == 0 and len(hold_down) == 0): #Hold (strong consolidation or possible swing back) i += window_range_front continue elif len(TP_hit) > 0 and len( SL_hit) == 0: #buy (no SL hit in period but TP is hit) X_buy.append(np.array(window_back.values)) Y_reg_buy.append(np.array(window_front.values)) i += window_range_front if save_img == True: img = return_img(window_back, dpi=dpi, rcparams=rcparams) X_buy_chart.append(img) if save_gasf == True: X_gasf = gasf.fit_transform( window_back.values.transpose([1, 0])) X_buy_gasf.append(X_gasf) elif len(TP_hit) > 0 and len( SL_hit ) > 0: #buy (both tp and sl hit but SL is hit after TP so trade won) TP_hit = TP_hit.iloc[0] SL_hit = SL_hit.iloc[0] if TP_hit.timestamp < SL_hit.timestamp: X_buy.append(np.array(window_back.values)) Y_reg_buy.append(np.array(window_front.values)) i += window_range_front if save_img == True: img = return_img(window_back, dpi=dpi, rcparams=rcparams) X_buy_chart.append(img) if save_gasf == True: X_gasf = gasf.fit_transform( window_back.values.transpose([1, 0])) X_buy_gasf.append(X_gasf) else: i += 1 else: i += 1 #look for sells and holds i = start while (i < end): #sell_ops window_back = rates[i - window_range_back:i] window_front = rates[i:i + window_range_front] if len(window_front) < window_range_front: i = end continue lastclose = window_back.Close.iloc[-1] SL_hit = window_front[window_front.High >= lastclose + SL] TP_hit = window_front[window_front.Low <= lastclose - TP] hold_up = window_front[window_front.High > lastclose + (SL_hold + 0.00100)] hold_down = window_front[window_front.Low <= lastclose - (SL_hold + 0.00100)] if (len(TP_hit) == 0 and len(SL_hit) == 0) and (len(hold_up) == 0 and len(hold_down) == 0): #Hold (strong consolidation) i += window_range_front continue elif len(TP_hit) > 0 and len( SL_hit) == 0: #buy (no SL hit in period but TP is hit) X_sell.append(np.array(window_back.values)) Y_reg_sell.append(np.array(window_front.values)) i += window_range_front if save_img == True: img = return_img(window_back, dpi=dpi, rcparams=rcparams) X_sell_chart.append(img) if save_gasf == True: X_gasf = gasf.fit_transform( window_back.values.transpose([1, 0])) X_sell_gasf.append(X_gasf) elif len(TP_hit) > 0 and len( SL_hit ) > 0: #buy (both tp and sl hit but SL is hit after TP so trade won) TP_hit = TP_hit.iloc[0] SL_hit = SL_hit.iloc[0] if TP_hit.timestamp < SL_hit.timestamp: X_sell.append(np.array(window_back.values)) Y_reg_sell.append(np.array(window_front.values)) i += window_range_front if save_img == True: img = return_img(window_back, dpi=dpi, rcparams=rcparams) X_sell_chart.append(img) if save_gasf == True: X_gasf = gasf.fit_transform( window_back.values.transpose([1, 0])) X_sell_gasf.append(X_gasf) else: i += 1 else: i += 1 X_buy = np.array(X_buy) #72 OHLC candles window X_buy_chart = np.array(X_buy_chart) X_buy_gasf = np.array(X_buy_gasf) Y_reg_buy = np.array(Y_reg_buy) #Next 72 values X_sell = np.array(X_sell) X_sell_chart = np.array(X_sell_chart) X_sell_gasf = np.array(X_sell_gasf) Y_reg_sell = np.array(Y_reg_sell) X_hold = np.array(X_hold) X_hold_chart = np.array(X_hold_chart) X_hold_gasf = np.array(X_hold_gasf) Y_reg_hold = np.array(Y_reg_hold) print(X_buy.shape) print(Y_reg_buy.shape) print(X_buy_gasf.shape) print(X_buy_chart.shape) print(X_sell.shape) print(Y_reg_sell.shape) print(X_sell_gasf.shape) print(X_sell_chart.shape) print(X_hold.shape) print(Y_reg_hold.shape) print(X_hold_gasf.shape) print(X_hold_chart.shape) if save_npy == True: print('Saving npy candle data [to be added]') return X_buy, X_buy_chart, X_buy_gasf, Y_reg_buy, X_sell, X_sell_chart, X_sell_gasf, Y_reg_sell, X_hold, X_hold_chart, X_sell_gasf, Y_reg_hold
check = 0 serial_port.write("STOP".encode()) print(app.bcolors.FAIL + '#' + app.bcolors.ENDC) sys.stdout.flush() current_state = state_values[j] X[0][:] = [] X[1][:] = [] X[0].extend(np.interp(data_amplitude[0], (0, 4096), (0, 1))) X[1].extend(np.interp(data_deltatime[0], (0, 100), (0, 1))) csv_write_row(np.interp(data_amplitude[0], (0, 4096), (0, 1))) gasf = GramianAngularField(image_size=sample_count, method='summation') X_gasf = gasf.fit_transform(X) plt.imsave("img/sample_" + str(i) + "_state_" + states[j] + ".jpg", X_gasf[0], format='jpg') file.write("img/sample_" + str(i) + "_state_" + states[j] + ".jpg," + str(current_state) + "\n") del gasf del X_gasf data_amplitude[:] = [] data_deltatime[:] = [] data_electrodes_num = 0 print("") file.close()
def to_gaf(df): gaf = GramianAngularField(image_size=image_size, sample_range=sample_range, method=method, overlapping=overlapping, flatten=flatten) return gaf.fit_transform(df.values)
keys = create_zigzag(rates, pct=0.3) addp3 = mpf.make_addplot(rates[keys]) mpf.plot(rates, type='candle', volume=True, addplot=[addp3], style='yahoo', show_nontrading=False, block=False) price = rates.Close err_allowed = 0.1 / 100 # Find peaks for i in range(100, len(price)): current_idx, current_pat, start, end = peak_detect(price.values[:i]) plt.plot(np.arange(start, i), price.values[start:i]) plt.scatter(current_idx, current_pat, c='r') plt.show() gasf = GramianAngularField(image_size=16, method='summation') X_gasf = gasf.fit_transform(X_buy[0].transpose([1, 0])) print(X_gasf[3].shape) plt.imshow(X_gasf[3, :]) plt.show() while (True): msg = input('Close? [N],y\n') if msg == 'y': break