def run_LSTM(X_train, X_test, y_train, y_test, symbol, price): y_train = to_categorical(y_train) y_test = to_categorical(y_test) LSTM = create_LSTM(X_train) history = LSTM.fit(X_train, y_train, epochs=100, batch_size=32) print(LSTM.summary()) results = LSTM.evaluate(X_test, y_test, batch_size=32) print("test loss, test acc:", results) # summarize history for accuracy plt.plot(history.history['accuracy'], label='Train') plt.title('model accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.show() prediction = LSTM.predict(X_test) y_prediction = prediction[:, -1, :] y_prediction = np.argmax(prediction, axis=1) print(y_prediction) NAV_history = Generate_nav(10000, symbol, price, y_prediction)
def predcoin(coinname, dataset): result = [] global xs_xrp, ys_xrp, LSTM_xrp, lgb1_xrp, lgb2_xrp, xgb_xrp global xs_cosm, ys_cosm, LSTM_cosm, lgb1_cosm, lgb2_cosm, xgb_cosm global xs_eth, ys_eth, LSTM_eth, lgb1_eth, lgb2_eth, xgb_eth global xs_knc, ys_knc, LSTM_knc, lgb1_knc, lgb2_knc, xgb_knc global xs_powr, ys_powr, LSTM_powr, lgb1_powr, lgb2_powr, xgb_powr if coinname == 'XRP': xs, ys, LSTM, lgb1, lgb2, xgb = xs_xrp, ys_xrp, LSTM_xrp, lgb1_xrp, lgb2_xrp, xgb_xrp elif coinname == 'COSM': xs, ys, LSTM, lgb1, lgb2, xgb = xs_cosm, ys_cosm, LSTM_cosm, lgb1_cosm, lgb2_cosm, xgb_cosm elif coinname == 'ETH': xs, ys, LSTM, lgb1, lgb2, xgb = xs_eth, ys_eth, LSTM_eth, lgb1_eth, lgb2_eth, xgb_eth elif coinname == 'KNC': xs, ys, LSTM, lgb1, lgb2, xgb = xs_knc, ys_knc, LSTM_knc, lgb1_knc, lgb2_knc, xgb_knc elif coinname == 'POWR': xs, ys, LSTM, lgb1, lgb2, xgb = xs_powr, ys_powr, LSTM_powr, lgb1_powr, lgb2_powr, xgb_powr data = Indicators(dataset) nptf = np.array(data[-50:]) nptf = pd.DataFrame(xs.transform(nptf), columns=data.columns) nptf = np.array(nptf) nptf = nptf.reshape(1, 50, -1) result.append(ys.inverse_transform(LSTM.predict(nptf))) nptf = np.array(data.iloc[-1]) nptf = nptf.reshape(1, -1) result.append(lgb1.predict(nptf)) result.append(lgb2.predict(nptf)) result.append(xgb.predict(nptf)) return result
# This can be a simple Dense layer of size 16 as well #model = Flatten()(inputs) model = LSTM(LATENT_SIZE, activation='relu')(inputs) model = Dense(2)(model) model = Model(inputs, model) model.compile(loss=univariate_gaussian_loss, optimizer=OPTIMIZER) model.summary() callbacks = [ TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP + ' ' + SCRIPT_NAME, write_graph=False), DecypherAll(lambda x: str(x)), EarlyStopping(patience=40, min_delta=1e-3) ] history = model.fit(x=training_vectors, y=target_vectors, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_split=TRAIN_VALIDATE_SPLIT, callbacks=callbacks).history prediction = model.predict(training_vectors[0:1000]) error = np.sum(np.abs(prediction[:, 0] - target_vectors[0:1000, 0])) / 1000 print('Error factor:', error) notify(TIMESTAMP, SCRIPT_NAME, 'validation loss of ' + str(history['val_loss'][-1])) print(SCRIPT_NAME, 'finished successfully')
def go(options): tbw = SummaryWriter(log_dir=options.tb_dir) if options.task == 'wikisimple': x, w21, i2w = \ util.load_words(util.DIR + '/datasets/wikisimple.txt', vocab_size=options.top_words, limit=options.limit) # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) numwords = len(i2w) print('max sequence length ', x_max_len) print(numwords, 'distinct words') x = util.batch_pad(x, options.batch, add_eos=True) elif options.task == 'europarl-50': x, w21, i2w = \ util.load_words(util.DIR + '/datasets/europarl.50.txt', vocab_size=options.top_words, limit=options.limit) # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) numwords = len(i2w) print('max sequence length ', x_max_len) print(numwords, 'distinct words') x = util.batch_pad(x, options.batch, add_eos=True) elif options.task == 'coco': x, w21, i2w = \ util.load_words(util.DIR + '/datasets/coco.valannotations.txt', vocab_size=options.top_words, limit=options.limit) # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) numwords = len(i2w) print('max sequence length ', x_max_len) print(numwords, 'distinct words') x = util.batch_pad(x, options.batch, add_eos=True) elif options.task == 'europarl-100': x, w21, i2w = \ util.load_words(util.DIR + '/datasets/europarl.100.txt', vocab_size=options.top_words, limit=options.limit) # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) numwords = len(i2w) print('max sequence length ', x_max_len) print(numwords, 'distinct words') x = util.batch_pad(x, options.batch, add_eos=True) elif options.task == 'file': x, w21, i2w = \ util.load_words(options.data_dir, vocab_size=options.top_words, limit=options.limit) # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) numwords = len(i2w) print('max sequence length ', x_max_len) print(numwords, 'distinct words') x = util.batch_pad(x, options.batch, add_eos=True) else: raise Exception('Task {} not recognized.'.format(options.task)) def decode(seq): return ' '.join(i2w[id] for id in seq) print('Finished data loading. ', sum([b.shape[0] for b in x]), ' sentences loaded') ## Define encoder input = Input(shape=(None, ), name='inp') embedding = Embedding(numwords, options.embedding_size, input_length=None) embedded = embedding(input) encoder = LSTM( options.lstm_capacity) if options.rnn_type == 'lstm' else GRU( options.lstm_capacity) h = Bidirectional(encoder)(embedded) tozmean = Dense(options.hidden) zmean = tozmean(h) tozlsigma = Dense(options.hidden) zlsigma = tozlsigma(h) ## Define KL Loss and sampling kl = util.KLLayer( weight=K.variable(1.0)) # computes the KL loss and stores it for later zmean, zlsigma = kl([zmean, zlsigma]) eps = Input(shape=(options.hidden, ), name='inp-epsilon') sample = util.Sample() zsample = sample([zmean, zlsigma, eps]) ## Define decoder input_shifted = Input(shape=(None, ), name='inp-shifted') expandz_h = Dense(options.lstm_capacity, input_shape=(options.hidden, )) z_exp_h = expandz_h(zsample) if options.rnn_type == 'lstm': expandz_c = Dense(options.lstm_capacity, input_shape=(options.hidden, )) z_exp_c = expandz_c(zsample) state = [z_exp_h, z_exp_c] else: state = z_exp_h seq = embedding(input_shifted) seq = SpatialDropout1D(rate=options.dropout)(seq) if options.rnn_type == 'lstm': decoder_rnn = LSTM(options.lstm_capacity, return_sequences=True) else: decoder_rnn = GRU(options.lstm_capacity, return_sequences=True) h = decoder_rnn(seq, initial_state=state) towords = TimeDistributed(Dense(numwords)) out = towords(h) auto = Model([input, input_shifted, eps], out) ## Extract the encoder and decoder models form the autoencoder # - NB: This isn't exactly DRY. It seems much nicer to build a separate encoder and decoder model and then build a # an autoencoder model that chains the two. For the life of me, I couldn't get it to work. For some reason the # gradients don't seem to propagate down to the encoder. Let me know if you have better luck. encoder = Model(input, [zmean, zlsigma]) z_in = Input(shape=(options.hidden, )) s_in = Input(shape=(None, )) seq = embedding(s_in) z_exp_h = expandz_h(z_in) if options.rnn_type == 'lstm': z_exp_c = expandz_c(z_in) state = [z_exp_h, z_exp_c] else: state = z_exp_h h = decoder_rnn(seq, initial_state=state) out = towords(h) decoder = Model([s_in, z_in], out) ## Compile the autoencoder model for training opt = keras.optimizers.Adam(lr=options.lr) auto.compile(opt, sparse_loss) auto.summary() instances_seen = 0 for epoch in range(options.epochs + 1): klw = anneal(epoch, options.epochs) print('EPOCH {:03}: Set KL weight to {}'.format(epoch, klw)) K.set_value(kl.weight, klw) for batch in tqdm(x): n, l = batch.shape batch_shifted = np.concatenate([np.ones((n, 1)), batch], axis=1) # prepend start symbol batch_out = np.concatenate([batch, np.zeros((n, 1))], axis=1)[:, :, None] # append pad symbol eps = np.random.randn( n, options.hidden) # random noise for the sampling layer loss = auto.train_on_batch([batch, batch_shifted, eps], batch_out) instances_seen += n tbw.add_scalar('seq2seq/batch-loss', float(loss) / l, instances_seen) if epoch % options.out_every == 0: # show samples for some sentences from random batches for i in range(CHECK): # CHECK 1. Generate some sentences from a z vector for a random # sentence from the corpus b = random.choice(x) z, _ = encoder.predict(b) z = z[None, 0, :] print('in ', decode(b[0, :])) l = 60 if options.clip_length is None else options.clip_length gen = generate_seq(decoder, z=z, size=l) print('out 1 ', decode(gen)) gen = generate_seq(decoder, z=z, size=l, temperature=0.05) print('out 2 (t0.05) ', decode(gen)) gen = generate_seq(decoder, z=z, size=l, temperature=0.0) print('out 3 (greedy) ', decode(gen)) # CHECK 2. Show the argmax reconstruction (i n, _ = b.shape b_shifted = np.concatenate([np.ones((n, 1)), b], axis=1) # prepend start symbol eps = np.random.randn( n, options.hidden) # random noise for the sampling layer out = auto.predict([b, b_shifted, eps])[None, 0, :] out = np.argmax(out[0, ...], axis=1) print(out) print('recon ', decode([int(o) for o in out])) print() for i in range(CHECK): # CHECK 3: Sample two z's from N(0,1) and interpolate between them # Here we use use greedy decoding: i.e. we pick the word that gets the highest # probability zfrom, zto = np.random.randn(1, options.hidden), np.random.randn( 1, options.hidden) for d in np.linspace(0, 1, num=NINTER): z = zfrom * (1 - d) + zto * d gen = generate_seq(decoder, z=z, size=l, temperature=0.0) print('out (d={:.1}) \t'.format(d), decode(gen)) print()
write_graph=True) epoch_callback = EpochLogger(input_func=GeoVectorizer.decypher, target_func=lambda x: str(x), predict_func=lambda x: str(x), aggregate_func=None, stdout=True) model.fit(x=training_vectors, y=target_vectors, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_split=TRAIN_VALIDATE_SPLIT, callbacks=[epoch_callback, tb_callback]) plot_sample = training_vectors[-10000:] prediction = model.predict(plot_sample) intersecting_error = np.abs(prediction[:, 0] - target_vectors[-10000:]) triangle_vectors = plot_sample.reshape(10000, 6, 2) triangle_sets = np.array( [[Polygon(point_set[0:3]).wkt, Polygon(point_set[3:]).wkt] for point_set in triangle_vectors]) zipped = zip(triangle_sets, target_vectors[-10000:], prediction[:, 0]) sorted_results = sorted(zipped, key=lambda record: abs(record[2] - record[1])) print('Intersection surface area mean:', np.mean(target_vectors)) print('Intersecting error mean:', np.mean(intersecting_error)) plot_samples = 50 print('Saving top and bottom', plot_samples, 'results as plots, this will take a few minutes...')
def go(options): slength = options.max_length lstm_hidden = options.lstm_capacity print('devices', device_lib.list_local_devices()) tbw = SummaryWriter(log_dir=options.tb_dir) if options.task == 'file': dir = options.data_dir x, x_vocab_len, x_word_to_ix, x_ix_to_word = \ util.load_sentences(options.data_dir, vocab_size=options.top_words) if options.clip_length is not None: x = [(sentence if len(sentence) < options.clip_length else sentence[:options.clip_length]) for sentence in x] # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) print('max sequence length ', x_max_len) print(len(x_ix_to_word), 'distinct words') x = util.batch_pad(x, options.batch, add_eos=True) def decode(seq): return ' '.join(x_ix_to_word[id] for id in seq) elif options.task == 'europarl': dir = options.data_dir x, x_vocab_len, x_word_to_ix, x_ix_to_word, _, _, _, _ = \ util.load_data(dir+os.sep+'europarl-v8.fi-en.en', dir+os.sep+'europarl-v8.fi-en.fi', vocab_size=options.top_words) # Finding the length of the longest sequence x_max_len = max([len(sentence) for sentence in x]) print('max sequence length ', x_max_len) print(len(x_ix_to_word), 'distinct words') x = util.batch_pad(x, options.batch) # Padding zeros to make all sequences have a same length with the longest one # x = sequence.pad_sequences(x, maxlen=slength, dtype='int32', padding='post', truncating='post') # y = sequence.pad_sequences(y, maxlen=slength, dtype='int32', padding='post', truncating='post') def decode(seq): print(seq) return ' '.join(x_ix_to_word[id] for id in seq) else: # Load only training sequences (x, _), _ = imdb.load_data(num_words=options.top_words) # rm start symbol x = [l[1:] for l in x] # x = sequence.pad_sequences(x, maxlen=slength+1, padding='post', truncating='post') # x = x[:, 1:] # rm start symbol x = util.batch_pad(x, options.batch) decode = decode_imdb print('Data Loaded.') print(sum([b.shape[0] for b in x]), ' sentences loaded') # for i in range(3): # print(x[i, :]) # print(decode(x[i, :])) num_words = len(x_ix_to_word) ## Define encoder input = Input(shape=(None, ), name='inp') embedding = Embedding(num_words, options.embedding_size, input_length=None) embedded = embedding(input) encoder = LSTM(lstm_hidden) if options.rnn_type == 'lstm' else GRU(lstm_hidden) h = Bidirectional(encoder)(embedded) tozmean = Dense(options.hidden) zmean = tozmean(h) tozlsigma = Dense(options.hidden) zlsigma = tozlsigma(h) ## Define KL Loss and sampling kl = util.KLLayer(weight = K.variable(1.0)) # computes the KL loss and stores it for later zmean, zlsigma = kl([zmean, zlsigma]) eps = Input(shape=(options.hidden,), name='inp-epsilon') sample = util.Sample() zsample = sample([zmean, zlsigma, eps]) ## Define decoder # zsample = Input(shape=(options.hidden,), name='inp-decoder-z') input_shifted = Input(shape=(None, ), name='inp-shifted') if options.rnn_type == 'lstm': expandz_h = Dense(lstm_hidden, input_shape=(options.hidden,)) expandz_c = Dense(lstm_hidden, input_shape=(options.hidden,)) z_exp_h = expandz_h(zsample) z_exp_c = expandz_c(zsample) state = [z_exp_h, z_exp_c] else: expandz = Dense(lstm_hidden, input_shape=(options.hidden,)) state = expandz(zsample) seq = embedding(input_shifted) seq = SpatialDropout1D(rate=options.dropout)(seq) decoder_rnn = LSTM(lstm_hidden, return_sequences=True) if options.rnn_type == 'lstm' else GRU(lstm_hidden, return_sequences=True) h = decoder_rnn(seq, initial_state=state) towords = TimeDistributed(Dense(num_words)) out = towords(h) auto = Model([input, input_shifted, eps], out) ## Extract the encoder and decoder models form the autoencoder # - NB: This isn't exactly DRY. It seems much nicer to build a separate encoder and decoder model and then build a # an autoencoder model that chains the two. For the life of me, I couldn't get it to work. For some reason the # gradients don't seem to propagate down to the encoder. Let me know if you have better luck. encoder = Model(input, [zmean, zlsigma]) z_in = Input(shape=(options.hidden,)) s_in = Input(shape=(None,)) seq = embedding(s_in) if options.rnn_type == 'lstm': z_exp_h = expandz_h(z_in) z_exp_c = expandz_c(z_in) state = [z_exp_h, z_exp_c] else: state = expandz(z_in) h = decoder_rnn(seq, initial_state=state) out = towords(h) decoder = Model([s_in, z_in], out) ## Compile the autoencoder model if options.num_gpu is not None: auto = multi_gpu_model(auto, gpus=options.num_gpu) opt = keras.optimizers.Adam(lr=options.lr) auto.compile(opt, sparse_loss) auto.summary() epoch = 0 instances_seen = 0 # DEBUG # x = x[:20] while epoch < options.epochs: klw = anneal(epoch, options.epochs) print('EPOCH {:03}: Set KL weight to {}'.format(epoch, klw)) K.set_value(kl.weight, klw) for batch in tqdm(x): n, l = batch.shape batch_shifted = np.concatenate([np.ones((n, 1)), batch], axis=1) # prepend start symbol batch_out = np.concatenate([batch, np.zeros((n, 1))], axis=1)[:, :, None] # append pad symbol eps = np.random.randn(n, options.hidden) # random noise for the sampling layer loss = auto.train_on_batch([batch, batch_shifted, eps], batch_out) instances_seen += n tbw.add_scalar('seq2seq/batch-loss', float(loss)/l, instances_seen) epoch += 1 if epoch % options.out_every == 0: # show samples for some sentences from random batches for i in range(CHECK): # CHECK 1. Generate some sentences from a z vector for a random # sentence from the corpus b = random.choice(x) z, _ = encoder.predict(b) z = z[None, 0, :] print('in ', decode(b[0, :])) l = 60 if options.clip_length is None else options.clip_length gen = generate_seq(decoder, z=z, size=l) print('out 1 ', decode(gen)) gen = generate_seq(decoder, z=z, size=l, temperature=0.05) print('out 2 (t0.05) ', decode(gen)) gen = generate_seq(decoder, z=z, size=l, temperature=0.0) print('out 3 (greedy) ', decode(gen)) # CHECK 2. Show the argmax reconstruction (i n, _ = b.shape b_shifted = np.concatenate([np.ones((n, 1)), b], axis=1) # prepend start symbol eps = np.random.randn(n, options.hidden) # random noise for the sampling layer out = auto.predict([b, b_shifted, eps])[None, 0, :] out = np.argmax(out[0, ...], axis=1) print(out) print('recon ', decode([int(o) for o in out])) print() for i in range(CHECK): # CHECK 3: Sample two z's from N(0,1) and interpolate between them # Here we use use greedy decoding: i.e. we pick the word that gets the highest # probability zfrom, zto = np.random.randn(1, options.hidden), np.random.randn(1, options.hidden) for d in np.linspace(0, 1, num=NINTER): z = zfrom * (1-d) + zto * d gen = generate_seq(decoder, z=z, size=l, temperature=0.0) print('out (d={:.1}) \t'.format(d), decode(gen)) print()
scaler = preprocessing.MinMaxScaler() scaler.fit_transform(X_train) train_X = np.array(X_train).reshape((X_train.shape[0], 1, X_train.shape[1])) train_y = np.array(y_train).reshape((y_train.shape[0], 1, 1)) lstm_features = [] lstm_features.append(DATA_FESTIVA) lstm_features.append(FERIADO) lstm_features.append(QTD_CONCORRENTES) lstm_features.append(UMIDADE) lstm_features.append(VENDAS_ONTEM) lstm_features = scaler.transform([lstm_features]) lstm_features = np.array(lstm_features).reshape((lstm_features.shape[0], 1, lstm_features.shape[1])) lstm_y_pred = LSTM.predict(lstm_features).round().astype(int) ######################################################################## x = data.drop(columns=get_features_to_drop_gb(), axis=1) y = data["VENDAS"] X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.05, random_state=1, shuffle=False) scaler = preprocessing.MinMaxScaler() scaler.fit_transform(X_train) gb_features = [] gb_features.append(ALTA_TEMPORADA) gb_features.append(DATA_FESTIVA) gb_features.append(FERIADO) gb_features.append(QTD_CONCORRENTES) gb_features.append(VENDAS_ONTEM)
model.summary() callbacks = [ TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP + ' ' + SCRIPT_NAME, write_graph=False), DecypherAll(lambda x: str(x)), EarlyStopping(patience=40, min_delta=0.001) ] history = model.fit(x=training_vectors, y=target_vectors, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_split=TRAIN_VALIDATE_SPLIT, callbacks=callbacks).history val_set_start = -round(data_points * TRAIN_VALIDATE_SPLIT) prediction = model.predict(training_vectors[val_set_start:]) error = prediction[:, 0] - target_vectors[val_set_start:, 0] fig, ax = plt.subplots() plt.text(0.01, 0.94, r'prediction error $\mu: $' + str(np.round(np.mean(error), 4)), transform=ax.transAxes) plt.text(0.01, 0.88, r'prediction error $\sigma: $' + str(np.round(np.std(error), 4)), transform=ax.transAxes) plt.xlabel('Error') plt.ylabel('Frequency') plt.title('Geometric distance error') n, bins, patches = plt.hist(error, 50, facecolor='g', normed=False, alpha=0.75) os.makedirs(str(PLOT_DIR), exist_ok=True) plt.savefig(PLOT_DIR + '/plt_' + TIMESTAMP + '.png') notify(TIMESTAMP, SCRIPT_NAME, 'validation loss of ' + str(history['val_loss'][-1])) print(SCRIPT_NAME, 'finished successfully')