def run_LSTM(X_train, X_test, y_train, y_test, symbol, price):
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
LSTM = create_LSTM(X_train)
history = LSTM.fit(X_train, y_train, epochs=100, batch_size=32)
print(LSTM.summary())
results = LSTM.evaluate(X_test, y_test, batch_size=32)
print("test loss, test acc:", results)
# summarize history for accuracy
plt.plot(history.history['accuracy'], label='Train')
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
prediction = LSTM.predict(X_test)
y_prediction = prediction[:, -1, :]
y_prediction = np.argmax(prediction, axis=1)
print(y_prediction)
NAV_history = Generate_nav(10000, symbol, price, y_prediction)
def predcoin(coinname, dataset):
result = []
global xs_xrp, ys_xrp, LSTM_xrp, lgb1_xrp, lgb2_xrp, xgb_xrp
global xs_cosm, ys_cosm, LSTM_cosm, lgb1_cosm, lgb2_cosm, xgb_cosm
global xs_eth, ys_eth, LSTM_eth, lgb1_eth, lgb2_eth, xgb_eth
global xs_knc, ys_knc, LSTM_knc, lgb1_knc, lgb2_knc, xgb_knc
global xs_powr, ys_powr, LSTM_powr, lgb1_powr, lgb2_powr, xgb_powr
if coinname == 'XRP':
xs, ys, LSTM, lgb1, lgb2, xgb = xs_xrp, ys_xrp, LSTM_xrp, lgb1_xrp, lgb2_xrp, xgb_xrp
elif coinname == 'COSM':
xs, ys, LSTM, lgb1, lgb2, xgb = xs_cosm, ys_cosm, LSTM_cosm, lgb1_cosm, lgb2_cosm, xgb_cosm
elif coinname == 'ETH':
xs, ys, LSTM, lgb1, lgb2, xgb = xs_eth, ys_eth, LSTM_eth, lgb1_eth, lgb2_eth, xgb_eth
elif coinname == 'KNC':
xs, ys, LSTM, lgb1, lgb2, xgb = xs_knc, ys_knc, LSTM_knc, lgb1_knc, lgb2_knc, xgb_knc
elif coinname == 'POWR':
xs, ys, LSTM, lgb1, lgb2, xgb = xs_powr, ys_powr, LSTM_powr, lgb1_powr, lgb2_powr, xgb_powr
data = Indicators(dataset)
nptf = np.array(data[-50:])
nptf = pd.DataFrame(xs.transform(nptf), columns=data.columns)
nptf = np.array(nptf)
nptf = nptf.reshape(1, 50, -1)
result.append(ys.inverse_transform(LSTM.predict(nptf)))
nptf = np.array(data.iloc[-1])
nptf = nptf.reshape(1, -1)
result.append(lgb1.predict(nptf))
result.append(lgb2.predict(nptf))
result.append(xgb.predict(nptf))
return result
# This can be a simple Dense layer of size 16 as well
#model = Flatten()(inputs)
model = LSTM(LATENT_SIZE, activation='relu')(inputs)
model = Dense(2)(model)
model = Model(inputs, model)
model.compile(loss=univariate_gaussian_loss, optimizer=OPTIMIZER)
model.summary()
callbacks = [
TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP + ' ' + SCRIPT_NAME,
write_graph=False),
DecypherAll(lambda x: str(x)),
EarlyStopping(patience=40, min_delta=1e-3)
]
history = model.fit(x=training_vectors,
y=target_vectors,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_split=TRAIN_VALIDATE_SPLIT,
callbacks=callbacks).history
prediction = model.predict(training_vectors[0:1000])
error = np.sum(np.abs(prediction[:, 0] - target_vectors[0:1000, 0])) / 1000
print('Error factor:', error)
notify(TIMESTAMP, SCRIPT_NAME,
'validation loss of ' + str(history['val_loss'][-1]))
print(SCRIPT_NAME, 'finished successfully')
def go(options):
tbw = SummaryWriter(log_dir=options.tb_dir)
if options.task == 'wikisimple':
x, w21, i2w = \
util.load_words(util.DIR + '/datasets/wikisimple.txt', vocab_size=options.top_words, limit=options.limit)
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
numwords = len(i2w)
print('max sequence length ', x_max_len)
print(numwords, 'distinct words')
x = util.batch_pad(x, options.batch, add_eos=True)
elif options.task == 'europarl-50':
x, w21, i2w = \
util.load_words(util.DIR + '/datasets/europarl.50.txt', vocab_size=options.top_words, limit=options.limit)
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
numwords = len(i2w)
print('max sequence length ', x_max_len)
print(numwords, 'distinct words')
x = util.batch_pad(x, options.batch, add_eos=True)
elif options.task == 'coco':
x, w21, i2w = \
util.load_words(util.DIR + '/datasets/coco.valannotations.txt', vocab_size=options.top_words, limit=options.limit)
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
numwords = len(i2w)
print('max sequence length ', x_max_len)
print(numwords, 'distinct words')
x = util.batch_pad(x, options.batch, add_eos=True)
elif options.task == 'europarl-100':
x, w21, i2w = \
util.load_words(util.DIR + '/datasets/europarl.100.txt', vocab_size=options.top_words, limit=options.limit)
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
numwords = len(i2w)
print('max sequence length ', x_max_len)
print(numwords, 'distinct words')
x = util.batch_pad(x, options.batch, add_eos=True)
elif options.task == 'file':
x, w21, i2w = \
util.load_words(options.data_dir, vocab_size=options.top_words, limit=options.limit)
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
numwords = len(i2w)
print('max sequence length ', x_max_len)
print(numwords, 'distinct words')
x = util.batch_pad(x, options.batch, add_eos=True)
else:
raise Exception('Task {} not recognized.'.format(options.task))
def decode(seq):
return ' '.join(i2w[id] for id in seq)
print('Finished data loading. ', sum([b.shape[0] for b in x]),
' sentences loaded')
## Define encoder
input = Input(shape=(None, ), name='inp')
embedding = Embedding(numwords, options.embedding_size, input_length=None)
embedded = embedding(input)
encoder = LSTM(
options.lstm_capacity) if options.rnn_type == 'lstm' else GRU(
options.lstm_capacity)
h = Bidirectional(encoder)(embedded)
tozmean = Dense(options.hidden)
zmean = tozmean(h)
tozlsigma = Dense(options.hidden)
zlsigma = tozlsigma(h)
## Define KL Loss and sampling
kl = util.KLLayer(
weight=K.variable(1.0)) # computes the KL loss and stores it for later
zmean, zlsigma = kl([zmean, zlsigma])
eps = Input(shape=(options.hidden, ), name='inp-epsilon')
sample = util.Sample()
zsample = sample([zmean, zlsigma, eps])
## Define decoder
input_shifted = Input(shape=(None, ), name='inp-shifted')
expandz_h = Dense(options.lstm_capacity, input_shape=(options.hidden, ))
z_exp_h = expandz_h(zsample)
if options.rnn_type == 'lstm':
expandz_c = Dense(options.lstm_capacity,
input_shape=(options.hidden, ))
z_exp_c = expandz_c(zsample)
state = [z_exp_h, z_exp_c]
else:
state = z_exp_h
seq = embedding(input_shifted)
seq = SpatialDropout1D(rate=options.dropout)(seq)
if options.rnn_type == 'lstm':
decoder_rnn = LSTM(options.lstm_capacity, return_sequences=True)
else:
decoder_rnn = GRU(options.lstm_capacity, return_sequences=True)
h = decoder_rnn(seq, initial_state=state)
towords = TimeDistributed(Dense(numwords))
out = towords(h)
auto = Model([input, input_shifted, eps], out)
## Extract the encoder and decoder models form the autoencoder
# - NB: This isn't exactly DRY. It seems much nicer to build a separate encoder and decoder model and then build a
# an autoencoder model that chains the two. For the life of me, I couldn't get it to work. For some reason the
# gradients don't seem to propagate down to the encoder. Let me know if you have better luck.
encoder = Model(input, [zmean, zlsigma])
z_in = Input(shape=(options.hidden, ))
s_in = Input(shape=(None, ))
seq = embedding(s_in)
z_exp_h = expandz_h(z_in)
if options.rnn_type == 'lstm':
z_exp_c = expandz_c(z_in)
state = [z_exp_h, z_exp_c]
else:
state = z_exp_h
h = decoder_rnn(seq, initial_state=state)
out = towords(h)
decoder = Model([s_in, z_in], out)
## Compile the autoencoder model for training
opt = keras.optimizers.Adam(lr=options.lr)
auto.compile(opt, sparse_loss)
auto.summary()
instances_seen = 0
for epoch in range(options.epochs + 1):
klw = anneal(epoch, options.epochs)
print('EPOCH {:03}: Set KL weight to {}'.format(epoch, klw))
K.set_value(kl.weight, klw)
for batch in tqdm(x):
n, l = batch.shape
batch_shifted = np.concatenate([np.ones((n, 1)), batch],
axis=1) # prepend start symbol
batch_out = np.concatenate([batch, np.zeros((n, 1))],
axis=1)[:, :, None] # append pad symbol
eps = np.random.randn(
n, options.hidden) # random noise for the sampling layer
loss = auto.train_on_batch([batch, batch_shifted, eps], batch_out)
instances_seen += n
tbw.add_scalar('seq2seq/batch-loss',
float(loss) / l, instances_seen)
if epoch % options.out_every == 0:
# show samples for some sentences from random batches
for i in range(CHECK):
# CHECK 1. Generate some sentences from a z vector for a random
# sentence from the corpus
b = random.choice(x)
z, _ = encoder.predict(b)
z = z[None, 0, :]
print('in ', decode(b[0, :]))
l = 60 if options.clip_length is None else options.clip_length
gen = generate_seq(decoder, z=z, size=l)
print('out 1 ', decode(gen))
gen = generate_seq(decoder, z=z, size=l, temperature=0.05)
print('out 2 (t0.05) ', decode(gen))
gen = generate_seq(decoder, z=z, size=l, temperature=0.0)
print('out 3 (greedy) ', decode(gen))
# CHECK 2. Show the argmax reconstruction (i
n, _ = b.shape
b_shifted = np.concatenate([np.ones((n, 1)), b],
axis=1) # prepend start symbol
eps = np.random.randn(
n, options.hidden) # random noise for the sampling layer
out = auto.predict([b, b_shifted, eps])[None, 0, :]
out = np.argmax(out[0, ...], axis=1)
print(out)
print('recon ', decode([int(o) for o in out]))
print()
for i in range(CHECK):
# CHECK 3: Sample two z's from N(0,1) and interpolate between them
# Here we use use greedy decoding: i.e. we pick the word that gets the highest
# probability
zfrom, zto = np.random.randn(1,
options.hidden), np.random.randn(
1, options.hidden)
for d in np.linspace(0, 1, num=NINTER):
z = zfrom * (1 - d) + zto * d
gen = generate_seq(decoder, z=z, size=l, temperature=0.0)
print('out (d={:.1}) \t'.format(d), decode(gen))
print()
write_graph=True)
epoch_callback = EpochLogger(input_func=GeoVectorizer.decypher,
target_func=lambda x: str(x),
predict_func=lambda x: str(x),
aggregate_func=None,
stdout=True)
model.fit(x=training_vectors,
y=target_vectors,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_split=TRAIN_VALIDATE_SPLIT,
callbacks=[epoch_callback, tb_callback])
plot_sample = training_vectors[-10000:]
prediction = model.predict(plot_sample)
intersecting_error = np.abs(prediction[:, 0] - target_vectors[-10000:])
triangle_vectors = plot_sample.reshape(10000, 6, 2)
triangle_sets = np.array(
[[Polygon(point_set[0:3]).wkt,
Polygon(point_set[3:]).wkt] for point_set in triangle_vectors])
zipped = zip(triangle_sets, target_vectors[-10000:], prediction[:, 0])
sorted_results = sorted(zipped, key=lambda record: abs(record[2] - record[1]))
print('Intersection surface area mean:', np.mean(target_vectors))
print('Intersecting error mean:', np.mean(intersecting_error))
plot_samples = 50
print('Saving top and bottom', plot_samples,
'results as plots, this will take a few minutes...')
def go(options):
slength = options.max_length
lstm_hidden = options.lstm_capacity
print('devices', device_lib.list_local_devices())
tbw = SummaryWriter(log_dir=options.tb_dir)
if options.task == 'file':
dir = options.data_dir
x, x_vocab_len, x_word_to_ix, x_ix_to_word = \
util.load_sentences(options.data_dir, vocab_size=options.top_words)
if options.clip_length is not None:
x = [(sentence if len(sentence) < options.clip_length else sentence[:options.clip_length]) for sentence in x]
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
print('max sequence length ', x_max_len)
print(len(x_ix_to_word), 'distinct words')
x = util.batch_pad(x, options.batch, add_eos=True)
def decode(seq):
return ' '.join(x_ix_to_word[id] for id in seq)
elif options.task == 'europarl':
dir = options.data_dir
x, x_vocab_len, x_word_to_ix, x_ix_to_word, _, _, _, _ = \
util.load_data(dir+os.sep+'europarl-v8.fi-en.en', dir+os.sep+'europarl-v8.fi-en.fi', vocab_size=options.top_words)
# Finding the length of the longest sequence
x_max_len = max([len(sentence) for sentence in x])
print('max sequence length ', x_max_len)
print(len(x_ix_to_word), 'distinct words')
x = util.batch_pad(x, options.batch)
# Padding zeros to make all sequences have a same length with the longest one
# x = sequence.pad_sequences(x, maxlen=slength, dtype='int32', padding='post', truncating='post')
# y = sequence.pad_sequences(y, maxlen=slength, dtype='int32', padding='post', truncating='post')
def decode(seq):
print(seq)
return ' '.join(x_ix_to_word[id] for id in seq)
else:
# Load only training sequences
(x, _), _ = imdb.load_data(num_words=options.top_words)
# rm start symbol
x = [l[1:] for l in x]
# x = sequence.pad_sequences(x, maxlen=slength+1, padding='post', truncating='post')
# x = x[:, 1:] # rm start symbol
x = util.batch_pad(x, options.batch)
decode = decode_imdb
print('Data Loaded.')
print(sum([b.shape[0] for b in x]), ' sentences loaded')
# for i in range(3):
# print(x[i, :])
# print(decode(x[i, :]))
num_words = len(x_ix_to_word)
## Define encoder
input = Input(shape=(None, ), name='inp')
embedding = Embedding(num_words, options.embedding_size, input_length=None)
embedded = embedding(input)
encoder = LSTM(lstm_hidden) if options.rnn_type == 'lstm' else GRU(lstm_hidden)
h = Bidirectional(encoder)(embedded)
tozmean = Dense(options.hidden)
zmean = tozmean(h)
tozlsigma = Dense(options.hidden)
zlsigma = tozlsigma(h)
## Define KL Loss and sampling
kl = util.KLLayer(weight = K.variable(1.0)) # computes the KL loss and stores it for later
zmean, zlsigma = kl([zmean, zlsigma])
eps = Input(shape=(options.hidden,), name='inp-epsilon')
sample = util.Sample()
zsample = sample([zmean, zlsigma, eps])
## Define decoder
# zsample = Input(shape=(options.hidden,), name='inp-decoder-z')
input_shifted = Input(shape=(None, ), name='inp-shifted')
if options.rnn_type == 'lstm':
expandz_h = Dense(lstm_hidden, input_shape=(options.hidden,))
expandz_c = Dense(lstm_hidden, input_shape=(options.hidden,))
z_exp_h = expandz_h(zsample)
z_exp_c = expandz_c(zsample)
state = [z_exp_h, z_exp_c]
else:
expandz = Dense(lstm_hidden, input_shape=(options.hidden,))
state = expandz(zsample)
seq = embedding(input_shifted)
seq = SpatialDropout1D(rate=options.dropout)(seq)
decoder_rnn = LSTM(lstm_hidden, return_sequences=True) if options.rnn_type == 'lstm' else GRU(lstm_hidden, return_sequences=True)
h = decoder_rnn(seq, initial_state=state)
towords = TimeDistributed(Dense(num_words))
out = towords(h)
auto = Model([input, input_shifted, eps], out)
## Extract the encoder and decoder models form the autoencoder
# - NB: This isn't exactly DRY. It seems much nicer to build a separate encoder and decoder model and then build a
# an autoencoder model that chains the two. For the life of me, I couldn't get it to work. For some reason the
# gradients don't seem to propagate down to the encoder. Let me know if you have better luck.
encoder = Model(input, [zmean, zlsigma])
z_in = Input(shape=(options.hidden,))
s_in = Input(shape=(None,))
seq = embedding(s_in)
if options.rnn_type == 'lstm':
z_exp_h = expandz_h(z_in)
z_exp_c = expandz_c(z_in)
state = [z_exp_h, z_exp_c]
else:
state = expandz(z_in)
h = decoder_rnn(seq, initial_state=state)
out = towords(h)
decoder = Model([s_in, z_in], out)
## Compile the autoencoder model
if options.num_gpu is not None:
auto = multi_gpu_model(auto, gpus=options.num_gpu)
opt = keras.optimizers.Adam(lr=options.lr)
auto.compile(opt, sparse_loss)
auto.summary()
epoch = 0
instances_seen = 0
# DEBUG
# x = x[:20]
while epoch < options.epochs:
klw = anneal(epoch, options.epochs)
print('EPOCH {:03}: Set KL weight to {}'.format(epoch, klw))
K.set_value(kl.weight, klw)
for batch in tqdm(x):
n, l = batch.shape
batch_shifted = np.concatenate([np.ones((n, 1)), batch], axis=1) # prepend start symbol
batch_out = np.concatenate([batch, np.zeros((n, 1))], axis=1)[:, :, None] # append pad symbol
eps = np.random.randn(n, options.hidden) # random noise for the sampling layer
loss = auto.train_on_batch([batch, batch_shifted, eps], batch_out)
instances_seen += n
tbw.add_scalar('seq2seq/batch-loss', float(loss)/l, instances_seen)
epoch += 1
if epoch % options.out_every == 0:
# show samples for some sentences from random batches
for i in range(CHECK):
# CHECK 1. Generate some sentences from a z vector for a random
# sentence from the corpus
b = random.choice(x)
z, _ = encoder.predict(b)
z = z[None, 0, :]
print('in ', decode(b[0, :]))
l = 60 if options.clip_length is None else options.clip_length
gen = generate_seq(decoder, z=z, size=l)
print('out 1 ', decode(gen))
gen = generate_seq(decoder, z=z, size=l, temperature=0.05)
print('out 2 (t0.05) ', decode(gen))
gen = generate_seq(decoder, z=z, size=l, temperature=0.0)
print('out 3 (greedy) ', decode(gen))
# CHECK 2. Show the argmax reconstruction (i
n, _ = b.shape
b_shifted = np.concatenate([np.ones((n, 1)), b], axis=1) # prepend start symbol
eps = np.random.randn(n, options.hidden) # random noise for the sampling layer
out = auto.predict([b, b_shifted, eps])[None, 0, :]
out = np.argmax(out[0, ...], axis=1)
print(out)
print('recon ', decode([int(o) for o in out]))
print()
for i in range(CHECK):
# CHECK 3: Sample two z's from N(0,1) and interpolate between them
# Here we use use greedy decoding: i.e. we pick the word that gets the highest
# probability
zfrom, zto = np.random.randn(1, options.hidden), np.random.randn(1, options.hidden)
for d in np.linspace(0, 1, num=NINTER):
z = zfrom * (1-d) + zto * d
gen = generate_seq(decoder, z=z, size=l, temperature=0.0)
print('out (d={:.1}) \t'.format(d), decode(gen))
print()
scaler = preprocessing.MinMaxScaler()
scaler.fit_transform(X_train)
train_X = np.array(X_train).reshape((X_train.shape[0], 1, X_train.shape[1]))
train_y = np.array(y_train).reshape((y_train.shape[0], 1, 1))
lstm_features = []
lstm_features.append(DATA_FESTIVA)
lstm_features.append(FERIADO)
lstm_features.append(QTD_CONCORRENTES)
lstm_features.append(UMIDADE)
lstm_features.append(VENDAS_ONTEM)
lstm_features = scaler.transform([lstm_features])
lstm_features = np.array(lstm_features).reshape((lstm_features.shape[0], 1, lstm_features.shape[1]))
lstm_y_pred = LSTM.predict(lstm_features).round().astype(int)
########################################################################
x = data.drop(columns=get_features_to_drop_gb(), axis=1)
y = data["VENDAS"]
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.05, random_state=1, shuffle=False)
scaler = preprocessing.MinMaxScaler()
scaler.fit_transform(X_train)
gb_features = []
gb_features.append(ALTA_TEMPORADA)
gb_features.append(DATA_FESTIVA)
gb_features.append(FERIADO)
gb_features.append(QTD_CONCORRENTES)
gb_features.append(VENDAS_ONTEM)
model.summary()
callbacks = [
TensorBoard(log_dir='./tensorboard_log/' + TIMESTAMP + ' ' + SCRIPT_NAME, write_graph=False),
DecypherAll(lambda x: str(x)),
EarlyStopping(patience=40, min_delta=0.001)
]
history = model.fit(x=training_vectors,
y=target_vectors,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_split=TRAIN_VALIDATE_SPLIT,
callbacks=callbacks).history
val_set_start = -round(data_points * TRAIN_VALIDATE_SPLIT)
prediction = model.predict(training_vectors[val_set_start:])
error = prediction[:, 0] - target_vectors[val_set_start:, 0]
fig, ax = plt.subplots()
plt.text(0.01, 0.94, r'prediction error $\mu: $' + str(np.round(np.mean(error), 4)), transform=ax.transAxes)
plt.text(0.01, 0.88, r'prediction error $\sigma: $' + str(np.round(np.std(error), 4)), transform=ax.transAxes)
plt.xlabel('Error')
plt.ylabel('Frequency')
plt.title('Geometric distance error')
n, bins, patches = plt.hist(error, 50, facecolor='g', normed=False, alpha=0.75)
os.makedirs(str(PLOT_DIR), exist_ok=True)
plt.savefig(PLOT_DIR + '/plt_' + TIMESTAMP + '.png')
notify(TIMESTAMP, SCRIPT_NAME, 'validation loss of ' + str(history['val_loss'][-1]))
print(SCRIPT_NAME, 'finished successfully')
