def target_RNN(wv, tweet_max_length, aspect_max_length, classes=2, **kwargs):
######################################################
# HyperParameters
######################################################
noise = kwargs.get("noise", 0)
trainable = kwargs.get("trainable", False)
rnn_size = kwargs.get("rnn_size", 75)
rnn_type = kwargs.get("rnn_type", LSTM)
final_size = kwargs.get("final_size", 100)
final_type = kwargs.get("final_type", "linear")
use_final = kwargs.get("use_final", False)
drop_text_input = kwargs.get("drop_text_input", 0.)
drop_text_rnn = kwargs.get("drop_text_rnn", 0.)
drop_text_rnn_U = kwargs.get("drop_text_rnn_U", 0.)
drop_target_rnn = kwargs.get("drop_target_rnn", 0.)
drop_rep = kwargs.get("drop_rep", 0.)
drop_final = kwargs.get("drop_final", 0.)
activity_l2 = kwargs.get("activity_l2", 0.)
clipnorm = kwargs.get("clipnorm", 5)
bi = kwargs.get("bi", False)
lr = kwargs.get("lr", 0.001)
attention = kwargs.get("attention", "simple")
#####################################################
shared_RNN = get_RNN(rnn_type, rnn_size, bi=bi, return_sequences=True,
dropout_U=drop_text_rnn_U)
input_tweet = Input(shape=[tweet_max_length], dtype='int32')
input_aspect = Input(shape=[aspect_max_length], dtype='int32')
# Embeddings
tweets_emb = embeddings_layer(max_length=tweet_max_length, embeddings=wv,
trainable=trainable, masking=True)(
input_tweet)
tweets_emb = GaussianNoise(noise)(tweets_emb)
tweets_emb = Dropout(drop_text_input)(tweets_emb)
aspects_emb = embeddings_layer(max_length=aspect_max_length, embeddings=wv,
trainable=trainable, masking=True)(
input_aspect)
aspects_emb = GaussianNoise(noise)(aspects_emb)
# Recurrent NN
h_tweets = shared_RNN(tweets_emb)
h_tweets = Dropout(drop_text_rnn)(h_tweets)
h_aspects = shared_RNN(aspects_emb)
h_aspects = Dropout(drop_target_rnn)(h_aspects)
h_aspects = MeanOverTime()(h_aspects)
h_aspects = RepeatVector(tweet_max_length)(h_aspects)
# Merge of Aspect + Tweet
representation = concatenate([h_tweets, h_aspects])
# apply attention over the hidden outputs of the RNN's
att_layer = AttentionWithContext if attention == "context" else Attention
representation = att_layer()(representation)
representation = Dropout(drop_rep)(representation)
if use_final:
if final_type == "maxout":
representation = MaxoutDense(final_size)(representation)
else:
representation = Dense(final_size, activation=final_type)(
representation)
representation = Dropout(drop_final)(representation)
######################################################
# Probabilities
######################################################
probabilities = Dense(1 if classes == 2 else classes,
activation="sigmoid" if classes == 2 else "softmax",
activity_regularizer=l2(activity_l2))(representation)
model = Model(input=[input_aspect, input_tweet], output=probabilities)
loss = "binary_crossentropy" if classes == 2 else "categorical_crossentropy"
model.compile(optimizer=Adam(clipnorm=clipnorm, lr=lr), loss=loss)
return model
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
chars = '0123456789'
ctable = CharacterTable(chars, MAXLEN)
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
# note: in a situation where your input sequences have a variable length,
# use input_shape=(None, nb_feature).
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
# For the decoder's input, we repeat the encoded input for each time step
model.add(RepeatVector(DIGITS))
# The decoder RNN could be multiple layers stacked or a single layer
for _ in range(LAYERS):
model.add(RNN(HIDDEN_SIZE, return_sequences=True))
# For each of step of the output sequence, decide which character should be chosen
model.add(TimeDistributed(Dense(len(chars))))
model.add(Activation('softmax'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print (model.summary())
print (X_train[0])
print (y_train[0])
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X,
Y,
test_size=0.2,
random_state=42)
EMBED_SIZE = 128
HIDDEN_SIZE = 64
BATCH_SIZE = 32
NUM_EPOCHS = 1
# GRU
model = Sequential()
model.add(Embedding(s_vocabsize, EMBED_SIZE, input_length=MAX_SEQLEN))
model.add(Dropout(0.2))
model.add(GRU(HIDDEN_SIZE, dropout=0.2, recurrent_dropout=0.2))
model.add(RepeatVector(MAX_SEQLEN))
model.add(GRU(HIDDEN_SIZE, return_sequences=True))
model.add(TimeDistributed(Dense(t_vocabsize)))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
model.fit(Xtrain,
Ytrain,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=[Xtest, Ytest])
score, acc = model.evaluate(Xtest, Ytest, batch_size=BATCH_SIZE)
print("Test score: {:.3f}, accuracy: {:.3f}".format(score, acc))
# LSTM
input_img = Input((32, 100, 3))
# CNN
x = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2))(x)
x = Conv2D(64, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2))(x)
x = Flatten()(x)
# FC_layer
x = Dense(1024, activation='relu')(x)
x = RepeatVector(18)(x)
# RNN
x = LSTM(512, return_sequences=True)(x)
x = LSTM(512, return_sequences=True)(x)
output_word = TimeDistributed(Dense(64, activation='softmax'),
input_shape=(18, 64))(x)
recognizer = Model(input_img, output_word)
with open('./models/Reco_archi.json', 'w') as f:
f.write(recognizer.to_json())
print(recognizer.summary())
file_path = 'snopes.npy'
with open(file_path, 'rb') as f:
record = pickle.load(f)
emb_idx = get_emb_idx('glove.6B.100d.txt')
word_emb = create_emb_matrix(
vocabulary_size, emb_idx,
record['claim_text'] + record['evidence']) # + record['evidence_source'])
art_wrd = Input(shape=(100, ))
clm_wrd = Input(shape=(100, ))
clm_wrd_emb = Embedding(vocabulary_size,
100,
input_length=100,
weights=[word_emb],
trainable=False)(clm_wrd)
mean_clm_wrd_emb = RepeatVector(100)(mean(clm_wrd_emb, axis=-1))
art_wrd_emb = Embedding(vocabulary_size,
100,
input_length=100,
weights=[word_emb],
trainable=False)(art_wrd)
ip_to_dense = Concatenate(axis=-1)([mean_clm_wrd_emb, art_wrd_emb])
attn_weights = Dense(128, activation='tanh')(clm_wrd_emb)
attn_weights = Activation('softmax')(attn_weights)
model_attn = Model(inputs=[art_wrd, clm_wrd], outputs=attn_weights)
lstm_op = Bidirectional(LSTM(lstm_op_dim, return_sequences=True),
merge_mode='concat')(art_wrd_emb)
model_lstm = Model(inputs=art_wrd, outputs=lstm_op)
inner_pdt = Dot(axes=1)([model_attn.output, model_lstm.output])
def model_construct():
# CONFIG
config = ConfigParser()
config.read('./config.ini')
question_input = Input(shape=(config.getint('pre',
'question_maximum_length'), ),
dtype='int32',
name="question_input")
relation_all_input = Input(shape=(config.getint(
'pre', 'relation_word_maximum_length'), ),
dtype='int32',
name="relation_all_input")
relation_input = Input(shape=(config.getint('pre',
'relation_maximum_length'), ),
dtype='int32',
name="relation_input")
question_emd = np.load('./question_emd_matrix.npy')
relation_emd = np.load('./relation_emd_matrix.npy')
relation_all_emd = np.load('./relation_all_emd_matrix.npy')
question_emd = Embedding(question_emd.shape[0],
config.getint('pre', 'word_emd_length'),
weights=[question_emd],
input_length=config.getint(
'pre', 'question_maximum_length'),
trainable=False,
name="question_emd")(question_input)
sharedEmbd_r_w = Embedding(relation_all_emd.shape[0],
config.getint('pre', 'word_emd_length'),
weights=[relation_all_emd],
input_length=config.getint(
'pre', 'relation_word_maximum_length'),
trainable=False,
name="sharedEmbd_r_w")
relation_word_emd = sharedEmbd_r_w(relation_all_input)
sharedEmbd_r = Embedding(relation_emd.shape[0],
config.getint('pre', 'word_emd_length'),
weights=[relation_emd],
input_length=config.getint(
'pre', 'relation_maximum_length'),
trainable=True,
name="sharedEmbd_r")
relation_emd = sharedEmbd_r(relation_input)
bilstem_layer = Bidirectional(LSTM(units=200,
return_sequences=True,
implementation=2),
name="bilstem_layer")
question_bilstm_1 = bilstem_layer(question_emd)
# question_bilstm_2 = Bidirectional(LSTM(units=200, return_sequences=True, implementation=2),name="question_bilstm_2")(question_bilstm_1)
relation_word_bilstm = bilstem_layer(relation_word_emd)
relation_bilstm = bilstem_layer(relation_emd)
# question_res = Add()([question_bilstm_1, question_bilstm_2])
relation_con = Concatenate(axis=-2)(
[relation_word_bilstm, relation_bilstm])
relation_res = MaxPooling1D(400, padding='same')(relation_con)
relation_flatten = Flatten()(relation_res)
fc_layer1 = Dense(400, use_bias=True, activation='tanh')
fc_layer2 = Dense(1, use_bias=False, activation='softmax')
rel_expand = RepeatVector(30)(relation_flatten)
inputs = Concatenate()([question_bilstm_1, rel_expand])
weights = fc_layer2(fc_layer1(inputs))
question_att = MaxPooling1D(400, padding='same')(
Multiply()([question_bilstm_1, weights]))
# relation_maxpool = MaxPooling1D(400, padding='same')(relation_bilstm)
# relation_word_maxpool = MaxPooling1D(400, padding='same')(relation_word_bilstm)
# relation_res = Add()([relation_maxpool, relation_word_maxpool])
result = Dot(axes=-1, normalize=True)([question_att, relation_flatten])
model = Model(inputs=[
question_input,
relation_input,
relation_all_input,
],
outputs=result)
model.compile(optimizer=Adam(), loss=ranking_loss)
return model
# $$
#
# For safety, $y_0$ is defined as $\vec{0}$.
#
#
# In[7]:
# Define part of the attention layer gloablly so as to
# share the same layers for each attention step.
def softmax(x):
return K.softmax(x, axis=1)
at_repeat = RepeatVector(Tx)
at_concatenate = Concatenate(axis=-1)
at_dense1 = Dense(8, activation="tanh")
at_dense2 = Dense(1, activation="relu")
at_softmax = Activation(softmax, name='attention_weights')
at_dot = Dot(axes=1)
def one_step_of_attention(h_prev, a):
"""
Get the context.
Input:
h_prev - Previous hidden state of a RNN layer (m, n_h)
a - Input data, possibly processed (m, Tx, n_a)
X = X.reshape((1, X.shape[0], X.shape[1]))
y = y.reshape((1, y.shape[0], y.shape[1]))
return (X, y)
X, y = get_pair(5, 2, 50)
n_features = 50
n_timesteps_in = 5
n_timesteps_out = 2
# model architecture
model = Sequential()
model.add(LSTM(150, input_shape=(n_timesteps_in, n_features)))
model.add(RepeatVector(n_timesteps_in))
model.add(LSTM(150, return_sequences=True))
model.add(TimeDistributed(Dense(n_features, activation='softmax')))
# Summary
print(model.summary())
# compile
model.compile(loss='categorical_crossentropy',
optimizer='adam', metrics=['acc'])
for epoch in range(5000):
X, y = get_pair(n_timesteps_in, n_timesteps_out, n_features)
model.fit(X, y)
# Evaluate
count = 0
# ## Let's create the model
# In[48]:
embedding_size = 300
# Input dimension is 4096 since we will feed it the encoded version of the image.
# In[49]:
image_model = Sequential([
Dense(embedding_size, input_shape=(2048, ), activation='relu'),
RepeatVector(max_len)
])
# Since we are going to predict the next word using the previous words(length of previous words changes with every iteration over the caption), we have to set return_sequences = True.
# In[50]:
caption_model = Sequential([
Embedding(vocab_size, embedding_size, input_length=max_len),
LSTM(256, return_sequences=True),
TimeDistributed(Dense(300))
])
# Merging the models and creating a softmax classifier
# In[51]:
def build_model(self,
rnn_layer_sizes=np.array([20, 20, 20]),
dense_layer_sequential_sizes=np.array([32, 20]),
dropout_prob_rnn=0.3,
dropout_prob_dense=0.3,
lambda_reg_rnn=0.001,
lambda_reg_dense=0.001,
multiple_concatenate=False):
self.rnn_layer_sizes = rnn_layer_sizes
self.dense_layer_sequential_sizes = dense_layer_sequential_sizes
self.dropout_prob_rnn = dropout_prob_rnn
self.dropout_prob_dense = dropout_prob_dense
self.lambda_reg_rnn = lambda_reg_rnn
self.lambda_reg_dense = lambda_reg_dense
self.multiple_concatenate = multiple_concatenate
# define inputs
tracks_input = Input(self.input_shape_tracks,
dtype='float32',
name='tracks_input')
session_input = Input(self.input_shape_sessions,
dtype='float32',
name='session_input')
# Concatenate sessions and tracks
session_rep = RepeatVector(20)(session_input)
x_input = concatenate([tracks_input, session_rep], axis=-1)
# RNN part
x = LSTM(self.rnn_layer_sizes[0],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(x_input)
if self.multiple_concatenate:
for i in range(1, self.rnn_layer_sizes.size):
x = concatenate([x, session_rep], axis=-1)
x = LSTM(self.rnn_layer_sizes[i],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(x)
else:
for i in range(1, self.rnn_layer_sizes.size):
x = LSTM(self.rnn_layer_sizes[i],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(x)
x = BatchNormalization()(x)
x = Dropout(self.dropout_prob_rnn)(x)
for i in range(self.dense_layer_sequential_sizes.size - 1):
x = Dense(self.dense_layer_sequential_sizes[i],
activation='relu',
kernel_regularizer=l2(self.lambda_reg_dense))(x)
x = Dense(self.dense_layer_sequential_sizes[-1],
activation='linear',
kernel_regularizer=l2(self.lambda_reg_dense))(x)
output = Reshape(self.output_shape, name='output')(x)
# create model
self.model = K_Model(inputs=[tracks_input, session_input],
outputs=[output])
z = Lambda(sampling, name='LatentVector',
output_shape=(latent_dim, ))([z_mean, z_log_sigma])
#VAE Loss
def vae_loss(inputs, decoded):
xent_loss = K.sum(K.binary_crossentropy(inputs, decoded), axis=1)
kl_loss = -0.5 * K.sum(
1 + z_log_sigma - K.square(z_mean) - K.exp(z_log_sigma), axis=-1)
return K.mean(xent_loss + kl_loss)
#decoder LSTM
decoded = RepeatVector(7, name='EmbeddingtoTimeSeries')(z) #timesteps
decoded = LSTM(Intermediate_dim, name='DecoderLSTM1',
return_sequences=True)(decoded) #intermediate dimensions
decoded = LSTM(1, name='DecoderLSTM2',
return_sequences=True)(decoded) #input_dim
#decoded=TimeDistributed(Dense(1, name='Wrapper'), name='TimeDistributed')(decoded)
v_autoencoder = Model(inputs, decoded)
encoder = Model(inputs, z_mean)
#v_autoencoder.summary()
v_autoencoder.compile(optimizer=optimizer, loss=vae_loss)
v_autoencoder.fit(X, X, nb_epoch=nb_epoch, batch_size=batch_size)
def build_model(self,
rnn_layer_sizes=np.array([20, 20, 20]),
dense_layer_parallel_sizes=np.array([10, 20]),
dense_layer_sequential_sizes=np.array([32, 20]),
dropout_prob_rnn=0.3,
dropout_prob_dense=0.3,
lambda_reg_rnn=0.001,
lambda_reg_dense=0.001,
merge='multiply'):
self.rnn_layer_sizes = rnn_layer_sizes
self.dense_layer_parallel_sizes = dense_layer_parallel_sizes
self.dense_layer_sequential_sizes = dense_layer_sequential_sizes
self.dropout_prob_rnn = dropout_prob_rnn
self.dropout_prob_dense = dropout_prob_dense
self.lambda_reg_rnn = lambda_reg_rnn
self.lambda_reg_dense = lambda_reg_dense
self.merge = merge
if dense_layer_parallel_sizes[-1] != rnn_layer_sizes[-1]:
print(
'Dimensions of last layers of RNN and of parallel dense network must agree!'
)
return
# define inputs
tracks_input = Input(self.input_shape_tracks,
dtype='float32',
name='tracks_input')
session_input = Input(self.input_shape_sessions,
dtype='float32',
name='session_input')
# RNN side
x_rnn = LSTM(self.rnn_layer_sizes[0],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(tracks_input)
for i in range(1, self.rnn_layer_sizes.size):
x_rnn = LSTM(self.rnn_layer_sizes[i],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(x_rnn)
x_rnn = BatchNormalization()(x_rnn)
out_rnn = Dropout(self.dropout_prob_rnn)(x_rnn)
# dense side
x_fc = Dense(self.dense_layer_parallel_sizes[0],
activation='relu',
kernel_regularizer=l2(
self.lambda_reg_dense))(session_input)
for i in range(1, self.dense_layer_parallel_sizes.size):
x_fc = Dense(self.dense_layer_parallel_sizes[i],
activation='relu',
kernel_regularizer=l2(self.lambda_reg_dense))(x_fc)
x_fc = BatchNormalization()(x_fc)
out_fc = Dropout(self.dropout_prob_dense)(x_fc)
x = []
# merge RNN and dense side
if self.merge == 'multiply':
x = multiply([out_rnn, out_fc])
elif self.merge == 'add':
out_fc = RepeatVector(20)(out_fc)
x = add([out_rnn, out_fc])
elif self.merge == 'concatenate':
out_fc = RepeatVector(20)(out_fc)
x = concatenate([out_rnn, out_fc], axis=-1)
elif self.merge == 'maximum':
out_fc = RepeatVector(20)(out_fc)
x = maximum([out_rnn, out_fc])
else:
print(
'Choose proper merge variation: multiply, add, concatenate or maximum'
)
for i in range(self.dense_layer_sequential_sizes.size - 1):
x = Dense(self.dense_layer_sequential_sizes[i],
activation='relu',
kernel_regularizer=l2(self.lambda_reg_dense))(x)
x = Dense(self.dense_layer_sequential_sizes[-1],
activation='linear',
kernel_regularizer=l2(self.lambda_reg_dense))(x)
output = Reshape(self.output_shape, name='output')(x)
# create model and compile it
self.model = K_Model(inputs=[tracks_input, session_input],
outputs=[output])
def main(model=None):
X, y = prepare_data(TRAINING_SIZE)
print('Split data ... ')
# Shuffle (X, y)
#indices = np.arange(len(y))
#np.random.shuffle(indices)
#X = X[indices]
#y = y[indices]
# Explicitly set apart 10% for validation data that we never train over
split_at = len(X) - len(X) / 10
(X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
(y_train, y_val) = (y[:split_at], y[split_at:])
print(X_train.shape)
print(y_train.shape)
#print(X_train)
#print(y_train)
#return X_train, y_train
if model is None:
print('Build model...')
model = Sequential()
model.add(RNN(HIDDEN_SIZE, input_shape=(MAXLEN_q, MAXW_q)))
model.add(RepeatVector(MAXLEN_a))
for _ in range(LAYERS):
model.add(RNN(HIDDEN_SIZE, return_sequences=True))
model.add(TimeDistributed(Dense(MAXW_a)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
else:
print('Use existed model...')
# Train the model each generation and show predictions against the validation dataset
for iteration in range(1, 50):
print()
print('-' * 50)
print('Iteration', iteration)
model.fit(
X_train,
y_train,
batch_size=BATCH_SIZE,
nb_epoch=10, #validation_split=0.1,
validation_data=(X_val, y_val))
###
# Select samples from the validation set at random so we can visualize errors
for i in range(5):
ind = np.random.randint(0, len(X_val))
rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])]
preds = model.predict_classes(rowX, verbose=0)
#print(rowX)
#print(rowy)
#print(preds)
q = rowX[0]
correct = rowy[0]
guess = preds[0]
#print('Q', q)
print('T', correct)
#print('G', preds)
print(
colors.ok + '☑' +
colors.close if correct[0][guess[0]] > 0 else colors.fail +
'☒' + colors.close, guess)
print('---')
print('Save model ... ')
model.save('stk_rnn2.h5')
sequence_in = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9]) # rehape input into [samples, timesteps, features] timestep_in = len(sequence_in) sequence_in = sequence_in.reshape((1, timestep_in, 1)) # Prepare output sequence sequence_out = sequence_in[:, 1:, :] sequence_out timestep_out = timestep_in - 1 # define model model = Sequential() # Encoder model.add(LSTM(100, activation='relu', input_shape=(timestep_in, 1))) model.add(RepeatVector(timestep_out)) model.add(LSTM(100, activation='relu', return_sequences=True)) model.add(TimeDistributed(Dense(1))) model.compile(optimizer='adam', loss='mse') #plot_model(model, show_shapes=True, to_file='predict_lstm_autoencoder.png') # fit model model.fit(sequence_in, sequence_out, epochs=100) # demonstrate prediction yhat = model.predict(sequence_in) print(yhat) # Test on new data sequence_test1 = np.array([11, 12, 13, 14, 15, 16, 17, 18, 19])
def extreme(self):
print('Now operaring in Extreme.........')
#input channel 1
start = time.time()
input1 = Input(shape=(2, 1, 2, self.X_train.shape[2]))
conv1 = ConvLSTM2D(filters=32, kernel_size=(1, 2),
activation='relu')(input1)
#pool1 = MaxPooling1D(pool_size=2)(conv1)
# conv12 = Conv1D(filters=32,
# activation='relu',
# kernel_size=1)(pool1)
# pool12 =MaxPooling1D(pool_size=1)(conv12)
# flat1 = Flatten()(pool12)
# #input channel 2
# inputs2 =Input(shape=(self.X_train.shape[1],self.X_train.shape[2]))
# conv2= Conv1D(filters=96,
# kernel_size=3,
# activation='relu')(inputs2)
# pool2 = MaxPooling1D(pool_size=2)(conv2)
# conv22 = Conv1D(filters=32,
# activation='relu',
# kernel_size=1)(pool2)
# pool22 =MaxPooling1D(pool_size=1)(conv22)
# flat2 = Flatten()(pool22)
# #input channel 3
# inputs3 =Input(shape=(self.X_train.shape[1],self.X_train.shape[2]))
# conv3= Conv1D(filters=96,
# kernel_size=3,
# activation='relu')(inputs3)
# pool3 = MaxPooling1D(pool_size=2)(conv3)
# conv32 = Conv1D(filters=32,
# activation='relu',
# kernel_size=1)(pool3)
# pool32 =MaxPooling1D(pool_size=1)(conv32)
# flat3 = Flatten()(pool32)
# #merged
# merged =concatenate([flat1,flat2, flat3])
flat1 = Flatten()(conv1)
rv = RepeatVector(self.w)(flat1)
lstm1 = LSTM(128,
activation='relu',
return_sequences=True,
dropout=0.2)(rv, training=True)
lstm2 = LSTM(32, activation='relu', dropout=0.2)(lstm1)
dense1 = Dense(50)(lstm2)
out10 = Dense(1)(dense1)
out50 = Dense(1)(dense1)
out90 = Dense(1)(dense1)
model = Model([input1], [out10, out50, out90])
losses = [
lambda y, f: self.loss(self.q, y, f),
lambda y, f: self.loss(0.5, y, f),
lambda y, f: self.loss(1 - self.q, y, f)
]
#optimizer=BayesianOptimization()
model.compile(loss=losses,
optimizer='adam',
metrics=['mae'],
loss_weights=[0.2, 0.2, 0.2])
history = model.fit(
[self.X_train.reshape(-1, 2, 1, 2, self.X_train.shape[2])],
[self.y_train, self.y_train, self.y_train],
epochs=1,
batch_size=self.batch_size,
verbose=1,
shuffle=True)
self.modele, self.historye = model, history
ypred = np.array(
self.modele.predict(self.X_test.reshape(-1, 2, 1, 2,
self.X_train.shape[2]),
verbose=1)).reshape(3, -1)
self.yextreme = ypred[1]
self.extremet = time.time() - start
print('Execution time: {}'.format(self.extremet), end='\n')
def get_model_helper():
repeator = RepeatVector(Tx)
concatenator = Concatenate(axis=-1)
densor1 = Dense(10, activation="tanh")
densor2 = Dense(1, activation="relu")
activator = Activation(softmax,
name='attention_weights') # customed softmax
dotor = Dot(axes=1)
def one_step_attention(a, s_prev):
"""
Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights
"alphas" and the hidden states "a" of the Bi-LSTM.
Arguments:
a -- hidden state output of the Bi-LSTM, numpy-array of shape (m, Tx, 2*n_a)
s_prev -- previous hidden state of the (post-attention) LSTM, numpy-array of shape (m, n_s)
Returns:
context -- context vector, input of the next (post-attetion) LSTM cell
"""
# Use repeator to repeat s_prev to be of shape (m, Tx, n_s) so that you can concatenate it with all hidden states "a" (≈ 1 line)
s_prev = repeator(s_prev)
# Use concatenator to concatenate a and s_prev on the last axis (≈ 1 line)
concat = concatenator([a, s_prev])
# Use densor1 to propagate concat through a small fully-connected neural network to compute the "intermediate energies" variable e. (≈1 lines)
e = densor1(concat)
# Use densor2 to propagate e through a small fully-connected neural network to compute the "energies" variable energies. (≈1 lines)
energies = densor2(e)
# Use "activator" on "energies" to compute the attention weights "alphas" (≈ 1 line)
alphas = activator(energies)
# Use dotor together with "alphas" and "a" to compute the context vector to be given to the next (post-attention) LSTM-cell (≈ 1 line)
context = dotor([alphas, a])
return context
#define inference model for evaluation
def inference_model(vocab_inp_size, vocab_tar_size, Tx, Ty, n_s,
embedding_dim):
"""
Arguments:
Tx -- length of the input sequence
Ty -- length of the output sequence
embedding_dim -- embedding layer output size
n_s -- hidden state size of the post-attention LSTM
vocab_inp_size -- size of the python dictionary "vocab_inp_size"
vocab_tar_size -- size of the python dictionary "vocab_tar_size"
Returns:
inference_model -- Keras inference model instance
"""
# Define the inputs of your model with a shape (Tx, vocab_inp_size)
# Define s0 and c0, initial hidden state for the decoder LSTM of shape (n_s,)
X0 = Input(shape=(Tx, ), name='X')
# (m,Tx)
X = encoder_embedding(X0)
# (m,Tx,embedding_dim)
s0 = Input(shape=(n_s, ), name='s0')
c0 = Input(shape=(n_s, ), name='c0')
s = s0
c = c0
outputs = []
decoder_X0 = Input(shape=(1, ))
decoder_X = decoder_X0
#shape=(m, 1)
#shape is not (m, Ty) because we manually iterate Ty timesteps
#Define encoder as Bi-LSTM
encoder_output, forward_h, forward_c, backward_h, backward_c = encoder_layer(
X)
encoder_hidden = Concatenate(axis=-1)([forward_h, backward_h])
encoder_cell = Concatenate(axis=-1)([forward_c, backward_c])
s = encoder_hidden
c = encoder_cell
for t in range(Ty):
# one step of the attention mechanism to get back the context vector at step t
context = one_step_attention(encoder_output, encoder_hidden)
#(m, 1, n_s)
decoder_X = decoder_embedding(decoder_X)
#(m,1) - (m,1,embedding_dim)
decoder_inputs = enc_concat([decoder_X, context])
#shape--(m, 1, n_s+embedding_dim)
# Apply the post-attention LSTM cell to the "context" vector.
#initial_state = [hidden state, cell state]
decoder_X, s, c = decoder_cell(decoder_inputs,
initial_state=[s, c])
#decoder_X.shape--(m,n_s)
# Step 2.C: Apply Dense layer to the hidden state output of the decoder LSTM
decoder_X = output_layer(decoder_X)
#(m,vocab_tar_size)
out = decoder_X
#trick to add a dimension of 1 to tensor
decoder_X = RepeatVector(1)(decoder_X)
decoder_X = Lambda(lambda x: K.argmax(x))(decoder_X) #sampling
#shape--(m,1) so that it can fit embedding layer
outputs.append(out)
model = Model([X0, s0, c0, decoder_X0], outputs)
return model
inf_model = inference_model(ger_vocab_size, eng_vocab_size, ger_length,
eng_length, n_s, embedding_dim)
inf_model.compile(optimizer=optimizers.RMSprop(lr=0.001),
loss='categorical_crossentropy')
#there is issues serializing the model architecture, so only save weights
inf_model.load_weights('../models/inf_model_wts.h5')
return inf_model
def one_hot(x):
x = K.argmax(x)
x = tf.one_hot(x, 78)
x = RepeatVector(1)(x)
return x
def inference_model(vocab_inp_size, vocab_tar_size, Tx, Ty, n_s,
embedding_dim):
"""
Arguments:
Tx -- length of the input sequence
Ty -- length of the output sequence
embedding_dim -- embedding layer output size
n_s -- hidden state size of the post-attention LSTM
vocab_inp_size -- size of the python dictionary "vocab_inp_size"
vocab_tar_size -- size of the python dictionary "vocab_tar_size"
Returns:
inference_model -- Keras inference model instance
"""
# Define the inputs of your model with a shape (Tx, vocab_inp_size)
# Define s0 and c0, initial hidden state for the decoder LSTM of shape (n_s,)
X0 = Input(shape=(Tx, ), name='X')
# (m,Tx)
X = encoder_embedding(X0)
# (m,Tx,embedding_dim)
s0 = Input(shape=(n_s, ), name='s0')
c0 = Input(shape=(n_s, ), name='c0')
s = s0
c = c0
outputs = []
decoder_X0 = Input(shape=(1, ))
decoder_X = decoder_X0
#shape=(m, 1)
#shape is not (m, Ty) because we manually iterate Ty timesteps
#Define encoder as Bi-LSTM
encoder_output, forward_h, forward_c, backward_h, backward_c = encoder_layer(
X)
encoder_hidden = Concatenate(axis=-1)([forward_h, backward_h])
encoder_cell = Concatenate(axis=-1)([forward_c, backward_c])
s = encoder_hidden
c = encoder_cell
for t in range(Ty):
# one step of the attention mechanism to get back the context vector at step t
context = one_step_attention(encoder_output, encoder_hidden)
#(m, 1, n_s)
decoder_X = decoder_embedding(decoder_X)
#(m,1) - (m,1,embedding_dim)
decoder_inputs = enc_concat([decoder_X, context])
#shape--(m, 1, n_s+embedding_dim)
# Apply the post-attention LSTM cell to the "context" vector.
#initial_state = [hidden state, cell state]
decoder_X, s, c = decoder_cell(decoder_inputs,
initial_state=[s, c])
#decoder_X.shape--(m,n_s)
# Step 2.C: Apply Dense layer to the hidden state output of the decoder LSTM
decoder_X = output_layer(decoder_X)
#(m,vocab_tar_size)
out = decoder_X
#trick to add a dimension of 1 to tensor
decoder_X = RepeatVector(1)(decoder_X)
decoder_X = Lambda(lambda x: K.argmax(x))(decoder_X) #sampling
#shape--(m,1) so that it can fit embedding layer
outputs.append(out)
model = Model([X0, s0, c0, decoder_X0], outputs)
return model
from keras import backend as K
from keras.callbacks import LearningRateScheduler as LRS
import numpy as np
import random
import sys
import glob
import pickle
import re
WIDTH = 2029
MAXLEN = 31
INPUTLEN = 1000
inputs = Input(shape=(INPUTLEN, ))
#enc = Dense(2024, activation='linear')( inputs )
#enc = Dense(1024, activation='tanh')( enc )
repeat = RepeatVector(31)(inputs)
generated = Bi(GRU(256, return_sequences=True))(repeat)
generated = TD(Dense(2049, activation='relu'))(generated)
generated = TD(Dense(2049, activation='softmax'))(generated)
#generator = Model( inputs, generated )
#generated = Lambda( lambda x:x*2.0 )(generated)
generator = Model(inputs, generated)
generator.compile(optimizer=Adam(), loss='categorical_crossentropy')
def train_base():
for ge, name in enumerate(glob.glob('utils/dataset_*.pkl')[:1]):
dataset = pickle.loads(open(name, 'rb').read())
xs, ys = dataset
X.append(data[in_start:in_end, :])
y.append(data[in_end:out_end, 0])
# move along one time step
in_start += 1
# print(np.array(X).shape)
train_x, train_y = np.array(X), np.array(y)
(750, 10, 5)
# define parameters
epochs, batch_size = 50, 60
n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
# reshape output into [samples, timesteps, features]
train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))
# define model
model = Sequential()
model.add(LSTM(200, activation='relu', input_shape=(n_timesteps, n_features)))
model.add(RepeatVector(n_outputs))
model.add(LSTM(200, activation='relu', return_sequences=True))
model.add(TimeDistributed(Dense(100, activation='relu')))
model.add(TimeDistributed(Dense(1)))
model.compile(loss='mse', optimizer='adam')
# fit network
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size)
history = [x for x in train]
predictions = list()
for i in range(len(test)):
# flatten data
data = np.array(history)
data = data.reshape((data.shape[0]*data.shape[1], data.shape[2]))
# retrieve last observations for input data
input_x = data[-n_input:, :]
z_mean = Dense(latent_dim)(h)
z_log_var = Dense(latent_dim)(h)
def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(batch_size, latent_dim),
mean=0.,
stddev=epsilon_std)
return z_mean + K.exp(z_log_var / 2) * epsilon
# note that "output_shape" isn't necessary with the TensorFlow backend
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
# we instantiate these layers separately so as to reuse them later
repeated_context = RepeatVector(max_len)
decoder_h = LSTM(intermediate_dim,
return_sequences=True,
recurrent_dropout=0.2)
decoder_mean = Dense(
NB_WORDS, activation='linear'
) #softmax is applied in the seq2seqloss by tf #TimeDistributed()
h_decoded = decoder_h(repeated_context(z))
x_decoded_mean = decoder_mean(h_decoded)
# placeholder loss
def zero_loss(y_true, y_pred):
return K.zeros_like(y_pred)
trainX=encodedData[:round(len(encodedData)*0.9)]
testX=encodedData[round(len(encodedData)*0.9):]
trainY=oneHotLabels[:round(len(oneHotLabels)*0.9)]
testY=oneHotLabels[round(len(oneHotLabels)*0.9):]
print(trainX.shape)
print(testX.shape)
print(trainY.shape)
print(testY.shape)
exists = os.path.isfile("Task2Model.h5")
if not exists:
N_NERURONS=256
model= Sequential()
model.add(Embedding(len(encodedData),N_NERURONS,input_length=src_timesteps,mask_zero=True))
model.add(LSTM(N_NERURONS, input_shape=(trainY.shape[1],trainY.shape[2]),return_sequences=False))
model.add(RepeatVector(trainY.shape[1]))
model.add(LSTM(N_NERURONS//2, return_sequences=True))
model.add(TimeDistributed(Dense(TARGET_SIZE,activation="softmax")))
model.compile(optimizer="adam",loss="categorical_crossentropy",metrics=["accuracy"])
model.fit(trainX,trainY,validation_data=(testX,testY),epochs=30,batch_size=50)
model.save("Task2Model.h5")
model.save_weights("Task2Model_weights.h5")
else:
model = load_model("Task2Model.h5")
model.load_weights("Task2Model_weights.h5")
# SourceFileList = list(open("wwhitman-clean-processed.txt", "r"))
SourceFileList =list(open("poem.txt", "r"))
source_tokenizer = helper.create_tokenizer(SourceFileList)
source_encoded_sequence = helper.encode_sequences(source_tokenizer,trainX.shape[1],SourceFileList)
Xs.append(X.iloc[i:(i + time_steps)].values)
ys.append(y.iloc[i + time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[['Close']], train['Close'])
X_test, y_test = create_sequences(test[['Close']], test['Close'])
print(f'Training shape: {X_train.shape}')
print(f'Testing shape: {X_test.shape}')
model = Sequential()
model.add(LSTM(128, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(rate=0.2))
model.add(RepeatVector(X_train.shape[1]))
model.add(LSTM(128, return_sequences=True))
model.add(Dropout(rate=0.2))
model.add(TimeDistributed(Dense(X_train.shape[2])))
model.compile(optimizer='adam', loss='mae')
model.summary()
history = model.fit(X_train,
y_train,
epochs=100,
batch_size=32,
validation_split=0.1,
callbacks=[
keras.callbacks.EarlyStopping(monitor='val_loss',
patience=3,
mode='min')
print("Yoh.shape:", Yoh.shape)
index = 0
print("Source date:", dataset[index][0])
print("Target date:", dataset[index][1])
print()
print("Source after preprocessing (indices):", X[index])
print("Target after preprocessing (indices):", Y[index])
print()
print("Source after preprocessing (one-hot):", Xoh[index])
print("Target after preprocessing (one-hot):", Yoh[index])
# Defined shared layers as global variables
repeator = RepeatVector(Tx)
concatenator = Concatenate(axis=-1)
densor1 = Dense(10, activation = "tanh")
densor2 = Dense(1, activation = "relu")
activator = Activation(softmax, name='attention_weights') # We are using a custom softmax(axis = 1) loaded in this notebook
dotor = Dot(axes = 1)
# GRADED FUNCTION: one_step_attention
def one_step_attention(a, s_prev):
"""
Performs one step of attention: Outputs a context vector computed as a dot product of the attention weights
"alphas" and the hidden states "a" of the Bi-LSTM.
def make_model_1d(l2_lambda, clip_lenth, dimension):
input_holder = Input(shape=(clip_lenth, dimension))
x = Conv1D(filters=8,
kernel_size=15,
padding='same',
activation='relu',
input_shape=(clip_lenth, dimension),
kernel_regularizer=l2(l2_lambda))(input_holder)
x = BatchNormalization()(x)
x = Conv1D(filters=8,
kernel_size=10,
padding='same',
activation='relu',
input_shape=(clip_lenth, dimension),
kernel_regularizer=l2(l2_lambda))(x)
x = BatchNormalization()(x)
x = MaxPooling1D(2, padding='same')(x)
x = Conv1D(filters=16,
kernel_size=5,
padding='same',
activation='relu',
input_shape=(clip_lenth, dimension),
kernel_regularizer=l2(l2_lambda))(x)
x = BatchNormalization()(x)
x = Conv1D(filters=16,
kernel_size=5,
padding='same',
activation='relu',
input_shape=(clip_lenth, dimension),
kernel_regularizer=l2(l2_lambda))(x)
x = BatchNormalization()(x)
#####_______________________________________________________________________
u = GlobalMaxPooling1D()(x)
u_broadcast = RepeatVector(x.shape[1])(u)
def op(inputs):
x, y = inputs
return K.pow((x - y), 2)
Z = Lambda(op)([u_broadcast, x])
v = GlobalMaxPooling1D()(Z)
x = concatenate([u, v])
#####______________________________Multi task_________________________________________
# ['DP','BD','E','FS','A','RC']
number_of_N = 16
DP = 0.2
y1 = Dense(
number_of_N,
kernel_initializer='he_uniform',
# kernel_regularizer=l2(l2_lambda)
)(x)
y1 = BatchNormalization()(y1)
y1 = Activation('relu')(y1)
y1 = Dropout(DP)(
y1) # add some dropout for regularization after conv layers
y1 = Dense(1,
activation='sigmoid',
kernel_initializer='glorot_uniform',
name='DP'
# kernel_regularizer=l2(l2_lambda)
)(y1)
y2 = Dense(
number_of_N,
kernel_initializer='he_uniform',
# kernel_regularizer=l2(l2_lambda)
)(x)
y2 = BatchNormalization()(y2)
y2 = Activation('relu')(y2)
y2 = Dropout(DP)(
y2) # add some dropout for regularization after conv layers
y2 = Dense(1,
activation='sigmoid',
kernel_initializer='glorot_uniform',
name='BD'
# kernel_regularizer=l2(l2_lambda)
)(y2)
y3 = Dense(
number_of_N,
kernel_initializer='he_uniform',
# kernel_regularizer=l2(l2_lambda)
)(x)
y3 = BatchNormalization()(y3)
y3 = Activation('relu')(y3)
y3 = Dropout(DP)(
y3) # add some dropout for regularization after conv layers
y3 = Dense(1,
activation='sigmoid',
kernel_initializer='glorot_uniform',
name='E'
# kernel_regularizer=l2(l2_lambda)
)(y3)
y4 = Dense(
number_of_N,
kernel_initializer='he_uniform',
# kernel_regularizer=l2(l2_lambda)
)(x)
y4 = BatchNormalization()(y4)
y4 = Activation('relu')(y4)
y4 = Dropout(DP)(
y4) # add some dropout for regularization after conv layers
y4 = Dense(1,
activation='sigmoid',
kernel_initializer='glorot_uniform',
name='FS'
# kernel_regularizer=l2(l2_lambda)
)(y4)
y5 = Dense(
number_of_N,
kernel_initializer='he_uniform',
# kernel_regularizer=l2(l2_lambda)
)(x)
y5 = BatchNormalization()(y5)
y5 = Activation('relu')(y5)
y5 = Dropout(DP)(
y5) # add some dropout for regularization after conv layers
y5 = Dense(1,
activation='sigmoid',
kernel_initializer='glorot_uniform',
name='A'
# kernel_regularizer=l2(l2_lambda)
)(y5)
y6 = Dense(
number_of_N,
kernel_initializer='he_uniform',
# kernel_regularizer=l2(l2_lambda)
)(x)
y6 = BatchNormalization()(y6)
y6 = Activation('relu')(y6)
y6 = Dropout(DP)(
y6) # add some dropout for regularization after conv layers
y6 = Dense(1,
activation='sigmoid',
kernel_initializer='glorot_uniform',
name='RC'
# kernel_regularizer=l2(l2_lambda)
)(y6)
#####______________________________Multi task_________________________________________
model = Model(inputs=input_holder, outputs=[y1, y2, y3, y4, y5, y6])
# ['DP','BD','E','FS','A','RC']
losses = {
"DP": "mean_squared_error",
"BD": "mean_squared_error",
"E": "mean_squared_error",
"FS": "mean_squared_error",
"A": "mean_squared_error",
"RC": "mean_squared_error",
}
model.compile( #loss='mean_squared_error', # 'categorical_crossentropy' 'mean_squared_error' 'mean_absolute_percentage_error'
loss=losses,
optimizer='adam') # 'adadelta' 'rmsprop'
# model.summary()
return model
def create_lstm_vae(input_dim,
timesteps,
batch_size,
intermediate_dim,
latent_dim,
epsilon_std=1.):
"""
Creates an LSTM Variational Autoencoder (VAE). Returns VAE, Encoder, Generator.
# Arguments
input_dim: int.
timesteps: int, input timestep dimension.
batch_size: int.
intermediate_dim: int, output shape of LSTM.
latent_dim: int, latent z-layer shape.
epsilon_std: float, z-layer sigma.
# References
- [Building Autoencoders in Keras](https://blog.keras.io/building-autoencoders-in-keras.html)
- [Generating sentences from a continuous space](https://arxiv.org/abs/1511.06349)
"""
x = Input(shape=(
timesteps,
input_dim,
))
# LSTM encoding
h1 = LSTM(intermediate_dim + 20, return_sequences=True,
activation='relu')(x)
h = LSTM(intermediate_dim, activation='relu')(h1)
# VAE Z layer
z_mean = Dense(latent_dim, activation='relu')(h)
z_log_sigma = Dense(latent_dim, activation='relu')(h)
def sampling(args):
z_mean, z_log_sigma = args
epsilon = K.random_normal(shape=(batch_size, latent_dim),
mean=0.,
stddev=epsilon_std)
return Add()([z_mean, Multiply()([z_log_sigma, epsilon])])
# note that "output_shape" isn't necessary with the TensorFlow backend
# so you could write `Lambda(sampling)([z_mean, z_log_sigma])`
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_sigma])
# decoded LSTM layer
decoder_h = LSTM(intermediate_dim,
return_sequences=True,
activation='relu')
decoder_h1 = LSTM(intermediate_dim + 20,
return_sequences=True,
activation='relu')
decoder_mean = LSTM(input_dim, return_sequences=True, activation='tanh')
h_decoded = RepeatVector(timesteps)(z)
h_decoded = decoder_h(h_decoded)
# decoded layer
h1_decoded = decoder_h1(h_decoded)
x_decoded_mean = decoder_mean(h1_decoded)
# end-to-end autoencoder
vae = Model(x, x_decoded_mean)
# encoder, from inputs to latent space
encoder = Model(x, z_mean)
# generator, from latent space to reconstructed inputs
decoder_input = Input(shape=(latent_dim, ))
_h_decoded = RepeatVector(timesteps)(decoder_input)
_h_decoded = decoder_h(_h_decoded)
_h_decoded = decoder_h1(_h_decoded)
_x_decoded_mean = decoder_mean(_h_decoded)
generator = Model(decoder_input, _x_decoded_mean)
def vae_loss(x, x_decoded_mean):
xent_loss = keras.losses.logcosh(x, x_decoded_mean)
#xent_loss = objectives.mse(x, x_decoded_mean)
kl_loss = -0.5 * K.mean(1 + z_log_sigma - K.square(z_mean) -
K.exp(z_log_sigma))
loss = xent_loss + kl_loss
return loss
#sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
vae.compile(optimizer='rmsprop', loss=vae_loss)
# The standard was rmsprop
# generator.compile(optimizer='rmsprop', loss=vae_loss)
return vae, encoder, generator
print('X.shape = {}'.format(X.shape))
print('Xq.shape = {}'.format(Xq.shape))
print('Y.shape = {}'.format(Y.shape))
print('tX.shape = {}'.format(tX.shape))
print('tXq.shape = {}'.format(tXq.shape))
print('tY.shape = {}'.format(tY.shape))
print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))
print('Build model...')
print(vocab_size, vocab_answer_size)
sentrnn = Sequential()
sentrnn.add(Embedding(vocab_size, EMBED_HIDDEN_SIZE,
input_length=story_maxlen))
sentrnn.add(Dropout(0.3))
sentrnn.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False))
sentrnn.add(RepeatVector(story_maxlen))
qrnn = Sequential()
qrnn.add(Embedding(vocab_size, EMBED_HIDDEN_SIZE,
input_length=query_maxlen))
qrnn.add(Dropout(0.3))
qrnn.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False))
qrnn.add(RepeatVector(story_maxlen))
model = Sequential()
model.add(Merge([sentrnn, qrnn], mode='sum'))
model.add(RNN(EMBED_HIDDEN_SIZE, return_sequences=False))
model.add(Dropout(0.3))
model.add(Dense(vocab_answer_size, activation='softmax'))
# loaded_model = load_model('my_model.h5')
def repeat_vector(layer, layer_in, layerId, tensor=True):
out = {layerId: RepeatVector(layer['params']['n'])}
if tensor:
out[layerId] = out[layerId](*layer_in)
return out
# Shuffle (x, y) in unison as the later parts of x will almost all be larger
# digits.
indices = np.arange(len(y))
np.random.shuffle(indices)
x = x[indices]
y = y[indices]
# Explicitly set apart 10% for validation data that we never train over.
split_at = len(x) - len(x) // 10
(x_train, x_val) = x[:split_at], x[split_at:]
(y_train, y_val) = y[:split_at], y[split_at:]
model = Sequential()
model.add(LSTM(config.hidden_size, input_shape=(maxlen, len(chars))))
model.add(RepeatVector(config.digits + 1))
model.add(LSTM(config.hidden_size, return_sequences=True))
model.add(TimeDistributed(Dense(len(chars), activation='softmax')))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
model.fit(x_train,
y_train,
batch_size=config.batch_size,
epochs=100,
validation_data=(x_val, y_val),
callbacks=[WandbCallback(), log_table_callback])
# Train the model each generation and show predictions against the validation
# dataset.
n_features = timeseries.shape[1]
timeseries
timesteps = 3
X, y = temporalize(X = timeseries, y = np.zeros(len(timeseries)), lookback = timesteps)
n_features = 2
X = np.array(X)
X = X.reshape(X.shape[0], timesteps, n_features)
# define model
model = Sequential()
model.add(LSTM(128, activation='relu', input_shape=(timesteps,n_features), return_sequences=True))
model.add(LSTM(128, activation='relu', return_sequences=True))
model.add(LSTM(64, activation='relu', return_sequences=False))
model.add(RepeatVector(timesteps))
model.add(LSTM(64, activation='relu', return_sequences=True))
model.add(LSTM(64, activation='relu', return_sequences=True))
model.add(LSTM(128, activation='relu', return_sequences=True))
model.add(TimeDistributed(Dense(n_features)))
model.compile(optimizer='adam', loss='mse')
model.summary()
# fit model
model.fit(X, X, epochs=300, batch_size=5, verbose=0)
# demonstrate reconstruction
yhat = model.predict(X, verbose=0)
print('---Predicted---')
print(np.round(yhat,3))
print('---Actual---')
print(np.round(X, 3))
