number_of_features = 1000
# Load data and target vector from movie review data
(data_train,
target_train), (data_test,
target_test) = imdb.load_data(num_words=number_of_features)
# Use padding or truncation to make each observation have 400 features
features_train = sequence.pad_sequences(data_train, maxlen=400)
features_test = sequence.pad_sequences(data_test, maxlen=400)
# Start neural network
network = models.Sequential()
# Add an embedding layer
network.add(layers.Embedding(input_dim=number_of_features, output_dim=128))
# Add a long short-term memory layer with 128 units
network.add(layers.LSTM(units=128))
# Add fully connected layer with a sigmoid activation function
network.add(layers.Dense(units=1, activation="sigmoid"))
# Compile neural network
network.compile(
loss="binary_crossentropy", # Cross-entropy
optimizer="Adam", # Adam optimization
metrics=["accuracy"]) # Accuracy performance metric
# Train neural network
history = network.fit(
model = Sequential()
model.add(Embedding(max_features, 32))
model.add(LSTM(32)) # other params: default values are all right.
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
history = model.fit(
input_train, y_train, epochs=10, batch_size=128, validation_split=0.2
) # 20% train data is used to validation. refer to P171 or cloud note
# if para shuffle is used in fit() with validation_split. validation_split first, shuffle later.P171
# build network model with bio-direction RNN
from keras import layers
bio_direction_model = Sequential()
bio_direction_model.add(layers.Embedding(max_features, 32))
bio_direction_model.add(layers.Bidirectional(layers.LSTM(32)))
bio_direction_model.add(layers.Dense(1, activation='sigmoid'))
bio_direction_model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc'])
bio_direction_history = bio_direction_model.fit(input_train,
y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
# plot
import matplotlib.pyplot as plt
story_maxlen = max(map(len, (x for x, _, _ in train + test)))
query_maxlen = max(map(len, (x for _, x, _ in train + test)))
x, xq, y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)
tx, txq, ty = vectorize_stories(test, word_idx, story_maxlen, query_maxlen)
print('vocab = {}'.format(vocab))
print('x.shape = {}'.format(x.shape))
print('xq.shape = {}'.format(xq.shape))
print('y.shape = {}'.format(y.shape))
print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))
print('Build model...')
sentence = layers.Input(shape=(story_maxlen, ), dtype='int32')
encoded_sentence = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(sentence)
encoded_sentence = RNN(SENT_HIDDEN_SIZE)(encoded_sentence)
question = layers.Input(shape=(query_maxlen, ), dtype='int32')
encoded_question = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(question)
encoded_question = RNN(QUERY_HIDDEN_SIZE)(encoded_question)
merged = layers.concatenate([encoded_sentence, encoded_question])
preds = layers.Dense(vocab_size, activation='softmax')(merged)
model = Model([sentence, question], preds)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print('Training')
max_features = 10000 # number of words to consider as features
max_len = 500 # cut texts after this number of words (among top max_features most common words)
print('Loading data...')
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
from keras.models import Sequential
from keras import layers
from keras.optimizers import RMSprop
model = Sequential()
model.add(layers.Embedding(max_features, 128, input_length=max_len))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer=RMSprop(lr=1e-4),
loss='binary_crossentropy',
metrics=['acc'])
history = model.fit(x_train, y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
import matplotlib.pyplot as plt
acc = history.history['acc']
tokenizer = Tokenizer(num_words=5000)
tokenizer.fit_on_texts(x_train)
X_train = tokenizer.texts_to_sequences(x_train)
X_test = tokenizer.texts_to_sequences(x_test)
vocab_size = len(tokenizer.word_index) + 1
# maxlen = 470420
maxlen = 2543
X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)
X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)
# because spacy largest model uses 300
embedding_dim = 300
model = Sequential()
model.add(layers.Embedding(input_dim=vocab_size, output_dim=embedding_dim, input_length=maxlen))
model.add(layers.Flatten())
model.add(layers.Dense(10, activation='relu'))
# model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(layers.Dense(20, activation='softmax')) #20 classes
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print(model.summary())
history = model.fit(X_train, y_train,
epochs=20,
verbose=False,
validation_data=(X_test, y_test),
batch_size=16)
embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
if i < max_words:
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
# 模型定义
category_num = 1258
conv_filter_size = 128
dense_hidden_size = 4096
# Inputs
text_input = Input(shape=(maxlen,))
# Embeddings layers
embedded_text = layers.Embedding(max_words, embedding_dim, weights=[embedding_matrix])(text_input)
# Conv layers
convs = []
filter_sizes = [1, 2, 3, 4]
for fsz in filter_sizes:
conv_1 = layers.Conv1D(filters=conv_filter_size, kernel_size=fsz, activation='relu')(embedded_text)
batchNorm_1 = layers.BatchNormalization()(conv_1)
conv_2 = layers.Conv1D(filters=conv_filter_size, kernel_size=fsz, activation='relu')(batchNorm_1)
batchNorm_2 = layers.BatchNormalization()(conv_2)
globalMaxPool = layers.GlobalMaxPooling1D()(batchNorm_2)
convs.append(globalMaxPool)
merge = layers.concatenate(convs, axis=-1)
# Classifier
dense_1 = layers.Dense(dense_hidden_size, activation='relu')(merge)
tokenizer = Tokenizer(num_words=2000)
tokenizer.fit_on_texts(sentences)
#getting the vocabulary of data
sentences = tokenizer.texts_to_sequences(sentences)
padded_docs= pad_sequences(sentences,maxlen=max_review_len)
le = preprocessing.LabelEncoder()
y = le.fit_transform(y)
X_train, X_test, y_train, y_test = train_test_split(padded_docs[:10000], y[:10000], test_size=0.25, random_state=1000)
# Number of features
# print(input_dim)
vocab_size = 2000
model = Sequential()
model.add(layers.Embedding(vocab_size, 50, input_length=max_review_len))
model.add(layers.Flatten())
model.add(layers.Dense(300, activation='relu'))
model.add(layers.Dense(3, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',optimizer='adam',metrics=['acc'])
history=model.fit(X_train,y_train, epochs=2, verbose=True, validation_data=(X_test,y_test), batch_size=256)
[test_loss, test_acc] = model.evaluate(X_test, y_test)
print("Evaluation result on Test Data : Loss = {}, accuracy = {}".format(test_loss, test_acc))
print("Evaluation result on Test Data for model 2 : Loss = {}, accuracy = {}".format(test_loss, test_acc))
prediction = np.argmax(model.predict(X_test[[0], ]))
print("Prediction: ", prediction)
# Part 3 of ICP10
# Most probable label
best_label = labels[predicted.argmax()]
return best_label
#%%
predict("Which car is faster?"), predict(texts_test[9])
#%%
# Model 2 - Graph
#
input_tensor = Input(shape=(MAX_LENGTH,))
embedding_layer = layers.Embedding(MAX_WORDS, EMBEDDING_DIM)
embedding_layer.trainable = EMBEDDING_TRAIN
x = embedding_layer(input_tensor)
#x = layers.Dropout(0.5)(x)
# Branch a
a = layers.Conv1D(CNN_FILTERS_L1, CNN_LENGTH_1, activation='relu', padding='same')(x)
a = layers.GlobalMaxPool1D()(a)
# Branch b
b = layers.Conv1D(CNN_FILTERS_L1, CNN_LENGTH_2, activation='relu', padding='same')(x)
b = layers.GlobalMaxPool1D()(b)
# Branch c
c = layers.Conv1D(CNN_FILTERS_L1, CNN_LENGTH_3, activation='relu', padding='same')(x)
c = layers.GlobalMaxPool1D()(c)
# Concatenate a and b and c
y = layers.concatenate([a,b,c], axis=-1)
from keras import layers
from keras import optimizers
if __name__ == '__main__':
args = argparse.ArgumentParser()
args.add_argument('--strmaxlen', type=int, default=150)
args.add_argument('--epochs', type=int, default=100)
args.add_argument('--batch', type=int, default=10)
args.add_argument('--embedding', type=int, default=256)
args.add_argument('--featuresize', type=int,
default=129) # ascii code 기준 0~127 + 1
config = args.parse_args()
inputs = layers.Input((config.strmaxlen, ))
layer = layers.Embedding(config.featuresize,
config.embedding,
input_length=config.strmaxlen,
mask_zero=True)(inputs)
layer = layers.Bidirectional(layers.GRU(256, return_sequences=True))(layer)
layer = layers.Bidirectional(layers.GRU(256,
return_sequences=False))(layer)
layer_dense = layers.Dense(3)(layer)
outputs_softmax = layers.Activation('softmax')(layer_dense)
model = models.Model(inputs=inputs, outputs=outputs_softmax)
model.summary()
model.compile(optimizer=optimizers.Adam(lr=0.001, amsgrad=True),
loss='binary_crossentropy',
metrics=['accuracy'])
file_train_instances = "sample50.csv" #데이터셋 파일 이름
if word in embeddings_dict:
# If the words are in the embeddings, we fill them with a value
embedding_matrix[word_idx[word]] = embeddings_dict[word]
print('Shape of embedding matrix:', embedding_matrix.shape)
print('Embedding of table', embedding_matrix[word_idx['table']])
print('Embedding of the padding symbol, idx 0, random numbers',
embedding_matrix[0])
# If model has not yet been trained and saved
if not models.load_model('ModelForNameEntityRecognition'):
# We build the model
model = models.Sequential()
model.add(
layers.Embedding(len(vocabulary_words) + 2,
EMBEDDING_DIM,
mask_zero=True,
input_length=None))
model.layers[0].set_weights([embedding_matrix])
# The default is True
model.layers[
0].trainable = False # Should be false according to Chollet p.191 - Change from true as in Pierre's ex.
model.add(SimpleRNN(100, return_sequences=True))
model.add(layers.Dropout(0.5))
# model.add(Bidirectional(SimpleRNN(100, return_sequences=True)))
# model.add(Bidirectional(LSTM(100, return_sequences=True)))
model.add(Dense(NB_CLASSES + 2, activation='softmax'))
# We fit the model
model.compile(loss='categorical_crossentropy',
optimizer=OPTIMIZER,
metrics=['acc'])
from keras import Input, layers, Model
import numpy as np
vocabulary_size = 50000
num_income_groups = 10
post_inputs = Input(shape=(None, ), dtype='int32', name='posts')
embedded_posts = layers.Embedding(256, vocabulary_size)(post_inputs)
x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(128, activation='relu')(x)
age_prediction = layers.Dense(1, name='age')(x)
income_prediction = layers.Dense(num_income_groups,
activation='softmax',
name='income')(x)
gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)
model = Model(post_inputs,
[age_prediction, income_prediction, gender_prediction])
model.compile(optimizer='rmsprop',
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'],
loss_weights=[-0.25, 1., 10.]) # weights need to be balanced
#Load data as lists of integers.
(train_samples, train_labels), (test_samples,
test_labels) = imdb.load_data(num_words=tokens)
#Reshape lists of integers to 2Dtensor of shape (samples,seq_length)
train_samples = preprocessing.sequence.pad_sequences(train_samples,
maxlen=seq_length)
test_samples = preprocessing.sequence.pad_sequences(test_samples,
maxlen=seq_length)
#Start.
model = models.Sequential()
#Embedding layer.
model.add(layers.Embedding(tokens, 8, input_length=seq_length))
#Network architecture.
model.add(layers.Flatten())
model.add(layers.Dense(1, activation='sigmoid'))
#Compilation.
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
#Summary.
model.summary()
#Training.
history = model.fit(train_samples,
train_labels,
epochs=10,
class decoder:
# [batch_size, IMG_EMBED_SIZE] of CNN image features
img_embeds = tf.placeholder('float32', [None, IMG_EMBED_SIZE])
# [batch_size, time steps] of word ids
sentences = tf.placeholder('int32', [None, None])
# we use bottleneck here to reduce the number of parameters
# image embedding -> bottleneck
img_embed_to_bottleneck = L.Dense(IMG_EMBED_BOTTLENECK,
input_shape=(None, IMG_EMBED_SIZE),
activation='elu')
# image embedding bottleneck -> lstm initial state
img_embed_bottleneck_to_h0 = L.Dense(LSTM_UNITS,
input_shape=(None,
IMG_EMBED_BOTTLENECK),
activation='elu')
# word -> embedding
word_embed = L.Embedding(len(vocab), WORD_EMBED_SIZE)
# lstm cell (from tensorflow)
lstm = tf.nn.rnn_cell.LSTMCell(LSTM_UNITS)
# we use bottleneck here to reduce model complexity
# lstm output -> logits bottleneck
token_logits_bottleneck = L.Dense(LOGIT_BOTTLENECK, activation="elu")
# logits bottleneck -> logits for next token prediction
token_logits = L.Dense(len(vocab))
# initial lstm cell state of shape (None, LSTM_UNITS),
# we need to condition it on `img_embeds` placeholder.
c0 = h0 = img_embed_bottleneck_to_h0(img_embed_to_bottleneck(img_embeds))
# embed all tokens but the last for lstm input,
# remember that L.Embedding is callable,
# use `sentences` placeholder as input.
word_embeds = word_embed(sentences[:, :-1])
# during training we use ground truth tokens `word_embeds` as context for next token prediction.
# that means that we know all the inputs for our lstm and can get
# all the hidden states with one tensorflow operation (tf.nn.dynamic_rnn).
# `hidden_states` has a shape of [batch_size, time steps, LSTM_UNITS].
hidden_states, _ = tf.nn.dynamic_rnn(
lstm, word_embeds, initial_state=tf.nn.rnn_cell.LSTMStateTuple(c0, h0))
# now we need to calculate token logits for all the hidden states
# first, we reshape `hidden_states` to [-1, LSTM_UNITS]
flat_hidden_states = tf.reshape(hidden_states, [-1, LSTM_UNITS])
# then, we calculate logits for next tokens using `token_logits` layer
flat_token_logits = token_logits(
token_logits_bottleneck(flat_hidden_states))
# then, we flatten the ground truth token ids.
# remember, that we predict next tokens for each time step,
# use `sentences` placeholder.
flat_ground_truth = tf.reshape(sentences[:, 1:], [-1])
# we need to know where we have real tokens (not padding) in `flat_ground_truth`,
# we don't want to propagate the loss for padded output tokens,
# fill `flat_loss_mask` with 1.0 for real tokens (not vocab[PAD]) and 0.0 otherwise.
flat_loss_mask = tf.cast(tf.not_equal(flat_ground_truth, pad_idx),
'float32')
# compute cross-entropy between `flat_ground_truth` and `flat_token_logits` predicted by lstm
xent = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=flat_ground_truth, logits=flat_token_logits)
# compute average `xent` over tokens with nonzero `flat_loss_mask`.
# we don't want to account misclassification of PAD tokens, because that doesn't make sense,
# we have PAD tokens for batching purposes only!
loss = tf.reduce_sum(xent * flat_loss_mask) / tf.reduce_sum(flat_loss_mask)
def genModel(self, ):
cat_vars_ind = False
cont_vars_ind = False
# INPUTS (TENSORS & DATA)
self.input_tensors = []
self.input_data_reform = []
# CONTINOUS FEATURES AS INPUT
if (self.df.shape[1] > len(self.cat_vars)):
cont_vars_ind = True
self.cont_df = self.df.drop(self.cat_vars,
axis=1).values.astype('float32')
self.cont_input = layers.Input(shape=(self.cont_df.shape[1], ))
if (self.bn_start):
cont = layers.BatchNormalization()(self.cont_input)
cont = layers.Dropout(self.cont_input_drop)(cont)
else:
cont = layers.Dropout(self.cont_input_drop)(self.cont_input)
self.input_tensors.append(self.cont_input)
self.input_data_reform.append(self.cont_df)
# CATEGORICAL FEATURES AS SEPERATE ARRAYS
self.cat_values = []
self.cat_df = self.df[self.cat_vars]
for n, c in self.cat_df.items():
self.cat_values.append(c.values.astype('float32'))
# CREATE EMBEDDING LAYERS
self.embeddings = []
self.emb_inputs = []
self.emb_models = []
if (len(self.cat_vars) > 0): #check we have cat vars
cat_vars_ind = True
for dict_sz, out_sz in self.emb_szs:
input_layer_i = layers.Input(shape=(1, ))
emb_layer_i = layers.Embedding(dict_sz, out_sz,
input_length=1)(input_layer_i)
self.emb_inputs.append(input_layer_i)
self.embeddings.append(emb_layer_i)
self.emb_models.append(
keras.Model(inputs=input_layer_i, outputs=emb_layer_i))
ent_emb = keras.layers.Concatenate(
name=self.prefix + 'Embedding_layer')(self.embeddings)
ent_emb = layers.Flatten()(ent_emb)
ent_emb = layers.Dropout(self.emb_drop)(ent_emb)
self.input_tensors += self.emb_inputs
self.input_data_reform += self.cat_values
# self.input_tensors = [self.cont_input]+self.emb_inputs
# self.input_data_reform = [self.cont_df]+self.cat_values
# CONCATENATE CONTINOUS AND EMBEDDED CATEGORICAL
if (cat_vars_ind and cont_vars_ind):
self.joint_tensor = keras.layers.Concatenate(
name=self.prefix + 'All_features')([ent_emb, cont])
elif (cat_vars_ind):
self.joint_tensor = ent_emb
else:
self.joint_tensor = cont
# DENSE HIDDEN LAYERS
self.output_tensor, self.mid_output_tensors = self.simpleDNN(
self.joint_tensor)
# DEFINE AND COMPILE MODEL
self.defineAndCompile()
X_train = tokenizer.texts_to_sequences(training_tweets)
X_test = tokenizer.texts_to_sequences(test_tweets)
y_train = [1] * 4000 + [0] * 4000
y_test = [1] * 1000 + [0] * 1000
vocab_size = len(tokenizer.word_index) + 1
maxlen = 100
X_train = pad_sequences(X_train, padding='post', maxlen=maxlen)
X_test = pad_sequences(X_test, padding='post', maxlen=maxlen)
embedding_dim = 100
model = Sequential()
model.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen))
model.add(layers.Conv1D(128, 5, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(10, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
history = model.fit(X_train,
y_train,
epochs=10,
verbose=False,
batch_size=10,
validation_split=0.1)
class EvaluateAccuracy(keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
sys.stdout.flush()
print("\nMeasuring validation accuracy...")
acc = compute_test_accuracy(self.model)
print("\nValidation accuracy: %.5f\n" % acc)
sys.stdout.flush()
model = keras.models.Sequential()
model = keras.models.Sequential()
model.add(L.InputLayer([None], dtype='int32'))
model.add(L.Embedding(len(all_words), 50))
model.add(L.TimeDistributed(L.Dense(96, activation='tanh')))
model.add(L.Dropout(0.25))
model.add(L.TimeDistributed(L.Dense(96, activation='tanh')))
model.add(L.Dropout(0.25))
#model.add(L.Conv1D(32,3,padding='same',activation='tanh'))
model.add(
L.Bidirectional(
L.GRU(128,
return_sequences=True,
activation='tanh',
recurrent_dropout=0.2,
dropout=0.2)))
model.add(L.TimeDistributed(L.Dense(128, activation='tanh')))
model.add(L.Dropout(0.25))
def build_model(char_size=27, dim=64, iterations=4, training=True, pca=False):
"""Build the model."""
# Inputs
# Context: (rules, preds, chars,)
context = L.Input(shape=(
None,
None,
None,
),
name='context',
dtype='int32')
query = L.Input(shape=(None, ), name='query', dtype='int32')
# Flatten preds to embed entire rules
var_flat = L.Lambda(lambda x: K.reshape(
x, K.stack([K.shape(x)[0], -1,
K.prod(K.shape(x)[2:])])),
name='var_flat')
flat_ctx = var_flat(context) # (?, rules, preds*chars)
# Onehot embedding
# Contextual embeddeding of symbols
onehot_weights = np.eye(char_size)
onehot_weights[0, 0] = 0 # Clear zero index
onehot = L.Embedding(char_size,
char_size,
trainable=False,
weights=[onehot_weights],
name='onehot')
embedded_ctx = onehot(flat_ctx) # (?, rules, preds*chars*char_size)
embedded_q = onehot(query) # (?, chars, char_size)
embed_pred = ZeroGRU(dim, go_backwards=True, name='embed_pred')
embedded_predq = embed_pred(embedded_q) # (?, dim)
# Embed every rule
embedded_rules = NestedTimeDist(embed_pred,
name='rule_embed')(embedded_ctx)
# (?, rules, dim)
# Reused layers over iterations
repeat_toctx = L.RepeatVector(K.shape(embedded_ctx)[1],
name='repeat_to_ctx')
diff_sq = L.Lambda(lambda xy: K.square(xy[0] - xy[1]),
output_shape=(None, dim),
name='diff_sq')
concat = L.Lambda(lambda xs: K.concatenate(xs, axis=2),
output_shape=(None, dim * 5),
name='concat')
att_dense1 = L.TimeDistributed(L.Dense(dim,
activation='tanh',
name='att_dense1'),
name='d_att_dense1')
att_dense2 = L.TimeDistributed(L.Dense(1,
activation='sigmoid',
name='att_dense2'),
name='d_att_dense2')
squeeze2 = L.Lambda(lambda x: K.squeeze(x, 2), name='sequeeze2')
# expand = L.Lambda(lambda x: K.expand_dims(x, axis=2), name='expand')
rule_mask = L.Lambda(lambda x: K.cast(
K.any(K.not_equal(x, 0), axis=-1, keepdims=True), 'float32'),
name='rule_mask')(embedded_rules)
episodic_mem = EpisodicMemory(dim, name='episodic_mem')
# Reasoning iterations
state = embedded_predq
repeated_q = repeat_toctx(embedded_predq)
outs = list()
for _ in range(iterations):
# Compute attention between rule and query state
ctx_state = repeat_toctx(state) # (?, rules, dim)
s_s_c = diff_sq([ctx_state, embedded_rules])
s_m_c = L.multiply([embedded_rules, state]) # (?, rules, dim)
sim_vec = concat([s_s_c, s_m_c, ctx_state, embedded_rules, repeated_q])
sim_vec = att_dense1(sim_vec) # (?, rules, dim)
sim_vec = att_dense2(sim_vec) # (?, rules, 1)
# sim_vec = squeeze2(sim_vec) # (?, rules)
# sim_vec = L.Softmax(axis=1)(sim_vec)
# sim_vec = expand(sim_vec) # (?, rules, 1)
sim_vec = L.multiply([sim_vec, rule_mask])
state = episodic_mem([state, sim_vec, embedded_rules])
sim_vec = squeeze2(sim_vec) # (?, rules)
outs.append(sim_vec)
# Predication
out = L.Dense(1, activation='sigmoid', name='out')(state)
if pca:
model = Model([context, query], [embedded_rules])
elif training:
model = Model([context, query], [out])
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['acc'])
else:
model = Model([context, query], outs + [out])
return model
def model(reader):
inshape = (None, )
known_in = L.Input(shape=inshape, name='known_input')
unknown_in = L.Input(shape=inshape, name='unknown_input')
embedding = L.Embedding(
len(reader.vocabulary_above_cutoff) + 2,
5,
input_length=reader.max_len,
name='embedding')
known_embed = embedding(known_in)
unknown_embed = embedding(unknown_in)
# Convolution 1
conv8_1 = L.Convolution1D(
filters=500,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8_1')
max_1_1 = L.MaxPooling1D()
conv8_1_known = max_1_1(conv8_1(known_embed))
conv8_1_unknown = max_1_1(conv8_1(unknown_embed))
# Convolution 2
conv8_2 = L.Convolution1D(
filters=250,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8_2')
max_2_2 = L.MaxPooling1D()
conv8_2_known = max_2_2(conv8_2(conv8_1_known))
conv8_2_unknown = max_2_2(conv8_2(conv8_1_unknown))
# Convolution 3
conv8_3 = L.Convolution1D(
filters=100,
kernel_size=8,
strides=1,
activation='relu',
name='convolutional_8_3')
# Glob Max Pool
repr_known2 = L.GlobalMaxPooling1D(name='known_repr_8')(
conv8_3(conv8_2_known))
repr_unknown2 = L.GlobalMaxPooling1D(name='unknown_repr_8')(
conv8_3(conv8_2_unknown))
abs_diff = L.merge(
inputs=[repr_known2, repr_unknown2],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
dense2 = L.Dense(500, activation='relu')(dense1)
dense3 = L.Dense(500, activation='relu')(dense2)
dense4 = L.Dense(500, activation='relu')(dense3)
pruned = L.Dropout(0.3)(dense4)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def biased_mean_squared_error(
y_true: np.ndarray, y_pred: np.ndarray, *, beta: float = 0.0
):
return K.mean(K.square(y_pred - y_true) * K.square(1 + (y_true * beta)), axis=-1)
K.clear_session()
word_embedding_input = layers.Input(
shape=(num_tokens_at_once, embedding_dim), name="w_emb"
)
entity_type_input = layers.Input(shape=(num_tokens_at_once,), name="entity_type_input")
event_type_input = layers.Input(shape=(1,), name="event_type_input")
entity_type_embedding = layers.Embedding(
num_entity_classes,
dim_ent,
name="ent_emb",
embeddings_initializer=ScaledRandomNormal(seed=seed_value, scale_factor=0.01),
embeddings_regularizer=regularizers.l2(l2_weight),
)(entity_type_input)
event_type_embedding = layers.Embedding(
num_event_classes,
hidden_dim,
name="evt_emb",
embeddings_initializer=ScaledRandomNormal(seed=seed_value, scale_factor=0.01),
embeddings_regularizer=regularizers.l2(l2_weight),
)(event_type_input)
concat = layers.concatenate([word_embedding_input, entity_type_embedding])
mask = layers.Masking(mask_value=MASK_VALUE_IGNORE_POSITION)(concat)
encoder = layers.LSTM(
hidden_dim,
kernel_regularizer=regularizers.l2(l2_weight),
from keras.datasets import imdb from keras.preprocessing import sequence # number of words to consider as features max_features = 2000 # cuts off texts after this number of words (among max_features most commom words) max_len = 500 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) x_train = sequence.pad_sequences(x_train, maxlen=max_len) x_test = sequence.pad_sequences(x_test, maxlen=max_len) model = keras.models.Sequential() model.add(layers.Embedding(max_features, 128, input_length=max_len, name='embed')) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.MaxPooling1D(5)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.GlobalMaxPooling1D()) model.add(layers.Dense(1)) model.summary() model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc']) # records activation histograms every 1 epoch # records embedding data every 1 epoch callbacks = [keras.callbacks.TensorBoard(log_dir='/home/valter/Python/my_log_dir', histogram_freq=1, embeddings_freq=1)] history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.2, callbacks=callbacks)
def main(i, RNN_TYPE):
# TODO: change to GRU, Recurrent, and SimpleRNN
# print(i, RNN_TYPE)
RNN = recurrent.LSTM
if RNN_TYPE == "gru":
# print("Starting a GRU: ")
RNN = recurrent.GRU
file_name = "history_gru_" + str(i) + ".csv"
elif RNN_TYPE == "recurrent":
# print("Starting a Recurrent Unit: ")
RNN = recurrent.Recurrent
file_name = "history_recurrent_" + str(i) + ".csv"
elif RNN_TYPE == "simplernn":
# print("Starting a SimpleRNN: ")
RNN = recurrent.SimpleRNN
file_name = "history_simple_" + str(i) + ".csv"
else:
# print("Starting an LSTM")
file_name = "history_lstm_" + str(i) + ".csv"
EMBED_HIDDEN_SIZE = 100
SENT_HIDDEN_SIZE = 100
QUERY_HIDDEN_SIZE = 100
BATCH_SIZE = 50
EPOCHS = 100
# print('RNN / Embed / Sent / Query = {}, {}, {}, {}'.format(RNN,
# EMBED_HIDDEN_SIZE,
# SENT_HIDDEN_SIZE,
# QUERY_HIDDEN_SIZE))
# print(os.getcwd())
file_list = (os.getcwd())
base_file = os.getcwd() + "/tasks_1-20_v1-2/en/"
file_list = (os.listdir(base_file))
# print(file_list)
test_file = ""
train_file = ""
for file in file_list:
if file.startswith("qa" + str(i) + "_"):
if file.endswith("_test.txt"):
test_file = file
# print(test_file)
elif file.endswith("_train.txt"):
train_file = file
# print(train_file)
print(train_file)
print(test_file)
f_train = open(base_file + train_file)
f_test = open(base_file + test_file)
# try:
# path = get_file('babi-tasks-v1-2.tar.gz',
# origin='https://s3.amazonaws.com/text-datasets/babi_tasks_1-20_v1-2.tar.gz')
# except:
# print('Error downloading dataset, please download it manually:\n'
# '$ wget http://www.thespermwhale.com/jaseweston/babi/tasks_1-20_v1-2.tar.gz\n'
# '$ mv tasks_1-20_v1-2.tar.gz ~/.keras/datasets/babi-tasks-v1-2.tar.gz')
# raise
# tar = tarfile.open(path)
# # Default QA1 with 1000 samples
# # challenge = 'tasks_1-20_v1-2/en/qa1_single-supporting-fact_{}.txt'
# # QA1 with 10,000 samples
# # challenge = 'tasks_1-20_v1-2/en-10k/qa1_single-supporting-fact_{}.txt'
# # QA2 with 1000 samples
# challenge = 'tasks_1-20_v1-2/en/qa2_two-supporting-facts_{}.txt'
# # QA2 with 10,000 samples
# # challenge = 'tasks_1-20_v1-2/en-10k/qa2_two-supporting-facts_{}.txt'
# train = get_stories(tar.extractfile(challenge.format('train')))
# test = get_stories(tar.extractfile(challenge.format('test')))
# print("training stories:")
train = get_stories(f_train)
# print(len(train))
# print("testing stories:")
test = get_stories(f_test)
# print(len(test))
vocab = set()
for story, q, answer in train + test:
vocab |= set(story + q + [answer])
vocab = sorted(vocab)
# check_existence(vocab)
# get_word_vectors_from_pretr_embeddings(train, test, vocab)
# Reserve 0 for masking via pad_sequences
vocab_size = len(vocab) + 1
print("Vocabulary size: ", vocab_size)
word_idx = dict((c, i + 1) for i, c in enumerate(vocab))
story_maxlen = max(map(len, (x for x, _, _ in train + test)))
query_maxlen = max(map(len, (x for _, x, _ in train + test)))
x, xq, y = vectorize_stories(train, word_idx, story_maxlen, query_maxlen)
tx, txq, ty = vectorize_stories(test, word_idx,story_maxlen, query_maxlen)
print('vocab = {}'.format(vocab))
print('x.shape = {}'.format(x.shape))
print('xq.shape = {}'.format(xq.shape))
print('y.shape = {}'.format(y.shape))
print('story_maxlen, query_maxlen = {}, {}'.format(story_maxlen, query_maxlen))
print('Build model...')
pre_trained_emb_weights = get_pre_trained_emb(vocab)
sentence = layers.Input(shape=(story_maxlen,), dtype='float32')
encoded_sentence = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(sentence)
encoded_sentence = layers.Dropout(0.3)(encoded_sentence)
question = layers.Input(shape=(query_maxlen,), dtype='float32')
encoded_question = layers.Embedding(vocab_size, EMBED_HIDDEN_SIZE)(question)
encoded_question = layers.Dropout(0.3)(encoded_question)
encoded_question = RNN(EMBED_HIDDEN_SIZE)(encoded_question)
encoded_question = layers.RepeatVector(story_maxlen)(encoded_question)
merged = layers.add([encoded_sentence, encoded_question])
merged = RNN(EMBED_HIDDEN_SIZE)(merged)
merged = layers.Dropout(0.3)(merged)
preds = layers.Dense(vocab_size, activation='softmax')(merged)
model = Model([sentence, question], preds)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
print('Training')
history = model.fit([x, xq], y, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.05)
pandas.DataFrame(history.history).to_csv("__pre_"+file_name)
loss, acc = model.evaluate([tx, txq], ty,
batch_size=BATCH_SIZE)
pandas.DataFrame([str(loss)+"_"+ str(acc)]).to_csv("__test_"+RNN_TYPE+"_"+str(i)+".csv")
print('Test loss / test accuracy = {:.4f} / {:.4f}'.format(loss, acc))
def Execute_Model(self):
EMBEDDING_DIM = 50
MAX_SEQUENCE_LENGTH = 10
RATE_DROP_LSTM = 0.17
RATE_DROP_DENSE = 0.25
NUMBER_LSTM = 50
NUMBER_DENSE_UNITS = 50
ACTIVATION_FUNCTION = 'relu'
VALIDATION_SPLIT = 0.1
sentences1 = list(self.df['question1'].astype(str))
sentences2 = list(self.df['question2'].astype(str))
is_similar = list(self.df['is_duplicate'])
tokenizer, embedding_matrix = word_embed_meta_data(
sentences1 + sentences2, EMBEDDING_DIM)
embedding_meta_data = {
'tokenizer': tokenizer,
'embedding_matrix': embedding_matrix
}
sentences_pair = [(x1, x2) for x1, x2 in zip(sentences1, sentences2)]
nb_words = len(tokenizer.word_index) + 1
embedding_layer = layers.Embedding(nb_words,
siamese_config['EMBEDDING_DIM'],
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
lstm_layer = layers.Bidirectional(
layers.LSTM(NUMBER_LSTM,
dropout=RATE_DROP_LSTM,
recurrent_dropout=RATE_DROP_LSTM))
sequence_1_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences_1 = embedding_layer(sequence_1_input)
left_output = lstm_layer(embedded_sequences_1)
sequence_2_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences_2 = embedding_layer(sequence_2_input)
right_output = lstm_layer(embedded_sequences_2)
merged = layers.concatenate([left_output, right_output], axis=-1)
merged = BatchNormalization()(merged)
merged = layers.Dropout(0.1)(merged)
merged = layers.Dense(128, activation='relu')(merged)
merged = BatchNormalization()(merged)
merged = layers.Dropout(0.1)(merged)
predictions = layers.Dense(1, activation='sigmoid')(merged)
model = Model([sequence_1_input, sequence_2_input], predictions)
model.compile(loss='binary_crossentropy',
optimizer='nadam',
metrics=['acc'])
model.summary()
train_data_x1, train_data_x2, train_labels, leaks_train, \
val_data_x1, val_data_x2, val_labels, leaks_val = create_train_dev_set(tokenizer, sentences_pair,
is_similar, MAX_SEQUENCE_LENGTH,
VALIDATION_SPLIT)
callbacks = [
keras.callbacks.TensorBoard(
log_dir=
'E:\workdirectory\Code Name Val Halen\DS Sup\DL\Chapter 15\logs',
histogram_freq=1)
]
self.history = model.fit([train_data_x1, train_data_x2],
train_labels,
validation_data=([val_data_x1,
val_data_x2], val_labels),
epochs=200,
batch_size=64,
shuffle=True,
callbacks=callbacks)
def test_embedding(self):
input_data = np.random.randint(1000, size=(32, 10))
zlayer = ZLayer.Embedding(1000, 64, input_shape=(10, ))
klayer = KLayer.Embedding(1000, 64, input_length=10)
self.compare_layer(klayer, zlayer, input_data,
WeightsConverter.convert_embedding)
from keras import layers
from keras.models import Sequential
max_features = 10000
max_len = 500
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
x_train = [x[::-1] for x in x_train]
x_test = [x[::-1] for x in x_test]
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
model = Sequential()
model.add(layers.Embedding(max_features, 128))
model.add(layers.LSTM(32))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
plot_loss(history.history['loss'], history.history['val_loss'])
#%%
# Using biderectionnal layer
model = Sequential()
def train_sequence_model(ctx, primary_voice, secondary_voice, name, **kwargs):
"""Train a LSTM neural network on sequence data from a performance"""
model_name = '{}.h5'.format(name)
model_path = os.path.join(os.getcwd(), MODELS_FOLDER, model_name)
resume = False
if os.path.isfile(model_path):
click.confirm(
'Found model with same name! Do you want to resume training?',
abort=True)
resume = True
# Parameters and Hyperparameters
num_classes = kwargs.get('num_classes')
batch_size = kwargs.get('batch_size')
data_split = kwargs.get('data_split')
seq_len = kwargs.get('seq_len')
dropout = kwargs.get('dropout')
epochs = kwargs.get('epochs')
num_layers = kwargs.get('num_layers')
num_units = kwargs.get('num_units')
ctx.log('\nParameters:')
ctx.log(line(length=32))
ctx.log('name:\t\t{}'.format(name))
ctx.log('num_classes:\t{}'.format(num_classes))
ctx.log('batch_size:\t{}'.format(batch_size))
ctx.log('data_split:\t{}'.format(data_split))
ctx.log('seq_len:\t{}'.format(seq_len))
ctx.log('epochs:\t\t{}'.format(epochs))
if not resume:
ctx.log('num_layers:\t{}'.format(num_layers))
ctx.log('num_units:\t{}'.format(num_units))
ctx.log(line(length=32))
ctx.log('')
primary_voice = Voice(primary_voice)
secondary_voice = Voice(secondary_voice)
# Generate training data from voice sequences
ctx.log(click.style('1. Generate training data from voices', bold=True))
ctx.log('Primary voice: "{}"'.format(primary_voice.name))
ctx.log('Secondary voice: "{}"'.format(secondary_voice.name))
data = generate_sequence(ctx,
primary_voice,
secondary_voice,
save_sequence=kwargs.get('save_sequence'))
ctx.log('')
# Encode data before training
ctx.log(click.style('2. Encode data before training', bold=True))
encoded_data, kmeans = k_means_encode_data(data, num_classes)
ctx.log('Number of classes: {}\n'.format(num_classes))
# Split in 3 sets for training, validation and testing
ctx.log(click.style('3. Split data in sets', bold=True))
validation_steps = round((data_split / 2) * len(data))
train_max = len(data) - (validation_steps * 2)
val_min = train_max + 1
val_max = train_max + validation_steps + 1
test_min = train_max + validation_steps + 2
test_max = len(data) - 1
training_steps = test_max - test_min
train_gen = generator(encoded_data,
seq_len=seq_len,
batch_size=batch_size,
min_index=0,
max_index=train_max)
val_gen = generator(encoded_data,
seq_len=seq_len,
batch_size=batch_size,
min_index=val_min,
max_index=val_max)
test_gen = generator(encoded_data,
seq_len=seq_len,
batch_size=batch_size,
min_index=test_min,
max_index=test_max)
steps_per_epoch = train_max // batch_size
ctx.log('Batch size: {}'.format(batch_size))
ctx.log('Steps per epoch: {}'.format(steps_per_epoch))
ctx.log('Split for validation & test @ {0:.2f}%'.format(data_split * 100))
ctx.log('Training set: {}-{}'.format(0, train_max))
ctx.log('Validation set: {}-{}'.format(val_min, val_max))
ctx.log('Test set: {}-{}\n'.format(test_min, test_max))
# Define model
ctx.log(click.style('4. Define a model', bold=True))
if resume:
ctx.log('Load existing model to resume training ..')
try:
model = load_model(model_path)
except ValueError as err:
ctx.elog('Could not load model: {}'.format(err))
sys.exit(1)
else:
model = Sequential()
model.add(
layers.Embedding(input_dim=num_classes,
output_dim=num_units,
input_length=seq_len))
for n in range(num_layers - 1):
model.add(layers.LSTM(num_units, return_sequences=True))
if dropout > 0.0:
model.add(layers.Dropout(dropout))
model.add(layers.LSTM(num_units))
if dropout > 0.0:
model.add(layers.Dropout(dropout))
model.add(layers.Dense(num_classes, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
model.summary()
ctx.log('')
# Training!
ctx.log(click.style('5. Training!', bold=True))
model.fit_generator(train_gen,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=val_gen,
validation_steps=validation_steps)
ctx.log('Finished training.\n')
# Evaluate training
ctx.log(click.style('6. Evaluation', bold=True))
scores = []
max_dist = .25
for i in range((training_steps // batch_size)):
# Predict point from model
samples, targets = next(test_gen)
results = model.predict(samples)
for j, result in enumerate(results):
# Decode data
result_value = np.argmax(result)
position = k_means_decode_data([[result_value]], kmeans).flatten()
position_target = k_means_decode_data([[targets[j]]], kmeans)
position_target = position_target.flatten()
# Calculate distance between prediction and actual test target
dist = max_dist - min(max_dist,
np.linalg.norm(position - position_target))
scores.append(0.0 if dist == 0.0 else dist / max_dist)
score = np.average(scores)
ctx.log('Score: {0:.2f}%\n'.format(score * 100))
# Save model
ctx.log(click.style('7. Store model weights', bold=True))
ctx.log('Stored weights at "{}"'.format(model_path))
model.save(model_path)
ctx.log('Done!')
from keras import Input, layers, Model
import numpy as np
from vizualization import plt_loss, plt_accuracy
text_vocabulary_size = 10000
question_vocabulary_size = 10000
answer_vocabulary_size = 500
text_input = Input(shape=(None, ), dtype='int32', name='text')
embedded_text = layers.Embedding(64, text_vocabulary_size)(text_input)
encoded_text = layers.LSTM(32)(embedded_text)
question_input = Input(shape=(None, ), dtype='int32', name='question')
embedded_question = layers.Embedding(32,
question_vocabulary_size)(question_input)
encoded_question = layers.LSTM(16)(embedded_question)
concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1)
answer = layers.Dense(answer_vocabulary_size,
activation='softmax')(concatenated)
model = Model([text_input, question_input], answer)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['acc'])
# generate random data
num_samples = 1000
max_length = 100
(3D float tensor of (samples, sequence_length, embedding_size)),
flatten the tensor to 2D, and train a single Dense layer on top for classification.
'''
from keras.datasets import imdb
from keras import preprocessing
from keras import models, layers, Input
max_features = 10000 # Number of words to consider as features
maxlen = 20 # Cuts off the text after this number of words
# (among the max_features most common words)
# read data
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
# turn the lists of integers into a 2D integer tensor of shape (samples, maxlen)
x_train = preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = preprocessing.sequence.pad_sequences(x_test, maxlen=maxlen)
input_tensor = Input((maxlen, ))
kmodel = layers.Embedding(10000, 8, input_length=maxlen)(input_tensor)
kmodel = layers.Flatten()(kmodel)
output_tensor = layers.Dense(1, activation='sigmoid')(kmodel)
model = models.Model(input_tensor, output_tensor)
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
model.summary()
history = model.fit(x_train,
y_train,
epochs=10,
batch_size=32,
validation_split=0.2)
content = [word for word in content if not isinstance(word, int)]
y.append(int(row[4]))
x.append(' '.join(content))
vectorizer.fit_transform(x).toarray()
vocab_size = len(vectorizer.get_feature_names())
x = vectorizer.fit_transform(x).toarray()
le = LabelEncoder()
y = le.fit_transform(y)
print(x)
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
if mode == 'create':
model = Sequential([
layers.Embedding(vocab_size, 64),
layers.Conv1D(64, 5, activation='relu'),
layers.MaxPooling1D(5),
layers.Conv1D(64, 5, activation='relu'),
layers.MaxPooling1D(35),
layers.Conv1D(64, 5, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dropout(0.5),
layers.Dense(1, activation='softmax')
])
'''
model = Sequential([
layers.Embedding(vocab_size, 64),
layers.Bidirectional(layers.LSTM(64, return_sequences=True)),
layers.Bidirectional(layers.LSTM(32)),
layers.Dense(64, activation='relu'),
def test_TimeDistributed():
# first, test with Dense layer
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 2)),
epochs=1,
batch_size=10)
# test config
model.get_config()
# test when specifying a batch_input_shape
test_input = np.random.random((1, 3, 4))
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(
wrappers.TimeDistributed(layers.Dense(2), batch_input_shape=(1, 3, 4)))
reference.add(layers.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Embedding
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.Embedding(5, 6),
batch_input_shape=(10, 3, 4),
dtype='int32'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.randint(5, size=(10, 3, 4), dtype='int32'),
np.random.random((10, 3, 4, 6)),
epochs=1,
batch_size=10)
# compare to not using batch_input_shape
test_input = np.random.randint(5, size=(10, 3, 4), dtype='int32')
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(
wrappers.TimeDistributed(layers.Embedding(5, 6),
input_shape=(3, 4),
dtype='int32'))
reference.compile(optimizer='rmsprop', loss='mse')
reference.layers[0].set_weights(weights)
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Conv2D
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.Conv2D(5, (2, 2), padding='same'),
input_shape=(2, 4, 4, 3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.random.random((1, 2, 4, 4, 3)),
np.random.random((1, 2, 4, 4, 5)))
model = model_from_json(model.to_json())
model.summary()
# test stacked layers
model = Sequential()
model.add(wrappers.TimeDistributed(layers.Dense(2), input_shape=(3, 4)))
model.add(wrappers.TimeDistributed(layers.Dense(3)))
model.add(layers.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test wrapping Sequential model
model = Sequential()
model.add(layers.Dense(3, input_dim=2))
outer_model = Sequential()
outer_model.add(wrappers.TimeDistributed(model, input_shape=(3, 2)))
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test with functional API
x = Input(shape=(3, 2))
y = wrappers.TimeDistributed(model)(x)
outer_model = Model(x, y)
outer_model.compile(optimizer='rmsprop', loss='mse')
outer_model.fit(np.random.random((10, 3, 2)),
np.random.random((10, 3, 3)),
epochs=1,
batch_size=10)
# test with BatchNormalization
model = Sequential()
model.add(
wrappers.TimeDistributed(layers.BatchNormalization(center=True,
scale=True),
name='bn',
input_shape=(10, 2)))
model.compile(optimizer='rmsprop', loss='mse')
# Assert that mean and variance are 0 and 1.
td = model.layers[0]
assert np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Train
model.train_on_batch(np.random.normal(loc=2, scale=2, size=(1, 10, 2)),
np.broadcast_to(np.array([0, 1]), (1, 10, 2)))
# Assert that mean and variance changed.
assert not np.array_equal(td.get_weights()[2], np.array([0, 0]))
assert not np.array_equal(td.get_weights()[3], np.array([1, 1]))
# Verify input_map has one mapping from inputs to reshaped inputs.
uid = object_list_uid(model.inputs)
assert len(td._input_map.keys()) == 1
assert uid in td._input_map
assert K.int_shape(td._input_map[uid]) == (None, 2)
xtrain=pad_sequences(xtrain,padding='post', maxlen=maxlen)
xtest=pad_sequences(xtest,padding='post', maxlen=maxlen)
print(X_train[3])
print(xtrain[3])
"""#### Defining the LSTM model
* We apply the Embedding layer for input data before adding the LSTM layer into the Keras sequential model. The model definition goes as a following.
"""
embedding_dim=50
model=Sequential()
model.add(layers.Embedding(input_dim=100,
output_dim=embedding_dim,
input_length=maxlen))
model.add(layers.LSTM(units=20,return_sequences=True))
model.add(layers.LSTM(units=10))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(28))
model.add(layers.Dense(1, activation="sigmoid"))
model.compile(optimizer="adam", loss="binary_crossentropy",
metrics=['accuracy'])
model.summary()
"""#### Finally, we'll train the model and check the training accuracy."""
history = model.fit(xtrain,y_train, epochs=20, batch_size=64)
"""#### Visualize the training loss and te validation accuracy to see if the model is overfitting"""
