#%%
print('Build model...')
# Inputs
review_input = Input(shape=(maxsents,maxlen), dtype='int32')
# Embedding Layer
embedding_layer = Embedding(max_features, embedding_dims,
input_length=maxlen)
# WORD-LEVEL
sentence_input = Input(shape=(maxlen,), dtype='int32')
embedded_sequences = embedding_layer(sentence_input)
# Bidirectional GRU
l_gru = Bidirectional(GRU(gru_output_size, return_sequences=True))(embedded_sequences)
l_dense = TimeDistributed(Dense(units=gru_output_size))(l_gru)
# Word-Level Attention Layer
l_att = AttLayer()(l_dense)
sentEncoder = Model(sentence_input, l_att)
review_encoder = TimeDistributed(sentEncoder)(review_input)
# SENTENCE_LEVEL
# Bidirectional GRU
l_gru_sent = Bidirectional(GRU(gru_output_size, return_sequences=True))(review_encoder)
l_dense_sent = TimeDistributed(Dense(units=gru_output_size))(l_gru_sent)
# Sentence-Level Attention Layer
postp = AttLayer()(l_dense_sent)
# Embedding Average
sentEmbed = Model(sentence_input, embedded_sequences)
def construct_model(label_name):
output_parameters()
model = Sequential()
# mask_zero 在 MaxPooling 层中不能支持
model.add(
Embedding(creative_id_window, embedding_size, input_length=max_len))
if model_type == 'MLP':
model.add(Flatten())
model.add(Dropout(0.2))
model.add(
Dense(embedding_size * max_len // 4,
activation='relu',
kernel_regularizer=l2(0.001)))
model.add(Dropout(0.2))
elif model_type == 'Conv1D':
model.add(
Conv1D(32, 7, activation='relu', kernel_regularizer=l2(0.001)))
model.add(
Conv1D(32, 7, activation='relu', kernel_regularizer=l2(0.001)))
model.add(GlobalMaxPooling1D())
elif model_type == 'GlobalMaxPooling1D':
model.add(GlobalMaxPooling1D())
elif model_type == 'GRU':
model.add(GRU(embedding_size, dropout=0.2, recurrent_dropout=0.2))
# model.add(LSTM(128, dropout = 0.5, recurrent_dropout = 0.5))
elif model_type == 'Conv1D+LSTM':
model.add(
Conv1D(32, 5, activation='relu', kernel_regularizer=l2(0.001)))
model.add(
Conv1D(32, 5, activation='relu', kernel_regularizer=l2(0.001)))
model.add(LSTM(16, dropout=0.5, recurrent_dropout=0.5))
elif model_type == 'Bidirectional-LSTM':
model.add(
Bidirectional(
LSTM(embedding_size, dropout=0.2, recurrent_dropout=0.2)))
else:
raise Exception("错误的网络模型类型")
if label_name == "age":
model.add(
Dense(embedding_size * 10,
activation='relu',
kernel_regularizer=l2(0.001)))
model.add(Dense(10, activation='softmax',
kernel_regularizer=l2(0.001)))
print("%s——模型构建完成！" % model_type)
print("* 编译模型")
model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.sparse_categorical_crossentropy,
metrics=[metrics.sparse_categorical_accuracy])
elif label_name == 'gender':
model.add(
Dense(embedding_size,
activation='relu',
kernel_regularizer=l2(0.001)))
model.add(Dense(1, activation='sigmoid', kernel_regularizer=l2(0.001)))
print("%s——模型构建完成！" % model_type)
print("* 编译模型")
model.compile(optimizer=optimizers.RMSprop(lr=RMSProp_lr),
loss=losses.binary_crossentropy,
metrics=[metrics.binary_accuracy])
else:
raise Exception("错误的标签类型！")
return model
def getModel(maxWordLen, data_dim):
# mlp = Sequential()
# mlp.add(Dense(512, input_dim=data_dim, init='uniform', activation='tanh'))
# mlp.add(Dropout(0.5))
# mlp.add(Dense(128, init='uniform', activation='tanh'))
# mlp.add(Dropout(0.5))
# mlp.add(Dense(1, init='uniform', activation='sigmoid'))
# sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
# mlp.compile(loss='mean_squared_error',
# optimizer=sgd,
# metrics=['accuracy'])
# # train
# mlp.fit(x_train, y_train,
# batch_size=68, nb_epoch=5,
# validation_data=(x_val, y_val))
# # test
# score = mlp.evaluate(x_test, y_test, show_accuracy=True, verbose=0)
# print('Test score:', score[0])
# print('Test accuracy:', score[1])
# , dtype='int32'
# inp = Input(shape=(maxWordLen+1, data_dim))
# # embedded = Embedding(data_dim, 128, input_length=maxWordLen+1)(inp)
# # # # apply forwards LSTM
# forwards = LSTM(
# 128, return_sequences=False)(inp)
# # # apply backwards LSTM
# backwards = LSTM(
# 128, return_sequences=False, go_backwards=True)(inp)
# # # # concatenate the outputs of the 2 LSTMs
# merged = merge([forwards, backwards], mode='concat', concat_axis=-1)
# # after_dp = Dropout(0.5)(merged)
# # # l2 = LSTM(16)(merged)
# output = Dense(2, activation='softmax')(merged)
# lstm = Model(input=inp, output=output)
lstm = Sequential()
# lstm.add(Embedding(data_dim, 128, input_length=maxWordLen+1))
# embedded = Embedding(max_features, 128, input_length=maxlen)(sequence)
# lecun_uniform
lstm.add(GRU(
output_dim=62,
return_sequences=False,
# batch_input_shape=(),
input_shape=(maxWordLen+1, data_dim),
consume_less="cpu",
go_backwards=False,
dropout_W=0.25
# W_regularizer=l2(0.01),
# b_regularizer=l2(0.01)
))
# lstm.add(GRU(8, return_sequences=False, consume_less="cpu"))
# lstm.add(LSTMpeephole(
# output_dim=1,
# return_sequences=False,
# # batch_input_shape=(),
# input_shape=(maxWordLen+1, data_dim),
# consume_less="cpu",
# go_backwards=False
# # dropout_W=0.5
# # W_regularizer=l2(0.01)
# # b_regularizer=l2(0.01)
# ))
# lstm.add(LSTM(
# 128, return_sequences=False, consume_less="cpu",
# go_backwards=True))
# lstm.add(LSTM(
# hidden_units, return_sequences=True, consume_less="cpu",
# go_backwards=False))
# lstm.add(LSTM(
# hidden_units, return_sequences=False, consume_less="cpu",
# go_backwards=False))
# lstm.add(LSTM(
# hidden_units, return_sequences=True, consume_less="cpu",
# go_backwards=True))
# lstm.add(LSTM(
# hidden_units,
# return_sequences=True,
# # Varies by word length
# input_length=None,
# input_dim=data_dim))
# lstm.add(LSTM(hidden_units, return_sequences=True))
# lstm.add(LSTM(hidden2_units, return_sequences=False, consume_less="cpu"))
# lstm.add(Dense(256, activation="tanh"))
# lstm.add(Dense(32, activation="tanh"))
lstm.add(Dense(1, activation="sigmoid"))
# lstm.add(Dense(2, activation="softmax"))
return lstm
def create_model(self, ckpt_file=None):
if ckpt_file is None:
if self.arch == "vanilla":
self.model = Sequential()
self.model.add(self.embedding_layer)
self.model.add(
LSTM(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "gru":
self.model = Sequential()
self.model.add(self.embedding_layer)
self.model.add(
GRU(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "conv":
conv_filters = []
for filter_size in self.filter_sizes:
conv_filters.append(Sequential())
conv_filters[-1].add(self.embedding_layer)
conv_filters[-1].add(
Conv1D(filters=self.num_filters,
kernel_size=filter_size,
strides=1,
padding='valid',
activation='relu'))
conv_filters[-1].add(
MaxPooling1D(pool_size=(self.seq_length - filter_size +
1)))
self.model = Sequential()
self.model.add(Merge(conv_filters, mode='concat'))
self.model.add(Flatten())
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(512, activation='softmax'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "seq_conv1":
self.model = Sequential()
filter_size = 3
self.model.add(self.embedding_layer)
self.model.add(
Conv1D(filters=self.num_filters / 2,
kernel_size=filter_size,
strides=1,
padding='valid',
activation='relu'))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(
Conv1D(filters=int(self.num_filters / 4),
kernel_size=filter_size,
strides=1,
padding='valid',
activation='relu'))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(Flatten())
self.model.add(Dense(1024, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "seq_conv2":
self.model = Sequential()
filter_size = 3
self.model.add(self.embedding_layer)
self.model.add(
Conv1D(filters=self.num_filters,
kernel_size=filter_size,
strides=1,
padding='same',
activation='relu'))
self.model.add(
Conv1D(filters=int(self.num_filters / 2),
kernel_size=filter_size,
strides=1,
padding='same',
activation='relu'))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(
Conv1D(filters=int(self.num_filters / 4),
kernel_size=filter_size,
strides=1,
padding='same',
activation='relu'))
self.model.add(
Conv1D(filters=int(self.num_filters / 4),
kernel_size=filter_size,
strides=1,
padding='same',
activation='relu'))
self.model.add(MaxPooling1D(pool_size=2))
self.model.add(Flatten())
self.model.add(Dense(1024, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(512, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "conv_lstm":
# Ensemble of sentence-level embedding obtained from an LSTM and
# a sentence-level embedding obtained from a CNN.
print("Start")
branch1 = Sequential()
branch1.add(self.embedding_layer)
branch1.add(
LSTM(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True))
print("Created first model")
conv_filters = []
for filter_size in self.filter_sizes:
conv_filters.append(Sequential())
conv_filters[-1].add(self.embedding_layer)
conv_filters[-1].add(
Conv1D(filters=self.num_filters,
kernel_size=filter_size,
strides=1,
padding='valid',
activation='relu'))
conv_filters[-1].add(
MaxPooling1D(pool_size=(self.seq_length - filter_size +
1)))
branch2 = Sequential()
branch2.add(Merge(conv_filters, mode='concat'))
branch2.add(Flatten())
branch2.add(Dropout(2 * DROPOUT))
print("Created second model")
self.model = Sequential()
self.model.add(Merge([branch1, branch2], mode='concat'))
self.model.add(Dense(1024, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "ensemble":
# Ensemble of sentence-level embedding obtained from an LSTM and
# a sentence-level embedding obtained from the OpenAI features.
print("Start")
branch1 = Sequential()
branch1.add(self.embedding_layer)
branch1.add(
LSTM(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True))
print("Created first model")
branch2 = Sequential()
branch2.add(
Dense(OPENAI_REDUCED_SIZE,
activation="linear",
input_shape=(OPENAI_FEATURE_SIZE, )))
print("Created second model")
self.model = Sequential()
self.model.add(Merge([branch1, branch2], mode='concat'))
self.model.add(Dense(1024, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "multi_layer":
self.model = Sequential()
self.model.add(self.embedding_layer)
# Two LSTM layers.
self.model.add(
LSTM(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True,
return_sequences=True))
self.model.add(
LSTM(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True))
# Add 2 fully-connected layers.
self.model.add(Dense(1024, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(512, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
elif self.arch == "bidi":
self.model = Sequential()
self.model.add(self.embedding_layer)
lstm_layer = LSTM(LSTM_SIZE,
dropout=self.drop_prob,
recurrent_dropout=self.drop_prob,
implementation=2,
unroll=True)
self.model.add(Bidirectional(lstm_layer, merge_mode='sum'))
self.model.add(Dense(512, activation='relu'))
self.model.add(Dropout(2 * DROPOUT))
self.model.add(Dense(2, activation='softmax'))
else:
raise NotImplementedError("Architecture is not implemented:",
self.arch)
else:
# Load model from checkpoint file.
self.model = load_model(ckpt_file)
sen_batch1 = BatchNormalization()(sen_conv1)
sen_max1 = MaxPooling1D(pool_size=MAX1_SIZE)(sen_batch1)
sen_conv3 = Conv1D(CONV_NUM2,
kernel_size=(CONV_MERGE_LEN3),
strides=(STRIDE_MERGE_LEN3),
padding='causal',
activation='relu')(sen_batch1)
sen_batch3 = BatchNormalization()(sen_conv3)
sen_max3 = MaxPooling1D(pool_size=MAX2_SIZE)(sen_batch3)
sen_flatten = Flatten()(sen_max3)
merged_model = Model(inputs=win_input, outputs=sen_flatten)
timed = TimeDistributed(merged_model)(input)
#elkészült conv struktúra
print(merged_model.summary())
grucell = GRU(INTER_DIM, return_sequences=True)(timed)
grucell2 = GRU(INTER_DIM, return_sequences=True)(grucell)
avg = Lambda(avg_layer)(grucell2)
if (len(PERSONS) > 2):
dens = Dense(len(PERSONS), activation="softmax")(
avg) #sigmoid binárisnál, softmax categorinál
else:
dens = Dense(1, activation="sigmoid")(avg)
model = Model(inputs=input, outputs=dens)
#teljes modell
print(model.summary())
if (len(PERSONS) > 2):
model.compile(loss='categorical_crossentropy',
optimizer='RMSprop',
metrics=['accuracy']) # rms prop # binary hogyha bináris
#model.add(Bidirectional(GRU(64,dropout_W=dropout, dropout_U=dropout, return_sequences=True), merge_mode='concat'))
model.add(Bidirectional(GRU(64, return_sequences=True), merge_mode='concat', input_shape=(50, 10)))
#model.add(GRU(128,dropout_W=dropout, dropout_U=dropout, return_sequences=False))
#model.add(GRU(64, return_sequences=False))
#model.add(Dropout(dropout))
#model.add(Dense(64, activation="tanh"))
model.add(AttentionLayer())
model.add(Dense(num_classes, activation="softmax"))
'''
x_input = Input(shape=input_shape, dtype='float32')
#x_input = Input(shape=(50,10), dtype='float32')
l_lstm = Bidirectional(GRU(64, return_sequences=True),
merge_mode='concat')(x_input)
#l_dense = TimeDistributed(Dense(256))(l_lstm)
att = AttentionLayer()(l_lstm)
preds = Dense(num_classes, activation="softmax")(att)
model = Model(x_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.summary()
#exit()
model.fit(
train_features,
def lstm(n_features=4,
n_timesteps=12,
n_train=10,
n_window=5,
n_units=100,
n_epochs=50,
with_att=False,
methods='lstm',
lr=0.001):
"""
:param n_features: 4 or 10, using 4 features or 10 features
:param n_train: training timesteps
:param n_window: width of training window, for example, [0 1 2 3 4]->[5], n_window = 5
:param n_units: LSTM units
:param n_epochs: trainning epochs
:return:
"""
data = []
for i in range(len(name_node_pairs)):
f = open('./data/features_{}/{}_temp_link_ft.csv'.format(
n_features, name_node_pairs[i]))
df = pd.read_csv(f)
data.append(df.values)
data = np.array(data)
print('data: {}, \ndata.shape(): {}'.format(data, data.shape))
# define train, test
# scaler = MinMaxScaler(feature_range=(0, 1))
n_samples, n_timesteps, n_features = data.shape
scaled_data = data.reshape((n_samples, n_timesteps * n_features))
# scaled_data = scaler.fit_transform(scaled_data)
scaled_data = scaled_data.reshape((n_samples, n_timesteps, n_features))
# define problem properties
n_test = 12 - n_train
# define LSTM
# sequential
# model = Sequential()
# model.add(Bidirectional(LSTM(n_units, input_shape=(n_window, n_features))))
# model.add(Dense(1))
#
# model.compile(loss='mse', optimizer='adam')
# Model
inputs = Input(shape=(n_window, n_features))
return_sequences = False
if with_att == True:
return_sequences = True
if methods == 'lstm':
att_in = Bidirectional(
LSTM(n_units,
input_shape=(n_window, n_features),
return_sequences=return_sequences))(inputs)
elif methods == 'gru':
att_in = Bidirectional(
GRU(n_units,
input_shape=(n_window, n_features),
return_sequences=return_sequences))(inputs)
elif methods == 'rnn':
att_in = Bidirectional(
SimpleRNN(n_units,
input_shape=(n_window, n_features),
return_sequences=return_sequences))(inputs)
if with_att == True:
att_out = attention()(att_in)
outputs = Dense(1)(att_out)
else:
outputs = Dense(1)(att_in)
model = Model(inputs, outputs)
opt = optimizers.Adam(lr=lr)
model.compile(loss='mse', optimizer=opt)
# fit network
for i in range(n_train - n_window):
history = model.fit(scaled_data[:, i:i + n_window, :],
scaled_data[:, i + n_window, 1],
epochs=n_epochs)
# plot history
# plt.plot(history.history['loss'])
# plt.show()
# make prediction
inv_yhat = []
for i in range(n_test):
yhat = model.predict(scaled_data[:, n_train - n_window + i:n_train +
i, :])
inv_yhat.append(yhat)
inv_yhat = np.array(inv_yhat)
print('inv_yhat.shape:{}'.format(
inv_yhat.shape)) #inv_yhat.shape:(3, 736, 1)
inv_yhat = inv_yhat.reshape((inv_yhat.shape[0], inv_yhat.shape[1]))
print('inv_yhat.shape:{}'.format(inv_yhat.shape)) #inv_yhat.shape:(3, 736)
inv_yhat = inv_yhat.T
print('inv_yhat.shape:{}'.format(inv_yhat.shape)) #inv_yhat.shape:(736, 3)
inv_yhat = np.concatenate((scaled_data[:, n_train:, 0], inv_yhat), axis=1)
inv_yhat = inv_yhat.reshape((n_samples, n_test, 2))
inv_yhat = np.concatenate((inv_yhat, scaled_data[:, n_train:, 2:]), axis=2)
print('inv_yhat.shape:{}'.format(inv_yhat.shape))
inv_yhat = np.concatenate((scaled_data[:, :n_train, :], inv_yhat), axis=1)
print('inv_yhat.shape:{}'.format(inv_yhat.shape))
inv_yhat = inv_yhat.reshape(n_samples, n_timesteps * n_features)
print('inv_yhat.shape:{}'.format(inv_yhat.shape))
# inv_yhat = scaler.inverse_transform(inv_yhat)
inv_yhat = inv_yhat.reshape(n_samples, n_timesteps, n_features)
inv_yhat[inv_yhat < 0] = 0 # transform negative values to zero
prediction = inv_yhat[:, -3:, 1]
prediction = prediction.reshape(prediction.shape[0], prediction.shape[1],
1)
original = data[:, -3:, 1]
original = original.reshape(original.shape[0], original.shape[1], 1)
concat = np.concatenate((original, prediction), axis=2)
print('concat.shape:{}'.format(concat.shape))
np.set_printoptions(threshold=1e6)
print('concat\n{}'.format(concat))
concat = concat.reshape(concat.shape[0] * concat.shape[1], concat.shape[2])
df = pd.DataFrame(concat)
df.columns = ['original', 'prediction']
df.to_csv('./data/LSTM/prediction_LSTM_{}.csv'.format(n_features),
index=False)
rmse = sqrt(mean_squared_error(inv_yhat[:, -3:, 1], data[:, -3:, 1]))
print('rmse: {}'.format(rmse))
print("------------------------------------------------------")
print(Y[:10])
#print(Y.shape)
MAX_LENGTH = 6
X = pad_sequences(X, maxlen=MAX_LENGTH, padding='post')
print(X.shape)
vocab_size = len(vocab)
Y = to_categorical(Y, num_classes=vocab_size)
print(Y.shape)
model = Sequential()
model.add(Embedding(vocab_size, 50, input_length=6))
model.add(GRU(150, recurrent_dropout=0.1, dropout=0.1))
model.add(Dense(vocab_size, activation='softmax'))
model.compile(loss='categorical_crossentropy',
metrics=['acc'],
optimizer='adam')
print(model.summary())
model.fit(X, Y, epochs=100, verbose=2)
def spell_checking(sentence):
X, _ = generate_training_data(sentence, word_to_id, 3)
for i in range(len(X)):
yhat = model.predict_classes(np.array([X[i]]))
word = id_to_word[int(yhat)]
print(i)
from keras.layers import GRU
from keras.layers import SimpleRNN
from keras.layers import Dropout
from keras import losses
import h5py
from keras import backend as K
import numpy as np
print('Build model...')
main_input = Input(shape=(None, 22), name='main_input')
#x = Dense(44, activation='relu')(main_input)
#x = GRU(44, dropout=0.0, recurrent_dropout=0.0, activation='tanh', recurrent_activation='sigmoid', return_sequences=True)(x)
x = main_input
x = GRU(128,
activation='tanh',
recurrent_activation='sigmoid',
return_sequences=True)(x)
#x = GRU(128, return_sequences=True)(x)
#x = GRU(22, activation='relu', return_sequences=True)(x)
x = Dense(22, activation='sigmoid')(x)
#x = Dense(22, activation='softplus')(x)
model = Model(inputs=main_input, outputs=x)
batch_size = 32
print('Loading data...')
with h5py.File('denoise_data.h5', 'r') as hf:
all_data = hf['denoise_data'][:]
print('done.')
window_size = 500
for n, tags in enumerate(Y_test):
for t, tag in enumerate(tags):
Y_test_onehot[n, t, tag] = 1
# 参数
epochs = 30
batch_size = 32
hidden_layer_size = 10
embedding_dim = 10
# 建立模型 Input只说明输入列维度
# NxT
input_ = Input(shape=(max_sequence_length, ))
# NxTxD
x = Embedding(vocab_size, embedding_dim)(input_)
x = GRU(hidden_layer_size, return_sequences=True)(x)
# NxTxM
output = Dense(tag_size, activation='softmax')(x)
model = Model(input_, output)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=1e-2),
metrics=['accuracy'])
print('Training model...')
# keras 的好处就是不用像tensorflow一样，输入前后都要进行维度转换
r = model.fit(X_train,
Y_train_onehot,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_test, Y_test_onehot))
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
y_tr_one = get_onehot(y_train, 4)
y_ts_one = get_onehot(y_test, 4)
# creating the model
gru_input = Input(shape=(maxlen, ))
x = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=False)(gru_input)
x = SpatialDropout1D(0.2)(x)
x = Bidirectional(
GRU(128,
return_sequences=True,
reset_after=True,
recurrent_activation='sigmoid'))(x)
avg_pool = GlobalAveragePooling1D()(x)
max_pool = GlobalMaxPooling1D()(x)
x = concatenate([avg_pool, max_pool])
x = Dense(128, activation='relu')(x)
x = Dropout(0.1)(x)
gru_output = Dense(4, activation="sigmoid")(x)
gru_model = Model(gru_input, gru_output)
gru_model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# training parameters
epochs = 20
batch_size = 128
def main(rnn_model):
def message_to_array(msg):
msg = msg.lower().split(' ')
test_seq = np.array([word_index[word] for word in msg])
test_seq = np.pad(test_seq, (500-len(test_seq), 0), 'constant', constant_values=(0))
test_seq = test_seq.reshape(1, 500)
return test_seq
data = pd.read_csv("./spam_text_message_data.csv")
print(data.head())
print(data.tail())
messages = []
labels = []
for index, row in data.iterrows():
messages.append(row['Message'])
if row['Category'] == 'ham':
labels.append(0)
else:
labels.append(1)
messages = np.asarray(messages)
labels = np.asarray(labels)
# print("Number of messages: ", len(messages))
#print("Number of labels: ", len(labels))
max_vocab = 10000
max_len = 500
# Ignore all words except the 10000 most common words
tokenizer = Tokenizer(num_words=max_vocab)
# Calculate the frequency of words
tokenizer.fit_on_texts(messages)
# Convert array of messages to list of sequences of integers
sequences = tokenizer.texts_to_sequences(messages)
# Dict keeping track of words to integer index
word_index = tokenizer.word_index
# Convert the array of sequences(of integers) to 2D array with padding
# maxlen specifies the maximum length of sequence (truncated if longer, padded if shorter)
data = pad_sequences(sequences, maxlen=max_len)
print("data shape: ", data.shape)
# We will use 80% of data for training & validation(80% train, 20% validation) and 20% for testing
train_samples = int(len(messages)*0.8)
messages_train = data[:train_samples]
labels_train = labels[:train_samples]
messages_test = data[train_samples:len(messages)-2]
labels_test = labels[train_samples:len(messages)-2]
embedding_mat_columns=40
# Construct the SimpleRNN model
model = Sequential()
## Add embedding layer to convert integer encoding to word embeddings(the model learns the
## embedding matrix during training), embedding matrix has max_vocab as no. of rows and chosen
## no. of columns
model.add(Embedding(input_dim=max_vocab, output_dim=embedding_mat_columns, input_length=max_len))
model.add(Dropout(0.2))
model.add(Conv1D(64, 5, activation='sigmoid'))
model.add(MaxPooling1D(pool_size=4))
if rnn_model == 'SimpleRNN':
model.add(SimpleRNN(units=embedding_mat_columns))
elif rnn_model == 'LSTM':
model.add(LSTM(units=embedding_mat_columns))
else:
model.add(GRU(units=embedding_mat_columns))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
model.summary()
#plot_model(model, to_file='model.png', show_shapes=True, show_layer_names=True)
# Training the model
model.fit(messages_train, labels_train, epochs=1, batch_size=60, validation_split=0.2)
# Testing the model
pred = model.predict_classes(messages_test)
acc = model.evaluate(messages_test, labels_test)
print("Test loss is {0:.2f} accuracy is {1:.2f} ".format(acc[0],acc[1]))
# =============================================================================
# print(pred)
# print(labels_test)
# =============================================================================
print("precision is about ", metrics.precision_score(labels_test, pred,average="macro"))
print("recall is about ", metrics.recall_score(labels_test, pred, average="macro"))
print("f-score is about ", metrics.f1_score(labels_test, pred, average="macro"))
print(metrics.confusion_matrix(labels_test, pred))
if len(sys.argv) == 2:
print("load " + sys.argv[1] + " to keep training")
model = load_model(sys.argv[1])
else:
model = Sequential()
wm = np.load('wm.npy')
model.add(
Embedding(wm.shape[0],
output_dim=wm.shape[1],
weights=[wm],
input_length=MAX_LENGTH,
trainable=False))
model.add(
Bidirectional(
GRU(units=256,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)))
model.add(Dropout(0.4))
model.add(
Bidirectional(
GRU(units=256,
return_sequences=False,
dropout=0.2,
recurrent_dropout=0.2)))
model.add(Dropout(0.6))
model.add(Dense(units=1, activation='sigmoid'))
rms = optimizers.RMSprop(clipnorm=1.)
model.compile(optimizer=rms,
loss='binary_crossentropy',
metrics=['accuracy'])
print(model.summary())
def get_model_5(params):
input_tensor=None
include_top=True
# Determine proper input shape
if K.image_dim_ordering() == 'th':
input_shape = (1, 96, 1366)
else:
input_shape = (96, 1366, 1)
if input_tensor is None:
melgram_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
melgram_input = Input(tensor=input_tensor, shape=input_shape)
else:
melgram_input = input_tensor
# Determine input axis
if K.image_dim_ordering() == 'th':
channel_axis = 1
freq_axis = 2
time_axis = 3
else:
channel_axis = 3
freq_axis = 1
time_axis = 2
# Input block
x = ZeroPadding2D(padding=(0, 37))(melgram_input)
x = BatchNormalization(axis=freq_axis, name='bn_0_freq')(x)
x = Permute((1, 3, 2))(x)
# Conv block 1
x = Convolution2D(64, 3, 3, border_mode='same', name='conv1')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn1')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(x)
x = Dropout(0.1, name='dropout1')(x)
# Conv block 2
x = Convolution2D(128, 3, 3, border_mode='same', name='conv2')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn2')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(3, 3), strides=(3, 3), name='pool2')(x)
x = Dropout(0.1, name='dropout2')(x)
# Conv block 3
x = Convolution2D(128, 3, 3, border_mode='same', name='conv3')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn3')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool3')(x)
x = Dropout(0.1, name='dropout3')(x)
# Conv block 4
x = Convolution2D(128, 3, 3, border_mode='same', name='conv4')(x)
x = BatchNormalization(axis=channel_axis, mode=0, name='bn4')(x)
x = ELU()(x)
x = MaxPooling2D(pool_size=(4, 4), strides=(4, 4), name='pool4')(x)
x = Dropout(0.1, name='dropout4')(x)
# reshaping
if K.image_dim_ordering() == 'th':
x = Permute((3, 2, 1))(x)
x = Reshape((15, 128))(x)
# GRU block 1, 2, output
x = GRU(32, return_sequences=True, name='gru1')(x)
x = GRU(32, return_sequences=False, name='gru2')(x)
x = Dropout(0.3)(x)
if include_top:
x = Dense(params["n_out"], activation=params['final_activation'], name='output')(x)
if params['final_activation'] == 'linear':
reg = Lambda(lambda x :K.l2_normalize(x, axis=1))
x = reg(x)
# Create model
model = Model(melgram_input, x)
return model
test_data = X[trainSize:len(training_data), :, :]
test_labels = y[trainSize:len(training_data), :]
test_data2 = test_features[trainSize:len(training_data), :]
test_labels2 = labels[trainSize:len(training_data)]
#network
#tanh/sigmoid scale[0,1]/[-1,1]
#class_weight=class_weight.compute_class_weight('balanced',np.unique(labels),labels)
nb_epoch = 5
BS = 64
my_init = keras.initializers.glorot_uniform(seed=seed)
model = Sequential()
model.add(
Masking(mask_value=0,
input_shape=(train_data.shape[1], train_data.shape[2])))
model.add(GRU(units=200, kernel_initializer=my_init, stateful=False))
model.add(Activation('tanh'))
#model.add(Dropout(0.2))
'''
model.add(LSTM(units=200, kernel_initializer=my_init, return_sequences=True))
model.add(Activation('tanh'))
'''
model.add(Dense(units=len(itemId2), kernel_initializer=my_init))
#model.add(Dense(units = len(itemId2), kernel_initializer=my_init))
model.add(Activation('softmax')) #Output
#model.add(Activation('tanh'))#Output
#activation function
#sgd = optimizers.Adagrad(lr=0.001, epsilon=None, decay=1e-6)
#sgd = optimizers.SGD(lr=0.001, decay=1e-6, momentum=0.98, nesterov=True )
#sgd = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-8, decay=0.0)
#sgd = optimizers.adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=1e-6, amsgrad=False )
eij = K.tanh(K.dot(x, self.W))
ai = K.exp(eij)
weights = ai / K.sum(ai, axis=1).dimshuffle(0, 'x')
weighted_input = x * weights.dimshuffle(0, 1, 'x') #.decode('utf-8')
print(weighted_input)
return weighted_input.sum(axis=1)
def get_output_shape_for(self, input_shape):
return (input_shape[0], input_shape[-1])
sentence_input = Input(shape=(max_text_len, ), dtype='int32')
embedded_sequences = embedding_layer(sentence_input)
l_lstm = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences)
l_dense = TimeDistributed(Dense(200))(l_lstm)
l_att = AttLayer()(l_dense)
sentEncoder = Model(sentence_input, l_att)
review_input = Input(shape=(max_text_num, max_text_len), dtype='int32')
review_encoder = TimeDistributed(sentEncoder)(review_input)
l_lstm_sent = Bidirectional(GRU(100, return_sequences=True))(review_encoder)
l_dense_sent = TimeDistributed(Dense(200))(l_lstm_sent)
l_att_sent = AttLayer()(l_dense_sent)
preds = Dense(2, activation='softmax')(l_att_sent)
model = Model(review_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
def get_model():
#Inputs
name = Input(shape=[X_train["name"].shape[1]], name="name")
item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
brand = Input(shape=[1], name="brand")
#category = Input(shape=[1], name="category")
category_name_split = Input(
shape=[X_train["category_name_split"].shape[1]],
name="category_name_split")
item_condition = Input(shape=[1], name="item_condition")
num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
#Embeddings layers
emb_size = 60
emb_name = Embedding(MAX_TEXT, emb_size)(name) # , mask_zero=True
emb_item_desc = Embedding(MAX_TEXT,
emb_size)(item_desc) # , mask_zero=True#
emb_category_name_split = Embedding(MAX_TEXT, emb_size // 3)(
category_name_split) # , mask_zero=True
emb_brand = Embedding(MAX_BRAND, 8)(brand)
emb_item_condition = Embedding(MAX_CONDITION, 5)(item_condition)
conv1 = Conv1D(16, 3, activation='relu', padding="same")(emb_item_desc)
conv1 = MaxPooling1D(2)(conv1)
conv1 = Conv1D(32, 3, activation='relu', padding="same")(conv1)
conv1 = MaxPooling1D(2)(conv1)
conv1 = Conv1D(32, 3, activation='relu', padding="same")(conv1)
conv1 = MaxPooling1D(2)(conv1)
conv2 = Conv1D(16, 3, activation='relu',
padding="same")(emb_category_name_split)
conv2 = MaxPooling1D(2)(conv2)
conv2 = Conv1D(32, 3, activation='relu', padding="same")(conv2)
#conv2 = MaxPooling1D(2)(conv2)
conv3 = Conv1D(16, 3, activation='relu', padding="same")(emb_name)
conv3 = MaxPooling1D(2)(conv3)
conv3 = Conv1D(32, 3, activation='relu', padding="same")(conv3)
#conv3 = MaxPooling1D(2)(conv3)
rnn_layer3 = GRU(32,
recurrent_dropout=0.0)(concatenate([conv3, conv2, conv1]))
#main layer
main_l = concatenate([
Flatten()(emb_brand)
#, Flatten() (emb_category)
,
Flatten()(emb_item_condition)
#, rnn_layer1
#, rnn_layer2
,
rnn_layer3,
num_vars
])
main_l = Dropout(dr)(Dense(512, activation='relu')(main_l))
main_l = Dropout(dr)(Dense(64, activation='relu')(main_l))
#output
output = Dense(1, activation="linear")(main_l)
#model
model = Model(
[
name,
item_desc,
brand #, cat1, cat2, cat3
,
category_name_split # category_name, category,
,
item_condition,
num_vars
],
output)
#optimizer = optimizers.RMSprop()
optimizer = optimizers.Adam()
model.compile(loss='mse', optimizer=optimizer)
return model
maxlen = np.max([len(x) for x in T])
print(maxlen)
# Convert characters to int and pad
T = [[valid_chars[y] for y in x] for x in T]
X_test = sequence.pad_sequences(T, maxlen=max_len)
#y_train = np.array(trainlabel)
y_test = np.array(testlabel)
embedding_vecor_length = 128
model = Sequential()
model.add(Embedding(max_features, embedding_vecor_length,
input_length=max_len))
model.add(GRU(128))
model.add(Dropout(0.1))
model.add(Dense(1))
model.add(Activation('sigmoid'))
'''
model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
checkpointer = callbacks.ModelCheckpoint(filepath="logs/gru/checkpoint-{epoch:02d}.hdf5", verbose=1, save_best_only=True, monitor='val_acc',mode='max')
csv_logger = CSVLogger('logs/gru/training_set_gruanalysis.csv',separator=',', append=False)
model.fit(X_train, y_train, batch_size=32, nb_epoch=1000,validation_split=0.33, shuffle=True,callbacks=[checkpointer,csv_logger])
score, acc = model.evaluate(X_test, y_test, batch_size=128)
print('Test score:', score)
print('Test accuracy:', acc)
'''
embedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
l_gru = Bidirectional(GRU(100, return_sequences=True))(embedded_sequences)
l_att = AttLayer()(l_gru)
preds = Dense(2, activation='softmax')(l_att)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
print("model fitting - attention GRU network")
model.summary()
model.fit(x_train,
y_train,
validation_data=(x_val, y_val),
nb_epoch=10,
batch_size=50)
# pad documents to a max length of 4 words
tpadded_docs = pad_sequences(tencoded_docs, maxlen=max_length, padding='post')
#print(padded_docs)
# define model
model = Sequential()
e = Embedding(vocab_size,
100,
weights=[embedding_matrix],
input_length=max_length,
trainable=False)
model.add(e)
model.add(
Bidirectional(
GRU(gru_output_size,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)))
model.add(Dense(nclass, activation='softmax'))
# compile the model
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
# summarize the model
print(model.summary())
# fit the model
model.fit(padded_docs,
y_train,
epochs=1,
verbose=0,
validation_data=(vpadded_docs, y_valid))
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.5,
random_state=0)
#Reshape for LSTM
X_train = X_train.reshape(X_train.shape + (1, ))
X_test = X_test.reshape(X_test.shape + (1, ))
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
# create the model
model = Sequential()
model.add(GRU(200, input_shape=X_train.shape[1:]))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
history = model.fit(X_train,
y_train,
batch_size=64,
nb_epoch=3,
validation_data=(X_test, y_test))
"""embedding_vecor_length = 32
model = Sequential()
model.add(Embedding(1000, embedding_vecor_length, input_length=10))
model.add(Convolution1D(nb_filter=32, filter_length=3, border_mode='same', activation='relu'))
model.add(MaxPooling1D(pool_length=2))
mask_zero=mask)
sequence_input_c = Input(shape=(
padsize,
MAX_WORD_LENGTH,
), dtype='int32')
embedded_sequences_c = embedding_layer_c(sequence_input_c)
rone = Lambda(reshape_one)(embedded_sequences_c)
merge_m = raw_input('Enter merge mode for GRU Karakter: ')
dropout = input('Enter dropout for GRU: ')
rec_dropout = dropout # input('Enter GRU Karakter recurrent dropout: ')
gru_karakter = Bidirectional(GRU(CHAR_EMBEDDING_DIM,
return_sequences=False,
dropout=dropout,
recurrent_dropout=rec_dropout),
merge_mode=merge_m,
weights=None)(rone)
rtwo = Lambda(reshape_two)(gru_karakter)
"""
Combine word + char model
"""
from keras.layers import Add, Subtract, Multiply, Average, Maximum
print "Model Choice:"
model_choice = input('Enter 1 for WE only, 2 for CE only, 3 for both: ')
merge_m = raw_input('Enter merge mode for GRU Kata: ')
# dropout = input('Enter GRU Karakter dropout: ')
# rec_dropout = input('Enter GRU Karakter recurrent dropout: ')
max_sequence_length = max_recipe_length + 1
max_name_sequence_length = max_name_length + 1
print('Building Inference model...')
hiddenStateSize = 512
hiddenStateSize2 = 256
hiddenStateSize3 = 128
hiddenLayerSize = 128
recipe_node = Sequential()
recipe_node.add(
GRU(hiddenStateSize,
batch_input_shape=(1, max_sequence_length, len(char2id)),
stateful=True))
recipe_node.add(Dense(hiddenStateSize3))
recipe_node.add(Activation('relu'))
title_node = Sequential()
title_node.add(
GRU(hiddenStateSize2,
batch_input_shape=(1, 1, len(char2id)),
stateful=True))
title_node.add(Dense(hiddenStateSize3))
title_node.add(Activation('relu'))
inference_model = Sequential()
inference_model.add(
def cinq_wordEmb_model(args, embedding_matrix, isTeacherForcing=True):
'''Inputs P and Q'''
P_Input_word = Input(shape=(args.c_max_len, ), name='Pword')
Q_Input_word = Input(shape=(args.q_max_len, ), name='Qword')
P_Input_char = Input(shape=(args.c_max_len, ), name='Pchar')
Q_Input_char = Input(shape=(args.q_max_len, ), name='Qchar')
'''Input context vector'''
cinq_vector = Input(shape=(args.c_max_len, args.context_vector_level))
gru_cinq = Bidirectional(GRU(64, return_sequences=True))(cinq_vector)
punctuation_vector = Input(shape=(args.c_max_len, args.punctuation_level))
P_char_encode, P_word_encode, passage_encoding, \
Q_char_encode, Q_word_encode, question_encoding \
= sentence_encode(args, P_Input_word, Q_Input_word, P_Input_char, Q_Input_char, embedding_matrix)
q_char_attention_vector, q_word_attention_vector, question_attention_vector \
= get_attentions(args, P_char_encode, P_word_encode, passage_encoding,
Q_char_encode, Q_word_encode, question_encoding)
# Answer start prediction
output_start = Lambda(
lambda arg: concatenate([arg[i] for i in range(len(arg))]))([
passage_encoding, q_char_attention_vector, q_word_attention_vector,
question_attention_vector, gru_cinq, punctuation_vector,
multiply([P_char_encode, q_char_attention_vector]),
multiply([P_word_encode, q_word_attention_vector]),
multiply([passage_encoding, question_attention_vector])
])
output_start = TimeDistributed(Dense(args.hidden_size,
activation='relu'))(output_start)
output_start = TimeDistributed(Dense(1))(output_start)
output_start = Flatten()(output_start)
output_start = Activation(K.softmax)(output_start)
'''Inputs answer_start '''
answer_start_input = Input(shape=(args.c_max_len, ))
if isTeacherForcing == True:
answer_start = answer_start_input
else:
answer_start = output_start
# answer_start = Reshape((args.c_max_len, 1), input_shape=(args.c_max_len, ))(answer_start)
# Answer end prediction depends on the start prediction
def s_answer_feature(x):
maxind = K.argmax(
x,
axis=1,
)
return maxind
x = Lambda(lambda x: K.tf.cast(s_answer_feature(x), dtype=K.tf.int32))(
answer_start)
start_feature = Lambda(lambda arg: K.tf.gather_nd(
arg[0],
K.tf.stack(
[K.tf.range(K.tf.shape(arg[1])[0]),
K.tf.cast(arg[1], K.tf.int32)],
axis=1)))([passage_encoding, x])
start_feature = RepeatVector(args.c_max_len)(start_feature)
start_position = Lambda(
lambda x: K.tf.one_hot(K.argmax(x), args.c_max_len))(answer_start)
start_position = Reshape((args.c_max_len, 1),
input_shape=(args.c_max_len, ))(start_position)
start_position = Bidirectional(GRU(8,
return_sequences=True))(start_position)
# Answer end prediction
output_end = Lambda(
lambda arg: concatenate([arg[i] for i in range(len(arg))]))([
start_position, start_feature, passage_encoding,
q_char_attention_vector, q_word_attention_vector,
question_attention_vector, gru_cinq, punctuation_vector,
multiply([P_char_encode, q_char_attention_vector]),
multiply([P_word_encode, q_word_attention_vector]),
multiply([passage_encoding, question_attention_vector]),
multiply([passage_encoding, start_feature])
])
output_end = TimeDistributed(Dense(args.hidden_size,
activation='relu'))(output_end)
output_end = TimeDistributed(Dense(1))(output_end)
output_end = Flatten()(output_end)
output_end = Activation(K.softmax)(output_end)
# define model in/out and compile
inputs = [
cinq_vector, punctuation_vector, P_Input_word, Q_Input_word,
P_Input_char, Q_Input_char, answer_start_input
]
outputs = [output_start, output_end]
model = Model(inputs, outputs)
model.compile(optimizer=args.optimizer, loss=args.loss, metrics=['acc'])
return model
def get_model(height, nclass):
input = Input(shape=(height, None, 1), name='the_input')
m = Conv2D(64,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv1')(input)
m = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool1')(m)
m = Conv2D(128,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv2')(m)
m = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), name='pool2')(m)
m = Conv2D(256,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv3')(m)
m = Conv2D(256,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv4')(m)
m = ZeroPadding2D(padding=(0, 1))(m)
m = MaxPooling2D(pool_size=(2, 2),
strides=(2, 1),
padding='valid',
name='pool3')(m)
m = Conv2D(512,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv5')(m)
m = BatchNormalization(axis=1)(m)
m = Conv2D(512,
kernel_size=(3, 3),
activation='relu',
padding='same',
name='conv6')(m)
m = BatchNormalization(axis=1)(m)
m = ZeroPadding2D(padding=(0, 1))(m)
m = MaxPooling2D(pool_size=(2, 2),
strides=(2, 1),
padding='valid',
name='pool4')(m)
m = Conv2D(512,
kernel_size=(2, 2),
activation='relu',
padding='valid',
name='conv7')(m)
m = Permute((2, 1, 3), name='permute')(m)
m = TimeDistributed(Flatten(), name='timedistrib')(m)
m = Bidirectional(GRU(rnnunit, return_sequences=True), name='blstm1')(m)
m = Dense(rnnunit, name='blstm1_out', activation='linear')(m)
m = Bidirectional(GRU(rnnunit, return_sequences=True), name='blstm2')(m)
y_pred = Dense(nclass, name='blstm2_out', activation='softmax')(m)
basemodel = Model(inputs=input, outputs=y_pred)
labels = Input(name='the_labels', shape=[
None,
], dtype='float32')
input_length = Input(name='input_length', shape=[1], dtype='int64')
label_length = Input(name='label_length', shape=[1], dtype='int64')
loss_out = Lambda(ctc_lambda_func, output_shape=(1, ),
name='ctc')([y_pred, labels, input_length, label_length])
model = Model(inputs=[input, labels, input_length, label_length],
outputs=[loss_out])
sgd = SGD(lr=0.001, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5)
#model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer='adadelta')
model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=sgd)
model.summary()
return model, basemodel
def __init__(self, param_dict, Item_size):
config = tf.ConfigProto(
intra_op_parallelism_threads=THREAD,
inter_op_parallelism_threads=THREAD,
allow_soft_placement=True,
)
session = tf.Session(config=config)
backend.set_session(session)
self.batch_size = int(param_dict['batch_size'] * 1.25)
hand_feature_cols = len(Item_size['hand_feature'])
name_seq_len = param_dict['name_Len']
desc_seq_len = param_dict['description_Len']
denselayer_units = param_dict['denselayer_units']
embed_name = param_dict['embed_name']
embed_desc = param_dict['embed_desc']
embed_brand = param_dict['embed_brand']
embed_cat_2 = param_dict['embed_cat_2']
embed_cat_3 = param_dict['embed_cat_3']
rnn_dim_name = param_dict['rnn_dim_name']
rnn_dim_desc = param_dict['rnn_dim_desc']
dense_drop = param_dict['dense_drop']
name_voc_size = Item_size['name']
desc_voc_size = Item_size['item_description']
brand_voc_size = Item_size['brand_name']
cat1_voc_size = Item_size['category_1']
cat2_voc_size = Item_size['category_2']
cat3_voc_size = Item_size['category_name']
# Inputs
X_seq_name = Input(shape=[name_seq_len],
name="X_seq_name",
dtype='int32')
X_seq_item_description = Input(shape=[desc_seq_len],
name="X_seq_item_description",
dtype='int32')
X_brand_name = Input(shape=[1], name="X_brand_name", dtype='int32')
X_category_1 = Input(shape=[1], name="X_category_1", dtype='int32')
X_category_2 = Input(shape=[1], name="X_category_2", dtype='int32')
X_category_name = Input(shape=[1],
name="X_category_name",
dtype='int32')
X_item_condition_id = Input(shape=[1],
name="X_item_condition_id",
dtype='uint8')
X_shipping = Input(shape=[1], name="X_shipping", dtype='float32')
X_hand_feature = Input(shape=[hand_feature_cols],
name="X_hand_feature",
dtype='float32')
# Embeddings layers
name = Embedding(name_voc_size, embed_name)(X_seq_name)
item_desc = Embedding(desc_voc_size,
embed_desc)(X_seq_item_description)
brand = Embedding(brand_voc_size, embed_brand)(X_brand_name)
cat_2 = Embedding(cat2_voc_size, embed_cat_2)(X_category_2)
cat_3 = Embedding(cat3_voc_size, embed_cat_3)(X_category_name)
# RNN layers
name = GRU(rnn_dim_name)(name)
item_desc = GRU(rnn_dim_desc)(item_desc)
# OneHot layers
cond = Lambda(one_hot,
arguments={'num_classes': 5},
output_shape=(1, 5))(X_item_condition_id)
cat_1 = Lambda(one_hot,
arguments={'num_classes': cat1_voc_size},
output_shape=(1, cat1_voc_size))(X_category_1)
# main layer
main_l = concatenate([
name,
item_desc,
Flatten()(cat_1),
Flatten()(cat_2),
Flatten()(cat_3),
Flatten()(brand),
Flatten()(cond),
X_shipping,
X_hand_feature,
])
main_l = Dropout(dense_drop)(Dense(denselayer_units,
activation='relu')(main_l))
output = Dense(1, activation="linear")(main_l)
# model
model = Model([
X_seq_name, X_seq_item_description, X_brand_name, X_category_1,
X_category_2, X_category_name, X_item_condition_id, X_shipping,
X_hand_feature
], output)
optimizer = optimizers.Adam(lr=param_dict['lr'])
model.compile(loss='mse', optimizer=optimizer)
self.model = model
embedding_vector = embedding_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
print(num_words)
#define model
model_nn = Sequential()
embedding_layer = Embedding(num_words,
EMBEDDING_DIM,
embeddings_initializer=Constant(embedding_matrix),
input_length=max_length,
trainable=False)
model_nn.add(embedding_layer)
model_nn.add(GRU(units=32, dropout=0.2, recurrent_dropout=0.2))
model_nn.add(Dense(1, activation='sigmoid'))
model_nn.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model_nn.summary()
VALIDATION_SPLIT = 0.2
indices = np.arange(review_pad.shape[0])
#np.random.shuffle(indices)
review_pad = review_pad[indices]
sentiment = sentiment[indices]
num_validation_samples = int(VALIDATION_SPLIT * review_pad.shape[0])
X = X.reshape((X.shape[0], X.shape[1], n_features))
X_train, X_test = X[:split], X[split:]
y_train, y_test = y[:split], y[split:]
model = Sequential()
# model.add(LSTM(100,activation='relu',input_shape=(n_steps_in,n_features)))
# model.add(LSTM(100,activation='relu',return_sequences=True,input_shape=(n_steps_in,n_features)))
# model.add(LSTM(100,activation='relu'))
# model.add(Bidirectional( LSTM(200,activation='relu',input_shape=(n_steps_in,n_features)) ))
model.add(
GRU(150,
activation='relu',
return_sequences=True,
input_shape=(n_steps_in, n_features)))
model.add(GRU(150, activation='relu', return_sequences=True))
model.add(GRU(150, activation='relu'))
model.add(Dense(n_steps_out))
model.compile(loss='mse', optimizer='adam')
model.fit(X_train,
y_train,
epochs=50,
batch_size=1000,
shuffle=False,
verbose=1)
y_pred = model.predict(X_test)
print(decoder_inputs[:,:,0]) decoder_lstm = LSTM(n_units, return_sequences=True, return_state=True) decoder_outputs, _, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states) decoder_dense = Dense(n_headway_features,activation='relu') decoder_outputs = decoder_dense(decoder_outputs) model = Model(inputs = [encoder_inputs, decoder_inputs],outputs = decoder_outputs) ''' encoder_inputs = Input(shape=(None, n_stop_features)) encoder = GRU(n_units, return_state=True) encoder_outputs, state_h = encoder(encoder_inputs) decoder_inputs = Input(shape=(None, n_headway_features)) decoder_gru = GRU(n_units, return_sequences=True) decoder_outputs = decoder_gru(decoder_inputs, initial_state=state_h) decoder_dense = Dense(n_headway_features, activation='relu') decoder_outputs = decoder_dense(decoder_outputs) model = Model([encoder_inputs, decoder_inputs], decoder_outputs) model.compile(optimizer=optimiser, loss='mse', metrics=['acc']) X1_test_stopfeatures, X2_test_headway, y_test_target_headway = get_test_dataset( ) model.fit([X1_train_stopfeatures, X2_train_headway],
def build_seq2seq_model(word_emb_dim, hidden_state_dim, encoder_seq_len,
num_encoder_tokens, num_decoder_tokens):
"""
Builds architecture for sequence to sequence model.
Encoder and Decoder layer consist of one GRU Layer each. User
can specify the dimensionality of the word embedding and the hidden state.
Parameters
----------
word_emb_dim : int
dimensionality of the word embeddings
hidden_state_dim : int
dimensionality of the hidden state in the encoder and decoder.
encoder_seq_len : int
the length of the sequences that are input into the encoder. The
sequences are expected to all be padded to the same size.
num_encoder_tokens : int
the vocabulary size of the corpus relevant to the encoder.
num_decoder_tokens : int
the vocabulary size of the corpus relevant to the decoder.
Returns
-------
Keras.models.Model
"""
#### Encoder Model ####
encoder_inputs = Input(shape=(encoder_seq_len, ), name='Encoder-Input')
# Word embeding for encoder (ex: Issue Titles, Code)
x = Embedding(num_encoder_tokens,
word_emb_dim,
name='Body-Word-Embedding',
mask_zero=False)(encoder_inputs)
x = BatchNormalization(name='Encoder-Batchnorm-1')(x)
# We do not need the `encoder_output` just the hidden state.
_, state_h = GRU(hidden_state_dim,
return_state=True,
name='Encoder-Last-GRU',
dropout=.5)(x)
# Encapsulate the encoder as a separate entity so we can just
# encode without decoding if we want to.
encoder_model = Model(inputs=encoder_inputs,
outputs=state_h,
name='Encoder-Model')
seq2seq_encoder_out = encoder_model(encoder_inputs)
#### Decoder Model ####
decoder_inputs = Input(shape=(None, ),
name='Decoder-Input') # for teacher forcing
# Word Embedding For Decoder (ex: Issue Titles, Docstrings)
dec_emb = Embedding(num_decoder_tokens,
word_emb_dim,
name='Decoder-Word-Embedding',
mask_zero=False)(decoder_inputs)
dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb)
# Set up the decoder, using `decoder_state_input` as initial state.
decoder_gru = GRU(hidden_state_dim,
return_state=True,
return_sequences=True,
name='Decoder-GRU',
dropout=.5)
decoder_gru_output, _ = decoder_gru(dec_bn,
initial_state=seq2seq_encoder_out)
x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output)
# Dense layer for prediction
decoder_dense = Dense(num_decoder_tokens,
activation='softmax',
name='Final-Output-Dense')
decoder_outputs = decoder_dense(x)
#### Seq2Seq Model ####
seq2seq_Model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return seq2seq_Model