model4.add(Dropout(0.2))
model4.add(Dense(300))
model4.add(Dropout(0.2))
model4.add(BatchNormalization())
model5 = Sequential()
model5.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2))
model5.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
model6 = Sequential()
model6.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2))
model6.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
merged_model = Sequential()
merged_model.add(
Merge([model1, model2, model3, model4, model5, model6], mode='concat'))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
def what_d(runtimes=1, renew=True, maxlen=100, file_id=3):
[
glove, [char_X_100, char_X_010, char_X_001, char_Y_10, char_Y_01],
[word_lens]
] = readfile(file_id=file_id)
char_X_010 = min(char_X_010, maxlen)
vocab = []
X = np.zeros((char_X_100, char_X_010, char_X_001), dtype=np.bool)
y = np.zeros((char_Y_10, char_Y_01), dtype=np.float64)
ii = 0
for i in range(0, word_lens):
ttt = glove[i].split()
ttt_lens = len(ttt)
lists = ["".join(ttt[0:ttt_lens - char_Y_01])
] + ttt[ttt_lens - char_Y_01:]
lists[0] = re.sub("[^0-9a-zA-Z]", "", lists[0].lower())
if 0 < len(lists[0]) <= maxlen:
#print(ii, i)
vocab.append(lists[0])
text = lists[0].ljust(char_X_010)
for j in range(0, char_X_010):
X[ii, j, char_indices[text[j]]] = 1
for k in range(1, char_Y_01 + 1):
y[ii, k - 1] = lists[k]
ii = ii + 1
if i % 40000 == 0:
print(i)
# Find par.
lens = []
for word in vocab:
lens.append(len(word))
print(max(lens)) # min(maxlen, char_X_010)
print(len(vocab)) # 399488
char_X_100 = len(vocab)
char_Y_10 = len(vocab)
X = X[0:len(vocab)]
y = y[0:len(vocab)]
# First time: build the model: a bidirectional SimpleRNN
if renew == True:
print('Build model...')
left = Sequential()
left.add(
GRU(char_Y_01,
input_shape=(char_X_010, char_X_001),
activation='tanh',
inner_activation='sigmoid',
dropout_W=0.5,
dropout_U=0.5))
#dropout_W=0.5, dropout_U=0.5))
right = Sequential()
right.add(
GRU(char_Y_01,
input_shape=(char_X_010, char_X_001),
activation='tanh',
inner_activation='sigmoid',
dropout_W=0.5,
dropout_U=0.5,
go_backwards=True))
#dropout_W=0.5, dropout_U=0.5, go_backwards=True))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense((char_Y_01), activation='sigmoid'))
model.compile('Adadelta', 'MSE', metrics=['accuracy'])
model.fit([X, X], y, batch_size=512, nb_epoch=1)
model.save(path + "layer_2/bi_GRU_merge__dense" + str(file_id) + ".pk")
# Not first time: build the model: a bidirectional LSTM
print('Load model...')
model = load_model(path + "layer_2/bi_GRU_merge__dense" + str(file_id) +
".pk")
for j in range(0, runtimes - 1):
print('Build model...')
model.fit([X, X], y, batch_size=512, nb_epoch=1)
model.save(path + "layer_2/bi_GRU_merge__dense" + str(file_id) + ".pk")
# Test cosine similarity, train set
print('Test cosine similarity, train set')
cos = []
for i in range(0, len(vocab)):
text = vocab[i].ljust(char_X_010)
x = np.zeros((1, char_X_010, char_X_001), dtype=np.bool)
for j in range(0, len(text)):
x[0, j, char_indices[text[j]]] = 1
map_LSTM = model.predict([x, x], verbose=0)
map_GloVe = y[i]
cos.append(1 - spatial.distance.cosine(map_LSTM, map_GloVe))
f = open(path + "layer_2/cosine.txt", 'a')
f.write("20 times bi_GRU_merge__dense" + str(file_id) +
" cosine similarity: " + str(sum(cos) / len(cos)) + "\n")
f.close()
# Test cosine similarity, misspelling
print('Test cosine similarity, misspelling')
cos = []
change_engs = [
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'
]
for i in range(0, len(vocab)):
misspelling = vocab[i]
if len(misspelling) > 4:
loc = int(np.random.uniform(0, 1, 1) * len(misspelling))
cha = int(np.random.uniform(0, 1, 1) * 26)
tem = list(misspelling)
tem[loc] = change_engs[cha]
misspelling = "".join(tem)
text = misspelling.ljust(char_X_010)
x = np.zeros((1, char_X_010, char_X_001), dtype=np.bool)
for j in range(0, len(text)):
x[0, j, char_indices[text[j]]] = 1
map_LSTM = model.predict([x, x], verbose=0)
map_GloVe = y[i]
cos.append(1 - spatial.distance.cosine(map_LSTM, map_GloVe))
f = open(path + "layer_2/cosine.txt", 'a')
f.write("20 times bi_GRU_merge__dense" + str(file_id) +
" misspelling cosine similarity : " + str(sum(cos) / len(cos)) +
", len: " + str(len(cos)) + "\n")
f.close()
# Model for wld
model_wld = Sequential()
model_wld.add(
Dense(numFeatures_wld,
input_dim=numFeatures_wld,
init='glorot_uniform',
activation='relu'))
model_wld.add(GaussianNoise(3))
model_wld.add(Dropout(0.3))
model_wld.add(Dense(numFeatures_wld, activation='relu'))
model_wld.add(Dropout(0.3))
#Merge all models
model_merged = Sequential()
model_merged.add(
Merge([model_bsif, model_lpq, model_wld], mode='concat', concat_axis=1))
model_merged.add(Dense(1, activation='sigmoid'))
#Show some debug output
print(model_merged.summary())
#compile model
opt = Adadelta()
model_merged.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
#fit the model
hist = model_merged.fit(
[feature_train_bsif, feature_train_lpq, feature_train_wld],
target_train_bsif,
def build(self):
enc_size = self.size_of_env_observation()
argument_size = IntegerArguments.size_of_arguments
input_enc = InputLayer(batch_input_shape=(self.batch_size, enc_size),
name='input_enc')
input_arg = InputLayer(batch_input_shape=(self.batch_size,
argument_size),
name='input_arg')
input_prg = Embedding(input_dim=PROGRAM_VEC_SIZE,
output_dim=PROGRAM_KEY_VEC_SIZE,
input_length=1,
batch_input_shape=(self.batch_size, 1))
f_enc = Sequential(name='f_enc')
f_enc.add(Merge([input_enc, input_arg], mode='concat'))
f_enc.add(MaxoutDense(128, nb_feature=4))
self.f_enc = f_enc
program_embedding = Sequential(name='program_embedding')
program_embedding.add(input_prg)
f_enc_convert = Sequential(name='f_enc_convert')
f_enc_convert.add(f_enc)
f_enc_convert.add(RepeatVector(1))
f_lstm = Sequential(name='f_lstm')
f_lstm.add(Merge([f_enc_convert, program_embedding], mode='concat'))
f_lstm.add(
LSTM(256,
return_sequences=False,
stateful=True,
W_regularizer=l2(0.0000001)))
f_lstm.add(Activation('relu', name='relu_lstm_1'))
f_lstm.add(RepeatVector(1))
f_lstm.add(
LSTM(256,
return_sequences=False,
stateful=True,
W_regularizer=l2(0.0000001)))
f_lstm.add(Activation('relu', name='relu_lstm_2'))
# plot(f_lstm, to_file='f_lstm.png', show_shapes=True)
f_end = Sequential(name='f_end')
f_end.add(f_lstm)
f_end.add(Dense(1, W_regularizer=l2(0.001)))
f_end.add(Activation('sigmoid', name='sigmoid_end'))
f_prog = Sequential(name='f_prog')
f_prog.add(f_lstm)
f_prog.add(Dense(PROGRAM_KEY_VEC_SIZE, activation="relu"))
f_prog.add(Dense(PROGRAM_VEC_SIZE, W_regularizer=l2(0.0001)))
f_prog.add(Activation('softmax', name='softmax_prog'))
# plot(f_prog, to_file='f_prog.png', show_shapes=True)
f_args = []
for ai in range(1, IntegerArguments.max_arg_num + 1):
f_arg = Sequential(name='f_arg%s' % ai)
f_arg.add(f_lstm)
f_arg.add(Dense(IntegerArguments.depth, W_regularizer=l2(0.0001)))
f_arg.add(Activation('softmax', name='softmax_arg%s' % ai))
f_args.append(f_arg)
# plot(f_arg, to_file='f_arg.png', show_shapes=True)
self.model = Model([input_enc.input, input_arg.input, input_prg.input],
[f_end.output, f_prog.output] +
[fa.output for fa in f_args],
name="npi")
self.compile_model()
plot(self.model, to_file='model.png', show_shapes=True)
model1.add(input_0)
model2 = Sequential()
model2.add(input_45)
if debug: print model1.output_shape
model = Sequential()
model.add(
Merge(
[model1, model2],
mode=kaggle_input,
output_shape=lambda x: (BATCH_SIZE * 4 * N_INPUT_VARIATION,
NUM_INPUT_FEATURES, PART_SIZE, PART_SIZE),
arguments={
'part_size': PART_SIZE,
'n_input_var': N_INPUT_VARIATION,
'include_flip': False,
'random_flip': True
}))
if debug: print model.output_shape
model.add(fPermute((1, 2, 3, 0)))
if debug: print model.output_shape
model.add(
kerasCudaConvnetConv2DLayer(n_filters=32, filter_size=6,
untie_biases=True))
model4.add(GlobalMaxPooling1D()) model4.add(Dropout(0.2)) model4.add(Dense(200)) model4.add(Dropout(0.2)) model4.add(BatchNormalization()) model5 = Sequential() model5.add(Embedding(len(word_index) + 1, 300, input_length=max_len, dropout=0.2)) model5.add(LSTM(300, dropout_W=0.2, dropout_U=0.2)) model6 = Sequential() model6.add(Embedding(len(word_index) + 1, 300, input_length=max_len, dropout=0.2)) model6.add(LSTM(300, dropout_W=0.2, dropout_U=0.2)) merged_model = Sequential() merged_model.add(Merge([model1, model2, model3, model4], mode='concat')) # model1, model2,, model5, model6 merged_model.add(BatchNormalization()) merged_model.add(Dense(200)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(200)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(200)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2))
model1.add(LSTM(300, dropout_W=0.2, dropout_U=0.2)) model2 = Sequential() model2.add(Embedding(len(word_index) + 1, 300, input_length=40, dropout=0.2)) model2.add(LSTM(300, dropout_W=0.2, dropout_U=0.2)) ''' distance_model = Sequential() distance_model.add(Lambda(euclidean_distance)([model1, model2])) angle_model = Sequential() angle_model.add(Lambda(dot_prod)([model1, model2])) ''' merged_model = Sequential() merged_model.add(Merge([model1, model2], mode='concat')) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU()) merged_model.add(Dropout(0.2)) merged_model.add(BatchNormalization()) merged_model.add(Dense(300)) merged_model.add(PReLU())
tr = records[:splitPoint, ]
ts = records[splitPoint:, ]
L = tr.shape[0]
M = ts.shape[0]
user_count = 6040 + 1
item_count = 3952 + 1
model_left = Sequential()
model_left.add(Embedding(user_count, 20, input_length=1))
model_right = Sequential()
model_right.add(Embedding(item_count, 20, input_length=1))
model = Sequential()
model.add(Merge([model_left, model_right], mode='concat'))
print(model.output_shape)
model.add(Flatten())
print(model.output_shape)
model.add(Dense(20))
model.add(Activation('sigmoid'))
model.add(Dense(1))
model.compile(loss='mean_squared_error',
optimizer=SGD(lr=0.1, momentum=0., decay=0., nesterov=False))
model.fit(
[tr[:, 0].reshape((L, 1)), tr[:, 1].reshape((L, 1))],
tr[:, 2].reshape((L, 1)),
batch_size=32,
nb_epoch=20,
validation_data=([ts[:, 0].reshape((M, 1)), ts[:, 1].reshape(
y_train = x_train['is_duplicate'].apply(pd.to_numeric).values
model = Sequential()
print('Build model...')
model5 = Sequential()
model5.add(Embedding(120000 + 1, 300, input_length=300, dropout=0.2))
model5.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
model6 = Sequential()
model6.add(Embedding(120000 + 1, 300, input_length=300, dropout=0.2))
model6.add(LSTM(300, dropout_W=0.2, dropout_U=0.2))
merged_model = Sequential()
merged_model.add(Merge([model5, model6], mode='concat'))
merged_model.add(BatchNormalization())
merged_model.add(Dense(300))
merged_model.add(PReLU())
merged_model.add(Dropout(0.2))
merged_model.add(BatchNormalization())
#merged_model.add(Dense(300))
#merged_model.add(PReLU())
#merged_model.add(Dropout(0.2))
#merged_model.add(BatchNormalization())
#
#merged_model.add(Dense(300))
#merged_model.add(PReLU())
#merged_model.add(Dropout(0.2))