print("\nCompiling model\n")
tgt = Input(shape=(max_sents, SENT_DIM), dtype='float32')
srcs = Input(shape=(max_sents, SENT_DIM), dtype='float32')
encode = Bidirectional(LSTM(SENT_DIM / 2,
return_sequences=False,
dropout_W=0.0,
dropout_U=0.0),
input_shape=(max_sents, SENT_DIM))
tgt_vec = encode(tgt)
src_vec = encode(srcs)
# attention = _Attention(max_sents,SENT_DIM,dropout=0.2)(tgt,srcs)
# align1= _SoftAlignment(max_sents,SENT_DIM)(srcs,attention)
# align2= _SoftAlignment(max_sents,SENT_DIM)(tgt,attention,transpose=True)
# vec_l = _Comparison(max_sents,SENT_DIM,dropout=0.2)(tgt,align1)
# vec_r = _Comparison(max_sents,SENT_DIM,dropout=0.2)(srcs,align2)
pds = _Entailment(SENT_DIM, NUM_CLASSES, dropout=0.4)(tgt_vec, src_vec)
model = Model(inputs=[tgt, srcs], outputs=pds)
model.summary()
model.compile(loss="categorical_crossentropy",
optimizer=Adam(lr=0.001),
metrics=["accuracy"])
#get_attention_matrix = K.function([model.layers[0].input,model.layers[1].input,K.learning_phase()],[model.layers[3].output])
NUM_EPOCHS = 10
BATCH_SIZE = 10
MODEL_NAME = "dlnd_bilstm_mlp_best_model_fold_" + str(fold - 1) + ".h5"
train_target_vecs, valid_target_vecs, train_source_vecs, valid_source_vecs, train_golds, valid_golds = train_test_split(
target_vecs[train],
source_vecs[train],
gold[train],
def process_topic():
# LOG.debug('Processing topic: %s', topic)
# ids = [idx for idx in range(len(subtopics)) if subtopics[idx].split('/')[0] == topic]
# tgt_ids = [all_tgt_ids[idx] for idx in ids]
# src_ids = [all_src_ids[idx] for idx in ids]
# golds = [all_golds[idx] for idx in ids]
tgt_ids = all_tgt_ids
src_ids = all_src_ids
golds = all_golds
max_sents= max([len(doc) for doc in src_ids + tgt_ids])
LOG.debug("Max sentences: %d", max_sents)
# target_vecs = [np.vstack([vocab[sent_idx] for sent_idx in doc]) for doc in tgt_ids]
# source_vecs = [np.vstack([vocab[sent_idx] for sent_idx in doc]) for doc in src_ids]
# target_vecs = np.array([pad_or_truncate1(mat,max_sents) for mat in target_vecs])
# source_vecs = np.array([pad_or_truncate1(mat,max_sents) for mat in source_vecs])
tgt_ids = np.array([pad_or_truncate1(doc, max_sents) for doc in tgt_ids])
src_ids = np.array([pad_or_truncate1(doc, max_sents) for doc in src_ids])
gold_list = [i for i in golds]
golds = to_categorical(golds)
train, test = train_test_split(
list(range(len(gold_list))),
train_size=0.8,
random_state=9274,
shuffle=True,
stratify=gold_list)
LOG.debug("Compiling model")
emb = Embedding(
vocab.shape[0],
vocab.shape[1],
weights=[vocab],
input_length=max_sents,
trainable=False)
# tgt = Input(shape=(max_sents,SENT_DIM), dtype='float32')
# srcs = Input(shape=(max_sents,SENT_DIM), dtype='float32')
tgt = Input(shape=(max_sents,), dtype='int32')
with tf.device('/device:CPU:0'):
tgt_emb = emb(tgt)
srcs = Input(shape=(max_sents,), dtype='int32')
with tf.device('/device:CPU:0'):
srcs_emb = emb(srcs)
encode = Bidirectional(LSTM(SENT_DIM/2, return_sequences=False,
dropout_W=0.0, dropout_U=0.0),
input_shape=(max_sents, SENT_DIM))
# tgt_vec = encode(tgt)
# src_vec = encode(srcs)
tgt_vec = encode(tgt_emb)
src_vec = encode(srcs_emb)
pds = _Entailment(SENT_DIM,NUM_CLASSES,dropout=0.2)(tgt_vec,src_vec)
model = Model(input=[tgt,srcs],output=pds)
model.summary()
model.compile(loss="categorical_crossentropy",optimizer=Adam(lr=0.001),metrics=["accuracy"])
NUM_EPOCHS = 25
BATCH_SIZE = 32
LOG.debug("Training model")
model.fit(x=[tgt_ids[train],src_ids[train]],y=golds[train],batch_size=BATCH_SIZE,nb_epoch=NUM_EPOCHS,shuffle=True,verbose=2)
preds = model.predict([tgt_ids[test],src_ids[test]])
preds = np.argmax(preds,axis=1)
gold_test = np.argmax(golds[test],axis=1)
test_acc = accuracy_score(gold_test, preds)
LOG.debug("Testing accuracy: %0.3f", test_acc)
LOG.debug("Confusion matrix:\n%s\n\n", confusion_matrix(gold_test,preds))
# Write completed topic
# with open(completed_topics_file, 'a') as fh:
# fh.write('\n{}'.format(topic))
# Cleanup memory
del model
gc.collect()
K.clear_session()
tf.reset_default_graph()
if fold <= folds_complete:
fold += 1
continue
print("\nFold: " + str(fold))
fold += 1
print("\nCompiling model\n")
tgt = Input(shape=(max_sents, SENT_DIM), dtype='float32')
srcs = Input(shape=(max_sents, SENT_DIM), dtype='float32')
attention = _Attention(max_sents, SENT_DIM, dropout=0.2)(tgt, srcs)
align1 = _SoftAlignment(max_sents, SENT_DIM)(srcs, attention)
align2 = _SoftAlignment(max_sents, SENT_DIM)(tgt,
attention,
transpose=True)
vec_l = _Comparison(max_sents, SENT_DIM, dropout=0.2)(tgt, align1)
vec_r = _Comparison(max_sents, SENT_DIM, dropout=0.2)(srcs, align2)
pds = _Entailment(SENT_DIM, NUM_CLASSES, dropout=0.2)(vec_l, vec_r)
model = Model(inputs=[tgt, srcs], outputs=pds)
model.summary()
model.compile(loss="categorical_crossentropy",
optimizer=Adam(lr=0.001),
metrics=["accuracy"])
cb = [
ModelCheckpoint("temp2_model.hdf5",
monitor="val_loss",
verbose=1,
save_weights_only=True,
save_best_only=True)
]
get_attention_matrix = K.function(