def load(cls, model_path, nlp, sentence_length, get_features):
with (model_path / 'config.json').open() as file_:
model = model_from_json(file_.read(), custom_objects={'Attention': Attention, 'AttentionWithContext': AttentionWithContext})
with (model_path / 'model').open('rb') as file_:
weights = pickle.load(file_)
embeddings, vocab = get_embeddings(nlp.vocab)
model.set_weights([embeddings] + weights)
return cls(model, vocab, sentence_length, get_features)
def train(train_texts,
train_labels,
dev_texts,
dev_labels,
sentence_length=100,
batch_size=100,
nb_epoch=5):
embeddings, vocab = get_embeddings(nlp.vocab)
print embeddings.shape
print("Creating model...")
model, config = create_model(embeddings, sentence_length=sentence_length)
model.compile(optimizer='adam',
loss='mse',
loss_weights=[1.],
metrics=['accuracy'])
print(model.summary())
print("Creating input...")
train_X = get_features(nlp, train_texts, vocab, sentence_length)
dev_X = get_features(nlp, dev_texts, vocab, sentence_length)
# print train_X['input_1'][:2]
# print train_X['input_2'][:2]
# print train_labels[:2]
# print dev_X['input_1'][:2]
# print dev_X['input_2'][:2]
# print dev_labels[:2]
print("Class weights...")
from sklearn.utils import class_weight
class_weight = class_weight.compute_class_weight('balanced',
np.unique(train_labels),
train_labels)
class_weight = dict(zip(xrange(len(class_weight)), class_weight))
print class_weight
print("Train...")
model.fit(train_X,
train_labels,
validation_data=(dev_X, dev_labels),
epochs=nb_epoch,
batch_size=batch_size,
class_weight=class_weight)
return model
def evaluate(model_path, texts, labels, sentence_length=100):
from keras.models import model_from_json
import pickle
from spacy_data import get_embeddings
from attention_layer import Attention, AttentionWithContext
with (model_path / 'config.json').open() as file_:
model = model_from_json(file_.read(),
custom_objects={
'Attention': Attention,
'AttentionWithContext':
AttentionWithContext
})
with (model_path / 'model').open('rb') as file_:
weights = pickle.load(file_)
embeddings, vocab = get_embeddings(nlp.vocab)
model.set_weights([embeddings] + weights)
Xs = get_features(nlp, texts, vocab, sentence_length)
acc = precision = recall = f1 = 0
tp = 0
tn = 0
fp = 0
fn = 0
t = 0.5
ys = model.predict(Xs)
for i, y in enumerate(ys):
# print y, labels[i]
claim_score = y[0]
tp += claim_score >= t and bool(labels[i]) == True
tn += claim_score < t and bool(labels[i]) == False
fp += claim_score >= t and bool(labels[i]) == False
fn += claim_score < t and bool(labels[i]) == True
acc = float(tp + tn) / (tp + tn + fp + fn)
precision = float(tp) / (tp + fp)
recall = float(tp) / (tp + fn)
f1 = float(2 * tp) / (2 * tp + fp + fn)
return acc, precision, recall, f1
def train(train_texts,
train_labels,
dev_texts,
dev_labels,
test_texts,
test_labels,
sentence_length=100,
batch_size=100,
nb_epoch=5):
embeddings, vocab = get_embeddings(nlp.vocab)
print embeddings.shape
print("Creating model...")
from sklearn import svm
from spacy_features import anygram_kernel
# 0.2 -- 0.3 0.754
# 1.0 -- 0.1 0.75
for c, l in [(0.8, 0.8)]:
# for c in [0.1, 0.2, 0.3, 0.4, 0.8, 1.0]:
# for l in [0.1, 0.3, 0.5, 0.8, 1.0]:
C = c
clf = svm.SVC(C=C,
kernel='precomputed',
class_weight='balanced',
verbose=True)
print("Parsing texts...")
train_docs = list(nlp.pipe(train_texts))
dev_docs = list(nlp.pipe(dev_texts))
test_docs = list(nlp.pipe(test_texts))
print("Creating input...")
train_X = anygram_kernel(train_docs, train_docs, l=l)
print train_X.shape
# train_X = get_features(nlp, train_texts, vocab, sentence_length)
# dev_X = get_features(nlp, dev_texts, vocab, sentence_length)
print("Class weights...")
from sklearn.utils import class_weight
class_weight = class_weight.compute_class_weight(
'balanced', np.unique(train_labels), train_labels)
class_weight = dict(zip(xrange(len(class_weight)), class_weight))
print class_weight
train_labels = [int(b) for b in train_labels]
dev_labels = [int(b) for b in dev_labels]
test_labels = [int(b) for b in test_labels]
print("Train...")
model = clf.fit(train_X, train_labels)
print("Test...")
# p = model.predict(anygram_kernel(dev_docs[:10], train_docs))
# print p
s = model.score(anygram_kernel(test_docs, train_docs), test_labels)
#s = model.score(anygram_kernel(dev_docs, train_docs, l=l), dev_labels)
print c, l, s
return model