def on_epoch_end(self, epoch, logs={}):
prtr = ev.eval_sts(self.model.predict(self.train_gr)['classes'],
loader.sts_categorical2labels(self.train_gr['classes']), 'Train', quiet=True).Pearson
prval = ev.eval_sts(self.model.predict(self.val_gr)['classes'],
loader.sts_categorical2labels(self.val_gr['classes']), 'Val', quiet=True).Pearson
print(' train Pearson %f val Pearson %f' % (prtr, prval))
logs['pearson'] = prval
def on_epoch_end(self, epoch, logs={}):
prtr = ev.eval_sts(self.model.predict(self.train_gr)['classes'],
loader.sts_categorical2labels(self.train_gr['classes']), 'Train', quiet=True).Pearson
prval = ev.eval_sts(self.model.predict(self.val_gr)['classes'],
loader.sts_categorical2labels(self.val_gr['classes']), 'Val', quiet=True).Pearson
print(' train Pearson %f val Pearson %f' % (prtr, prval))
logs['pearson'] = prval
def train_and_eval(runid, module_prep_model, c, glove, vocab, gr, grt):
print('Model')
model = build_model(glove, vocab, module_prep_model, c)
print('Training')
# XXX: samples_per_epoch is in brmson/keras fork, TODO fit_generator()?
model.fit(gr, validation_data=grt,
callbacks=[STSPearsonCB(gr, grt),
ModelCheckpoint('sts-weights-'+runid+'-bestval.h5', save_best_only=True, monitor='pearson', mode='max'),
EarlyStopping(monitor='pearson', mode='max', patience=3)],
batch_size=160, nb_epoch=32)
model.save_weights('sts-weights-'+runid+'-final.h5', overwrite=True)
print('Predict&Eval')
model.load_weights('sts-weights-'+runid+'-bestval.h5')
ev.eval_sts(model.predict(gr)['classes'], gr['classes'], 'Train')
ev.eval_sts(model.predict(grt)['classes'], grt['classes'], 'Val')
def eval(self, model):
res = []
for gr, fname in [(self.gr, self.trainf), (self.grv, self.valf), (self.grt, self.testf)]:
if gr is None:
res.append(None)
continue
ypred = self.predict(model, gr)
res.append(ev.eval_sts(ypred, gr['classes'], fname))
return tuple(res)
def eval(self, model):
res = []
for gr, fname in [(self.gr, self.trainf), (self.grv, self.valf), (self.grt, self.testf)]:
if gr is None:
res.append(None)
continue
ypred = model.predict(gr)['classes']
res.append(ev.eval_sts(ypred, gr['classes'], fname))
return tuple(res)
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr,
grv):
# We construct both original and sts model, then copy over
# the weights from the original model
print('Model')
umodel = anssel_train.build_model(glove,
vocab,
module_prep_model,
c,
s0pad=spad,
s1pad=spad,
do_compile=False)
model = sts_train.build_model(glove,
vocab,
module_prep_model,
c,
spad=spad,
optimizer=c['opt'],
fix_layers=c['fix_layers'])
print('Model (weights)')
umodel.load_weights(weightsf)
for n in umodel.nodes.keys():
model.nodes[n].set_weights(umodel.nodes[n].get_weights())
ev.eval_sts(
model.predict(grv)['classes'][:, 0], grv['classes'],
'sts Val (bef. train)')
print('Training')
model.fit(gr,
validation_data=grv,
callbacks=[
STSPearsonCB(gr, grv),
ModelCheckpoint('sts-weights-' + runid + '-bestval.h5',
save_best_only=True),
EarlyStopping(patience=4)
],
batch_size=conf['batch_size'],
nb_epoch=conf['nb_epoch'])
model.save_weights('sts-weights-' + runid + '-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('sts-weights-' + runid + '-bestval.h5')
ev.eval_sts(model.predict(grv)['classes'][:, 0], grv['classes'], 'sts Val')
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr, grv):
# We construct both original and sts model, then copy over
# the weights from the original model
print('Model')
umodel = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=spad, s1pad=spad, do_compile=False)
model = sts_train.build_model(glove, vocab, module_prep_model, c, spad=spad, optimizer=c['opt'], fix_layers=c['fix_layers'])
print('Model (weights)')
umodel.load_weights(weightsf)
for n in umodel.nodes.keys():
model.nodes[n].set_weights(umodel.nodes[n].get_weights())
ev.eval_sts(model.predict(grv)['classes'][:,0], grv['classes'], 'sts Val (bef. train)')
print('Training')
model.fit(gr, validation_data=grv,
callbacks=[STSPearsonCB(gr, grv),
ModelCheckpoint('sts-weights-'+runid+'-bestval.h5', save_best_only=True),
EarlyStopping(patience=4)],
batch_size=conf['batch_size'], nb_epoch=conf['nb_epoch'])
model.save_weights('sts-weights-'+runid+'-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('sts-weights-'+runid+'-bestval.h5')
ev.eval_sts(model.predict(grv)['classes'][:,0], grv['classes'], 'sts Val')
def train_and_eval(runid,
module_prep_model,
c,
glove,
vocab,
gr,
grt,
do_eval=True):
print('Model')
model = build_model(glove, vocab, module_prep_model, c)
print('Training')
# XXX: samples_per_epoch is in brmson/keras fork, TODO fit_generator()?
model.fit(gr,
validation_data=grt,
callbacks=[
STSPearsonCB(gr, grt),
ModelCheckpoint('sts-weights-' + runid + '-bestval.h5',
save_best_only=True,
monitor='pearson',
mode='max'),
EarlyStopping(monitor='pearson', mode='max', patience=3)
],
batch_size=c['batch_size'],
nb_epoch=c['nb_epoch'])
model.save_weights('sts-weights-' + runid + '-final.h5', overwrite=True)
if c['ptscorer'] is None:
model.save_weights('sts-weights-' + runid + '-bestval.h5',
overwrite=True)
model.load_weights('sts-weights-' + runid + '-bestval.h5')
if do_eval:
print('Predict&Eval (best val epoch)')
ev.eval_sts(model.predict(gr)['classes'], gr['classes'], 'Train')
ev.eval_sts(model.predict(grt)['classes'], grt['classes'], 'Val')
return model
args = parser.parse_args()
glove = emb.GloVe(N=args.N)
if args.sick == 1:
Xtrain, ytrain = load_set(glove, 'data/sts/sick2014/SICK_train.txt', loader.load_sick2014)
Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014)
else:
Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*')
Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*')
model = prep_model(glove)
model.compile(loss={'classes': 'categorical_crossentropy'}, optimizer='adam')
model.fit({'e0': Xtrain[0], 'e1': Xtrain[1], 'classes': loader.sts_labels2categorical(ytrain)},
batch_size=20, nb_epoch=100,
validation_data={'e0': Xtest[0], 'e1': Xtest[1], 'classes': loader.sts_labels2categorical(ytest)})
ev.eval_sts(model.predict({'e0': Xtrain[0], 'e1': Xtrain[1]})['classes'], ytrain, 'Train')
ev.eval_sts(model.predict({'e0': Xtest[0], 'e1': Xtest[1]})['classes'], ytest, 'Test')
"""
Performance tuning (100 iters) on sick2014:
* Just elementwise-mul:
4500/4500 [==============================] - 0s - loss: 1.2332 - acc: 0.5016 - val_loss: 1.1926 - val_acc: 0.4770
Train Pearson: 0.721662
Train Spearman: 0.593711
Train MSE: 0.529364
Test Pearson: 0.701661
Test Spearman: 0.571329
Test MSE: 0.540898
prv = []
prt = []
for i in range(niter):
runid = '%s-%x-%02d' % (modelname, h, i)
print('RunID: %s (%s)' % (runid, ps))
model = sts_train.train_and_eval(runid, module.prep_model, conf, glove, vocab, gr, grv, do_eval=False)
print('Predict&Eval (best val epoch)')
ypred = model.predict(gr)['classes']
ypredv = model.predict(grv)['classes']
ypredt = dict()
for ti in range(len(testf)):
ypredt[testf[ti]] = model.predict(lgrt[ti])['classes']
pr.append(ev.eval_sts(ypred, gr['classes'], 'Train'))
prv.append(ev.eval_sts(ypredv, grv['classes'], 'Val'))
dprt = dict()
for ti in range(len(testf)):
dprt[testf[ti]] = ev.eval_sts(ypredt[testf[ti]], lgrt[ti]['classes'], testf[ti])
prt.append(dprt)
rdata = {'ps': ps, 'ypred': (ypred, ypredv, ypredt), 'pr': (pr[-1], prv[-1], prt[-1])}
pickle.dump(rdata, open('%s-res.pickle' % (runid,), 'wb'), protocol=2)
# prt view indexed by testf, not by iteration
prtt = dict()
for ti in range(len(testf)):
prtt[testf[ti]] = [prt0[testf[ti]] for prt0 in prt]
mprt = [np.mean(prtt[testf[ti]]) for ti in range(len(testf))] # (n_testf,) mean across epochs
model.fit(
{
'e0': Xtrain[0],
'e1': Xtrain[1],
'classes': loader.sts_labels2categorical(ytrain)
},
batch_size=20,
nb_epoch=100,
validation_data={
'e0': Xtest[0],
'e1': Xtest[1],
'classes': loader.sts_labels2categorical(ytest)
})
ev.eval_sts(
model.predict({
'e0': Xtrain[0],
'e1': Xtrain[1]
})['classes'], ytrain, 'Train')
ev.eval_sts(
model.predict({
'e0': Xtest[0],
'e1': Xtest[1]
})['classes'], ytest, 'Test')
"""
Performance tuning (100 iters) on sick2014:
* Just elementwise-mul:
4500/4500 [==============================] - 0s - loss: 1.2332 - acc: 0.5016 - val_loss: 1.1926 - val_acc: 0.4770
Train Pearson: 0.721662
Train Spearman: 0.593711
model1.add(RepeatVector(6)) # [nclass]
model = Sequential()
model.add(Merge([model0, model1], mode='dot', dot_axes=([2], [2])))
model.add(Flatten()) # 6x6 matrix with cross-activations -> 36 vector
model.add(Dense(6, W_regularizer=l2(l2reg))) # 36 vector -> 6 vector, ugh
model.add(Activation('softmax'))
return model
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Benchmark yu1412 on semantic relatedness regression-classification (sts)")
parser.add_argument("-N", help="GloVe dim", type=int, default=300)
parser.add_argument("--sick", help="whether to run on SICK2014 inst. of sts2012-14/15 dataset", type=int, default=0)
args = parser.parse_args()
glove = emb.GloVe(N=args.N)
if args.sick == 1:
Xtrain, ytrain = load_set(glove, 'data/sts/sick2014/SICK_train.txt', loader.load_sick2014)
Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014)
else:
Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*')
Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*')
model = prep_model(glove)
model.compile(loss='categorical_crossentropy', optimizer='adam')
model.fit(Xtrain, loader.sts_labels2categorical(ytrain), batch_size=80, nb_epoch=200, show_accuracy=True,
validation_data=(Xtest, loader.sts_labels2categorical(ytest)))
ev.eval_sts(model.predict_proba(Xtrain), ytrain, 'Train')
ev.eval_sts(model.predict_proba(Xtest), ytest, 'Test')
