def on_epoch_end(self, epoch, logs={}): prtr = ev.eval_sts(self.model.predict(self.train_gr)['classes'], loader.sts_categorical2labels(self.train_gr['classes']), 'Train', quiet=True).Pearson prval = ev.eval_sts(self.model.predict(self.val_gr)['classes'], loader.sts_categorical2labels(self.val_gr['classes']), 'Val', quiet=True).Pearson print(' train Pearson %f val Pearson %f' % (prtr, prval)) logs['pearson'] = prval
def train_and_eval(runid, module_prep_model, c, glove, vocab, gr, grt): print('Model') model = build_model(glove, vocab, module_prep_model, c) print('Training') # XXX: samples_per_epoch is in brmson/keras fork, TODO fit_generator()? model.fit(gr, validation_data=grt, callbacks=[STSPearsonCB(gr, grt), ModelCheckpoint('sts-weights-'+runid+'-bestval.h5', save_best_only=True, monitor='pearson', mode='max'), EarlyStopping(monitor='pearson', mode='max', patience=3)], batch_size=160, nb_epoch=32) model.save_weights('sts-weights-'+runid+'-final.h5', overwrite=True) print('Predict&Eval') model.load_weights('sts-weights-'+runid+'-bestval.h5') ev.eval_sts(model.predict(gr)['classes'], gr['classes'], 'Train') ev.eval_sts(model.predict(grt)['classes'], grt['classes'], 'Val')
def eval(self, model): res = [] for gr, fname in [(self.gr, self.trainf), (self.grv, self.valf), (self.grt, self.testf)]: if gr is None: res.append(None) continue ypred = self.predict(model, gr) res.append(ev.eval_sts(ypred, gr['classes'], fname)) return tuple(res)
def eval(self, model): res = [] for gr, fname in [(self.gr, self.trainf), (self.grv, self.valf), (self.grt, self.testf)]: if gr is None: res.append(None) continue ypred = model.predict(gr)['classes'] res.append(ev.eval_sts(ypred, gr['classes'], fname)) return tuple(res)
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr, grv): # We construct both original and sts model, then copy over # the weights from the original model print('Model') umodel = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=spad, s1pad=spad, do_compile=False) model = sts_train.build_model(glove, vocab, module_prep_model, c, spad=spad, optimizer=c['opt'], fix_layers=c['fix_layers']) print('Model (weights)') umodel.load_weights(weightsf) for n in umodel.nodes.keys(): model.nodes[n].set_weights(umodel.nodes[n].get_weights()) ev.eval_sts( model.predict(grv)['classes'][:, 0], grv['classes'], 'sts Val (bef. train)') print('Training') model.fit(gr, validation_data=grv, callbacks=[ STSPearsonCB(gr, grv), ModelCheckpoint('sts-weights-' + runid + '-bestval.h5', save_best_only=True), EarlyStopping(patience=4) ], batch_size=conf['batch_size'], nb_epoch=conf['nb_epoch']) model.save_weights('sts-weights-' + runid + '-final.h5', overwrite=True) print('Predict&Eval (best epoch)') model.load_weights('sts-weights-' + runid + '-bestval.h5') ev.eval_sts(model.predict(grv)['classes'][:, 0], grv['classes'], 'sts Val')
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr, grv): # We construct both original and sts model, then copy over # the weights from the original model print('Model') umodel = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=spad, s1pad=spad, do_compile=False) model = sts_train.build_model(glove, vocab, module_prep_model, c, spad=spad, optimizer=c['opt'], fix_layers=c['fix_layers']) print('Model (weights)') umodel.load_weights(weightsf) for n in umodel.nodes.keys(): model.nodes[n].set_weights(umodel.nodes[n].get_weights()) ev.eval_sts(model.predict(grv)['classes'][:,0], grv['classes'], 'sts Val (bef. train)') print('Training') model.fit(gr, validation_data=grv, callbacks=[STSPearsonCB(gr, grv), ModelCheckpoint('sts-weights-'+runid+'-bestval.h5', save_best_only=True), EarlyStopping(patience=4)], batch_size=conf['batch_size'], nb_epoch=conf['nb_epoch']) model.save_weights('sts-weights-'+runid+'-final.h5', overwrite=True) print('Predict&Eval (best epoch)') model.load_weights('sts-weights-'+runid+'-bestval.h5') ev.eval_sts(model.predict(grv)['classes'][:,0], grv['classes'], 'sts Val')
def train_and_eval(runid, module_prep_model, c, glove, vocab, gr, grt, do_eval=True): print('Model') model = build_model(glove, vocab, module_prep_model, c) print('Training') # XXX: samples_per_epoch is in brmson/keras fork, TODO fit_generator()? model.fit(gr, validation_data=grt, callbacks=[ STSPearsonCB(gr, grt), ModelCheckpoint('sts-weights-' + runid + '-bestval.h5', save_best_only=True, monitor='pearson', mode='max'), EarlyStopping(monitor='pearson', mode='max', patience=3) ], batch_size=c['batch_size'], nb_epoch=c['nb_epoch']) model.save_weights('sts-weights-' + runid + '-final.h5', overwrite=True) if c['ptscorer'] is None: model.save_weights('sts-weights-' + runid + '-bestval.h5', overwrite=True) model.load_weights('sts-weights-' + runid + '-bestval.h5') if do_eval: print('Predict&Eval (best val epoch)') ev.eval_sts(model.predict(gr)['classes'], gr['classes'], 'Train') ev.eval_sts(model.predict(grt)['classes'], grt['classes'], 'Val') return model
args = parser.parse_args() glove = emb.GloVe(N=args.N) if args.sick == 1: Xtrain, ytrain = load_set(glove, 'data/sts/sick2014/SICK_train.txt', loader.load_sick2014) Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014) else: Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*') Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*') model = prep_model(glove) model.compile(loss={'classes': 'categorical_crossentropy'}, optimizer='adam') model.fit({'e0': Xtrain[0], 'e1': Xtrain[1], 'classes': loader.sts_labels2categorical(ytrain)}, batch_size=20, nb_epoch=100, validation_data={'e0': Xtest[0], 'e1': Xtest[1], 'classes': loader.sts_labels2categorical(ytest)}) ev.eval_sts(model.predict({'e0': Xtrain[0], 'e1': Xtrain[1]})['classes'], ytrain, 'Train') ev.eval_sts(model.predict({'e0': Xtest[0], 'e1': Xtest[1]})['classes'], ytest, 'Test') """ Performance tuning (100 iters) on sick2014: * Just elementwise-mul: 4500/4500 [==============================] - 0s - loss: 1.2332 - acc: 0.5016 - val_loss: 1.1926 - val_acc: 0.4770 Train Pearson: 0.721662 Train Spearman: 0.593711 Train MSE: 0.529364 Test Pearson: 0.701661 Test Spearman: 0.571329 Test MSE: 0.540898
prv = [] prt = [] for i in range(niter): runid = '%s-%x-%02d' % (modelname, h, i) print('RunID: %s (%s)' % (runid, ps)) model = sts_train.train_and_eval(runid, module.prep_model, conf, glove, vocab, gr, grv, do_eval=False) print('Predict&Eval (best val epoch)') ypred = model.predict(gr)['classes'] ypredv = model.predict(grv)['classes'] ypredt = dict() for ti in range(len(testf)): ypredt[testf[ti]] = model.predict(lgrt[ti])['classes'] pr.append(ev.eval_sts(ypred, gr['classes'], 'Train')) prv.append(ev.eval_sts(ypredv, grv['classes'], 'Val')) dprt = dict() for ti in range(len(testf)): dprt[testf[ti]] = ev.eval_sts(ypredt[testf[ti]], lgrt[ti]['classes'], testf[ti]) prt.append(dprt) rdata = {'ps': ps, 'ypred': (ypred, ypredv, ypredt), 'pr': (pr[-1], prv[-1], prt[-1])} pickle.dump(rdata, open('%s-res.pickle' % (runid,), 'wb'), protocol=2) # prt view indexed by testf, not by iteration prtt = dict() for ti in range(len(testf)): prtt[testf[ti]] = [prt0[testf[ti]] for prt0 in prt] mprt = [np.mean(prtt[testf[ti]]) for ti in range(len(testf))] # (n_testf,) mean across epochs
model.fit( { 'e0': Xtrain[0], 'e1': Xtrain[1], 'classes': loader.sts_labels2categorical(ytrain) }, batch_size=20, nb_epoch=100, validation_data={ 'e0': Xtest[0], 'e1': Xtest[1], 'classes': loader.sts_labels2categorical(ytest) }) ev.eval_sts( model.predict({ 'e0': Xtrain[0], 'e1': Xtrain[1] })['classes'], ytrain, 'Train') ev.eval_sts( model.predict({ 'e0': Xtest[0], 'e1': Xtest[1] })['classes'], ytest, 'Test') """ Performance tuning (100 iters) on sick2014: * Just elementwise-mul: 4500/4500 [==============================] - 0s - loss: 1.2332 - acc: 0.5016 - val_loss: 1.1926 - val_acc: 0.4770 Train Pearson: 0.721662 Train Spearman: 0.593711
model1.add(RepeatVector(6)) # [nclass] model = Sequential() model.add(Merge([model0, model1], mode='dot', dot_axes=([2], [2]))) model.add(Flatten()) # 6x6 matrix with cross-activations -> 36 vector model.add(Dense(6, W_regularizer=l2(l2reg))) # 36 vector -> 6 vector, ugh model.add(Activation('softmax')) return model if __name__ == "__main__": parser = argparse.ArgumentParser(description="Benchmark yu1412 on semantic relatedness regression-classification (sts)") parser.add_argument("-N", help="GloVe dim", type=int, default=300) parser.add_argument("--sick", help="whether to run on SICK2014 inst. of sts2012-14/15 dataset", type=int, default=0) args = parser.parse_args() glove = emb.GloVe(N=args.N) if args.sick == 1: Xtrain, ytrain = load_set(glove, 'data/sts/sick2014/SICK_train.txt', loader.load_sick2014) Xtest, ytest = load_set(glove, 'data/sts/sick2014/SICK_test_annotated.txt', loader.load_sick2014) else: Xtrain, ytrain = load_set(glove, 'data/sts/semeval-sts/all/201[0-4]*') Xtest, ytest = load_set(glove, 'data/sts/semeval-sts/all/2015*') model = prep_model(glove) model.compile(loss='categorical_crossentropy', optimizer='adam') model.fit(Xtrain, loader.sts_labels2categorical(ytrain), batch_size=80, nb_epoch=200, show_accuracy=True, validation_data=(Xtest, loader.sts_labels2categorical(ytest))) ev.eval_sts(model.predict_proba(Xtrain), ytrain, 'Train') ev.eval_sts(model.predict_proba(Xtest), ytest, 'Test')