def train_and_eval(runid, module_prep_model, c, glove, vocab, gr, grt):
print('Model')
model = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=s0pad, s1pad=s1pad)
print('Training')
model.fit_generator(sample_pairs(gr, c, c['batch_size']),
callbacks=[#ValSampleCB(grt, c), # loss function & ubuntu metrics
AnsSelCB(grt['si0'], grt), # MRR
ModelCheckpoint('ubu-weights-'+runid+'-bestval.h5', save_best_only=True, monitor='mrr', mode='max'),
EarlyStopping(monitor='mrr', mode='max', patience=6)],
nb_epoch=32, samples_per_epoch=200000)
model.save_weights('ubu-weights-'+runid+'-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('ubu-weights-'+runid+'-bestval.h5')
ypredt = model.predict(grt)['score'][:,0]
ev.eval_ubuntu(ypredt, grt['si0'], grt['score'], 'Val')
def eval(self, model):
res = [None]
for gr, fname in [(self.grv, self.valf), (self.grt, self.testf)]:
if gr is None:
res.append(None)
continue
ypred = model.predict(gr)['score'][:, 0]
res.append(ev.eval_ubuntu(ypred, gr['si0'], gr['score'], fname))
return tuple(res)
def eval(self, model):
res = [None]
for gr, fname in [(self.grv, self.valf), (self.grt, self.testf)]:
if gr is None:
res.append(None)
continue
ypred = self.predict(model, gr)
res.append(ev.eval_ubuntu(ypred, np.array(gr['si0']) + np.array(gr['sj0']), gr['score'], fname))
return tuple(res)
def eval(self, model):
res = [None]
for gr, fname in [(self.grv, self.valf), (self.grt, self.testf)]:
if gr is None:
res.append(None)
continue
ypred = model.predict(gr)['score'][:,0]
res.append(ev.eval_ubuntu(ypred, gr['si0'], gr['score'], fname))
return tuple(res)
print('GloVe')
glove = emb.GloVe(N=conf['embdim'])
print('Dataset (vocab)')
vocab = pickle.load(open(vocabf, "rb")) # use plain pickle because unicode
print('%d words loaded' % (len(vocab.word_idx),))
print('Dataset (val)')
grt = ubuntu_train.load_set(valf, vocab)
print('Padding (val)')
ubuntu_train.pad_graph(grt)
print('Model')
model = anssel_train.build_model(glove, vocab, module.prep_model, conf, s0pad=ubuntu_train.s0pad, s1pad=ubuntu_train.s1pad)
print('%d parameters (except embedding matrix)' % (model.count_params() - model.nodes['emb'].count_params(),))
print('Weights')
model.load_weights(weightsfile)
print('Prediction (val)')
ypredt = model.predict(grt)['score'][:,0]
ev.eval_ubuntu(ypredt, grt['si0'], grt['score'], valf)
# print('Dataset (train)')
# gr = ubuntu_train.load_set(trainf, vocab)
# print('Prediction (train)')
# ypred = model.predict(gr)['score'][:,0]
# ev.eval_ubuntu(ypred, gr['si0'], gr['score'], trainf)
print('%d words loaded' % (len(vocab.word_idx), ))
print('Dataset (val)')
grt = ubuntu_train.load_set(valf, vocab)
print('Padding (val)')
ubuntu_train.pad_graph(grt)
print('Model')
model = anssel_train.build_model(glove,
vocab,
module.prep_model,
conf,
s0pad=ubuntu_train.s0pad,
s1pad=ubuntu_train.s1pad)
print('%d parameters (except embedding matrix)' %
(model.count_params() - model.nodes['emb'].count_params(), ))
print('Weights')
model.load_weights(weightsfile)
print('Prediction (val)')
ypredt = model.predict(grt)['score'][:, 0]
ev.eval_ubuntu(ypredt, grt['si0'], grt['score'], valf)
# print('Dataset (train)')
# gr = ubuntu_train.load_set(trainf, vocab)
# print('Prediction (train)')
# ypred = model.predict(gr)['score'][:,0]
# ev.eval_ubuntu(ypred, gr['si0'], gr['score'], trainf)