def load_model(trainf, model_name, weights_file):
params = []
module = importlib.import_module('.'+modelname, 'models')
conf, ps, h = anssel_train.config(module.config, params)
s0, s1, y, vocab, gr = anssel_train.load_set(trainf)
model = anssel_train.build_model(glove, vocab, module.prep_model, conf)
model.load_weights(weights_file)
return model, vocab
def load_model(trainf, model_name, weights_file):
params = []
module = importlib.import_module('.' + modelname, 'models')
conf, ps, h = anssel_train.config(module.config, params)
s0, s1, y, vocab, gr = anssel_train.load_set(trainf)
model = anssel_train.build_model(glove, vocab, module.prep_model, conf)
model.load_weights(weights_file)
return model, vocab
def train_and_eval(runid, module_prep_model, c, glove, vocab, gr, grt):
print('Model')
model = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=s0pad, s1pad=s1pad)
print('Training')
model.fit_generator(sample_pairs(gr, c, c['batch_size']),
callbacks=[#ValSampleCB(grt, c), # loss function & ubuntu metrics
AnsSelCB(grt['si0'], grt), # MRR
ModelCheckpoint('ubu-weights-'+runid+'-bestval.h5', save_best_only=True, monitor='mrr', mode='max'),
EarlyStopping(monitor='mrr', mode='max', patience=6)],
nb_epoch=32, samples_per_epoch=200000)
model.save_weights('ubu-weights-'+runid+'-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('ubu-weights-'+runid+'-bestval.h5')
ypredt = model.predict(grt)['score'][:,0]
ev.eval_ubuntu(ypredt, grt['si0'], grt['score'], 'Val')
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr,
grv):
# We construct both original and sts model, then copy over
# the weights from the original model
print('Model')
umodel = anssel_train.build_model(glove,
vocab,
module_prep_model,
c,
s0pad=spad,
s1pad=spad,
do_compile=False)
model = sts_train.build_model(glove,
vocab,
module_prep_model,
c,
spad=spad,
optimizer=c['opt'],
fix_layers=c['fix_layers'])
print('Model (weights)')
umodel.load_weights(weightsf)
for n in umodel.nodes.keys():
model.nodes[n].set_weights(umodel.nodes[n].get_weights())
ev.eval_sts(
model.predict(grv)['classes'][:, 0], grv['classes'],
'sts Val (bef. train)')
print('Training')
model.fit(gr,
validation_data=grv,
callbacks=[
STSPearsonCB(gr, grv),
ModelCheckpoint('sts-weights-' + runid + '-bestval.h5',
save_best_only=True),
EarlyStopping(patience=4)
],
batch_size=conf['batch_size'],
nb_epoch=conf['nb_epoch'])
model.save_weights('sts-weights-' + runid + '-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('sts-weights-' + runid + '-bestval.h5')
ev.eval_sts(model.predict(grv)['classes'][:, 0], grv['classes'], 'sts Val')
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr, grv):
# We construct both original and sts model, then copy over
# the weights from the original model
print('Model')
umodel = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=spad, s1pad=spad, do_compile=False)
model = sts_train.build_model(glove, vocab, module_prep_model, c, spad=spad, optimizer=c['opt'], fix_layers=c['fix_layers'])
print('Model (weights)')
umodel.load_weights(weightsf)
for n in umodel.nodes.keys():
model.nodes[n].set_weights(umodel.nodes[n].get_weights())
ev.eval_sts(model.predict(grv)['classes'][:,0], grv['classes'], 'sts Val (bef. train)')
print('Training')
model.fit(gr, validation_data=grv,
callbacks=[STSPearsonCB(gr, grv),
ModelCheckpoint('sts-weights-'+runid+'-bestval.h5', save_best_only=True),
EarlyStopping(patience=4)],
batch_size=conf['batch_size'], nb_epoch=conf['nb_epoch'])
model.save_weights('sts-weights-'+runid+'-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('sts-weights-'+runid+'-bestval.h5')
ev.eval_sts(model.predict(grv)['classes'][:,0], grv['classes'], 'sts Val')
def transfer_eval(runid, weightsf, module_prep_model, c, glove, vocab, gr, grv):
print('Model')
model = anssel_train.build_model(glove, vocab, module_prep_model, c, s0pad=s0pad, s1pad=s1pad, optimizer=c['opt'], fix_layers=c['fix_layers'])
print('Model (weights)')
model.load_weights(weightsf)
ev.eval_anssel(model.predict(grv)['score'][:,0], grv['si0'], grv['score'], 'anssel Val (bef. train)')
print('Training')
if c.get('balance_class', False):
one_ratio = np.sum(gr['score'] == 1) / len(gr['score'])
class_weight = {'score': {0: one_ratio, 1: 0.5}}
else:
class_weight = {}
model.fit(gr, validation_data=grv,
callbacks=[AnsSelCB(s0v, grv),
ModelCheckpoint('weights-'+runid+'-bestval.h5', save_best_only=True, monitor='mrr', mode='max'),
EarlyStopping(monitor='mrr', mode='max', patience=4)],
class_weight=class_weight,
batch_size=conf['batch_size'], nb_epoch=conf['nb_epoch'], samples_per_epoch=int(len(gr['score'])*conf['epoch_fract']))
model.save_weights('weights-'+runid+'-final.h5', overwrite=True)
print('Predict&Eval (best epoch)')
model.load_weights('weights-'+runid+'-bestval.h5')
ev.eval_anssel(model.predict(grv)['score'][:,0], grv['si0'], grv['score'], 'anssel Val')
if __name__ == "__main__":
modelname, weightsfile, trainf, valf, trec_qrels_file, trec_top_file = sys.argv[1:7]
params = sys.argv[7:]
module = importlib.import_module('.'+modelname, 'models')
conf, ps, h = anssel_train.config(module.config, params)
print('GloVe')
glove = emb.GloVe(N=conf['embdim'])
print('Dataset')
s0, s1, y, vocab, gr = anssel_train.load_set(trainf)
s0t, s1t, yt, _, grt = anssel_train.load_set(valf, vocab)
print('Model')
model = anssel_train.build_model(glove, vocab, module.prep_model, conf)
print('Weights')
model.load_weights(weightsfile)
print('Prediction')
ypred = model.predict(gr)['score'][:,0]
ypredt = model.predict(grt)['score'][:,0]
ev.eval_anssel(ypred, s0, y, trainf)
ev.eval_anssel(ypredt, s0t, yt, valf)
with open(trec_qrels_file, 'wt') as f:
save_trec_qrels(f, s0t, s1t, yt)
with open(trec_top_file, 'wt') as f:
save_trec_top(f, s0t, s1t, ypredt, modelname)
cdim={
1: [0, 0, 1 / 2, 1, 2],
2: [0, 0, 1 / 2, 1, 2, 0],
3: [0, 0, 1 / 2, 1, 2, 0],
4: [0, 0, 1 / 2, 1, 2, 0],
5: [0, 0, 1 / 2, 1, 2]
},
project=[True, True, False],
pdim=[1, 2, 2.5, 3],
ptscorer=[B.mlp_ptscorer],
Ddim=[1, 2, 2.5, 3])
for ps, h, pardict in rs():
print(' ...... %s .................... %s' % (h, ps))
conf, ps, h = anssel_train.config(module.config, ps)
model = anssel_train.build_model(glove, vocab, module.prep_model, conf)
runid = '%s_%x' % (modelname, h)
model.fit(gr,
validation_data=grt,
callbacks=[
AnsSelCB(s0t, grt),
ModelCheckpoint('weights-' + runid + '.h5',
save_best_only=True,
monitor='mrr',
mode='max'),
EarlyStopping(monitor='mrr', mode='max', patience=1)
],
batch_size=160,
nb_epoch=16,
samples_per_epoch=5000)
conf, ps, h = anssel_train.config(module.config, params)
print('GloVe')
glove = emb.GloVe(N=conf['embdim'])
print('Dataset (vocab)')
vocab = pickle.load(open(vocabf, "rb")) # use plain pickle because unicode
print('%d words loaded' % (len(vocab.word_idx),))
print('Dataset (val)')
grt = ubuntu_train.load_set(valf, vocab)
print('Padding (val)')
ubuntu_train.pad_graph(grt)
print('Model')
model = anssel_train.build_model(glove, vocab, module.prep_model, conf, s0pad=ubuntu_train.s0pad, s1pad=ubuntu_train.s1pad)
print('%d parameters (except embedding matrix)' % (model.count_params() - model.nodes['emb'].count_params(),))
print('Weights')
model.load_weights(weightsfile)
print('Prediction (val)')
ypredt = model.predict(grt)['score'][:,0]
ev.eval_ubuntu(ypredt, grt['si0'], grt['score'], valf)
# print('Dataset (train)')
# gr = ubuntu_train.load_set(trainf, vocab)
# print('Prediction (train)')
# ypred = model.predict(gr)['score'][:,0]
# ev.eval_ubuntu(ypred, gr['si0'], gr['score'], trainf)
print('GloVe')
glove = emb.GloVe(N=conf['embdim'])
print('Dataset (vocab)')
vocab = pickle.load(open(vocabf, "rb")) # use plain pickle because unicode
print('%d words loaded' % (len(vocab.word_idx), ))
print('Dataset (val)')
grt = ubuntu_train.load_set(valf, vocab)
print('Padding (val)')
ubuntu_train.pad_graph(grt)
print('Model')
model = anssel_train.build_model(glove,
vocab,
module.prep_model,
conf,
s0pad=ubuntu_train.s0pad,
s1pad=ubuntu_train.s1pad)
print('%d parameters (except embedding matrix)' %
(model.count_params() - model.nodes['emb'].count_params(), ))
print('Weights')
model.load_weights(weightsfile)
print('Prediction (val)')
ypredt = model.predict(grt)['score'][:, 0]
ev.eval_ubuntu(ypredt, grt['si0'], grt['score'], valf)
# print('Dataset (train)')
# gr = ubuntu_train.load_set(trainf, vocab)