def load_data(self, trainf, valf, testf=None):
self.trainf = trainf
self.valf = valf
self.testf = testf
self.gr, self.y, self.vocab = self.load_set(trainf)
self.grv, self.yv, _ = self.load_set(valf)
if testf is not None:
self.grt, self.yt, _ = self.load_set(testf)
else:
self.grt, self.yt = (None, None)
if self.c.get('adapt_ubuntu', False):
self.gr = loader.graph_adapt_ubuntu(self.gr, self.vocab)
self.grv = loader.graph_adapt_ubuntu(self.grv, self.vocab)
if self.grt is not None:
self.grt = loader.graph_adapt_ubuntu(self.grt, self.vocab)
def load_data(self, trainf, valf, testf=None):
self.trainf = trainf # train file
self.valf = valf # value file
self.testf = testf # test file
self.gr, self.y, self.vocab = self.load_set(trainf)
self.grv, self.yv, _ = self.load_set(valf)
if testf is not None:
self.grt, self.yt, _ = self.load_set(testf)
else:
self.grt, self.yt = (None, None)
if self.c.get('adapt_ubuntu', False):
self.vocab.add_word('__eou__')
self.vocab.add_word('__eot__')
self.gr = loader.graph_adapt_ubuntu(self.gr, self.vocab)
self.grv = loader.graph_adapt_ubuntu(self.grv, self.vocab)
if self.grt is not None:
self.grt = loader.graph_adapt_ubuntu(self.grt, self.vocab)
print('Predict&Eval (best epoch)')
model.load_weights('weights-'+runid+'-bestval.h5')
ev.eval_anssel(model.predict(grv)['score'][:,0], grv['si0'], grv['score'], 'anssel Val')
if __name__ == "__main__":
modelname, weightsf, vocabf, trainf, valf = sys.argv[1:6]
params = sys.argv[6:]
module = importlib.import_module('.'+modelname, 'models')
conf, ps, h = config(module.config, params)
runid = '%s-%x' % (modelname, h)
print('RunID: %s (%s)' % (runid, ps))
print('GloVe')
glove = emb.GloVe(N=conf['embdim'])
print('Dataset (vocab)')
vocab = pickle.load(open(vocabf, "rb")) # use plain pickle because unicode
print('Dataset (anssel train)')
s0, s1, y, _, gr_ = anssel_train.load_set(trainf, vocab, s0pad=s0pad, s1pad=s1pad)
gr = loader.graph_adapt_ubuntu(gr_, vocab)
print('Dataset (anssel val)')
s0v, s1v, yv, _, grv_ = anssel_train.load_set(valf, vocab, s0pad=s0pad, s1pad=s1pad)
grv = loader.graph_adapt_ubuntu(grv_, vocab)
transfer_eval(runid, weightsf, module.prep_model, conf, glove, vocab, gr, grv)