def load_data(self, trainf, valf, testf=None): self.trainf = trainf self.valf = valf self.testf = testf self.gr, self.y, self.vocab = self.load_set(trainf) self.grv, self.yv, _ = self.load_set(valf) if testf is not None: self.grt, self.yt, _ = self.load_set(testf) else: self.grt, self.yt = (None, None) if self.c.get('adapt_ubuntu', False): self.gr = loader.graph_adapt_ubuntu(self.gr, self.vocab) self.grv = loader.graph_adapt_ubuntu(self.grv, self.vocab) if self.grt is not None: self.grt = loader.graph_adapt_ubuntu(self.grt, self.vocab)
def load_data(self, trainf, valf, testf=None): self.trainf = trainf # train file self.valf = valf # value file self.testf = testf # test file self.gr, self.y, self.vocab = self.load_set(trainf) self.grv, self.yv, _ = self.load_set(valf) if testf is not None: self.grt, self.yt, _ = self.load_set(testf) else: self.grt, self.yt = (None, None) if self.c.get('adapt_ubuntu', False): self.vocab.add_word('__eou__') self.vocab.add_word('__eot__') self.gr = loader.graph_adapt_ubuntu(self.gr, self.vocab) self.grv = loader.graph_adapt_ubuntu(self.grv, self.vocab) if self.grt is not None: self.grt = loader.graph_adapt_ubuntu(self.grt, self.vocab)
print('Predict&Eval (best epoch)') model.load_weights('weights-'+runid+'-bestval.h5') ev.eval_anssel(model.predict(grv)['score'][:,0], grv['si0'], grv['score'], 'anssel Val') if __name__ == "__main__": modelname, weightsf, vocabf, trainf, valf = sys.argv[1:6] params = sys.argv[6:] module = importlib.import_module('.'+modelname, 'models') conf, ps, h = config(module.config, params) runid = '%s-%x' % (modelname, h) print('RunID: %s (%s)' % (runid, ps)) print('GloVe') glove = emb.GloVe(N=conf['embdim']) print('Dataset (vocab)') vocab = pickle.load(open(vocabf, "rb")) # use plain pickle because unicode print('Dataset (anssel train)') s0, s1, y, _, gr_ = anssel_train.load_set(trainf, vocab, s0pad=s0pad, s1pad=s1pad) gr = loader.graph_adapt_ubuntu(gr_, vocab) print('Dataset (anssel val)') s0v, s1v, yv, _, grv_ = anssel_train.load_set(valf, vocab, s0pad=s0pad, s1pad=s1pad) grv = loader.graph_adapt_ubuntu(grv_, vocab) transfer_eval(runid, weightsf, module.prep_model, conf, glove, vocab, gr, grv)