def test_agent_online(robot, game, model, budget): # to address game -> we have a new game here i = 0 queried_x = [] queried_y = [] performance = [] test_sents = helpers.data2sents(game.test_x, game.test_y) game.reboot() while i < budget: sel_ind = game.current_frame # construct the observation observation = game.get_frame(model) action = robot.get_action(observation) if action[1] == 1: sentence = game.train_x[sel_ind] labels = game.train_y[sel_ind] queried_x.append(sentence) queried_y.append(labels) i += 1 train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(test_sents)) reward, observation2, terminal = game.feedback(action, model) # game robot.update(observation, action, reward, observation2, terminal) # train a crf and evaluate it train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(test_sents)) print("***TEST", performance)
def test_agent_online(robot, game, model, budget): # to address game -> we have a new game here i = 0 queried_x = [] queried_y = [] performance = [] test_sents = helpers.data2sents(game.test_x, game.test_y) X_test = [tagger.sent2features(s) for s in test_sents] Y_true = [tagger.sent2labels(s) for s in test_sents] game.reboot(model) while i < budget: sel_ind = game.current_frame # construct the observation observation = game.get_frame(model) action = robot.get_action(observation) if action[1] == 1: sentence = game.train_x[sel_ind] labels = game.train_y[sel_ind] queried_x.append(sentence) queried_y.append(labels) i += 1 train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(X_test, Y_true)) reward, observation2, terminal = game.feedback(action, model) # game robot.update(observation, action, reward, observation2, terminal) # train a crf and evaluate it train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(X_test, Y_true)) logging.debug('***TEST {}'.format(performance))
def test_agent_batch(robot, game, model, budget): i = 0 queried_x = [] queried_y = [] performance = [] cost = [] test_sents = helpers.data2sents(game.test_x, game.test_y) game.replay() while i < budget: sel_ind = game.current_frame # construct the observation observation = game.get_frame(model) action = robot.get_action(observation) if action[1] == 1: sentence = game.train_x[sel_ind] labels = game.train_y[sel_ind] queried_x.append(sentence) queried_y.append(labels) i += 1 train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(test_sents)) cost.append(i) game.current_frame += 1 print('***COST', cost) print("***TEST", performance) return (cost, performance)
def test_agent_batch(robot, game, model, budget): i = 0 queried_x = [] queried_y = [] performance = [] test_sents = helpers.data2sents(game.test_x, game.test_y) X_test = [tagger.sent2features(s) for s in test_sents] Y_true = [tagger.sent2labels(s) for s in test_sents] game.reboot(model) while i < budget: sel_ind = game.current_frame # construct the observation observation = game.get_frame(model) action = robot.get_action(observation) if action[1] == 1: sentence = game.train_x[sel_ind] labels = game.train_y[sel_ind] queried_x.append(sentence) queried_y.append(labels) i += 1 train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(X_test, Y_true)) game.current_frame += 1 # train a crf and evaluate it train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(X_test, Y_true)) logging.debug('***TEST {}'.format(performance))
def test_agent_batch(robot, game, model, budget): i = 0 queried_x = [] queried_y = [] performance = [] test_sents = helpers.data2sents(game.test_x, game.test_y) game.reboot() while i < budget: sel_ind = game.current_frame # construct the observation observation = game.get_frame(model) action = robot.get_action(observation) if action[1] == 1: sentence = game.train_x[sel_ind] labels = game.train_y[sel_ind] queried_x.append(sentence) queried_y.append(labels) i += 1 train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(test_sents)) game.current_frame += 1 # train a crf and evaluate it train_sents = helpers.data2sents(queried_x, queried_y) model.train(train_sents) performance.append(model.test(test_sents)) print "***TEST", performance
def get_performance(self, tagger): # train with {queried_set_x, queried_set_y} # train with examples: self.model.train(self.queried_set_x, # self.queried_set_y) if tagger.name == "RNN": tagger.train(self.queried_set_idx, self.queried_set_y) performance = tagger.test(self.dev_idx, self.dev_y) return performance print (len(self.queried_set_x), len(self.queried_set_y)) train_sents = helpers.data2sents( self.queried_set_x, self.queried_set_y) # print train_sents tagger.train(train_sents) # test on development data test_sents = helpers.data2sents(self.dev_x, self.dev_y) performance = tagger.test(test_sents) #performance = self.model.test2conlleval(self.dev_x, self.dev_y) return performance
def get_performance(self, tagger): # train with {queried_set_x, queried_set_y} # train with examples: self.model.train(self.queried_set_x, self.queried_set_y) if tagger.name == 'RNN': tagger.train(self.queried_set_idx, self.queried_set_idy) performance = tagger.test(features=self.dev_idx, labels=self.dev_y) return performance else: # CRF case train_sents = helpers.data2sents(self.queried_set_x, self.queried_set_y) tagger.train(train_sents) # test on development data performance = tagger.test(self.X_dev, self.Y_dev) # performance = self.model.test2conlleval(self.dev_x, self.dev_y) return performance
def __init__(self, story, dev, max_len, w2v, budget, model_name): # build environment # load data as story logging.info('Initializing the game:') # import story self.train_x, self.train_y, self.train_idx, self.train_idy = story self.dev_x, self.dev_y, self.dev_idx, self.dev_idy = dev # TODO this should be a function # Cache only for CRF model if model_name == 'CRF': dev_sents = helpers.data2sents(self.dev_x, self.dev_y) self.X_dev = [tagger.sent2features(s) for s in dev_sents] self.Y_dev = [tagger.sent2labels(s) for s in dev_sents] self.max_len = max_len self.w2v = w2v logging.info('Story: length = {}'.format(len(self.train_x))) self.order = list(range(0, len(self.train_x))) # if re-order, use random.shuffle(self.order) # load word embeddings, pretrained - w2v logging.info('Dictionary size {} Embedding size {}'.format( len(self.w2v), len(self.w2v[0]))) # when queried times is 'budget', then stop self.budget = budget self.queried_times = 0 # TODO use ndarrays for everything # select pool self.queried_set_x = [] self.queried_set_y = [] self.queried_set_idx = [] self.queried_set_idy = [] # let's start self.episode = 0 # story frame self.current_frame = 0 #self.nextFrame = self.current_frame + 1 self.terminal = False self.make_query = False self.performance = 0