def test_qgen(sess, testset, tokenizer, qgen, cpu_pool, batch_size, logger):
qgen_sources = qgen.get_sources(sess)
qgen_evaluator = Evaluator(qgen_sources, qgen.scope_name, network=qgen, tokenizer=tokenizer)
qgen_batchifier = QuestionerBatchifier(tokenizer, qgen_sources, status=('success',))
qgen_iterator = Iterator(testset, pool=cpu_pool,
batch_size=batch_size,
batchifier=qgen_batchifier)
[qgen_loss] = qgen_evaluator.process(sess, qgen_iterator, outputs=[qgen.ml_loss])
logger.info("QGen test loss: {}".format(qgen_loss))
class GuesserWrapper(object):
def __init__(self, guesser, batchifier, tokenizer, listener):
self.guesser = guesser
self.batchifier = batchifier
self.tokenizer = tokenizer
self.listener = listener
self.evaluator = None
def initialize(self, sess):
self.evaluator = Evaluator(self.guesser.get_sources(sess),
self.guesser.scope_name)
def find_object(self, sess, games):
# the guesser may need to split the input
iterator = BasicIterator(games,
batch_size=len(games),
batchifier=self.batchifier)
# sample
self.evaluator.process(sess,
iterator,
outputs=[],
listener=self.listener,
show_progress=False)
results = self.listener.results()
# Update games
new_games = []
# for game in games:
for game in games:
res = results[game.dialogue_id]
# print("--")
# print(att)
game.id_guess_object = res["id_guess_object"]
game.user_data.get("softmax", []).append(res["softmax"])
game.status = "success" if res["success"] else "failure"
game.is_full_dialogue = True
new_games.append(game)
return new_games
def test_guesser(sess, testset, tokenizer, guesser, cpu_pool, batch_size, logger):
guesser_sources = guesser.get_sources(sess)
guesser_evaluator = Evaluator(guesser_sources, guesser.scope_name, network=guesser, tokenizer=tokenizer)
guesser_batchifier = QuestionerBatchifier(tokenizer, guesser_sources, status=('success',))
guesser_iterator = Iterator(testset, pool=cpu_pool,
batch_size=batch_size,
batchifier=guesser_batchifier)
[guesser_loss, guesser_error] = guesser_evaluator.process(sess, guesser_iterator, [guesser.loss, guesser.error])
logger.info("Guesser test loss: {}".format(guesser_loss))
logger.info("Guesser test error: {}".format(guesser_error))
def test_oracle(sess, testset, tokenizer, oracle, cpu_pool, batch_size, logger):
oracle_dataset = OracleDataset(testset)
oracle_sources = oracle.get_sources(sess)
oracle_evaluator = Evaluator(oracle_sources, oracle.scope_name, network=oracle, tokenizer=tokenizer)
oracle_batchifier = OracleBatchifier(tokenizer, oracle_sources, status=('success',))
oracle_iterator = Iterator(oracle_dataset, pool=cpu_pool,
batch_size=batch_size,
batchifier=oracle_batchifier)
[oracle_loss, oracle_error] = oracle_evaluator.process(sess, oracle_iterator, [oracle.loss, oracle.error])
logger.info("Oracle test loss: {}".format(oracle_loss))
logger.info("Oracle test error: {}".format(oracle_error))
network=network,
tokenizer=tokenizer)
batchifier = QuestionerBatchifier(tokenizer,
sources,
status=('success', ))
for t in range(start_epoch, no_epoch):
logger.info('Epoch {}..'.format(t + 1))
train_iterator = Iterator(trainset,
batch_size=batch_size,
pool=cpu_pool,
batchifier=batchifier,
shuffle=True)
train_loss, train_accuracy = evaluator.process(sess,
train_iterator,
outputs=outputs +
[optimizer])
valid_iterator = Iterator(validset,
pool=cpu_pool,
batch_size=batch_size * 2,
batchifier=batchifier,
shuffle=False)
valid_loss, valid_accuracy = evaluator.process(sess,
valid_iterator,
outputs=outputs)
logger.info("Training loss: {}".format(train_loss))
logger.info("Training error: {}".format(1 - train_accuracy))
logger.info("Validation loss: {}".format(valid_loss))
logger.info("Validation error: {}".format(1 - valid_accuracy))
logger.info('Epoch {}..'.format(t + 1))
# logger.info('Epoch {}..'.format(t + 1))
logger.info(" train_oracle | Iterator ...")
t1 = time.time()
train_iterator = Iterator(trainset,
batch_size=batch_size, pool=cpu_pool,
batchifier=batchifier,
shuffle=True)
t2 = time.time()
logger.info(" train_oracle | Iterator...Total=".format(t2-t1))
t1 = time.time()
train_loss, train_accuracy = evaluator.process(sess, train_iterator, outputs=outputs + [optimizer],out_net=best_param)
t2 = time.time()
logger.info(" train_oracle | evaluatorator...Total=".format(t2-t1))
t1 = time.time()
valid_iterator = Iterator(validset, pool=cpu_pool,
batch_size=batch_size*2,
batchifier=batchifier,
shuffle=False)
t2 = time.time()
batchifier = QuestionerBatchifier(tokenizer,
sources,
status=('success', ))
best_val_loss = 1e5
for t in range(0, config['optimizer']['no_epoch']):
logger.info('Epoch {}..'.format(t + 1))
train_iterator = Iterator(trainset,
batch_size=batch_size,
pool=cpu_pool,
batchifier=batchifier,
shuffle=True)
[train_loss] = evaluator.process(sess,
train_iterator,
outputs=outputs + [optimizer])
valid_iterator = Iterator(validset,
pool=cpu_pool,
batch_size=batch_size * 2,
batchifier=batchifier,
shuffle=False)
[valid_loss] = evaluator.process(sess,
valid_iterator,
outputs=outputs)
logger.info("Training loss: {}".format(train_loss))
logger.info("Validation loss: {}".format(valid_loss))
if valid_loss < best_val_loss:
for t in range(start_epoch, no_epoch):
if args.skip_training:
logger.info("Skip training...")
break
logger.info('Epoch {}..'.format(t + 1))
# Create cpu pools (at each iteration otherwise threads may become zombie - python bug)
cpu_pool = create_cpu_pool(args.no_thread, use_process=use_process)
train_iterator = Iterator(trainset,
batch_size=batch_size,
pool=cpu_pool,
batchifier=batchifier,
shuffle=True)
train_loss, _ = evaluator.process(sess,
train_iterator,
outputs=outputs + [optimizer],
listener=listener)
train_accuracy = listener.accuracy(
) # Some guessers needs to go over the full dataset before comuting the accuracy, thus we use an intermediate listener
valid_iterator = Iterator(validset,
pool=cpu_pool,
batch_size=batch_size * 2,
batchifier=batchifier,
shuffle=False)
valid_loss, _ = evaluator.process(sess,
valid_iterator,
outputs=outputs,
listener=listener)
valid_accuracy = listener.accuracy()
best_val_loss = 1e5
for t in range(0, config['optimizer']['no_epoch']):
logger.info('Epoch {}..'.format(t + 1))
train_iterator = Iterator(trainset,
batch_size=batch_size,
pool=cpu_pool,
batchifier=batchifier,
shuffle=True)
# Changed for
[train_loss, _] = evaluator.process(sess,
train_iterator,
outputs=outputs + [optimizer] +
[network.summary],
n_batch=global_train_step,
writer=writer_t,
mod_val=config["freq"])
print "The Golbal Train Step is : %d" % (global_train_step[0])
# Don't know why the batch size is doubled???
# valid_iterator = Iterator(validset, pool=cpu_pool,
# batch_size=batch_size*2,
# batchifier=batchifier,
# shuffle=False)
valid_iterator = Iterator(validset,
pool=cpu_pool,
batch_size=batch_size,
batchifier=batchifier,
# CPU
cpu_pool = create_cpu_pool(
args.no_thread,
use_process=image_builder.require_multiprocess())
logger.info('Epoch {}/{}..'.format(t + 1, no_epoch))
train_iterator = Iterator(trainset,
batch_size=batch_size,
batchifier=batchifier,
shuffle=True,
pool=cpu_pool)
[train_loss, train_accuracy_fake
] = evaluator.process(sess,
train_iterator,
outputs=outputs + [optimize],
listener=listener)
train_accuracy = listener.evaluate()
valid_iterator = Iterator(validset,
batch_size=batch_size * 2,
batchifier=batchifier,
shuffle=False,
pool=cpu_pool)
[valid_loss,
valid_accuracy_fake] = evaluator.process(sess,
valid_iterator,
outputs=outputs,
listener=listener)
valid_accuracy = listener.evaluate()
shuffle=True,
pool=cpu_pool)
[train_loss, train_accuracy] = train_evaluator.process(sess, train_iterator, outputs=outputs + [optimizer])
valid_loss, valid_accuracy = 0,0
if not merge_dataset:
valid_iterator = Iterator(validset,
batch_size=batch_size*2,
batchifier=eval_batchifier,
shuffle=False,
pool=cpu_pool)
# Note : As we need to dump a compute VQA accuracy, we can only use a single-gpu evaluator
[valid_loss, valid_accuracy] = eval_evaluator.process(sess, valid_iterator,
outputs=[networks[0].loss, networks[0].accuracy],
listener=vqa_eval_listener)
logger.info("Training loss: {}".format(train_loss))
logger.info("Training accuracy: {}".format(train_accuracy))
logger.info("Validation loss: {}".format(valid_loss))
logger.info("Validation accuracy: {}".format(valid_accuracy))
logger.info(vqa_eval_listener.get_accuracy())
if valid_accuracy >= best_val_acc:
best_train_acc = train_accuracy
best_val_acc = valid_accuracy
saver.save(sess, save_path.format('params.ckpt'))
logger.info("checkpoint saved...")
pickle_dump({'epoch': t}, save_path.format('status.pkl'))
sources = network.get_sources(sess)
logger.info("Sources: " + ', '.join(sources))
saver.restore(sess, save_path.format('params.ckpt'))
if not os.path.exists(args.features):
os.makedirs(args.features)
# create training tools
evaluator = Evaluator(sources, network.scope_name, network=network, tokenizer=tokenizer)
batchifier = QuestionerBatchifier(tokenizer, sources, status=('success',))
train_iterator = Iterator(trainset,
batch_size=batch_size * 2, pool=cpu_pool,
batchifier=batchifier,
shuffle=False)
_, train_states = evaluator.process(sess, train_iterator, outputs=outputs, output_dialogue_states=True)
save_dialogue_states(args.features, "train", *train_states)
valid_iterator = Iterator(validset, pool=cpu_pool,
batch_size=batch_size * 2,
batchifier=batchifier,
shuffle=False)
_, valid_states = evaluator.process(sess, valid_iterator, outputs=outputs, output_dialogue_states=True)
save_dialogue_states(args.features, "valid", *valid_states)
test_iterator = Iterator(testset, pool=cpu_pool,
batch_size=batch_size * 2,
batchifier=batchifier,
shuffle=False)
class QGenWrapper(object):
def __init__(self, qgen, batchifier, tokenizer, max_length, k_best):
self.qgen = qgen
self.batchifier = batchifier
self.tokenizer = tokenizer
self.ops = dict()
self.ops["sampling"] = qgen.create_sampling_graph(
start_token=tokenizer.start_token,
stop_token=tokenizer.stop_token,
max_tokens=max_length)
self.ops["greedy"] = qgen.create_greedy_graph(
start_token=tokenizer.start_token,
stop_token=tokenizer.stop_token,
max_tokens=max_length)
beam_predicted_ids, seq_length, att = qgen.create_beam_graph(
start_token=tokenizer.start_token,
stop_token=tokenizer.stop_token,
max_tokens=max_length,
k_best=k_best)
# print('b',beam_predicted_ids)
# print('s',seq_length)
# Only keep best beam
self.ops[
"beam"] = beam_predicted_ids[:,
0, :], seq_length[:,
0], beam_predicted_ids[:,
0, :] * 0, att
self.evaluator = None
def initialize(self, sess):
self.evaluator = Evaluator(self.qgen.get_sources(sess),
self.qgen.scope_name,
network=self.qgen,
tokenizer=self.tokenizer)
def policy_update(self, sess, games, optimizer):
# ugly hack... to allow training on RL
batchifier = copy.copy(self.batchifier)
batchifier.generate = False
batchifier.supervised = False
iterator = BasicIterator(games,
batch_size=len(games),
batchifier=batchifier)
# Check whether the gradient is accumulated
if isinstance(optimizer, AccOptimizer):
sess.run(optimizer.zero) # reset gradient
local_optimizer = optimizer.accumulate
else:
local_optimizer = optimizer
# Compute the gradient
self.evaluator.process(sess,
iterator,
outputs=[local_optimizer],
show_progress=False)
if isinstance(optimizer, AccOptimizer):
sess.run(optimizer.update) # Apply accumulated gradient
def sample_next_question(self, sess, games, att_dict, beta_dict, mode):
# ugly hack... to allow training on RL
batchifier = copy.copy(self.batchifier)
batchifier.generate = True
batchifier.supervised = False
# create the training batch
batch = batchifier.apply(games, skip_targets=True)
batch["is_training"] = False
batch["is_dynamic"] = True
# Sample
tokens, seq_length, state_values, atts = self.evaluator.execute(
sess, output=self.ops[mode], batch=batch)
# tokens, seq_length, state_values, atts, betas = self.evaluator.execute(sess, output=self.ops[mode], batch=batch)
# Update game
new_games = []
for game, question_tokens, l, state_value, att in zip(
games, tokens, seq_length, state_values, atts):
# for game, question_tokens, l, state_value, att, beta in zip(games, tokens, seq_length, state_values, atts, betas):
if not game.user_data[
"has_stop_token"]: # stop adding question if dialogue is over
# clean tokens after stop_dialogue_tokens
if self.tokenizer.stop_dialogue in question_tokens:
game.user_data["has_stop_token"] = True
l = np.nonzero(
question_tokens == self.tokenizer.stop_dialogue
)[0][0] + 1 # find the first stop_dialogue occurrence
# Append the newly generated question
game.questions.append(
self.tokenizer.decode(question_tokens[:l]))
game.question_ids.append(len(game.question_ids))
game.user_data["state_values"] = game.user_data.get(
"state_values", [])
game.user_data["state_values"].append(state_value[:l].tolist())
att = att.tolist()
att_i = np.argsort(att).tolist()
att_3 = np.sort(att).tolist()
if game.dialogue_id not in att_dict:
att_dict[game.dialogue_id] = []
att_dict[game.dialogue_id].append((att_i, att_3))
else:
att_dict[game.dialogue_id].append((att_i, att_3))
# beta = beta.tolist()
# if game.dialogue_id not in beta_dict:
# beta_dict[game.dialogue_id] = []
# beta_dict[game.dialogue_id].append(beta)
# else:
# beta_dict[game.dialogue_id].append(beta)
new_games.append(game)
return new_games, att_dict #, beta_dict