def train_speaker(train_env, tok, n_iters, log_every=500, val_envs={}):
writer = SummaryWriter(logdir=log_dir)
listner = Seq2SeqAgent(train_env, "", tok, args.maxAction)
speaker = Speaker(train_env, listner, tok)
if args.fast_train:
log_every = 40
best_bleu = defaultdict(lambda: 0)
best_loss = defaultdict(lambda: 1232)
for idx in range(0, n_iters, log_every):
interval = min(log_every, n_iters - idx)
# Train for log_every interval
speaker.env = train_env
speaker.train(interval) # Train interval iters
print()
print("Iter: %d" % idx)
# Evaluation
for env_name, (env, evaluator) in val_envs.items():
if 'train' in env_name: # Ignore the large training set for the efficiency
continue
print("............ Evaluating %s ............." % env_name)
speaker.env = env
path2inst, loss, word_accu, sent_accu = speaker.valid()
path_id = next(iter(path2inst.keys()))
print("Inference: ", tok.decode_sentence(path2inst[path_id]))
print("GT: ", evaluator.gt[str(path_id)]['instructions'])
bleu_score, precisions = evaluator.bleu_score(path2inst)
# Tensorboard log
writer.add_scalar("bleu/%s" % (env_name), bleu_score, idx)
writer.add_scalar("loss/%s" % (env_name), loss, idx)
writer.add_scalar("word_accu/%s" % (env_name), word_accu, idx)
writer.add_scalar("sent_accu/%s" % (env_name), sent_accu, idx)
writer.add_scalar("bleu4/%s" % (env_name), precisions[3], idx)
# Save the model according to the bleu score
if bleu_score > best_bleu[env_name]:
best_bleu[env_name] = bleu_score
print('Save the model with %s BEST env bleu %0.4f' % (env_name, bleu_score))
speaker.save(idx, os.path.join(log_dir, 'state_dict', 'best_%s_bleu' % env_name))
if loss < best_loss[env_name]:
best_loss[env_name] = loss
print('Save the model with %s BEST env loss %0.4f' % (env_name, loss))
speaker.save(idx, os.path.join(log_dir, 'state_dict', 'best_%s_loss' % env_name))
# Screen print out
print("Bleu 1: %0.4f Bleu 2: %0.4f, Bleu 3 :%0.4f, Bleu 4: %0.4f" % tuple(precisions))
def valid_speaker(tok, val_envs):
import tqdm
listner = Seq2SeqAgent(None, "", tok, args.maxAction)
speaker = Speaker(None, listner, tok)
speaker.load(os.path.join(log_dir, 'state_dict', 'best_val_seen_bleu'))
# speaker.load(os.path.join(log_dir, 'state_dict', 'best_val_unseen_loss'))
for args.beam in [False, True]:
print("Using Beam Search %s" % args.beam)
for env_name, (env, evaluator) in val_envs.items():
if env_name == 'train':
continue
print("............ Evaluating %s ............." % env_name)
speaker.env = env
path2inst, loss, word_accu, sent_accu = speaker.valid(
beam=args.beam, wrapper=tqdm.tqdm)
path_id = next(iter(path2inst.keys()))
print("Inference: ", tok.decode_sentence(path2inst[path_id]))
print("GT: ", evaluator.gt[path_id]['instructions'])
bleu_score, precisions, _ = evaluator.bleu_score(path2inst)
print(
"Bleu, Loss, Word_Accu, Sent_Accu for %s is: %0.4f, %0.4f, %0.4f, %0.4f"
% (env_name, bleu_score, loss, word_accu, sent_accu))
print(
"Bleu 1: %0.4f Bleu 2: %0.4f, Bleu 3 :%0.4f, Bleu 4: %0.4f" %
tuple(precisions))
print("Average Length %0.4f" % utils.average_length(path2inst))
def valid_speaker(train_env, tok, val_envs):
import tqdm
listner = Seq2SeqAgent(train_env, "", tok, args.maxAction)
speaker = Speaker(train_env, listner, tok)
speaker.load(args.load)
for env_name, (env, evaluator) in val_envs.items():
if env_name == 'train':
continue
print("............ Evaluating %s ............." % env_name)
speaker.env = env
path2inst, loss, word_accu, sent_accu = speaker.valid(wrapper=tqdm.tqdm)
path_id = next(iter(path2inst.keys()))
print("Inference: ", tok.decode_sentence(path2inst[path_id]))
print("GT: ", evaluator.gt[str(path_id)]['instructions'])
bleu_score, precisions = evaluator.bleu_score(path2inst)
print(len(env.data), len(path2inst.keys()))
import pdb; pdb.set_trace()
def valid_speaker(tok, val_envs):
import tqdm
listner = Seq2SeqAgent(None, "", tok, args.maxAction)
speaker = Speaker(None, listner, tok)
speaker.load(args.load)
for env_name, (env, evaluator) in val_envs.items():
if env_name == 'train':
continue
print("............ Evaluating %s ............." % env_name)
speaker.env = env
path2inst, loss, word_accu, sent_accu = speaker.valid(wrapper=tqdm.tqdm)
path_id = next(iter(path2inst.keys()))
print("Inference: ", tok.decode_sentence(path2inst[path_id]))
print("GT: ", evaluator.gt[path_id]['instructions'])
pathXinst = list(path2inst.items())
name2score = evaluator.lang_eval(pathXinst, no_metrics={'METEOR'})
score_string = " "
for score_name, score in name2score.items():
score_string += "%s_%s: %0.4f " % (env_name, score_name, score)
print("For env %s" % env_name)
print(score_string)
print("Average Length %0.4f" % utils.average_length(path2inst))
def create_augment_data():
setup()
# Create a batch training environment that will also preprocess text
vocab = read_vocab(TRAIN_VOCAB)
tok = Tokenizer(vocab=vocab, encoding_length=args.maxInput)
# Load features
feat_dict = read_img_features(features)
candidate_dict = utils.read_candidates(CANDIDATE_FEATURES)
# The datasets to be augmented
print("Start to augment the data")
aug_envs = []
# aug_envs.append(
# R2RBatch(
# feat_dict, candidate_dict, batch_size=args.batchSize, splits=['train'], tokenizer=tok
# )
# )
# aug_envs.append(
# SemiBatch(False, 'tasks/R2R/data/all_paths_46_removetrain.json',
# feat_dict, candidate_dict, batch_size=args.batchSize, splits=['train', 'val_seen'], tokenizer=tok)
# )
aug_envs.append(
SemiBatch(False,
'tasks/R2R/data/all_paths_46_removevalunseen.json',
"unseen",
feat_dict,
candidate_dict,
batch_size=args.batchSize,
splits=['val_unseen'],
tokenizer=tok))
aug_envs.append(
SemiBatch(False,
'tasks/R2R/data/all_paths_46_removetest.json',
"test",
feat_dict,
candidate_dict,
batch_size=args.batchSize,
splits=['test'],
tokenizer=tok))
# aug_envs.append(
# R2RBatch(
# feat_dict, candidate_dict, batch_size=args.batchSize, splits=['val_seen'], tokenizer=tok
# )
# )
# aug_envs.append(
# R2RBatch(
# feat_dict, candidate_dict, batch_size=args.batchSize, splits=['val_unseen'], tokenizer=tok
# )
# )
for snapshot in os.listdir(os.path.join(log_dir, 'state_dict')):
# if snapshot != "best_val_unseen_bleu": # Select a particular snapshot to process. (O/w, it will make for every snapshot)
if snapshot != "best_val_unseen_bleu":
continue
# Create Speaker
listner = Seq2SeqAgent(aug_envs[0], "", tok, args.maxAction)
speaker = Speaker(aug_envs[0], listner, tok)
# Load Weight
load_iter = speaker.load(os.path.join(log_dir, 'state_dict', snapshot))
print("Load from iter %d" % (load_iter))
# Augment the env from aug_envs
for aug_env in aug_envs:
speaker.env = aug_env
# Create the aug data
import tqdm
path2inst = speaker.get_insts(beam=args.beam, wrapper=tqdm.tqdm)
data = []
for datum in aug_env.fake_data:
datum = datum.copy()
path_id = datum['path_id']
if path_id in path2inst:
datum['instructions'] = [
tok.decode_sentence(path2inst[path_id])
]
datum.pop('instr_encoding') # Remove Redundant keys
datum.pop('instr_id')
data.append(datum)
print("Totally, %d data has been generated for snapshot %s." %
(len(data), snapshot))
print("Average Length %0.4f" % utils.average_length(path2inst))
print(datum) # Print a Sample
# Save the data
import json
os.makedirs(os.path.join(log_dir, 'aug_data'), exist_ok=True)
beam_tag = "_beam" if args.beam else ""
json.dump(data,
open(
os.path.join(
log_dir, 'aug_data', '%s_%s%s.json' %
(snapshot, aug_env.name, beam_tag)), 'w'),
sort_keys=True,
indent=4,
separators=(',', ': '))
