def train_speaker(train_env, tok, n_iters, log_every=500, val_envs={}): writer = SummaryWriter(logdir=log_dir) listner = Seq2SeqAgent(train_env, "", tok, args.maxAction) speaker = Speaker(train_env, listner, tok) if args.fast_train: log_every = 40 best_bleu = defaultdict(lambda: 0) best_loss = defaultdict(lambda: 1232) for idx in range(0, n_iters, log_every): interval = min(log_every, n_iters - idx) # Train for log_every interval speaker.env = train_env speaker.train(interval) # Train interval iters print() print("Iter: %d" % idx) # Evaluation for env_name, (env, evaluator) in val_envs.items(): if 'train' in env_name: # Ignore the large training set for the efficiency continue print("............ Evaluating %s ............." % env_name) speaker.env = env path2inst, loss, word_accu, sent_accu = speaker.valid() path_id = next(iter(path2inst.keys())) print("Inference: ", tok.decode_sentence(path2inst[path_id])) print("GT: ", evaluator.gt[str(path_id)]['instructions']) bleu_score, precisions = evaluator.bleu_score(path2inst) # Tensorboard log writer.add_scalar("bleu/%s" % (env_name), bleu_score, idx) writer.add_scalar("loss/%s" % (env_name), loss, idx) writer.add_scalar("word_accu/%s" % (env_name), word_accu, idx) writer.add_scalar("sent_accu/%s" % (env_name), sent_accu, idx) writer.add_scalar("bleu4/%s" % (env_name), precisions[3], idx) # Save the model according to the bleu score if bleu_score > best_bleu[env_name]: best_bleu[env_name] = bleu_score print('Save the model with %s BEST env bleu %0.4f' % (env_name, bleu_score)) speaker.save(idx, os.path.join(log_dir, 'state_dict', 'best_%s_bleu' % env_name)) if loss < best_loss[env_name]: best_loss[env_name] = loss print('Save the model with %s BEST env loss %0.4f' % (env_name, loss)) speaker.save(idx, os.path.join(log_dir, 'state_dict', 'best_%s_loss' % env_name)) # Screen print out print("Bleu 1: %0.4f Bleu 2: %0.4f, Bleu 3 :%0.4f, Bleu 4: %0.4f" % tuple(precisions))
def valid_speaker(tok, val_envs): import tqdm listner = Seq2SeqAgent(None, "", tok, args.maxAction) speaker = Speaker(None, listner, tok) speaker.load(os.path.join(log_dir, 'state_dict', 'best_val_seen_bleu')) # speaker.load(os.path.join(log_dir, 'state_dict', 'best_val_unseen_loss')) for args.beam in [False, True]: print("Using Beam Search %s" % args.beam) for env_name, (env, evaluator) in val_envs.items(): if env_name == 'train': continue print("............ Evaluating %s ............." % env_name) speaker.env = env path2inst, loss, word_accu, sent_accu = speaker.valid( beam=args.beam, wrapper=tqdm.tqdm) path_id = next(iter(path2inst.keys())) print("Inference: ", tok.decode_sentence(path2inst[path_id])) print("GT: ", evaluator.gt[path_id]['instructions']) bleu_score, precisions, _ = evaluator.bleu_score(path2inst) print( "Bleu, Loss, Word_Accu, Sent_Accu for %s is: %0.4f, %0.4f, %0.4f, %0.4f" % (env_name, bleu_score, loss, word_accu, sent_accu)) print( "Bleu 1: %0.4f Bleu 2: %0.4f, Bleu 3 :%0.4f, Bleu 4: %0.4f" % tuple(precisions)) print("Average Length %0.4f" % utils.average_length(path2inst))
def valid_speaker(train_env, tok, val_envs): import tqdm listner = Seq2SeqAgent(train_env, "", tok, args.maxAction) speaker = Speaker(train_env, listner, tok) speaker.load(args.load) for env_name, (env, evaluator) in val_envs.items(): if env_name == 'train': continue print("............ Evaluating %s ............." % env_name) speaker.env = env path2inst, loss, word_accu, sent_accu = speaker.valid(wrapper=tqdm.tqdm) path_id = next(iter(path2inst.keys())) print("Inference: ", tok.decode_sentence(path2inst[path_id])) print("GT: ", evaluator.gt[str(path_id)]['instructions']) bleu_score, precisions = evaluator.bleu_score(path2inst) print(len(env.data), len(path2inst.keys())) import pdb; pdb.set_trace()
def valid_speaker(tok, val_envs): import tqdm listner = Seq2SeqAgent(None, "", tok, args.maxAction) speaker = Speaker(None, listner, tok) speaker.load(args.load) for env_name, (env, evaluator) in val_envs.items(): if env_name == 'train': continue print("............ Evaluating %s ............." % env_name) speaker.env = env path2inst, loss, word_accu, sent_accu = speaker.valid(wrapper=tqdm.tqdm) path_id = next(iter(path2inst.keys())) print("Inference: ", tok.decode_sentence(path2inst[path_id])) print("GT: ", evaluator.gt[path_id]['instructions']) pathXinst = list(path2inst.items()) name2score = evaluator.lang_eval(pathXinst, no_metrics={'METEOR'}) score_string = " " for score_name, score in name2score.items(): score_string += "%s_%s: %0.4f " % (env_name, score_name, score) print("For env %s" % env_name) print(score_string) print("Average Length %0.4f" % utils.average_length(path2inst))
def create_augment_data(): setup() # Create a batch training environment that will also preprocess text vocab = read_vocab(TRAIN_VOCAB) tok = Tokenizer(vocab=vocab, encoding_length=args.maxInput) # Load features feat_dict = read_img_features(features) candidate_dict = utils.read_candidates(CANDIDATE_FEATURES) # The datasets to be augmented print("Start to augment the data") aug_envs = [] # aug_envs.append( # R2RBatch( # feat_dict, candidate_dict, batch_size=args.batchSize, splits=['train'], tokenizer=tok # ) # ) # aug_envs.append( # SemiBatch(False, 'tasks/R2R/data/all_paths_46_removetrain.json', # feat_dict, candidate_dict, batch_size=args.batchSize, splits=['train', 'val_seen'], tokenizer=tok) # ) aug_envs.append( SemiBatch(False, 'tasks/R2R/data/all_paths_46_removevalunseen.json', "unseen", feat_dict, candidate_dict, batch_size=args.batchSize, splits=['val_unseen'], tokenizer=tok)) aug_envs.append( SemiBatch(False, 'tasks/R2R/data/all_paths_46_removetest.json', "test", feat_dict, candidate_dict, batch_size=args.batchSize, splits=['test'], tokenizer=tok)) # aug_envs.append( # R2RBatch( # feat_dict, candidate_dict, batch_size=args.batchSize, splits=['val_seen'], tokenizer=tok # ) # ) # aug_envs.append( # R2RBatch( # feat_dict, candidate_dict, batch_size=args.batchSize, splits=['val_unseen'], tokenizer=tok # ) # ) for snapshot in os.listdir(os.path.join(log_dir, 'state_dict')): # if snapshot != "best_val_unseen_bleu": # Select a particular snapshot to process. (O/w, it will make for every snapshot) if snapshot != "best_val_unseen_bleu": continue # Create Speaker listner = Seq2SeqAgent(aug_envs[0], "", tok, args.maxAction) speaker = Speaker(aug_envs[0], listner, tok) # Load Weight load_iter = speaker.load(os.path.join(log_dir, 'state_dict', snapshot)) print("Load from iter %d" % (load_iter)) # Augment the env from aug_envs for aug_env in aug_envs: speaker.env = aug_env # Create the aug data import tqdm path2inst = speaker.get_insts(beam=args.beam, wrapper=tqdm.tqdm) data = [] for datum in aug_env.fake_data: datum = datum.copy() path_id = datum['path_id'] if path_id in path2inst: datum['instructions'] = [ tok.decode_sentence(path2inst[path_id]) ] datum.pop('instr_encoding') # Remove Redundant keys datum.pop('instr_id') data.append(datum) print("Totally, %d data has been generated for snapshot %s." % (len(data), snapshot)) print("Average Length %0.4f" % utils.average_length(path2inst)) print(datum) # Print a Sample # Save the data import json os.makedirs(os.path.join(log_dir, 'aug_data'), exist_ok=True) beam_tag = "_beam" if args.beam else "" json.dump(data, open( os.path.join( log_dir, 'aug_data', '%s_%s%s.json' % (snapshot, aug_env.name, beam_tag)), 'w'), sort_keys=True, indent=4, separators=(',', ': '))