def main(): params = options.readCommandLine() split = params['evalSplit'] index = params['index'] img_name, word2ind = load_downloaded_json(params['inputJson'], split=split, index=index) vocab = Vocabulary(word2ind) params['continue'] = True params['vocabSize'] = vocab.size aBot, loadedParams, _ = utils.loadModel(params, 'abot', overwrite=False) assert aBot.encoder.vocabSize == vocab.size aBot.eval() aBot.reset() img_feat = load_download_img_feat(params['inputImg'], split=split, index=index) img_feat = torch.from_numpy(img_feat) cap, cap_len = load_processed_data(params['inputQues'], vocab, split=split, index=index, gt=params['gt']) cap, cap_len = process(cap, cap_len, vocab) img_feat = Variable(img_feat, volatile=True) cap = Variable(cap, volatile=True) cap_len = Variable(cap_len, volatile=True) aBot.observe(-1, image=img_feat[None, :], caption=cap, captionLens=cap_len) num_rounds = params['numRounds'] for r in range(num_rounds): q = input( "Round {} question (will add ? automatically) >>> ".format(r + 1)) q = tokenize_and_encode(q + '?', vocab) q_len = len(q) q, q_len = process(q, q_len, vocab) q = Variable(q, volatile=True) q_len = Variable(q_len, volatile=True) aBot.observe(r, ques=q, quesLens=q_len) answers, ansLens = aBot.forwardDecode(beamSize=5, inference='greedy') aBot.observe(r, ans=answers, ansLens=ansLens) print('A: {}'.format( decode_sent(answers.data[0].numpy(), ansLens.data[0], vocab)[8:]))
from utils import utilities as utils from utils.visualize import VisdomVisualize try: import nsml from nsml import IS_ON_NSML print('able to use nsml') except ImportError: pass #--------------------------------------------------------------------------------------- # Setup #--------------------------------------------------------------------------------------- # Read the command line options params = options.readCommandLine() # Seed rng for reproducibility random.seed(params['randomSeed']) torch.manual_seed(params['randomSeed']) if params['useGPU']: torch.cuda.manual_seed_all(params['randomSeed']) # Setup dataloader splits = ['train', 'val', 'test'] dataset = VisDialDataset(params, splits) # Params to transfer from dataset transfer = ['vocabSize', 'numOptions', 'numRounds'] for key in transfer:
def main(): params = options.readCommandLine() data_params = options.data_params(params) if params['dataset'] == 'VQA': dataset = VQAPoolDataset(data_params, ['train', 'val1', 'val2']) dataset.split = 'val1' elif params['dataset'] == 'CUB': dataset = CUBPoolDataset(data_params, ['train', 'val', 'test'], load_vis_info=True) dataset.split = 'val' elif params['dataset'] == 'AWA': dataset = AWAPoolDataset(data_params, ['train', 'val', 'test'], load_vis_info=True) dataset.split = 'val' dataloader = DataLoader(dataset, batch_size=params['batchSize'], shuffle=False, num_workers=10, drop_last=True, pin_memory=True) dliter = iter(dataloader) for _ in range(params['batchIndex'] + 1): batch = next(dliter) batch = { key: v.cuda() if hasattr(v, 'cuda') else v for key, v in batch.items() } qbot, abot, q_exp, qbf, abf, qep, aep = next(iter(load_models(params))) if params['visMode'] == 'latent': zsources = ['encoder', 'policy'] #zsources += ['interpolate'] #zsources += ['manual{}'.format(dim) for dim in list(range(128))[:3]] #zsources += ['prior'] inferences = [('greedy', 'greedy'), ('greedy', 'sample'), ('sample', 'greedy'), ('sample', 'sample')] qd = generate_z_info(batch, qbot, zsources, inferences, dataset.ind2word) html = render_webpage(batch, qbot, qd, zsources, inferences, dataset) save_file = pth.join( params['savePath'], '{}_z_examples.html'.format(params['evalSaveName'])) with open(save_file, 'w') as f: f.write(html) elif params['visMode'] == 'interpolate': zsources = ['policy', 'interpolate'] inferences = [('greedy', 'greedy'), ('sample', 'greedy'), ('sample', 'sample')] qd = generate_z_info(batch, qbot, zsources, inferences, dataset.ind2word) html = render_webpage(batch, qbot, qd, zsources, inferences, dataset) save_file = pth.join( params['savePath'], '{}_z_examples.html'.format(params['evalSaveName'])) with open(save_file, 'w') as f: f.write(html) elif params['visMode'] == 'dialog': html = generate_and_render_dialog(batch, qbot, abot, dataset, params['maxRounds'], q_exp, abf) save_file = pth.join( params['savePath'], '{}_dialog_examples.html'.format(params['evalSaveName'])) with open(save_file, 'w') as f: f.write(html) elif params['visMode'] == 'mturk': dialog_data = generate_and_render_dialog(batch, qbot, abot, dataset, params['maxRounds'], q_exp, abf, generate_only=True) os.makedirs(pth.join(params['savePath'], 'mturk'), exist_ok=True) save_file = pth.join( params['savePath'], 'mturk', '{}_dialog_data.joblib'.format(params['evalSaveName'])) joblib.dump(dialog_data, save_file, compress=True)
def main(): params = options.readCommandLine() # memory for unique samples is a concern; use smaller batch sizes n_samples = min(50, params['batchSize']) unique_batch_size = params['batchSize'] // n_samples unique_examples = 300 unique_num_batches = (unique_examples - 1) // unique_batch_size + 1 z_sources = ['policy', 'encoder', 'prior'] # z, dec inferences = [('greedy', 'greedy'), ('sample', 'greedy'), ('greedy', 'sample'), ('sample', 'sample')] data_params = options.data_params(params) if params['dataset'] == 'VQA': dataset = VQAPoolDataset(data_params, ['train', 'val1', 'val2']) val_split = { 'val1': 'val1', 'val2': 'val2', 'val': 'val1', 'test': 'val2', }[params['evalSplit']] elif params['dataset'] == 'CUB': dataset = CUBPoolDataset(data_params, ['train', 'val', 'test'], load_vis_info=True) val_split = params['evalSplit'] elif params['dataset'] == 'AWA': dataset = AWAPoolDataset(data_params, ['train', 'val', 'test'], load_vis_info=True) val_split = params['evalSplit'] dataset.split = val_split #unique_records = [] eval_records = [] for qbot, abot, q_exp, qbf, abf, qep, aep in load_models(params): # evaluate task performance params_ = params.copy() params_['trainMode'] = 'fine-qbot' print('evaluating {}'.format(str(dataset.params))) result = evalQBot(params_, qbot, dataset, val_split, aBot=abot, exampleLimit=params['evalLimit'], do_perplexity=True, do_relevance=True, do_diversity=True) result['abot_file'] = abf result['qbot_file'] = qbf result['qbot'] = q_exp result['qepoch'] = qep result['aepoch'] = aep result[ 'poolType'] = dataset.poolType if dataset.name == 'VQA' else dataset.pool_type result[ 'poolSize'] = dataset.poolSize if dataset.name == 'VQA' else dataset.pool_size result[ 'randQues'] = dataset.randQues if dataset.name == 'VQA' else False if 'lang_result' in result and 'byRound' in result['lang_result']: for d in result['lang_result']['byRound']: del d['imgToEval'] else: result['lang_result'] = {} eval_records.append(result) out_prefix = 'pt-{}_ps-{}'.format(params['poolType'], params['poolSize']) #unique_df = pd.DataFrame(unique_records) eval_df = pd.DataFrame(eval_records) out_file = pth.join(params['savePath'], '{}.joblib'.format(params['evalSaveName'])) joblib.dump( { #'unique_df': unique_df, 'eval_df': eval_df, }, out_file, compress=True)