def main():
params = options.readCommandLine()
split = params['evalSplit']
index = params['index']
img_name, word2ind = load_downloaded_json(params['inputJson'],
split=split,
index=index)
vocab = Vocabulary(word2ind)
params['continue'] = True
params['vocabSize'] = vocab.size
aBot, loadedParams, _ = utils.loadModel(params, 'abot', overwrite=False)
assert aBot.encoder.vocabSize == vocab.size
aBot.eval()
aBot.reset()
img_feat = load_download_img_feat(params['inputImg'],
split=split,
index=index)
img_feat = torch.from_numpy(img_feat)
cap, cap_len = load_processed_data(params['inputQues'],
vocab,
split=split,
index=index,
gt=params['gt'])
cap, cap_len = process(cap, cap_len, vocab)
img_feat = Variable(img_feat, volatile=True)
cap = Variable(cap, volatile=True)
cap_len = Variable(cap_len, volatile=True)
aBot.observe(-1, image=img_feat[None, :], caption=cap, captionLens=cap_len)
num_rounds = params['numRounds']
for r in range(num_rounds):
q = input(
"Round {} question (will add ? automatically) >>> ".format(r + 1))
q = tokenize_and_encode(q + '?', vocab)
q_len = len(q)
q, q_len = process(q, q_len, vocab)
q = Variable(q, volatile=True)
q_len = Variable(q_len, volatile=True)
aBot.observe(r, ques=q, quesLens=q_len)
answers, ansLens = aBot.forwardDecode(beamSize=5, inference='greedy')
aBot.observe(r, ans=answers, ansLens=ansLens)
print('A: {}'.format(
decode_sent(answers.data[0].numpy(), ansLens.data[0], vocab)[8:]))
from utils import utilities as utils
from utils.visualize import VisdomVisualize
try:
import nsml
from nsml import IS_ON_NSML
print('able to use nsml')
except ImportError:
pass
#---------------------------------------------------------------------------------------
# Setup
#---------------------------------------------------------------------------------------
# Read the command line options
params = options.readCommandLine()
# Seed rng for reproducibility
random.seed(params['randomSeed'])
torch.manual_seed(params['randomSeed'])
if params['useGPU']:
torch.cuda.manual_seed_all(params['randomSeed'])
# Setup dataloader
splits = ['train', 'val', 'test']
dataset = VisDialDataset(params, splits)
# Params to transfer from dataset
transfer = ['vocabSize', 'numOptions', 'numRounds']
for key in transfer:
def main():
params = options.readCommandLine()
data_params = options.data_params(params)
if params['dataset'] == 'VQA':
dataset = VQAPoolDataset(data_params, ['train', 'val1', 'val2'])
dataset.split = 'val1'
elif params['dataset'] == 'CUB':
dataset = CUBPoolDataset(data_params, ['train', 'val', 'test'],
load_vis_info=True)
dataset.split = 'val'
elif params['dataset'] == 'AWA':
dataset = AWAPoolDataset(data_params, ['train', 'val', 'test'],
load_vis_info=True)
dataset.split = 'val'
dataloader = DataLoader(dataset,
batch_size=params['batchSize'],
shuffle=False,
num_workers=10,
drop_last=True,
pin_memory=True)
dliter = iter(dataloader)
for _ in range(params['batchIndex'] + 1):
batch = next(dliter)
batch = {
key: v.cuda() if hasattr(v, 'cuda') else v
for key, v in batch.items()
}
qbot, abot, q_exp, qbf, abf, qep, aep = next(iter(load_models(params)))
if params['visMode'] == 'latent':
zsources = ['encoder', 'policy']
#zsources += ['interpolate']
#zsources += ['manual{}'.format(dim) for dim in list(range(128))[:3]]
#zsources += ['prior']
inferences = [('greedy', 'greedy'), ('greedy', 'sample'),
('sample', 'greedy'), ('sample', 'sample')]
qd = generate_z_info(batch, qbot, zsources, inferences,
dataset.ind2word)
html = render_webpage(batch, qbot, qd, zsources, inferences, dataset)
save_file = pth.join(
params['savePath'],
'{}_z_examples.html'.format(params['evalSaveName']))
with open(save_file, 'w') as f:
f.write(html)
elif params['visMode'] == 'interpolate':
zsources = ['policy', 'interpolate']
inferences = [('greedy', 'greedy'), ('sample', 'greedy'),
('sample', 'sample')]
qd = generate_z_info(batch, qbot, zsources, inferences,
dataset.ind2word)
html = render_webpage(batch, qbot, qd, zsources, inferences, dataset)
save_file = pth.join(
params['savePath'],
'{}_z_examples.html'.format(params['evalSaveName']))
with open(save_file, 'w') as f:
f.write(html)
elif params['visMode'] == 'dialog':
html = generate_and_render_dialog(batch, qbot, abot, dataset,
params['maxRounds'], q_exp, abf)
save_file = pth.join(
params['savePath'],
'{}_dialog_examples.html'.format(params['evalSaveName']))
with open(save_file, 'w') as f:
f.write(html)
elif params['visMode'] == 'mturk':
dialog_data = generate_and_render_dialog(batch,
qbot,
abot,
dataset,
params['maxRounds'],
q_exp,
abf,
generate_only=True)
os.makedirs(pth.join(params['savePath'], 'mturk'), exist_ok=True)
save_file = pth.join(
params['savePath'], 'mturk',
'{}_dialog_data.joblib'.format(params['evalSaveName']))
joblib.dump(dialog_data, save_file, compress=True)
def main():
params = options.readCommandLine()
# memory for unique samples is a concern; use smaller batch sizes
n_samples = min(50, params['batchSize'])
unique_batch_size = params['batchSize'] // n_samples
unique_examples = 300
unique_num_batches = (unique_examples - 1) // unique_batch_size + 1
z_sources = ['policy', 'encoder', 'prior']
# z, dec
inferences = [('greedy', 'greedy'), ('sample', 'greedy'),
('greedy', 'sample'), ('sample', 'sample')]
data_params = options.data_params(params)
if params['dataset'] == 'VQA':
dataset = VQAPoolDataset(data_params, ['train', 'val1', 'val2'])
val_split = {
'val1': 'val1',
'val2': 'val2',
'val': 'val1',
'test': 'val2',
}[params['evalSplit']]
elif params['dataset'] == 'CUB':
dataset = CUBPoolDataset(data_params, ['train', 'val', 'test'],
load_vis_info=True)
val_split = params['evalSplit']
elif params['dataset'] == 'AWA':
dataset = AWAPoolDataset(data_params, ['train', 'val', 'test'],
load_vis_info=True)
val_split = params['evalSplit']
dataset.split = val_split
#unique_records = []
eval_records = []
for qbot, abot, q_exp, qbf, abf, qep, aep in load_models(params):
# evaluate task performance
params_ = params.copy()
params_['trainMode'] = 'fine-qbot'
print('evaluating {}'.format(str(dataset.params)))
result = evalQBot(params_,
qbot,
dataset,
val_split,
aBot=abot,
exampleLimit=params['evalLimit'],
do_perplexity=True,
do_relevance=True,
do_diversity=True)
result['abot_file'] = abf
result['qbot_file'] = qbf
result['qbot'] = q_exp
result['qepoch'] = qep
result['aepoch'] = aep
result[
'poolType'] = dataset.poolType if dataset.name == 'VQA' else dataset.pool_type
result[
'poolSize'] = dataset.poolSize if dataset.name == 'VQA' else dataset.pool_size
result[
'randQues'] = dataset.randQues if dataset.name == 'VQA' else False
if 'lang_result' in result and 'byRound' in result['lang_result']:
for d in result['lang_result']['byRound']:
del d['imgToEval']
else:
result['lang_result'] = {}
eval_records.append(result)
out_prefix = 'pt-{}_ps-{}'.format(params['poolType'], params['poolSize'])
#unique_df = pd.DataFrame(unique_records)
eval_df = pd.DataFrame(eval_records)
out_file = pth.join(params['savePath'],
'{}.joblib'.format(params['evalSaveName']))
joblib.dump(
{
#'unique_df': unique_df,
'eval_df': eval_df,
},
out_file,
compress=True)
