def main():
params = options.readCommandLine()
split = params['evalSplit']
index = params['index']
img_name, word2ind = load_downloaded_json(params['inputJson'],
split=split,
index=index)
vocab = Vocabulary(word2ind)
params['continue'] = True
params['vocabSize'] = vocab.size
aBot, loadedParams, _ = utils.loadModel(params, 'abot', overwrite=False)
assert aBot.encoder.vocabSize == vocab.size
aBot.eval()
aBot.reset()
img_feat = load_download_img_feat(params['inputImg'],
split=split,
index=index)
img_feat = torch.from_numpy(img_feat)
cap, cap_len = load_processed_data(params['inputQues'],
vocab,
split=split,
index=index,
gt=params['gt'])
cap, cap_len = process(cap, cap_len, vocab)
img_feat = Variable(img_feat, volatile=True)
cap = Variable(cap, volatile=True)
cap_len = Variable(cap_len, volatile=True)
aBot.observe(-1, image=img_feat[None, :], caption=cap, captionLens=cap_len)
num_rounds = params['numRounds']
for r in range(num_rounds):
q = input(
"Round {} question (will add ? automatically) >>> ".format(r + 1))
q = tokenize_and_encode(q + '?', vocab)
q_len = len(q)
q, q_len = process(q, q_len, vocab)
q = Variable(q, volatile=True)
q_len = Variable(q_len, volatile=True)
aBot.observe(r, ques=q, quesLens=q_len)
answers, ansLens = aBot.forwardDecode(beamSize=5, inference='greedy')
aBot.observe(r, ans=answers, ansLens=ansLens)
print('A: {}'.format(
decode_sent(answers.data[0].numpy(), ansLens.data[0], vocab)[8:]))
if hasattr(dataset, key):
params[key] = getattr(dataset, key)
# Create save path and checkpoints folder
os.makedirs('checkpoints', exist_ok=True)
os.mkdir(params['savePath'])
# Loading Modules
parameters = []
aBot = None
qBot = None
aqmBot = None
# Loading A-Bot
if params['trainMode'] in ['sl-abot', 'rl-full-QAf', 'aqmbot-dep']:
aBot, loadedParams, optim_state = utils.loadModel(params, 'abot')
for key in loadedParams:
params[key] = loadedParams[key]
if params['trainMode'] not in ['aqmbot-dep']:
parameters.extend(aBot.parameters())
# aBot = nn.DataParallel(aBot)
# Loading Q-Bot
if params['trainMode'] in ['sl-qbot', 'rl-full-QAf']:
qBot, loadedParams, optim_state = utils.loadModel(params, 'qbot')
for key in loadedParams:
params[key] = loadedParams[key]
# Filtering parameters which require a gradient update
parameters.extend(filter(lambda p: p.requires_grad, qBot.parameters()))
# parameters.extend(qBot.parameters())
logging.info('=' * 80)
# Always load checkpoint parameters with continue flag
params['continue'] = True
excludeParams = ['batchSize', 'visdomEnv', 'startFrom', 'qstartFrom', 'trainMode', \
'evalModeList', 'inputImg', 'inputQues', 'inputJson', 'evalTitle', 'beamSize', \
'enableVisdom', 'visdomServer', 'visdomServerPort', \
'qaCategory','categoryMap']
aBot = None
qBot = None
# load aBot
if params['startFrom']:
aBot, loadedParams, _ = utils.loadModel(params, 'abot', overwrite=True)
assert aBot.encoder.vocabSize == dataset.vocabSize, "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aBot.eval()
# Retaining certain dataloder parameters
for key in excludeParams:
params[key] = dlparams[key]
# load qBot
if params['qstartFrom']:
qBot, loadedParams, _ = utils.loadModel(params, 'qbot', overwrite=True)
assert qBot.encoder.vocabSize == params[
'vocabSize'], "Vocab size mismatch!"
for key in loadedParams:
def main(params):
aqmSetting = None
if ("AQMBotRank" in params["evalModeList"]
or "AQMdialog" in params["evalModeList"]
or "AQMdemo" in params["evalModeList"]):
aqmSetting = getAQMSetting(params)
# setup dataloader
dlparams = params.copy()
dlparams['useIm'] = True
dlparams['useHistory'] = True
dlparams['numRounds'] = 10
splits = ['val', 'test']
dataset = VisDialDataset(dlparams, splits)
# Transferring dataset parameters
transfer = ['vocabSize', 'numOptions', 'numRounds']
for key in transfer:
if hasattr(dataset, key):
params[key] = getattr(dataset, key)
if 'numRounds' not in params:
params['numRounds'] = 10
# Always load checkpoint parameters with continue flag
params['continue'] = True
excludeParams = ['batchSize', 'visdomEnv', 'startFrom', 'qstartFrom', 'trainMode', \
'evalModeList', 'inputImg', 'inputQues', 'inputJson', 'evalTitle', 'beamSize', \
'enableVisdom', 'visdomServer', 'visdomServerPort', 'randomCaption', 'zeroCaption',
'numImg', 'numQ', 'numA', 'alpha',
'qbeamSize', 'gamma', 'delta', 'lambda',
'onlyGuesser', 'randQ', 'gen1Q', 'gtQ', 'randA', 'noHistory',
'slGuesser', 'resampleEveryDialog']
aBot = None
qBot = None
aqmBot = None
# load aBot
print('load aBot')
if params['startFrom']:
aBot, loadedParams, _ = utils.loadModel(params, 'abot', overwrite=True)
assert aBot.encoder.vocabSize == dataset.vocabSize, "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aBot.eval()
# Retaining certain dataloder parameters
for key in excludeParams:
params[key] = dlparams[key]
print('load qBot')
# load qBot
if params['qstartFrom'] and not params['aqmstartFrom']:
qBot, loadedParams, _ = utils.loadModel(params, 'qbot', overwrite=True)
assert qBot.encoder.vocabSize == params[
'vocabSize'], "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
qBot.eval()
# Retaining certain dataloder parameters
for key in excludeParams:
params[key] = dlparams[key]
print('load AQM-Bot')
# load aqmBot
if params['aqmstartFrom']: # abot of AQM
assert params['qstartFrom'] # qbot of AQM
aqmBot, loadedParams, _ = utils.loadModel(params,
'AQM-qbot',
overwrite=True)
assert aqmBot.questioner.encoder.vocabSize == params[
'vocabSize'], "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aqmBot.eval()
# load qBot
for key in excludeParams:
params[key] = dlparams[key]
aqmQ, loadedParams, _ = utils.loadModel(params, 'qbot', overwrite=True)
assert aqmQ.encoder.vocabSize == params[
'vocabSize'], "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aqmQ.eval()
for key in excludeParams:
params[key] = dlparams[key]
aqmBot.setQuestioner(aqmQ)
elif params['aqmQStartFrom']:
from visdial.models.aqm_questioner import AQMQuestioner
aqmBot = AQMQuestioner()
aqmBot.eval()
params['qstartFrom'] = params['aqmQStartFrom']
aqmQ, loadedParams, _ = utils.loadModel(params, 'qbot', overwrite=True)
assert aqmQ.encoder.vocabSize == params[
'vocabSize'], "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aqmQ.eval()
for key in excludeParams:
params[key] = dlparams[key]
aqmBot.setQuestioner(aqmQ)
params['startFrom'] = params['aqmAStartFrom']
aqmA, loadedParams, _ = utils.loadModel(params, 'abot', overwrite=True)
assert aqmA.encoder.vocabSize == dataset.vocabSize, "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aqmA.eval()
aqmBot.setAppAnswerer(aqmA)
for key in excludeParams:
params[key] = dlparams[key]
pprint.pprint(params)
#viz.addText(pprint.pformat(params, indent=4))
print("Running evaluation!")
numRounds = params['numRounds']
if 'ckpt_iterid' in params:
iterId = params['ckpt_iterid'] + 1
else:
iterId = -1
if 'test' in splits:
split = 'test'
splitName = 'test - {}'.format(params['evalTitle'])
else:
split = 'val'
splitName = 'full Val - {}'.format(params['evalTitle'])
print("Using split %s" % split)
dataset.split = split
if 'ABotRank' in params['evalModeList']:
if params['aqmstartFrom']:
aBot = aqmBot.appAnswerer
print('evaluating appBot of AQM')
print("Performing ABotRank evaluation")
rankMetrics = rankABot(aBot,
dataset,
split,
scoringFunction=utils.maskedNll,
expLowerLimit=params['expLowerLimit'],
expUpperLimit=params['expUpperLimit'])
print(rankMetrics)
for metric, value in rankMetrics.items():
plotName = splitName + ' - ABot Rank'
#viz.linePlot(iterId, value, plotName, metric, xlabel='Iterations')
if 'QBotRank' in params['evalModeList']:
print("Performing QBotRank evaluation")
rankMetrics, roundRanks = rankQBot(
qBot,
dataset,
split,
expLowerLimit=params['expLowerLimit'],
expUpperLimit=params['expUpperLimit'],
verbose=1)
for metric, value in rankMetrics.items():
plotName = splitName + ' - QBot Rank'
#viz.linePlot(iterId, value, plotName, metric, xlabel='Iterations')
for r in range(numRounds + 1):
for metric, value in roundRanks[r].items():
plotName = '[Iter %d] %s - QABots Rank Roundwise' % \
(iterId, splitName)
#viz.linePlot(r, value, plotName, metric, xlabel='Round')
if 'QABotsRank' in params['evalModeList']:
print("Performing QABotsRank evaluation")
outputPredFile = "data/visdial/visdial/output_predictions_rollout.h5"
rankMetrics, roundRanks = rankQABots(
qBot,
aBot,
dataset,
split,
beamSize=params['beamSize'],
expLowerLimit=params['expLowerLimit'],
expUpperLimit=params['expUpperLimit'],
zeroCaption=params['zeroCaption'],
randomCaption=params['randomCaption'],
numRounds=params['runRounds'])
for metric, value in rankMetrics.items():
plotName = splitName + ' - QABots Rank'
#viz.linePlot(iterId, value, plotName, metric, xlabel='Iterations')
for r in range(numRounds + 1):
for metric, value in roundRanks[r].items():
plotName = '[Iter %d] %s - QBot All Metrics vs Round'%\
(iterId, splitName)
#viz.linePlot(r, value, plotName, metric, xlabel='Round')
if 'AQMBotRank' in params['evalModeList']:
print("Performing AQMBotRank evaluation")
outputPredFile = "data/visdial/visdial/output_predictions_rollout.h5"
rankMetrics, roundRanks = AQMRunner(
aqmBot,
aBot,
dataset,
split,
beamSize=params['beamSize'],
realQA=params['aqmRealQA'],
saveLogs=params['saveLogs'],
showQA=params['showQA'],
expLowerLimit=params['expLowerLimit'],
expUpperLimit=params['expUpperLimit'],
selectedBatchIdxs=params['selectedBatchIdxs'],
numRounds=params['runRounds'],
lda=params['lambda'],
onlyGuesser=params['onlyGuesser'],
numQ=params['numQ'],
qbeamSize=params['qbeamSize'],
numImg=params['numImg'],
alpha=params['alpha'],
numA=params['numA'],
randQ=params['randQ'],
randA=params['randA'],
zeroCaption=params['zeroCaption'],
randomCaption=params['randomCaption'],
gamma=params['gamma'],
delta=params['delta'],
gen1Q=params['gen1Q'],
gtQ=params['gtQ'],
noHistory=params['noHistory'],
slGuesser=params['slGuesser'],
resampleEveryDialog=params['resampleEveryDialog'],
aqmSetting=aqmSetting,
).rankQuestioner()
for metric, value in rankMetrics.items():
plotName = splitName + ' - QABots Rank'
#viz.linePlot(iterId, value, plotName, metric, xlabel='Iterations')
for r in range(numRounds + 1):
for metric, value in roundRanks[r].items():
plotName = '[Iter %d] %s - QBot All Metrics vs Round'%\
(iterId, splitName)
#viz.linePlot(r, value, plotName, metric, xlabel='Round')
if 'dialog' in params['evalModeList']:
print("Performing dialog generation...")
split = 'test'
outputFolder = "dialog_output/results"
os.makedirs(outputFolder, exist_ok=True)
outputPath = os.path.join(outputFolder, "results.json")
dialogDump(params,
dataset,
split,
aBot=aBot,
qBot=qBot,
expLowerLimit=params['expLowerLimit'],
expUpperLimit=params['expUpperLimit'],
beamSize=params['beamSize'],
savePath=outputPath)
if 'AQMdialog' in params['evalModeList']:
print("Performing AQM dialog generation...")
split = 'test'
AQMRunner(
aqmBot,
aBot,
dataset,
split,
beamSize=params['beamSize'],
realQA=params['aqmRealQA'],
saveLogs=params['saveLogs'],
showQA=params['showQA'],
expLowerLimit=params['expLowerLimit'],
expUpperLimit=params['expUpperLimit'],
selectedBatchIdxs=params['selectedBatchIdxs'],
numRounds=params['runRounds'],
lda=params['lambda'],
onlyGuesser=params['onlyGuesser'],
numQ=params['numQ'],
qbeamSize=params['qbeamSize'],
numImg=params['numImg'],
alpha=params['alpha'],
numA=params['numA'],
randQ=params['randQ'],
randA=params['randA'],
zeroCaption=params['zeroCaption'],
randomCaption=params['randomCaption'],
gamma=params['gamma'],
delta=params['delta'],
gen1Q=params['gen1Q'],
gtQ=params['gtQ'],
noHistory=params['noHistory'],
slGuesser=params['slGuesser'],
resampleEveryDialog=params['resampleEveryDialog'],
aqmSetting=aqmSetting,
).dialogDump(params)
"inputImg",
"inputQues",
"inputJson",
"evalTitle",
"beamSize",
"enableVisdom",
"visdomServer",
"visdomServerPort",
]
aBot = None
qBot = None
# load aBot
if params["startFrom"]:
aBot, loadedParams, _ = utils.loadModel(params, "abot", overwrite=True)
assert aBot.encoder.vocabSize == dataset.vocabSize, "Vocab size mismatch!"
for key in loadedParams:
params[key] = loadedParams[key]
aBot.eval()
# Retaining certain dataloder parameters
for key in excludeParams:
params[key] = dlparams[key]
# load qBot
if params["qstartFrom"]:
qBot, loadedParams, _ = utils.loadModel(params, "qbot", overwrite=True)
assert qBot.encoder.vocabSize == params[
"vocabSize"], "Vocab size mismatch!"
for key in loadedParams:
def __init__(self):
'''
'''
# Read the command line options and create options table to initialize dataloader
self.opt = options.readCommandLine()
splits = ['train', 'val', 'test']
self.dataloader = VisDialDataset(self.opt, subsets=splits)
# Params to transfer from dataset
transfer = ['vocabSize', 'numOptions', 'numRounds']
for key in transfer:
if hasattr(self.dataloader, key):
self.opt[key] = getattr(self.dataloader, key)
# Seed rng for reproducibility
random.seed(self.opt['randomSeed'])
torch.manual_seed(self.opt['randomSeed'])
if self.opt['useGPU']:
torch.cuda.manual_seed_all(self.opt['randomSeed'])
self.device = torch.device("cuda:0" if (torch.cuda.is_available() and self.opt['useGPU']) else "cpu")
print("Device:", self.device)
# -- Load Questioner and Answerer model
self.questionerModel, loadedParams, optim_state = utils.loadModel(self.opt, 'qbot')
self.questionerModelParams = loadedParams
self.answererModel, loadedParams, optim_state = utils.loadModel(self.opt, 'abot')
self.answererModelParams = loadedParams
# -- changing savepath in checkpoints
self.questionerModelParams['model_name'] = 'im-hist-enc-dec-questioner'
self.answererModelParams['model_name'] = 'im-hist-enc-dec-answerer'
# -- Print Questioner and Answerer
print('Questioner', self.questionerModelParams['model_name'])
print('Answerer', self.answererModelParams['model_name'])
# -- Add flags for various configurations
if 'hist' in self.questionerModelParams['model_name']:
self.questionerModelParams['useHistory'] = True
if 'hist' in self.answererModelParams['model_name']:
self.answererModelParams['useHistory'] = True
if 'im' in self.answererModelParams['model_name']:
self.answererModelParams['useIm'] = True
# -- Setup both Qbot and Abot
print('Using models from', self.questionerModelParams['model_name'])
print('Using models from', self.answererModelParams['model_name'])
self.questionerModel.to(self.device)
self.answererModel.to(self.device)
self.questionerModel.eval()
self.answererModel.eval()
# load pre-trained VGG 19 (used to extract image features)
print("Loading image feature extraction model")
self.feat_extract_model = torchvision.models.vgg19(pretrained=True)
self.feat_extract_model.classifier = nn.Sequential(*list(self.feat_extract_model.classifier.children())[:-3])
self.feat_extract_model.eval()
self.feat_extract_model.to(self.device)
