def main(params):
dataDir = params['dataDir']
vqaDir = params['vqaDir']
taskType = 'OpenEnded'
dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType = 'train2014'
annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType,
dataSubType)
quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType,
dataType, dataSubType)
imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType)
dataDir = params['dataDir'] + "/"
vqaDir = params['vqaDir'] + "/"
vqa = VQA(annFile, quesFile)
data = []
annIds = list(set(vqa.getImgIds()))[:params['num']]
for id in annIds:
imgFilename = 'mscoco/train2014/COCO_' + dataSubType + '_' + str(
id).zfill(12) + '.jpg'
copyfile(dataDir + imgFilename, vqaDir + imgFilename)
caption = []
caption2 = []
for i in vqa.imgToQA[id]:
caption.append(vqa.qqa[i['question_id']]['question'])
caption.append(vqa.qqap[i]['question_id']['choices'])
data.append({"file_path": imgFilename, "captions": caption})
with open('data.json', 'w') as outfile:
json.dump(data, outfile)
def main(params):
dataDir=params['dataDir']
vqaDir=params['vqaDir']
taskType='OpenEnded'
dataType='mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType='train2014'
annFile='%s/Annotations/%s_%s_annotations.json'%(dataDir, dataType, dataSubType)
quesFile='%s/Questions/%s_%s_%s_questions.json'%(dataDir, taskType, dataType, dataSubType)
imgDir = '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType)
dataDir=params['dataDir']+"/"
vqaDir=params['vqaDir']+"/"
vqa=VQA(annFile, quesFile)
data=[]
annIds = list(set(vqa.getImgIds()))[:params['num']]
for id in annIds:
imgFilename = 'mscoco/train2014/COCO_' + dataSubType + '_'+ str(id).zfill(12) + '.jpg'
copyfile(dataDir+imgFilename, vqaDir+imgFilename)
caption=[]
caption2=[]
for i in vqa.imgToQA[id]:
caption.append(vqa.qqa[i['question_id']]['question'])
caption.append(vqa.qqap[i]['question_id']['choices'])
data.append({"file_path":imgFilename,"captions":caption})
with open('data.json','w') as outfile:
json.dump(data,outfile)
def main(params):
dataDir=params['dataDir']
vqaDir=params['vqaDir']
copyfileBool= True if params['copyfile']=='True' else False
taskType='MultipleChoice'
dataType='mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType='train2014' #folder in which it is contained
annFile='%s/Annotations/%s_%s_annotations.json'%(dataDir, dataType, dataSubType)
quesFile='%s/Questions/%s_%s_%s_questions.json'%(dataDir, taskType, dataType, dataSubType)
imgDir = '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType)
dataDir=params['dataDir']+"/"
vqaDir=params['vqaDir']+"/"
vqa=VQA(annFile, quesFile)
data=[]
data2=[]
annIds = list(set(vqa.getImgIds()))[:params['num']]
for id in annIds:
imgFilename = 'mscoco/train2014/COCO_' + dataSubType + '_'+ str(id).zfill(12) + '.jpg'
if copyfileBool:
copyfile(dataDir+imgFilename, vqaDir+imgFilename)
caption=[]
options=[]
for i in vqa.imgToQA[id]:
caption.append(vqa.qqa[i['question_id']]['question'])
choices= ', '.join(vqa.qqa[i['question_id']]['multiple_choices'])
options.append(choices)
data.append({"file_path":imgFilename,"captions":caption})
data2.append({"file_path":imgFilename,"captions":options})
with open('data.json','w') as outfile:
json.dump(data,outfile)
with open('data2.json','w') as outfile:
json.dump(data2,outfile)
what color what kind what are what type is the
is this how many are does where
is there why which do what does
what time who what sport what animal what brand
"""
##### Load VQA dataset
print('Enter the quesTypes (\'what color\', \'is this\', ..., \'all\')')
# quesTypes = input()
quesTypes = 'what is'
if quesTypes == 'all':
annIdsA = vqa.getQuesIds()
tannIdsA = tvqa.getQuesIds()
imgIdsA = vqa.getImgIds()
timgIdsA = tvqa.getImgIds()
else:
annIdsA = vqa.getQuesIds(quesTypes=quesTypes)
tannIdsA = tvqa.getQuesIds(quesTypes=quesTypes)
imgIdsA = vqa.getImgIds(quesTypes=quesTypes)
timgIdsA = tvqa.getImgIds(quesTypes=quesTypes)
annsA = vqa.loadQA(annIdsA)
tannsA = tvqa.loadQA(tannIdsA)
if len(annsA) > TR_LIMIT_SIZE:
annsA[TR_LIMIT_SIZE:] = []
imgIdsA[TR_LIMIT_SIZE:] = []
if len(tannsA) > VAL_LIMIT_SIZE:
"""
annIds = vqa.getQuesIds(ansTypes='yes/no')
anns = vqa.loadQA(annIds)
randomAnn = random.choice(anns)
vqa.showQA([randomAnn])
imgId = randomAnn['image_id']
imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg'
if os.path.isfile(imgDir + imgFilename):
I = io.imread(imgDir + imgFilename)
plt.imshow(I)
plt.axis('off')
plt.show()
# load and display QA annotations for given images
"""
Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[])
Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types.
"""
ids = vqa.getImgIds()
annIds = vqa.getQuesIds(imgIds=random.sample(ids, 5))
anns = vqa.loadQA(annIds)
randomAnn = random.choice(anns)
vqa.showQA([randomAnn])
imgId = randomAnn['image_id']
imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg'
if os.path.isfile(imgDir + imgFilename):
I = io.imread(imgDir + imgFilename)
plt.imshow(I)
plt.axis('off')
plt.show()
qtype = ['what color','what is on the','what sport is']
# path to images
data_dir = '/srv/share/data/mscoco/coco/images/train2014/'
model = '/home/ashwin/caffe/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel'
prototxt = '/home/ashwin/caffe/models/bvlc_reference_caffenet/deploy.prototxt'
# load QAs
vqa = VQA(annFile, quesFile)
# add question type
annIds = []
anns = []
ids = []
for qitem in qtype:
annIds = vqa.getQuesIds(quesTypes= qtype)
anns.extend(vqa.loadQA(annIds))
ids.extend(vqa.getImgIds(quesTypes = qtype))
UIDs = list(np.unique(np.array(ids)))
# extract fc7 features
caffe.set_mode_gpu()
caffe.set_device(1)
net = caffe.Net(prototxt,model,caffe.TEST)
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2,0,1))
transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel
transformer.set_raw_scale('data', 255) # the reference model operates on images in [0,255] range instead of [0,1]
transformer.set_channel_swap('data', (2,1,0)) # the reference model has channels in BGR order instead of RGB
net.blobs['data'].reshape(1,3,227,227)
fc7_feat = []
what color what kind what are what type is the
is this how many are does where
is there why which do what does
what time who what sport what animal what brand
"""
##### Load VQA dataset
print("Enter the quesTypes ('what color', 'is this', ..., 'all')")
# quesTypes = input()
# quesTypes = 'all' # defined above
if quesTypes == "all":
annIdsA = vqa.getQuesIds()
tannIdsA = tvqa.getQuesIds()
imgIdsA = vqa.getImgIds()
timgIdsA = tvqa.getImgIds()
else:
annIdsA = vqa.getQuesIds(quesTypes=quesTypes)
tannIdsA = tvqa.getQuesIds(quesTypes=quesTypes)
imgIdsA = vqa.getImgIds(quesTypes=quesTypes)
timgIdsA = tvqa.getImgIds(quesTypes=quesTypes)
annsA = vqa.loadQA(annIdsA)
tannsA = tvqa.loadQA(tannIdsA)
if len(annsA) > TR_LIMIT_SIZE:
annsA[TR_LIMIT_SIZE:] = []
imgIdsA[TR_LIMIT_SIZE:] = []
if len(tannsA) > VAL_LIMIT_SIZE:
"""
annIds = vqa.getQuesIds(ansTypes="yes/no")
anns = vqa.loadQA(annIds)
randomAnn = random.choice(anns)
vqa.showQA([randomAnn])
imgId = randomAnn["image_id"]
imgFilename = "COCO_" + dataSubType + "_" + str(imgId).zfill(12) + ".jpg"
if os.path.isfile(imgDir + imgFilename):
I = io.imread(imgDir + imgFilename)
plt.imshow(I)
plt.axis("off")
plt.show()
# load and display QA annotations for given images
"""
Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[])
Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types.
"""
ids = vqa.getImgIds()
annIds = vqa.getQuesIds(imgIds=random.sample(ids, 5))
anns = vqa.loadQA(annIds)
randomAnn = random.choice(anns)
vqa.showQA([randomAnn])
imgId = randomAnn["image_id"]
imgFilename = "COCO_" + dataSubType + "_" + str(imgId).zfill(12) + ".jpg"
if os.path.isfile(imgDir + imgFilename):
I = io.imread(imgDir + imgFilename)
plt.imshow(I)
plt.axis("off")
plt.show()
