def main(params): dataDir = params['dataDir'] vqaDir = params['vqaDir'] taskType = 'OpenEnded' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) dataDir = params['dataDir'] + "/" vqaDir = params['vqaDir'] + "/" vqa = VQA(annFile, quesFile) data = [] annIds = list(set(vqa.getImgIds()))[:params['num']] for id in annIds: imgFilename = 'mscoco/train2014/COCO_' + dataSubType + '_' + str( id).zfill(12) + '.jpg' copyfile(dataDir + imgFilename, vqaDir + imgFilename) caption = [] caption2 = [] for i in vqa.imgToQA[id]: caption.append(vqa.qqa[i['question_id']]['question']) caption.append(vqa.qqap[i]['question_id']['choices']) data.append({"file_path": imgFilename, "captions": caption}) with open('data.json', 'w') as outfile: json.dump(data, outfile)
def main(params): dataDir=params['dataDir'] vqaDir=params['vqaDir'] taskType='OpenEnded' dataType='mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType='train2014' annFile='%s/Annotations/%s_%s_annotations.json'%(dataDir, dataType, dataSubType) quesFile='%s/Questions/%s_%s_%s_questions.json'%(dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType) dataDir=params['dataDir']+"/" vqaDir=params['vqaDir']+"/" vqa=VQA(annFile, quesFile) data=[] annIds = list(set(vqa.getImgIds()))[:params['num']] for id in annIds: imgFilename = 'mscoco/train2014/COCO_' + dataSubType + '_'+ str(id).zfill(12) + '.jpg' copyfile(dataDir+imgFilename, vqaDir+imgFilename) caption=[] caption2=[] for i in vqa.imgToQA[id]: caption.append(vqa.qqa[i['question_id']]['question']) caption.append(vqa.qqap[i]['question_id']['choices']) data.append({"file_path":imgFilename,"captions":caption}) with open('data.json','w') as outfile: json.dump(data,outfile)
def main(params): dataDir=params['dataDir'] vqaDir=params['vqaDir'] copyfileBool= True if params['copyfile']=='True' else False taskType='MultipleChoice' dataType='mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType='train2014' #folder in which it is contained annFile='%s/Annotations/%s_%s_annotations.json'%(dataDir, dataType, dataSubType) quesFile='%s/Questions/%s_%s_%s_questions.json'%(dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' %(dataDir, dataType, dataSubType) dataDir=params['dataDir']+"/" vqaDir=params['vqaDir']+"/" vqa=VQA(annFile, quesFile) data=[] data2=[] annIds = list(set(vqa.getImgIds()))[:params['num']] for id in annIds: imgFilename = 'mscoco/train2014/COCO_' + dataSubType + '_'+ str(id).zfill(12) + '.jpg' if copyfileBool: copyfile(dataDir+imgFilename, vqaDir+imgFilename) caption=[] options=[] for i in vqa.imgToQA[id]: caption.append(vqa.qqa[i['question_id']]['question']) choices= ', '.join(vqa.qqa[i['question_id']]['multiple_choices']) options.append(choices) data.append({"file_path":imgFilename,"captions":caption}) data2.append({"file_path":imgFilename,"captions":options}) with open('data.json','w') as outfile: json.dump(data,outfile) with open('data2.json','w') as outfile: json.dump(data2,outfile)
what color what kind what are what type is the is this how many are does where is there why which do what does what time who what sport what animal what brand """ ##### Load VQA dataset print('Enter the quesTypes (\'what color\', \'is this\', ..., \'all\')') # quesTypes = input() quesTypes = 'what is' if quesTypes == 'all': annIdsA = vqa.getQuesIds() tannIdsA = tvqa.getQuesIds() imgIdsA = vqa.getImgIds() timgIdsA = tvqa.getImgIds() else: annIdsA = vqa.getQuesIds(quesTypes=quesTypes) tannIdsA = tvqa.getQuesIds(quesTypes=quesTypes) imgIdsA = vqa.getImgIds(quesTypes=quesTypes) timgIdsA = tvqa.getImgIds(quesTypes=quesTypes) annsA = vqa.loadQA(annIdsA) tannsA = tvqa.loadQA(tannIdsA) if len(annsA) > TR_LIMIT_SIZE: annsA[TR_LIMIT_SIZE:] = [] imgIdsA[TR_LIMIT_SIZE:] = [] if len(tannsA) > VAL_LIMIT_SIZE:
""" annIds = vqa.getQuesIds(ansTypes='yes/no') anns = vqa.loadQA(annIds) randomAnn = random.choice(anns) vqa.showQA([randomAnn]) imgId = randomAnn['image_id'] imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' if os.path.isfile(imgDir + imgFilename): I = io.imread(imgDir + imgFilename) plt.imshow(I) plt.axis('off') plt.show() # load and display QA annotations for given images """ Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[]) Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types. """ ids = vqa.getImgIds() annIds = vqa.getQuesIds(imgIds=random.sample(ids, 5)) anns = vqa.loadQA(annIds) randomAnn = random.choice(anns) vqa.showQA([randomAnn]) imgId = randomAnn['image_id'] imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' if os.path.isfile(imgDir + imgFilename): I = io.imread(imgDir + imgFilename) plt.imshow(I) plt.axis('off') plt.show()
qtype = ['what color','what is on the','what sport is'] # path to images data_dir = '/srv/share/data/mscoco/coco/images/train2014/' model = '/home/ashwin/caffe/models/bvlc_reference_caffenet/bvlc_reference_caffenet.caffemodel' prototxt = '/home/ashwin/caffe/models/bvlc_reference_caffenet/deploy.prototxt' # load QAs vqa = VQA(annFile, quesFile) # add question type annIds = [] anns = [] ids = [] for qitem in qtype: annIds = vqa.getQuesIds(quesTypes= qtype) anns.extend(vqa.loadQA(annIds)) ids.extend(vqa.getImgIds(quesTypes = qtype)) UIDs = list(np.unique(np.array(ids))) # extract fc7 features caffe.set_mode_gpu() caffe.set_device(1) net = caffe.Net(prototxt,model,caffe.TEST) transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape}) transformer.set_transpose('data', (2,0,1)) transformer.set_mean('data', np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy').mean(1).mean(1)) # mean pixel transformer.set_raw_scale('data', 255) # the reference model operates on images in [0,255] range instead of [0,1] transformer.set_channel_swap('data', (2,1,0)) # the reference model has channels in BGR order instead of RGB net.blobs['data'].reshape(1,3,227,227) fc7_feat = []
what color what kind what are what type is the is this how many are does where is there why which do what does what time who what sport what animal what brand """ ##### Load VQA dataset print("Enter the quesTypes ('what color', 'is this', ..., 'all')") # quesTypes = input() # quesTypes = 'all' # defined above if quesTypes == "all": annIdsA = vqa.getQuesIds() tannIdsA = tvqa.getQuesIds() imgIdsA = vqa.getImgIds() timgIdsA = tvqa.getImgIds() else: annIdsA = vqa.getQuesIds(quesTypes=quesTypes) tannIdsA = tvqa.getQuesIds(quesTypes=quesTypes) imgIdsA = vqa.getImgIds(quesTypes=quesTypes) timgIdsA = tvqa.getImgIds(quesTypes=quesTypes) annsA = vqa.loadQA(annIdsA) tannsA = tvqa.loadQA(tannIdsA) if len(annsA) > TR_LIMIT_SIZE: annsA[TR_LIMIT_SIZE:] = [] imgIdsA[TR_LIMIT_SIZE:] = [] if len(tannsA) > VAL_LIMIT_SIZE:
""" annIds = vqa.getQuesIds(ansTypes="yes/no") anns = vqa.loadQA(annIds) randomAnn = random.choice(anns) vqa.showQA([randomAnn]) imgId = randomAnn["image_id"] imgFilename = "COCO_" + dataSubType + "_" + str(imgId).zfill(12) + ".jpg" if os.path.isfile(imgDir + imgFilename): I = io.imread(imgDir + imgFilename) plt.imshow(I) plt.axis("off") plt.show() # load and display QA annotations for given images """ Usage: vqa.getImgIds(quesIds=[], quesTypes=[], ansTypes=[]) Above method can be used to retrieve imageIds for given question Ids or given question types or given answer types. """ ids = vqa.getImgIds() annIds = vqa.getQuesIds(imgIds=random.sample(ids, 5)) anns = vqa.loadQA(annIds) randomAnn = random.choice(anns) vqa.showQA([randomAnn]) imgId = randomAnn["image_id"] imgFilename = "COCO_" + dataSubType + "_" + str(imgId).zfill(12) + ".jpg" if os.path.isfile(imgDir + imgFilename): I = io.imread(imgDir + imgFilename) plt.imshow(I) plt.axis("off") plt.show()