def get_image_feat(feat_type, image_folder, orig_split, indices, real_split): feats = defaultdict(int) prefix = 'abstract_v002_%s2015_' % (orig_split) if 'fc7' in feat_type: # set some parameters folder = os.path.join(image_folder, 'scene_img', 'img_%s2015' % (orig_split)) + '/' print "Preparing the VGG 19 Net" net = demo.build_convnet() print "Extracting Features" with open('temp_{}.txt'.format(orig_split), 'w') as image_file: for item in tqdm(indices): image_file.write(imname(prefix, item) + '\n') image_file.close() feats = demo.compute_fromfile(net, 'temp_{}.txt'.format(orig_split), base_path=folder) elif 'hdf5' in feat_type: try: folder = os.path.join(image_folder, 'scene_img', 'img_%s2015' % (orig_split)) + '/' images = np.zeros((len(indices), 3, 224, 224)) # TODO: Low Priority, make general for index, item in tqdm(enumerate(indices)): images[index] = demo.load_abstract_image(folder + imname(prefix, item)) with h5py.File('/ssd_local/rama/datasets/abstract-hdf5/{}.h5'.format(real_split), 'w') as outfile: outfile['images'] = images return True except: print "problem" return False else: folder = os.path.join(image_folder, 'scene_json', 'scene_%s2015_indv' % (orig_split)) # create the abstract feature instance AF = pickle.load(open('extract_features/af_dump.p', 'r')) # TODO: Figure out a better place to initialize all this out_dir = '/srv/share/vqa/release_data/abstract_v002/scene_json/features_v2/' keep_or_remove = 'keep' get_names = False tags = feat_type # path to metafeature directory metafeat_dir = af.dir_path(os.path.join(out_dir, 'metafeatures')) for item in tqdm(indices): metafeat_fn = '{}_instances-{}.cpickle'.format(item, AF.instance_ordering) cur_metafeat_fn = os.path.join(metafeat_dir, metafeat_fn) with open(cur_metafeat_fn, 'rb') as fp: cur_metafeats = pickle.load(fp) cur_feats, _ = AF.scene_metafeatures_to_features(cur_metafeats, tags, keep_or_remove, get_names) feats[item] = cur_feats return feats
def __init__(self,model_path_dict): """Initialization requires embedding model path """ self.model_path = model_path_dict # compile image feature extractor self.vggnet = demo.build_convnet() self._get_image_features = theano.function( inputs = [self.vggnet['input'].input_var], outputs = L.get_output(self.vggnet['fc7'],deterministic=True), allow_input_downcast = True ) # load up pretrained VSEM model self.model = tools.load_model( path_to_model=self.model_path['vse_model'] )
# class MyList(list): # def __init__(self, *args): # super(MyList, self).__init__(args) # # def __sub__(self, other): # return self.__class__(*[item for item in self if item not in other]) # read caption f = open('43_captions', 'rb') text_list = array([x.strip() for x in f.readlines()]) print 'read caption' # read images net = demo.build_convnet() img_list = demo.compute_fromfile(net, './43_img_list', '/home/czq/visual-semantic-embedding/43_img/') # select indexes which is not empty string non_empty_indexes = [] for k, v in enumerate(text_list): if v.strip() != '': non_empty_indexes.append(k) non_empty_indexes = sorted(non_empty_indexes) # filter by non-empty string non_empty_img_list = img_list[non_empty_indexes, :] non_empty_text_list = text_list[non_empty_indexes] # sampling to make train/dev/test to 8/1/1
# -*- coding: utf-8 -*- """ Created on Sun Mar 19 09:22:17 2017 @author: chahak """ import demo, tools, datasets net = demo.build_convnet() model = tools.load_model() train = datasets.load_dataset('f8k', load_train=True)[0] vectors = tools.encode_sentences(model, train[0], verbose=False) demo.retrieve_captions(model, net, train[0], vectors, 'child.jpg', k=5)