def get_image_feat(feat_type, image_folder, orig_split, indices, real_split):
feats = defaultdict(int)
prefix = 'abstract_v002_%s2015_' % (orig_split)
if 'fc7' in feat_type:
# set some parameters
folder = os.path.join(image_folder, 'scene_img', 'img_%s2015' % (orig_split)) + '/'
print "Preparing the VGG 19 Net"
net = demo.build_convnet()
print "Extracting Features"
with open('temp_{}.txt'.format(orig_split), 'w') as image_file:
for item in tqdm(indices):
image_file.write(imname(prefix, item) + '\n')
image_file.close()
feats = demo.compute_fromfile(net, 'temp_{}.txt'.format(orig_split),
base_path=folder)
elif 'hdf5' in feat_type:
try:
folder = os.path.join(image_folder, 'scene_img', 'img_%s2015' % (orig_split)) + '/'
images = np.zeros((len(indices), 3, 224, 224)) # TODO: Low Priority, make general
for index, item in tqdm(enumerate(indices)):
images[index] = demo.load_abstract_image(folder + imname(prefix, item))
with h5py.File('/ssd_local/rama/datasets/abstract-hdf5/{}.h5'.format(real_split), 'w') as outfile:
outfile['images'] = images
return True
except:
print "problem"
return False
else:
folder = os.path.join(image_folder, 'scene_json', 'scene_%s2015_indv' % (orig_split))
# create the abstract feature instance
AF = pickle.load(open('extract_features/af_dump.p', 'r'))
# TODO: Figure out a better place to initialize all this
out_dir = '/srv/share/vqa/release_data/abstract_v002/scene_json/features_v2/'
keep_or_remove = 'keep'
get_names = False
tags = feat_type
# path to metafeature directory
metafeat_dir = af.dir_path(os.path.join(out_dir, 'metafeatures'))
for item in tqdm(indices):
metafeat_fn = '{}_instances-{}.cpickle'.format(item,
AF.instance_ordering)
cur_metafeat_fn = os.path.join(metafeat_dir,
metafeat_fn)
with open(cur_metafeat_fn, 'rb') as fp:
cur_metafeats = pickle.load(fp)
cur_feats, _ = AF.scene_metafeatures_to_features(cur_metafeats,
tags,
keep_or_remove,
get_names)
feats[item] = cur_feats
return feats
def __init__(self,model_path_dict):
"""Initialization requires embedding model path
"""
self.model_path = model_path_dict
# compile image feature extractor
self.vggnet = demo.build_convnet()
self._get_image_features = theano.function(
inputs = [self.vggnet['input'].input_var],
outputs = L.get_output(self.vggnet['fc7'],deterministic=True),
allow_input_downcast = True
)
# load up pretrained VSEM model
self.model = tools.load_model(
path_to_model=self.model_path['vse_model']
)
# class MyList(list):
# def __init__(self, *args):
# super(MyList, self).__init__(args)
#
# def __sub__(self, other):
# return self.__class__(*[item for item in self if item not in other])
# read caption
f = open('43_captions', 'rb')
text_list = array([x.strip() for x in f.readlines()])
print 'read caption'
# read images
net = demo.build_convnet()
img_list = demo.compute_fromfile(net, './43_img_list', '/home/czq/visual-semantic-embedding/43_img/')
# select indexes which is not empty string
non_empty_indexes = []
for k, v in enumerate(text_list):
if v.strip() != '':
non_empty_indexes.append(k)
non_empty_indexes = sorted(non_empty_indexes)
# filter by non-empty string
non_empty_img_list = img_list[non_empty_indexes, :]
non_empty_text_list = text_list[non_empty_indexes]
# sampling to make train/dev/test to 8/1/1
# -*- coding: utf-8 -*-
"""
Created on Sun Mar 19 09:22:17 2017
@author: chahak
"""
import demo, tools, datasets
net = demo.build_convnet()
model = tools.load_model()
train = datasets.load_dataset('f8k', load_train=True)[0]
vectors = tools.encode_sentences(model, train[0], verbose=False)
demo.retrieve_captions(model, net, train[0], vectors, 'child.jpg', k=5)