def loadArbitraryFeatures(params, idxes,auxidxes = []):
feat_all = []
aux_all = []
if params.get('multi_model',0) == 0:
params['nmodels'] = 1
for i in xrange(params['nmodels']):
#----------------------------- Loop -------------------------------------#
features_path = params['feat_file'][i] if params.get('multi_model',0) else params['feat_file']
if features_path.rsplit('.',1)[1] == 'mat':
features_struct = scipy.io.loadmat(features_path)
features = features_struct['feats'][:,idxes].astype(theano.config.floatX) # this is a 4096 x N numpy array of features
elif features_path.rsplit('.',1)[1] == 'hdf5':
#If the file is one of Vik's HDF5 Files
features_struct = h5py.File(features_path,'r')
features = features_struct['feats'][idxes,:].astype(theano.config.floatX) # this is a N x 2032128 array of features
elif features_path.rsplit('.',1)[1] == 'bin':
features_struct = picsom_bin_data(features_path)
features = np.array(features_struct.get_float_list(idxes)).T.astype(theano.config.floatX) # this is a 4096 x N numpy array of features
print "Working on Bin file now"
elif features_path.rsplit('.',1)[1] == 'txt':
#This is for feature concatenation.
# NOTE: Assuming same order of features in all the files listed in the txt file
feat_Flist = open(features_path, 'r').read().splitlines()
feat_list = []
for f in feat_Flist:
f_struct = picsom_bin_data(f)
feat_list.append(np.array(f_struct.get_float_list(idxes)).T)
print feat_list[-1].shape
# this is a 4096 x N numpy array of features
features = np.concatenate(feat_list, axis=0).astype(theano.config.floatX)
print "Combined all the features. Final size is %d %d"%(features.shape[0],features.shape[1])
aux_inp = []
aux_inp_file = params['aux_inp_file'][i] if params.get('multi_model',0) else params.get('aux_inp_file',None)
if aux_inp_file != None:
# Load Auxillary input file, one vec per image
# NOTE: Assuming same order as feature file
auxidxes = idxes if auxidxes == [] else auxidxes
if aux_inp_file.rsplit('.',1)[1] == 'bin':
f_struct = picsom_bin_data(aux_inp_file)
aux_inp = np.array(f_struct.get_float_list(auxidxes)).T.astype(theano.config.floatX)
elif aux_inp_file.rsplit('.',1)[1] == 'txt':
feat_Flist = open(aux_inp_file, 'r').read().splitlines()
feat_list = []
for f in feat_Flist:
f_struct = picsom_bin_data(f)
feat_list.append(np.array(f_struct.get_float_list(idxes)).T.astype(theano.config.floatX))
print feat_list[-1].shape
# this is a 4096 x N numpy array of features
aux_inp = np.concatenate(feat_list, axis=0)
feat_all.append(features)
aux_all.append(aux_inp)
if params.get('multi_model',0) == 0:
return features, aux_inp
else:
return feat_all, aux_all
def loadArbitraryFeatures(params, idxes):
feat_all = []
aux_all = []
if params.get('multi_model', 0) == 0:
params['nmodels'] = 1
for i in xrange(params['nmodels']):
#----------------------------- Loop -------------------------------------#
features_path = params['feat_file'][i] if params.get(
'multi_model', 0) else params['feat_file']
if features_path.rsplit('.', 1)[1] == 'mat':
features_struct = scipy.io.loadmat(features_path)
features = features_struct['feats'][:, idxes].astype(
theano.config.floatX
) # this is a 4096 x N numpy array of features
elif features_path.rsplit('.', 1)[1] == 'hdf5':
#If the file is one of Vik's HDF5 Files
features_struct = h5py.File(features_path, 'r')
features = features_struct['feats'][idxes, :].astype(
theano.config.floatX
) # this is a N x 2032128 array of features
elif features_path.rsplit('.', 1)[1] == 'bin':
features_struct = picsom_bin_data(features_path)
features = np.array(
features_struct.get_float_list(idxes)).T.astype(
theano.config.floatX
) # this is a 4096 x N numpy array of features
print "Working on Bin file now"
elif features_path.rsplit('.', 1)[1] == 'txt':
#This is for feature concatenation.
# NOTE: Assuming same order of features in all the files listed in the txt file
feat_Flist = open(features_path, 'r').read().splitlines()
feat_list = []
for f in feat_Flist:
f_struct = picsom_bin_data(f)
feat_list.append(np.array(f_struct.get_float_list(idxes)).T)
print feat_list[-1].shape
# this is a 4096 x N numpy array of features
features = np.concatenate(feat_list,
axis=0).astype(theano.config.floatX)
print "Combined all the features. Final size is %d %d" % (
features.shape[0], features.shape[1])
aux_inp = []
aux_inp_file = params['aux_inp_file'][i] if params.get(
'multi_model', 0) else params['aux_inp_file']
if aux_inp_file != None:
# Load Auxillary input file, one vec per image
# NOTE: Assuming same order as feature file
f_struct = picsom_bin_data(aux_inp_file)
aux_inp = np.array(f_struct.get_float_list(idxes)).T.astype(
theano.config.floatX)
feat_all.append(features)
aux_all.append(aux_inp)
if params.get('multi_model', 0) == 0:
return features, aux_inp
else:
return feat_all, aux_all
def __init__(self, params):
dataset = params.get('dataset', 'coco')
feature_file = params.get('feature_file', 'vgg_feats.mat')
data_file = params.get('data_file', 'dataset.json')
mat_new_ver = params.get('mat_new_ver', -1)
print 'Initializing data provider for dataset %s...' % (dataset, )
self.hdf5Flag = 0 #Flag indicating whether the dataset is an HDF5 File.
#Large HDF5 files are stored (by Vik) as one image
# per row, going against the conventions of the other
# storage formats
# !assumptions on folder structure
self.dataset_root = os.path.join('data', dataset)
self.image_root = os.path.join('data', dataset, 'imgs')
# load the dataset into memory
dataset_path = os.path.join(self.dataset_root, data_file)
print 'BasicDataProvider: reading %s' % (dataset_path, )
self.dataset = json.load(open(dataset_path, 'r'))
# load the image features into memory
features_path = os.path.join(self.dataset_root, feature_file)
print 'BasicDataProvider: reading %s' % (features_path, )
if feature_file.rsplit('.', 1)[1] == 'mat':
if mat_new_ver == 1:
features_struct = h5py.File(features_path)
self.features = np.array(
features_struct[features_struct.keys()[0]],
dtype=theano.config.floatX)
else:
features_struct = scipy.io.loadmat(features_path)
self.features = features_struct['feats']
#The condition below makes consuming HDF5 features easy
# This is what I (Vik) use for features extracted in an unsupervised
# manner
elif feature_file.rsplit('.', 1)[1] == 'hdf5':
self.hdf5Flag = 1
features_struct = h5py.File(features_path)
self.features = features_struct[
'features'] #The dataset in the HDF5 file is named 'features'
elif feature_file.rsplit('.', 1)[1] == 'bin':
features_struct = picsom_bin_data(features_path)
self.features = np.array(
features_struct.get_float_list(-1)).T.astype(
theano.config.floatX)
# this is a 4096 x N numpy array of features
print "Working on Bin file now"
elif feature_file.rsplit('.', 1)[1] == 'txt':
#This is for feature concatenation.
# NOTE: Assuming same order of features in all the files listed in the txt file
feat_Flist = open(features_path, 'r').read().splitlines()
feat_list = []
for f in feat_Flist:
f_struct = picsom_bin_data(os.path.join(self.dataset_root, f))
feat_list.append(
np.array(f_struct.get_float_list(-1)).T.astype(
theano.config.floatX))
print feat_list[-1].shape
# this is a 4096 x N numpy array of features
self.features = np.concatenate(feat_list, axis=0)
print "Combined all the features. Final size is %d %d" % (
self.features.shape[0], self.features.shape[1])
if self.hdf5Flag == 1:
#Because the HDF5 file is currently stored as one feature per row
self.img_feat_size = self.features.shape[1]
else:
self.img_feat_size = self.features.shape[0]
self.aux_pres = 0
self.aux_inp_size = 0
if params.get('en_aux_inp', 0):
# Load Auxillary input file, one vec per image
# NOTE: Assuming same order as feature file
f_struct = picsom_bin_data(
os.path.join(self.dataset_root, params['aux_inp_file']))
self.aux_inputs = np.array(f_struct.get_float_list(-1)).T.astype(
theano.config.floatX)
self.aux_pres = 1
self.aux_inp_size = self.aux_inputs.shape[0]
# group images by their train/val/test split into a dictionary -> list structure
self.split = defaultdict(list)
for img in self.dataset['images']:
self.split[img['split']].append(img)
if img['split'] != 'train':
self.split['allval'].append(img)
# Build tables for length based sampling
lenHist = defaultdict(int)
self.lenMap = defaultdict(list)
if (dataset == 'coco'):
self.min_len = 7
self.max_len = 27
for iid, img in enumerate(self.split['train']):
for sid, sent in enumerate(img['sentences']):
ix = max(min(len(sent['tokens']), self.max_len),
self.min_len)
lenHist[ix] += 1
self.lenMap[ix].append((iid, sid))
else:
raise ValueError(
'ERROR: Dont know how to do len splitting for this dataset')
self.lenCdist = np.cumsum(lenHist.values())
def __init__(self, params):
dataset = params.get('dataset', 'coco')
feature_file = params.get('feature_file', 'vgg_feats.mat')
data_file = params.get('data_file', 'dataset.json')
mat_new_ver = params.get('mat_new_ver', -1)
print 'Initializing data provider for dataset %s...' % (dataset, )
self.hdf5Flag = 0 #Flag indicating whether the dataset is an HDF5 File.
#Large HDF5 files are stored (by Vik) as one image
# per row, going against the conventions of the other
# storage formats
# !assumptions on folder structure
self.dataset_root = os.path.join('data', dataset)
self.image_root = os.path.join('data', dataset, 'imgs')
# load the dataset into memory
dataset_path = os.path.join(self.dataset_root, data_file)
print 'BasicDataProvider: reading %s' % (dataset_path, )
self.dataset = json.load(open(dataset_path, 'r'))
# load the image features into memory
features_path = os.path.join(self.dataset_root, feature_file)
print 'BasicDataProvider: reading %s' % (features_path, )
if feature_file.rsplit('.',1)[1] == 'mat':
if mat_new_ver == 1:
features_struct = h5py.File(features_path)
self.features = np.array(features_struct[features_struct.keys()[0]],dtype=theano.config.floatX)
else:
features_struct = scipy.io.loadmat(features_path)
self.features = features_struct['feats']
#The condition below makes consuming HDF5 features easy
# This is what I (Vik) use for features extracted in an unsupervised
# manner
elif feature_file.rsplit('.',1)[1] == 'hdf5':
self.hdf5Flag = 1
features_struct = h5py.File(features_path)
self.features = features_struct['features'] #The dataset in the HDF5 file is named 'features'
elif feature_file.rsplit('.',1)[1] == 'bin':
features_struct = picsom_bin_data(features_path)
self.features = np.array(features_struct.get_float_list(-1)).T.astype(theano.config.floatX)
# this is a 4096 x N numpy array of features
print "Working on Bin file now"
elif feature_file.rsplit('.',1)[1] == 'txt':
#This is for feature concatenation.
# NOTE: Assuming same order of features in all the files listed in the txt file
feat_Flist = open(features_path, 'r').read().splitlines()
feat_list = []
for f in feat_Flist:
f_struct = picsom_bin_data(os.path.join(self.dataset_root,f))
feat_list.append(np.array(f_struct.get_float_list(-1)).T.astype(theano.config.floatX))
print feat_list[-1].shape
# this is a 4096 x N numpy array of features
self.features = np.concatenate(feat_list, axis=0)
print "Combined all the features. Final size is %d %d"%(self.features.shape[0],self.features.shape[1])
if self.hdf5Flag == 1:
#Because the HDF5 file is currently stored as one feature per row
self.img_feat_size = self.features.shape[1]
else:
self.img_feat_size = self.features.shape[0]
self.aux_pres = 0
self.aux_inp_size = 0
if params.get('en_aux_inp',0):
# Load Auxillary input file, one vec per image
# NOTE: Assuming same order as feature file
f_struct = picsom_bin_data(os.path.join(self.dataset_root,params['aux_inp_file']))
self.aux_inputs = np.array(f_struct.get_float_list(-1)).T.astype(theano.config.floatX)
self.aux_pres = 1
self.aux_inp_size = self.aux_inputs.shape[0]
# group images by their train/val/test split into a dictionary -> list structure
self.split = defaultdict(list)
for img in self.dataset['images']:
self.split[img['split']].append(img)
if img['split'] != 'train':
self.split['allval'].append(img)
# Build tables for length based sampling
lenHist = defaultdict(int)
self.lenMap = defaultdict(list)
if(dataset == 'coco'):
self.min_len = 7
self.max_len = 27
for iid, img in enumerate(self.split['train']):
for sid, sent in enumerate(img['sentences']):
ix = max(min(len(sent['tokens']),self.max_len),self.min_len)
lenHist[ix] += 1
self.lenMap[ix].append((iid,sid))
else:
raise ValueError('ERROR: Dont know how to do len splitting for this dataset')
self.lenCdist = np.cumsum(lenHist.values())
