def vg_make_meta_hdf():
train_stats = zl.load('output/train_stats.pkl')
test_stats = zl.load('output/test_stats.pkl')
#train_stats = zl.sort_dict_by_val(train_stats)
obj_names_sorted = sorted(train_stats.keys())
classes = ('__background__', ) # always index 0
for i in xrange(len(obj_names_sorted)):
classes += (obj_names_sorted[i], )
train_pre_stats = zl.load('output/train_pre_stats.pkl')
test_pre_stats = zl.load('output/test_pre_stats.pkl')
pre_names_sorted = sorted(train_pre_stats.keys())
predicates = ()
for i in range(len(pre_names_sorted)):
predicates += (pre_names_sorted[i], )
h5f = 'data/vg1_2_meta.h5'
h5f = h5py.File(h5f)
for i in range(201):
h5f['meta/cls/name2idx/' + classes[i]] = str(i)
h5f['meta/cls/idx2name/' + str(i)] = classes[i]
for i in range(100):
h5f['meta/pre/name2idx/' + predicates[i]] = str(i)
h5f['meta/pre/idx2name/' + str(i)] = predicates[i]
def test_sparse_coder():
db = coco.COCO('/media/zawlin/ssd/coco/annotations/captions_train2014.json')
ilsvrc_word2vec = zl.load('ilsvrc_word2vec')
test_word2vec = zl.load('test_word2vec')
D = []
idx2word = []
for i in xrange(len(ilsvrc_word2vec.keys())):
idx2word.append(ilsvrc_word2vec.keys()[i])
D.append(ilsvrc_word2vec[ilsvrc_word2vec.keys()[i]])
idx2word = np.array(idx2word)
D = np.array(D)
print 'loading word2vec'
model = gensim.models.Word2Vec.load_word2vec_format('./GoogleNews-vectors-negative300.bin', binary=True)
coder = SparseCoder(dictionary=D, transform_n_nonzero_coefs=5,
transform_alpha=None, transform_algorithm='lasso_cd')
#random.shuffle(db.anns)
for k in db.anns:
cap = db.anns[k]['caption']
splited = cap.split(' ')
for s in splited:
if s.lower() in model:
y = model[s.lower()]
x = coder.transform(y.reshape(1,-1))[0]
print '%s = %s'%(s.lower(),idx2word[np.argsort(x)[::-1]])
print x[np.argsort(x)[::-1]]
c = raw_input('press q to quit')
if c== 'q':
exit(0)
def setup(self, bottom, top):
"""Setup the RoIDataLayer."""
self._cur_idx = 0
self.gt_labels = {}
self.meta = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r')
self.cache = h5py.File('output/sg_vrd_cache.h5','r')
self.cache_mem = {}
if os.path.exists('output/cache/sg_vrd_gt.pkl'):
self.gt_labels = zl.load('output/cache/sg_vrd_gt.pkl')
glog.info('loaded gt data from cache')
else:
glog.info( 'Preloading gt')
zl.tic()
for k in self.meta['gt/train'].keys():
rlp_labels = self.meta['gt/train/%s/rlp_labels'%k][...]
sub_boxes = self.meta['gt/train/%s/sub_boxes'%k][...].astype(np.float)
obj_boxes = self.meta['gt/train/%s/obj_boxes'%k][...].astype(np.float)
if sub_boxes.shape[0]>0:
zeros = np.zeros((sub_boxes.shape[0],1), dtype=np.float)
# first index is always zero since we do one image by one image
sub_boxes = np.concatenate((zeros, sub_boxes),axis=1)
obj_boxes = np.concatenate((zeros, obj_boxes),axis=1)
self.gt_labels[k] = {}
self.gt_labels[k]['rlp_labels']=rlp_labels
self.gt_labels[k]['sub_boxes']=sub_boxes
self.gt_labels[k]['obj_boxes']=obj_boxes
glog.info('done preloading gt %f'%zl.toc())
zl.save('output/cache/sg_vrd_gt.pkl',self.gt_labels)
self.imids = []
for k in self.gt_labels.keys():
self.imids.append(k)
self.imidx =0
random.shuffle(self.imids)
# parse the layer parameter string, which must be valid YAML
layer_params = yaml.load(self.param_str)
self._num_classes = layer_params['num_classes']
self._name_to_top_map = {}
# data blob: holds a batch of N images, each with 3 channels
idx = 0
top[idx].reshape(cfg.TRAIN.IMS_PER_BATCH, 1024,
50,50)
self._name_to_top_map['conv_new_1'] = idx
idx += 1
top[idx].reshape(1, 5, 1, 1)
self._name_to_top_map['sub_boxes'] = idx
idx += 1
top[idx].reshape(1, 5, 1, 1)
self._name_to_top_map['obj_boxes'] = idx
idx += 1
# labels blob: R categorical labels in [0, ..., K] for K foreground
# classes plus background
top[idx].reshape(1, 1, 1, 1)
self._name_to_top_map['labels'] = idx
def vg_data_prep2():
objects_all = zl.load('output/objects_all.pkl')
objects = zl.load('output/objects.pkl')
subjects = zl.load('output/subjects.pkl')
predicate = zl.load('output/predicate.pkl')
objects_all_sorted = sorted(objects_all.items(),
key=operator.itemgetter(1),
reverse=True)
objects_sorted = sorted(objects.items(),
key=operator.itemgetter(1),
reverse=True)
subjects_sorted = sorted(subjects.items(),
key=operator.itemgetter(1),
reverse=True)
predicate_sorted = sorted(predicate.items(),
key=operator.itemgetter(1),
reverse=True)
wnl = WordNetLemmatizer()
spl = SpellingReplacer()
cnt = 0
print objects_all_sorted[:100]
def merge_pickled_files():
import os
h5f = h5py.File(C.coco_eb_h5_path, 'w')
cnt = 0
zl.tick()
for path, subdirs, files in os.walk(C.coco_eb_dir):
for name in files:
cnt += 1
if cnt % 1000 == 0:
print cnt, zl.tock()
zl.tick()
fpath = os.path.join(path, name)
fid = name.replace('.eb', '')
bbs = np.array(zl.load(fpath)).astype(np.float16)
h5f[fid] = bbs
def setup(self, bottom, top):
self._cur_idx = 0
self.vgg_data = {}
vgg_h5 = h5py.File(
"output/precalc/vg1_2_2016_predicate_exp_train.hdf5", 'r')
layer_params = yaml.load(self.param_str_)
self.imids = []
for k in vgg_h5.keys():
self.imids.append(k)
self.imidx = 0
if os.path.exists('output/cache/vg1_2_2016_pre_train_concat.pkl'):
self.vgg_data = zl.load(
'output/cache/vg1_2_2016_pre_train_concat.pkl')
print 'loaded train data from cache'
else:
print 'Preloading training data'
zl.tick()
for k in vgg_h5.keys():
sub_visual = vgg_h5[k]['sub_visual'][...]
obj_visual = vgg_h5[k]['obj_visual'][...]
pre_label = vgg_h5[k]['pre_label'][...]
self.vgg_data[k] = {}
self.vgg_data[k]['sub_visual'] = sub_visual
self.vgg_data[k]['obj_visual'] = obj_visual
self.vgg_data[k]['pre_label'] = pre_label
print 'done preloading training data %f' % zl.tock()
zl.save('output/cache/vg1_2_2016_pre_train_concat.pkl',
self.vgg_data)
vgg_h5.close()
self._batch_size = layer_params['batch_size']
self.train_data = []
self._name_to_top_map = {}
# data blob: holds a batch of N images, each with 3 channels
# top[0].reshape(self._batch_size, 4096 * 2 )
top[0].reshape(self._batch_size, 2 * 4096)
top[1].reshape(self._batch_size)
self._name_to_top_map['visual'] = 0
self._name_to_top_map['label'] = 1
def prep_train():
cnt = 0
root = '/media/zawlin/ssd/iccv2017/data/voc/gen_eb/train'
h5path = '/media/zawlin/ssd/iccv2017/data/voc/gen_eb.h5'
h5f = h5py.File(h5path)
for path, subdirs, files in os.walk(root):
for name in files:
cnt += 1
if cnt % 100 == 0:
print cnt
fpath = os.path.join(path, name)
fname = name[:-3]
eb_data = zl.load(fpath)
eb_data = np.array(eb_data).astype(np.float16)
# h5f.create_dataset('train/%s'%fname,dtype='float1', data=eb_data)
# exit(0)
h5f['train/%s' % fname] = eb_data
def prep_train():
cnt = 0
root = 'data/sg_vrd_2016/EB/sg_train_images'
h5path = 'data/sg_vrd_2016/EB/eb2.h5'
h5f = h5py.File(h5path, driver='core')
data = {}
for path, subdirs, files in os.walk(root):
for name in files:
cnt += 1
if cnt % 100 == 0:
print cnt
fpath = os.path.join(path, name)
fname = name[:-3]
eb_data = zl.load(fpath)
eb_data = np.array(eb_data).astype(np.float16)
data[fname] = eb_data
# h5f.create_dataset('train/%s'%fname,dtype='float1', data=eb_data)
# exit(0)
for fname in data.keys():
h5f['train/%s' % fname] = data[fname]
def vg_choose_final_set():
objects_all = zl.load('output/objects_all.pkl')
objects = zl.load('output/objects.pkl')
subjects = zl.load('output/subjects.pkl')
predicate = zl.load('output/predicate.pkl')
objects_all_sorted = sorted(objects_all.items(),
key=operator.itemgetter(1),
reverse=True)
objects_sorted = sorted(objects.items(),
key=operator.itemgetter(1),
reverse=True)
subjects_sorted = sorted(subjects.items(),
key=operator.itemgetter(1),
reverse=True)
predicate_sorted = sorted(predicate.items(),
key=operator.itemgetter(1),
reverse=True)
# for pre in predicate_sorted:
# print pre[0],pre[1]
# exit(0)
#objects_all_sorted = objects_all_sorted[:300]
#predicate_sorted = predicate_sorted[:100]
objects_final = {i[0]: i[1] for i in objects_all_sorted[:202]}
del objects_final['woman']
del objects_final['background']
# print len(sorted(objects_final.keys()))
# exit(0)
#predicate_final = {i[0]:i[1] for i in predicate_sorted[:100]}
predicate_final = {}
mappings_p = make_p_mappings()
for k in mappings_p.keys():
if mappings_p[k] in predicate:
predicate_final[mappings_p[k]] = predicate[mappings_p[k]]
client = MongoClient("mongodb://localhost:27017")
db = client.visual_genome_1_2
db_results = db.relationships_cannon.find(no_cursor_timeout=True)
cnt = 0
start = time.time()
last = time.time()
subjects = {}
objects = {}
objects_all = {}
predicate = {}
cnt = 0
docs = []
for doc in db_results:
if cnt % 10000 == 0:
print cnt
docs.append(doc)
cnt += 1
cnt = 0
random.shuffle(docs)
for doc in docs:
cnt += 1
if cnt % 10000 == 0:
print cnt
rcnt = 0
doc_object = dict(doc)
doc_relationship = dict(doc)
doc_object.pop('relationships', None)
doc_relationship.pop('relationships', None)
doc_object['objects'] = []
doc_relationship['relationships'] = []
for r in doc['relationships']:
#doc_object['objects'].append()
pre = r['predicate']
obj = r['object']['name']
sub = r['subject']['name']
if sub in objects_final: doc_object['objects'].append(r['subject'])
if obj in objects_final: doc_object['objects'].append(r['object'])
if sub in objects_final and obj in objects_final and pre in predicate_final:
doc_relationship['relationships'].append(r)
if cnt <= 80000:
db.relationships_all_train.insert(doc_relationship)
db.relationships_objects_train.insert(doc_object)
else:
db.relationships_all_test.insert(doc_relationship)
db.relationships_objects_test.insert(doc_object)
def relation_make_meta():
spo_info = zl.load('output/spo_info_vg.pkl')
spo_list = []
spo_dict = {}
for k in spo_info.keys():
if spo_info[k] >= 5:
spo_list.append(k)
for spo in spo_list:
spo_dict[spo] = 0
spo_list = spo_dict
blacklist = [
'VG_100K/2363098',
'VG_100K_2/2402233',
'VG_100K/2365839',
'VG_100K_2/2398948',
'VG_100K/2315697',
'VG_100K_2/2403354',
]
client = MongoClient("mongodb://localhost:27017")
db = client.visual_genome_1_2
imdatas = {}
for imdata in db.image_data.find(no_cursor_timeout=True):
imid = imdata['image_id']
imdatas[imid] = imdata
db_results_train = db.relationships_all_train.find(no_cursor_timeout=True)
m = h5py.File('data/vg1_2_meta.h5')
cnt = 0
for doc in db_results_train:
if cnt % 1000 == 0:
print cnt
cnt += 1
imid = doc['image_id']
im_data = imdatas[imid]
if im_data['width'] < 100 or im_data['height'] < 100: continue
im_path_full = im_data['url'].replace(
'https://cs.stanford.edu/people/rak248/', '')
im_path_folder = im_path_full.split('/')[0]
im_path_file = im_path_full.split('/')[1]
im_index = im_path_folder + '/' + im_path_file.replace('.jpg', '')
if im_index in blacklist: continue
obj_boxes = []
sub_boxes = []
rlp_labels = []
imid = doc['image_id']
imdata = imdatas[imid]
for r in doc['relationships']:
pre = r['predicate']
sub_name = r['subject']['name']
obj_name = r['object']['name']
spo = sub_name + '_' + pre + '_' + obj_name
if spo in spo_list:
sidx = zl.name2idx_cls(m, sub_name)
oidx = zl.name2idx_cls(m, obj_name)
pidx = zl.name2idx_pre(m, pre)
ox1, oy1, ow, oh = r['object']['x'], r['object']['y'], r[
'object']['w'], r['object']['h']
sx1, sy1, sw, sh = r['subject']['x'], r['subject']['y'], r[
'subject']['w'], r['subject']['h']
ox2, oy2 = ox1 + ow, oy1 + oh
sx2, sy2 = sx1 + sw, sy1 + sh
rlp_labels.append([sidx, pidx, oidx])
sub_boxes.append([sx1, sy1, sx2, sy2])
obj_boxes.append([ox1, oy1, ox2, oy2])
m.create_dataset('gt/train/%s/rlp_labels' % imid,
data=np.array(rlp_labels).astype(np.int16))
m.create_dataset('gt/train/%s/sub_boxes' % imid,
data=np.array(sub_boxes).astype(np.int16))
m.create_dataset('gt/train/%s/obj_boxes' % imid,
data=np.array(obj_boxes).astype(np.int16))
db_results_test = db.relationships_all_test.find(no_cursor_timeout=True)
for doc in db_results_test:
if cnt % 1000 == 0:
print cnt
cnt += 1
imid = doc['image_id']
im_data = imdatas[imid]
if im_data['width'] < 100 or im_data['height'] < 100: continue
im_path_full = im_data['url'].replace(
'https://cs.stanford.edu/people/rak248/', '')
im_path_folder = im_path_full.split('/')[0]
im_path_file = im_path_full.split('/')[1]
im_index = im_path_folder + '/' + im_path_file.replace('.jpg', '')
if im_index in blacklist: continue
obj_boxes = []
sub_boxes = []
rlp_labels = []
imid = doc['image_id']
imdata = imdatas[imid]
for r in doc['relationships']:
pre = r['predicate']
sub_name = r['subject']['name']
obj_name = r['object']['name']
spo = sub_name + '_' + pre + '_' + obj_name
if spo in spo_list:
sidx = zl.name2idx_cls(m, sub_name)
oidx = zl.name2idx_cls(m, obj_name)
pidx = zl.name2idx_pre(m, pre)
ox1, oy1, ow, oh = r['object']['x'], r['object']['y'], r[
'object']['w'], r['object']['h']
sx1, sy1, sw, sh = r['subject']['x'], r['subject']['y'], r[
'subject']['w'], r['subject']['h']
ox2, oy2 = ox1 + ow, oy1 + oh
sx2, sy2 = sx1 + sw, sy1 + sh
rlp_labels.append([sidx, pidx, oidx])
sub_boxes.append([sx1, sy1, sx2, sy2])
obj_boxes.append([ox1, oy1, ox2, oy2])
m.create_dataset('gt/test/%s/rlp_labels' % imid,
data=np.array(rlp_labels).astype(np.int16))
m.create_dataset('gt/test/%s/sub_boxes' % imid,
data=np.array(sub_boxes).astype(np.int16))
m.create_dataset('gt/test/%s/obj_boxes' % imid,
data=np.array(obj_boxes).astype(np.int16))
def setup(self, bottom, top):
self._cur_idx = 0
self.vgg_data = {}
self.gt_labels = {}
vgg_h5 = h5py.File("output/precalc/vg1_2_2016_train.hdf5", 'r')
if os.path.exists('output/cache/vg1_2_2016_train.pkl'):
self.vgg_data = zl.load('output/cache/vg1_2_2016_train.pkl')
print 'loaded train data from cache'
else:
print 'Preloading training data'
zl.tick()
for k in vgg_h5.keys():
classemes = vgg_h5[k]['classemes'][...]
visuals = vgg_h5[k]['visuals'][...]
locations = vgg_h5[k]['locations'][...]
cls_confs = vgg_h5[k]['cls_confs'][...]
self.vgg_data[k] = {}
self.vgg_data[k]['classemes'] = classemes
self.vgg_data[k]['visuals'] = visuals
self.vgg_data[k]['cls_confs'] = cls_confs
self.vgg_data[k]['locations'] = locations
print 'done preloading training data %f' % zl.tock()
zl.save('output/cache/vg1_2_2016_train.pkl', self.vgg_data)
vgg_h5.close()
self.meta = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r')
if os.path.exists('output/cache/vg1_2_2016_gt.pkl'):
self.gt_labels = zl.load('output/cache/vg1_2_2016_gt.pkl')
print 'loaded gt data from cache'
else:
print 'Preloading gt'
zl.tick()
for k in self.meta['gt/train'].keys():
rlp_labels = self.meta['gt/train/%s/rlp_labels' % k][...]
sub_boxes = self.meta['gt/train/%s/sub_boxes' % k][...].astype(
np.float)
obj_boxes = self.meta['gt/train/%s/obj_boxes' % k][...].astype(
np.float)
self.gt_labels[k] = {}
self.gt_labels[k]['rlp_labels'] = rlp_labels
self.gt_labels[k]['sub_boxes'] = sub_boxes
self.gt_labels[k]['obj_boxes'] = obj_boxes
print 'done preloading gt %f' % zl.tock()
zl.save('output/cache/vg1_2_2016_gt.pkl', self.gt_labels)
self.imids = []
for k in self.vgg_data.keys():
self.imids.append(k)
self.imidx = 0
random.shuffle(self.imids)
layer_params = yaml.load(self.param_str_)
self._batch_size = layer_params['batch_size']
self.train_data = []
self._name_to_top_map = {}
# top[0].reshape(self._batch_size, 201*2)
top[0].reshape(self._batch_size, 4096 * 2)
# top[0].reshape(self._batch_size, 4*2)
top[1].reshape(self._batch_size)
# self._name_to_top_map['classeme'] = 0
self._name_to_top_map['visual'] = 0
# self._name_to_top_map['location'] = 0
self._name_to_top_map['label'] = 1
def run_retrieval_vp():
h5f = h5py.File('output/precalc/ours_retr.hdf5')
retr_meta = zl.load('output/pkl/vr_retr_meta.pkl')
# print h5f[h5f.keys()[0]].keys()
# exit(0)
#h5f = h5py.file('output/results/lu_method_results.hdf5')
# data_root='/home/zawlin/g/py-faster-rcnn/data/vg1_2_2016/Data/test/'
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
thresh = 0.0
filters = []
rois = []
images = {}
imids = h5f.keys()
results = {}
cnt = 0
pre = 'teddy bear_sit on_blanket'
# pre = 'paw_in front of_cat'
# pre = 'racket_hold by_player'
r_idx = zl.name2idx_tri(m_vp, pre)
# sub_lbl = pre.split('_')[0]
# pre_lbl = pre.split('_')[1]
# obj_lbl = pre.split('_')[2]
# sub_idx = zl.name2idx_cls(m,sub_lbl)
# pre_idx = zl.name2idx_pre(m,pre_lbl)
# obj_idx = zl.name2idx_cls(m,obj_lbl)
# rlp_label = np.array([sub_idx,pre_idx,obj_idx]).astype(np.int16)
results = {}
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.toc()
zl.tic()
cnt += 1
# rlp_labels = h5f[imid+'/labels'][...]
rlp_confs = h5f[imid + '/confs'][:, r_idx]
ind = np.argsort(rlp_confs)[::-1]
rlp_confs = rlp_confs[ind[:5]]
results[imid] = np.average(rlp_confs)
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
zl.tic()
indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
example_folder = 'output/examples_retr_vg_vp/%s/' % pre
zl.make_dirs(example_folder)
cnt = 0
for imid, v in results_sorted[:200]:
boxes = h5f[imid + '/boxes'][...]
rlp_labels = h5f[imid + '/labels'][...]
rlp_confs = h5f[imid + '/confs'][...]
# indexor = np.arange(rlp_labels.shape[0])
# ind = indexor[np.all(rlp_labels==rlp_label,axis=1)]
ind = np.where(rlp_labels == r_idx)[0]
if ind.shape[0] != 0:
boxes = boxes[ind]
sb = boxes[0]
# ob = obj_boxes[0]
im = cv2.imread(data_root + zl.imid2path(m, imid))
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
# cv2.rectangle(im,(ob[0],ob[1]),(ob[2],ob[3]),(0,0,200),1)
cv2.imwrite(example_folder + str(cnt) + '_%s.jpg' % imid, im)
#cv2.imshow('im',im)
cnt += 1
def run_retrieval_vp_v2():
# h5_path = 'output/sg_vrd_2016_result.classeme.hdf5'
h5_path = 'output/precalc/vg1_2_vp2016_test_nms2_.4.hdf5'
m = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_meta.h5', 'r', 'core')
m_vp = h5py.File('/home/zawlin/Dropbox/proj/vg1_2_vp_meta.h5', 'r', 'core')
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
data_root = '/home/zawlin/data/data_vrd/vg_1.2/'
rlp_map = []
for i in xrange(1, len(m_vp['meta/tri/idx2name'].keys())):
tri = str(m_vp['meta/tri/idx2name'][str(i)][...])
s_lbl = tri.split('_')[0]
p_lbl = tri.split('_')[1]
o_lbl = tri.split('_')[2]
rlp_label = [
zl.name2idx_cls(m, s_lbl),
zl.name2idx_pre(m, p_lbl),
zl.name2idx_cls(m, o_lbl)
]
rlp_map.append(rlp_label)
rlp_map = np.array(rlp_map)
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['labels'] = h5_in[i]['labels'][...]
h5f[i]['confs'] = h5_in[i]['confs'][...]
h5f[i]['boxes'] = h5_in[i]['boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
total = 0
retr_meta = zl.load('output/pkl/vg_retr_meta.pkl')
for k in xrange(len(retr_meta['rlp_labels'])):
total += retr_meta['counts'][k]
if k > 1000:
print total / 1000.
exit(0)
break
continue
cnt += 1
rlp_label = retr_meta['rlp_labels'][k]
gt_files = retr_meta['files'][k]
# print gt_files
# exit(0)
# for f in gt_files:
# impath= zl.imid2path(m,f)
# print impath
# im= cv2.imread(data_root+impath)
# cv2.imshow('im',im)
# cv2.waitKey(0)
results = {}
zl.tick()
ranks = []
for imid in imids:
labels = h5f[imid]['labels'] - 1
rlp_confs = h5f[imid]['confs']
rlp_labels = rlp_map[labels]
if rlp_labels.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
delay = 0
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
impath = zl.imid2path(m, imid)
if found and i >= 5:
break
# print gt_files
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < 5:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+impath)
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / 5 #total_gt
r_100 = tp_100 / 5 #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.tock())
def show_retrieval_results():
results = zl.load('output/results/retr_ours.pkl')
scores = {}
rel = 'person_has_phone;1_2_50;12'
# rel = 'bowl_on_table;64_0_6;13'
# #rel = 'glasses_on_table;11_0_6;26'
# #rel = 'person_carry_bag;1_36_27;17'
# #rel = 'sunglasses_on_person;63_0_1;15'
# rel = 'person_wear_tie;1_1_89;15'
#rel = 'cat_sleep on_car;62_62_9;1'
#rel = 'sunglasses_on_person;63_0_1;15'
rel = 'car_above_wheel;9_9_22;5'
rel = 'person_with_skis;1_30_78;3'
rel = 'sofa_beneath_person;52_29_1;4'
#rel = 'cat_watch_monitor;62_44_21;3'
#rel = 'person_play with_laptop;1_61_29;3'
#rel = 'trash can_below_trees;83_23_15;5'
rel_str = rel.split(';')[1]
rel_lbl_str = rel.split(';')[0]
root = '/home/zawlin/g/py-faster-rcnn/data/sg_vrd_2016/Data/sg_test_images/'
#h5f = h5py.File('output/results/lu_method_results.hdf5')
h5f = h5py.File('output/results/ours_retr.hdf5')
zl.tick()
for i in results.keys():
scores[i] = 0
if rel_str in results[i]:
scores[i] = results[i][rel_str]
scores_sorted = sorted(scores.items(),
key=operator.itemgetter(1),
reverse=True)
for i in xrange(0, 20):
im_folder = 'output/results/examples_zeroshot_retr/ours/' + rel_lbl_str + '/'
if not os.path.exists(im_folder):
os.makedirs(im_folder)
imid = scores_sorted[i][0]
sub_boxes = h5f[imid + '/sub_boxes'][...]
obj_boxes = h5f[imid + '/obj_boxes'][...]
rlp_labels = h5f[imid + '/rlp_labels'][...]
rlp_confs = h5f[imid + '/rlp_confs'][...]
rlp_label = np.array([
int(rel_str.split('_')[0]),
int(rel_str.split('_')[1]),
int(rel_str.split('_')[2])
])
ind = np.all(rlp_labels == rlp_label, axis=1)
sub_boxes = sub_boxes[ind]
obj_boxes = obj_boxes[ind]
rlp_labels = rlp_labels[ind]
rlp_confs = rlp_confs[ind]
ind_sorted = np.argsort(rlp_confs)[::-1]
sub_boxes = sub_boxes[ind_sorted]
obj_boxes = obj_boxes[ind_sorted]
rlp_labels = rlp_labels[ind_sorted]
rlp_confs = rlp_confs[ind_sorted]
if sub_boxes.shape[0] == 0: continue
im = cv2.imread(root + imid + '.jpg')
sb = sub_boxes[0]
ob = obj_boxes[0]
cv2.rectangle(im, (sb[0], sb[1]), (sb[2], sb[3]), (0, 200, 0), 1)
cv2.rectangle(im, (ob[0], ob[1]), (ob[2], ob[3]), (0, 0, 200), 1)
cv2.imwrite(im_folder + str(i) + '_%s.jpg' % imid, im)
cv2.imshow('im', im)
cv2.waitKey(0)
def run_relation(model_type, iteration):
cache_h5 = h5py.File('output/sg_vrd_cache.h5')['test']
result = h5py.File('output/sg_vrd_2016_result_' + model_type + '_' +
iteration + '.hdf5')
m = h5py.File('data/sg_vrd_meta.h5')
#--------------cache boxes-----------------
h5_boxes = h5py.File('output/precalc/sg_vrd_objs.hdf5')
cache_boxes = {}
if os.path.exists('output/cache/sg_vrd_objs_test.pkl'):
cache_boxes = zl.load('output/cache/sg_vrd_objs_test.pkl')
glog.info('loaded obj data from cache')
else:
glog.info('Preloading obj')
zl.tic()
for k in h5_boxes['test'].keys():
boxes = h5_boxes['test/%s/boxes' % k][...]
cache_boxes[k] = boxes
glog.info('done preloading obj %f' % zl.toc())
zl.save('output/cache/sg_vrd_obj_test.pkl', cache_boxes)
#--------------cache boxes-----------------
#--------------cache old boxes-------------
h5_boxes = h5py.File('output/sg_vrd_2016_test.hdf5')
cache_old_boxes = {}
if os.path.exists('output/cache/sg_vrd_objs_test_vgg.pkl'):
cache_old_boxes = zl.load('output/cache/sg_vrd_objs_test_vgg.pkl')
glog.info('loaded obj data from cache')
else:
glog.info('Preloading obj')
zl.tic()
for k in h5_boxes.keys():
locations = h5_boxes['%s/locations' % k][...]
cls_confs = h5_boxes['%s/cls_confs' % k][...]
boxes = np.concatenate((locations, cls_confs[:, 1, np.newaxis],
cls_confs[:, 0, np.newaxis]),
axis=1)
cache_old_boxes[k] = boxes
glog.info('done preloading obj %f' % zl.toc())
zl.save('output/cache/sg_vrd_obj_test_vgg.pkl', cache_old_boxes)
#--------------cache old boxes-------------
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
keep = 100
thresh = 0.0001
prototxt = 'models/sg_vrd/rel_pre_iccv/test_' + model_type + '.prototxt'
model_path = 'output/rel_iccv/' + model_type + '_iter_' + iteration + '.caffemodel'
net = caffe.Net(prototxt, model_path, caffe.TEST)
# sio.savemat('output/'+model_type+'.mat',{'params_'+model_type:net.params['relation'][0].data})
# exit(0)
#net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST)
cnt = 0
for imid in cache_h5.keys():
cnt += 1
if cnt % 10 == 0:
glog.info(cnt)
obj_boxes_gt = m['gt/test'][imid]['obj_boxes'][...]
sub_boxes_gt = m['gt/test'][imid]['sub_boxes'][...]
rlp_labels_gt = m['gt/test'][imid]['rlp_labels'][...]
rlp_labels = []
rlp_confs = []
sub_boxes = []
obj_boxes = []
#boxes = cache_boxes[imid]
boxes = cache_old_boxes[imid]
if boxes.shape[0] >= 2:
for s in xrange(boxes[:20].shape[0]):
for o in xrange(boxes[:20].shape[0]):
if s == o: continue
if boxes[s][4] < 0.01: continue
if boxes[o][4] < 0.01: continue
sbox = boxes[s][:4]
obox = boxes[o][:4]
rlp_labels.append([boxes[s, 5], -1, boxes[o, 5]])
rlp_confs.append(boxes[s, 4] + boxes[o, 4])
sub_boxes.append(sbox)
obj_boxes.append(obox)
if len(sub_boxes) <= 0: continue
#sub_box = np.array(sub_box)
#obj_box = np.array(obj_box)
im_path = C.get_sg_vrd_path_test(imid)
im_detect(net, im_path, sub_boxes, obj_boxes)
relation_prob = net.blobs['relation_prob'].data[...]
for r in xrange(relation_prob.shape[0]):
argmax = np.argmax(relation_prob[r, ...])
rs = relation_prob[r, argmax].squeeze()
rlp_labels[r][1] = argmax
rlp_confs[r] = rlp_confs[r] + rs
im = cv2.imread(data_root + imid + '.jpg')
for i in xrange(len(sub_boxes)):
sb = sub_boxes[i]
cv2.rectangle(im, (int(sb[0]), int(sb[1])),
(int(sb[2]), int(sb[3])), (255, 0, 0), 2)
pass
cv2.imshow('im', im)
#if cv2.waitKey(0)==27:
# exit(0)
#rlp_confs.append(rlp_conf)
#rlp_labels.append(rlp_label)
#sub_boxes.append(sub_box)
#obj_boxes.append(obj_box)
# for i in xrange(70):
# rs = relation_score[i]
# if rs>0.0:
# predicate =i
# #print relation_score[predicate]
# rlp_label = np.array([sub_cls[s],predicate,obj_cls[s]]).astype(np.int32)
# #print '%s %s %s %f'%(m['meta/cls/idx2name/'+str(rlp_label[0])][...],m['meta/pre/idx2name/'+str(rlp_label[1])][...],m['meta/cls/idx2name/'+str(rlp_label[2])][...],rs)
# rlp_conf = rs#+sub_score+obj_score#relation_score[predicate]
# rlp_confs.append(rlp_conf)
# rlp_labels.append(rlp_label)
# sub_boxes.append(sub_boxes_gt[s])
# obj_boxes.append(obj_boxes_gt[s])
result.create_dataset(imid + '/rlp_confs',
dtype='float16',
data=np.array(rlp_confs).astype(np.float16))
result.create_dataset(imid + '/sub_boxes',
dtype='float16',
data=np.array(sub_boxes).astype(np.float16))
result.create_dataset(imid + '/obj_boxes',
dtype='float16',
data=np.array(obj_boxes).astype(np.float16))
result.create_dataset(imid + '/rlp_labels',
dtype='float16',
data=np.array(rlp_labels).astype(np.float16))
def run_retrieval_zeroshot():
# h5_path = 'output/sg_vrd_2016_result_all_19500.hdf5'
# h5_path = 'output/sg_vrd_2016_result_diff_all_5000.hdf5'
# h5_path= 'output/results/lu_method_results.hdf5'
# h5_path = 'output/sg_vrd_2016_result.hdf5.dd'
# h5_path = 'output/results/lu_method_results_max.hdf5'
h5_path = 'output/results/lu_visual_method_results.hdf5'
data_root = 'data/sg_vrd_2016/Data/sg_test_images/'
m = h5py.File('/home/zawlin/Dropbox/proj/sg_vrd_meta.h5', 'r', 'core')
zeroshots = m['meta/zeroshots'][...]
gt_cache_path = 'output/cache/sg_vrd_gt_cache.pkl'
gt_h5f = {}
np.random.seed(76)
if os.path.exists(gt_cache_path):
print 'load gt from cache'
gt_h5f = zl.load(gt_cache_path)
else:
print 'cacheing gt'
for k in m['gt/test']:
gt_h5f[k] = {}
sub_boxes = m['gt/test'][k]['sub_boxes'][...]
rlp_labels = m['gt/test'][k]['rlp_labels'][...]
obj_boxes = m['gt/test'][k]['obj_boxes'][...]
gt_h5f[k]['sub_boxes'] = sub_boxes
gt_h5f[k]['obj_boxes'] = obj_boxes
gt_h5f[k]['rlp_labels'] = rlp_labels
print 'caching gt done'
zl.save(gt_cache_path, gt_h5f)
cache_path = 'output/cache/%s.pkl' % h5_path.split('/')[-1]
# h5f = h5py.File('output/sg_vrd_2016_result.classeme.hdf5')
if os.path.exists(cache_path):
print 'load from cache'
h5f = zl.load(cache_path)
else:
h5_in = h5py.File(h5_path, 'r')
h5f = {}
print 'preloading data'
for i in h5_in:
h5f[i] = {}
h5f[i]['rlp_labels'] = h5_in[i]['rlp_labels'][...]
h5f[i]['rlp_confs'] = h5_in[i]['rlp_confs'][...]
h5f[i]['sub_boxes'] = h5_in[i]['sub_boxes'][...]
h5f[i]['obj_boxes'] = h5_in[i]['obj_boxes'][...]
zl.save(cache_path, h5f)
print 'preloading data done'
#h5f = h5py.file('output/results/lu_method_results.hdf5')
retr_meta = zl.load('output/pkl/vr_retr_meta.pkl')
thresh = 0.0
images = {}
imids = h5f.keys()
results = {}
cnt = 0
r_acc_100 = 0
r_acc_50 = 0
tp_total = 0
gt_total = 0
median = []
for k in xrange(len(retr_meta['rlp_labels'])):
# if k>1000:
# break
rlp_label = retr_meta['rlp_labels'][k]
if not np.any(np.all(zeroshots == rlp_label, axis=1)): continue
gt_files = retr_meta['files'][k]
cnt += 1
results = {}
zl.tic()
for imid in imids:
rlp_labels = h5f[imid]['rlp_labels']
rlp_confs = h5f[imid]['rlp_confs']
if rlp_confs.shape[0] == 0:
results[imid] = 0.0
continue
indexor = np.arange(rlp_labels.shape[0])
ind = indexor[np.all(rlp_labels == rlp_label, axis=1)]
if ind.shape[0] == 0:
results[imid] = 0.0
continue
confs = rlp_confs[ind]
results[imid] = np.random.uniform(-1, 1) #np.average(confs)
results_sorted = zl.sort_dict_by_val(results)
total_gt = len(gt_files) + 0.0
gt_total += total_gt + 0.0
tp_50 = 0.
tp_100 = 0.
found = False
delay = 0
s_lbl = zl.idx2name_cls(m, rlp_label[0])
p_lbl = zl.idx2name_pre(m, rlp_label[1])
o_lbl = zl.idx2name_cls(m, rlp_label[2])
lbl_str = '%s_%s_%s' % (s_lbl, p_lbl, o_lbl)
r_at_k = 5
for i in xrange(len(results_sorted)):
imid, v = results_sorted[i]
if found and i >= r_at_k:
break
cor = imid in gt_files
if cor:
if not found:
found = True
median.append(i)
if i < r_at_k:
tp_100 += 1
tp_total += 1
if i < 50: tp_50 += 1
# if True:
# cor_or_not = str(cor)
# if cor :delay=0
# if delay ==0:
# im = cv2.imread(data_root+imid+'.jpg')
# cv2.putText(im, cor_or_not, (50, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, lbl_str, (50, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.putText(im, str(i), (50, 100), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 1)
# cv2.imshow('im',im)
# c = cv2.waitKey(delay)&0xFF
# if c ==27:
# exit(0)
# if c == ord('s'):
# delay = 1-delay
# if c == ord('c'):
# delay = 1
r_50 = tp_50 / r_at_k #total_gt
r_100 = tp_100 / r_at_k #total_gt
r_acc_50 += r_50
r_acc_100 += r_100
med = np.median(median)
print '%d %f %f %f %f %d %f' % (cnt, r_50, r_100, r_acc_50 / cnt,
r_acc_100 / cnt, med, zl.toc())
def run_relation_batch(model_type, iteration):
vgg_h5 = h5py.File('output/precalc/vg1_2_2016_test.hdf5')
vgg_data = {}
if os.path.exists('output/cache/vg1_2_2016_test.pkl'):
vgg_data = zl.load('output/cache/vg1_2_2016_test.pkl')
print 'loaded test data from cache'
else:
print 'Preloading testing data'
zl.tick()
for k in vgg_h5.keys():
classemes = vgg_h5[k]['classemes'][...]
visuals = vgg_h5[k]['visuals'][...]
locations = vgg_h5[k]['locations'][...]
cls_confs = vgg_h5[k]['cls_confs'][...]
vgg_data[k] = {}
vgg_data[k]['classemes'] = classemes
vgg_data[k]['visuals'] = visuals
vgg_data[k]['cls_confs'] = cls_confs
vgg_data[k]['locations'] = locations
print 'done preloading testing data %f' % zl.tock()
zl.save('output/cache/vg1_2_2016_test.pkl', vgg_data)
vgg_h5.close()
result = h5py.File('output/vg_results/vg1_2_2016_result_' + model_type +
'_' + iteration + '.hdf5')
m = h5py.File('data/vg1_2_meta.h5')
data_root = 'data/vg1_2_2016/Data/test/'
keep = 100
thresh = 0.0001
net = caffe.Net(
'models/vg1_2/relation/test_' + model_type + '.prototxt',
'output/relation/vg/relation_vgg16_' + model_type + '_iter_' +
iteration + '.caffemodel', caffe.TEST)
#net = caffe.Net('models/sg_vrd/relation/test.prototxt','output/models/sg_vrd_relation_vgg16_iter_264000.caffemodel',caffe.TEST)
cnt = 1
zl.tick()
imids = sorted(vgg_data.keys())
for imid in imids:
if cnt % 100 == 0:
print cnt, zl.tock()
zl.tick()
cnt += 1
if imid in result: continue
classemes = vgg_data[imid]['classemes']
visuals = vgg_data[imid]['visuals']
locations = vgg_data[imid]['locations']
cls_confs = vgg_data[imid]['cls_confs']
# im = cv2.imread(data_root+imid+'.jpg')
# #print cls_confs
# # for box in locations:
# # b=box[:4].astype(np.int32)
# # cv2.rectangle(im,(b[0],b[1]),(b[2],b[3]),(255,0,0))
# w,h = im.shape[2],im.shape[1]
rlp_labels = []
rlp_confs = []
sub_boxes = []
obj_boxes = []
relation_vectors = []
classemes_in = []
visuals_in = []
locations_in = []
cls_confs_in = []
sub_cls_in = []
obj_cls_in = []
sub_score_in = []
obj_score_in = []
sub_boxes = []
obj_boxes = []
for s in xrange(len(locations)):
for o in xrange(len(locations)):
if s == o: continue
sub = locations[s]
obj = locations[o]
sub_visual = visuals[s]
obj_visual = visuals[o]
sub_cls = cls_confs[s, 0]
obj_cls = cls_confs[o, 0]
sub_score = cls_confs[s, 1]
obj_score = cls_confs[o, 1]
sub_classme = classemes[s]
obj_classme = classemes[o]
sub_loc_encoded = bbox_transform(np.array([obj[:4]]),
np.array([sub[:4]]))[0]
obj_loc_encoded = bbox_transform(np.array([sub[:4]]),
np.array([obj[:4]]))[0]
visual = np.hstack((sub_visual, obj_visual)).reshape(8192)
classeme = np.hstack((sub_classme, obj_classme)).reshape(402)
loc = np.hstack((sub_loc_encoded, obj_loc_encoded)).reshape(8)
classemes_in.append(classeme)
visuals_in.append(visual)
locations_in.append(loc)
sub_cls_in.append(sub_cls)
obj_cls_in.append(obj_cls)
sub_score_in.append(sub_score)
obj_score_in.append(obj_score)
sub_boxes.append(sub[:4])
obj_boxes.append(obj[:4])
if 'all' in model_type:
blob = {
'classeme': np.array(classemes_in),
'visual': np.array(visuals_in),
'location': np.array(locations_in)
}
net.blobs['classeme'].reshape(*blob['classeme'].shape)
net.blobs['visual'].reshape(*blob['visual'].shape)
net.blobs['location'].reshape(*blob['location'].shape)
elif 'visual' in model_type:
blob = {
'visual': np.array(visuals_in),
}
net.blobs['visual'].reshape(*blob['visual'].shape)
elif 'classeme' in model_type:
blob = {
'classeme': np.array(classemes_in),
}
net.blobs['classeme'].reshape(*blob['classeme'].shape)
elif 'location' in model_type:
blob = {'location': np.array(locations_in)}
#batch this
net.blobs['location'].reshape(*blob['location'].shape)
if len(locations_in) == 0:
rlp_confs = []
sub_boxes = []
obj_boxes = []
rlp_labels = []
else:
net.forward_all(**blob)
relation_score = net.blobs['relation_prob'].data.copy()
argmax = np.argmax(relation_score, axis=1)
rs = relation_score[np.arange(relation_score.shape[0]), argmax]
rlp_labels = np.vstack((sub_cls_in, argmax, obj_cls_in)).T
rlp_confs = np.array(sub_score_in) + np.array(rs) + np.array(
obj_score_in)
result.create_dataset(imid + '/rlp_confs',
dtype='float16',
data=np.array(rlp_confs).astype(np.float16))
result.create_dataset(imid + '/sub_boxes',
dtype='float16',
data=np.array(sub_boxes).astype(np.float16))
result.create_dataset(imid + '/obj_boxes',
dtype='float16',
data=np.array(obj_boxes).astype(np.float16))
result.create_dataset(imid + '/rlp_labels',
dtype='float16',
data=np.array(rlp_labels).astype(np.float16))
def setup(self, bottom, top):
self._cur_idx = 0
self.rdata = sio.loadmat('data/meta/vrd/annotation_train.mat',
struct_as_record=False,
squeeze_me=True)
self.meta = h5py.File('data/sg_vrd_meta.h5', 'r')
self.gt_labels = {}
if os.path.exists('output/cache/sg_vrd_gt.pkl'):
self.gt_labels = zl.load('output/cache/sg_vrd_gt.pkl')
glog.info('loaded gt data from cache')
else:
glog.info('Preloading gt')
zl.tic()
for k in self.meta['gt/train'].keys():
rlp_labels = self.meta['gt/train/%s/rlp_labels' % k][...]
sub_boxes = self.meta['gt/train/%s/sub_boxes' % k][...].astype(
np.float)
obj_boxes = self.meta['gt/train/%s/obj_boxes' % k][...].astype(
np.float)
if sub_boxes.shape[0] > 0:
zeros = np.zeros((sub_boxes.shape[0], 1), dtype=np.float)
# first index is always zero since we do one image by one image
sub_boxes = np.concatenate((zeros, sub_boxes), axis=1)
obj_boxes = np.concatenate((zeros, obj_boxes), axis=1)
self.gt_labels[k] = {}
self.gt_labels[k]['rlp_labels'] = rlp_labels
self.gt_labels[k]['sub_boxes'] = sub_boxes
self.gt_labels[k]['obj_boxes'] = obj_boxes
glog.info('done preloading gt %f' % zl.toc())
vgg_h5 = h5py.File("output/sg_vrd_2016_train_predicate_exp_train.hdf5",
'r', 'core')
if os.path.exists('output/cache/sg_vrd_2016_train.pkl'):
self.vgg_data = zl.load('output/cache/sg_vrd_2016_train.pkl')
glog.info('loaded train data from cache')
else:
glog.info('Preloading training data')
zl.tic()
self.vgg_data = {}
for k in vgg_h5.keys():
sub_visual = vgg_h5[k]['sub_classeme'][...]
obj_visual = vgg_h5[k]['obj_classeme'][...]
pre_label = vgg_h5[k]['pre_label'][...]
self.vgg_data[k] = {}
self.vgg_data[k]['sub_visual'] = sub_visual
self.vgg_data[k]['obj_visual'] = obj_visual
self.vgg_data[k]['pre_label'] = pre_label
glog.info('done preloading training data %f' % zl.toc())
zl.save('output/cache/sg_vrd_2016_train.pkl', self.vgg_data)
vgg_h5.close()
self.meta = h5py.File('data/sg_vrd_meta.h5', 'r', 'core')
layer_params = yaml.load(self.param_str)
self._batch_size = layer_params['batch_size']
self.train_data = []
self._name_to_top_map = {}
# data blob: holds a batch of N images, each with 3 channels
# top[0].reshape(self._batch_size, 4096 * 2 )
top[0].reshape(self._batch_size, 2 * 101)
top[1].reshape(self._batch_size)
# self._name_to_top_map['visual'] = 0
# self._name_to_top_map['classeme'] = 0
self._name_to_top_map['visual'] = 0
# self._name_to_top_map['location'] = 1
self._name_to_top_map['label'] = 1
