def get_minibatch_multi(roidb, num_classes):
"""Given a mini batch of roidb, construct a data blob from it."""
num_images = len(roidb)
# Sample random scales to use for each image in this batch
random_scale_inds = npr.randint(0, high=len(cfg.TRAIN.SCALES),
size=num_images)
#assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), 'num_images ({}) must divide BATCH_SIZE ({})'.format(num_images, cfg.TRAIN.BATCH_SIZE)
cfg.TRAIN.BATCH_SIZE = num_images * 10
rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
rois_per_image = 100
fg_rois_per_image = 100 #np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)
im_timer = Timer()
im_timer.tic()
im_blob, im_scales = _get_image_blob(roidb, random_scale_inds)
im_timer.toc()
blobs = {'ims': im_blob}
# Now, build the region of interest and label blobs
rois_blob = np.zeros((0, 5), dtype=np.float32)
oids_blob = np.zeros((0,1),dtype=np.int32)
quadric_blob = np.zeros((0, 28), dtype=np.float32)
one_hot = np.zeros((0,51), dtype=np.float32)
direct_value = np.zeros((0,1), dtype=np.float32)
labels_blob = np.zeros((0), dtype=np.float32)
#rels_blob = np.zeros((0, 3), dtype=np.int32)
rels_blob = np.zeros((0, roidb[0]['gt_relations'].shape[1]), dtype=np.int32)
bbox_targets_blob = np.zeros((0, 4 * num_classes), dtype=np.float32)
bbox_inside_blob = np.zeros(bbox_targets_blob.shape, dtype=np.float32)
all_overlaps = []
dbs=np.ones((num_images), dtype=np.int32)
box_idx_offset = 0
d_timer = Timer()
d_timer.tic()
for im_i in xrange(num_images):
# sample graph
#print 'boxes',roidb[im_i]['boxes']
roi_inds, rels = _sample_graph(roidb[im_i],
fg_rois_per_image,
rois_per_image,
num_neg_rels=cfg.TRAIN.NUM_NEG_RELS)
if rels.size == 0:
print('batch skipped')
return None
# gather all samples based on the sampled graph
# does not seem to change rels
rels, labels, overlaps, im_rois, bbox_targets, bbox_inside_weights =\
_gather_samples(roidb[im_i], roi_inds, rels, num_classes) # gather samples will decide which rois overlap enough with a labelled bbx to be a "fg" roi. And it will re-organise the relations and the other variables accordingly
#print 'some of them', im_rois, fg_rois_per_image, cfg.TRAIN.FG_FRACTION , cfg.TRAIN.BATCH_SIZE, num_images , rois_per_image
# Add to RoIs blob
rois = _project_im_rois(im_rois, im_scales[im_i])
batch_ind = im_i * np.ones((rois.shape[0], 1)) #im id for roi_pooling
rois_blob_this_image = np.hstack((batch_ind, rois))
rois_blob = np.vstack((rois_blob, rois_blob_this_image))
quadrics = roidb[im_i]['quadric_rois']
dbs[im_i]=roidb[im_i]['db_idx']
#print im_i,roidb[im_i]['db_idx'],quadrics.shape, batch_ind[0:quadrics.shape[0]], rois.shape, rois
quadric_blob_this_image = np.hstack((batch_ind[0:quadrics.shape[0]], quadrics))
quadric_blob = np.vstack((quadric_blob, quadric_blob_this_image))
quadric_blob = np.vstack((quadric_blob, quadric_blob_this_image)) # because, there is one for the GT, one for the proposal. They evaluate the relationship loss on the GT and the proposals. So they will build new relations between the GT and the proposed rois. I guess this done somewhere in _sample_graph or _gather_samples
oids_blob = np.vstack((oids_blob, roidb[im_i]['oids']))
# Add to labels, bbox targets, and bbox loss blobs
labels_blob = np.hstack((labels_blob, labels))
bbox_targets_blob = np.vstack((bbox_targets_blob, bbox_targets))
bbox_inside_blob = np.vstack((bbox_inside_blob, bbox_inside_weights))
all_overlaps = np.hstack((all_overlaps, overlaps))
# offset the relationship reference idx the number of previously
# added box
rels_offset = rels.copy()
rels_offset[:, :2] += box_idx_offset
#one_hot_blob = np.zeros((rels_offset.shape[0],51), dtype=np.float32)
#for i in range(rels_offset.shape[0]):
# one_hot_blob[i,rels_offset[i,2]]=1
#one_hot = np.vstack([one_hot, one_hot_blob])
direct_value_blob = np.zeros((rels_offset.shape[0],1), dtype=np.float32)
for i in range(rels_offset.shape[0]):
direct_value_blob[i] = rels_offset[i,2]
direct_value = np.vstack([direct_value, direct_value_blob])
rels_blob = np.vstack([rels_blob, rels_offset])
box_idx_offset += rois.shape[0]
#viz_inds = np.where(overlaps == 1)[0] # ground truth
#viz_inds = npr.choice(np.arange(rois.shape[0]), size=50, replace=False) # random sample
#viz_inds = np.where(overlaps > cfg.TRAIN.FG_THRESH)[0] # foreground
#viz_scene_graph(im_blob[im_i], rois, labels, viz_inds, rels)
blobs['rois'] = rois_blob.copy()
blobs['labels'] = labels_blob.copy().astype(np.int32)
blobs['relations'] = rels_blob[:,:2].copy().astype(np.int32)
blobs['predicates'] = rels_blob[:,2:].copy().astype(np.int32)
blobs['bbox_targets'] = bbox_targets_blob.copy()
blobs['bbox_inside_weights'] = bbox_inside_blob.copy()
blobs['bbox_outside_weights'] = \
np.array(bbox_inside_blob > 0).astype(np.float32).copy()
blobs['oids'] = oids_blob
num_roi = rois_blob.shape[0]
num_rel = rels_blob.shape[0]
blobs['rel_rois'] = data_utils.compute_rel_rois(num_rel,
rois_blob,
rels_blob)
blobs['quadric_rois'] = quadric_blob # direct_value #one_hot #quadric_blob #roidb[im_i]['quadric_rois'] # here I can replace with some gt for sanity check
blobs['db_idx'] = dbs
d_timer.toc()
graph_dict = data_utils.create_graph_data(num_roi, num_rel, rels_blob[:, :2])
for k in graph_dict:
blobs[k] = graph_dict[k]
return blobs
def get_minibatch(roidb, num_classes):
"""Given a mini batch of roidb, construct a data blob from it."""
num_images = len(roidb)
# Sample random scales to use for each image in this batch
random_scale_inds = npr.randint(0, high=len(cfg.TRAIN.SCALES),
size=num_images)
assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), \
'num_images ({}) must divide BATCH_SIZE ({})'. \
format(num_images, cfg.TRAIN.BATCH_SIZE)
rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
fg_rois_per_image = int(np.round(cfg.TRAIN.FG_FRACTION * rois_per_image))
im_timer = Timer()
im_timer.tic()
im_blob, im_scales = _get_image_blob(roidb, random_scale_inds)
im_timer.toc()
blobs = {'ims': im_blob}
# Now, build the region of interest and label blobs
rois_blob = np.zeros((0, 5), dtype=np.float32)
labels_blob = np.zeros((0), dtype=np.float32)
rels_blob = np.zeros((0, 3), dtype=np.int32)
bbox_targets_blob = np.zeros((0, 4 * num_classes), dtype=np.float32)
bbox_inside_blob = np.zeros(bbox_targets_blob.shape, dtype=np.float32)
all_overlaps = []
box_idx_offset = 0
d_timer = Timer()
d_timer.tic()
for im_i in xrange(num_images):
# sample graph
roi_inds, rels = _sample_graph(roidb[im_i],
fg_rois_per_image,
rois_per_image,
num_neg_rels=cfg.TRAIN.NUM_NEG_RELS)
if rels.size == 0:
print('batch skipped')
return None
# gather all samples based on the sampled graph
rels, labels, overlaps, im_rois, bbox_targets, bbox_inside_weights =\
_gather_samples(roidb[im_i], roi_inds, rels, num_classes)
# Add to RoIs blob
rois = _project_im_rois(im_rois, im_scales[im_i])
batch_ind = im_i * np.ones((rois.shape[0], 1)) #im id for roi_pooling
rois_blob_this_image = np.hstack((batch_ind, rois))
rois_blob = np.vstack((rois_blob, rois_blob_this_image))
# Add to labels, bbox targets, and bbox loss blobs
labels_blob = np.hstack((labels_blob, labels))
bbox_targets_blob = np.vstack((bbox_targets_blob, bbox_targets))
bbox_inside_blob = np.vstack((bbox_inside_blob, bbox_inside_weights))
all_overlaps = np.hstack((all_overlaps, overlaps))
# offset the relationship reference idx the number of previously
# added box
rels_offset = rels.copy()
rels_offset[:, :2] += box_idx_offset
rels_blob = np.vstack([rels_blob, rels_offset])
box_idx_offset += rois.shape[0]
#viz_inds = np.where(overlaps == 1)[0] # ground truth
#viz_inds = npr.choice(np.arange(rois.shape[0]), size=50, replace=False) # random sample
#viz_inds = np.where(overlaps > cfg.TRAIN.FG_THRESH)[0] # foreground
#viz_scene_graph(im_blob[im_i], rois, labels, viz_inds, rels)
blobs['rois'] = rois_blob.copy()
blobs['labels'] = labels_blob.copy().astype(np.int32)
blobs['relations'] = rels_blob[:,:2].copy().astype(np.int32)
blobs['predicates'] = rels_blob[:,2].copy().astype(np.int32)
blobs['bbox_targets'] = bbox_targets_blob.copy()
blobs['bbox_inside_weights'] = bbox_inside_blob.copy()
blobs['bbox_outside_weights'] = \
np.array(bbox_inside_blob > 0).astype(np.float32).copy()
num_roi = rois_blob.shape[0]
num_rel = rels_blob.shape[0]
blobs['rel_rois'] = data_utils.compute_rel_rois(num_rel,
rois_blob,
rels_blob)
d_timer.toc()
graph_dict = data_utils.create_graph_data(num_roi, num_rel, rels_blob[:, :2])
for k in graph_dict:
blobs[k] = graph_dict[k]
return blobs
def get_minibatch(roidb, num_classes):
"""Given a mini batch of roidb, construct a data blob from it."""
num_images = len(roidb)
# Sample random scales to use for each image in this batch
random_scale_inds = npr.randint(0,
high=len(cfg.TRAIN.SCALES),
size=num_images)
#assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), 'num_images ({}) must divide BATCH_SIZE ({})'.format(num_images, cfg.TRAIN.BATCH_SIZE)
cfg.TRAIN.BATCH_SIZE = num_images * 10
#rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
#fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)
rois_per_image = 100
fg_rois_per_image = 100 #np.round(cfg.TRAIN.FG_FRACTION * rois_per_image)
im_timer = Timer()
im_timer.tic()
im_blob, im_scales = _get_image_blob(roidb, random_scale_inds)
im_timer.toc()
blobs = {'ims': im_blob}
# Now, build the region of interest and label blobs
rois_blob = np.zeros((0, 5), dtype=np.float32)
oids_blob = np.zeros(
(0, 1), dtype=np.int32
) # this one contains the database object id, I keep for later debugging
oids_blob_4_graph = np.zeros(
(0, 1), dtype=np.int32
) # this one I use to build the graph, I will give different id for the duplicates
quadric_blob = np.zeros((0, 28), dtype=np.float32)
direct_value = np.zeros((0, 1), dtype=np.float32)
labels_blob = np.zeros((0), dtype=np.float32)
rels_blob = np.zeros((0, 3), dtype=np.int32)
bbox_targets_blob = np.zeros((0, 4 * num_classes), dtype=np.float32)
bbox_inside_blob = np.zeros(bbox_targets_blob.shape, dtype=np.float32)
#rels_feat2d = np.zeros((0,400), dtype=np.float32)
#rels_feat3d = np.zeros((0,400), dtype=np.float32)
all_overlaps = []
dbs = np.ones((num_images), dtype=np.int32)
box_idx_offset = 0
d_timer = Timer()
d_timer.tic()
for im_i in xrange(num_images):
# sample graph
roi_inds, rels = _sample_graph(roidb[im_i],
fg_rois_per_image,
rois_per_image,
num_neg_rels=cfg.TRAIN.NUM_NEG_RELS)
if rels.size == 0:
print('batch skipped')
return None
# gather all samples based on the sampled graph
# does not seem to change rels
rels, labels, overlaps, im_rois, bbox_targets, bbox_inside_weights =\
_gather_samples(roidb[im_i], roi_inds, rels, num_classes)
# Add to RoIs blob
rois = _project_im_rois(im_rois, im_scales[im_i])
batch_ind = im_i * np.ones((rois.shape[0], 1)) #im id for roi_pooling
rois_blob_this_image = np.hstack((batch_ind, rois))
rois_blob = np.vstack((rois_blob, rois_blob_this_image))
quadrics = roidb[im_i]['quadric_rois']
dbs[im_i] = roidb[im_i]['db_idx']
#print im_i,quadrics.shape[0]
quadric_blob_this_image = np.hstack(
(batch_ind[0:quadrics.shape[0]], quadrics))
#print quadric_blob_this_image.shape
quadric_blob = np.vstack((quadric_blob, quadric_blob_this_image))
quadric_blob = np.vstack(
(quadric_blob, quadric_blob_this_image)
) # because, there is one for the GT, one for the proposal. They evaluate the relationship loss on the GT and the proposals
#(f2d, f3d) = data_utils.get_rel_blob(roidb[im_i]['rel_geo_2d'], roidb[im_i]['rel_geo_3d'], roidb[im_i]['gt_relations'], rels)
#rels_feat2d = np.vstack((rels_feat2d, f2d ))
#rels_feat3d = np.vstack((rels_feat3d, f3d ))
#quadric_blob_this_image = np.vstack((quadric_blob_this_image, quadric_blob_this_image))
#quadric_blob = np.vstack((quadric_blob, quadric_blob_this_image[roi_inds,:])) # because, there is one for the GT, one for the proposal. They evaluate the relationship loss on the GT and the proposals
oids_this_image = np.vstack((roidb[im_i]['oids'], roidb[im_i]['oids']))
oids_blob = np.vstack((oids_blob, oids_this_image))
oids_4_factor_this_image = np.vstack(
(roidb[im_i]['oids'], 10000 + roidb[im_i]['oids']
)) # because in the graph, I will need to differentiate the two
oids_blob_4_graph = np.vstack(
(oids_blob_4_graph, oids_4_factor_this_image))
# Add to labels, bbox targets, and bbox loss blobs
labels_blob = np.hstack((labels_blob, labels))
bbox_targets_blob = np.vstack((bbox_targets_blob, bbox_targets))
bbox_inside_blob = np.vstack((bbox_inside_blob, bbox_inside_weights))
all_overlaps = np.hstack((all_overlaps, overlaps))
# offset the relationship reference idx the number of previously
# added box
rels_offset = rels.copy()
rels_offset[:, :2] += box_idx_offset
direct_value_blob = np.zeros((rels_offset.shape[0], 1),
dtype=np.float32)
for i in range(rels_offset.shape[0]):
direct_value_blob[i] = rels_offset[i, 2]
direct_value = np.vstack([direct_value, direct_value_blob])
rels_blob = np.vstack([rels_blob, rels_offset])
box_idx_offset += rois.shape[0]
#viz_inds = np.where(overlaps == 1)[0] # ground truth
#viz_inds = npr.choice(np.arange(rois.shape[0]), size=50, replace=False) # random sample
#viz_inds = np.where(overlaps > cfg.TRAIN.FG_THRESH)[0] # foreground
#viz_scene_graph(im_blob[im_i], rois, labels, viz_inds, rels)
#print oids_blob.shape, dbs, rois_blob.shape
blobs['rois'] = rois_blob.copy()
blobs['labels'] = labels_blob.copy().astype(np.int32)
blobs['relations'] = rels_blob[:, :2].copy().astype(np.int32)
blobs['predicates'] = rels_blob[:, 2].copy().astype(np.int32)
blobs['bbox_targets'] = bbox_targets_blob.copy()
#blobs['rels_feat2d'] = rels_feat2d
#blobs['rels_feat3d'] = rels_feat3d
blobs['bbox_inside_weights'] = bbox_inside_blob.copy()
blobs['bbox_outside_weights'] = \
np.array(bbox_inside_blob > 0).astype(np.float32).copy()
blobs['oids'] = oids_blob
num_roi = rois_blob.shape[0]
num_rel = rels_blob.shape[0]
blobs['rel_rois'] = data_utils.compute_rel_rois(num_rel, rois_blob,
rels_blob)
blobs[
'quadric_rois'] = quadric_blob # direct_value #quadric_blob #roidb[im_i]['quadric_rois']
blobs['db_idx'] = dbs
d_timer.toc()
graph_dict = data_utils.create_graph_data_fus(num_roi, num_rel,
rels_blob[:, :2], oids_blob)
for k in graph_dict:
blobs[k] = graph_dict[k]
return blobs
def get_minibatch_test(roidb, num_classes):
"""
Given a mini batch of roidb, construct a data blob from it.
Difference with the non test version is
-the graph is constructed directly by taking the set of all possible relations
-no need to double the set of rois
"""
num_images = len(roidb)
# Sample random scales to use for each image in this batch
im_timer = Timer()
im_timer.tic()
im_blob, im_scales = _get_image_blob_test(roidb, cfg.TEST.SCALES[0])
im_timer.toc()
blobs = {'ims': im_blob}
# Now, build the region of interest and label blobs
rois_blob = np.zeros((0, 5), dtype=np.float32)
oids_blob = np.zeros((0, 1), dtype=np.int32)
quadric_blob = np.zeros((0, 28), dtype=np.float32)
labels_blob = np.zeros((0), dtype=np.float32)
rels_blob = np.zeros((0, 2), dtype=np.int32)
#rels_feat2d = np.zeros((0,400), dtype=np.float32)
#rels_feat3d = np.zeros((0,400), dtype=np.float32)
dbs = np.ones((num_images), dtype=np.int32)
box_idx_offset = 0
d_timer = Timer()
d_timer.tic()
for im_i in xrange(num_images):
# sample graph
#roi_inds, rels = _sample_graph(roidb[im_i], fg_rois_per_image, rois_per_image, num_neg_rels=cfg.TRAIN.NUM_NEG_RELS)
im_rois = roidb[im_i]['boxes']
no = im_rois.shape[0]
rels = make_rels(no)
#rels, labels, overlaps, im_rois, bbox_targets, bbox_inside_weights = _gather_samples(roidb[im_i], roi_inds, rels, num_classes)
# Add to RoIs blob
rois = _project_im_rois(im_rois, im_scales[im_i])
batch_ind = im_i * np.ones((rois.shape[0], 1)) #im id for roi_pooling
rois_blob_this_image = np.hstack((batch_ind, rois))
rois_blob = np.vstack((rois_blob, rois_blob_this_image))
quadrics = roidb[im_i]['quadric_rois']
dbs[im_i] = roidb[im_i]['db_idx']
quadric_blob_this_image = np.hstack((batch_ind, quadrics))
quadric_blob = np.vstack((quadric_blob, quadric_blob_this_image))
#(f2d, f3d) = data_utils.get_rel_blob(roidb[im_i]['rel_geo_2d'], roidb[im_i]['rel_geo_3d'], roidb[im_i]['gt_relations'], rels)
#rels_feat2d = np.vstack((rels_feat2d, f2d ))
#rels_feat3d = np.vstack((rels_feat3d, f3d ))
#quadric_blob_this_image = np.vstack((quadric_blob_this_image, quadric_blob_this_image))
#quadric_blob = np.vstack((quadric_blob, quadric_blob_this_image[roi_inds,:])) # because, there is one for the GT, one for the proposal. They evaluate the relationship loss on the GT and the proposals
oids_blob = np.vstack((oids_blob, roidb[im_i]['oids']))
# offset the relationship reference idx the number of previously
# added box
rels_offset = rels.copy()
rels_offset += box_idx_offset
rels_blob = np.vstack([rels_blob, rels_offset])
box_idx_offset += rois.shape[0]
#viz_inds = np.where(overlaps == 1)[0] # ground truth
#viz_inds = npr.choice(np.arange(rois.shape[0]), size=50, replace=False) # random sample
#viz_inds = np.where(overlaps > cfg.TRAIN.FG_THRESH)[0] # foreground
#viz_scene_graph(im_blob[im_i], rois, labels, viz_inds, rels)
#print oids_blob.shape, dbs, rois_blob.shape
blobs['rois'] = rois_blob.copy()
blobs['relations'] = rels_blob[:, :2].copy().astype(np.int32)
blobs['oids'] = oids_blob
#blobs['rels_feat2d'] = rels_feat2d
#blobs['rels_feat3d'] = rels_feat3d
num_roi = rois_blob.shape[0]
num_rel = rels_blob.shape[0]
blobs['rel_rois'] = data_utils.compute_rel_rois(num_rel, rois_blob,
rels_blob)
blobs['quadric_rois'] = quadric_blob
blobs['db_idx'] = dbs
d_timer.toc()
graph_dict = data_utils.create_graph_data_fus(num_roi, num_rel, rels_blob,
oids_blob)
for k in graph_dict:
blobs[k] = graph_dict[k]
return blobs
def get_minibatch(roidb, num_classes, sigma=cfg.SIGMA):
"""Given a mini batch of roidb, construct a data blob from it."""
num_images = len(roidb)
# Sample random scales to use for each image in this batch
random_scale_inds = npr.randint(0,
high=len(cfg.TRAIN.SCALES),
size=num_images)
assert(cfg.TRAIN.BATCH_SIZE % num_images == 0), \
'num_images ({}) must divide BATCH_SIZE ({})'. \
format(num_images, cfg.TRAIN.BATCH_SIZE)
rois_per_image = cfg.TRAIN.BATCH_SIZE / num_images
fg_rois_per_image = np.round(cfg.TRAIN.FG_FRACTION *
rois_per_image).astype(np.int)
im_timer = Timer()
im_timer.tic()
im_blob, im_scales = _get_image_blob(roidb, random_scale_inds)
im_timer.toc()
blobs = {'ims': im_blob}
# Now, build the region of interest and label blobs
rois_blob = np.zeros((0, 5), dtype=np.float32)
labels_blob = np.zeros((0), dtype=np.float32)
rels_blob = np.zeros((0, 3), dtype=np.int32)
bbox_targets_blob = np.zeros((0, 4 * num_classes), dtype=np.float32)
bbox_inside_blob = np.zeros(bbox_targets_blob.shape, dtype=np.float32)
all_overlaps = []
box_idx_offset = 0
d_timer = Timer()
d_timer.tic()
for im_i in xrange(num_images):
# sample graph
roi_inds, rels = _sample_graph(roidb[im_i],
fg_rois_per_image,
rois_per_image,
num_neg_rels=cfg.TRAIN.NUM_NEG_RELS)
if rels.size == 0:
print('batch skipped')
return None
# gather all samples based on the sampled graph
rels, labels, overlaps, im_rois, bbox_targets, bbox_inside_weights =\
_gather_samples(roidb[im_i], roi_inds, rels, num_classes)
# Add to RoIs blob
rois = _project_im_rois(im_rois, im_scales[im_i])
batch_ind = im_i * np.ones((rois.shape[0], 1)) #im id for roi_pooling
rois_blob_this_image = np.hstack((batch_ind, rois))
rois_blob = np.vstack((rois_blob, rois_blob_this_image))
# Add to labels, bbox targets, and bbox loss blobs
labels_blob = np.hstack((labels_blob, labels))
bbox_targets_blob = np.vstack((bbox_targets_blob, bbox_targets))
bbox_inside_blob = np.vstack((bbox_inside_blob, bbox_inside_weights))
all_overlaps = np.hstack((all_overlaps, overlaps))
# offset the relationship reference idx the number of previously
# added box
rels_offset = rels.copy()
rels_offset[:, :2] += box_idx_offset
rels_blob = np.vstack([rels_blob, rels_offset])
box_idx_offset += rois.shape[0]
#viz_inds = np.where(overlaps == 1)[0] # ground truth
#viz_inds = npr.choice(np.arange(rois.shape[0]), size=50, replace=False) # random sample
#viz_inds = np.where(overlaps > cfg.TRAIN.FG_THRESH)[0] # foreground
#viz_scene_graph(im_blob[im_i], rois, labels, viz_inds, rels)
blobs['rois'] = rois_blob.copy()
blobs['labels'] = labels_blob.copy().astype(np.int32)
blobs['relations'] = rels_blob[:, :2].copy().astype(np.int32)
blobs['predicates'] = rels_blob[:, 2].copy().astype(np.int32)
blobs['bbox_targets'] = bbox_targets_blob.copy()
blobs['bbox_inside_weights'] = bbox_inside_blob.copy()
blobs['bbox_outside_weights'] = \
np.array(bbox_inside_blob > 0).astype(np.float32).copy()
num_roi = rois_blob.shape[0]
num_rel = rels_blob.shape[0]
blobs['rel_rois'] = data_utils.compute_rel_rois(num_rel, rois_blob,
rels_blob)
d_timer.toc()
graph_dict = data_utils.create_graph_data(num_roi, num_rel,
rels_blob[:, :2])
for k in graph_dict:
blobs[k] = graph_dict[k]
import scipy.io as sio
Xmat = sio.loadmat('X_r_102517.mat')
Xr = Xmat['X_r']
# matlab's X_r is in (pred, sub, obj). swap to make it (sub, obj, pred)
Xr = np.swapaxes(Xr, 0, 2)
Xr = np.swapaxes(Xr, 0, 1)
# pad with 0
# Xr = np.lib.pad(Xr, ((1, 0), (1, 0), (1, 0)), 'constant', constant_values=0)
# I need to explicitly get the 0-based class indices by -1 from class labels
# classes_idx = classes - 1;
# predicate_preds = X_r[:, classes_idx, :][classes_idx, :, :]
# predicates = np.argmax(predicate_preds, 2).ravel() + 1
# predicate_scores = predicate_preds.max(axis=2).ravel()
blobs['Xr'] = Xr
blobs['sigma'] = sigma
means = np.full([num_rel, 51], sigma, dtype=np.float32)
rand_val = np.random.uniform(0.0, 1.0, size=(num_rel, 51))
# mask = tf.where(rand_val - means > 0,
# tf.ones([self.data["num_rel"], 51]), tf.zeros([self.data["num_rel"], 51]))
mask = np.ceil(rand_val - means) + 0 # trivial: convert sign of 0
blobs['mask'] = mask
# print 'get_minibatch sigma:%s' % sigma
# print means
# print "----------------------"
# print rand_val
# print "----------------------"
# print mask
return blobs