def im_detect(sess, net, inputs, im, boxes, bbox_reg, multi_iter):
blobs, im_scales = _get_blobs(im, boxes)
# 两两配对，n^2组pairs
relations = []
for i in range(boxes.shape[0]):
for j in range(boxes.shape[0]):
if i != j:
relations.append([i, j])
relations = np.array(relations, dtype=np.int32) # all possible combinations
num_roi = blobs['rois'].shape[0]
num_rel = relations.shape[0]
inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)
feed_dict = {inputs['ims']: blobs['data'],
inputs['rois']: blobs['rois'],
inputs['relations']: relations,
net.keep_prob: 1}
for k in inputs_feed:
feed_dict[inputs[k]] = inputs_feed[k]
# compute relation rois
feed_dict[inputs['rel_rois']] = \
data_utils.compute_rel_rois(num_rel, blobs['rois'], relations)
ops = {}
ops['bbox_deltas'] = net.bbox_pred_output(multi_iter)
ops['rel_probs'] = net.rel_pred_output(multi_iter)
ops['cls_probs'] = net.cls_pred_output(multi_iter)
ops_value = sess.run(ops, feed_dict=feed_dict)
out_dict = {}
for mi in multi_iter:
rel_probs = None
rel_probs_flat = ops_value['rel_probs'][mi]
rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
for i, rel in enumerate(relations):
rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]
cls_probs = ops_value['cls_probs'][mi]
if bbox_reg:
# Apply bounding-box regression deltas
pred_boxes = bbox_transform_inv(boxes, ops_value['bbox_deltas'][mi])
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))
out_dict[mi] = {'scores': cls_probs.copy(),
'boxes': pred_boxes.copy(),
'relations': rel_probs.copy()}
return out_dict
def run_single(sess, net, inputs, outputs, im, boxes, relations, bbox_reg,
multi_iter):
blobs, im_scales = _get_blobs(im, boxes)
relations = np.array(relations,
dtype=np.int32) # all possible combinations
num_roi = blobs['rois'].shape[0]
num_rel = relations.shape[0]
inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)
feed_dict = {
inputs['ims']: blobs['data'],
inputs['rois']: blobs['rois'],
inputs['relations']: relations,
net.keep_prob: 1
}
for k in inputs_feed:
feed_dict[inputs[k]] = inputs_feed[k]
# compute relation rois
feed_dict[inputs['rel_rois']] = \
data_utils.compute_rel_rois(num_rel, blobs['rois'], relations)
ops_value = sess.run(outputs, feed_dict=feed_dict)
mi = multi_iter[-1]
rel_probs_flat = ops_value['rel_probs'][mi]
rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
for i, rel in enumerate(relations):
rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]
cls_probs = ops_value['cls_probs'][mi]
if bbox_reg:
# Apply bounding-box regression deltas
pred_boxes = bbox_transform_inv(boxes, ops_value['bbox_deltas'][mi])
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))
return {'scores': cls_probs, 'boxes': pred_boxes, 'relations': rel_probs}
def im_detect(sess, net, inputs, im, boxes, bbox_reg, multi_iter, quadric_rois,
relations, labels):
blobs, im_scales = _get_blobs(im, boxes)
num_roi = blobs['rois'].shape[0]
num_rel = relations.shape[0]
inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)
#blobs['rois'] = blobs['rois'][:,[0,2,1,4,3]]
feed_dict = {
inputs['ims']:
blobs['data'],
inputs['rois']:
blobs['rois'],
inputs['relations']:
relations,
inputs['quadric_rois']:
quadric_rois,
inputs['labels']:
labels,
#inputs['rels_feat2d']: rels_feat2d,
#inputs['rels_feat3d']: rels_feat3d,
net.keep_prob:
1
}
for k in inputs_feed:
feed_dict[inputs[k]] = inputs_feed[k]
# compute relation rois
feed_dict[inputs['rel_rois']] = \
data_utils.compute_rel_rois(num_rel, blobs['rois'], relations)
ops = {}
ops['bbox_deltas'] = net.bbox_pred_output(multi_iter)
ops['rel_probs'] = net.rel_pred_output(multi_iter)
ops['cls_probs'] = net.cls_pred_output(multi_iter)
ops_value = sess.run(ops, feed_dict=feed_dict)
out_dict = {}
for mi in multi_iter:
rel_probs = None
rel_probs_flat = ops_value['rel_probs'][mi]
rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
for i, rel in enumerate(relations):
rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]
cls_probs = ops_value['cls_probs'][mi]
if bbox_reg:
# Apply bounding-box regression deltas
pred_boxes = bbox_transform_inv(boxes,
ops_value['bbox_deltas'][mi])
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))
out_dict[mi] = {
'scores': cls_probs.copy(),
'boxes': pred_boxes.copy(),
'relations': rel_probs.copy()
}
#import pdb; pdb.set_trace()
return out_dict
def run_batch(sess, net, inputs, outputs, ims, batch_boxes, batch_relations,
bbox_reg, multi_iter):
mi = multi_iter[-1]
results = list()
#t_start = time.time()
# Convert an image and RoIs within that image into network inputs.
im_scaled_list, rois_list = list(), list()
for im, boxes in zip(ims, batch_boxes):
im_scaled, im_scale_factors = _get_image_blob(im)
rois = _get_rois_blob(boxes, im_scale_factors)
im_scaled_list.append(im_scaled[0])
rois_list.append(rois)
conv_outs = sess.run(net.layers['conv_out'],
feed_dict={
inputs['ims']: np.stack(im_scaled_list, axis=0),
net.keep_prob: 1,
})
#print 'VGG takes', time.time() - t_start
#t_start = time.time()
for i in range(len(rois_list)):
conv_out, rois, relations = np.expand_dims(
conv_outs[i], axis=0), rois_list[i], batch_relations[i]
relations = np.array(relations,
dtype=np.int32) # all possible combinations
num_roi = rois.shape[0]
num_rel = relations.shape[0]
feed_dict = {
net.layers['conv_out']:
conv_out,
inputs['rois']:
rois,
inputs['relations']:
relations,
inputs['rel_rois']:
data_utils.compute_rel_rois(num_rel, rois, relations),
net.keep_prob:
1
}
inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)
for k in inputs_feed:
feed_dict[inputs[k]] = inputs_feed[k]
ops_value = sess.run(outputs, feed_dict=feed_dict)
rel_probs = None
rel_probs_flat = ops_value['rel_probs'][mi]
rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
for i, rel in enumerate(relations):
rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]
cls_probs = ops_value['cls_probs'][mi]
if bbox_reg:
# Apply bounding-box regression deltas
pred_boxes = bbox_transform_inv(boxes,
ops_value['bbox_deltas'][mi])
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))
results.append({
'scores': cls_probs,
'boxes': pred_boxes,
'relations': rel_probs
})
#print 'Scene takes', time.time() - t_start
return results
def im_detect(sess, net, inputs, im, boxes, relations, bbox_reg, multi_iter):
blobs, im_scales = _get_blobs(im, boxes)
relations = np.array(relations,
dtype=np.int32) # all possible combinations
num_roi = blobs['rois'].shape[0]
num_rel = relations.shape[0]
inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)
feed_dict = {
inputs['ims']: blobs['data'],
inputs['rois']: blobs['rois'],
inputs['relations']: relations,
net.keep_prob: 1
}
for k in inputs_feed:
feed_dict[inputs[k]] = inputs_feed[k]
# compute relation rois
feed_dict[inputs['rel_rois']] = \
data_utils.compute_rel_rois(num_rel, blobs['rois'], relations)
# Stage 1: VGG feature extraction.
#ops_vgg = {
# 'conv_out': net.layers['conv_out'],
#}
# Stage 2.
ops = {
'bbox_deltas': net.bbox_pred_output(multi_iter),
'rel_probs': net.rel_pred_output(multi_iter),
'cls_probs': net.cls_pred_output(multi_iter)
}
#t_start = time.time()
#ops_vgg_value = sess.run(ops_vgg, feed_dict={
# inputs['ims']: blobs['data'],
# net.keep_prob: 1,
#})
#print 'VGG takes', time.time() - t_start
#del feed_dict[inputs['ims']]
#feed_dict[ops_vgg['conv_out']] = ops_vgg_value['conv_out']
#t_start = time.time()
ops_value = sess.run(ops, feed_dict=feed_dict)
#print 'Scene takes', time.time() - t_start
out_dict = {}
for mi in multi_iter:
rel_probs = None
rel_probs_flat = ops_value['rel_probs'][mi]
rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
for i, rel in enumerate(relations):
rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]
cls_probs = ops_value['cls_probs'][mi]
if bbox_reg:
# Apply bounding-box regression deltas
pred_boxes = bbox_transform_inv(boxes,
ops_value['bbox_deltas'][mi])
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))
out_dict[mi] = {
'scores': cls_probs.copy(),
'boxes': pred_boxes.copy(),
'relations': rel_probs.copy()
}
return out_dict
def im_detect(sess, net, inputs, im, boxes, bbox_reg, multi_iter):
blobs, im_scales = _get_blobs(im, boxes)
im_orig = im.astype(np.float32, copy=True)
im_orig -= cfg.PIXEL_MEANS
im_shape = im_orig.shape
im_size_min = np.min(im_shape[0:2])
relations = []
num_box_per_image = int(boxes.shape[0] / 10)
for act_i in range(10):
curr_box = boxes[act_i * num_box_per_image:(act_i + 1) *
num_box_per_image]
for i in range(num_box_per_image):
for j in range(num_box_per_image):
# if i != j: # and i<j
x1 = curr_box[i][0]
y1 = curr_box[i][1]
x2 = curr_box[j][0]
y2 = curr_box[j][1]
if i < j and math.sqrt(
(x1 - x2)**2 + (y1 - y2)**2
) < im_size_min / 2 and i != num_box_per_image - 1 and j != num_box_per_image - 1:
relations.append([
i + act_i * num_box_per_image,
j + act_i * num_box_per_image
])
if len(relations) == 0:
relations.append([0, 1])
print(len(relations))
relations = np.array(relations,
dtype=np.int32) # all possible combinations
spa_relations = relations.copy()
num_roi = blobs['rois'].shape[0]
num_rel = relations.shape[0]
num_spa_rel = spa_relations.shape[0]
inputs_feed = data_utils.create_graph_data(num_roi, num_rel, relations)
feed_dict = {
inputs['ims']: blobs['data'],
inputs['rois']: blobs['rois'],
inputs['relations']: relations,
net.keep_prob: 1
}
for k in inputs_feed:
feed_dict[inputs[k]] = inputs_feed[k]
# compute relation rois
feed_dict[inputs['rel_rois']] = \
data_utils.compute_rel_rois(num_spa_rel, blobs['rois'], spa_relations)
ops = {}
# ops['bbox_deltas'] = net.bbox_pred_output(multi_iter)
ops['rel_probs'] = net.rel_pred_output(multi_iter)
ops['cls_probs'] = net.cls_pred_output(multi_iter)
ops['vert'] = net.getver()
ops_value = sess.run(ops, feed_dict=feed_dict)
out_dict = {}
for mi in multi_iter:
rel_probs = None
rel_probs_flat = ops_value['rel_probs'][mi]
rel_probs = np.zeros([num_roi, num_roi, rel_probs_flat.shape[1]])
for i, rel in enumerate(relations):
rel_probs[rel[0], rel[1], :] = rel_probs_flat[i, :]
cls_probs = ops_value['cls_probs'][mi]
if bbox_reg:
# Apply bounding-box regression deltas
pred_boxes = bbox_transform_inv(boxes,
ops_value['bbox_deltas'][mi])
pred_boxes = clip_boxes(pred_boxes, im.shape)
else:
# Simply repeat the boxes, once for each class
pred_boxes = np.tile(boxes, (1, cls_probs.shape[1]))
out_dict[mi] = {
'scores': cls_probs.copy(),
'boxes': pred_boxes.copy(),
'relations': rel_probs.copy(),
'vert': ops_value['vert'].copy()
}
return out_dict