def main_batch_from_rgb_detection(output_filename, result_dir=None):
ps_list = []
segp_list = []
center_list = []
heading_cls_list = []
heading_res_list = []
size_cls_list = []
size_res_list = []
rot_angle_list = []
score_list = []
test_idxs = np.arange(0, len(TEST_DATASET))
print(len(TEST_DATASET))
raw_input()
batch_size = 32
num_batches = int((len(TEST_DATASET)+batch_size-1)/batch_size)
batch_data_to_feed = np.zeros((batch_size, NUM_POINT, NUM_CHANNEL))
batch_one_hot_to_feed = np.zeros((batch_size, NUM_CLASS))
sess, ops = get_model(batch_size=batch_size, num_point=NUM_POINT)
for batch_idx in range(num_batches):
print(batch_idx)
start_idx = batch_idx * batch_size
end_idx = min(len(TEST_DATASET), (batch_idx+1) * batch_size)
cur_batch_size = end_idx - start_idx
batch_data, batch_rot_angle, batch_rgb_prob, batch_one_hot_vec = get_batch(TEST_DATASET, test_idxs, start_idx, end_idx, NUM_POINT, NUM_CHANNEL, from_rgb_detection=True)
batch_data_to_feed[0:cur_batch_size,...] = batch_data
batch_one_hot_to_feed[0:cur_batch_size,:] = batch_one_hot_vec
batch_output, batch_center_pred, batch_hclass_pred, batch_hres_pred, batch_sclass_pred, batch_sres_pred, batch_scores = inference(sess, ops, batch_data_to_feed, batch_one_hot_to_feed, batch_size=batch_size)
print(batch_hclass_pred.shape, batch_hres_pred.shape)
print(batch_sclass_pred.shape, batch_sres_pred.shape)
for i in range(cur_batch_size):
ps_list.append(batch_data[i,...])
segp_list.append(batch_output[i,...])
center_list.append(batch_center_pred[i,:])
heading_cls_list.append(batch_hclass_pred[i])
heading_res_list.append(batch_hres_pred[i])
size_cls_list.append(batch_sclass_pred[i])
size_res_list.append(batch_sres_pred[i,:])
rot_angle_list.append(batch_rot_angle[i])
#score_list.append(batch_scores[i] + np.log(batch_rgb_prob[i])) # Combine 3D BOX score and 2D RGB detection score
score_list.append(batch_rgb_prob[i]) # 2D RGB detection score
if FLAGS.dump_result:
save_zipped_pickle([ps_list, segp_list, center_list, heading_cls_list, heading_res_list, size_cls_list, size_res_list, rot_angle_list, score_list], output_filename)
# Write detection results for KITTI evaluation
print(len(ps_list))
raw_input()
write_detection_results(result_dir, TEST_DATASET.id_list, TEST_DATASET.type_list, TEST_DATASET.box2d_list, center_list, heading_cls_list, heading_res_list, size_cls_list, size_res_list, rot_angle_list, score_list)
# Make sure for each frame (no matter if we have measurment for that frame), there is a TXT file
output_dir = os.path.join(result_dir, 'data')
to_fill_filename_list = [line.rstrip()+'.txt' for line in open(FLAGS.idx_path)]
fill_files(output_dir, to_fill_filename_list)
def main_batch(output_filename, result_dir=None):
ps_list = []
seg_list = []
segp_list = []
center_list = []
heading_cls_list = []
heading_res_list = []
size_cls_list = []
size_res_list = []
rot_angle_list = []
score_list = []
test_idxs = np.arange(0, len(TEST_DATASET))
batch_size = 32
num_batches = int((len(TEST_DATASET)+batch_size-1)/batch_size)
batch_data_to_feed = np.zeros((batch_size, NUM_POINT, NUM_CHANNEL))
batch_one_hot_to_feed = np.zeros((batch_size, NUM_CLASS))
sess, ops = get_model(batch_size=batch_size, num_point=NUM_POINT)
correct_cnt = 0
for batch_idx in range(num_batches):
#if batch_idx == 50: break #TODO: remove this line!!
print(batch_idx)
start_idx = batch_idx * batch_size
end_idx = min(len(TEST_DATASET), (batch_idx+1) * batch_size)
cur_batch_size = end_idx - start_idx
batch_data, batch_label, batch_center, batch_hclass, batch_hres, batch_sclass, batch_sres, batch_rot_angle, batch_one_hot_vec = get_batch(TEST_DATASET, test_idxs, start_idx, end_idx, NUM_POINT, NUM_CHANNEL)
batch_data_to_feed[0:cur_batch_size,...] = batch_data
batch_one_hot_to_feed[0:cur_batch_size,:] = batch_one_hot_vec
batch_output, batch_center_pred, batch_hclass_pred, batch_hres_pred, batch_sclass_pred, batch_sres_pred, batch_scores = inference(sess, ops, batch_data_to_feed, batch_one_hot_to_feed, batch_size=batch_size)
print(batch_hclass_pred.shape, batch_hres_pred.shape)
print(batch_sclass_pred.shape, batch_sres_pred.shape)
#raw_input()
correct_cnt += np.sum(batch_output[0:cur_batch_size,...]==batch_label[0:cur_batch_size,...])
for i in range(cur_batch_size):
ps_list.append(batch_data[i,...])
seg_list.append(batch_label[i,...])
segp_list.append(batch_output[i,...])
center_list.append(batch_center_pred[i,:])
heading_cls_list.append(batch_hclass_pred[i])
heading_res_list.append(batch_hres_pred[i])
size_cls_list.append(batch_sclass_pred[i])
size_res_list.append(batch_sres_pred[i,:])
rot_angle_list.append(batch_rot_angle[i])
score_list.append(batch_scores[i])
print("Accuracy: ", correct_cnt / float(len(TEST_DATASET)*NUM_POINT))
save_zipped_pickle([ps_list, segp_list, center_list, heading_cls_list, heading_res_list, size_cls_list, size_res_list, rot_angle_list, score_list], output_filename)
# Write detection results for KITTI evaluation
write_detection_results(result_dir, TEST_DATASET.id_list, TEST_DATASET.type_list, TEST_DATASET.box2d_list, center_list, heading_cls_list, heading_res_list, size_cls_list, size_res_list, rot_angle_list, score_list)
def extract_roi_seg_from_rgb_detection(det_folder,
split,
output_filename,
viz,
valid_id_list=None,
type_whitelist=[
'bed', 'table', 'sofa', 'chair',
'toilet', 'desk', 'dresser',
'night_stand', 'bookshelf',
'bathtub'
]):
''' Extract data pairs for RoI point set segmentation from RGB detector outputed 2D boxes.
Input:
det_folder: contains files for each frame, lines in each file are type -1 -10 -10 xmin ymin xmax ymax ... prob
split: corresponding to official either trianing or testing
output_filename: the name for output .pickle file
valid_id_list: specify a list of valid image IDs
Output:
None (will write a .pickle file to the disk)
Usage: extract_roi_seg_from_rgb_detection("val_result_folder", "training", "roi_seg_val_rgb_detector_0908.pickle")
'''
dataset = sunrgbd_object(
'/home/haianyt/frustum-pointnets/sunrgbd/sunrgbd_data/matlab/SUNRGBDtoolbox/mysunrgbd',
split)
det_id_list, det_type_list, det_box2d_list, det_prob_list = read_det_folder(
det_folder)
cache_id = -1
cache = None
id_list = []
type_list = []
box2d_list = []
prob_list = []
input_list = [] # channel number = 4, xyz,intensity in rect camera coord
frustum_angle_list = [] # angle of 2d box center from pos x-axis
for det_idx in range(len(det_id_list)):
data_idx = det_id_list[det_idx]
if valid_id_list is not None and data_idx not in valid_id_list:
continue
print('det idx: %d/%d, data idx: %d' %
(det_idx, len(det_id_list), data_idx))
if cache_id != data_idx:
calib = dataset.get_calibration(data_idx)
pc_upright_depth = dataset.get_depth(data_idx)
pc_upright_camera = np.zeros_like(pc_upright_depth)
pc_upright_camera[:, 0:
3] = calib.project_upright_depth_to_upright_camera(
pc_upright_depth[:, 0:3])
pc_upright_camera[:, 3:] = pc_upright_depth[:, 3:]
img = dataset.get_image(data_idx)
img_height, img_width, img_channel = img.shape
pc_image_coord, _ = calib.project_upright_depth_to_image(
pc_upright_depth)
cache = [calib, pc_upright_camera, pc_image_coord]
cache_id = data_idx
else:
calib, pc_upright_camera, pc_image_coord = cache
#if det_type_list[det_idx]!='Car': continue
if det_type_list[det_idx] not in type_whitelist: continue
# 2D BOX: Get pts rect backprojected
xmin, ymin, xmax, ymax = det_box2d_list[det_idx]
box_fov_inds = (pc_image_coord[:, 0] < xmax) & (
pc_image_coord[:, 0] >= xmin) & (pc_image_coord[:, 1] < ymax) & (
pc_image_coord[:, 1] >= ymin)
pc_in_box_fov = pc_upright_camera[box_fov_inds, :]
# Get frustum angle (according to center pixel in 2D BOX)
box2d_center = np.array([(xmin + xmax) / 2.0, (ymin + ymax) / 2.0])
uvdepth = np.zeros((1, 3))
uvdepth[0, 0:2] = box2d_center
uvdepth[0, 2] = 20 # some random depth
box2d_center_upright_camera = calib.project_image_to_upright_camerea(
uvdepth)
frustum_angle = -1 * np.arctan2(
box2d_center_upright_camera[0, 2], box2d_center_upright_camera[
0, 0]) # angle as to positive x-axis as in the Zoox paper
# Subsample points..
num_point = pc_in_box_fov.shape[0]
if num_point > 2048:
choice = np.random.choice(pc_in_box_fov.shape[0],
2048,
replace=False)
pc_in_box_fov = pc_in_box_fov[choice, :]
# Pass objects that are too small
if len(pc_in_box_fov) < 5:
continue
id_list.append(data_idx)
type_list.append(det_type_list[det_idx])
box2d_list.append(det_box2d_list[det_idx])
prob_list.append(det_prob_list[det_idx])
input_list.append(pc_in_box_fov)
frustum_angle_list.append(frustum_angle)
utils.save_zipped_pickle([
id_list, box2d_list, input_list, type_list, frustum_angle_list,
prob_list
], output_filename)
if viz:
for i in range(10):
p1 = input_list[i]
fig = mlab.figure(figure=None,
bgcolor=(0.4, 0.4, 0.4),
fgcolor=None,
engine=None,
size=(500, 500))
mlab.points3d(p1[:, 0],
p1[:, 1],
p1[:, 2],
p1[:, 1],
mode='point',
colormap='gnuplot',
scale_factor=1,
figure=fig)
fig = mlab.figure(figure=None,
bgcolor=(0.4, 0.4, 0.4),
fgcolor=None,
engine=None,
size=(500, 500))
mlab.points3d(p1[:, 2],
-p1[:, 0],
-p1[:, 1],
seg,
mode='point',
colormap='gnuplot',
scale_factor=1,
figure=fig)
raw_input()
def extract_roi_seg(idx_filename,
split,
output_filename,
viz,
perturb_box2d=False,
augmentX=1,
type_whitelist=[
'bed', 'table', 'sofa', 'chair', 'toilet', 'desk',
'dresser', 'night_stand', 'bookshelf', 'bathtub'
]):
dataset = sunrgbd_object(
'/home/haianyt/frustum-pointnets/sunrgbd/sunrgbd_data/matlab/SUNRGBDtoolbox/mysunrgbd',
split)
data_idx_list = [int(line.rstrip()) for line in open(idx_filename)]
id_list = [] # int number
box2d_list = [] # [xmin,ymin,xmax,ymax]
box3d_list = [] # (8,3) array in upright depth coord
input_list = [] # channel number = 6, xyz,rgb in upright depth coord
label_list = [] # 1 for roi object, 0 for clutter
type_list = [] # string e.g. bed
heading_list = [
] # face of object angle, radius of clockwise angle from positive x axis in upright camera coord
box3d_size_list = [] # array of l,w,h
frustum_angle_list = [
] # angle of 2d box center from pos x-axis (clockwise)
pos_cnt = 0
all_cnt = 0
#debuglimit = 10
for data_idx in data_idx_list:
# debuglimit -=1
# if debuglimit <=0:
# break
print('------------- ', data_idx)
calib = dataset.get_calibration(data_idx)
objects = dataset.get_label_objects(data_idx)
pc_upright_depth = dataset.get_depth(data_idx)
pc_upright_camera = np.zeros_like(pc_upright_depth)
pc_upright_camera[:,
0:3] = calib.project_upright_depth_to_upright_camera(
pc_upright_depth[:, 0:3])
pc_upright_camera[:, 3:] = pc_upright_depth[:, 3:]
if viz:
mlab.points3d(pc_upright_camera[:, 0],
pc_upright_camera[:, 1],
pc_upright_camera[:, 2],
pc_upright_camera[:, 1],
mode='point')
mlab.orientation_axes()
raw_input()
img = dataset.get_image(data_idx)
img_height, img_width, img_channel = img.shape
pc_image_coord, _ = calib.project_upright_depth_to_image(
pc_upright_depth)
#print('PC image coord: ', pc_image_coord)
for obj_idx in range(len(objects)):
obj = objects[obj_idx]
if obj.classname not in type_whitelist: continue
print("object pass the type_whitelist augmentX = ", augmentX)
# 2D BOX: Get pts rect backprojected
box2d = obj.box2d
for _ in range(augmentX):
# Augment data by box2d perturbation
if perturb_box2d:
xmin, ymin, xmax, ymax = random_shift_box2d(box2d)
print(xmin, ymin, xmax, ymax)
else:
xmin, ymin, xmax, ymax = box2d
box_fov_inds = (pc_image_coord[:, 0] <
xmax) & (pc_image_coord[:, 0] >= xmin) & (
pc_image_coord[:, 1] <
ymax) & (pc_image_coord[:, 1] >= ymin)
pc_in_box_fov = pc_upright_camera[box_fov_inds, :]
# Get frustum angle (according to center pixel in 2D BOX)
box2d_center = np.array([(xmin + xmax) / 2.0,
(ymin + ymax) / 2.0])
uvdepth = np.zeros((1, 3))
uvdepth[0, 0:2] = box2d_center
uvdepth[0, 2] = 20 # some random depth
box2d_center_upright_camera = calib.project_image_to_upright_camerea(
uvdepth)
print('UVdepth, center in upright camera: ', uvdepth,
box2d_center_upright_camera)
frustum_angle = -1 * np.arctan2(
box2d_center_upright_camera[0, 2],
box2d_center_upright_camera[0, 0]
) # angle as to positive x-axis as in the Zoox paper
print('Frustum angle: ', frustum_angle)
# 3D BOX: Get pts velo in 3d box
box3d_pts_2d, box3d_pts_3d = utils.compute_box_3d(obj, calib)
box3d_pts_3d = calib.project_upright_depth_to_upright_camera(
box3d_pts_3d)
_, inds = extract_pc_in_box3d(pc_in_box_fov, box3d_pts_3d)
print(len(inds))
label = np.zeros((pc_in_box_fov.shape[0]))
label[inds] = 1
# Get 3D BOX heading
print('Orientation: ', obj.orientation)
print('Heading angle: ', obj.heading_angle)
# Get 3D BOX size
box3d_size = np.array([2 * obj.l, 2 * obj.w, 2 * obj.h])
print('Box3d size: ', box3d_size)
print('Type: ', obj.classname)
print('Num of point: ', pc_in_box_fov.shape[0])
# Subsample points..
num_point = pc_in_box_fov.shape[0]
if num_point > 2048:
choice = np.random.choice(pc_in_box_fov.shape[0],
2048,
replace=False)
pc_in_box_fov = pc_in_box_fov[choice, :]
label = label[choice]
# Reject object with too few points
if np.sum(label) < 5:
continue
id_list.append(data_idx)
box2d_list.append(np.array([xmin, ymin, xmax, ymax]))
box3d_list.append(box3d_pts_3d)
input_list.append(pc_in_box_fov)
label_list.append(label)
type_list.append(obj.classname)
heading_list.append(obj.heading_angle)
box3d_size_list.append(box3d_size)
frustum_angle_list.append(frustum_angle)
# collect statistics
pos_cnt += np.sum(label)
all_cnt += pc_in_box_fov.shape[0]
# VISUALIZATION
if viz:
img2 = np.copy(img)
cv2.rectangle(img2, (int(obj.xmin), int(obj.ymin)),
(int(obj.xmax), int(obj.ymax)), (0, 255, 0),
2)
utils.draw_projected_box3d(img2, box3d_pts_2d)
Image.fromarray(img2).show()
p1 = input_list[-1]
seg = label_list[-1]
fig = mlab.figure(figure=None,
bgcolor=(0.4, 0.4, 0.4),
fgcolor=None,
engine=None,
size=(500, 500))
mlab.points3d(p1[:, 0],
p1[:, 1],
p1[:, 2],
seg,
mode='point',
colormap='gnuplot',
scale_factor=1,
figure=fig)
mlab.points3d(0,
0,
0,
color=(1, 1, 1),
mode='sphere',
scale_factor=0.2)
draw_gt_boxes3d([box3d_pts_3d], fig=fig)
mlab.orientation_axes()
raw_input()
print('Average pos ratio: ', pos_cnt / float(all_cnt))
print('Average npoints: ', float(all_cnt) / len(id_list))
utils.save_zipped_pickle([
id_list, box2d_list, box3d_list, input_list, label_list, type_list,
heading_list, box3d_size_list, frustum_angle_list
], output_filename)
def extract_roi_seg(idx_filename,
split,
output_filename,
viz,
perturb_box2d=False,
augmentX=1,
type_whitelist=[
'bed', 'table', 'sofa', 'chair', 'toilet', 'desk',
'dresser', 'night_stand', 'bookshelf', 'bathtub'
]):
print('Extracting roi_seg from: %s' % idx_filename)
dataset = sunrgbd_object(SUNRGBD_DATASET_DIR, split)
data_idx_list = [int(line.rstrip()) for line in open(idx_filename)]
id_list = [] # int number
box2d_list = [] # [xmin,ymin,xmax,ymax]
box3d_list = [] # (8,3) array in upright depth coord
image_list = [
] # (h_i, w_i, 3) - different height and width for each image
input_list = [] # channel number = 6, xyz,rgb in upright depth coord
label_list = [] # 1 for roi object, 0 for clutter
type_list = [] # string e.g. bed
heading_list = [
] # face of object angle, radius of clockwise angle from positive x axis in upright camera coord
box3d_size_list = [] # array of l,w,h
rtilts_list = [] # (3,3) array used for projection onto image
ks_list = [] # (3,3) array used for projection onto image
frustum_angle_list = [
] # angle of 2d box center from pos x-axis (clockwise)
img_dims_list = [] # dimensions of the full images (not just the object)
pos_cnt = 0
all_cnt = 0
for data_idx in data_idx_list:
print('------------- ', data_idx)
calib, objects, pc_upright_camera, img, pc_image_coord = load_data(
dataset, data_idx)
#print('PC image coord: ', pc_image_coord)
for obj_idx in range(len(objects)):
obj = objects[obj_idx]
if obj.classname not in type_whitelist: continue
# 2D BOX: Get pts rect backprojected
for _ in range(augmentX):
#try:
box2d, box3d_pts_3d, cropped_image, pc_in_box_fov, label, box3d_size, frustum_angle, img_dims = \
process_object(calib, obj, pc_upright_camera, img, pc_image_coord, perturb_box2d=perturb_box2d)
# Reject object with too few points
if np.sum(label) < 5:
continue
id_list.append(data_idx)
box2d_list.append(box2d)
box3d_list.append(box3d_pts_3d)
image_list.append(cropped_image)
input_list.append(pc_in_box_fov)
label_list.append(label)
type_list.append(obj.classname)
heading_list.append(obj.heading_angle)
box3d_size_list.append(box3d_size)
rtilts_list.append(calib.Rtilt)
ks_list.append(calib.K)
frustum_angle_list.append(frustum_angle)
img_dims_list.append(img_dims)
# collect statistics
pos_cnt += np.sum(label)
all_cnt += pc_in_box_fov.shape[0]
print('Average pos ratio: ', pos_cnt / float(all_cnt))
print('Average npoints: ', float(all_cnt) / len(id_list))
utils.save_zipped_pickle([id_list, box2d_list, box3d_list, image_list, input_list, label_list, \
type_list, heading_list, box3d_size_list, rtilts_list, ks_list, \
frustum_angle_list, img_dims_list], output_filename)
def extract_roi_seg_from_rgb_detection(det_folder, split, output_filename, viz, valid_id_list=None, type_whitelist=['bed','table','sofa','chair','toilet','desk','dresser','night_stand','bookshelf','bathtub']):
''' Extract data pairs for RoI point set segmentation from RGB detector outputed 2D boxes.
Input:
det_folder: contains files for each frame, lines in each file are type -1 -10 -10 xmin ymin xmax ymax ... prob
split: corresponding to official either trianing or testing
output_filename: the name for output .pickle file
valid_id_list: specify a list of valid image IDs
Output:
None (will write a .pickle file to the disk)
Usage: extract_roi_seg_from_rgb_detection("val_result_folder", "training", "roi_seg_val_rgb_detector_0908.pickle")
'''
dataset = sunrgbd_object('/home/rqi/Data/mysunrgbd', split)
det_id_list, det_type_list, det_box2d_list, det_prob_list = read_det_folder(det_folder)
cache_id = -1
cache = None
id_list = []
type_list = []
box2d_list = []
prob_list = []
input_list = [] # channel number = 4, xyz,intensity in rect camera coord
frustum_angle_list = [] # angle of 2d box center from pos x-axis
for det_idx in range(len(det_id_list)):
data_idx = det_id_list[det_idx]
if valid_id_list is not None and data_idx not in valid_id_list: continue
print('det idx: %d/%d, data idx: %d' % (det_idx, len(det_id_list), data_idx))
if cache_id != data_idx:
calib = dataset.get_calibration(data_idx)
pc_upright_depth = dataset.get_depth(data_idx)
pc_upright_camera = np.zeros_like(pc_upright_depth)
pc_upright_camera[:,0:3] = calib.project_upright_depth_to_upright_camera(pc_upright_depth[:,0:3])
pc_upright_camera[:,3:] = pc_upright_depth[:,3:]
img = dataset.get_image(data_idx)
img_height, img_width, img_channel = img.shape
pc_image_coord,_ = calib.project_upright_depth_to_image(pc_upright_depth)
cache = [calib,pc_upright_camera,pc_image_coord]
cache_id = data_idx
else:
calib,pc_upright_camera,pc_image_coord = cache
#if det_type_list[det_idx]!='Car': continue
if det_type_list[det_idx] not in type_whitelist: continue
# 2D BOX: Get pts rect backprojected
xmin,ymin,xmax,ymax = det_box2d_list[det_idx]
box_fov_inds = (pc_image_coord[:,0]<xmax) & (pc_image_coord[:,0]>=xmin) & (pc_image_coord[:,1]<ymax) & (pc_image_coord[:,1]>=ymin)
pc_in_box_fov = pc_upright_camera[box_fov_inds,:]
# Get frustum angle (according to center pixel in 2D BOX)
box2d_center = np.array([(xmin+xmax)/2.0, (ymin+ymax)/2.0])
uvdepth = np.zeros((1,3))
uvdepth[0,0:2] = box2d_center
uvdepth[0,2] = 20 # some random depth
box2d_center_upright_camera = calib.project_image_to_upright_camerea(uvdepth)
frustum_angle = -1 * np.arctan2(box2d_center_upright_camera[0,2], box2d_center_upright_camera[0,0]) # angle as to positive x-axis as in the Zoox paper
# Subsample points..
num_point = pc_in_box_fov.shape[0]
if num_point > 2048:
choice = np.random.choice(pc_in_box_fov.shape[0], 2048, replace=False)
pc_in_box_fov = pc_in_box_fov[choice,:]
# Pass objects that are too small
if len(pc_in_box_fov)<5:
continue
id_list.append(data_idx)
type_list.append(det_type_list[det_idx])
box2d_list.append(det_box2d_list[det_idx])
prob_list.append(det_prob_list[det_idx])
input_list.append(pc_in_box_fov)
frustum_angle_list.append(frustum_angle)
utils.save_zipped_pickle([id_list, box2d_list, input_list, type_list, frustum_angle_list, prob_list], output_filename)
if viz:
for i in range(10):
p1 = input_list[i]
fig = mlab.figure(figure=None, bgcolor=(0.4,0.4,0.4), fgcolor=None, engine=None, size=(500, 500))
mlab.points3d(p1[:,0], p1[:,1], p1[:,2], p1[:,1], mode='point', colormap='gnuplot', scale_factor=1, figure=fig)
fig = mlab.figure(figure=None, bgcolor=(0.4,0.4,0.4), fgcolor=None, engine=None, size=(500, 500))
mlab.points3d(p1[:,2], -p1[:,0], -p1[:,1], seg, mode='point', colormap='gnuplot', scale_factor=1, figure=fig)
raw_input()
def extract_roi_seg(idx_filename, split, output_filename, viz, perturb_box2d=False, augmentX=1, type_whitelist=['bed','table','sofa','chair','toilet','desk','dresser','night_stand','bookshelf','bathtub']):
dataset = sunrgbd_object('/home/rqi/Data/mysunrgbd', split)
data_idx_list = [int(line.rstrip()) for line in open(idx_filename)]
id_list = [] # int number
box2d_list = [] # [xmin,ymin,xmax,ymax]
box3d_list = [] # (8,3) array in upright depth coord
input_list = [] # channel number = 6, xyz,rgb in upright depth coord
label_list = [] # 1 for roi object, 0 for clutter
type_list = [] # string e.g. bed
heading_list = [] # face of object angle, radius of clockwise angle from positive x axis in upright camera coord
box3d_size_list = [] # array of l,w,h
frustum_angle_list = [] # angle of 2d box center from pos x-axis (clockwise)
pos_cnt = 0
all_cnt = 0
for data_idx in data_idx_list:
print('------------- ', data_idx)
calib = dataset.get_calibration(data_idx)
objects = dataset.get_label_objects(data_idx)
pc_upright_depth = dataset.get_depth(data_idx)
pc_upright_camera = np.zeros_like(pc_upright_depth)
pc_upright_camera[:,0:3] = calib.project_upright_depth_to_upright_camera(pc_upright_depth[:,0:3])
pc_upright_camera[:,3:] = pc_upright_depth[:,3:]
if viz:
mlab.points3d(pc_upright_camera[:,0], pc_upright_camera[:,1], pc_upright_camera[:,2], pc_upright_camera[:,1], mode='point')
mlab.orientation_axes()
raw_input()
img = dataset.get_image(data_idx)
img_height, img_width, img_channel = img.shape
pc_image_coord,_ = calib.project_upright_depth_to_image(pc_upright_depth)
#print('PC image coord: ', pc_image_coord)
for obj_idx in range(len(objects)):
obj = objects[obj_idx]
if obj.classname not in type_whitelist: continue
# 2D BOX: Get pts rect backprojected
box2d = obj.box2d
for _ in range(augmentX):
try:
# Augment data by box2d perturbation
if perturb_box2d:
xmin,ymin,xmax,ymax = random_shift_box2d(box2d)
print(xmin,ymin,xmax,ymax)
else:
xmin,ymin,xmax,ymax = box2d
box_fov_inds = (pc_image_coord[:,0]<xmax) & (pc_image_coord[:,0]>=xmin) & (pc_image_coord[:,1]<ymax) & (pc_image_coord[:,1]>=ymin)
pc_in_box_fov = pc_upright_camera[box_fov_inds,:]
# Get frustum angle (according to center pixel in 2D BOX)
box2d_center = np.array([(xmin+xmax)/2.0, (ymin+ymax)/2.0])
uvdepth = np.zeros((1,3))
uvdepth[0,0:2] = box2d_center
uvdepth[0,2] = 20 # some random depth
box2d_center_upright_camera = calib.project_image_to_upright_camerea(uvdepth)
print('UVdepth, center in upright camera: ', uvdepth, box2d_center_upright_camera)
frustum_angle = -1 * np.arctan2(box2d_center_upright_camera[0,2], box2d_center_upright_camera[0,0]) # angle as to positive x-axis as in the Zoox paper
print('Frustum angle: ', frustum_angle)
# 3D BOX: Get pts velo in 3d box
box3d_pts_2d, box3d_pts_3d = utils.compute_box_3d(obj, calib)
box3d_pts_3d = calib.project_upright_depth_to_upright_camera(box3d_pts_3d)
_,inds = extract_pc_in_box3d(pc_in_box_fov, box3d_pts_3d)
print(len(inds))
label = np.zeros((pc_in_box_fov.shape[0]))
label[inds] = 1
# Get 3D BOX heading
print('Orientation: ', obj.orientation)
print('Heading angle: ', obj.heading_angle)
# Get 3D BOX size
box3d_size = np.array([2*obj.l,2*obj.w,2*obj.h])
print('Box3d size: ', box3d_size)
print('Type: ', obj.classname)
print('Num of point: ', pc_in_box_fov.shape[0])
# Subsample points..
num_point = pc_in_box_fov.shape[0]
if num_point > 2048:
choice = np.random.choice(pc_in_box_fov.shape[0], 2048, replace=False)
pc_in_box_fov = pc_in_box_fov[choice,:]
label = label[choice]
# Reject object with too few points
if np.sum(label) < 5:
continue
id_list.append(data_idx)
box2d_list.append(np.array([xmin,ymin,xmax,ymax]))
box3d_list.append(box3d_pts_3d)
input_list.append(pc_in_box_fov)
label_list.append(label)
type_list.append(obj.classname)
heading_list.append(obj.heading_angle)
box3d_size_list.append(box3d_size)
frustum_angle_list.append(frustum_angle)
# collect statistics
pos_cnt += np.sum(label)
all_cnt += pc_in_box_fov.shape[0]
# VISUALIZATION
if viz:
img2 = np.copy(img)
cv2.rectangle(img2, (int(obj.xmin),int(obj.ymin)), (int(obj.xmax),int(obj.ymax)), (0,255,0), 2)
utils.draw_projected_box3d(img2, box3d_pts_2d)
Image.fromarray(img2).show()
p1 = input_list[-1]
seg = label_list[-1]
fig = mlab.figure(figure=None, bgcolor=(0.4,0.4,0.4), fgcolor=None, engine=None, size=(500, 500))
mlab.points3d(p1[:,0], p1[:,1], p1[:,2], seg, mode='point', colormap='gnuplot', scale_factor=1, figure=fig)
mlab.points3d(0, 0, 0, color=(1,1,1), mode='sphere', scale_factor=0.2)
draw_gt_boxes3d([box3d_pts_3d], fig=fig)
mlab.orientation_axes()
raw_input()
except:
pass
print('Average pos ratio: ', pos_cnt/float(all_cnt))
print('Average npoints: ', float(all_cnt)/len(id_list))
utils.save_zipped_pickle([id_list,box2d_list,box3d_list,input_list,label_list,type_list,heading_list,box3d_size_list,frustum_angle_list],output_filename)
def main_batch(test_dataset,
test_classes,
num_class,
num_point,
num_channel,
prefix='',
semi_type=None,
use_boxpc_fit_prob=False,
sess_ops=None,
output_filename=None,
result_dir=None,
verbose=False):
ps_list = []
seg_gt_list = []
seg_pred_list = []
center_list = []
heading_cls_list = []
heading_res_list = []
size_cls_list = []
size_res_list = []
rot_angle_list = []
score_list = []
cls_list = []
file_num_list = []
box2d_list = []
box3d_list = []
test_idxs = np.arange(0, len(test_dataset))
batch_size = 32
num_batches = int((len(test_dataset) + batch_size - 1) / batch_size)
batch_data_to_feed = np.zeros((batch_size, num_point, num_channel))
batch_one_hot_to_feed = np.zeros((batch_size, num_class))
if sess_ops is None:
sess, ops = get_model(batch_size=batch_size,
num_point=num_point,
num_channel=num_channel)
else:
sess, ops = sess_ops
idx = 0
iou_sum = 0
for batch_idx in range(num_batches):
if verbose and batch_idx % 10 == 0: print(batch_idx)
start_idx = batch_idx * batch_size
end_idx = min(len(test_dataset), (batch_idx + 1) * batch_size)
cur_batch_size = end_idx - start_idx
batch_data, batch_image, batch_label, batch_center, batch_hclass, batch_hres, \
batch_sclass, batch_sres, batch_box2d, batch_rtilt, batch_k, batch_rot_angle, \
batch_img_dims, batch_one_hot_vec = \
test_dataset.get_batch(test_idxs, start_idx, end_idx, num_point, num_channel)
batch_data_to_feed[0:cur_batch_size, ...] = batch_data
batch_one_hot_to_feed[0:cur_batch_size, :] = batch_one_hot_vec
batch_output, batch_center_pred, batch_hclass_pred, batch_hres_pred, batch_sclass_pred, \
batch_sres_pred, batch_scores = inference(sess, ops, batch_data_to_feed, \
batch_one_hot_to_feed,
use_boxpc_fit_prob=use_boxpc_fit_prob,
batch_size=batch_size,
prefix=prefix)
# The point cloud has been duplicated, we only want to calculate the IOU
# on the points that are not duplicated
# correct_cnt += np.sum(batch_output==batch_label)
for i, (point_cloud, seg_pred,
seg_gt) in enumerate(zip(batch_data, batch_output,
batch_label)):
if start_idx + i >= len(test_dataset.idx_l):
continue
_, unique_idx = np.unique(point_cloud, axis=0, return_index=True)
y_pred = seg_pred[unique_idx]
y_true = seg_gt[unique_idx]
iou = float(
np.sum(y_pred & y_true)) / (np.sum(y_pred | y_true) + 1)
iou_sum += iou
for i in range(cur_batch_size):
ps_list.append(batch_data[i, ...])
seg_gt_list.append(batch_label[i, ...])
seg_pred_list.append(batch_output[i, ...])
center_list.append(batch_center_pred[i, :])
heading_cls_list.append(batch_hclass_pred[i])
heading_res_list.append(batch_hres_pred[i])
size_cls_list.append(batch_sclass_pred[i])
size_res_list.append(batch_sres_pred[i, :])
rot_angle_list.append(batch_rot_angle[i])
score_list.append(batch_scores[i])
cls_list.append(np.argmax(batch_one_hot_vec[i]))
file_num_list.append(test_dataset.idx_l[idx])
box2d_list.append(test_dataset.box2d_l[idx])
box3d_list.append(test_dataset.box3d_l[idx])
idx += 1
print("Mean segmentation IOU: %f" % (iou_sum / len(test_dataset.idx_l)))
predictions = [ps_list, seg_gt_list, seg_pred_list, center_list, heading_cls_list,
heading_res_list, size_cls_list, size_res_list, rot_angle_list, \
score_list, cls_list, file_num_list, box2d_list, box3d_list]
if output_filename is not None:
save_zipped_pickle(predictions, output_filename)
if result_dir is not None:
# Write detection results for MATLAB evaluation
write_detection_results(result_dir, test_classes, test_dataset.idx_l, test_dataset.cls_type_l, \
test_dataset.box2d_l, center_list, heading_cls_list, heading_res_list, size_cls_list, \
size_res_list, rot_angle_list, score_list)
return predictions
def main_batch_from_rgb_detection(test_dataset,
test_classes,
num_class,
num_point,
num_channel,
prefix='',
semi_type=None,
use_boxpc_fit_prob=False,
use_oracle_mask=False,
sess_ops=None,
output_filename=None,
result_dir=None,
verbose=False):
ps_list = []
image_list = []
segp_list = []
center_list = []
heading_cls_list = []
heading_res_list = []
size_cls_list = []
size_res_list = []
rot_angle_list = []
score_list = []
cls_list = []
file_num_list = []
box2d_list = []
test_idxs = np.arange(0, len(test_dataset))
batch_size = 32
num_batches = int((len(test_dataset) + batch_size - 1) / batch_size)
batch_data_to_feed = np.zeros((batch_size, num_point, num_channel))
batch_one_hot_to_feed = np.zeros((batch_size, num_class))
batch_oracle_mask_to_feed = np.zeros((batch_size, num_point))
if sess_ops is None:
sess, ops = get_model(batch_size=batch_size,
num_point=num_point,
num_channel=num_channel,
use_oracle_mask=use_oracle_mask)
else:
sess, ops = sess_ops
idx = 0
for batch_idx in range(num_batches):
if verbose and batch_idx % 10 == 0: print(batch_idx)
start_idx = batch_idx * batch_size
end_idx = min(len(test_dataset), (batch_idx + 1) * batch_size)
cur_batch_size = end_idx - start_idx
batch_data, batch_image, batch_rot_angle, batch_rgb_prob, batch_one_hot_vec, \
batch_oracle_y_seg = \
test_dataset.get_batch(test_idxs, start_idx, end_idx, num_point, num_channel,
from_rgb_detection=True)
batch_data_to_feed[0:cur_batch_size, ...] = batch_data
batch_one_hot_to_feed[0:cur_batch_size, :] = batch_one_hot_vec
if use_oracle_mask:
batch_oracle_mask_to_feed[0:cur_batch_size,
...] = batch_oracle_y_seg
else:
batch_oracle_mask_to_feed = None
batch_output, batch_center_pred, batch_hclass_pred, batch_hres_pred, batch_sclass_pred, \
batch_sres_pred, batch_scores = inference(sess, ops, batch_data_to_feed, \
batch_one_hot_to_feed,
use_boxpc_fit_prob=use_boxpc_fit_prob,
oracle_mask=batch_oracle_mask_to_feed,
batch_size=batch_size,
prefix=prefix)
for i in range(cur_batch_size):
ps_list.append(batch_data[i, ...])
segp_list.append(batch_output[i, ...])
center_list.append(batch_center_pred[i, :])
heading_cls_list.append(batch_hclass_pred[i])
heading_res_list.append(batch_hres_pred[i])
size_cls_list.append(batch_sclass_pred[i])
size_res_list.append(batch_sres_pred[i, :])
rot_angle_list.append(batch_rot_angle[i])
# Combine 3D BOX score and 2D RGB detection score
# score_list.append(batch_scores[i] + np.log(batch_rgb_prob[i]))
score_list.append(batch_rgb_prob[i]) # 2D RGB detection score
cls_list.append(np.argmax(batch_one_hot_vec[i]))
file_num_list.append(test_dataset.idx_l[idx])
box2d_list.append(test_dataset.box2d_l[idx])
idx += 1
predictions = [ps_list, None, segp_list, center_list, heading_cls_list, \
heading_res_list, size_cls_list, size_res_list, rot_angle_list, \
score_list, cls_list, file_num_list, box2d_list, None]
if output_filename is not None:
save_zipped_pickle(predictions, output_filename)
if result_dir is not None:
# Write detection results for MATLAB evaluation
write_detection_results(result_dir, test_classes, test_dataset.idx_l, test_dataset.cls_type_l, \
test_dataset.box2d_l, center_list, heading_cls_list, heading_res_list, size_cls_list, \
size_res_list, rot_angle_list, score_list)
return predictions
for age, task_idx in enumerate(range(nb_tasks)):
print("Age is {}".format(age))
X_train, y_train = training_data[task_idx]
cl.pre_task_updates()
cl.fit(X_train, y_train, protocol['epochs_per_task'])
ftask = []
for X_valid, y_valid in valid_data[:task_idx + 1]:
f_ = cl.evaluate(X_valid, y_valid)
ftask.append(f_)
print(np.mean(ftask))
evals.append(ftask)
next_task = True if task_idx != nb_tasks - 1 else False
cl.post_task_updates(X_train, y_train, next_task=next_task)
return evals, tmp_evals
total_evals = []
evals, tmp_evals = run_fits(training_dataset, validation_dataset)
print([np.mean(e) for e in evals])
print([np.mean(e) for e in tmp_evals])
utils.save_zipped_pickle(evals, datafile_name)
evals = utils.load_zipped_pickle(datafile_name)
