def get_depth_coord_sample(labelweights, image_path, depth_path, label_path,
int_param, rotation, translation):
color_img = Image.open(image_path) # load color
color_img = np.array(color_img)
label_img = Image.open(label_path) # load label
label_img = np.array(label_img)
label_img = np.expand_dims(label_img, axis=-1)
# detph to camera xyz
depth_img = np.load(depth_path)['depth'] # load depth
assert color_img.shape[0:2] == label_img.shape[0:2] == depth_img.shape[0:2],\
'height, width not equal corlor {}, label {}, depth {}'\
.format(color_img.shape, label_img.shape, depth_img.shape)
height = depth_img.shape[0]
width = depth_img.shape[1]
#pixel_z = np.array(depth_img) / 1000.0
pixel_z = np.squeeze(depth_img, axis=-1)
valid_idx = (pixel_z != 0)
pixel_x, pixel_y = np.meshgrid(np.linspace(0, width, width, endpoint=False), \
np.linspace(0, height, height, endpoint=False))
cx = int_param[0, 2] * (width / (int_param[0, 2] * 2))
fx = int_param[0, 0] * (width / (int_param[0, 2] * 2))
cy = int_param[1, 2] * (height / (int_param[1, 2] * 2))
fy = int_param[1, 1] * (height / (int_param[1, 2] * 2))
pixel_x = pixel_z * (pixel_x - cx) / fx
pixel_y = pixel_z * (pixel_y - cy) / fy
pixel_x = np.expand_dims(pixel_x, axis=-1)
pixel_y = np.expand_dims(pixel_y, axis=-1)
pixel_z = np.expand_dims(pixel_z, axis=-1)
pixel_xyz = np.concatenate((pixel_x, pixel_y, pixel_z), axis=-1)
# depth world xyz without translation
coord_img = pixel_xyz.reshape(-1, 3)
valid_pts = np.where(coord_img[:, 2] != 0)[0]
#print(valid_pts)
coord_img = np.matmul(rotation, coord_img.T).T
coord_xy_center = np.mean(coord_img[valid_pts, 0:2], axis=0)
coord_img[valid_pts, 0:2] = coord_img[valid_pts, 0:2] - coord_xy_center
# align
x_vec = np.array([[1.], [0.], [0.]])
x_rotated = np.matmul(rotation, x_vec)
x_rotated[2, 0] = 0.
horizontal_shift = np.arccos(np.matmul(x_rotated.T, x_vec))
if x_rotated[1, 0] > 0:
align_z = eua.euler2mat(z=-horizontal_shift, y=0, x=0)
else:
align_z = eua.euler2mat(z=horizontal_shift, y=0, x=0)
coord_img = np.matmul(align_z, coord_img.T).T
coord_img[valid_pts, 2] = coord_img[valid_pts, 2] + translation[2]
# reshape to image shape
coord_img = coord_img.reshape(height, width, 3)
coord_img = np.concatenate([coord_img, color_img], axis=-1)
# coord weight
pixel_weight = labelweights[label_img.reshape(-1)] # get pixel weight
coord_weight = pixel_weight.reshape((height, width, 1)) * \
np.expand_dims(valid_idx.astype(np.int32), axis=-1)
#print('from depth', coord_img.shape, label_img.shape, coord_weight.shape, valid_idx.shape)
return coord_img, label_img, coord_weight, valid_idx
def draw_point_cloud(input_points, canvasSize=500, space=200, diameter=25,
xrot=0, yrot=0, zrot=0, switch_xyz=[0, 1, 2], normalize=True):
""" Render point cloud to image with alpha channel.
Input:
points: Nx3 numpy array (+y is up direction)
Output:
gray image as numpy array of size canvasSizexcanvasSize
"""
image = np.zeros((canvasSize, canvasSize))
if input_points is None or input_points.shape[0] == 0:
return image
points = input_points[:, switch_xyz]
M = euler2mat(zrot, yrot, xrot)
points = (np.dot(M, points.transpose())).transpose()
# Normalize the point cloud
# We normalize scale to fit points in a unit sphere
if normalize:
centroid = np.mean(points, axis=0)
points -= centroid
furthest_distance = np.max(np.sqrt(np.sum(abs(points) ** 2, axis=-1)))
points /= furthest_distance
# Pre-compute the Gaussian disk
radius = (diameter - 1) / 2.0
disk = np.zeros((diameter, diameter))
for i in range(diameter):
for j in range(diameter):
if (i - radius) * (i - radius) + (j - radius) * (j - radius) <= radius * radius:
disk[i, j] = np.exp((-(i - radius) ** 2 - (j - radius) ** 2) / (radius ** 2))
mask = np.argwhere(disk > 0)
dx = mask[:, 0]
dy = mask[:, 1]
dv = disk[disk > 0]
# Order points by z-buffer
zorder = np.argsort(points[:, 2])
points = points[zorder, :]
points[:, 2] = (points[:, 2] - np.min(points[:, 2])) / (np.max(points[:, 2] - np.min(points[:, 2])))
max_depth = np.max(points[:, 2])
for i in range(points.shape[0]):
j = points.shape[0] - i - 1
x = points[j, 0]
y = points[j, 1]
xc = canvasSize / 2 + (x * space)
yc = canvasSize / 2 + (y * space)
xc = int(np.round(xc))
yc = int(np.round(yc))
px = dx + xc
py = dy + yc
image[px, py] = image[px, py] * 0.7 + dv * (max_depth - points[j, 2]) * 0.3
image = image / np.max(image)
return image
def data_argment(train_data):
# Enriched Normailzation
# Rotate through X-axis
out_data = []
for xrot in (np.arange(-1, 1, 0.25) * math.pi):
M = euler2mat(0, xrot, 0)
out_data.append(np.dot(train_data, M.transpose()))
return np.concatenate(out_data)
def get_coord_sample(labelweights, image_path, pixcoord_path, label_path,
int_param, rotation, translation):
color_img = Image.open(image_path) # load color
color_img = np.array(color_img)
label_img = Image.open(label_path) # load label
label_img = np.array(label_img)
label_img = np.expand_dims(label_img, axis=-1)
# detph to camera xyz
pixcoord_img = np.load(pixcoord_path)['pixcoord'] # load depth
assert color_img.shape[0:2] == label_img.shape[0:2] == pixcoord_img.shape[0:2],\
'height, width not equal corlor {}, label {}, pixel coord {}'\
.format(color_img.shape, label_img.shape, pixcoord_img.shape)
height = pixcoord_img.shape[0]
width = pixcoord_img.shape[1]
#pixel_z = np.array(depth_img) / 1000.0
valid_idx = np.sum(pixcoord_img != np.array([0, 0, 0]), axis=-1) > 0
coord_img = pixcoord_img.reshape(-1, 3)
valid_pts = np.where(coord_img != np.array([0, 0, 0]))[0]
#coord_img[valid_idx] = coord_img[valid_idx] - translation
#coord_img = np.matmul(np.linalg.inv(rotation),coord_img[valid_idx].T)
# depth world xyz without translation
coord_xy_center = np.mean(coord_img[valid_pts, 0:2], axis=0)
coord_img[valid_pts, 0:2] = coord_img[valid_pts, 0:2] - coord_xy_center
# align
x_vec = np.array([[1.], [0.], [0.]])
x_rotated = np.matmul(rotation, x_vec)
x_rotated[2, 0] = 0.
horizontal_shift = np.arccos(np.matmul(x_rotated.T, x_vec))
if x_rotated[1, 0] > 0:
align_z = eua.euler2mat(z=-horizontal_shift, y=0, x=0)
else:
align_z = eua.euler2mat(z=horizontal_shift, y=0, x=0)
coord_img = np.matmul(align_z, coord_img.T).T
# reshape to image shape
coord_img = coord_img.reshape(height, width, 3)
coord_img = np.concatenate([coord_img, color_img], axis=-1)
# coord weight
pixel_weight = labelweights[label_img.reshape(-1)] # get pixel weight
coord_weight = pixel_weight.reshape((height, width, 1)) * \
np.expand_dims(valid_idx.astype(np.int32), axis=-1)
#print('from coord', coord_img.shape, label_img.shape, coord_weight.shape, valid_idx.shape)
return coord_img, label_img, coord_weight, valid_idx
def main():
opt = parse_config()
if opt.net_type == "default_residual":
opt.net_type = "default"
opt.hidden_size = [[
64, 64, 64, 128, 128, 128, 256, 256, 256, 512, 1024
], [512, 256, 256, 256, 64, 64, 64]]
elif opt.net_type == "default_big":
opt.net_type = "default"
opt.hidden_size = [[64, 128, 256, 512, 1024, 2048],
[1024, 512, 256, 64]]
opt.swap_axis = True
opt.do_evaluation = 0
np.random.seed(opt.seed)
if opt.cuda == 1:
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
# Prepare training data
categories = [
"toilet", "table", "sofa", "night_stand", "monitor", "dresser", "desk",
"chair", "bed", "bathtub"
]
train_data, train_labels, train_idx = [], [], []
for idx, cat in enumerate(categories):
d = np.load('data/%s_train.npy' % cat)
train_data.append(d)
train_labels.append(np.ones(d.shape[0]) * idx)
train_idx.append(np.array([idx * 10000 + i
for i in range(d.shape[0])]))
train_data = np.concatenate(train_data)
train_labels = np.concatenate(train_labels)
train_idx = np.concatenate(train_idx)
if opt.argment_mode:
print("Argment data by rotating through x axis. %d Data" %
len(train_data))
out_data = []
for xrot in (np.arange(-1, 1, 0.25) * math.pi):
M = euler2mat(0, xrot, 0)
out_data.append(np.dot(train_data, M.transpose()))
train_data = np.concatenate(out_data).astype(np.float32)
train_labels = train_labels.repeat(8, axis=0)
train_idx = train_idx.repeat(8, axis=0)
print("Argmented. %d Data" % len(train_data))
idx = np.random.permutation(train_data.shape[0])
train_data = train_data[idx, :opt.num_point]
train_labels = train_labels[idx]
train_idx = train_idx[idx]
np.save("%s/train_idx.npy" % opt.output_dir, idx)
test_data, test_labels = [], []
for idx, cat in enumerate(categories):
d = np.load('data/%s_test.npy' % cat)
test_data.append(d)
test_labels.append(np.ones(d.shape[0]) * idx)
test_data = np.concatenate(test_data)
test_labels = np.concatenate(test_labels)
idx = np.random.permutation(test_data.shape[0])
test_data = test_data[idx, :opt.num_point]
test_labels = test_labels[idx]
np.save("%s/train_labels.npy" % opt.output_dir, train_labels)
np.save("%s/test_labels.npy" % opt.output_dir, test_labels)
max_val = train_data.max()
min_val = train_data.min()
train_data = ((train_data - min_val) / (max_val - min_val)) * 2 - 1
test_data = ((test_data - min_val) / (max_val - min_val)) * 2 - 1
if opt.tpp and os.path.exists(opt.output_dir + "/train_feature.npy"):
train_feature = np.load(opt.output_dir + "/train_feature.npy")
test_feature = np.load(opt.output_dir + "/test_feature.npy")
print("pretrained data loaded.")
print(train_feature.shape)
if opt.robust_test:
model = load_model(opt)
else:
model = load_model(opt)
train_feature = extract_feature(opt, model, train_data)
test_feature = extract_feature(opt, model, test_data)
np.save(opt.output_dir + "/train_feature.npy", train_feature)
np.save(opt.output_dir + "/test_feature.npy", test_feature)
# voxel_mean = train_feature.mean()
# voxel_var = np.sqrt(train_feature.var())
# voxel_mean = np.mean(train_feature, 0, keepdims=True)
# voxel_var = np.sqrt(np.var(train_feature, 0, keepdims=True))
# train_feature = (train_feature - voxel_mean) / voxel_var
# test_feature = (test_feature - voxel_mean) / voxel_var
if opt.robust_test:
robust_test(opt, model, train_feature, train_labels, test_feature,
test_labels, test_data)
else:
train_svm_par(train_feature, train_labels, test_feature, test_labels)
def load_data(category, arg_mode):
cate_temp = category.split("_")[0]
train_data = []
if cate_temp == "modelnet40":
categories = [
'cup', 'bookshelf', 'lamp', 'stool', 'desk', 'toilet',
'night_stand', 'bowl', 'door', 'flower_pot', 'plant', 'stairs',
'bottle', 'mantel', 'sofa', 'laptop', 'xbox', 'tent', 'piano',
'car', 'wardrobe', 'tv_stand', 'cone', 'range_hood', 'bathtub',
'curtain', 'sink', 'glass_box', 'bed', 'chair', 'person', 'radio',
'dresser', 'bench', 'airplane', 'guitar', 'keyboard', 'table',
'monitor', 'vase'
]
for cat in categories:
d = np.load('data/%s_train.npy' % cat)
print(d.var(), d.mean(), d.max(), d.min())
train_data.append(d)
elif cate_temp == "modelnet10":
for cat in categories:
d = np.load('data/%s_train.npy' % cat)
print(d.var(), d.mean(), d.max(), d.min())
train_data.append(d)
elif cate_temp == "partnet":
partnet_base = "/home/fei960922/Documents/Dataset/PointCloud/partnet/sem_seg_h5"
if len(category.split("_")) > 1:
categories = [category.split("_")[1]]
else:
categories = os.listdir(partnet_base)
for cat in categories:
for name in [
s for s in os.listdir("%s/%s" % (partnet_base, cat))
if s[:3] == "tra" and s[-1] == "5"
]:
h5file = h5py.File("%s/%s/%s" % (partnet_base, cat, name))
d = np.asarray(h5file['data'])
# seg = h5file['label_seg']
print(d.shape, d.var(), d.mean(), d.max(), d.min())
train_data.append(d)
elif cate_temp == "shapenetpart":
pcd, label, seg = [], [], []
for i in range(10):
try:
f = h5py.File("data/hdf5_data/ply_data_train%d.h5" % i)
pcd.append(f['data'][:])
label.append(f['label'][:])
seg.append(f['pid'][:])
except:
break
train_data, train_label, train_seg = np.concatenate(
pcd), np.concatenate(label), np.concatenate(seg)
# data stored in list as a pickle file. range from [-1, 1]
# path = "data/shapenetpart_training.pkl"
# with open(path, "rb") as f:
# data = pickle.load(f)
if len(category.split("_")) > 1:
idx = int(category.split("_")[1])
train_data = train_data[train_label == idx]
elif cate_temp == "mnist":
# train_data = []
train_data = pickle.load(open("data/mnist_normal.pkl", "rb"))
# for pnt in temp_data:
# train_data.append(np.concatenate([pnt, np.zeros(([pnt.shape[0], 1]))], 1))
else:
train_data = [np.load('data/%s_train.npy' % category)]
if len(train_data[0].shape) == 3:
train_data = np.concatenate(train_data)
if arg_mode == 1:
print("Argment data by rotating through x axis. %d Data" %
len(train_data))
train_data = data_argment(train_data)
print("Argmented. %d Data" % len(train_data))
else:
# data stored in pickle
if arg_mode == 1:
print("Argment data by rotating through x axis. %d Data" %
len(train_data))
new_train_data = []
for pcd in train_data:
for xrot in (np.arange(-1, 1, 0.25) * math.pi):
M = euler2mat(0, xrot, 0)
new_train_data.append(np.dot(pcd, M.transpose()))
train_data = new_train_data
print("Argmented. %d Data" % len(train_data))
idx = np.random.permutation(len(train_data))
if type(train_data) is list:
train_data = [train_data[i] for i in idx]
else:
train_data = train_data[idx]
return train_data