def config_datasets(data):
train_path = data["train_path"]
valid_path = data["valid_path"]
minibatch_size = int(data["minibatch_size"])
rgb_noise = float(data["data_augmentation"]["rgb_noise"])
depth_noise = float(data["data_augmentation"]["depth_noise"])
occluder_path = data["data_augmentation"]["occluder_path"]
background_path = data["data_augmentation"]["background_path"]
blur_noise = int(data["data_augmentation"]["blur_noise"])
h_noise = float(data["data_augmentation"]["h_noise"])
s_noise = float(data["data_augmentation"]["s_noise"])
v_noise = float(data["data_augmentation"]["v_noise"])
channel_hide = data["data_augmentation"]["channel_hide"] == "True"
data_augmentation = DataAugmentation()
data_augmentation.set_rgb_noise(rgb_noise)
data_augmentation.set_depth_noise(depth_noise)
if occluder_path != "":
data_augmentation.set_occluder(occluder_path)
if background_path != "":
data_augmentation.set_background(background_path)
if channel_hide:
data_augmentation.set_channel_hide(0.25)
data_augmentation.set_blur(blur_noise)
data_augmentation.set_hsv_noise(h_noise, s_noise, v_noise)
message_logger.info("Setup Train : {}".format(train_path))
train_dataset = Dataset(train_path, minibatch_size=minibatch_size)
if not train_dataset.load():
message_logger.error("Train dataset empty")
sys.exit(-1)
train_dataset.set_data_augmentation(data_augmentation)
train_dataset.compute_mean_std()
message_logger.info("Computed mean : {}\nComputed Std : {}".format(
train_dataset.mean, train_dataset.std))
message_logger.info("Setup Valid : {}".format(valid_path))
valid_dataset = Dataset(valid_path,
minibatch_size=minibatch_size,
max_samples=20000)
if not valid_dataset.load():
message_logger.error("Valid dataset empty")
sys.exit(-1)
valid_dataset.set_data_augmentation(data_augmentation)
valid_dataset.mean = train_dataset.mean
valid_dataset.std = train_dataset.std
return train_dataset, valid_dataset
def config_datasets(data):
train_path = data["train_path"]
valid_path = data["valid_path"]
minibatch_size = int(data["minibatch_size"])
rgb_noise = float(data["data_augmentation"]["rgb_noise"])
depth_noise = float(data["data_augmentation"]["depth_noise"])
occluder_path = data["data_augmentation"]["occluder_path"]
background_path = data["data_augmentation"]["background_path"]
blur_noise = int(data["data_augmentation"]["blur_noise"])
hue_noise = float(data["data_augmentation"]["hue_noise"])
data_augmentation = DataAugmentation()
data_augmentation.set_rgb_noise(rgb_noise)
data_augmentation.set_depth_noise(depth_noise)
if occluder_path != "":
data_augmentation.set_occluder(occluder_path)
if background_path != "":
data_augmentation.set_background(background_path)
data_augmentation.set_blur(blur_noise)
data_augmentation.set_hue_noise(hue_noise)
train_dataset = Dataset(train_path, minibatch_size=minibatch_size)
if not train_dataset.load():
message_logger.error("Train dataset empty")
sys.exit(-1)
train_dataset.set_data_augmentation(data_augmentation)
train_dataset.compute_mean_std()
message_logger.info("Computed mean : {}\nComputed Std : {}".format(train_dataset.mean, train_dataset.std))
valid_dataset = Dataset(valid_path, minibatch_size=minibatch_size)
if not valid_dataset.load():
message_logger.error("Valid dataset empty")
sys.exit(-1)
valid_dataset.set_data_augmentation(data_augmentation)
valid_dataset.mean = train_dataset.mean
valid_dataset.std = train_dataset.std
return train_dataset, valid_dataset
"""
Sanity tests for dataset folder
- Make sure all images in viewpoints.json are in the folder
...
"""
from deeptracking.data.dataset import Dataset
import sys
if __name__ == '__main__':
dataset_path = "/home/mathieu/Dataset/DeepTrack/skull"
dataset = Dataset(dataset_path)
if not dataset.load():
print("[Error]: Train dataset empty")
sys.exit(-1)
# check if all viewpoints are there
for frame, pose in dataset.data_pose:
if not frame.exists(dataset.path):
print("[Error]: Missing pose frame {}".format(frame.id))
sys.exit(-1)
# check if all pairs are there
for key, value in dataset.data_pair.items():
for frame, pose in value:
if not frame.exists(dataset.path):
print("[Error]: Missing pair frame {}".format(frame.id))
"object_width": "250"
}]
MODELS_3D = models
SHADER_PATH = "/home/mathieu/source/deeptracking/deeptracking/data/shaders"
OBJECT_WIDTH = int(MODELS_3D[0]["object_width"])
MODEL_3D_PATH = MODELS_3D[0]["model_path"]
try:
MODEL_3D_AO_PATH = MODELS_3D[0]["ambiant_occlusion_model"]
except KeyError:
MODEL_3D_AO_PATH = None
frame_download_path = None
video_data = Dataset(VIDEO_PATH)
if not video_data.load():
print("[ERROR] Error while loading video...")
sys.exit(-1)
frame_download_path = video_data.path
# Makes the list a generator for compatibility with sensor's generator
gen = lambda alist: [(yield i) for i in alist]
frame_generator = gen(video_data.data_pose)
camera = video_data.camera
# Renderer
window = InitOpenGL(camera.width, camera.height)
vpRender = ModelRenderer(MODEL_3D_PATH, SHADER_PATH, camera, window,
(camera.width, camera.height))
vpRender.load_ambiant_occlusion_map(MODEL_3D_AO_PATH)
for i, (current_frame, ground_truth_pose) in enumerate(frame_generator):
REAL_PATH = data["real_path"]
OUTPUT_PATH = data["output_path"]
SAMPLE_QUANTITY = int(data["sample_quantity"])
TRANSLATION_RANGE = float(data["translation_range"])
ROTATION_RANGE = math.radians(float(data["rotation_range"]))
SPHERE_MIN_RADIUS = float(data["sphere_min_radius"])
SPHERE_MAX_RADIUS = float(data["sphere_max_radius"])
IMAGE_SIZE = (int(data["image_size"]), int(data["image_size"]))
PRELOAD = data["preload"] == "True"
SATURATION_THRESHOLD = int(data["saturation_threshold"])
if not os.path.exists(OUTPUT_PATH):
os.mkdir(OUTPUT_PATH)
real_dataset = Dataset(REAL_PATH)
real_dataset.load()
camera = Camera.load_from_json(real_dataset.path)
real_dataset.camera = camera
output_dataset = Dataset(OUTPUT_PATH, frame_class=data["save_type"])
output_dataset.camera = camera
window_size = (real_dataset.camera.width, real_dataset.camera.height)
window = InitOpenGL(*window_size)
model = MODELS[0]
vpRender = ModelRenderer(model["model_path"], SHADER_PATH,
real_dataset.camera, window, window_size)
vpRender.load_ambiant_occlusion_map(model["ambiant_occlusion_model"])
OBJECT_WIDTH = int(model["object_width"])
metadata = {}
metadata["translation_range"] = str(TRANSLATION_RANGE)
class DataAugmentation:
def __init__(self):
self.occluder = None
self.background = None
self.rgb_noise = None
self.depth_noise = None
self.blur_kernel = None
self.jitter = None
self.hue_noise = None
def set_background(self, path):
self.background = RGBDDataset(path)
def set_occluder(self, path):
self.occluder = Dataset(path)
self.occluder.load()
def set_rgb_noise(self, gaussian_std):
self.rgb_noise = gaussian_std
def set_depth_noise(self, gaussian_std):
self.depth_noise = gaussian_std
def set_hue_noise(self, offset):
"""
offset is the % of random hue offset distribution
:param offset:
:return:
"""
self.hue_noise = offset
def set_blur(self, size):
self.blur_kernel = size
def set_jitter(self, max_x, max_y):
self.jitter = (max_x, max_y)
def augment(self, rgb, depth, prior, real=False):
ret_rgb = rgb
ret_depth = depth
if real and self.occluder:
if random.uniform(0, 1) < 0.75:
rand_id = random.randint(0, self.occluder.size() - 1)
occluder_rgb, occluder_depth, occ_pose = self.occluder.load_image(rand_id)
if random.randint(0, 1):
occluder_rgb, occluder_depth, _ = self.occluder.load_pair(rand_id, 0)
occluder_depth = occluder_depth.astype(np.float32)
# Z offset of occluder to be closer to the occluded object ( with random distance in front of the object)
offset = -occ_pose.matrix[2, 3] + prior.matrix[2, 3] - random.uniform(0.07, 0.01)
occluder_depth += offset
occluder_rgb = self.add_hue_noise(occluder_rgb, 1)
occluder_rgb = imresize(occluder_rgb, ret_depth.shape, interp='nearest')
occluder_depth = imresize(occluder_depth, ret_depth.shape, interp='nearest', mode="F").astype(np.int16)
ret_rgb, ret_depth = self.depth_blend(ret_rgb, ret_depth, occluder_rgb, occluder_depth)
if real and self.hue_noise:
if random.uniform(0, 1) > 0.05:
ret_rgb = self.add_hue_noise(ret_rgb, self.hue_noise)
if self.jitter:
self.x_jitter = random.randint(-self.jitter[0], self.jitter[0])
self.y_jitter = random.randint(-self.jitter[1], self.jitter[1])
if self.x_jitter > 0:
ret_rgb = np.pad(ret_rgb, ((self.x_jitter, 0), (0, 0), (0, 0)), mode='constant')[:-self.x_jitter, :, :]
ret_depth = np.pad(ret_depth, ((self.x_jitter, 0), (0, 0)), mode='constant')[:-self.x_jitter, :]
else:
ret_rgb = np.pad(ret_rgb, ((0, abs(self.x_jitter)), (0, 0), (0, 0)), mode='constant')[
abs(self.x_jitter):, :, :]
ret_depth = np.pad(ret_depth, ((0, abs(self.x_jitter)), (0, 0)), mode='constant')[abs(self.x_jitter):,
:]
if self.y_jitter > 0:
ret_rgb = np.pad(ret_rgb, ((0, 0), (self.y_jitter, 0), (0, 0)), mode='constant')[:, :-self.y_jitter, :]
ret_depth = np.pad(ret_depth, ((0, 0), (self.y_jitter, 0)), mode='constant')[:, :-self.y_jitter]
else:
ret_rgb = np.pad(ret_rgb, ((0, 0), (0, abs(self.y_jitter)), (0, 0)), mode='constant')[:,
abs(self.y_jitter):, :]
ret_depth = np.pad(ret_depth, ((0, 0), (0, abs(self.y_jitter))), mode='constant')[:,
abs(self.y_jitter):]
if real and self.background:
color_background, depth_background = self.background.load_random_image(ret_rgb.shape[1])
depth_background = depth_background.astype(np.int32)
ret_rgb, ret_depth = self.color_blend(ret_rgb, ret_depth, color_background, depth_background)
if real and self.rgb_noise:
if random.uniform(0, 1) > 0.05:
noise = random.uniform(0, self.rgb_noise)
ret_rgb = self.add_noise(ret_rgb, noise)
if real and self.depth_noise:
if random.uniform(0, 1) > 0.05:
noise = random.uniform(0, self.depth_noise)
ret_depth = self.add_noise(ret_depth, noise)
if real and self.blur_kernel is not None:
if random.uniform(0, 1) < 0.75:
kernel_size = random.randint(3, self.blur_kernel)
kernel = self.gkern(kernel_size)
ret_rgb[0, :, :] = scipy.signal.convolve2d(ret_rgb[0, :, :], kernel, mode='same')
ret_rgb[1, :, :] = scipy.signal.convolve2d(ret_rgb[1, :, :], kernel, mode='same')
ret_rgb[2, :, :] = scipy.signal.convolve2d(ret_rgb[2, :, :], kernel, mode='same')
if random.uniform(0, 1) < 0.75:
kernel_size = random.randint(3, self.blur_kernel)
kernel = self.gkern(kernel_size)
ret_depth[:, :] = scipy.signal.convolve2d(ret_depth[:, :], kernel, mode='same')
return ret_rgb.astype(np.uint8), ret_depth
@staticmethod
def add_noise(img, gaussian_std):
type = img.dtype
copy = img.astype(np.float)
gaussian_noise = np.random.normal(0, gaussian_std, img.shape)
copy = (gaussian_noise + copy)
if type == np.uint8:
copy[copy < 0] = 0
copy[copy > 255] = 255
return copy.astype(type)
@staticmethod
def add_hue_noise(rgb, hue_offset):
hsv = rgb2hsv(rgb)
hsv[:, :, 0] = (hsv[:, :, 0] + random.uniform(-hue_offset, hue_offset)) % 1
rgb = hsv2rgb(hsv) * 255
return rgb.astype(np.uint8)
@staticmethod
def color_blend(rgb1, depth1, rgb2, depth2):
mask = np.all(rgb1 == 0, axis=2)
mask = ndimage.binary_dilation(mask).astype(mask.dtype)
depth1[mask] = 0
rgb1[mask, :] = 0
mask = mask.astype(np.uint8)
new_depth = depth2 * mask + depth1
new_color = rgb2 * mask[:, :, np.newaxis] + rgb1
return new_color.astype(np.uint8), new_depth
@staticmethod
def depth_blend(rgb1, depth1, rgb2, depth2):
new_depth2 = depth2.copy()
new_depth1 = depth1.copy()
rgb1_mask = np.all(rgb1 == 0, axis=2)
rgb2_mask = np.all(rgb2 == 0, axis=2)
rgb1_mask = ndimage.binary_dilation(rgb1_mask)
new_depth2[rgb2_mask] = -100000
new_depth1[rgb1_mask] = -100000
mask = (new_depth1 < new_depth2)
pos_mask = mask.astype(np.uint8)
neg_mask = (mask == False).astype(np.uint8)
masked_rgb_occluder = rgb1 * pos_mask[:, :, np.newaxis]
masked_rgb_object = rgb2 * neg_mask[:, :, np.newaxis]
masked_depth_occluder = depth1 * pos_mask
masked_depth_object = depth2 * neg_mask
blend_rgb = masked_rgb_occluder + masked_rgb_object
blend_depth = masked_depth_occluder + masked_depth_object
return blend_rgb, blend_depth
@staticmethod
def gkern(kernlen=21, nsig=3.5):
"""Returns a 2D Gaussian kernel array."""
interval = (2 * nsig + 1.) / (kernlen)
x = np.linspace(-nsig - interval / 2., nsig + interval / 2., kernlen + 1)
kern1d = np.diff(st.norm.cdf(x))
kernel_raw = np.sqrt(np.outer(kern1d, kern1d))
kernel = kernel_raw / kernel_raw.sum()
return kernel
window = InitOpenGL(camera.width, camera.height)
detector = ArucoDetector(camera, DETECTOR_PATH)
vpRender = ModelRenderer(MODELS[0]["model_path"], SHADER_PATH, camera,
window, (camera.width, camera.height))
vpRender.load_ambiant_occlusion_map(MODELS[0]["ambiant_occlusion_model"])
cv2.namedWindow('image')
cv2.createTrackbar('transparency', 'image', 0, 100, trackbar)
# todo, read from file?
detection_offset = Transform()
rgbd_record = False
save_next_rgbd_pose = False
lock_offset = False
if PRELOAD:
dataset.load()
offset_path = os.path.join(dataset.path, "offset.npy")
if os.path.exists(offset_path):
detection_offset = Transform.from_matrix(np.load(offset_path))
lock_offset = True
while True:
start_time = time.time()
bgr, depth = sensor.get_frame()
bgr = cv2.resize(bgr, (int(1920 / ratio), int(1080 / ratio)))
depth = cv2.resize(depth, (int(1920 / ratio), int(1080 / ratio)))
screen = bgr.copy()
if rgbd_record:
# here we add a dummy pose, we will compute the pose as a post operation
dataset.add_pose(bgr, depth, Transform())
metadata["save_type"] = data["save_type"]
metadata["object_width"] = {}
for model in MODELS:
metadata["object_width"][model["name"]] = str(model["object_width"])
metadata["min_radius"] = str(SPHERE_MIN_RADIUS)
metadata["max_radius"] = str(SPHERE_MAX_RADIUS)
camera = Camera.load_from_json(data["camera_path"])
dataset = Dataset(OUTPUT_PATH, frame_class=data["save_type"])
dataset.camera = camera
window_size = (camera.width, camera.height)
window = InitOpenGL(*window_size)
sphere_sampler = UniformSphereSampler(SPHERE_MIN_RADIUS, SPHERE_MAX_RADIUS)
preload_count = 0
if PRELOAD:
if dataset.load():
preload_count = dataset.size()
print("This Dataset already contains {} samples".format(
preload_count))
# Iterate over all models from config files
for model in MODELS:
vpRender = ModelRenderer(model["model_path"], SHADER_PATH,
dataset.camera, window, window_size)
vpRender.load_ambiant_occlusion_map(model["ambiant_occlusion_model"])
OBJECT_WIDTH = int(model["object_width"])
for i in tqdm(range(SAMPLE_QUANTITY - preload_count)):
random_pose = sphere_sampler.get_random()
random_transform = Transform.random(
(-TRANSLATION_RANGE, TRANSLATION_RANGE),
(-ROTATION_RANGE, ROTATION_RANGE))
pair = combine_view_transform(random_pose, random_transform)
import numpy as np
import matplotlib.pyplot as plt
from deeptracking.data.dataset import Dataset
from deeptracking.data.dataaugmentation import DataAugmentation
if __name__ == '__main__':
object_path = "/home/mathieu/Dataset/DeepTrack/skull/train_cyclegan"
occluder_path = "/home/mathieu/Dataset/DeepTrack/mixed/test"
background_path = "/home/mathieu/Dataset/RGBD/SUN3D"
object_dataset = Dataset(object_path)
object_dataset.load()
data_augmentation = DataAugmentation()
data_augmentation.set_rgb_noise(2)
data_augmentation.set_depth_noise(2)
data_augmentation.set_hue_noise(0.07)
data_augmentation.set_occluder(occluder_path)
data_augmentation.set_background(background_path)
data_augmentation.set_blur(5)
# data_augmentation.set_jitter(20, 20)
for i in range(object_dataset.size()):
rgb, depth, pose = object_dataset.load_image(i)
rgb, depth, label = object_dataset.load_pair(i, 0)
rgb_augmented, depth_augmented = data_augmentation.augment(
rgb, depth, pose, True)
plt.figure(0)
plt.imshow(rgb)