class TestDatasetMethods(unittest.TestCase): @classmethod def setUpClass(cls): # run these once as they take time cls.dummy_rgb = np.zeros((150, 150, 3), dtype=np.float32) cls.dummy_depth = np.zeros((150, 150), dtype=np.float32) cls.dummy_pose = Transform() def setUp(self): self.populated_dataset = Dataset("data") for i in range(10): self.populated_dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) def tearDown(self): pass def test_it_should_have_size_0_at_init(self): dataset = Dataset("data") self.assertEqual(dataset.size(), 0) def test_it_should_add_sample(self): dataset = Dataset("data") dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) self.assertEqual(dataset.size(), 1) def test_it_should_return_index_after_adding_pose(self): dataset = Dataset("data") index0 = dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) self.assertEqual(index0, 0) index1 = dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) self.assertEqual(index1, 1) def test_it_return_0_if_no_pair(self): self.assertEqual(self.populated_dataset.pair_size(1), 0) def test_it_should_add_pair(self): self.populated_dataset.add_pair(self.dummy_rgb, self.dummy_depth, self.dummy_pose, 1) self.assertEqual(self.populated_dataset.pair_size(1), 1) self.populated_dataset.add_pair(self.dummy_rgb, self.dummy_depth, self.dummy_pose, 1) self.assertEqual(self.populated_dataset.pair_size(1), 2) def test_it_should_raise_indexerror_if_pose_id_does_not_exists(self): self.assertRaises(IndexError, self.populated_dataset.add_pair, self.dummy_rgb, self.dummy_depth, self.dummy_pose, 20) self.assertRaises(IndexError, self.populated_dataset.add_pair, self.dummy_rgb, self.dummy_depth, self.dummy_pose, 10)
"/home/mathieu/Dataset/3D_models/skull/skull_ao.ply", "object_width": "250" }] MODELS_3D = models SHADER_PATH = "/home/mathieu/source/deeptracking/deeptracking/data/shaders" OBJECT_WIDTH = int(MODELS_3D[0]["object_width"]) MODEL_3D_PATH = MODELS_3D[0]["model_path"] try: MODEL_3D_AO_PATH = MODELS_3D[0]["ambiant_occlusion_model"] except KeyError: MODEL_3D_AO_PATH = None frame_download_path = None video_data = Dataset(VIDEO_PATH) if not video_data.load(): print("[ERROR] Error while loading video...") sys.exit(-1) frame_download_path = video_data.path # Makes the list a generator for compatibility with sensor's generator gen = lambda alist: [(yield i) for i in alist] frame_generator = gen(video_data.data_pose) camera = video_data.camera # Renderer window = InitOpenGL(camera.width, camera.height) vpRender = ModelRenderer(MODEL_3D_PATH, SHADER_PATH, camera, window, (camera.width, camera.height)) vpRender.load_ambiant_occlusion_map(MODEL_3D_AO_PATH)
SHADER_PATH = data["shader_path"] REAL_PATH = data["real_path"] OUTPUT_PATH = data["output_path"] SAMPLE_QUANTITY = int(data["sample_quantity"]) TRANSLATION_RANGE = float(data["translation_range"]) ROTATION_RANGE = math.radians(float(data["rotation_range"])) SPHERE_MIN_RADIUS = float(data["sphere_min_radius"]) SPHERE_MAX_RADIUS = float(data["sphere_max_radius"]) IMAGE_SIZE = (int(data["image_size"]), int(data["image_size"])) PRELOAD = data["preload"] == "True" SATURATION_THRESHOLD = int(data["saturation_threshold"]) if not os.path.exists(OUTPUT_PATH): os.mkdir(OUTPUT_PATH) real_dataset = Dataset(REAL_PATH) real_dataset.load() camera = Camera.load_from_json(real_dataset.path) real_dataset.camera = camera output_dataset = Dataset(OUTPUT_PATH, frame_class=data["save_type"]) output_dataset.camera = camera window_size = (real_dataset.camera.width, real_dataset.camera.height) window = InitOpenGL(*window_size) model = MODELS[0] vpRender = ModelRenderer(model["model_path"], SHADER_PATH, real_dataset.camera, window, window_size) vpRender.load_ambiant_occlusion_map(model["ambiant_occlusion_model"]) OBJECT_WIDTH = int(model["object_width"]) metadata = {}
def test_it_should_return_index_after_adding_pose(self): dataset = Dataset("data") index0 = dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) self.assertEqual(index0, 0) index1 = dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) self.assertEqual(index1, 1)
def test_it_should_add_sample(self): dataset = Dataset("data") dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose) self.assertEqual(dataset.size(), 1)
def test_it_should_have_size_0_at_init(self): dataset = Dataset("data") self.assertEqual(dataset.size(), 0)
def setUp(self): self.populated_dataset = Dataset("data") for i in range(10): self.populated_dataset.add_pose(self.dummy_rgb, self.dummy_depth, self.dummy_pose)
import numpy as np import matplotlib.pyplot as plt from deeptracking.data.dataset import Dataset from deeptracking.data.dataaugmentation import DataAugmentation if __name__ == '__main__': object_path = "/home/mathieu/Dataset/DeepTrack/skull/train_cyclegan" occluder_path = "/home/mathieu/Dataset/DeepTrack/mixed/test" background_path = "/home/mathieu/Dataset/RGBD/SUN3D" object_dataset = Dataset(object_path) object_dataset.load() data_augmentation = DataAugmentation() data_augmentation.set_rgb_noise(2) data_augmentation.set_depth_noise(2) data_augmentation.set_hue_noise(0.07) data_augmentation.set_occluder(occluder_path) data_augmentation.set_background(background_path) data_augmentation.set_blur(5) # data_augmentation.set_jitter(20, 20) for i in range(object_dataset.size()): rgb, depth, pose = object_dataset.load_image(i) rgb, depth, label = object_dataset.load_pair(i, 0) rgb_augmented, depth_augmented = data_augmentation.augment( rgb, depth, pose, True) plt.figure(0) plt.imshow(rgb)
def set_occluder(self, path): self.occluder = Dataset(path) self.occluder.load()
class DataAugmentation: def __init__(self): self.occluder = None self.background = None self.rgb_noise = None self.depth_noise = None self.blur_kernel = None self.jitter = None self.hue_noise = None def set_background(self, path): self.background = RGBDDataset(path) def set_occluder(self, path): self.occluder = Dataset(path) self.occluder.load() def set_rgb_noise(self, gaussian_std): self.rgb_noise = gaussian_std def set_depth_noise(self, gaussian_std): self.depth_noise = gaussian_std def set_hue_noise(self, offset): """ offset is the % of random hue offset distribution :param offset: :return: """ self.hue_noise = offset def set_blur(self, size): self.blur_kernel = size def set_jitter(self, max_x, max_y): self.jitter = (max_x, max_y) def augment(self, rgb, depth, prior, real=False): ret_rgb = rgb ret_depth = depth if real and self.occluder: if random.uniform(0, 1) < 0.75: rand_id = random.randint(0, self.occluder.size() - 1) occluder_rgb, occluder_depth, occ_pose = self.occluder.load_image(rand_id) if random.randint(0, 1): occluder_rgb, occluder_depth, _ = self.occluder.load_pair(rand_id, 0) occluder_depth = occluder_depth.astype(np.float32) # Z offset of occluder to be closer to the occluded object ( with random distance in front of the object) offset = -occ_pose.matrix[2, 3] + prior.matrix[2, 3] - random.uniform(0.07, 0.01) occluder_depth += offset occluder_rgb = self.add_hue_noise(occluder_rgb, 1) occluder_rgb = imresize(occluder_rgb, ret_depth.shape, interp='nearest') occluder_depth = imresize(occluder_depth, ret_depth.shape, interp='nearest', mode="F").astype(np.int16) ret_rgb, ret_depth = self.depth_blend(ret_rgb, ret_depth, occluder_rgb, occluder_depth) if real and self.hue_noise: if random.uniform(0, 1) > 0.05: ret_rgb = self.add_hue_noise(ret_rgb, self.hue_noise) if self.jitter: self.x_jitter = random.randint(-self.jitter[0], self.jitter[0]) self.y_jitter = random.randint(-self.jitter[1], self.jitter[1]) if self.x_jitter > 0: ret_rgb = np.pad(ret_rgb, ((self.x_jitter, 0), (0, 0), (0, 0)), mode='constant')[:-self.x_jitter, :, :] ret_depth = np.pad(ret_depth, ((self.x_jitter, 0), (0, 0)), mode='constant')[:-self.x_jitter, :] else: ret_rgb = np.pad(ret_rgb, ((0, abs(self.x_jitter)), (0, 0), (0, 0)), mode='constant')[ abs(self.x_jitter):, :, :] ret_depth = np.pad(ret_depth, ((0, abs(self.x_jitter)), (0, 0)), mode='constant')[abs(self.x_jitter):, :] if self.y_jitter > 0: ret_rgb = np.pad(ret_rgb, ((0, 0), (self.y_jitter, 0), (0, 0)), mode='constant')[:, :-self.y_jitter, :] ret_depth = np.pad(ret_depth, ((0, 0), (self.y_jitter, 0)), mode='constant')[:, :-self.y_jitter] else: ret_rgb = np.pad(ret_rgb, ((0, 0), (0, abs(self.y_jitter)), (0, 0)), mode='constant')[:, abs(self.y_jitter):, :] ret_depth = np.pad(ret_depth, ((0, 0), (0, abs(self.y_jitter))), mode='constant')[:, abs(self.y_jitter):] if real and self.background: color_background, depth_background = self.background.load_random_image(ret_rgb.shape[1]) depth_background = depth_background.astype(np.int32) ret_rgb, ret_depth = self.color_blend(ret_rgb, ret_depth, color_background, depth_background) if real and self.rgb_noise: if random.uniform(0, 1) > 0.05: noise = random.uniform(0, self.rgb_noise) ret_rgb = self.add_noise(ret_rgb, noise) if real and self.depth_noise: if random.uniform(0, 1) > 0.05: noise = random.uniform(0, self.depth_noise) ret_depth = self.add_noise(ret_depth, noise) if real and self.blur_kernel is not None: if random.uniform(0, 1) < 0.75: kernel_size = random.randint(3, self.blur_kernel) kernel = self.gkern(kernel_size) ret_rgb[0, :, :] = scipy.signal.convolve2d(ret_rgb[0, :, :], kernel, mode='same') ret_rgb[1, :, :] = scipy.signal.convolve2d(ret_rgb[1, :, :], kernel, mode='same') ret_rgb[2, :, :] = scipy.signal.convolve2d(ret_rgb[2, :, :], kernel, mode='same') if random.uniform(0, 1) < 0.75: kernel_size = random.randint(3, self.blur_kernel) kernel = self.gkern(kernel_size) ret_depth[:, :] = scipy.signal.convolve2d(ret_depth[:, :], kernel, mode='same') return ret_rgb.astype(np.uint8), ret_depth @staticmethod def add_noise(img, gaussian_std): type = img.dtype copy = img.astype(np.float) gaussian_noise = np.random.normal(0, gaussian_std, img.shape) copy = (gaussian_noise + copy) if type == np.uint8: copy[copy < 0] = 0 copy[copy > 255] = 255 return copy.astype(type) @staticmethod def add_hue_noise(rgb, hue_offset): hsv = rgb2hsv(rgb) hsv[:, :, 0] = (hsv[:, :, 0] + random.uniform(-hue_offset, hue_offset)) % 1 rgb = hsv2rgb(hsv) * 255 return rgb.astype(np.uint8) @staticmethod def color_blend(rgb1, depth1, rgb2, depth2): mask = np.all(rgb1 == 0, axis=2) mask = ndimage.binary_dilation(mask).astype(mask.dtype) depth1[mask] = 0 rgb1[mask, :] = 0 mask = mask.astype(np.uint8) new_depth = depth2 * mask + depth1 new_color = rgb2 * mask[:, :, np.newaxis] + rgb1 return new_color.astype(np.uint8), new_depth @staticmethod def depth_blend(rgb1, depth1, rgb2, depth2): new_depth2 = depth2.copy() new_depth1 = depth1.copy() rgb1_mask = np.all(rgb1 == 0, axis=2) rgb2_mask = np.all(rgb2 == 0, axis=2) rgb1_mask = ndimage.binary_dilation(rgb1_mask) new_depth2[rgb2_mask] = -100000 new_depth1[rgb1_mask] = -100000 mask = (new_depth1 < new_depth2) pos_mask = mask.astype(np.uint8) neg_mask = (mask == False).astype(np.uint8) masked_rgb_occluder = rgb1 * pos_mask[:, :, np.newaxis] masked_rgb_object = rgb2 * neg_mask[:, :, np.newaxis] masked_depth_occluder = depth1 * pos_mask masked_depth_object = depth2 * neg_mask blend_rgb = masked_rgb_occluder + masked_rgb_object blend_depth = masked_depth_occluder + masked_depth_object return blend_rgb, blend_depth @staticmethod def gkern(kernlen=21, nsig=3.5): """Returns a 2D Gaussian kernel array.""" interval = (2 * nsig + 1.) / (kernlen) x = np.linspace(-nsig - interval / 2., nsig + interval / 2., kernlen + 1) kern1d = np.diff(st.norm.cdf(x)) kernel_raw = np.sqrt(np.outer(kern1d, kern1d)) kernel = kernel_raw / kernel_raw.sum() return kernel
""" Resize all frames of a dataset """ from deeptracking.data.dataset import Dataset import sys import cv2 import os if __name__ == '__main__': folder = "/home/mathieu/Dataset/DeepTrack/dragon/" dataset_path = os.path.join(folder, "train_raw_real") new_dataset_path = os.path.join(folder, "train_raw_real_resized") if not os.path.exists(new_dataset_path): os.mkdir(new_dataset_path) dataset = Dataset(dataset_path) if not dataset.load(): print("[Error]: Train dataset empty") sys.exit(-1) new_dataset = Dataset(new_dataset_path) new_dataset.camera = dataset.camera.copy() new_dataset.camera.set_ratio(2) for i in range(dataset.size()): rgb, depth, pose = dataset.load_image(i) new_rgb = cv2.resize( rgb, (new_dataset.camera.width, new_dataset.camera.height)) new_depth = cv2.resize( depth, (new_dataset.camera.width, new_dataset.camera.height)) new_dataset.add_pose(new_rgb, new_depth, pose) if i % (1 * dataset.size() / 100) == 0:
def config_datasets(data): train_path = data["train_path"] valid_path = data["valid_path"] minibatch_size = int(data["minibatch_size"]) rgb_noise = float(data["data_augmentation"]["rgb_noise"]) depth_noise = float(data["data_augmentation"]["depth_noise"]) occluder_path = data["data_augmentation"]["occluder_path"] background_path = data["data_augmentation"]["background_path"] blur_noise = int(data["data_augmentation"]["blur_noise"]) hue_noise = float(data["data_augmentation"]["hue_noise"]) data_augmentation = DataAugmentation() data_augmentation.set_rgb_noise(rgb_noise) data_augmentation.set_depth_noise(depth_noise) if occluder_path != "": data_augmentation.set_occluder(occluder_path) if background_path != "": data_augmentation.set_background(background_path) data_augmentation.set_blur(blur_noise) data_augmentation.set_hue_noise(hue_noise) train_dataset = Dataset(train_path, minibatch_size=minibatch_size) if not train_dataset.load(): message_logger.error("Train dataset empty") sys.exit(-1) train_dataset.set_data_augmentation(data_augmentation) train_dataset.compute_mean_std() message_logger.info("Computed mean : {}\nComputed Std : {}".format(train_dataset.mean, train_dataset.std)) valid_dataset = Dataset(valid_path, minibatch_size=minibatch_size) if not valid_dataset.load(): message_logger.error("Valid dataset empty") sys.exit(-1) valid_dataset.set_data_augmentation(data_augmentation) valid_dataset.mean = train_dataset.mean valid_dataset.std = train_dataset.std return train_dataset, valid_dataset
datasets_path = [ "/media/mathieu/e912e715-2be7-4fa2-8295-5c3ef1369dd0/dataset/deeptracking_train/dragon/real_large/train", "/media/mathieu/e912e715-2be7-4fa2-8295-5c3ef1369dd0/dataset/deeptracking_train/dragon/synth_large/train" ] output_path = "/media/mathieu/e912e715-2be7-4fa2-8295-5c3ef1369dd0/dataset/deeptracking_train/dragon/real_synth_large/train" if not os.path.exists(output_path): os.mkdir(output_path) print("Merging datasets :") for path in datasets_path: print(path) print("Into : {}".format(output_path)) # load datasets datasets = [Dataset(x) for x in datasets_path] for dataset in datasets: dataset.load() # metadata sanity check for dataset_check in datasets: for other_dataset in datasets: if dataset_check.metadata != other_dataset.metadata: print(dataset_check.metadata) raise RuntimeError( "Dataset {} have different metadata than {}\n{}\n{}". format(dataset_check.path, other_dataset.path, dataset_check.metadata, other_dataset.metadata)) metadata = datasets[0].metadata camera = datasets[0].camera
SHADER_PATH = data["shader_path"] OUTPUT_PATH = data["output_path"] IMAGE_SIZE = (int(data["image_size"]), int(data["image_size"])) CAMERA_PATH = data["camera_path"] DETECTOR_PATH = data["detector_layout_path"] PRELOAD = data["preload"] == "True" if not os.path.exists(OUTPUT_PATH): os.mkdir(OUTPUT_PATH) sensor = Kinect2(CAMERA_PATH) camera = sensor.intrinsics() ratio = 2 camera.set_ratio(ratio) sensor.start() dataset = Dataset(OUTPUT_PATH) dataset.camera = camera window = InitOpenGL(camera.width, camera.height) detector = ArucoDetector(camera, DETECTOR_PATH) vpRender = ModelRenderer(MODELS[0]["model_path"], SHADER_PATH, camera, window, (camera.width, camera.height)) vpRender.load_ambiant_occlusion_map(MODELS[0]["ambiant_occlusion_model"]) cv2.namedWindow('image') cv2.createTrackbar('transparency', 'image', 0, 100, trackbar) # todo, read from file? detection_offset = Transform() rgbd_record = False save_next_rgbd_pose = False lock_offset = False
os.mkdir(OUTPUT_PATH) # Write important misc data to file metadata = {} metadata["translation_range"] = str(TRANSLATION_RANGE) metadata["rotation_range"] = str(ROTATION_RANGE) metadata["image_size"] = str(IMAGE_SIZE[0]) metadata["save_type"] = data["save_type"] metadata["object_width"] = {} for model in MODELS: metadata["object_width"][model["name"]] = str(model["object_width"]) metadata["min_radius"] = str(SPHERE_MIN_RADIUS) metadata["max_radius"] = str(SPHERE_MAX_RADIUS) camera = Camera.load_from_json(data["camera_path"]) dataset = Dataset(OUTPUT_PATH, frame_class=data["save_type"]) dataset.camera = camera window_size = (camera.width, camera.height) window = InitOpenGL(*window_size) sphere_sampler = UniformSphereSampler(SPHERE_MIN_RADIUS, SPHERE_MAX_RADIUS) preload_count = 0 if PRELOAD: if dataset.load(): preload_count = dataset.size() print("This Dataset already contains {} samples".format( preload_count)) # Iterate over all models from config files for model in MODELS: vpRender = ModelRenderer(model["model_path"], SHADER_PATH, dataset.camera, window, window_size) vpRender.load_ambiant_occlusion_map(model["ambiant_occlusion_model"])
def config_datasets(data): train_path = data["train_path"] valid_path = data["valid_path"] minibatch_size = int(data["minibatch_size"]) rgb_noise = float(data["data_augmentation"]["rgb_noise"]) depth_noise = float(data["data_augmentation"]["depth_noise"]) occluder_path = data["data_augmentation"]["occluder_path"] background_path = data["data_augmentation"]["background_path"] blur_noise = int(data["data_augmentation"]["blur_noise"]) h_noise = float(data["data_augmentation"]["h_noise"]) s_noise = float(data["data_augmentation"]["s_noise"]) v_noise = float(data["data_augmentation"]["v_noise"]) channel_hide = data["data_augmentation"]["channel_hide"] == "True" data_augmentation = DataAugmentation() data_augmentation.set_rgb_noise(rgb_noise) data_augmentation.set_depth_noise(depth_noise) if occluder_path != "": data_augmentation.set_occluder(occluder_path) if background_path != "": data_augmentation.set_background(background_path) if channel_hide: data_augmentation.set_channel_hide(0.25) data_augmentation.set_blur(blur_noise) data_augmentation.set_hsv_noise(h_noise, s_noise, v_noise) message_logger.info("Setup Train : {}".format(train_path)) train_dataset = Dataset(train_path, minibatch_size=minibatch_size) if not train_dataset.load(): message_logger.error("Train dataset empty") sys.exit(-1) train_dataset.set_data_augmentation(data_augmentation) train_dataset.compute_mean_std() message_logger.info("Computed mean : {}\nComputed Std : {}".format( train_dataset.mean, train_dataset.std)) message_logger.info("Setup Valid : {}".format(valid_path)) valid_dataset = Dataset(valid_path, minibatch_size=minibatch_size, max_samples=20000) if not valid_dataset.load(): message_logger.error("Valid dataset empty") sys.exit(-1) valid_dataset.set_data_augmentation(data_augmentation) valid_dataset.mean = train_dataset.mean valid_dataset.std = train_dataset.std return train_dataset, valid_dataset
""" Sanity tests for dataset folder - Make sure all images in viewpoints.json are in the folder ... """ from deeptracking.data.dataset import Dataset import sys if __name__ == '__main__': dataset_path = "/home/mathieu/Dataset/DeepTrack/skull" dataset = Dataset(dataset_path) if not dataset.load(): print("[Error]: Train dataset empty") sys.exit(-1) # check if all viewpoints are there for frame, pose in dataset.data_pose: if not frame.exists(dataset.path): print("[Error]: Missing pose frame {}".format(frame.id)) sys.exit(-1) # check if all pairs are there for key, value in dataset.data_pair.items(): for frame, pose in value: if not frame.exists(dataset.path): print("[Error]: Missing pair frame {}".format(frame.id))
from deeptracking.utils.camera import Camera from deeptracking.utils.transform import Transform import cv2 import os import numpy as np from deeptracking.detector.detector_aruco import ArucoDetector if __name__ == '__main__': dataset_path = "/media/mathieu/e912e715-2be7-4fa2-8295-5c3ef1369dd0/dataset/deeptracking/sequences/skull" detector_path = "../deeptracking/detector/aruco_layout.xml" model_path = "/home/mathieu/Dataset/3D_models/skull/skull.ply" model_ao_path = "/home/mathieu/Dataset/3D_models/skull/skull_ao.ply" shader_path = "../deeptracking/data/shaders" dataset = Dataset(dataset_path) offset = Transform.from_matrix( np.load(os.path.join(dataset.path, "offset.npy"))) camera = Camera.load_from_json(dataset_path) dataset.camera = camera files = [ f for f in os.listdir(dataset_path) if os.path.splitext(f)[-1] == ".png" and 'd' not in os.path.splitext(f)[0] ] detector = ArucoDetector(camera, detector_path) window = InitOpenGL(camera.width, camera.height) vpRender = ModelRenderer(model_path, shader_path, camera, window, (camera.width, camera.height)) vpRender.load_ambiant_occlusion_map(model_ao_path) ground_truth_pose = None