def quick_pose(image): image = resize_image(image, (128, 128)) # show_image(resize_image(image, (256 * 3, 256 * 3))) keypoints = estimate_keypoints(image) points2D = keypoints['points2D'] points3D = keypoints['points3D'] # points3D[:, 2:3] = 0.0 points2D = denormalize_points2D(points2D, 128, 128) success, rotation, translation = predict_pose(points3D, points2D) quaternion = rotation_vector_to_quaternion(rotation) pose6D = Pose6D(quaternion, translation, 'solar_panel') poses6D = [pose6D] # show_image(image) points = [[points2D, points3D]] image = draw_masks(image, points, object_sizes) image = image.astype('float') image = draw_poses6D(image, poses6D, cube_points3D, camera_intrinsics) image = image.astype('uint8') image = resize_image(image, (256 * 3, 256 * 3)) show_image(image)
keypoints = self.denormalize_keypoints(keypoints, cropped_image) keypoints = self.change_coordinates(keypoints, box2D) keypoints2D.append(keypoints) contour = self.draw_probabilities(cropped_image, probabilities) contours.append(contour) image = self.draw_keypoints(image, keypoints) image = self.draw_boxes2D(image, boxes2D) return self.wrap(image, boxes2D, keypoints2D, contours) if __name__ == '__main__': from paz.abstract import ProcessingSequence from paz.backend.image import show_image from facial_keypoints import FacialKeypoints data_manager = FacialKeypoints('dataset/', 'train') dataset = data_manager.load_data() augment_keypoints = AugmentKeypoints('train', with_partition=False) for arg in range(1, 100): sample = dataset[arg] predictions = augment_keypoints(sample) original_image = predictions['inputs']['image'][:, :, 0] original_image = original_image * 255.0 kp = predictions['labels']['keypoints'] kp = denormalize_keypoints(kp, 96, 96) original_image = draw_circles(original_image, kp.astype('int')) show_image(original_image.astype('uint8')) sequence = ProcessingSequence(augment_keypoints, 32, dataset, True) batch = sequence.__getitem__(0)
import os from paz.abstract import SequentialProcessor from paz.backend.image import show_image, load_image import paz.processors as pr from tensorflow.keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.utils import get_file # let's download a test image and put it inside our PAZ directory IMAGE_URL = ('https://github.com/oarriaga/altamira-data/releases/download' '/v0.9/image_augmentation.png') filename = os.path.basename(IMAGE_URL) image_fullpath = get_file(filename, IMAGE_URL, cache_subdir='paz/tutorials') # we load the original image and display it image = load_image(image_fullpath) show_image(image) # We construct a data augmentation pipeline using the built-in PAZ processors: augment = SequentialProcessor() augment.add(pr.RandomContrast()) augment.add(pr.RandomBrightness()) augment.add(pr.RandomSaturation()) # We can now apply our pipeline as a normal function: for _ in range(5): image = load_image(image_fullpath) # use it as a normal function image = augment(image) show_image(image) # We can add to our sequential pipeline other function anywhere i.e. arg 0:
name_to_sizes = { 'power_drill': np.array([1840, 1870, 520]), 'Solar_panel': np.array([15000, 15000, 2000]), 'Large_clamp': np.array([12000, 17100, 3900]), 'hammer': np.array([18210, 33272, 3280]) } name_to_weights = { 'power_drill': 'weights/UNET_weights_epochs-10_beta-3.hdf5', 'Solar_panel': 'weights/UNET-VGG_solar_panel_canonical_13.hdf5', 'Large_clamp': 'weights/UNET-VGG_large_clamp_canonical_10.hdf5', 'hammer': 'weights/UNET-VGG16_weights_hammer_10.hdf5' } segment = UNET_VGG16(num_classes, image_shape, freeze_backbone=True) valid_class_names = ['power_drill', 'Solar_panel', 'Large_clamp', 'hammer'] pipeline = MultiPix2Pose(detect, segment, camera, name_to_weights, name_to_sizes, valid_class_names, offsets, epsilon, draw=True) results = pipeline(image) predicted_image = results['image'] show_image(predicted_image)
H, W = load_image(image_fullpath).shape[:2] # The x_max, y_max are the **normalized** coordinates class_names = ['background', 'human', 'horse'] box_data = np.array([[200 / W, 60 / H, 300 / W, 200 / H, 1], [100 / W, 90 / H, 400 / W, 300 / H, 2]]) # Let's visualize our boxes! # first we transform our numpy array into our built-in ``Box2D`` messages to_boxes2D = pr.ToBoxes2D(class_names) denormalize = pr.DenormalizeBoxes2D() boxes2D = to_boxes2D(box_data) image = load_image(image_fullpath) boxes2D = denormalize(image, boxes2D) draw_boxes2D = pr.DrawBoxes2D(class_names) show_image(draw_boxes2D(image, boxes2D)) # As you can see, we were not able to put everything as a # ``SequentialProcessor``. This is because we are dealing with 2 inputs: # ``box_data`` and ``image``. We can join them into a single processor # using ``pr.ControlMap`` wrap. ``pr.ControlMap`` allows you to select which # arguments (``intro_indices``) are passed to your processor, and also where # you should put the output of your processor (``outro_indices``). draw_boxes = SequentialProcessor() draw_boxes.add(pr.ControlMap(to_boxes2D, intro_indices=[1], outro_indices=[1])) draw_boxes.add(pr.ControlMap(pr.LoadImage(), [0], [0])) draw_boxes.add(pr.ControlMap(denormalize, [0, 1], [1], keep={0: 0})) draw_boxes.add(pr.DrawBoxes2D(class_names)) draw_boxes.add(pr.ShowImage()) # now you have everything in a single packed function that loads and draws!
help='Dataset name') args = parser.parse_args() name_to_model = {'VOC': SSD300VOC, 'FAT': SSD300FAT, 'COCO': SSD512COCO, 'YCBVideo': SSD512YCBVideo} image = load_image(args.image_path) H = 1000 W = int((H / image.shape[0]) * image.shape[1]) # image = resize_image(image, (W, H)) focal_length = image.shape[1] image_center = (image.shape[1] / 2.0, image.shape[0] / 2.0) camera = Camera(args.camera_id) camera.distortion = np.zeros((4, 1)) camera.intrinsics = np.array([[focal_length, 0, image_center[0]], [0, focal_length, image_center[1]], [0, 0, 1]]) pipeline_A = DetectMiniXceptionFER([args.offset, args.offset]) pipeline_B = DetectFaceKeypointNet2D32() pipeline_C = HeadPoseKeypointNet2D32(camera) pipeline_D = name_to_model[args.dataset](args.score_thresh, args.nms_thresh) pipelines = [pipeline_A, pipeline_B, pipeline_C, pipeline_D] for pipeline in pipelines: predictions = pipeline(image.copy()) show_image(predictions['image'])
import numpy as np from paz.datasets import CityScapes from pipelines import PreprocessSegmentationIds from pipelines import PostprocessSegmentationIds from pipelines import PostProcessImage if __name__ == "__main__": from paz.backend.image import show_image label_path = '/home/octavio/Downloads/dummy/gtFine/' # label_path = '/home/octavio/Downloads/dummy/gtCoarse/' image_path = '/home/octavio/Downloads/dummy/RGB_images/leftImg8bit/' data_manager = CityScapes(image_path, label_path, 'train') dataset = data_manager.load_data() class_names = data_manager.class_names num_classes = len(class_names) preprocess = PreprocessSegmentationIds((128, 128), num_classes) postprocess_masks = PostprocessSegmentationIds(num_classes) postprocess_image = PostProcessImage() for sample in dataset: preprocessed_sample = preprocess(sample) image = preprocessed_sample['inputs']['input_1'] image = postprocess_image(image) masks = preprocessed_sample['labels']['masks'] masks = postprocess_masks(masks) mask_and_image = np.concatenate([masks, image], axis=1) show_image(mask_and_image)
# from meters to milimiters # object_sizes = renderer.mesh.mesh.extents * 100 object_sizes = renderer.mesh.mesh.extents * 10000 print(object_sizes) model = UNET_VGG16(3, image_shape, freeze_backbone=True) model.load_weights( 'experiments/UNET-VGG16_RUN_00_04-04-2022_12-29-44/model_weights.hdf5') # model.load_weights('experiments/UNET-VGG16_RUN_00_06-04-2022_11-20-18/model_weights.hdf5') # model.load_weights('experiments/UNET-VGG16_RUN_00_07-04-2022_13-28-04/model_weights.hdf5') # estimate_pose = RGBMaskToPose6D(model, object_sizes, camera, draw=True) # estimate_pose = SingleInferencePIX2POSE6D(model, object_sizes, camera) estimate_pose = SingleInstancePIX2POSE6D(model, object_sizes, camera) image, alpha, RGBA_mask = renderer.render() image = np.copy(image) # TODO: renderer outputs unwritable numpy arrays show_image(image) results = estimate_pose(image) show_image(results['image']) for arg in range(100): image, alpha, RGBA_mask = renderer.render() RGBA = concatenate_alpha_mask(image, alpha) background = make_random_plain_image(image.shape) image_with_background = blend_alpha_channel(RGBA, background) results = estimate_pose(image_with_background.copy()) image = np.concatenate( [image_with_background, RGBA_mask[..., 0:3], results['image']], axis=1) H, W = image.shape[:2] image = resize_image(image, (W * 3, H * 3)) show_image(image)
import argparse from pipelines import DetectGMMKeypointNet2D from paz.backend.image import show_image, load_image description = 'Demo for visualizing uncertainty in probabilistic keypoints' parser = argparse.ArgumentParser(description=description) parser.add_argument('-p', '--path', type=str, help='Path to image') args = parser.parse_args() pipeline = DetectGMMKeypointNet2D() image = load_image(args.path) inferences = pipeline(image) show_image(inferences['image']) show_image(inferences['contours'][0])
light, top_only, roll, shift) # check why this is needed in this object renderer.scene.ambient_light = [1.0, 1.0, 1.0] # pose = np.eye(4) # pose[1, 1] = +np.cos(np.deg2rad(90)) # pose[1, 2] = -np.sin(np.deg2rad(90)) # pose[2, 2] = +np.cos(np.deg2rad(90)) # pose[2, 1] = +np.sin(np.deg2rad(90)) # renderer.scene.set_pose(renderer.mesh, pose) # renderer.scene.set_pose(renderer.pixel_mesh, pose) from paz.backend.image import show_image from backend import rotate_image for _ in range(0): image, alpha, RGB_mask = renderer.render() RGB_mask = RGB_mask[..., 0:3] show_image(RGB_mask) angles = np.linspace(0, 2 * np.pi, 7)[0:6] images = [] for angle in angles: rotation_matrix = build_rotation_matrix_z(angle) rotated_image = rotate_image(RGB_mask, rotation_matrix) rotated_image = rotated_image.astype('uint8') images.append(rotated_image) images = np.concatenate(images, axis=1) show_image(images) inputs_to_shape = {'input_1': [H, W, num_channels]} labels_to_shape = {'masks': [H, W, 4]} processor = DomainRandomization(
super(PostprocessSegmentation, self).__init__() self.add(pr.UnpackDictionary(['image_path'])) self.add(pr.LoadImage()) self.add(pr.ResizeImage(model.input_shape[1:3])) self.add(pr.ConvertColorSpace(pr.RGB2BGR)) self.add(pr.SubtractMeanImage(pr.BGR_IMAGENET_MEAN)) self.add(pr.ExpandDims(0)) self.add(pr.Predict(model)) self.add(pr.Squeeze(0)) self.add(Round()) self.add(MasksToColors(model.output_shape[-1], colors)) self.add(pr.DenormalizeImage()) self.add(pr.CastImage('uint8')) self.add(ResizeImageWithNearestNeighbors((1024, 512))) # self.add(pr.ShowImage()) num_classes = len(data_manager.class_names) input_shape = (128, 128, 3) model = UNET_VGG16(num_classes, input_shape, 'imagenet', activation='softmax') post_process = PostprocessSegmentation(model) model.load_weights(args.weights_path) for sample in data: masks = post_process(sample) image = load_image(sample['image_path']) image = resize_image(image, (1024, 512)) image_with_masks = ((0.6 * image) + (0.4 * masks)).astype("uint8") # image_and_masks = np.concatenate([image, masks], axis=1) show_image(image_with_masks)
class_masks[0] = np.logical_not(class_masks[0]) for shape_arg, (shape, color, dimensions) in enumerate(shapes): mask_arg = self.name_to_arg[shape] class_mask = class_masks[mask_arg] class_mask = self._draw_shape(class_mask, shape, dimensions, 1) class_masks[mask_arg] = class_mask negative_mask = np.logical_not(class_mask) background_mask = class_masks[0].copy() class_masks[0] = np.logical_and(negative_mask, background_mask) masks = np.concatenate(class_masks, axis=-1).astype(np.uint8) return masks if __name__ == '__main__': from paz.backend.image import show_image data_manager = Shapes(1000, (128, 128), iou_thresh=0.3, max_num_shapes=3) dataset = data_manager.load_data() for sample in dataset: image = sample['image'] masks = (sample['masks'] * 255.0).astype('uint8') background_mask, masks = masks[..., 0:1], masks[..., 1:] background_mask = np.repeat(background_mask, 3, axis=-1) boxes = sample['box_data'] for box in boxes: coordinates, class_arg = box[:4], box[4] # coordinates = denormalize_box(coordinates, (128, 128)) class_name = data_manager.arg_to_name[class_arg] image = draw_box(image, coordinates, class_name, 1.0) print(background_mask.shape) show_image(np.concatenate([image, masks, background_mask], axis=1))