def __init__(self, renderer, image_shape, image_paths, inputs_to_shape, labels_to_shape, num_occlusions=1): super(DomainRandomization, self).__init__() H, W = image_shape[:2] self.add(pr.Render(renderer)) self.add(pr.ControlMap(RandomizeRender(image_paths), [0, 1], [0])) self.add(pr.ControlMap(pr.NormalizeImage(), [0], [0])) self.add(pr.ControlMap(pr.NormalizeImage(), [1], [1])) self.add(pr.SequenceWrapper({0: inputs_to_shape}, {1: labels_to_shape}))
def __init__(self, detector, keypoint_estimator, radius=3): super(ProbabilisticKeypointPrediction, self).__init__() # face detector RGB2GRAY = pr.ConvertColorSpace(pr.RGB2GRAY) self.detect = pr.Predict(detector, RGB2GRAY, pr.ToBoxes2D(['face'])) # creating pre-processing pipeline for keypoint estimator preprocess = SequentialProcessor() preprocess.add(pr.ResizeImage(keypoint_estimator.input_shape[1:3])) preprocess.add(pr.ConvertColorSpace(pr.RGB2GRAY)) preprocess.add(pr.NormalizeImage()) preprocess.add(pr.ExpandDims(0)) preprocess.add(pr.ExpandDims(-1)) # creating post-processing pipeline for keypoint esimtator # postprocess = SequentialProcessor() # postprocess.add(ToNumpyArray()) # postprocess.add(pr.Squeeze(1)) # keypoint estimator predictions self.estimate_keypoints = PredictMeanDistribution( keypoint_estimator, preprocess) # self.estimate_keypoints = pr.Predict( # keypoint_estimator, preprocess, postprocess) # used for drawing up keypoints in original image self.change_coordinates = pr.ChangeKeypointsCoordinateSystem() self.denormalize_keypoints = pr.DenormalizeKeypoints() self.crop_boxes2D = pr.CropBoxes2D() self.num_keypoints = len(keypoint_estimator.output_shape) self.draw = pr.DrawKeypoints2D(self.num_keypoints, radius, False) self.draw_boxes2D = pr.DrawBoxes2D(['face'], colors=[[0, 255, 0]]) self.wrap = pr.WrapOutput(['image', 'boxes2D'])
def test_image_cropping(): handsegnet = HandSegmentationNet() preprocess_image = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) postprocess_segmentation = PostProcessSegmentation(320, 320) localize_hand = pr.Predict(handsegnet, preprocess_image, postprocess_segmentation) image = load_image('./sample.jpg') hand_crop, segmentation_map, center, boxes, crop_sizes = localize_hand( image) box = boxes[0] xmin, ymin, xmax, ymax = box crop_size = crop_sizes[0] assert len(hand_crop.shape) == 4 assert hand_crop.shape == (1, 256, 256, 3) assert len(segmentation_map.shape) == 4 assert segmentation_map.shape == (1, 320, 320, 1) assert center == [[191.5, 194.5]] assert len(box) == 4 assert box == [114, 153, 269, 236] assert xmax > xmin and ymin > ymax assert round(crop_size[0], 2) == 1.32
def __init__(self, renderer, image_paths, num_occlusions, split=pr.TRAIN): super(DomainRandomizationProcessor, self).__init__() self.copy = pr.Copy() self.render = pr.Render(renderer) self.augment = RandomizeRenderedImage(image_paths, num_occlusions) preprocessors = [pr.ConvertColorSpace(pr.RGB2BGR), pr.NormalizeImage()] self.preprocess = SequentialProcessor(preprocessors) self.split = split
def __init__(self, shape, mean=pr.BGR_IMAGENET_MEAN): super(PreprocessImage, self).__init__() self.add(pr.ResizeImage(shape)) self.add(pr.CastImage(float)) if mean is None: self.add(pr.NormalizeImage()) else: self.add(pr.SubtractMeanImage(mean))
def test_preprocess_image(): preprocess_pipeline = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) image = load_image('./sample.jpg') processed_image = preprocess_pipeline(image) assert len(processed_image.shape) == 4 assert processed_image.shape == (1, 320, 320, 3)
def __init__(self, model, draw=True): super(GMMKeypoints, self).__init__() self.num_keypoints = len(model.output_shape) preprocess = SequentialProcessor() preprocess.add(pr.ResizeImage(model.input_shape[1:3])) preprocess.add(pr.ConvertColorSpace(pr.RGB2GRAY)) preprocess.add(pr.NormalizeImage()) preprocess.add(pr.ExpandDims(0)) preprocess.add(pr.ExpandDims(-1)) self.estimate_keypoints = PredictDistributions(model, preprocess) self.to_grid = ToProbabilityGrid(GRID) self.draw = draw self.draw_probabilities = DrawProbabilities(self.num_keypoints) self.wrap = pr.WrapOutput(['image', 'probabilities', 'distributions'])
def __init__(self, scene, keypoints, image_paths, num_occlusions): super(RandomKeypointsRender, self).__init__() H, W = scene.viewport_size render = pr.Render(scene) augment = RandomizeRenderedImage(image_paths, num_occlusions) augment.add(pr.NormalizeImage()) self.add(RenderRandomSample(render, augment, keypoints)) self.add( pr.SequenceWrapper({0: { 'image': [H, W, 3] }}, {1: { 'keypoints': [len(keypoints), 3] }}))
def test_segmentation_postprocess(): preprocess_pipeline = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) image = load_image('./sample.jpg') processed_image = preprocess_pipeline(image) localization_pipeline = PostProcessSegmentation(HandSegNet) localization_output = localization_pipeline(processed_image) assert len(localization_output) == 5 assert localization_output[0].shape == (1, 256, 256, 3) assert localization_output[1].shape == (1, 320, 320, 1) assert localization_output[2].shape == (1, 2) assert localization_output[3].shape == (1, 2, 2) assert localization_output[4].shape == (1, 1)
def test_keypoints2D_process(): preprocess_pipeline = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) image = load_image('./sample.jpg') processed_image = preprocess_pipeline(image) localization_pipeline = PostProcessSegmentation(HandSegNet) localization_output = localization_pipeline(processed_image) keypoints_pipeline = Process2DKeypoints(HandPoseNet) score_maps_dict = keypoints_pipeline( np.squeeze(localization_output[0], axis=0)) score_maps = score_maps_dict['score_maps'] assert score_maps.shape == (1, 32, 32, 21) assert len(score_maps) == 1
def __init__(self, phase, rotation_range=30, delta_scales=[0.2, 0.2], num_keypoints=15): super(AugmentKeypoints, self).__init__() self.add(pr.UnpackDictionary(['image', 'keypoints'])) if phase == 'train': self.add(pr.ControlMap(pr.RandomBrightness())) self.add(pr.ControlMap(pr.RandomContrast())) self.add(pr.RandomKeypointRotation(rotation_range)) self.add(pr.RandomKeypointTranslation(delta_scales)) self.add(pr.ControlMap(pr.NormalizeImage(), [0], [0])) self.add(pr.ControlMap(pr.ExpandDims(-1), [0], [0])) self.add(pr.ControlMap(pr.NormalizeKeypoints((96, 96)), [1], [1])) self.add( pr.SequenceWrapper({0: { 'image': [96, 96, 1] }}, {1: { 'keypoints': [num_keypoints, 2] }}))
def __init__(self, handsegnet, posenet, posepriornet, viewpointnet, image_size=320, crop_shape=(256, 256), num_keypoints=21): super(DetectHandKeypoints, self).__init__() self.preprocess_image = SequentialProcessor([ pr.NormalizeImage(), pr.ResizeImage((image_size, image_size)), pr.ExpandDims(0) ]) postprocess_segmentation = PostProcessSegmentation( image_size, crop_shape) self.localize_hand = pr.Predict(handsegnet, postprocess=postprocess_segmentation) self.resize_scoremaps = ResizeScoreMaps(crop_shape) self.merge_dictionaries = MergeDictionaries() self.wrap_input = WrapToDictionary(['hand_side']) self.predict_keypoints2D = pr.Predict(posenet) self.predict_keypoints3D = pr.Predict(posepriornet) self.predict_keypoints_angles = pr.Predict(viewpointnet) self.postprocess_keypoints = PostProcessKeypoints() self.resize = pr.ResizeImage(shape=crop_shape) self.extract_2D_keypoints = ExtractKeypoints() self.transform_keypoints = TransformKeypoints() self.draw_keypoint = pr.DrawKeypoints2D(num_keypoints, normalized=True, radius=4) self.denormalize = pr.DenormalizeImage() self.wrap = pr.WrapOutput(['image', 'keypoints2D', 'keypoints3D']) self.expand_dims = pr.ExpandDims(axis=0) self.draw_boxes = pr.DrawBoxes2D(['hand'], [[0, 1, 0]])
def __init__(self, phase, rotation_range=30, delta_scales=[0.2, 0.2], with_partition=False, num_keypoints=15): super(AugmentKeypoints, self).__init__() self.add(pr.UnpackDictionary(['image', 'keypoints'])) if phase == 'train': self.add(pr.ControlMap(pr.RandomBrightness())) self.add(pr.ControlMap(pr.RandomContrast())) self.add(pr.RandomKeypointRotation(rotation_range)) self.add(pr.RandomKeypointTranslation(delta_scales)) self.add(pr.ControlMap(pr.NormalizeImage(), [0], [0])) self.add(pr.ControlMap(pr.ExpandDims(-1), [0], [0])) self.add(pr.ControlMap(pr.NormalizeKeypoints((96, 96)), [1], [1])) labels_info = {1: {'keypoints': [num_keypoints, 2]}} if with_partition: outro_indices = list(range(1, 16)) self.add(pr.ControlMap(PartitionKeypoints(), [1], outro_indices)) labels_info = {} for arg in range(num_keypoints): labels_info[arg + 1] = {'keypoint_%s' % arg: [2]} self.add(pr.SequenceWrapper({0: {'image': [96, 96, 1]}}, labels_info))