def __init__(self, size, image_size, crop_size, variance): super(ExtractHandPose, self).__init__() self.unwrap_inputs = pr.UnpackDictionary( ['image', 'segmentation_label', 'annotations']) self.preprocess_image = pr.SequentialProcessor( [pr.LoadImage(), pr.ResizeImage((size, size))]) self.preprocess_segmentation_map = pr.SequentialProcessor( [pr.LoadImage(), pr.ResizeImage((size, size)), ExtractHandmask()]) self.extract_annotations = pr.UnpackDictionary(['xyz', 'uv_vis', 'K']) self.extract_2D_keypoints = Extract2DKeypoints() self.keypoints_to_palm = KeypointstoPalmFrame() self.visibility_to_palm = TransformVisibilityMask() self.extract_hand_side = ExtractHandsideandKeypoints() self.to_one_hot = ToOneHot(num_classes=2) self.normaliza_keypoints = NormalizeKeypoints() self.to_relative_frame = TransformtoRelativeFrame() self.canonical_transformations = GetCanonicalTransformation() self.flip_right_hand = FlipRightHandToLeftHand() self.get_matrix_inverse = CalculatePseudoInverse() self.extract_hand_visibility = ExtractDominantHandVisibility() self.extract_dominant_keypoints = ExtractDominantKeypoints2D() self.crop_image_from_mask = CropImageFromMask() self.create_scoremaps = CreateScoremaps(image_size=image_size, crop_size=crop_size, variance=variance) self.wrap = pr.WrapOutput( ['score_maps', 'hand_side', 'keypoints3D', 'rotation_matrix'])
def __init__(self, detector, keypoint_estimator, radius=3): super(ProbabilisticKeypointPrediction, self).__init__() # face detector RGB2GRAY = pr.ConvertColorSpace(pr.RGB2GRAY) self.detect = pr.Predict(detector, RGB2GRAY, pr.ToBoxes2D(['face'])) # creating pre-processing pipeline for keypoint estimator preprocess = SequentialProcessor() preprocess.add(pr.ResizeImage(keypoint_estimator.input_shape[1:3])) preprocess.add(pr.ConvertColorSpace(pr.RGB2GRAY)) preprocess.add(pr.NormalizeImage()) preprocess.add(pr.ExpandDims(0)) preprocess.add(pr.ExpandDims(-1)) # creating post-processing pipeline for keypoint esimtator # postprocess = SequentialProcessor() # postprocess.add(ToNumpyArray()) # postprocess.add(pr.Squeeze(1)) # keypoint estimator predictions self.estimate_keypoints = PredictMeanDistribution( keypoint_estimator, preprocess) # self.estimate_keypoints = pr.Predict( # keypoint_estimator, preprocess, postprocess) # used for drawing up keypoints in original image self.change_coordinates = pr.ChangeKeypointsCoordinateSystem() self.denormalize_keypoints = pr.DenormalizeKeypoints() self.crop_boxes2D = pr.CropBoxes2D() self.num_keypoints = len(keypoint_estimator.output_shape) self.draw = pr.DrawKeypoints2D(self.num_keypoints, radius, False) self.draw_boxes2D = pr.DrawBoxes2D(['face'], colors=[[0, 255, 0]]) self.wrap = pr.WrapOutput(['image', 'boxes2D'])
def test_image_cropping(): handsegnet = HandSegmentationNet() preprocess_image = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) postprocess_segmentation = PostProcessSegmentation(320, 320) localize_hand = pr.Predict(handsegnet, preprocess_image, postprocess_segmentation) image = load_image('./sample.jpg') hand_crop, segmentation_map, center, boxes, crop_sizes = localize_hand( image) box = boxes[0] xmin, ymin, xmax, ymax = box crop_size = crop_sizes[0] assert len(hand_crop.shape) == 4 assert hand_crop.shape == (1, 256, 256, 3) assert len(segmentation_map.shape) == 4 assert segmentation_map.shape == (1, 320, 320, 1) assert center == [[191.5, 194.5]] assert len(box) == 4 assert box == [114, 153, 269, 236] assert xmax > xmin and ymin > ymax assert round(crop_size[0], 2) == 1.32
def __init__(self, shape, mean=pr.BGR_IMAGENET_MEAN): super(PreprocessImage, self).__init__() self.add(pr.ResizeImage(shape)) self.add(pr.CastImage(float)) if mean is None: self.add(pr.NormalizeImage()) else: self.add(pr.SubtractMeanImage(mean))
def test_preprocess_image(): preprocess_pipeline = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) image = load_image('./sample.jpg') processed_image = preprocess_pipeline(image) assert len(processed_image.shape) == 4 assert processed_image.shape == (1, 320, 320, 3)
def __init__(self, shape, bkg_paths, mean=pr.BGR_IMAGENET_MEAN): super(AugmentImage, self).__init__() # self.add(LoadImage(4)) self.add(pr.ResizeImage(shape)) self.add(pr.BlendRandomCroppedBackground(bkg_paths)) self.add(pr.RandomContrast()) self.add(pr.RandomBrightness()) self.add(pr.RandomSaturation(0.7)) self.add(pr.RandomHue()) self.add(pr.ConvertColorSpace(pr.RGB2BGR))
def __init__(self, base, mean_face, shape=(48, 48), with_crop=True): super(CalculateFaceWeights, self).__init__() self.base, self.mean_face = base, mean_face self.preprocess = pr.SequentialProcessor() self.convert_to_gray = pr.ConvertColorSpace(pr.RGB2GRAY) if with_crop: self.preprocess.add(pe.CropFrontalFace()) self.preprocess.add(pr.ResizeImage(shape)) self.preprocess.add(pr.ExpandDims(-1)) self.subtract = pe.SubtractMeanFace() self.project = pe.ProjectToBase(self.base)
def __init__(self, size=320): super(ExtractHandSegmentation, self).__init__() self.add( pr.UnpackDictionary(['image', 'segmentation_label', 'annotations'])) preprocess_image = pr.SequentialProcessor( [pr.LoadImage(), pr.ResizeImage((size, size))]) preprocess_segmentation_map = pr.SequentialProcessor( [pr.LoadImage(), pr.ResizeImage((size, size)), ExtractHandmask()]) self.add(pr.ControlMap(preprocess_image, [0], [0])) self.add(pr.ControlMap(preprocess_segmentation_map, [1], [1])) self.add( pr.SequenceWrapper({0: { 'image': [size, size, 3] }}, {1: { 'hand_mask': [size, size] }}))
def __init__(self, handsegnet, posenet, posepriornet, viewpointnet, image_size=320, crop_shape=(256, 256), num_keypoints=21): super(DetectHandKeypoints, self).__init__() self.preprocess_image = SequentialProcessor([ pr.NormalizeImage(), pr.ResizeImage((image_size, image_size)), pr.ExpandDims(0) ]) postprocess_segmentation = PostProcessSegmentation( image_size, crop_shape) self.localize_hand = pr.Predict(handsegnet, postprocess=postprocess_segmentation) self.resize_scoremaps = ResizeScoreMaps(crop_shape) self.merge_dictionaries = MergeDictionaries() self.wrap_input = WrapToDictionary(['hand_side']) self.predict_keypoints2D = pr.Predict(posenet) self.predict_keypoints3D = pr.Predict(posepriornet) self.predict_keypoints_angles = pr.Predict(viewpointnet) self.postprocess_keypoints = PostProcessKeypoints() self.resize = pr.ResizeImage(shape=crop_shape) self.extract_2D_keypoints = ExtractKeypoints() self.transform_keypoints = TransformKeypoints() self.draw_keypoint = pr.DrawKeypoints2D(num_keypoints, normalized=True, radius=4) self.denormalize = pr.DenormalizeImage() self.wrap = pr.WrapOutput(['image', 'keypoints2D', 'keypoints3D']) self.expand_dims = pr.ExpandDims(axis=0) self.draw_boxes = pr.DrawBoxes2D(['hand'], [[0, 1, 0]])
def __init__(self, model, draw=True): super(GMMKeypoints, self).__init__() self.num_keypoints = len(model.output_shape) preprocess = SequentialProcessor() preprocess.add(pr.ResizeImage(model.input_shape[1:3])) preprocess.add(pr.ConvertColorSpace(pr.RGB2GRAY)) preprocess.add(pr.NormalizeImage()) preprocess.add(pr.ExpandDims(0)) preprocess.add(pr.ExpandDims(-1)) self.estimate_keypoints = PredictDistributions(model, preprocess) self.to_grid = ToProbabilityGrid(GRID) self.draw = draw self.draw_probabilities = DrawProbabilities(self.num_keypoints) self.wrap = pr.WrapOutput(['image', 'probabilities', 'distributions'])
def __init__(self, size, image_size, crop_size, variance): super(ExtractHandPose2D, self).__init__() self.unwrap_inputs = pr.UnpackDictionary( ['image', 'segmentation_label', 'annotations']) self.preprocess_image = pr.SequentialProcessor( [pr.LoadImage(), pr.ResizeImage((size, size))]) self.preprocess_segmentation_map = pr.SequentialProcessor( [pr.LoadImage(), pr.ResizeImage((size, size)), ExtractHandmask()]) self.extract_annotations = pr.UnpackDictionary(['xyz', 'uv_vis', 'K']) self.extract_2D_keypoints = Extract2DKeypoints() self.keypoints_to_palm = KeypointstoPalmFrame() self.visibility_to_palm = TransformVisibilityMask() self.extract_hand_side = ExtractHandsideandKeypoints() self.extract_visibility_dominant_hand = ExtractDominantHandVisibility() self.create_scoremaps = CreateScoremaps(image_size, crop_size, variance) self.crop_image_from_mask = CropImageFromMask() self.wrap = pr.WrapOutput( ['cropped_image', 'score_maps', 'keypoints_vis21'])
def __init__(self, class_names, prior_boxes, variances=[0.1, 0.1, 0.2, 0.2]): super(ShowBoxes, self).__init__() self.deprocess_boxes = SequentialProcessor([ pr.DecodeBoxes(prior_boxes, variances), pr.ToBoxes2D(class_names, True), pr.FilterClassBoxes2D(class_names[1:]) ]) self.denormalize_boxes2D = pr.DenormalizeBoxes2D() self.draw_boxes2D = pr.DrawBoxes2D(class_names) self.show_image = pr.ShowImage() self.resize_image = pr.ResizeImage((600, 600))
def __init__(self, model, colors=None): super(PostprocessSegmentation, self).__init__() self.add(pr.UnpackDictionary(['image_path'])) self.add(pr.LoadImage()) self.add(pr.ResizeImage(model.input_shape[1:3])) self.add(pr.ConvertColorSpace(pr.RGB2BGR)) self.add(pr.SubtractMeanImage(pr.BGR_IMAGENET_MEAN)) self.add(pr.ExpandDims(0)) self.add(pr.Predict(model)) self.add(pr.Squeeze(0)) self.add(Round()) self.add(MasksToColors(model.output_shape[-1], colors)) self.add(pr.DenormalizeImage()) self.add(pr.CastImage('uint8')) self.add(pr.ShowImage())
def test_segmentation_postprocess(): preprocess_pipeline = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) image = load_image('./sample.jpg') processed_image = preprocess_pipeline(image) localization_pipeline = PostProcessSegmentation(HandSegNet) localization_output = localization_pipeline(processed_image) assert len(localization_output) == 5 assert localization_output[0].shape == (1, 256, 256, 3) assert localization_output[1].shape == (1, 320, 320, 1) assert localization_output[2].shape == (1, 2) assert localization_output[3].shape == (1, 2, 2) assert localization_output[4].shape == (1, 1)
def test_keypoints2D_process(): preprocess_pipeline = SequentialProcessor( [pr.NormalizeImage(), pr.ResizeImage((320, 320)), pr.ExpandDims(0)]) image = load_image('./sample.jpg') processed_image = preprocess_pipeline(image) localization_pipeline = PostProcessSegmentation(HandSegNet) localization_output = localization_pipeline(processed_image) keypoints_pipeline = Process2DKeypoints(HandPoseNet) score_maps_dict = keypoints_pipeline( np.squeeze(localization_output[0], axis=0)) score_maps = score_maps_dict['score_maps'] assert score_maps.shape == (1, 32, 32, 21) assert len(score_maps) == 1
def __init__(self, image_shape, num_classes, input_name='input_1', dataset='CityScapes'): super(PreprocessSegmentationIds, self).__init__() self.add(pr.UnpackDictionary(['image_path', 'label_path'])) preprocess_image = pr.SequentialProcessor() preprocess_image.add(pr.LoadImage()) preprocess_image.add(pr.ResizeImage(image_shape)) preprocess_image.add(pr.ConvertColorSpace(pr.RGB2BGR)) preprocess_image.add(pr.SubtractMeanImage(pr.BGR_IMAGENET_MEAN)) preprocess_label = pr.SequentialProcessor() preprocess_label.add(pr.LoadImage()) preprocess_label.add(ResizeImageWithNearestNeighbors(image_shape)) preprocess_label.add(FromIdToMask(dataset)) self.add(pr.ControlMap(preprocess_image, [0], [0])) self.add(pr.ControlMap(preprocess_label, [1], [1])) H, W = image_shape[:2] self.add(pr.SequenceWrapper({0: {input_name: [H, W, 3]}}, {1: {'masks': [H, W, num_classes]}}))