コード例 #1
0
    def _getLabels(self, coords, im):

        # need lip_coords in pixel-coordinate units for generating masks
        lip_coords = (len(im) *
                      np.array(helenUtils.getLipCoords(coords))).astype(int)
        mask = utils.getMask([lip_coords], (len(im), len(im[0])),
                             (len(im), len(im[0])))
        mask = np.expand_dims(mask, axis=-1)
        return mask
コード例 #2
0
    def _getLabels(self, coords, im):
        """coords = np.reshape(coords, (self.num_coords, 2))
        heatmap = utils.coordsToHeatmapsFast(coords, self.pdfs)
        heatmap = np.moveaxis(heatmap, 0, -1)
        return heatmap"""

        # need lip_coords in pixel-coordinate units for generating masks
        lip_coords = (len(im) * np.array(
            helenUtils.getLipCoords(
                coords, ibug_version=self.ibug_version))).astype(int)
        mask = utils.getMask([lip_coords], (len(im), len(im[0])),
                             (len(im), len(im[0])))
        mask = np.expand_dims(mask, axis=-1)

        # bbox coords
        lip_coords_normalized = helenUtils.getLipCoords(coords)
        bbox = utils.getBbox(lip_coords_normalized)
        bbox = utils.getExpandedBbox(bbox, 0.5, 0.5)
        return [bbox, mask]
コード例 #3
0
    def _getLabels(self, coords, im):
        coords = np.reshape(coords, (self.num_coords, 2))
        lip_coords = helenUtils.getLipCoords(coords)

        labels = []
        for i in range(len(self.pdfs)):
            masks = utils.coordsToHeatmapsFast(lip_coords, self.pdfs[i])
            masks = np.moveaxis(masks, 0, -1)
            masks /= np.max(masks, axis=(0, 1))

            l = self.mask_side_len / 2**i
            masks = cv2.resize(masks, (l, l), interpolation=cv2.INTER_LINEAR)
            labels.append(masks)
        """
        masks_0 = utils.coordsToHeatmapsFast(coords, self.pdfs)
        heatmap = np.moveaxis(heatmap, 0, -1)
        heatmap /= np.sum(heatmap, axis=(0,1))
        summed = np.sum(heatmap, axis=-1)
        summed /= (np.max(summed) / 4.0)
        summed = np.minimum(summed, 1.0)
        summed = np.expand_dims(summed, axis=-1)
        return [heatmap, summed]
        """
        return labels
コード例 #4
0
    def getTrainingPair(self, im, coords, augment=False):

        if augment:
            # random rotation
            width = len(im[0])
            height = len(im)
            center = [(height - 1) / 2.0, (width - 1) / 2.0]
            max_rot_deg = 10
            rot_deg = max_rot_deg * np.random.rand(1)
            rot_rad = np.deg2rad(rot_deg)
            im = scipy.misc.imrotate(im, rot_deg)
            coords = utils.getRotatedPoints(coords, center, rot_rad)

        bbox = utils.getBbox(coords)
        square = utils.getSquareFromRect(bbox)

        if augment:
            # random scale and shift
            rect = utils.getRandomlyExpandedBbox(square, -0.2, 0.3)
            max_shift = 0.1 * (square[2] - square[0])
            shifts = max_shift * np.random.rand(2)
            rect = np.array(utils.getShiftedBbox(rect, shifts)).astype(int)
        else:
            rect = utils.getExpandedBbox(square, 0.0, 0.0)

        # make sure that rect does not go beyond image borders
        rect = utils.getClippedBbox(im, rect)

        # crop
        coords[:, 0] -= rect[0]
        coords[:, 1] -= rect[1]
        im = utils.getCropped(im, rect)
        """
        # brightness and saturation adjustments
        if augment:
            rand_v_delta = (np.random.rand() - 0.5) * 0.2 * 255
            rand_s_delta = (np.random.rand() - 0.7) * 0.6 * 255
            im_hsv = cv2.cvtColor(im, cv2.COLOR_RGB2HSV)
            im_hsv = im_hsv.astype(np.float32)
            im_hsv[:,:,1] += rand_s_delta
            im_hsv[:,:,2] += rand_v_delta
            im_hsv = np.clip(im_hsv, 0, 255)
            im_hsv = im_hsv.astype(np.uint8)
            im = cv2.cvtColor(im_hsv, cv2.COLOR_HSV2RGB)
        """

        # just the lip coords for now
        # flip
        flipped = bool(np.random.randint(0, 2)) if augment else False
        lip_coords = helenUtils.getLipCoords(coords,
                                             len(im[0]),
                                             flip_x=flipped,
                                             ibug_version=self.ibug_version)
        im = np.fliplr(im) if flipped else im

        # normalize the coords and image
        im = cv2.resize(im, (self.targ_im_len, self.targ_im_len))
        crop_width = rect[3] - rect[1]
        crop_height = rect[2] - rect[0]
        normalized_lip_coords = helenUtils.normalizeCoords(
            lip_coords, crop_width, crop_height)

        # mask from coords
        masks = utils.coordsToHeatmapsFast(normalized_lip_coords, self.pdfs)
        masks = np.moveaxis(masks, 0, -1)
        masks /= np.max(masks, axis=(0, 1))
        hd_masks = utils.coordsToHeatmapsFast(normalized_lip_coords,
                                              self.hd_pdfs)
        hd_masks = np.moveaxis(hd_masks, 0, -1)
        hd_masks /= np.max(hd_masks, axis=(0, 1))
        l = self.mask_side_len
        hd_l = self.hd_mask_side_len

        # try using hd masks for everything!
        masks = cv2.resize(masks, (hd_l, hd_l), interpolation=cv2.INTER_LINEAR)
        hd_masks = cv2.resize(hd_masks, (hd_l, hd_l),
                              interpolation=cv2.INTER_LINEAR)
        return [im], [masks, hd_masks]
コード例 #5
0
def testNormalizedDistanceError(model, batch_generator, ibug_version=True):
    """
    Parameters
    ----------
    batch_generator: 
        Should be class PointsBatchGenerator.
    """
    X, Y = batch_generator.getAllData()
    ims, labels = X[0], Y[0]
    overall_avg = 0.0
    all_avgs = []
    """
    for i in range(len(ims)):
        im = ims[i]
        factors = np.expand_dims([len(im), len(im[0])], axis=0)
        label = labels[i] * factors
        utils.visualizeCoords(ims[i], label)
    """

    #for i in range(len(ims)):
    #    utils.visualizeCoords(ims[i], 224 * labels[i])

    print 'Processing test set of images, counted ' + str(len(ims)) + ': '
    t1 = time.time()
    for i in range(len(ims)):
        im = ims[i]
        #labels[i] = helenUtils.normalizeCoords(labels[i], len(im[0]), len(im))
        eye_dist = helenUtils.getEyeDistance(labels[i],
                                             ibug_version=ibug_version)
        lip_labels = helenUtils.getLipCoords(labels[i],
                                             1.0,
                                             ibug_version=ibug_version)

        # vanilla version
        #lip_preds = np.array(getCoordsFromImage(model, im))

        # cascaded version
        base_masks, residual_masks = getNormalizedCascadedMasksFromImage(
            model, im)
        base_coords = utils.getCoordsFromPointMasks(base_masks, 224, 224,
                                                    'mean')
        residual_coords = utils.getCoordsFromPointMasks(
            residual_masks, 28, 28, 'mean')
        #max_coords = utils.getCoordsFromPointMasks(base_masks, width, height, 'max')
        lip_preds = np.add(base_coords, residual_coords) - 28 / 2.0

        lip_preds[:, 0] /= float(len(im))
        lip_preds[:, 1] /= float(len(im[0]))
        cur_avg = 0.0
        for j in range(0, len(lip_preds)):
            diff = lip_preds[j] - lip_labels[j]
            dist = np.sqrt(diff.dot(diff))
            normalized = dist / eye_dist
            cur_avg += normalized
        cur_avg /= len(lip_preds)

        # expand to image dimensions to visualize
        factors = np.expand_dims([len(im), len(im[0])], axis=0)
        lip_preds *= factors
        lip_labels *= factors

        leye_coord = helenUtils.getLeyeCenter(labels[i],
                                              ibug_version=ibug_version)
        reye_coord = helenUtils.getReyeCenter(labels[i],
                                              ibug_version=ibug_version)
        leye_coord *= np.squeeze(factors)
        reye_coord *= np.squeeze(factors)

        all_coords = np.concatenate([lip_preds, lip_labels], axis=0)
        all_coords = np.concatenate([all_coords, [leye_coord], [reye_coord]],
                                    axis=0)
        pred_indices = np.arange(0, len(all_coords) / 2)
        #utils.visualizeCoords(im, all_coords, pred_indices)
        #print 'eye to eye distance: ' + str(eye_dist)
        #print 'avg error across all points: ' + str(cur_avg)
        overall_avg += cur_avg
        all_avgs.append(cur_avg)
        utils.informProgress(i, len(ims))

    processing_time = time.time() - t1
    all_avgs = sorted(all_avgs)
    overall_avg /= len(ims)

    print '\nMedian normalized landmark error: ' + str(
        all_avgs[len(all_avgs) / 2])
    print 'Average normalized landmark error: ' + str(overall_avg)
    print 'Total processing time: ' + str(processing_time)
    print 'Per-image processing time: ' + str(
        processing_time / float(len(ims)))
コード例 #6
0
 def _getLabels(self, coords, im):
     lip_coords = helenUtils.getLipCoords(coords)
     line_mask = helenUtils.getLipLineMask(
         lip_coords, np.shape(im), (self.mask_side_len, self.mask_side_len))
     return [line_mask]