def get_segmentation_result(self):
        # detection threshold for each class
        # (this is adaptively set based on the max_per_set constraint)
        thresh = -np.inf * np.ones(self.num_classes)
        # top_scores will hold one min heap of scores per class (used to enforce
        # the max_per_set constraint)
        top_scores = [[] for _ in xrange(self.num_classes)]
        # all detections and segmentation are collected into a list:
        # Since the number of dets/segs are of variable size
        all_boxes = [[[] for _ in xrange(self.num_images)]
                     for _ in xrange(self.num_classes)]
        all_masks = [[[] for _ in xrange(self.num_images)]
                     for _ in xrange(self.num_classes)]

        _t = {'im_detect': Timer(), 'misc': Timer()}
        for i in xrange(self.num_images):
            im = cv2.imread(self.imdb.image_path_at(i))
            _t['im_detect'].tic()
            masks, boxes, seg_scores = self._segmentation_forward(im)
            _t['im_detect'].toc()
            if not cfg.TEST.USE_MASK_MERGE:
                for j in xrange(1, self.num_classes):
                    inds = np.where(seg_scores[:, j] > thresh[j])[0]
                    cls_scores = seg_scores[inds, j]
                    cls_boxes = boxes[inds, :]
                    cls_masks = masks[inds, :]
                    top_inds = np.argsort(-cls_scores)[:self.max_per_image]
                    cls_scores = cls_scores[top_inds]
                    cls_boxes = cls_boxes[top_inds, :]
                    cls_masks = cls_masks[top_inds, :]
                    # push new scores onto the min heap
                    for val in cls_scores:
                        heapq.heappush(top_scores[j], val)
                    # if we've collected more than the max number of detection,
                    # then pop items off the min heap and update the class threshold
                    if len(top_scores[j]) > self.max_per_set:
                        while len(top_scores[j]) > self.max_per_set:
                            heapq.heappop(top_scores[j])
                        thresh[j] = top_scores[j][0]
                    # Add new boxes into record
                    box_before_nms = np.hstack((cls_boxes, cls_scores[:, np.newaxis]))\
                        .astype(np.float32, copy=False)
                    mask_before_nms = cls_masks.astype(np.float32, copy=False)
                    all_boxes[j][i], all_masks[j][i] = apply_nms_mask_single(
                        box_before_nms, mask_before_nms, cfg.TEST.NMS)
            else:
                if cfg.TEST.USE_GPU_MASK_MERGE:
                    result_mask, result_box = gpu_mask_voting(
                        masks, boxes, seg_scores, self.num_classes,
                        self.max_per_image, im.shape[1], im.shape[0])
                else:
                    result_box, result_mask = cpu_mask_voting(
                        masks, boxes, seg_scores, self.num_classes,
                        self.max_per_image, im.shape[1], im.shape[0])
                # no need to create a min heap since the output will not exceed max number of detection
                for j in xrange(1, self.num_classes):
                    all_boxes[j][i] = result_box[j - 1]
                    all_masks[j][i] = result_mask[j - 1]

            print 'process image %d/%d, forward average time %f' % (
                i, self.num_images, _t['im_detect'].average_time)

        for j in xrange(1, self.num_classes):
            for i in xrange(self.num_images):
                inds = np.where(all_boxes[j][i][:, -1] > thresh[j])[0]
                all_boxes[j][i] = all_boxes[j][i][inds, :]
                all_masks[j][i] = all_masks[j][i][inds]

        return all_boxes, all_masks
        img_name = img_name.strip()
        im_name = os.path.join(img_name + '.tif')
        print 'Demo for data/demo/{}'.format(im_name)
        print os.path.join(data_path, im_name)
        gt_image = os.path.join(data_path, im_name)
        im = cv2.imread(gt_image)
        #im = img[:200,:200,:]
        print 'im size:{}'.format(im.shape)
        start = time.time()
        boxes, masks, seg_scores = im_detect(im, net)
        #print 'boxes{},masks{},seg_scores{}'.format(boxes.shape,masks.shape,seg_scores.shape)
        print 'boxes{}'.format(boxes.shape)
        end = time.time()
        print 'forward time %f' % (end - start)
        result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores,
                                                  len(CLASSES) + 1, 300,
                                                  im.shape[1], im.shape[0])
        #print 'res_box{},res_mask{}'.format(result_box.shape,result_mask.shape)
        #pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name, CLASSES)
        pred_dict = get_vis_dict(result_box, result_mask, data_path + im_name,
                                 CLASSES)

        img_width = im.shape[1]
        img_height = im.shape[0]
        inst_img, cls_img = _convert_pred_to_image(img_width, img_height,
                                                   pred_dict)
        color_map = _get_voc_color_map()
        target_cls_file = os.path.join(res_path,
                                       'cls_maskSeg_' + img_name + '.jpg')
        cls_out_img = np.zeros((img_height, img_width, 3))
        for i in xrange(img_height):
def process_jpg(CLASSES, net, im_name, jpg_dir, tif_name, tif_dir,
                geojson_name, geojson_dir, updated_geojson_name,
                updated_geojson_dir, vis_threshold):
    print "\n"
    print "Processing {}".format(jpg_dir + '/' + im_name)
    #Create initia geojson (has extra DN:0 polygons)

    gt_image = os.path.join(jpg_dir, im_name)
    im = cv2.imread(gt_image)

    img_width = im.shape[1]
    img_height = im.shape[0]

    boxes, masks, seg_scores = im_detect(im, net)

    result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores,
                                              len(CLASSES) + 1, 300,
                                              im.shape[1], im.shape[0])
    pred_dict = get_vis_dict(result_box, result_mask, jpg_dir + im_name,
                             CLASSES, vis_threshold)

    num_inst = len(pred_dict['boxes'])

    #Get image number
    image_number_search = re.search('(?<=img)\w+', im_name)
    image_number = image_number_search.group(0)

    #Open tif file
    print "Opening {}".format(tif_dir + '/' + tif_name)
    srcRas_ds = gdal.Open(tif_dir + '/' + tif_name)

    #Stuff to create before entering instance loop
    geom = srcRas_ds.GetGeoTransform()
    proj = srcRas_ds.GetProjection()
    memdrv = gdal.GetDriverByName('MEM')
    inst_img = np.zeros((img_height, img_width))
    src_ds = memdrv.Create('', inst_img.shape[1], inst_img.shape[0], num_inst)
    src_ds.SetGeoTransform(geom)
    src_ds.SetProjection(proj)

    #Create geojson data source
    drv = ogr.GetDriverByName("geojson")
    dst_ds = drv.CreateDataSource(geojson_dir + '/' + geojson_name)

    #Create layer
    dst_layername = "building_layer_name"
    dst_layer = dst_ds.CreateLayer(dst_layername, srs=None)
    dst_layer.CreateField(ogr.FieldDefn("DN", ogr.OFTInteger))

    for inst_index in xrange(num_inst):
        box = np.round(pred_dict['boxes'][inst_index]).astype(int)
        mask = pred_dict['masks'][inst_index]
        cls_num = pred_dict['cls_name'][inst_index]

        box[0] = min(max(box[0], 0), img_width - 1)
        box[1] = min(max(box[1], 0), img_height - 1)
        box[2] = min(max(box[2], 0), img_width - 1)
        box[3] = min(max(box[3], 0), img_height - 1)

        mask = cv2.resize(mask.astype(np.float32),
                          (box[2] - box[0] + 1, box[3] - box[1] + 1))
        mask = mask >= cfg.BINARIZE_THRESH
        mask = mask.astype(int)

        part1 = (1) * mask.astype(np.float32)
        part2 = np.multiply(np.logical_not(mask), inst_img[box[1]:box[3] + 1,
                                                           box[0]:box[2] + 1])

        #Reset instance image to 0's
        inst_img = np.zeros((img_height, img_width))
        inst_img[box[1]:box[3] + 1, box[0]:box[2] + 1] = part1 + part2
        inst_img = inst_img.astype(int)

        band = src_ds.GetRasterBand(1)
        band.WriteArray(inst_img)

        gdal.Polygonize(band, None, dst_layer, 0, [], callback=None)
    dst_ds = None

    #Now reformat the geojson we just created
    print 'Reformat geojson {}'.format(geojson_name)

    #set index to 0
    i = 0
    geojson_full_name = geojson_dir + '/' + geojson_name
    with open(geojson_full_name, 'r') as f:
        data = json.load(f)
        while (i < len(data['features'])):
            if (data['features'][i]['properties']['DN'] == 0):
                data['features'].pop(i)
            elif (area(data['features'][i]['geometry']) <= 50):
                data['features'].pop(i)
            else:
                i = i + 1

    #Write geojson
    updated_geojson_full_name = updated_geojson_dir + '/' + updated_geojson_name
    with open(updated_geojson_full_name, 'w') as outfile:
        json.dump(data, outfile, sort_keys=True)
        print "Geojson reformatted!"
Example #4
0
File: demo.py Project: kekedan/MNC
    for i in xrange(2):
        _, _, _ = im_detect(im, net)

    im_names = ['2008_000533.jpg', '2008_000910.jpg', '2008_001602.jpg',
                '2008_001717.jpg', '2008_008093.jpg']
    demo_dir = './data/demo'
    for im_name in im_names:
        print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
        print 'Demo for data/demo/{}'.format(im_name)
        gt_image = os.path.join(demo_dir, im_name)
        im = cv2.imread(gt_image)
        start = time.time()
        boxes, masks, seg_scores = im_detect(im, net)
        end = time.time()
        print 'forward time %f' % (end-start)
        result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores, len(CLASSES) + 1,
                                                  100, im.shape[1], im.shape[0])
        pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name, CLASSES)

        img_width = im.shape[1]
        img_height = im.shape[0]
        
        inst_img, cls_img = _convert_pred_to_image(img_width, img_height, pred_dict)
        color_map = _get_voc_color_map()
        target_cls_file = os.path.join(demo_dir, 'cls_' + im_name)
        cls_out_img = np.zeros((img_height, img_width, 3))
        for i in xrange(img_height):
            for j in xrange(img_width):
                cls_out_img[i][j] = color_map[cls_img[i][j]][::-1]
        cv2.imwrite(target_cls_file, cls_out_img)
        
        background = Image.open(gt_image)
Example #5
0
    def get_segmentation_result(self):
        # detection threshold for each class
        # (this is adaptively set based on the max_per_set constraint)
        thresh = -np.inf * np.ones(self.num_classes)
        # top_scores will hold one min heap of scores per class (used to enforce
        # the max_per_set constraint)
        top_scores = [[] for _ in xrange(self.num_classes)]
        # all detections and segmentation are collected into a list:
        # Since the number of dets/segs are of variable size
        all_boxes = [[[] for _ in xrange(self.num_images)]
                     for _ in xrange(self.num_classes)]
        all_masks = [[[] for _ in xrange(self.num_images)]
                     for _ in xrange(self.num_classes)]

        _t = {'im_detect': Timer(), 'misc': Timer()}
        for i in xrange(self.num_images):
            im = cv2.imread(self.imdb.image_path_at(i))
            _t['im_detect'].tic()
            masks, boxes, seg_scores = self._segmentation_forward(im)
            _t['im_detect'].toc()
            if not cfg.TEST.USE_MASK_MERGE:
                for j in xrange(1, self.num_classes):
                    inds = np.where(seg_scores[:, j] > thresh[j])[0]
                    cls_scores = seg_scores[inds, j]
                    cls_boxes = boxes[inds, :]
                    cls_masks = masks[inds, :]
                    top_inds = np.argsort(-cls_scores)[:self.max_per_image]
                    cls_scores = cls_scores[top_inds]
                    cls_boxes = cls_boxes[top_inds, :]
                    cls_masks = cls_masks[top_inds, :]
                    # push new scores onto the min heap
                    for val in cls_scores:
                        heapq.heappush(top_scores[j], val)
                    # if we've collected more than the max number of detection,
                    # then pop items off the min heap and update the class threshold
                    if len(top_scores[j]) > self.max_per_set:
                        while len(top_scores[j]) > self.max_per_set:
                            heapq.heappop(top_scores[j])
                        thresh[j] = top_scores[j][0]
                    # Add new boxes into record
                    box_before_nms = np.hstack((cls_boxes, cls_scores[:, np.newaxis]))\
                        .astype(np.float32, copy=False)
                    mask_before_nms = cls_masks.astype(np.float32, copy=False)
                    all_boxes[j][i], all_masks[j][i] = apply_nms_mask_single(box_before_nms, mask_before_nms, cfg.TEST.NMS)
            else:
                if cfg.TEST.USE_GPU_MASK_MERGE:
                    result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores, self.num_classes,
                                                              self.max_per_image, im.shape[1], im.shape[0])
                else:
                    result_box, result_mask = cpu_mask_voting(masks, boxes, seg_scores, self.num_classes,
                                                              self.max_per_image, im.shape[1], im.shape[0])
                # no need to create a min heap since the output will not exceed max number of detection
                for j in xrange(1, self.num_classes):
                    all_boxes[j][i] = result_box[j-1]
                    all_masks[j][i] = result_mask[j-1]

            print 'process image %d/%d, forward average time %f' % (i, self.num_images,
                                                                    _t['im_detect'].average_time)

        for j in xrange(1, self.num_classes):
            for i in xrange(self.num_images):
                inds = np.where(all_boxes[j][i][:, -1] > thresh[j])[0]
                all_boxes[j][i] = all_boxes[j][i][inds, :]
                all_masks[j][i] = all_masks[j][i][inds]

        return all_boxes, all_masks
Example #6
0
def mnc_pixlevel_detect(url_or_np_array, categories=['person']):
    demo_dir = './'
    start = time.time()
    if isinstance(url_or_np_array, basestring):
        im = url_to_image(url_or_np_array)
        im_name = url_or_np_array.split('/')[-1]
    else:
        im = url_or_np_array
        im_name = str(int(time.time())) + '.jpg'
    # resize to max dim of max_dim
    if im is None:
        print('uuhhh got None image')
        return None
    max_dim = 400
    h, w = im.shape[0:2]
    compress_factor = float(max(h, w)) / max_dim
    new_h = int(float(h) / compress_factor)
    new_w = int(float(w) / compress_factor)
    orig_im = im

    im = cv2.resize(im, (new_w, new_h))
    actual_new_h, actual_new_w = im.shape[0:2]
    print('old w,h {}x{}, planned {}x{}, actual {}x{}'.format(
        w, h, new_w, new_h, actual_new_w, actual_new_h))

    gt_image = os.path.join(demo_dir, im_name)
    # print gt_image
    cv2.imwrite(gt_image, im)

    boxes, masks, seg_scores = im_detect(im, net)
    end = time.time()
    print 'im_detect time %f' % (end - start)

    result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores,
                                              len(CLASSES) + 1, 100,
                                              im.shape[1], im.shape[0])

    #   return result_mask, result_box

    start = time.time()
    pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name,
                             CLASSES)
    end = time.time()
    print 'gpu vis dicttime %f' % (end - start)

    #    res_dict = {'image_name': img_name,
    #                'cls_name': cls_for_img,
    #                'boxes': box_for_img,
    #                'masks': mask_for_img}

    #    print('preddict:'+str( pred_dict))

    # find indices of desired cats - jr
    desired_categories = []
    for cat in categories:
        catno = CLASSES.index(cat) + 1
        desired_categories.append(catno)
    print('desired catnos:' + str(desired_categories))
    #########33end jr

    print('boxes:' + str(pred_dict['boxes']))
    start = time.time()
    img_width = im.shape[1]
    img_height = im.shape[0]

    inst_img, cls_img = _convert_pred_to_image(img_width, img_height,
                                               pred_dict)
    color_map = _get_voc_color_map()
    target_cls_file = os.path.join(demo_dir, 'cls_' + im_name)
    cls_out_img = np.zeros((img_height, img_width, 3))

    #####this loop over x,y is retarded
    for i in xrange(img_height):
        for j in xrange(img_width):
            #dont color the unwanted classes - jr
            if cls_img[i][j] in desired_categories:
                cls_out_img[i][j] = color_map[cls_img[i][j]][::-1]
# nonworking stab at replacement
#    cls_out_img = cls_img[color_map[cls_img][::-1] for cls_img in desired_categories]

#        cls_out_img[i][j] = color_map[cls_img[i][j]][::-1]
###########end jr (i took out line above too

    cv2.imwrite(target_cls_file, cls_out_img)
    end = time.time()

    print 'convert pred to image  %f' % (end - start)

    start = time.time()
    background = Image.open(gt_image)
    mask = Image.open(target_cls_file)
    background = background.convert('RGBA')
    mask = mask.convert('RGBA')

    end = time.time()
    print 'superimpose 0 time %f' % (end - start)
    start = time.time()

    #remove unwanted class boxes - jr
    print('classes:' + str(pred_dict['cls_name']))
    desired_boxes = []
    for i in range(len(pred_dict['boxes'])):
        current_classno = pred_dict['cls_name'][i]
        current_classname = CLASSES[current_classno - 1]
        print('i {} cat {} name {} box {}'.format(i, pred_dict['cls_name'][i],
                                                  current_classname,
                                                  pred_dict['boxes'][i]))
        if current_classname in categories:
            print('cat accepted')
            desired_boxes.append(pred_dict['boxes'][i])
# rescale the bbs - jr
    for bbox in desired_boxes:
        bbox[0] = int(bbox[0] * compress_factor)
        bbox[1] = int(bbox[1] * compress_factor)
        bbox[2] = int(bbox[2] * compress_factor)
        bbox[3] = int(bbox[3] * compress_factor)

    superimpose_image = Image.blend(background, mask, 0.8)
    superimpose_name = os.path.join(demo_dir, 'final_' + im_name)
    superimpose_image.save(superimpose_name, 'JPEG')
    im = cv2.imread(superimpose_name)

    for bbox in desired_boxes:
        cv2.rectangle(im, (int(bbox[0]), int(bbox[1])),
                      (int(bbox[2]), int(bbox[3])),
                      color=[255, 255, 100],
                      thickness=4)
        cv2.putText(im,
                    'person:' + str(round(bbox[4], 3)),
                    org=(int(bbox[0]), int(bbox[1]) - 10),
                    fontFace=cv2.FONT_HERSHEY_PLAIN,
                    fontScale=1,
                    color=[100, 100, 255])

    end = time.time()
    print 'superimpose 1 time %f' % (end - start)

    print('boxes:' + str(pred_dict['boxes']))
    print('accepted boxes:' + str(desired_boxes))
    return result_mask, result_box, im, im_name, orig_im, desired_boxes, compress_factor, superimpose_name
Example #7
0
        im = cv2.imread(gt_image)

        if do_resize:
            print("Resizing image to {}x{}.".format(args.inference_width,
                                                    args.inference_height))
            net_input = cv2.resize(im, (args.inference_width, args.inference_height))
            # print(net_input.shape)
            # print(im.shape)
        else:
            net_input = im

        start = time.time()
        boxes, masks, seg_scores = im_detect(net_input, net)
        end = time.time()
        print('forward time %f' % (end-start))
        result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores, len(CLASSES) + 1,
                                                  100, net_input.shape[1], net_input.shape[0])
        print('GPU mask voting OK')
        pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name, CLASSES)

        # TODO(andrei): If resizing image, blow the result back up.
        img_width = net_input.shape[1]
        img_height = net_input.shape[0]
        # img_width = im.shape[1]
        # img_height = im.shape[0]

        # TODO(andrei): Correctly handle bounding box repositioning based on
        # the initial resize.
        inst_img, cls_img, instances = _convert_pred_to_image(img_width, img_height, pred_dict)

        color_map = _get_voc_color_map()
        target_cls_file = os.path.join(demo_result_dir, 'cls_' + im_name)