def get_segmentation_result(self):
# detection threshold for each class
# (this is adaptively set based on the max_per_set constraint)
thresh = -np.inf * np.ones(self.num_classes)
# top_scores will hold one min heap of scores per class (used to enforce
# the max_per_set constraint)
top_scores = [[] for _ in xrange(self.num_classes)]
# all detections and segmentation are collected into a list:
# Since the number of dets/segs are of variable size
all_boxes = [[[] for _ in xrange(self.num_images)]
for _ in xrange(self.num_classes)]
all_masks = [[[] for _ in xrange(self.num_images)]
for _ in xrange(self.num_classes)]
_t = {'im_detect': Timer(), 'misc': Timer()}
for i in xrange(self.num_images):
im = cv2.imread(self.imdb.image_path_at(i))
_t['im_detect'].tic()
masks, boxes, seg_scores = self._segmentation_forward(im)
_t['im_detect'].toc()
if not cfg.TEST.USE_MASK_MERGE:
for j in xrange(1, self.num_classes):
inds = np.where(seg_scores[:, j] > thresh[j])[0]
cls_scores = seg_scores[inds, j]
cls_boxes = boxes[inds, :]
cls_masks = masks[inds, :]
top_inds = np.argsort(-cls_scores)[:self.max_per_image]
cls_scores = cls_scores[top_inds]
cls_boxes = cls_boxes[top_inds, :]
cls_masks = cls_masks[top_inds, :]
# push new scores onto the min heap
for val in cls_scores:
heapq.heappush(top_scores[j], val)
# if we've collected more than the max number of detection,
# then pop items off the min heap and update the class threshold
if len(top_scores[j]) > self.max_per_set:
while len(top_scores[j]) > self.max_per_set:
heapq.heappop(top_scores[j])
thresh[j] = top_scores[j][0]
# Add new boxes into record
box_before_nms = np.hstack((cls_boxes, cls_scores[:, np.newaxis]))\
.astype(np.float32, copy=False)
mask_before_nms = cls_masks.astype(np.float32, copy=False)
all_boxes[j][i], all_masks[j][i] = apply_nms_mask_single(
box_before_nms, mask_before_nms, cfg.TEST.NMS)
else:
if cfg.TEST.USE_GPU_MASK_MERGE:
result_mask, result_box = gpu_mask_voting(
masks, boxes, seg_scores, self.num_classes,
self.max_per_image, im.shape[1], im.shape[0])
else:
result_box, result_mask = cpu_mask_voting(
masks, boxes, seg_scores, self.num_classes,
self.max_per_image, im.shape[1], im.shape[0])
# no need to create a min heap since the output will not exceed max number of detection
for j in xrange(1, self.num_classes):
all_boxes[j][i] = result_box[j - 1]
all_masks[j][i] = result_mask[j - 1]
print 'process image %d/%d, forward average time %f' % (
i, self.num_images, _t['im_detect'].average_time)
for j in xrange(1, self.num_classes):
for i in xrange(self.num_images):
inds = np.where(all_boxes[j][i][:, -1] > thresh[j])[0]
all_boxes[j][i] = all_boxes[j][i][inds, :]
all_masks[j][i] = all_masks[j][i][inds]
return all_boxes, all_masks
img_name = img_name.strip()
im_name = os.path.join(img_name + '.tif')
print 'Demo for data/demo/{}'.format(im_name)
print os.path.join(data_path, im_name)
gt_image = os.path.join(data_path, im_name)
im = cv2.imread(gt_image)
#im = img[:200,:200,:]
print 'im size:{}'.format(im.shape)
start = time.time()
boxes, masks, seg_scores = im_detect(im, net)
#print 'boxes{},masks{},seg_scores{}'.format(boxes.shape,masks.shape,seg_scores.shape)
print 'boxes{}'.format(boxes.shape)
end = time.time()
print 'forward time %f' % (end - start)
result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores,
len(CLASSES) + 1, 300,
im.shape[1], im.shape[0])
#print 'res_box{},res_mask{}'.format(result_box.shape,result_mask.shape)
#pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name, CLASSES)
pred_dict = get_vis_dict(result_box, result_mask, data_path + im_name,
CLASSES)
img_width = im.shape[1]
img_height = im.shape[0]
inst_img, cls_img = _convert_pred_to_image(img_width, img_height,
pred_dict)
color_map = _get_voc_color_map()
target_cls_file = os.path.join(res_path,
'cls_maskSeg_' + img_name + '.jpg')
cls_out_img = np.zeros((img_height, img_width, 3))
for i in xrange(img_height):
def process_jpg(CLASSES, net, im_name, jpg_dir, tif_name, tif_dir,
geojson_name, geojson_dir, updated_geojson_name,
updated_geojson_dir, vis_threshold):
print "\n"
print "Processing {}".format(jpg_dir + '/' + im_name)
#Create initia geojson (has extra DN:0 polygons)
gt_image = os.path.join(jpg_dir, im_name)
im = cv2.imread(gt_image)
img_width = im.shape[1]
img_height = im.shape[0]
boxes, masks, seg_scores = im_detect(im, net)
result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores,
len(CLASSES) + 1, 300,
im.shape[1], im.shape[0])
pred_dict = get_vis_dict(result_box, result_mask, jpg_dir + im_name,
CLASSES, vis_threshold)
num_inst = len(pred_dict['boxes'])
#Get image number
image_number_search = re.search('(?<=img)\w+', im_name)
image_number = image_number_search.group(0)
#Open tif file
print "Opening {}".format(tif_dir + '/' + tif_name)
srcRas_ds = gdal.Open(tif_dir + '/' + tif_name)
#Stuff to create before entering instance loop
geom = srcRas_ds.GetGeoTransform()
proj = srcRas_ds.GetProjection()
memdrv = gdal.GetDriverByName('MEM')
inst_img = np.zeros((img_height, img_width))
src_ds = memdrv.Create('', inst_img.shape[1], inst_img.shape[0], num_inst)
src_ds.SetGeoTransform(geom)
src_ds.SetProjection(proj)
#Create geojson data source
drv = ogr.GetDriverByName("geojson")
dst_ds = drv.CreateDataSource(geojson_dir + '/' + geojson_name)
#Create layer
dst_layername = "building_layer_name"
dst_layer = dst_ds.CreateLayer(dst_layername, srs=None)
dst_layer.CreateField(ogr.FieldDefn("DN", ogr.OFTInteger))
for inst_index in xrange(num_inst):
box = np.round(pred_dict['boxes'][inst_index]).astype(int)
mask = pred_dict['masks'][inst_index]
cls_num = pred_dict['cls_name'][inst_index]
box[0] = min(max(box[0], 0), img_width - 1)
box[1] = min(max(box[1], 0), img_height - 1)
box[2] = min(max(box[2], 0), img_width - 1)
box[3] = min(max(box[3], 0), img_height - 1)
mask = cv2.resize(mask.astype(np.float32),
(box[2] - box[0] + 1, box[3] - box[1] + 1))
mask = mask >= cfg.BINARIZE_THRESH
mask = mask.astype(int)
part1 = (1) * mask.astype(np.float32)
part2 = np.multiply(np.logical_not(mask), inst_img[box[1]:box[3] + 1,
box[0]:box[2] + 1])
#Reset instance image to 0's
inst_img = np.zeros((img_height, img_width))
inst_img[box[1]:box[3] + 1, box[0]:box[2] + 1] = part1 + part2
inst_img = inst_img.astype(int)
band = src_ds.GetRasterBand(1)
band.WriteArray(inst_img)
gdal.Polygonize(band, None, dst_layer, 0, [], callback=None)
dst_ds = None
#Now reformat the geojson we just created
print 'Reformat geojson {}'.format(geojson_name)
#set index to 0
i = 0
geojson_full_name = geojson_dir + '/' + geojson_name
with open(geojson_full_name, 'r') as f:
data = json.load(f)
while (i < len(data['features'])):
if (data['features'][i]['properties']['DN'] == 0):
data['features'].pop(i)
elif (area(data['features'][i]['geometry']) <= 50):
data['features'].pop(i)
else:
i = i + 1
#Write geojson
updated_geojson_full_name = updated_geojson_dir + '/' + updated_geojson_name
with open(updated_geojson_full_name, 'w') as outfile:
json.dump(data, outfile, sort_keys=True)
print "Geojson reformatted!"
for i in xrange(2):
_, _, _ = im_detect(im, net)
im_names = ['2008_000533.jpg', '2008_000910.jpg', '2008_001602.jpg',
'2008_001717.jpg', '2008_008093.jpg']
demo_dir = './data/demo'
for im_name in im_names:
print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'
print 'Demo for data/demo/{}'.format(im_name)
gt_image = os.path.join(demo_dir, im_name)
im = cv2.imread(gt_image)
start = time.time()
boxes, masks, seg_scores = im_detect(im, net)
end = time.time()
print 'forward time %f' % (end-start)
result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores, len(CLASSES) + 1,
100, im.shape[1], im.shape[0])
pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name, CLASSES)
img_width = im.shape[1]
img_height = im.shape[0]
inst_img, cls_img = _convert_pred_to_image(img_width, img_height, pred_dict)
color_map = _get_voc_color_map()
target_cls_file = os.path.join(demo_dir, 'cls_' + im_name)
cls_out_img = np.zeros((img_height, img_width, 3))
for i in xrange(img_height):
for j in xrange(img_width):
cls_out_img[i][j] = color_map[cls_img[i][j]][::-1]
cv2.imwrite(target_cls_file, cls_out_img)
background = Image.open(gt_image)
def get_segmentation_result(self):
# detection threshold for each class
# (this is adaptively set based on the max_per_set constraint)
thresh = -np.inf * np.ones(self.num_classes)
# top_scores will hold one min heap of scores per class (used to enforce
# the max_per_set constraint)
top_scores = [[] for _ in xrange(self.num_classes)]
# all detections and segmentation are collected into a list:
# Since the number of dets/segs are of variable size
all_boxes = [[[] for _ in xrange(self.num_images)]
for _ in xrange(self.num_classes)]
all_masks = [[[] for _ in xrange(self.num_images)]
for _ in xrange(self.num_classes)]
_t = {'im_detect': Timer(), 'misc': Timer()}
for i in xrange(self.num_images):
im = cv2.imread(self.imdb.image_path_at(i))
_t['im_detect'].tic()
masks, boxes, seg_scores = self._segmentation_forward(im)
_t['im_detect'].toc()
if not cfg.TEST.USE_MASK_MERGE:
for j in xrange(1, self.num_classes):
inds = np.where(seg_scores[:, j] > thresh[j])[0]
cls_scores = seg_scores[inds, j]
cls_boxes = boxes[inds, :]
cls_masks = masks[inds, :]
top_inds = np.argsort(-cls_scores)[:self.max_per_image]
cls_scores = cls_scores[top_inds]
cls_boxes = cls_boxes[top_inds, :]
cls_masks = cls_masks[top_inds, :]
# push new scores onto the min heap
for val in cls_scores:
heapq.heappush(top_scores[j], val)
# if we've collected more than the max number of detection,
# then pop items off the min heap and update the class threshold
if len(top_scores[j]) > self.max_per_set:
while len(top_scores[j]) > self.max_per_set:
heapq.heappop(top_scores[j])
thresh[j] = top_scores[j][0]
# Add new boxes into record
box_before_nms = np.hstack((cls_boxes, cls_scores[:, np.newaxis]))\
.astype(np.float32, copy=False)
mask_before_nms = cls_masks.astype(np.float32, copy=False)
all_boxes[j][i], all_masks[j][i] = apply_nms_mask_single(box_before_nms, mask_before_nms, cfg.TEST.NMS)
else:
if cfg.TEST.USE_GPU_MASK_MERGE:
result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores, self.num_classes,
self.max_per_image, im.shape[1], im.shape[0])
else:
result_box, result_mask = cpu_mask_voting(masks, boxes, seg_scores, self.num_classes,
self.max_per_image, im.shape[1], im.shape[0])
# no need to create a min heap since the output will not exceed max number of detection
for j in xrange(1, self.num_classes):
all_boxes[j][i] = result_box[j-1]
all_masks[j][i] = result_mask[j-1]
print 'process image %d/%d, forward average time %f' % (i, self.num_images,
_t['im_detect'].average_time)
for j in xrange(1, self.num_classes):
for i in xrange(self.num_images):
inds = np.where(all_boxes[j][i][:, -1] > thresh[j])[0]
all_boxes[j][i] = all_boxes[j][i][inds, :]
all_masks[j][i] = all_masks[j][i][inds]
return all_boxes, all_masks
def mnc_pixlevel_detect(url_or_np_array, categories=['person']):
demo_dir = './'
start = time.time()
if isinstance(url_or_np_array, basestring):
im = url_to_image(url_or_np_array)
im_name = url_or_np_array.split('/')[-1]
else:
im = url_or_np_array
im_name = str(int(time.time())) + '.jpg'
# resize to max dim of max_dim
if im is None:
print('uuhhh got None image')
return None
max_dim = 400
h, w = im.shape[0:2]
compress_factor = float(max(h, w)) / max_dim
new_h = int(float(h) / compress_factor)
new_w = int(float(w) / compress_factor)
orig_im = im
im = cv2.resize(im, (new_w, new_h))
actual_new_h, actual_new_w = im.shape[0:2]
print('old w,h {}x{}, planned {}x{}, actual {}x{}'.format(
w, h, new_w, new_h, actual_new_w, actual_new_h))
gt_image = os.path.join(demo_dir, im_name)
# print gt_image
cv2.imwrite(gt_image, im)
boxes, masks, seg_scores = im_detect(im, net)
end = time.time()
print 'im_detect time %f' % (end - start)
result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores,
len(CLASSES) + 1, 100,
im.shape[1], im.shape[0])
# return result_mask, result_box
start = time.time()
pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name,
CLASSES)
end = time.time()
print 'gpu vis dicttime %f' % (end - start)
# res_dict = {'image_name': img_name,
# 'cls_name': cls_for_img,
# 'boxes': box_for_img,
# 'masks': mask_for_img}
# print('preddict:'+str( pred_dict))
# find indices of desired cats - jr
desired_categories = []
for cat in categories:
catno = CLASSES.index(cat) + 1
desired_categories.append(catno)
print('desired catnos:' + str(desired_categories))
#########33end jr
print('boxes:' + str(pred_dict['boxes']))
start = time.time()
img_width = im.shape[1]
img_height = im.shape[0]
inst_img, cls_img = _convert_pred_to_image(img_width, img_height,
pred_dict)
color_map = _get_voc_color_map()
target_cls_file = os.path.join(demo_dir, 'cls_' + im_name)
cls_out_img = np.zeros((img_height, img_width, 3))
#####this loop over x,y is retarded
for i in xrange(img_height):
for j in xrange(img_width):
#dont color the unwanted classes - jr
if cls_img[i][j] in desired_categories:
cls_out_img[i][j] = color_map[cls_img[i][j]][::-1]
# nonworking stab at replacement
# cls_out_img = cls_img[color_map[cls_img][::-1] for cls_img in desired_categories]
# cls_out_img[i][j] = color_map[cls_img[i][j]][::-1]
###########end jr (i took out line above too
cv2.imwrite(target_cls_file, cls_out_img)
end = time.time()
print 'convert pred to image %f' % (end - start)
start = time.time()
background = Image.open(gt_image)
mask = Image.open(target_cls_file)
background = background.convert('RGBA')
mask = mask.convert('RGBA')
end = time.time()
print 'superimpose 0 time %f' % (end - start)
start = time.time()
#remove unwanted class boxes - jr
print('classes:' + str(pred_dict['cls_name']))
desired_boxes = []
for i in range(len(pred_dict['boxes'])):
current_classno = pred_dict['cls_name'][i]
current_classname = CLASSES[current_classno - 1]
print('i {} cat {} name {} box {}'.format(i, pred_dict['cls_name'][i],
current_classname,
pred_dict['boxes'][i]))
if current_classname in categories:
print('cat accepted')
desired_boxes.append(pred_dict['boxes'][i])
# rescale the bbs - jr
for bbox in desired_boxes:
bbox[0] = int(bbox[0] * compress_factor)
bbox[1] = int(bbox[1] * compress_factor)
bbox[2] = int(bbox[2] * compress_factor)
bbox[3] = int(bbox[3] * compress_factor)
superimpose_image = Image.blend(background, mask, 0.8)
superimpose_name = os.path.join(demo_dir, 'final_' + im_name)
superimpose_image.save(superimpose_name, 'JPEG')
im = cv2.imread(superimpose_name)
for bbox in desired_boxes:
cv2.rectangle(im, (int(bbox[0]), int(bbox[1])),
(int(bbox[2]), int(bbox[3])),
color=[255, 255, 100],
thickness=4)
cv2.putText(im,
'person:' + str(round(bbox[4], 3)),
org=(int(bbox[0]), int(bbox[1]) - 10),
fontFace=cv2.FONT_HERSHEY_PLAIN,
fontScale=1,
color=[100, 100, 255])
end = time.time()
print 'superimpose 1 time %f' % (end - start)
print('boxes:' + str(pred_dict['boxes']))
print('accepted boxes:' + str(desired_boxes))
return result_mask, result_box, im, im_name, orig_im, desired_boxes, compress_factor, superimpose_name
im = cv2.imread(gt_image)
if do_resize:
print("Resizing image to {}x{}.".format(args.inference_width,
args.inference_height))
net_input = cv2.resize(im, (args.inference_width, args.inference_height))
# print(net_input.shape)
# print(im.shape)
else:
net_input = im
start = time.time()
boxes, masks, seg_scores = im_detect(net_input, net)
end = time.time()
print('forward time %f' % (end-start))
result_mask, result_box = gpu_mask_voting(masks, boxes, seg_scores, len(CLASSES) + 1,
100, net_input.shape[1], net_input.shape[0])
print('GPU mask voting OK')
pred_dict = get_vis_dict(result_box, result_mask, 'data/demo/' + im_name, CLASSES)
# TODO(andrei): If resizing image, blow the result back up.
img_width = net_input.shape[1]
img_height = net_input.shape[0]
# img_width = im.shape[1]
# img_height = im.shape[0]
# TODO(andrei): Correctly handle bounding box repositioning based on
# the initial resize.
inst_img, cls_img, instances = _convert_pred_to_image(img_width, img_height, pred_dict)
color_map = _get_voc_color_map()
target_cls_file = os.path.join(demo_result_dir, 'cls_' + im_name)