def random_image(self, height, width):
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
shapes = []
boxes = []
indexes = []
N = random.randint(self.min_leaf, self.max_leaf)
# AZ START INSERT
prev_angle = random.randint(0, 360)
min_x = math.floor(0.4 * width)
max_x = math.floor(0.6 * width)
min_y = math.floor(0.4 * height)
max_y = math.floor(0.6 * height)
x_location = random.randint(min_x, max_x)
y_location = random.randint(min_y, max_y)
# AZ END INSERT
for _ in range(N):
# AZ modified
if self.centered_leaves:
shape, location, scale, angle, index = self.random_shape_centered(height, width, x_location, y_location, prev_angle)
else:
shape, location, scale, angle, index = self.random_shape(height, width)
prev_angle = angle
y, x, _ = np.asarray(self.img2[index]).shape
shapes.append((shape, location, scale, angle, index))
boxes.append([location[1], location[0], location[1] + y, location[0] + x])
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N), 0.5) # 0.3
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
def random_image_multiple_plants(self, height, width):
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
shapes = []
boxes = []
num_plants = random.randint(self.min_plants, self.max_plants)
N = 0
for curr_plant in range(num_plants):
N_curr = random.randint(self.min_leaf, self.max_leaf)
N = N + N_curr
prev_angle = random.randint(0, 360)
min_x = math.floor(0.2 * width)
max_x = math.floor(0.8 * width)
min_y = math.floor(0.2 * height)
max_y = math.floor(0.8 * height)
x_location = random.randint(min_x, max_x)
y_location = random.randint(min_y, max_y)
for _ in range(N_curr):
if self.centered_leaves:
shape, location, scale, angle, index = self.random_shape_centered(height, width, x_location, y_location, prev_angle)
else:
shape, location, scale, angle, index = self.random_shape(height, width)
prev_angle = angle
y, x, channels = np.asarray(self.img2[index]).shape
shapes.append((shape, location, scale, angle, index))
boxes.append([location[1], location[0], location[1] + y, location[0] + x])
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N), 0.5) # 0.3
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
def filter_rois(refined_rois, class_ids, class_scores):
# Remove boxes classified as background
keep = np.where(class_ids > 0)[0]
# Filter out low confidence boxes
if config.DETECTION_MIN_CONFIDENCE:
conf_keep = np.where(
class_scores >= config.DETECTION_MIN_CONFIDENCE)[0]
keep = np.intersect1d(keep, conf_keep)
# Apply per-class non-max suppression
pre_nms_class_ids = class_ids[keep]
pre_nms_scores = class_scores[keep]
pre_nms_boxes = refined_rois[keep]
unique_pre_nms_class_ids = np.unique(pre_nms_class_ids)
nms_keep = []
for class_id in unique_pre_nms_class_ids:
# Pick detections of this class
ixs = np.where(pre_nms_class_ids == class_id)[0]
# Apply NMS
class_keep = utils.non_max_suppression(pre_nms_boxes[ixs],
pre_nms_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep = keep[ixs[class_keep]]
nms_keep = np.union1d(nms_keep, class_keep)
keep = np.intersect1d(keep, nms_keep).astype(np.int32)
return keep
def random_image(self, height, width):
"""Creates random specifications of an image with multiple shapes.
Returns the background color of the image and a list of shape
specifications that can be used to draw the image.
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs] 这句，i是range(N)范围内的数
"""
# Pick random background color
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# print('smy: ',bg_color)
# Generate a few random shapes and record their
# bounding boxes
shapes = []
boxes = []
N = random.randint(1, 4)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
x, y, s = dims
boxes.append([y - s, x - s, y + s, x + s])
# Apply non-max suppression wit 0.3 threshold to avoid
# shapes covering each other
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N),
0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
def random_image(self, height, width):
bg_color = np.array([np.random.randint(0, 255) for _ in range(3)])
shapes, boxes = [], []
N = np.random.randint(1, 4)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
X, Y, size = dims
boxes.append([Y - size, X - size, Y + size, X + size])
# Apply non-max suppression wit 0.3 threshold to avoid
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N),
0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
def random_image(self, height, width):
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# Generate a few random auto and record their
# bounding boxes
auto = []
boxes = []
N = random.randint(1, 4)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
auto.append((shape, color, dims))
x, y, s = dims
boxes.append([y - s, x - s, y + s, x + s])
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N),
0.3)
auto = [s for i, s in enumerate(auto) if i in keep_ixs]
return bg_color, auto
def random_image(self, height, width):
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# Generate a few random shapes and record their
# bounding boxes
shapes = []
boxes = []
N = random.randint(1, 4)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
x, y, s = dims
boxes.append([y - s, x - s, y + s, x + s])
# Apply non-max suppression wit 0.3 threshold to avoid
# shapes covering each other
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N),
0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
def random_image(self, height, width):
"""Creates random specifications of an image with multiple shapes.
Returns a list of shape specifications that can be used to draw the image.
"""
# Generate a few random shapes and record their
# bounding boxes
shapes = []
boxes = []
N = 2 # random.randint(4, 6)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
x, y, s = dims
boxes.append([y - s, x - s, y + s, x + s])
# Apply non-max suppression with 1 threshold to avoid
# shapes covering each other (it was 0.3)
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N), 1)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return shapes
def random_image(self, height, width):
"""Generate random images."""
# randomly pick bg color (3 channels)
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# genereate shapes and record their bouding boxes
shapes = []
boxes = []
N = random.randint(1, 4)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
x, y, s = dims
boxes.append([y - s, x - s, y + s, x + s])
""" threshold = 0.3 for non-max suppersion."""
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N),
0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
def random_image(self, height, width):
"""Creates random specifications of an image with multiple shapes.
Returns the background color of the image and a list of shape
specifications that can be used to draw the image.
"""
# Pick random background color
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# Generate a few random shapes and record their
# bounding boxes
shapes = []
boxes = []
N = random.randint(1, 4)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
x, y, s = dims
boxes.append([y - s, x - s, y + s, x + s])
# Apply non-max suppression wit 0.3 threshold to avoid
# shapes covering each other
keep_ixs = utils.non_max_suppression(
np.array(boxes), np.arange(N), 0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
#%% [markdown]
# #### Per-Class Non-Max Suppression
#%%
# Apply per-class non-max suppression
pre_nms_boxes = refined_proposals[keep]
pre_nms_scores = roi_scores[keep]
pre_nms_class_ids = roi_class_ids[keep]
nms_keep = []
for class_id in np.unique(pre_nms_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_class_ids == class_id)[0]
# Apply NMS
class_keep = utils.non_max_suppression(pre_nms_boxes[ixs],
pre_nms_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep = keep[ixs[class_keep]]
nms_keep = np.union1d(nms_keep, class_keep)
print("{:22}: {} -> {}".format(dataset.class_names[class_id][:20],
keep[ixs], class_keep))
keep = np.intersect1d(keep, nms_keep).astype(np.int32)
print("\nKept after per-class NMS: {}\n{}".format(keep.shape[0], keep))
#%%
# Show final detections
ixs = np.arange(len(keep)) # Display all
# ixs = np.random.randint(0, len(keep), 10) # Display random sample
captions = [
def random_image(self, image_id, height, width):
'''
Creates random specifications of an image with multiple shapes.
Returns the background color of the image and a list of shape
specifications that can be used to draw the image.
'''
# Pick random background color
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# Generate a few random shapes and record their
# bounding boxes
shapes = []
N = random.randint(1,
self.shapes_per_image) # number to shapes in image
shape_choices = ["person", "car", "sun", "building", "tree", "cloud"]
for _ in range(N):
shape = random.choice(shape_choices)
color, dims = self.random_shape(shape, height, width)
shapes.append((shape, color, dims))
if shape == "sun":
shape_choices.remove("sun")
# following two lines have been moved below, after removal of hidden_shapes
# x, y, sx, sy = dims
# boxes.append([y - sy, x - sx, y + sy, x + sx])
##--------------------------------------------------------------------------------
## Reorder shapes by increasing cy to simulate overlay
## (nearer shapes cover farther away shapes)
# order shape objects based on closeness to bottom of image (-1) or top (+1)
# this will result in items closer to the viewer have higher priority in NMS
#--------------------------------------------------------------------------------
sort_lst = [itm[2][1] for itm in shapes]
sorted_shape_ind = np.argsort(np.array(sort_lst))[::+1]
# print(" ===== Before final sort ===== ")
# p4.pprint(shapes)
# print(sort_lst)
# print(sorted_shape_ind)
tmp_shapes = []
for i in sorted_shape_ind:
tmp_shapes.append(shapes[i])
shapes = tmp_shapes
# print(' ===== Sahpes after sorting ===== ')
# p4.pprint(shapes)
##-------------------------------------------------------------------------------
## find and remove shapes completely covered by other shapes
##-------------------------------------------------------------------------------
hidden_shape_ixs = self.find_hidden_shapes(shapes, height, width)
if len(hidden_shape_ixs) > 0:
non_hidden_shapes = [
s for i, s in enumerate(shapes) if i not in hidden_shape_ixs
]
print(' ===> Image Id : (', image_id,
') ---- Zero Mask Encountered ')
print(' ------ Original Shapes ------')
p8.pprint(shapes)
print(
' ------ shapes after removal of totally hidden shapes ------'
)
p8.pprint(non_hidden_shapes)
print(' Number of shapes now is : ', len(non_hidden_shapes))
else:
non_hidden_shapes = shapes
##-------------------------------------------------------------------------------
## build boxes for to pass to non_max_suppression
##-------------------------------------------------------------------------------
boxes = []
for shp in non_hidden_shapes:
x, y, sx, sy = shp[2]
boxes.append([y - sy, x - sx, y + sy, x + sx])
##--------------------------------------------------------------------------------
## Non Maximal Suppression
##--------------------------------------------------------------------------------
# Suppress occulsions more than 0.3 IoU
# Apply non-max suppression with 0.3 threshold to avoid shapes covering each other
# object scores (which dictate the priority) are assigned in the order they were created
assert len(boxes) == len(
non_hidden_shapes), "Problem with the shape and box sizes matching"
N = len(boxes)
keep_ixs = non_max_suppression(np.array(boxes), np.arange(N), 0.29)
shapes = [s for i, s in enumerate(non_hidden_shapes) if i in keep_ixs]
# print('===> Original number of shapes {} # after NMS {}'.format(N, len(shapes)))
#--------------------------------------------------------------------------------
## Reorder shapes to simulate overlay (nearer shapes cover farther away shapes)
# order shape objects based on closeness to bottom of image (-1) or top (+1)
# this will result in items closer to the viewer have higher priority in NMS
#--------------------------------------------------------------------------------
sort_lst = [itm[2][1] for itm in shapes]
sorted_shape_ind = np.argsort(np.array(sort_lst))[::+1]
# print(" ===== Before final sort ===== ")
# p4.pprint(shapes)
# print(sort_lst)
# print(sorted_shape_ind)
tmp_shapes = []
for i in sorted_shape_ind:
tmp_shapes.append(shapes[i])
shapes = tmp_shapes
# print(' ===== Sahpes after sorting ===== ')
# p4.pprint(shapes)
return bg_color, shapes
def display_mrcnn_prediction():
resized_image, image_meta, gt_class_ids, gt_bboxes, gt_masks = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
# Get input and output to classifier and mask heads.
mrcnn = model.run_graph([resized_image], [
("proposals", model.keras_model.get_layer("ROI").output),
("probs", model.keras_model.get_layer("mrcnn_class").output),
("deltas", model.keras_model.get_layer("mrcnn_bbox").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
("detections", model.keras_model.get_layer("mrcnn_detection").output),
])
ax = get_ax(1, 4)
################################## display detections ###############################################
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
padding_start_ix = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:padding_start_ix]
detections = mrcnn['detections'][0, :padding_start_ix]
log('trimmed_detection', detections)
print("{} detections: {}".format(
padding_start_ix,
np.array(dataset.class_names)[det_class_ids]))
captions = [
"{} {:.3f}".format(dataset.class_names[int(class_id)], score)
if class_id > 0 else ""
for class_id, score in zip(detections[:, 4], detections[:, 5])
]
visualize.draw_boxes(resized_image.copy(),
refined_boxes=utils.denorm_boxes(
detections[:, :4], resized_image.shape[:2]),
visibilities=[2] * len(detections),
captions=captions,
title="Detections",
ax=ax[0])
################################### display proposals ##########################################
# Proposals are in normalized coordinates. Scale them to image coordinates.
h, w = resized_image.shape[:2]
proposals = np.around(mrcnn["proposals"][0] *
np.array([h, w, h, w])).astype(np.int32)
# Class ID, score, and mask per proposal
# mrcnn 的 shape 为 (batch_size, num_proposals=1000, num_classes)
proposal_class_ids = np.argmax(mrcnn["probs"][0], axis=1)
proposal_class_scores = mrcnn["probs"][
0, np.arange(proposal_class_ids.shape[0]), proposal_class_ids]
proposal_class_names = np.array(dataset.class_names)[proposal_class_ids]
proposal_positive_ixs = np.where(proposal_class_ids > 0)[0]
# How many ROIs vs empty rows?
print("{} valid proposals out of {}".format(
np.sum(np.any(proposals, axis=1)), proposals.shape[0]))
print("{} positive ROIs".format(len(proposal_positive_ixs)))
# Class counts
print(list(zip(*np.unique(proposal_class_names, return_counts=True))))
# Display a random sample of proposals.
# Proposals classified as background are dotted, and
# the rest show their class and confidence score.
limit = 200
ixs = np.random.randint(0, proposals.shape[0], limit)
captions = [
"{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else ""
for c, s in zip(proposal_class_ids[ixs], proposal_class_scores[ixs])
]
visualize.draw_boxes(resized_image.copy(),
boxes=proposals[ixs],
visibilities=np.where(proposal_class_ids[ixs] > 0, 2,
1),
captions=captions,
title="Proposals Before Refinement",
ax=ax[1])
#################################### apply bounding box refinement #############################
# Class-specific bounding box shifts.
# mrcnn['deltas'] 的 shape 为 (batch_size, num_proposals=1000, num_classes, 4)
proposal_deltas = mrcnn["deltas"][0,
np.arange(proposals.shape[0]),
proposal_class_ids]
log("proposals_deltas", proposal_deltas)
# Apply bounding box transformations
# Shape: (num_proposals=1000, (y1, x1, y2, x2)]
# NOTE: delta 是不分 normalized coordinates 和 pixel coordinates 的
refined_proposals = utils.apply_box_deltas(
proposals, proposal_deltas * config.BBOX_STD_DEV).astype(np.int32)
log("refined_proposals", refined_proposals)
# Show positive proposals
# ids = np.arange(proposals.shape[0]) # Display all
limit = 5
ids = np.random.randint(0, len(proposal_positive_ixs),
limit) # Display random sample
captions = [
"{} {:.3f}".format(dataset.class_names[class_id], score)
if class_id > 0 else "" for class_id, score in zip(
proposal_class_ids[proposal_positive_ixs][ids],
proposal_class_scores[proposal_positive_ixs][ids])
]
visualize.draw_boxes(
resized_image.copy(),
boxes=proposals[proposal_positive_ixs][ids],
refined_boxes=refined_proposals[proposal_positive_ixs][ids],
visibilities=np.where(
proposal_class_ids[proposal_positive_ixs][ids] > 0, 1, 0),
captions=captions,
title="ROIs After Refinement",
ax=ax[2])
#################################### more steps ################################################
# Remove boxes classified as background
keep_proposal_ixs = np.where(proposal_class_ids > 0)[0]
print("Remove background proposals. Keep {}:\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
# Remove low confidence detections
keep_proposal_ixs = np.intersect1d(
keep_proposal_ixs,
np.where(proposal_class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
print("Remove proposals below {} confidence. Keep {}:\n{}".format(
config.DETECTION_MIN_CONFIDENCE, keep_proposal_ixs.shape[0],
keep_proposal_ixs))
# Apply per-class non-max suppression
pre_nms_proposals = refined_proposals[keep_proposal_ixs]
pre_nms_proposal_scores = proposal_class_scores[keep_proposal_ixs]
pre_nms_proposal_class_ids = proposal_class_ids[keep_proposal_ixs]
nms_keep_proposal_ixs = []
for class_id in np.unique(pre_nms_proposal_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_proposal_class_ids == class_id)[0]
# Apply NMS
class_keep = utils.non_max_suppression(pre_nms_proposals[ixs],
pre_nms_proposal_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep_proposal_ixs = keep_proposal_ixs[ixs[class_keep]]
nms_keep_proposal_ixs = np.union1d(nms_keep_proposal_ixs,
class_keep_proposal_ixs)
print("{:12}: {} -> {}".format(dataset.class_names[class_id][:10],
keep_proposal_ixs[ixs],
class_keep_proposal_ixs))
keep_proposal_ixs = np.intersect1d(keep_proposal_ixs,
nms_keep_proposal_ixs).astype(np.int32)
print("\nKeep after per-class NMS: {}\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
#################################### Show final detections #####################################
ixs = np.arange(len(keep_proposal_ixs)) # Display all
# ixs = np.random.randint(0, len(keep), 10) # Display random sample
captions = [
"{} {:.3f}".format(dataset.class_names[c], s) if c > 0 else ""
for c, s in zip(proposal_class_ids[keep_proposal_ixs][ixs],
proposal_class_scores[keep_proposal_ixs][ixs])
]
visualize.draw_boxes(
resized_image.copy(),
boxes=proposals[keep_proposal_ixs][ixs],
refined_boxes=refined_proposals[keep_proposal_ixs][ixs],
visibilities=np.where(proposal_class_ids[keep_proposal_ixs][ixs] > 0,
1, 0),
captions=captions,
title="Detections after NMS",
ax=ax[3])
plt.show()
def muti_inf_remoteSensing_image(model, image_path=None):
'''
use multiple scale (different patch size) for inference, then merge them using non_max_suppression
:param model: trained model
:param image_path:
:return: True if successful, False otherwise
'''
# get parameters
inf_image_dir = parameters.get_string_parameters(para_file,
'inf_images_dir')
muti_patch_w = parameters.get_string_parameters(para_file,
"muti_inf_patch_width")
muti_patch_h = parameters.get_string_parameters(para_file,
"muti_inf_patch_height")
muti_overlay_x = parameters.get_string_parameters(
para_file, "muti_inf_pixel_overlay_x")
muti_overlay_y = parameters.get_string_parameters(
para_file, "muti_inf_pixel_overlay_y")
final_keep_classes = parameters.get_string_parameters(
para_file, "final_keep_classes")
nms_iou_threshold = parameters.get_digit_parameters(
para_file, "nms_iou_threshold", None, 'float')
patch_w_list = [int(item) for item in muti_patch_w.split(',')]
patch_h_list = [int(item) for item in muti_patch_h.split(',')]
overlay_x_list = [int(item) for item in muti_overlay_x.split(',')]
overlay_y_list = [int(item) for item in muti_overlay_y.split(',')]
if final_keep_classes == '':
final_keep_classes = None
else:
final_keep_classes = [
int(item) for item in final_keep_classes.split(',')
]
# inference and save to json files
for patch_w, patch_h, overlay_x, overlay_y in zip(patch_w_list,
patch_h_list,
overlay_x_list,
overlay_y_list):
inf_rs_image_json(model, patch_w, patch_h, overlay_x, overlay_y,
inf_image_dir)
# load all boxes of images
with open(inf_list_file) as file_obj:
files_list = file_obj.readlines()
for img_idx, image_name in enumerate(files_list):
file_pattern = os.path.join(
inf_output_dir, 'I%d_patches_*_*_*' %
img_idx) # e.g., I0_patches_320_320_80_80
proc = subprocess.Popen('ls -d ' + file_pattern,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
profiles, err = proc.communicate()
json_folder_list = profiles.split()
if len(json_folder_list) < 1:
raise IOError('No folder containing json files in %s' %
inf_output_dir)
# bytes to str
if isinstance(json_folder_list[0], bytes):
json_folder_list = [item.decode() for item in json_folder_list]
print('loading json files of image :%d, in %s' %
(img_idx, ','.join(json_folder_list)))
# load all the boxes and scores
mrcnn_r_list = [
] # the results dict from rcnn, contains all the information
mask_files = [] # mask file for instance, each box has a mask file
boxes = [] # boxes of instance
class_ids = [] # class id of each box
patch_indices = [] # index of patch, on which contains the box
scores = [] # scores of boxes
json_files_list = [] # json files of patches
for json_folder in json_folder_list:
file_list = io_function.get_file_list_by_ext('.txt',
json_folder,
bsub_folder=False)
json_files_list.extend(file_list)
# load and convert coordinates, don't load mask images in this stage
patch_idx = 0
for json_file in json_files_list:
mrcnn_r = build_RS_data.load_instances_patch(
json_file,
bNMS=True,
bReadMaks=False,
final_classes=final_keep_classes)
mrcnn_r_list.append(
mrcnn_r) # this corresponds to json_files_list
if mrcnn_r is not None: # this will ignore the patches without instances, it is fine hlc 2018-11-25
mask_files.extend(mrcnn_r['masks'])
boxes.extend(mrcnn_r['rois'])
scores.extend(mrcnn_r['scores'])
class_ids.extend(mrcnn_r['class_ids'])
patch_indices.extend([patch_idx] * len(mrcnn_r['rois']))
patch_idx += 1
# Apply non-max suppression
keep_idxs = utils.non_max_suppression(np.array(boxes),
np.array(scores),
nms_iou_threshold)
# boxes_keep = [r for i, r in enumerate(boxes) if i in keep_ixs]
# convert kept patches to label images
for idx, keep_idx in enumerate(keep_idxs):
patch_idx = patch_indices[
keep_idx] # the index in original patches
# load mask (in the previous step, we did not load masks)
mask_file = mask_files[keep_idx]
patch_dir = os.path.dirname(json_files_list[patch_idx])
org_img_name = mrcnn_r_list[patch_idx]['org_img']
b_dict = mrcnn_r_list[patch_idx]['patch_boundary']
patch_boundary = (b_dict['xoff'], b_dict['yoff'],
b_dict['xsize'], b_dict['ysize'])
img_patch = build_RS_data.patchclass(
os.path.join(inf_image_dir, org_img_name), patch_boundary)
# the mask only contains one instances
# masks can overlap each others, but instances should not overlap each other
# non-instance pixels are zeros,which will be set as non-data when performing gdal_merge.pyg
mask = cv2.imread(os.path.join(patch_dir, mask_file),
cv2.IMREAD_UNCHANGED)
mask[mask == 255] = class_ids[
keep_idx] # when save mask, mask*255 for display
print('Save mask of instances:%d on Image:%d , shape:(%d,%d)' %
(idx, img_idx, mask.shape[0], mask.shape[1]))
# short the file name to avoid error of " Argument list too long", hlc 2018-Oct-29
file_name = "I%d_%d" % (img_idx, idx)
save_path = os.path.join(inf_output_dir, file_name + '.tif')
if build_RS_data.save_patch_oneband_8bit(
img_patch, mask.astype(np.uint8), save_path) is False:
return False
return True
def show_mrcnn_prediction(image):
resized_image, window, scale, padding, crop = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
min_scale=config.IMAGE_MIN_SCALE,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
# Get input and output to classifier and mask heads.
mrcnn = model.run_graph([resized_image], [
("proposals", model.keras_model.get_layer("ROI").output),
("probs", model.keras_model.get_layer("mrcnn_class").output),
("deltas", model.keras_model.get_layer("mrcnn_bbox").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
("detections", model.keras_model.get_layer("mrcnn_detection").output),
])
################################## display detections ###############################################
# Get detection class IDs. Trim zero padding.
det_class_ids = mrcnn['detections'][0, :, 4].astype(np.int32)
padding_start_ix = np.where(det_class_ids == 0)[0][0]
det_class_ids = det_class_ids[:padding_start_ix]
detections = mrcnn['detections'][0, :padding_start_ix]
log('trimmed_detection', detections)
print("{} detections: {}".format(
padding_start_ix,
np.array(dataset.class_names)[det_class_ids]))
################################### display proposals ##########################################
# Proposals are in normalized coordinates. Scale them to image coordinates.
h, w = resized_image.shape[:2]
proposals = np.around(mrcnn["proposals"][0] *
np.array([h, w, h, w])).astype(np.int32)
# Class ID, score, and mask per proposal
# mrcnn 的 shape 为 (batch_size, num_proposals=1000, num_classes)
proposal_class_ids = np.argmax(mrcnn["probs"][0], axis=1)
proposal_class_scores = mrcnn["probs"][
0, np.arange(proposal_class_ids.shape[0]), proposal_class_ids]
proposal_class_names = np.array(dataset.class_names)[proposal_class_ids]
proposal_positive_ixs = np.where(proposal_class_ids > 0)[0]
# How many ROIs vs empty rows?
print("{} valid proposals out of {}".format(
np.sum(np.any(proposals, axis=1)), proposals.shape[0]))
print("{} positive ROIs".format(len(proposal_positive_ixs)))
# Class counts
print(list(zip(*np.unique(proposal_class_names, return_counts=True))))
# Display a random sample of proposals.
# Proposals classified as background are dotted, and
# the rest show their class and confidence score.
limit = 200
#################################### apply bounding box refinement #############################
# Class-specific bounding box shifts.
# mrcnn['deltas'] 的 shape 为 (batch_size, num_proposals=1000, num_classes, 4)
proposal_deltas = mrcnn["deltas"][0,
np.arange(proposals.shape[0]),
proposal_class_ids]
log("proposals_deltas", proposal_deltas)
# Apply bounding box transformations
# Shape: (num_proposals=1000, (y1, x1, y2, x2)]
# NOTE: delta 是不分 normalized coordinates 和 pixel coordinates 的
refined_proposals = utils.apply_box_deltas(
proposals, proposal_deltas * config.BBOX_STD_DEV).astype(np.int32)
log("refined_proposals", refined_proposals)
#################################### more steps ################################################
# Remove boxes classified as background
keep_proposal_ixs = np.where(proposal_class_ids > 0)[0]
print("Remove background proposals. Keep {}:\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
# Remove low confidence detections
keep_proposal_ixs = np.intersect1d(
keep_proposal_ixs,
np.where(proposal_class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
print("Remove proposals below {} confidence. Keep {}:\n{}".format(
config.DETECTION_MIN_CONFIDENCE, keep_proposal_ixs.shape[0],
keep_proposal_ixs))
# Apply per-class non-max suppression
pre_nms_proposals = refined_proposals[keep_proposal_ixs]
pre_nms_proposal_scores = proposal_class_scores[keep_proposal_ixs]
pre_nms_proposal_class_ids = proposal_class_ids[keep_proposal_ixs]
nms_keep_proposal_ixs = []
for class_id in np.unique(pre_nms_proposal_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_proposal_class_ids == class_id)[0]
# Apply NMS
class_keep = utils.non_max_suppression(pre_nms_proposals[ixs],
pre_nms_proposal_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep_proposal_ixs = keep_proposal_ixs[ixs[class_keep]]
nms_keep_proposal_ixs = np.union1d(nms_keep_proposal_ixs,
class_keep_proposal_ixs)
print("{:12}: {} -> {}".format(dataset.class_names[class_id][:10],
keep_proposal_ixs[ixs],
class_keep_proposal_ixs))
keep_proposal_ixs = np.intersect1d(keep_proposal_ixs,
nms_keep_proposal_ixs).astype(np.int32)
print("\nKeep after per-class NMS: {}\n{}".format(
keep_proposal_ixs.shape[0], keep_proposal_ixs))
#################################### Show final detections #####################################
ixs = np.arange(len(keep_proposal_ixs)) # Display all
refined_bboxes = refined_proposals[keep_proposal_ixs][ixs]
refined_bboxes -= np.array([window[0], window[1], window[0], window[1]])
bboxes = refined_bboxes.astype('float32') / scale
bboxes = bboxes.tolist()
return bboxes
def load_sample(self,
samples,
dataset,
scaled=False,
scaled_to=50,
show_fig=True):
"""load the requested number of images.
count: number of images to generate.
scaled: whether to resize image or not.
scaled_to: percentage to resize the image.
"""
# Add classes
self.add_class("shapes", 1, "Houses")
self.add_class("shapes", 2, "Buildings")
self.add_class("shapes", 3, "Sheds/Garages")
# pick samples randomly
self.samples = random.sample(range(0, len(dataset)), samples)
# MAIN Loop
for image_id, sample in enumerate(self.samples):
# resize images
frame_id = dataset[sample]
self.imagePath = os.path.join(RGB_DIR, frame_id)
self.image, self.width, self.height = self.scale_image(
plt.imread(self.imagePath), scaled=scaled, scaled_to=scaled_to)
# record polygons class their bounding boxes and areas
shapes = []
boxes = []
areas = []
list_vertices = []
# read polygon annotations
data = pd.read_json(
self.imagePath.replace('raw', 'annotations').replace(
'png', 'png-annotated.json'))
for shape in range(len(data.labels)):
print('found {} {}'.format(
len(data.labels[shape]['annotations']),
data.labels[shape]['name']))
# iterate thorough each polygons
for poly in range(len(data.labels[shape]['annotations'])):
# get vertices of polygons (house, building, garage)
vertices = np.array(
data.labels[shape]['annotations'][poly]
['segmentation'], np.int32)
vertices = vertices.reshape((-1, 1, 2))
# draw polygons on scaled image
if scaled == True:
scaled_vertices = []
for v in range(len(vertices)):
scaled_vertices.append(
int(vertices[v][0][0] * scaled_to / 100)) #x
scaled_vertices.append(
int(vertices[v][0][1] * scaled_to / 100)) #y
vertices = np.array(scaled_vertices).reshape(
(-1, 1, 2))
# draw polygons on scaled image to create segmentation
image, color, bbox, area = self.draw_polygons(
self.image, vertices, shape, draw_bbox=False)
# same length as total polygons
boxes.append(bbox)
areas.append(area)
shapes.append((data.labels[shape]['name'], color, bbox))
list_vertices.append(vertices)
# Pick random background color
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# collect all necessary data
self.add_image("shapes",
image_id=image_id,
path=self.imagePath,
width=self.width,
height=self.height,
bg_color=bg_color,
shapes=shapes,
list_vertices=list_vertices,
image=self.image)
# Apply non-max suppression wit 0.3 threshold to avoid shapes covering each other
keep_ixs = utils.non_max_suppression(np.array(boxes),
np.arange(len(boxes)), 0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
# create mask for each instances
mask, class_ids = self.load_mask(image_id)
if show_fig == True:
fig = plt.figure(figsize=(12, 8),
dpi=100,
facecolor='w',
edgecolor='k')
plt.imshow((image * 255).astype(np.uint8))
plt.show()
visualize.display_top_masks(self.image, mask, class_ids,
class_names)
def refine_detections(rois, probs, deltas, window, config):
'''
Refine classified proposals and filter overlaps and return final detections.
Inputs:
------
rois: rpn_rois - [N, (y1, x1, y2, x2)] in normalized coordinates
probs: mrcnn_class - [N, num_classes]. Class probabilities.
deltas: mrcnn_bbox - [N, num_classes, (dy, dx, log(dh), log(dw))].
Class-specific bounding box deltas.
window: (y1, x1, y2, x2) in image coordinates. The part of the image
that contains the image excluding the padding.
Returns:
--------
detections [N, (y1, x1, y2, x2, class_id, score)]
'''
## 1. Find Class IDs with higest scores for each per ROI
class_ids = np.argmax(probs, axis=1)
## 2. Get Class probability(score) and bbox delta of the top class of each ROI
class_scores = probs[np.arange(class_ids.shape[0]), class_ids]
deltas_specific = deltas[np.arange(deltas.shape[0]) , class_ids]
## 3. Apply bounding box delta to the corrsponding rpn_proposal
# Shape: [boxes, (y1, x1, y2, x2)] in normalized coordinates
refined_rois = apply_box_deltas(rois, deltas_specific * config.BBOX_STD_DEV)
## 4. Convert the refined roi coordiates from normalized to image domain
# TODO: better to keep them normalized until later
height, width = config.IMAGE_SHAPE[:2]
refined_rois *= np.array([height, width, height, width])
## 5. Clip boxes to image window
refined_rois = clip_to_window(window, refined_rois)
## 6. Round and cast to int since we're deadling with pixels now
refined_rois = np.rint(refined_rois).astype(np.int32)
## 7. TODO: Filter out boxes with zero area
## 8. Filter out background boxes
keep = np.where(class_ids > 0)[0]
# Filter out low confidence boxes
if config.DETECTION_MIN_CONFIDENCE:
keep = np.intersect1d(keep, np.where(class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
##----------------------------------------------------------------------------
## 9. Apply per-class NMS
##----------------------------------------------------------------------------
pre_nms_class_ids = class_ids[keep]
pre_nms_scores = class_scores[keep]
pre_nms_rois = refined_rois[keep]
nms_keep = []
# print(' apply per class nms')
for class_id in np.unique(pre_nms_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_class_ids == class_id)[0]
# print('class_id : ', class_id)
# print('pre_nms_rois.shape:', pre_nms_rois[ixs].shape)
# pp.pprint(pre_nms_rois[ixs])
# print('pre_nms_scores.shape :', pre_nms_scores[ixs].shape)
# pp.pprint(pre_nms_scores[ixs])
# Apply NMS
class_keep = non_max_suppression(pre_nms_rois[ixs],
pre_nms_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep = keep[ixs[class_keep]]
nms_keep = np.union1d(nms_keep, class_keep)
keep = np.intersect1d(keep, nms_keep).astype(np.int32)
##----------------------------------------------------------------------------
## 10. Keep top detections
##----------------------------------------------------------------------------
roi_count = config.DETECTION_MAX_INSTANCES
top_ids = np.argsort(class_scores[keep])[::-1][:roi_count]
keep = keep[top_ids]
##----------------------------------------------------------------------------
## 11. Arrange output as [N, (y1, x1, y2, x2, class_id, score)]
## Coordinates are in image domain.
##----------------------------------------------------------------------------
result = np.hstack((refined_rois[keep],
class_ids [keep][..., np.newaxis],
class_scores[keep][..., np.newaxis]))
return result
#creates random specifications of an image with multiple shapes
#returns the background color of the image
bg_color=np.array([random.randint(0,255) for _in range(3)])
shapes=[]
boxes=[]
N = random.randint(1, 4)
# decides the number of the shapes in the image (yes....randomly)
for _ in range(N):
shape, color, dims = self.random_shape(height, width)
shapes.append((shape, color, dims))
x, y, s = dims
boxes.append([y-s, x-s, y+s, x+s])\
# Apply non-max suppression with 0.3 threshold to avoid
# shapes covering each other (occlusion)
keep_ixs = utils.non_max_suppression(np.array(boxes), np.arange(N), 0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
return bg_color, shapes
# Preparing Different Datasets
#Forming an object for the Dataset
dataset_train=ShapesDataset()
# forming dataset for the algo to train on
dataset_train.load_shapes(500,config,IMAGE_SHAPE[0],config.IMAGE_SHAPE[1])
dataset_train.prepare()
# Preparing dataset for validation as well, just like above (we are not here to spoonfeed you)
validation_train=ShapesDataset()
validation_train.load_shapes(500,config,IMAGE_SHAPE[0],config.IMAGE_SHAPE[1])
validation_train.prepare()
# Displaying the images with the masks
def refine_detections(rois, probs, deltas, window, config):
'''
Refine classified proposals and filter overlaps and return final detections.
Inputs:
------
rois: rpn_rois - [N, (y1, x1, y2, x2)] in normalized coordinates
passed from PROPOSAL_LAYER
probs: mrcnn_class - [N, num_classes]. Class probabilities.
deltas: mrcnn_bbox - [N, num_classes, (dy, dx, log(dh), log(dw))].
Class-specific bounding box deltas.
passed from FPN_CLASSIFIER_GRAPH
window:
(y1, x1, y2, x2) in image coordinates. The part of the image
that contains the image excluding the padding.
Returns:
--------
detections [M, (y1, x1, y2, x2, class_id, score)]
M - determined by DETECTION_MAX_INSTANCES
detection bounding boxes -- these have had the corresponding
deltas applied, and their boundries clipped to the image window
'''
##----------------------------------------------------------------------------
## 1. Find Class IDs with higest scores for each per ROI
##----------------------------------------------------------------------------
class_ids = np.argmax(probs, axis=1)
##----------------------------------------------------------------------------
## 2. Get Class probability(score) and bbox delta of the top class of each ROI
##----------------------------------------------------------------------------
class_scores = probs[np.arange(class_ids.shape[0]), class_ids]
deltas_specific = deltas[np.arange(deltas.shape[0]), class_ids]
##----------------------------------------------------------------------------
## 3. Apply bounding box delta to the corrsponding rpn_proposal
##----------------------------------------------------------------------------
# Shape: [boxes, (y1, x1, y2, x2)] in normalized coordinates
refined_rois = apply_box_deltas_np(rois,
deltas_specific * config.BBOX_STD_DEV)
##----------------------------------------------------------------------------
## 4. Convert the refined roi coordiates from normalized to NN image domain
## 5. Clip boxes to image window
## 6. Round and cast to int since we're deadling with pixels now
##----------------------------------------------------------------------------
# TODO: better to keep them normalized until later
height, width = config.IMAGE_SHAPE[:2]
refined_rois *= np.array([height, width, height, width])
refined_rois = clip_to_window(window, refined_rois)
refined_rois = np.rint(refined_rois).astype(np.int32)
##----------------------------------------------------------------------------
## 7. TODO: Filter out boxes with zero area
##----------------------------------------------------------------------------
##----------------------------------------------------------------------------
## 8. Filter out background boxes
## keep : contains indices of non-zero elements in class_ids
## config.DETECTION_MIN_CONFIDENCE == 0
## np.intersect: find indices into class_ids that satisfy:
## - class_id > 0
## - class_scores >= config.DETECTION_MIN_CONFIDENCE (0.3)
##----------------------------------------------------------------------------
keep = np.where(class_ids > 0)[0]
# Filter out low confidence boxes
if config.DETECTION_MIN_CONFIDENCE:
keep = np.intersect1d(
keep,
np.where(class_scores >= config.DETECTION_MIN_CONFIDENCE)[0])
##----------------------------------------------------------------------------
## 9. Apply per-class NMS
##----------------------------------------------------------------------------
pre_nms_class_ids = class_ids[keep]
pre_nms_scores = class_scores[keep]
pre_nms_rois = refined_rois[keep]
nms_keep = []
# print(' apply per class nms')
# print(' keep ixs : ', keep.shape)
# print(' pre_nms_rois.shape :', pre_nms_rois.shape)
# print(' pre_nms_scores.shape :', pre_nms_scores.shape)
for class_id in np.unique(pre_nms_class_ids):
# Pick detections of this class
ixs = np.where(pre_nms_class_ids == class_id)[0]
# print()
# print('class_id : ', class_id)
# print('---------------------')
# print(' ixs : ', ixs.shape)
# pp.pprint(ixs)
# print('pre_nms_rois.shape:', pre_nms_rois[ixs].shape)
# pp.pprint(pre_nms_rois[ixs])
# print('pre_nms_scores.shape :', pre_nms_scores[ixs].shape)
# pp.pprint(pre_nms_scores[ixs])
# Apply NMS - Suppress anything with IoU higher than config.DETECTION_NMS_THRESHOLD
class_keep = non_max_suppression(pre_nms_rois[ixs],
pre_nms_scores[ixs],
config.DETECTION_NMS_THRESHOLD)
# Map indicies
class_keep = keep[ixs[class_keep]]
nms_keep = np.union1d(nms_keep, class_keep)
# print('Class keep: ', class_keep)
# print('NMS keep : ', nms_keep.astype(np.int))
# print('post_nms_rois.shape :', refined_rois[class_keep].shape)
# pp.pprint(refined_rois[class_keep])
# print('post nms_scores.shape for class :', class_scores[class_keep].shape)
# pp.pprint(class_scores[class_keep])
keep = np.intersect1d(keep, nms_keep).astype(np.int32)
##----------------------------------------------------------------------------
## 10. Keep top detections
##----------------------------------------------------------------------------
roi_count = config.DETECTION_MAX_INSTANCES
top_ids = np.argsort(class_scores[keep])[::-1][:roi_count]
keep = keep[top_ids]
##----------------------------------------------------------------------------
## 11. Add a detect_ind = +1 to differentiate from false postives added in eval layer
##----------------------------------------------------------------------------
detect_ind = np.ones((top_ids.shape[0], 1))
##----------------------------------------------------------------------------
## 11. Arrange output as [N, (y1, x1, y2, x2, class_id, score)]
## Coordinates are in image domain.
##----------------------------------------------------------------------------
result = np.hstack((refined_rois[keep], class_ids[keep][..., np.newaxis],
class_scores[keep][..., np.newaxis], detect_ind))
return result
def multiview_detect(model, im, standard=False, verbose=True) :
# Implements Multi-Transform detection (see multi-transform detection chapter in report)
od = model.detect([im])[0]
if standard :
# if standard more activated then return the first, normal , detection
return od
# run 3 more detections on rotated and flipped image
r90_im = skimage.transform.rotate(im, 90, resize=True, preserve_range=True)
r90 = model.detect([r90_im])[0]
rn90_im = skimage.transform.rotate(im, -90, resize=True, preserve_range=True)
rn90 = model.detect([rn90_im])[0]
ud_im = im[::-1]
ud = model.detect([ud_im])[0]
# get translated bounding boxes
r90_bbx = np.array([translatecoord(len(r90_im), len(r90_im[0]), b, "left") for b in r90['rois']] )
rn90_bbx = np.array([translatecoord(len(rn90_im), len(rn90_im[0]), b, "right") for b in rn90['rois']] )
ud_bbx = np.array([translatecoord(len(ud_im), len(ud_im[0]), b, "upsidedown") for b in ud['rois']] )
# rotate masks (todo: move this code to function similar to translatecoord)
rnm = rn90['masks'][:, :, 0]
rn90_mask = np.zeros([len(rnm[0]), len(rnm), rn90['masks'].shape[-1]], dtype=np.uint8)
for i in range(rn90['masks'].shape[-1] ) :
tmp = rn90['masks'][:,:,i]
tmp = tmp.transpose()
tmp = tmp[::-1]
rn90_mask[:, :, i] = tmp
rm = r90['masks'][:, :, 0]
r90_mask = np.zeros([len(rm[0]), len(rm), r90['masks'].shape[-1]], dtype=np.uint8)
for i in range(r90['masks'].shape[-1] ) :
tmp = r90['masks'][:,:,i]
tmp = tmp[::-1]
tmp = tmp.transpose()
r90_mask[:, :, i] = tmp
for i in range(ud['masks'].shape[-1]) :
tmp = ud['masks'][:, : , i]
ud['masks'][:, :, i] = tmp[::-1]
# make sure the masks are the same shape to ensure a successful overlay
if od["masks"].shape != r90_mask.shape or od["masks"].shape != rn90_mask.shape :
r90_mask.resize((len(od['masks']), len(od['masks'][0]), r90['masks'].shape[-1]))
rn90_mask.resize((len(od['masks']), len(od['masks'][0]), rn90['masks'].shape[-1]))
# merge masks and bounding box vectors in preparation for the nonm-maximum suppression process
all_masks = np.concatenate((od['masks'], ud['masks'], r90_mask, rn90_mask), axis=2)
all_bboxes = np.concatenate((od['rois'], ud_bbx, r90_bbx, rn90_bbx))
all_scores = np.concatenate(( od['scores'], ud['scores'], r90['scores'], rn90['scores']))
all_class_ids = np.concatenate(( od['class_ids'], ud['class_ids'], r90['class_ids'], rn90['class_ids']))
final = utils.non_max_suppression(all_bboxes, all_scores, 0.55)
# final contains the indices of the boxes to be kept, these are used to extract them as well as the masks, scores and other metadata corresponding to each
final_masks = np.take(all_masks, final, axis=2)
final_bboxes= np.take(all_bboxes, final, axis=0)
final_scores = np.take(all_scores, final)
final_class_ids = np.take(all_class_ids, final)
if verbose :
print("standard detection instances : " + str(len(od['rois'])) )
print("90 deg detection instances : " + str(len(r90_bbx)) )
print("-90 deg detection instances : " + str(len(rn90_bbx)) )
print("upside down detection instances : " + str(len(ud_bbx)) )
print("post nms detect : " + str(len(final_bboxes)))
#vis.display_instances(im, final_bboxes, final_masks, final_class_ids, ['', ''] , save=True, name="multiview_detect_{}_{}".format(sys.argv[2].split('/')[-1].split('.')[0], sys.argv[3]), savedir=".")
return {
'rois' : final_bboxes,
'scores' : final_scores,
'masks' : final_masks,
'class_ids' : final_class_ids
}
