def display_samples():
image_ids = np.random.choice(dataset_train.image_ids, 4)
for image_id in image_ids:
image = dataset_train.load_image(image_id)
mask, class_ids = dataset_train.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset_train.class_names)
def display_dataset(num_of_random_samples):
# Load and display random samples
if num_of_random_samples >= len(dataset.image_ids):
print(
"The number of samples cannot be larger than the amount of samples available"
)
print("\nSetting the amount of equal to the amount of samples")
num_of_random_samples = len(dataset.image_ids) - 1
image_ids = np.random.choice(dataset.image_ids, num_of_random_samples)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
def load_and_display_random_sample(dataset, datacfg, N=2):
"""Load and display random samples
"""
log.info("load_and_display_random_sample::-------------------------------->")
image_ids = np.random.choice(dataset.image_ids, N)
class_names = dataset.class_names
log.debug("dataset: len(image_ids): {}\nimage_ids: {}".format(len(image_ids), image_ids))
log.debug("dataset: len(class_names): {}\nclass_names: {}".format(len(class_names), class_names))
for image_id in image_ids:
image = dataset.load_image(image_id, datacfg)
mask, class_ids, keys, values = dataset.load_mask(image_id, datacfg)
log.debug("keys: {}".format(keys))
log.debug("values: {}".format(values))
log.debug("class_ids: {}".format(class_ids))
## Display image and instances
visualize.display_top_masks(image, mask, class_ids, class_names)
## Compute Bounding box
bbox = utils.extract_bboxes(mask)
log.debug("bbox: {}".format(bbox))
visualize.display_instances(image, bbox, mask, class_ids, class_names)
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = DangerDataset()
dataset_train.load_danger(args.dataset, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = DangerDataset()
dataset_val.load_danger(args.dataset, "val")
dataset_val.prepare()
image_ids = np.random.choice(dataset_train.image_ids, 1)
for image_id in image_ids:
image = dataset_train.load_image(image_id)
mask, class_ids = dataset_train.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset_train.class_names)
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training network heads")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=1,
layers='heads')
def show_data():
dataset_train, dataset_val = load_data()
image_ids = np.random.choice(dataset_train.image_ids, 4)
for image_id in image_ids:
image = dataset_train.load_image(image_id)
mask, class_ids = dataset_train.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset_train.class_names)
def display_mask(self):
""" Display the image with mask randomly inside the dataset.
"""
image_ids = np.random.choice(self.image_ids, 4)
for image_id in image_ids:
image = self.load_image(image_id)
mask, class_ids = self.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, self.class_names)
def display_mask():
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# 显示面积最大的 limit(默认为4) 个 class 的 mask
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
def detect(model, dataset, image_id):
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, dataset.class_names)
results = model.detect([image], verbose=1)
# Visualize results
r = results[0]
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
["BG", "infection"], r['scores'])
def sample_dataset(dataset):
for i in range(6):
image_id = random.choice(dataset.image_ids)
print("ImageId: {}\n".format(image_id))
print(dataset.image_reference(image_id))
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image,
mask,
class_ids,
dataset.class_names,
limit=4)
def show(self, howmany=1):
'''
'''
image_ids = np.random.choice(self.image_ids, howmany)
for image_id in image_ids:
image = self.load_image(image_id)
mask, class_ids = self.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
self.class_names)
def train(model):
"""Train the model."""
# Training dataset.
dataset_train = ObjectDetectionDataset() # This is a class
dataset_train.load_objects(args.dataset, "training")
dataset_train.prepare()
# Validation dataset
dataset_val = ObjectDetectionDataset() # This is a class
dataset_val.load_objects(args.dataset, "valing")
dataset_val.prepare()
'''
show and display random samples
'''
# Load and display random samples
image_ids = np.random.choice(dataset_train.image_ids, 4)
print("####")
print("all image loaded", dataset_train.image_ids)
print("images selected to show", image_ids)
for image_id in image_ids:
# print(image_id)
image = dataset_train.load_image(image_id)
print(image.shape)
# cv2.imshow('image', image)
# cv2.waitKey(0)
# cv2.destroyAllWindows()
# img_s = [image]
# titles = ['img']
mask, class_ids = dataset_train.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset_train.class_names)
# print(dataset_train.class_names)
# print(class_ids)
# visualize.display_top_masks(image, mask, class_ids, dataset_train.class_names, limit=5)
# *** This training schedule is an example. Update to your needs ***
# Since we're using a very small dataset, and starting from
# COCO trained weights, we don't need to train too long. Also,
# no need to train all layers, just the heads should do it.
print("Training **ALL** networks!")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=150,
layers='all')
def display_datasets(dataset_train, dataset_val):
print("Training dataset\nImages: {}\nClasses: {}".format(
len(dataset_train.image_ids), dataset_train.class_names))
print("Validation dataset\nImages: {}\nClasses: {}".format(
len(dataset_val.image_ids), dataset_val.class_names))
for name, dataset in [('training', dataset_train),
('validation', dataset_val)]:
print(f'Displaying examples from {name} dataset:')
image_ids = np.random.choice(dataset.image_ids, 3)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
def detect_image(model, filename):
image = np.flip(
skimage.color.gray2rgb(skimage.io.imread(filename + ".tif") * 255), 1)
masked_arr = np.load(filename + ".mask")
mask = np.flip(np.expand_dims(masked_arr[0], axis=2), 1)
visualize.display_top_masks(image, mask,
np.ones([mask.shape[-1]], dtype=np.int32),
["BG", "infection"])
results = model.detect([image], verbose=1)
# Visualize results
r = results[0]
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
["BG", "infection"], r['scores'])
def random_detect(dataset_val, inference_config, model, count: int = 5):
for i in range(0, count):
# Test on a random image
image_id = np.random.choice(dataset_val.image_ids)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask = False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
image = dataset_val.load_image(image_id)
mask, class_ids = dataset_val.load_mask(image_id)
image = map_image_to_rgb(image)
visualize.display_top_masks(image, mask, class_ids, dataset_val.class_names, limit = 1)
results = model.detect([original_image], verbose=1)
r = results[0]
disp_oritinal_image = map_image_to_rgb(original_image)
visualize.display_instances(disp_oritinal_image, r['rois'], r['masks'], r['class_ids'], dataset_val.class_names, r['scores'], ax = get_ax())
# load the 0-th training image and corresponding masks and
# class IDs in the masks
image = trainDataset.load_image(0)
(masks, classIDs) = trainDataset.load_mask(0)
# show the image spatial dimensions which is HxWxC
print("[INFO] image shape: {}".format(image.shape))
# show the masks shape which should have the same width and
# height of the images but the third dimension should be
# equal to the total number of instances in the image itself
print("[INFO] masks shape: {}".format(masks.shape))
# show the length of the class IDs list along with the values
# inside the list -- the length of the list should be equal
# to the number of instances dimension in the 'masks' array
print("[INFO] class IDs length: {}".format(len(classIDs)))
print("[INFO] class IDs: {}".format(classIDs))
# determine a sample of training image indexes and loop over
# them
for i in np.random.choice(trainDataset.image_ids, 100):
# load the image and masks for the sampled image
print("[INFO] investigating image index: {}".format(i))
image = trainDataset.load_image(i)
(masks, classIDs) = trainDataset.load_mask(i)
# visualize the masks for the current image
visualize.display_top_masks(image, masks, classIDs,
trainDataset.class_names)
train_dataset = CloudDataset(df[:training_set_size])
train_dataset.prepare()
valid_dataset = CloudDataset(df[training_set_size:training_set_size+validation_set_size])
valid_dataset.prepare()
#%%
#VISUALIZE image with masks ###################################################
for i in range(5):
image_id = random.choice(train_dataset.image_ids)
print(train_dataset.image_reference(image_id))
image = train_dataset.load_image(image_id)
mask, class_ids = train_dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5)
#%%
#TRAIN##########################################################################
LR = 1e-4
import warnings
from imgaug import augmenters as iaa
warnings.filterwarnings("ignore")
augmentation = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Flipud(0.5),iaa.OneOf([
iaa.Multiply((0.9, 1.1)),
# Validation dataset
# dataset_val = ShapesDataset()
# dataset_val.load_shapes(50, config.IMAGE_SHAPE[0], config.IMAGE_SHAPE[1])
# dataset_val.prepare()
dataset_val = drillerDat
# Load and display random samples
image_ids = np.random.choice(dataset_train.image_ids, 4)
print('choice image_ids = ', image_ids)
for image_id in image_ids:
image = dataset_train.load_image(image_id)
mask, class_ids = dataset_train.load_mask(image_id)
visualize.display_top_masks(image,
mask,
class_ids,
dataset_train.class_names,
limit=2)
# Create model in training mode
model = modellib.MaskRCNN(mode="training", config=config, model_dir=MODEL_DIR)
# Which weights to start with?
init_with = "coco" # imagenet, coco, or last
if init_with == "imagenet":
model.load_weights(model.get_imagenet_weights(), by_name=True)
elif init_with == "coco":
# Load weights trained on MS COCO, but skip layers that
# are different due to the different number of classes
# See README for instructions to download the COCO weights
def examin_data(model, image_path=None, option=None):
# visualization function. 4 basic options provided:
# activation visualization
# pre and post mold mask visualization
# augmentations visualization
#
# change as necessary
assert image_path or option
if image_path:
print("Running on {}".format(args.image))
image = skimage.io.imread(args.image)
# run selected graphs and save outputs for each
activations = model.run_graph(
[image],
[
("input_image",
tf.identity(
model.keras_model.get_layer("input_image").output)),
("res2c_out", model.keras_model.get_layer("res2c_out").output),
("res3c_out", model.keras_model.get_layer("res3c_out").output),
("res4w_out", model.keras_model.get_layer("res4w_out").output
), # for resnet100
("rpn_bbox", model.keras_model.get_layer("rpn_bbox").output),
("roi", model.keras_model.get_layer("ROI").output),
])
_ = plt.imshow(
modellib.unmold_image(activations["input_image"][0],
SiliqueConfig()))
display_images(np.transpose(activations["res3c_out"][0, :, :, :5],
[2, 0, 1]),
cols=5,
save=True,
name="activations_aa_" + args.image.split(".")[0],
savedir="Mask_RCNN/output")
elif option:
#prepare dataset and augmentation config
dataset = SiliqueDataset()
dataset.load_silique("new_dataset/", "val", ["white"])
dataset.prepare()
image_ids = np.random.choice(dataset.image_ids, 10)
augmentation = augs.Sometimes(
0.7,
augs.SomeOf((1, 3), [
augs.Flipud(0.5),
augs.Flipud(0.5),
augs.GaussianBlur(sigma=(0.0, 5.0)),
augs.Affine(scale={
"x": (0.8, 1.2),
"y": (0.8, 1.2)
}),
augs.Affine(rotate=(-90, 90))
],
random_order=True))
if option == "premold_masks":
# view images and masks before mold
for image_id in image_ids:
print("extracting {}".format(image_id))
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
display_top_masks(image,
mask,
class_ids,
dataset.class_names,
limit=1,
sve=True,
nme="white_masks_{}".format(image_id),
svedir="Mask_RCNN/output")
elif option == "postmold_masks":
print("Generating molded images...")
for image_id in image_ids:
print("extracting {}".format(image_id))
#load image after it has been processed by the model
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset,
SiliqueConfig(),
image_id,
use_mini_mask=False,
augmentation=augmentation)
display_top_masks(image,
mask,
class_ids,
dataset.class_names,
limit=1,
sve=True,
nme="white_masks_{}".format(image_id),
svedir="Mask_RCNN/output")
def display_masks(self, image, mask, class_ids, class_names):
# print("display_masks: {}, {}".format(class_ids, class_names))
visualize.display_top_masks(image, mask, class_ids, class_names)
dataset_val = LegoDataset()
dataset_val.load_lego("val")
dataset_val.prepare()
dataset_test = LegoDataset()
dataset_test.load_lego("test")
dataset_test.prepare()
# In[6]:
# Load and display random samples
image_ids = np.random.choice(dataset_train.image_ids, 12)
for image_id in []:
path, image = dataset_train.load_image(image_id)
mask, class_ids = dataset_train.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset_train.class_names, path)
# ## Ceate Model
# In[7]:
# Create model in training mode
model = modellib.MaskRCNN(mode="training", config=config, model_dir=MODEL_DIR)
# In[8]:
# Which weights to start with?
init_with = "coco" # imagenet, coco, or last
if init_with == "imagenet":
model.load_weights(model.get_imagenet_weights(), by_name=True)
def load_sample(self,
samples,
dataset,
scaled=False,
scaled_to=50,
show_fig=True):
"""load the requested number of images.
count: number of images to generate.
scaled: whether to resize image or not.
scaled_to: percentage to resize the image.
"""
# Add classes
self.add_class("shapes", 1, "Houses")
self.add_class("shapes", 2, "Buildings")
self.add_class("shapes", 3, "Sheds/Garages")
# pick samples randomly
self.samples = random.sample(range(0, len(dataset)), samples)
# MAIN Loop
for image_id, sample in enumerate(self.samples):
# resize images
frame_id = dataset[sample]
self.imagePath = os.path.join(RGB_DIR, frame_id)
self.image, self.width, self.height = self.scale_image(
plt.imread(self.imagePath), scaled=scaled, scaled_to=scaled_to)
# record polygons class their bounding boxes and areas
shapes = []
boxes = []
areas = []
list_vertices = []
# read polygon annotations
data = pd.read_json(
self.imagePath.replace('raw', 'annotations').replace(
'png', 'png-annotated.json'))
for shape in range(len(data.labels)):
print('found {} {}'.format(
len(data.labels[shape]['annotations']),
data.labels[shape]['name']))
# iterate thorough each polygons
for poly in range(len(data.labels[shape]['annotations'])):
# get vertices of polygons (house, building, garage)
vertices = np.array(
data.labels[shape]['annotations'][poly]
['segmentation'], np.int32)
vertices = vertices.reshape((-1, 1, 2))
# draw polygons on scaled image
if scaled == True:
scaled_vertices = []
for v in range(len(vertices)):
scaled_vertices.append(
int(vertices[v][0][0] * scaled_to / 100)) #x
scaled_vertices.append(
int(vertices[v][0][1] * scaled_to / 100)) #y
vertices = np.array(scaled_vertices).reshape(
(-1, 1, 2))
# draw polygons on scaled image to create segmentation
image, color, bbox, area = self.draw_polygons(
self.image, vertices, shape, draw_bbox=False)
# same length as total polygons
boxes.append(bbox)
areas.append(area)
shapes.append((data.labels[shape]['name'], color, bbox))
list_vertices.append(vertices)
# Pick random background color
bg_color = np.array([random.randint(0, 255) for _ in range(3)])
# collect all necessary data
self.add_image("shapes",
image_id=image_id,
path=self.imagePath,
width=self.width,
height=self.height,
bg_color=bg_color,
shapes=shapes,
list_vertices=list_vertices,
image=self.image)
# Apply non-max suppression wit 0.3 threshold to avoid shapes covering each other
keep_ixs = utils.non_max_suppression(np.array(boxes),
np.arange(len(boxes)), 0.3)
shapes = [s for i, s in enumerate(shapes) if i in keep_ixs]
# create mask for each instances
mask, class_ids = self.load_mask(image_id)
if show_fig == True:
fig = plt.figure(figsize=(12, 8),
dpi=100,
facecolor='w',
edgecolor='k')
plt.imshow((image * 255).astype(np.uint8))
plt.show()
visualize.display_top_masks(self.image, mask, class_ids,
class_names)
dataset = FashionDataset(image_df)
dataset.prepare()
for i in range(1):
image_id = random.choice(dataset.image_ids)
print(dataset.image_reference(image_id))
image = dataset.load_image(image_id)
mask, class_ids, attr_ids = dataset.load_mask(image_id)
# print("class_ids", class_ids)
# print("attr_ids", attr_ids)
# print(type(attr_ids))
visualize.display_top_masks(image,
mask,
class_ids,
attr_ids,
dataset.class_names,
dataset.attr_names,
limit=4)
# In[86]:
# This code partially supports k-fold training,
# you can specify the fold to train and the total number of folds here
FOLD = 0
N_FOLDS = 3
kf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)
splits = kf.split(
image_df) # ideally, this should be multilabel stratification
def get_labels(self, labels):
dims = labels.shape
unlabeled_labels = np.zeros((dims[0], dims[1], 1))
building_labels = np.zeros((dims[0], dims[1], 1))
fence_labels = np.zeros((dims[0], dims[1], 1))
other_labels = np.zeros((dims[0], dims[1], 1))
pedestrian_labels = np.zeros((dims[0], dims[1], 1))
pole_labels = np.zeros((dims[0], dims[1], 1))
road_line_labels = np.zeros((dims[0], dims[1], 1))
road_labels = np.zeros((dims[0], dims[1], 1))
sidewalk_labels = np.zeros((dims[0], dims[1], 1))
vegetation_labels = np.zeros((dims[0], dims[1], 1))
car_labels = np.zeros((dims[0], dims[1], 1))
wall_labels = np.zeros((dims[0], dims[1], 1))
traffic_sign_labels = np.zeros((dims[0], dims[1], 1))
unlabeled_index = np.all(labels == (0, 0, 0), axis=-1)
building_index = np.all(labels == (70, 70, 70), axis=-1)
fence_index = np.all(labels == (190, 153, 153), axis=-1)
other_index = np.all(labels == (250, 170, 160), axis=-1)
pedestrian_index = np.all(labels == (220, 20, 60), axis=-1)
pole_index = np.all(labels == (153, 153, 153), axis=-1)
road_line_index = np.all(labels == (157, 234, 50), axis=-1)
road_index = np.all(labels == (128, 64, 128), axis=-1)
sidewalk_index = np.all(labels == (244, 35, 232), axis=-1)
vegetation_index = np.all(labels == (107, 142, 35), axis=-1)
car_index = np.all(labels == (0, 0, 142), axis=-1)
wall_index = np.all(labels == (102, 102, 156), axis=-1)
traffic_sign_index = np.all(labels == (220, 220, 70), axis=-1)
unlabeled_labels[unlabeled_index] = 1
building_labels[building_index] = 10
fence_labels[fence_index] = 10
other_labels[other_index] = 10
pedestrian_labels[pedestrian_index] = 10
pole_labels[pole_index] = 10
road_line_labels[road_line_index] = 10
road_labels[road_index] = 10
sidewalk_labels[sidewalk_index] = 10
vegetation_labels[vegetation_index] = 1
car_labels[car_index] = 10
wall_labels[wall_index] = 10
traffic_sign_labels[traffic_sign_index] = 10
return np.dstack([unlabeled_labels, building_labels, fence_labels,
return np.dstack([unlabeled_labels, building_labels, fence_labels,
other_labels, pedestrian_labels, pole_labels,
road_line_labels, road_labels, sidewalk_labels, vegetation_labels,
car_labels, wall_labels, traffic_sign_labels])
def image_reference(self, image_id):
"""Return the carla data of the image."""
info = self.image_info[image_id]
if info["source"] == "carla":
return info["id"]
else:
super(self.__class__).image_reference(self, image_id)
config = CarlaConfig()
config.STEPS_PER_EPOCH = NUMBER_OF_TRAIN_DATA//config.BATCH_SIZE
config.VALIDATION_STEPS = NUMBER_OF_VAL_DATA//config.BATCH_SIZE
config.display()
dataset = carlaDataset()
dataset.load_images(dir=RGB_TRAIN_DIR, type='train')
# mask, a = train.load_mask(50)
# print(a)
dataset.prepare()
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id)
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image]+[mask[:,:,i] for i in range(min(mask.shape[-1], 7))])
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config, config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells // config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start+anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(level, level_anchors.shape[0],
backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] + center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center+anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=2, facecolor='none',
edgecolor=(i+1)*np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# Create data generator
random_rois = 4000
g = modellib.data_generator(
dataset, config, shuffle=True, random_rois=random_rois,
batch_size=4,
detection_targets=True)
# Get Next Image
if random_rois:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids, gt_boxes, gt_masks, rpn_rois, rois], \
[mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes, gt_masks], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log("rpn_match", rpn_match, )
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names, np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
visualize.draw_boxes(sample_image, boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors)
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image, boxes=anchors[np.random.choice(neutral_anchor_ids, 100)])
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b, np.arange(mrcnn_bbox.shape[1]), mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(rois[b].astype(np.float32), bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b, np.arange(mrcnn_mask.shape[1]), :, :, mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific, mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(), rois[b,i,:4].astype(np.int32), [255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64), [0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b,i]][:20])
display_images(images, titles, cols=4, cmap="Blues", interpolation="none")
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(
dataset, config, shuffle=True, random_rois=10000,
batch_size=1, detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois, positive_rois/ids.shape[1]))
print("Average percent: {:.2f}".format(total/(limit*ids.shape[1])))
exit()
image_df = pd.concat([image_df_1, image_df_2, image_df_3])
dataset = FloorDataset()
dataset.add_data(df=image_df)
dataset.prepare()
#SHOW SOME IMAGE
for i in range(3):
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
black_white_img = visualize.display_top_masks(image,
mask,
class_ids,
dataset.class_names,
limit=1)
# cv2.imwrite('./black_white_img_4/color_img_{}.jpg'.format(i), black_white_img)
# black_white_img = cv2.imread('./black_white_img_4/color_img_{}.jpg'.format(i), 0)
# cv2.imwrite('./black_white_img_4/color_img_{}.jpg'.format(i), image)
# cv2.imwrite('./black_white_img_4/img_{}.jpg'.format(i), ~black_white_img)
####-----------__TRAINING----------------------########
FOLD = 0
N_FOLDS = 5
kf = KFold(n_splits=N_FOLDS, random_state=42, shuffle=True)
splits = kf.split(
image_df) # ideally, this should be multilabel stratification
# Visualize results
r = results[0]
# Get all result
all_mask = visualize.display_instances(image, r['rois'], r['masks'],
r['class_ids'], class_names,
r['scores'])
# Get box
box_mask = visualize.draw_rois(image, r['rois'], r['rois'], r['masks'],
r['class_ids'], class_names)
# skimage.io.imsave("./save_result/box_"+num+".png",box_mask)
# Get mask
binery_mask = visualize.display_top_masks(image,
r['masks'],
r['class_ids'],
class_names,
limit=1)
exact_roi = binery_mask * image[:, :, 0]
bimage = binery_mask * 255
skimage.io.imsave("./save_result/mask_" + num + ".png", bimage)
# Find max rectangle from binery mask
image_path = "./save_result/mask_" + num + ".png"
coors = find_max_rectangle(image_path)
print(coors)
# Extract ROI from original images
roi_image = exact_roi[coors[1]:coors[3], coors[0]:coors[2]]
skimage.io.imshow(roi_image, cmap='gray')
# skimage.io.imsave("./save_result/roi_"+num+".png",roi_image)
# Must call before using the dataset
dataset.prepare()
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
# 遍历coco数据集类别：总共80类
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids, dataset.class_names)
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
def inspect_data(dataset, config):
print("Image Count: {}".format(len(dataset.image_ids)))
print("Class Count: {}".format(dataset.num_classes))
for i, info in enumerate(dataset.class_info):
print("{:3}. {:50}".format(i, info['name']))
# ## Display Samples
#
# Load and display images and masks.
# In[4]:
# Load and display random samples
image_ids = np.random.choice(dataset.image_ids, 4)
for image_id in image_ids:
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
visualize.display_top_masks(image, mask, class_ids,
dataset.class_names)
# ## Bounding Boxes
#
# Rather than using bounding box coordinates provided by the source datasets, we compute the bounding boxes from masks instead. This allows us to handle bounding boxes consistently regardless of the source dataset, and it also makes it easier to resize, rotate, or crop images because we simply generate the bounding boxes from the updates masks rather than computing bounding box transformation for each type of image transformation.
# In[5]:
# Load random image and mask.
image_id = random.choice(dataset.image_ids)
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id ", image_id, dataset.image_reference(image_id))
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Resize Images
#
# To support multiple images per batch, images are resized to one size (1024x1024). Aspect ratio is preserved, though. If an image is not square, then zero padding is added at the top/bottom or right/left.
# In[6]:
# Load random image and mask.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image = dataset.load_image(image_id)
mask, class_ids = dataset.load_mask(image_id)
original_shape = image.shape
# Resize
image, window, scale, padding, _ = utils.resize_image(
image,
min_dim=config.IMAGE_MIN_DIM,
max_dim=config.IMAGE_MAX_DIM,
mode=config.IMAGE_RESIZE_MODE)
mask = utils.resize_mask(mask, scale, padding)
# Compute Bounding box
bbox = utils.extract_bboxes(mask)
# Display image and additional stats
print("image_id: ", image_id, dataset.image_reference(image_id))
print("Original shape: ", original_shape)
log("image", image)
log("mask", mask)
log("class_ids", class_ids)
log("bbox", bbox)
# Display image and instances
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Mini Masks
#
# Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone.
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples.
# In[7]:
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[8]:
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# In[9]:
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[10]:
mask = utils.expand_mask(bbox, mask, image.shape)
visualize.display_instances(image, bbox, mask, class_ids,
dataset.class_names)
# ## Anchors
#
# The order of anchors is important. Use the same order in training and prediction phases. And it must match the order of the convolution execution.
#
# For an FPN network, the anchors must be ordered in a way that makes it easy to match anchors to the output of the convolution layers that predict anchor scores and shifts.
# * Sort by pyramid level first. All anchors of the first level, then all of the second and so on. This makes it easier to separate anchors by level.
# * Within each level, sort anchors by feature map processing sequence. Typically, a convolution layer processes a feature map starting from top-left and moving right row by row.
# * For each feature map cell, pick any sorting order for the anchors of different ratios. Here we match the order of ratios passed to the function.
#
# **Anchor Stride:**
# In the FPN architecture, feature maps at the first few layers are high resolution. For example, if the input image is 1024x1024 then the feature meap of the first layer is 256x256, which generates about 200K anchors (256*256*3). These anchors are 32x32 pixels and their stride relative to image pixels is 4 pixels, so there is a lot of overlap. We can reduce the load significantly if we generate anchors for every other cell in the feature map. A stride of 2 will cut the number of anchors by 4, for example.
#
# In this implementation we use an anchor stride of 2, which is different from the paper.
# In[11]:
# Generate Anchors
backbone_shapes = modellib.compute_backbone_shapes(config,
config.IMAGE_SHAPE)
anchors = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
# Print summary of anchors
num_levels = len(backbone_shapes)
anchors_per_cell = len(config.RPN_ANCHOR_RATIOS)
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = backbone_shapes[l][0] * backbone_shapes[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
# Visualize anchors of one cell at the center of the feature map of a specific level.
# In[12]:
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, _, _, _ = modellib.load_image_gt(
dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(backbone_shapes)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start +
anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], backbone_shapes[level]))
center_cell = backbone_shapes[level] // 2
center_cell_index = (center_cell[0] * backbone_shapes[level][1] +
center_cell[1])
level_center = center_cell_index * anchors_per_cell
center_anchor = anchors_per_cell * (
(center_cell[0] * backbone_shapes[level][1] / config.RPN_ANCHOR_STRIDE**2) \
+ center_cell[1] / config.RPN_ANCHOR_STRIDE)
level_center = int(center_anchor)
# Draw anchors. Brightness show the order in the array, dark to bright.
for i, rect in enumerate(level_anchors[level_center:level_center +
anchors_per_cell]):
y1, x1, y2, x2 = rect
p = patches.Rectangle(
(x1, y1),
x2 - x1,
y2 - y1,
linewidth=2,
facecolor='none',
edgecolor=(i + 1) * np.array(colors[level]) / anchors_per_cell)
ax.add_patch(p)
# ## Data Generator
#
# In[13]:
# Create data generator
random_rois = 2000
g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=random_rois,
batch_size=4,
detection_targets=True)
# In[14]:
# Uncomment to run the generator through a lot of images
# to catch rare errors
# for i in range(1000):
# print(i)
# _, _ = next(g)
# In[15]:
# Get Next Image
if random_rois:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_class_ids,
gt_boxes, gt_masks, rpn_rois, rois
], [mrcnn_class_ids, mrcnn_bbox, mrcnn_mask] = next(g)
log("rois", rois)
log("mrcnn_class_ids", mrcnn_class_ids)
log("mrcnn_bbox", mrcnn_bbox)
log("mrcnn_mask", mrcnn_mask)
else:
[
normalized_images, image_meta, rpn_match, rpn_bbox, gt_boxes,
gt_masks
], _ = next(g)
log("gt_class_ids", gt_class_ids)
log("gt_boxes", gt_boxes)
log("gt_masks", gt_masks)
log(
"rpn_match",
rpn_match,
)
log("rpn_bbox", rpn_bbox)
image_id = modellib.parse_image_meta(image_meta)["image_id"][0]
print("image_id: ", image_id, dataset.image_reference(image_id))
# Remove the last dim in mrcnn_class_ids. It's only added
# to satisfy Keras restriction on target shape.
mrcnn_class_ids = mrcnn_class_ids[:, :, 0]
# In[16]:
b = 0
# Restore original image (reverse normalization)
sample_image = modellib.unmold_image(normalized_images[b], config)
# Compute anchor shifts.
indices = np.where(rpn_match[b] == 1)[0]
refined_anchors = utils.apply_box_deltas(
anchors[indices], rpn_bbox[b, :len(indices)] * config.RPN_BBOX_STD_DEV)
log("anchors", anchors)
log("refined_anchors", refined_anchors)
# Get list of positive anchors
positive_anchor_ids = np.where(rpn_match[b] == 1)[0]
print("Positive anchors: {}".format(len(positive_anchor_ids)))
negative_anchor_ids = np.where(rpn_match[b] == -1)[0]
print("Negative anchors: {}".format(len(negative_anchor_ids)))
neutral_anchor_ids = np.where(rpn_match[b] == 0)[0]
print("Neutral anchors: {}".format(len(neutral_anchor_ids)))
# ROI breakdown by class
for c, n in zip(dataset.class_names,
np.bincount(mrcnn_class_ids[b].flatten())):
if n:
print("{:23}: {}".format(c[:20], n))
# Show positive anchors
fig, ax = plt.subplots(1, figsize=(16, 16))
visualize.draw_boxes(sample_image,
boxes=anchors[positive_anchor_ids],
refined_boxes=refined_anchors,
ax=ax)
# In[17]:
# Show negative anchors
visualize.draw_boxes(sample_image, boxes=anchors[negative_anchor_ids])
# In[18]:
# Show neutral anchors. They don't contribute to training.
visualize.draw_boxes(sample_image,
boxes=anchors[np.random.choice(
neutral_anchor_ids, 100)])
# ## ROIs
# In[19]:
if random_rois:
# Class aware bboxes
bbox_specific = mrcnn_bbox[b,
np.arange(mrcnn_bbox.shape[1]),
mrcnn_class_ids[b], :]
# Refined ROIs
refined_rois = utils.apply_box_deltas(
rois[b].astype(np.float32),
bbox_specific[:, :4] * config.BBOX_STD_DEV)
# Class aware masks
mask_specific = mrcnn_mask[b,
np.arange(mrcnn_mask.shape[1]), :, :,
mrcnn_class_ids[b]]
visualize.draw_rois(sample_image, rois[b], refined_rois, mask_specific,
mrcnn_class_ids[b], dataset.class_names)
# Any repeated ROIs?
rows = np.ascontiguousarray(rois[b]).view(
np.dtype((np.void, rois.dtype.itemsize * rois.shape[-1])))
_, idx = np.unique(rows, return_index=True)
print("Unique ROIs: {} out of {}".format(len(idx), rois.shape[1]))
# In[20]:
if random_rois:
# Dispalay ROIs and corresponding masks and bounding boxes
ids = random.sample(range(rois.shape[1]), 8)
images = []
titles = []
for i in ids:
image = visualize.draw_box(sample_image.copy(),
rois[b, i, :4].astype(np.int32),
[255, 0, 0])
image = visualize.draw_box(image, refined_rois[i].astype(np.int64),
[0, 255, 0])
images.append(image)
titles.append("ROI {}".format(i))
images.append(mask_specific[i] * 255)
titles.append(dataset.class_names[mrcnn_class_ids[b, i]][:20])
display_images(images,
titles,
cols=4,
cmap="Blues",
interpolation="none")
# In[21]:
# Check ratio of positive ROIs in a set of images.
if random_rois:
limit = 10
temp_g = modellib.data_generator(dataset,
config,
shuffle=True,
random_rois=10000,
batch_size=1,
detection_targets=True)
total = 0
for i in range(limit):
_, [ids, _, _] = next(temp_g)
positive_rois = np.sum(ids[0] > 0)
total += positive_rois
print("{:5} {:5.2f}".format(positive_rois,
positive_rois / ids.shape[1]))
print("Average percent: {:.2f}".format(total / (limit * ids.shape[1])))
