def compute_mean_AP(model, config, dataset, n_images):
""" Compute VOC-Style mAP @ IoU=0.5
"""
image_ids = np.random.choice(dataset.image_ids, n_images)
APs = []
for image_id in image_ids:
# Load image and ground truth data
result = modellib.load_image_gt(dataset,
config,
image_id,
use_mini_mask=False)
if len(result) == 5:
image, image_meta, class_ids, gt_bbox, gt_mask = result
else:
image, image_meta, gt_bbox, gt_mask = result
class_ids = gt_bbox[:, 4]
gt_bbox = gt_bbox[:, :4]
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, class_ids, r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
return np.mean(APs)
def evaluate_gdxray(model, dataset, eval_type="bbox", limit=0, image_ids=None):
"""Runs GDXRay evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
image_ids = image_ids or dataset.image_ids
random.shuffle(image_ids)
limit = int(limit)
num_processed = 0
APs = []
for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
# Run object detection
results = model.detect([image], verbose=0)
# Compute AP
r = results[0]
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r['rois'], r['class_ids'], r['scores'], r['masks'])
print("Image", image_id, "AP:", AP)
num_processed += 1
APs.append(AP)
if limit and num_processed > limit:
break
return np.mean(APs)
def evaluate_model(dataset, model, cfg):
APs = list()
for image_id in dataset.image_ids:
# load image, bounding boxes and masks for the image id
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, cfg, image_id, use_mini_mask=False)
# convert pixel values (e.g. center)
scaled_image = modellib.mold_image(image, cfg)
# convert image into one sample
sample = expand_dims(scaled_image, 0)
# make prediction
yhat = model.detect(sample, verbose=0)
# extract results for first sample
r = yhat[0]
try:
# calculate statistics, including AP
AP, _, _, _ = utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"],
r["scores"], r['masks'])
# store
APs.append(AP)
print("good image id : %d" % image_id)
print("AP : %.3f" % AP)
except:
print("bad image id : %d" % image_id)
# calculate the mean AP across all images
mAP = mean(APs)
return mAP
def test_dataset(model, dataset, nr_images):
for i in range(nr_images):
image_id = dataset.image_ids[i] if nr_images == len(dataset.image_ids) else random.choice(dataset.image_ids)
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
r = model.detect([image], verbose=0)[0]
print(r['class_ids'].shape)
if r['class_ids'].shape[0]>0:
r_fused = utils.fuse_instances(r)
else:
r_fused = r
fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(16, 16))
# Display predictions
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
dataset.class_names, r['scores'], title="Predictions", ax=ax1)
visualize.display_instances(image, r_fused['rois'], r_fused['masks'], r_fused['class_ids'],
dataset.class_names, r_fused['scores'], title="Predictions fused", ax=ax2)
# # Display ground truth
visualize.display_instances(image, gt_bbox, gt_mask, gt_class_id, dataset.class_names, title="GT", ax=ax3)
# Voilà
plt.show()
def test(model, config, limit=None):
dataset_test = S3DDataset()
dataset_test.load_s3d('test')
dataset_test.prepare()
print("Start inferencing on %d images..." % len(dataset_test.image_info))
APs = []
import tqdm
for i in tqdm.tqdm(range(len(dataset_test.image_info))):
if limit is not None:
if i >= int(limit):
continue
image, meta, gt_class_ids, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_test, config, i)
result = model.detect([image], verbose=0)[0]
bbox = result['rois']
mask = result['masks']
class_ids = result['class_ids']
scores = result['scores']
second_class_ids = result['second_class_ids']
second_scores = result['second_scores']
probs = result['probs'][0]
# print(class_ids)
# print(second_class_ids)
# print(scores)
# print(second_scores)
# print(bbox.shape, gt_bbox.shape)
# print(mask.shape, gt_mask.shape)
# print(class_ids.shape, gt_class_ids.shape)
# print(scores.shape)
# 基础的结果
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
delta = 0.0
# 计入概率次高分类之后的结果
for i in range(len(class_ids)):
ori = class_ids[i]
class_ids[i] = second_class_ids[i]
mAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
class_ids[i] = ori
if mAP - basemAP > 0:
delta += mAP - basemAP
if mAP >= 0 and mAP <= 1:
APs.append(basemAP + delta)
print('%.3f' % np.mean(APs))
def predictTests_Bulk(model, inference_config, validpath, plot, cnt=None):
print('BATCH SIZE', inference_config.IMAGES_PER_GPU)
dataset_valid = ShapesDataset()
dataset_valid.load_imgs(validpath)
dataset_valid.prepare()
new_test_ids = []
rles = []
tot = len(dataset_valid.image_ids)
prog = tqdm(total=tot)
for i in range(0, tot, inference_config.IMAGES_PER_GPU):
if cnt is not None and i > cnt:
break
imgs = []
img_metas = []
file_ids = []
for j in range(inference_config.IMAGES_PER_GPU):
id = i + j
image_id = dataset_valid.image_ids[id]
file_id = dataset_valid.image_info[image_id]['id']
scaled_image, image_meta, _, _, _ =\
modellib.load_image_gt(dataset_valid, inference_config,
image_id, use_mini_mask=True, augment=False)
file_ids.append(file_id)
imgs.append(scaled_image)
img_metas.append(image_meta)
results = model.detect(imgs, verbose=0)
for i in range(len(results)):
rle = convert_result(imgs[i], results[i], img_metas[i],
dataset_valid, plot)
file_id = file_ids[i]
rles.extend(rle)
new_test_ids.extend([file_id] * len(rle))
prog.update(inference_config.IMAGES_PER_GPU)
sub = pd.DataFrame()
sub['ImageId'] = new_test_ids
sub['EncodedPixels'] = pd.Series(rles).apply(
lambda x: ' '.join(str(y) for y in x))
sub.to_csv('../result/{}_org.csv'.format(inference_config.NAME),
index=False)
kaggle_util.save_result(sub,
'../result/{}.csv'.format(inference_config.NAME),
competition='airbus-ship-detection',
send=True,
index=False)
return sub
def test_dataset(model, dataset, nr_images):
for i in range(nr_images):
image_id = dataset.image_ids[
i] #if nr_images == len(dataset.image_ids) #else random.choice(dataset.image_ids)
image, image_meta = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
"""
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
"""
info = dataset.image_info[image_id]
#print("info",info)
#detection
r = model.detect([image], verbose=0)[0]
print(r['class_ids'].shape[0])
print(r['class_ids'].shape[0],
file=codecs.open('amount_' + str(i) + '.txt', 'w', 'utf-8'))
if r['class_ids'].shape[0] > 0:
r_fused = utils.fuse_instances(r)
else:
r_fused = r
fig, (ax1) = plt.subplots(1, 1, figsize=(16, 16))
#fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(16, 16))
#予測表示
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
dataset.class_names,
r['scores'],
title="Predictions",
ax=ax1)
"""
visualize.display_instances(image, r_fused['rois'], r_fused['masks'], r_fused['class_ids'],
dataset.class_names, r_fused['scores'], title="Predictions fused", ax=ax2)
"""
#正解データ表示
#visualize.display_instances(image, gt_bbox, gt_mask, gt_class_id, dataset.class_names, title="GT", ax=ax3)
# 画像表示
plt.show()
def compute_batch_ap(image_ids):
APs = []
for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
# Run object detection
results = model.detect([image], verbose=0)
# Compute AP
r = results[0]
AP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_id, r['rois'], r['class_ids'], r['scores'])
APs.append(AP)
return APs
def evaluate_gdxray(self, model, dataset):
"""Evalua la imagen.
model: modelo con el que se va a evaluar la imagen.
dataset: imagen a evaluar.
"""
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset, config, 0, use_mini_mask=False)
# Run object detection
results = model.detect([image], verbose=0)
# Compute AP
self.r = results[0]
visualize.display_instances(image, self.r['rois'], self.r['masks'], self.r['class_ids'], dataset.class_names, self.r['scores'], visualize = False)
self.show_image()
def visualization(model,dataset_val,inference_config,img_id=0):
print("Visualization (on random Test Image, Ground Truths)")
# Test on a random image
image_id = img_id
original_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id,dataset_val.class_names,figsize=(8, 8))
print("Detecting for test image")
results = model.detect([original_image], verbose=1)
print("Visualization (on random Test Image, Predicted)")
r = results[0]
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'],
dataset_val.class_names, r['scores'])
def test_dataset(model, dataset, nr_images):
for i in range(nr_images):
image_id = random.choice(dataset.image_ids)
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
r = model.detect([image], verbose=0)[0]
# Display results
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
dataset.class_names, r['scores'], title="Predictions")
# Display ground truth
#visualize.display_instances(image, gt_bbox, gt_mask, gt_class_id, dataset.class_names)
print(r['class_ids'])
def evaluate_wpif(model, config, dataset, eval_type="bbox", limit=0):
"""
Evaluation on WPIF dataset, using VOC-Style mAP # IoU=0.5 for bbox
@TODO: add segment evaluation
:param model:
:param config:
:param dataset:
:param eval_type:
:param limit:
:return:
"""
image_ids = dataset.image_ids
if limit:
image_ids = np.random.choice(dataset.image_ids, limit)
t_prediction = 0
t_start = time.time()
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
t = time.time()
results = model.detect([image], verbose=0)
r = results[0]
t_prediction += (time.time() - t)
# Compute AP
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id,
r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
print("mAP: ", np.mean(APs))
def simpleValidation():
class InferenceConfig(NucleusConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
inference_config = InferenceConfig()
model = modellib.MaskRCNN(mode="inference",
config=inference_config,
model_dir=MODEL_DIR)
model_path = model.find_last()[1]
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
dataset_valid = ShapesDataset()
dataset_valid.load_imgs(VALID_PATH)
dataset_valid.prepare()
image_id = dataset_valid.image_ids[3]
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_valid, inference_config,
image_id, use_mini_mask=True, augment=False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
results = model.detect([original_image], verbose=1)
r = results[0]
visualize.display_instances(original_image,
r['rois'],
r['masks'],
r['class_ids'],
dataset_valid.class_names,
r['scores'],
ax=get_ax())
def evaluate_apc(model, config, dataset, eval_type="bbox", limit=0):
"""
Evaluation on APC dataset, using VOC-Style mAP # IoU=0.5 for bbox
@TODO: add segment evaluation
:param model:
:param config:
:param dataset:
:param eval_type:
:param limit:
:return:
"""
image_ids = dataset.image_ids
if limit:
image_ids = np.random.choice(dataset.image_ids, limit)
t_prediction = 0
t_start = time.time()
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
t = time.time()
results = model.detect([image], verbose=0)
r = results[0]
t_prediction += (time.time() - t)
# Compute AP
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id,
r["rois"], r["class_ids"], r["scores"])
APs.append(AP)
print("Prediction time: {}. Average {}/image".format(
t_prediction, t_prediction / len(image_ids)))
print("Total time: ", time.time() - t_start)
print("mAP: ", np.mean(APs))
def evaluate_maskrcnn(dataset_val,inference_config = InferenceConfig()):
image_ids = np.random.choice(dataset_val.image_ids, 10)
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs.append(AP)
print("mAP: ", np.mean(APs))
return APs
def test_dataset(model, dataset, nr_images):
for i in range(nr_images):
image_id = random.choice(dataset.image_ids)
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
r = model.detect([image], verbose=0)[0]
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 16))
# Display predictions
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
dataset.class_names, r['scores'], title="Predictions", ax=ax1)
# Display ground truth
visualize.display_instances(image, gt_bbox, gt_mask, gt_class_id, dataset.class_names, title="GT", ax=ax2)
# Voilà
plt.show()
def evaluate_engine(model,
dataset,
inference_config,
eval_type="bbox",
limit=0,
image_ids=None):
"""Runs official COCO evaluation.
dataset: A Dataset object with valiadtion data
eval_type: "bbox" or "segm" for bounding box or segmentation evaluation
limit: if not 0, it's the number of images to use for evaluation
"""
# Pick images from the dataset
image_ids = np.random.choice(dataset.image_ids, 100)
# Limit to a subset
if limit:
image_ids = image_ids[:limit]
t_prediction = 0
t_start = time.time()
APs = []
for image_id in image_ids:
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, inference_config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(
modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP, precisions, recalls, overlaps =\
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs.append(AP)
print("mAP: ", np.mean(APs))
print("Total time: ", time.time() - t_start)
# https://github.com/matterport/Mask_RCNN/releases
# weights_path = "/path/to/mask_rcnn_pig.h5"
# Or, load the last model you trained
weights_path = model.find_last()[1]
# Load weights
print("Loading weights ", weights_path)
model.load_weights(weights_path, by_name=True)
# ## Run Detection
# In[9]:
image_id = random.choice(dataset.image_ids)
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id,
dataset.image_reference(image_id)))
# Run object detection
results = model.detect([image], verbose=1)
# Display results
ax = get_ax(1)
r = results[0]
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
dataset.class_names,
def run(img_dir,
annos_dir,
ONLY_TEST=1,
STEPS_IS_LEN_TRAIN_SET=0,
n_epochs=5,
layer_string="5+",
name="Faster_RCNN-"):
""" heads: The RPN, classifier and mask heads of the network
all: All the layers
3+: Train Resnet stage 3 and up
4+: Train Resnet stage 4 and up
5+: Train Resnet stage 5 and up
img_dir: path to directory containing images
annos_dir: path to directory containing annotations """
# torch.backends.cudnn.benchmark = True
start_time = time.process_time()
print("start time time(s): ", round(start_time, 2))
# CONFIGURATION
import config
config = config.Config()
config.NAME = name
config.display()
# TEST SET
test_set = custom_dataset.LampPostDataset()
test_set.load_dataset(img_dir, annos_dir, is_train=False)
test_set.prepare()
if not ONLY_TEST:
# TRAINING SET
train_set = custom_dataset.LampPostDataset()
train_set.load_dataset(img_dir, annos_dir, is_train=True)
train_set.prepare()
print("Train: %d, Test: %d images" %
(len(train_set.image_ids), len(test_set.image_ids)))
if STEPS_IS_LEN_TRAIN_SET:
config.STEPS_PER_EPOCH = len(train_set.image_info)
data_time = time.process_time()
print("load data time(s): ", round(data_time - start_time, 2),
"total elapsed: ", round(data_time, 2))
# LOAD MODEL
model = modellib.MaskRCNN(config=config, model_dir='./models/')
load_model_time = time.process_time()
print("loading model time(s): ", round(load_model_time - data_time, 2),
"total elapsed: ", round(load_model_time, 2))
# LOAD WEIGHTS
model.load_weights(
'./models/mask_rcnn_coco.pth',
callback=True) # exclude=["mrcnn_class_logits", "mrcnn_bbox_fc",
# "mrcnn_bbox", "mrcnn_mask"]
load_weights_time = time.process_time()
print("loading weights time(s): ",
round(load_weights_time - load_model_time, 2), "total elapsed: ",
round(load_weights_time, 2))
# Save final config before start training
config.to_txt(model.log_dir)
# TRAIN MODEL
# train heads with higher lr to speedup the learning
model.train_model(train_set,
test_set,
learning_rate=2 * config.LEARNING_RATE,
epochs=n_epochs,
layers=layer_string)
train_time = time.process_time()
print("training time(s): ",
round((train_time - load_weights_time) / 60, 2),
"total minutes elapsed: ", round(train_time, 2))
# TEST MODEL
modellib.device = torch.device(
"cuda:0" if torch.cuda.is_available() else "cpu")
model = modellib.MaskRCNN(config=config, models_dir='./models')
# loading the trained weights of the custom dataset
# last_model = model.find_last()[1]
last_model = "./models/resnet50_imagenet.pth"
print("loading model: ", last_model)
model.load_weights(last_model)
# Delete test model log directory
os.rmdir(model.log_dir)
image_id = 3
# 1 = TMX7316010203-001499_pano_0000_001233 - only a hanging lamp post
# 2 = TMX7316010203-001209_pano_0000_002760 - on the right, behind/above blue car
# 3 = TMX7316010203-001187_pano_0000_002097 - clearly in the middle (old one) and further down the road on the right
image, image_meta, gt_class_id, gt_bbox = modellib.load_image_gt(
test_set, config, image_id)
info = test_set.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"],
image_id,
test_set.image_reference(image_id)))
# Run object detection
results = model.detect([image])
# Display results
r = results[0]
visualize.display_instances(
image,
r['rois'],
r['class_ids'], # r['masks'],
test_set.class_names,
r['scores'],
title="Predictions")
print("Count: ", anchors.shape[0])
print("Scales: ", config.RPN_ANCHOR_SCALES)
print("ratios: ", config.RPN_ANCHOR_RATIOS)
print("Anchors per Cell: ", anchors_per_cell)
print("Levels: ", num_levels)
anchors_per_level = []
for l in range(num_levels):
num_cells = BACKBONE_SHAPES[l][0] * BACKBONE_SHAPES[l][1]
anchors_per_level.append(anchors_per_cell * num_cells //
config.RPN_ANCHOR_STRIDE**2)
print("Anchors in Level {}: {}".format(l, anchors_per_level[l]))
## Visualize anchors of one cell at the center of the feature map of a specific level
# Load and draw random image
image, image_meta, _, _, _ = modellib.load_image_gt(dataset, config, image_id)
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.imshow(image)
levels = len(BACKBONE_SHAPES)
for level in range(levels):
colors = visualize.random_colors(levels)
# Compute the index of the anchors at the center of the image
level_start = sum(
anchors_per_level[:level]) # sum of anchors of previous levels
level_anchors = anchors[level_start:level_start + anchors_per_level[level]]
print("Level {}. Anchors: {:6} Feature map Shape: {}".format(
level, level_anchors.shape[0], BACKBONE_SHAPES[level]))
center_cell = BACKBONE_SHAPES[level] // 2
center_cell_index = (center_cell[0] * BACKBONE_SHAPES[level][1] +
center_cell[1])
def test(model, config, limit=None, savefiledir=None):
dataset_test = S3DDataset()
dataset_test.load_s3d('test')
dataset_test.prepare()
print("Start inferencing on %d images..." % len(dataset_test.image_info))
APs1, APs2 = [], []
with open('details_04.txt', 'w') as fout:
import tqdm
for i in tqdm.tqdm(range(len(dataset_test.image_info))):
if limit is not None:
if i >= int(limit):
continue
image, meta, gt_class_ids, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_test, config, i)
result = model.detect([image],
dataset_test.load_rel_coefs(),
dataset_test.load_rel_bias(),
verbose=0)[0]
bbox = result['rois']
mask = result['masks']
class_ids = result['class_ids']
scores = result['scores']
second_class_ids = result['second_class_ids']
second_scores = result['second_scores']
probs = result['probs'][0]
dists_x = result['dists_x']
dists_y = result['dists_y']
coefs = result['coefs']
bias = result['bias']
def drawfig():
# 画loss热力图
fig = plt.figure(figsize=(16, 16))
ax = fig.add_subplot(221)
im = ax.imshow(dists + 2 * np.eye(len(dists)),
cmap='Blues',
interpolation='none',
vmin=0,
vmax=2,
aspect='equal')
plt.colorbar(im, shrink=0.5)
ax = fig.add_subplot(222)
im = ax.imshow(coefs,
cmap='Reds',
interpolation='none',
vmin=0,
aspect='equal')
plt.colorbar(im, shrink=0.5)
ax = fig.add_subplot(223)
im = ax.imshow((1. / np.exp(dists + 2 * np.eye(len(dists)))) *
(1. / np.exp(coefs)),
cmap='Greens',
interpolation='none',
vmin=0,
aspect='equal')
plt.colorbar(im, shrink=0.5)
plt.savefig('fig.jpg')
def savefig():
visualize.display_instances(
image,
gt_bbox,
gt_mask,
gt_class_ids, [
categories.category2name(i)
for i in range(categories.cate_cnt)
],
savefilename=os.path.join(save_visual_path,
'%05d_gt.jpg' % i))
visualize.display_instances(
image,
bbox,
mask,
class_ids, [
categories.category2name(i)
for i in range(categories.cate_cnt)
],
savefilename=os.path.join(save_visual_path,
'%05d_pred.jpg' % i))
# @timer
def secondClassResults():
# 基础的结果
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores,
mask)
delta = 0.0
# 计入概率次高分类之后的结果
for i in range(len(class_ids)):
ori = class_ids[i]
class_ids[i] = second_class_ids[i]
mAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids,
scores, mask)
class_ids[i] = ori
if mAP - basemAP > 0:
delta += mAP - basemAP
if basemAP >= 0 and basemAP <= 1:
APs2.append(basemAP + delta)
# @timer
def basicResults():
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores,
mask)
if basemAP >= 0 and basemAP <= 1:
APs1.append(basemAP)
fout.write('%d %f\n' % (i, basemAP))
# visualize.display_instances(image, gt_bbox, gt_mask, gt_class_ids, [categories.category2name(i) for i in range(categories.cate_cnt)], savefilename=os.path.join(savefiledir, 'visual', '%05d_A.jpg' % i))
# visualize.display_instances_second_class(image, bbox, mask, class_ids, second_class_ids, [categories.category2name(i) for i in range(categories.cate_cnt)], scores, second_scores, savefilename=os.path.join(savefiledir, 'visual', '%05d_B.jpg' % i))
basicResults()
print('%.3f' % np.mean(APs1))
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# ## Mini Masks
#
# Instance binary masks can get large when training with high resolution images. For example, if training with 1024x1024 image then the mask of a single instance requires 1MB of memory (Numpy uses bytes for boolean values). If an image has 100 instances then that's 100MB for the masks alone.
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box, rather than a mask of the full image. Most objects are small compared to the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56). For objects that are larger than the selected size we lose a bit of accuracy. But most object annotations are not very accuracy to begin with, so this loss is negligable for most practical purposes. Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness, we visualize some examples.
# In[7]:
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
# In[8]:
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# In[9]:
print ("Weights loaded")
# load dataset
test_dataset = CowDataset()
count = len(os.listdir(os.path.join(source, 'cow_imgs_val_large')))
#count = len(os.listdir(os.path.join(source, 'cow_imgs_val')))
test_dataset.load_cows(count=count, stage = 'testing_mydata')
test_dataset.prepare()
print("Images: {}\nClasses: {}".format(len(test_dataset.image_ids), test_dataset.class_names))
#print (test_dataset.image_ids)
all_AP_list=[]
thresholds = np.linspace(0.5,0.95,10)
for th in thresholds:
AP_list=[]
for img in test_dataset.image_ids:
#image, image_meta, gt_mask = modellib.load_image_gt(test_dataset, config, img)
image, image_meta, class_ids, gt_bbox, gt_mask= modellib.load_image_gt(test_dataset, config, img)
print (img)
plt.imsave(str(img)+'.png', image)
mask,gt_class_ids = test_dataset.load_mask(img)
#print ('mask', mask.shape, image.shape)
results = model.detect([image], verbose=1)
r = results[0]
#print (r.keys())
plt.imsave('gt' + str(img)+'.png', mask[:,:,0])
#print ("get class ids")
#print (r['rois'], r['class_ids'], r['scores'])
# for the base model-keep only cow predictions
present_class = r['class_ids']
for idx, cls in enumerate(present_class):
if cls not in good_classes:
np.delete(r['rois'], idx, axis=0)
def segment_data_generation(mode, data_base_dir, use_edgelist=False, debug=False):
if mode == 'both':
dataset_types = ['val', 'test']
else:
dataset_types = [mode]
caption_base_dir = 'data'
outputs_base_dir = 'outputs'
trained_model_dir = os.path.join(outputs_base_dir, 'snapshot')
edgelist_result_dir = os.path.join(outputs_base_dir, 'edgelist')
seg_data_save_base_dir = os.path.join(outputs_base_dir, 'inst_segm_output_data')
epochs = '0100'
model_path = os.path.join(trained_model_dir, 'mask_rcnn_sketchyscene_' + epochs + '.h5')
dataset_class_names = ['bg']
color_map_mat_path = os.path.join(data_base_dir, 'colorMapC46.mat')
colorMap = scipy.io.loadmat(color_map_mat_path)['colorMap']
for i in range(46):
cat_name = colorMap[i][0][0]
dataset_class_names.append(cat_name)
ROAD_LABEL = dataset_class_names.index('road')
CLASS_ORDERS = [[dataset_class_names.index('sun'), dataset_class_names.index('moon'),
dataset_class_names.index('star'), dataset_class_names.index('road')],
[dataset_class_names.index('tree')],
[dataset_class_names.index('cloud')],
[dataset_class_names.index('house')],
[dataset_class_names.index('bus'), dataset_class_names.index('car'),
dataset_class_names.index('truck')]]
config = SkeSegConfig()
model = modellib.MaskRCNN(mode="inference", config=config, model_dir='', log_dir='')
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
for dataset_type in dataset_types:
caption_json_path = os.path.join(caption_base_dir, 'sentence_instance_' + dataset_type + '.json')
fp = open(caption_json_path, "r")
json_data = fp.read()
json_data = json.loads(json_data)
print('data_len', len(json_data))
# val/test dataset
dataset = SketchDataset(data_base_dir)
dataset.load_sketches(dataset_type)
dataset.prepare()
split_seg_data_save_base_dir = os.path.join(seg_data_save_base_dir, dataset_type)
os.makedirs(split_seg_data_save_base_dir, exist_ok=True)
for data_idx in range(len(json_data)):
img_idx = json_data[data_idx]['key']
print('Processing', dataset_type, data_idx + 1, '/', len(json_data))
original_image, _, gt_class_id, gt_bbox, gt_mask, _ = \
modellib.load_image_gt(dataset, config, img_idx - 1, use_mini_mask=False)
## 1. inference
results = model.detect([original_image])
r = results[0]
pred_boxes = r["rois"] # (nRoIs, (y1, x1, y2, x2))
pred_class_ids = r["class_ids"] # (nRoIs)
pred_scores = r["scores"]
pred_masks = r["masks"] # (768, 768, nRoIs)
log("pred_boxes", pred_boxes)
log("pred_class_ids", pred_class_ids)
log("pred_masks", pred_masks)
## 2. Use original_image(768, 768, 3) {0, 255} to filter pred_masks
if config.IGNORE_BG:
pred_masks = np.transpose(pred_masks, (2, 0, 1)) # (nRoIs, 768, 768)
bin_input = original_image[:, :, 0] == 255
pred_masks[:, bin_input[:, :]] = 0 # (nRoIs, 768, 768)
pred_masks = np.transpose(pred_masks, (1, 2, 0)) # (768, 768, nRoIs)
if debug:
visualize.display_instances(original_image, pred_boxes, pred_masks, pred_class_ids,
dataset.class_names, pred_scores, figsize=(8, 8))
## 3. refine pred_masks(768, 768, nRoIs) with edge-list
if use_edgelist:
pred_masks = \
refine_mask_with_edgelist(img_idx, dataset_type, data_base_dir, edgelist_result_dir,
pred_masks.copy(), pred_boxes)
## 4. TODO: remove road prediction
# pred_boxes = pred_boxes.tolist()
# pred_masks = np.transpose(pred_masks, (2, 0, 1)).tolist()
# pred_scores = pred_scores.tolist()
# pred_class_ids = pred_class_ids.tolist()
#
# while ROAD_LABEL in pred_class_ids:
# road_idx = pred_class_ids.index(ROAD_LABEL)
# pred_boxes.remove(pred_boxes[road_idx])
# pred_masks.remove(pred_masks[road_idx])
# pred_scores.remove(pred_scores[road_idx])
# pred_class_ids.remove(ROAD_LABEL)
## 5. TODO: add road from semantic prediction
# sem_label_base_path = '../../../../Sketch-Segmentation-TF/Segment-Sketch-DeepLab-v2/edge-list/pred_semantic_label_edgelist/'
# sem_label_base_path = os.path.join(sem_label_base_path, dataset_type, 'mat')
# sem_label_path = os.path.join(sem_label_base_path, 'L0_sample' + str(img_idx) + '.mat')
# sem_label = scipy.io.loadmat(sem_label_path)['pred_label_edgelist'] # (750, 750), [0, 46]
#
# if ROAD_LABEL in sem_label:
# road_mask_img = np.zeros([sem_label.shape[0], sem_label.shape[1], 3], dtype=np.uint8)
# road_mask_img[sem_label == ROAD_LABEL] = [255, 255, 255] # (750, 750, 3), {0, 255}
# road_mask_img = scipy.misc.imresize(
# road_mask_img, (config.IMAGE_MAX_DIM, config.IMAGE_MAX_DIM), interp='nearest') # (768, 768, 3)
# road_mask = np.zeros(road_mask_img[:, :, 0].shape, dtype=np.uint8)
# road_mask[road_mask_img[:, :, 0] == 255] = 1 # (768, 768), {0, 1}
# # plt.imshow(road_mask)
# # plt.show()
#
# road_bbox = utils.extract_bboxes(np.expand_dims(road_mask, axis=2)) # [num_instances, (y1, x1, y2, x2)]
# road_bbox = road_bbox[0]
# pred_boxes.append(road_bbox)
# pred_masks.append(road_mask)
# pred_scores.append(1.)
# pred_class_ids.append(ROAD_LABEL)
# pred_boxes = np.array(pred_boxes, dtype=np.int32)
# pred_class_ids = np.array(pred_class_ids, dtype=np.int32)
# pred_scores = np.array(pred_scores, dtype=np.float32)
# pred_masks = np.array(pred_masks, dtype=np.uint8)
# pred_masks = np.transpose(pred_masks, [1, 2, 0]) # (768, 768, nRoIs?)
if debug:
visualize.display_instances(original_image, pred_boxes, pred_masks, pred_class_ids,
dataset.class_names, pred_scores, figsize=(8, 8))
## 8. sort instances
instance_sorted_index = []
for order_idx in range(len(CLASS_ORDERS)):
order_ids = CLASS_ORDERS[order_idx]
for cate_idx in range(pred_class_ids.shape[0]):
if pred_class_ids[cate_idx] in order_ids:
instance_sorted_index.append(cate_idx)
for cate_idx in range(pred_class_ids.shape[0]):
if cate_idx not in instance_sorted_index:
instance_sorted_index.append(cate_idx)
# print('pred_class_ids', pred_class_ids)
# print('instance_sorted_index', instance_sorted_index)
assert len(instance_sorted_index) == pred_class_ids.shape[0]
pred_class_ids_list = []
pred_masks_list = []
pred_boxes_list = []
for cate_idx_i in range(len(instance_sorted_index)):
pred_class_ids_list.append(pred_class_ids[instance_sorted_index[cate_idx_i]])
pred_box = pred_boxes[instance_sorted_index[cate_idx_i]]
y1, x1, y2, x2 = pred_box
pred_mask_large = pred_masks[:, :, instance_sorted_index[cate_idx_i]]
pred_mask = pred_mask_large[y1: y2 + 1, x1: x2 + 1]
pred_masks_list.append(pred_mask)
pred_boxes_list.append(pred_box)
# print('pred_class_ids_list', pred_class_ids_list)
assert len(pred_class_ids_list) == pred_class_ids.shape[0]
## 9. generate .npz data
npz_name = os.path.join(split_seg_data_save_base_dir, str(img_idx) + '_datas.npz')
np.savez(npz_name, pred_class_ids=pred_class_ids_list, pred_masks=pred_masks_list,
pred_boxes=pred_boxes_list)
def main():
# Root directory of the project
ROOT_DIR = os.getcwd()
# Directory to save logs and trained model
MODEL_DIR = os.path.join(ROOT_DIR, "logs")
# MODEL_DIR = '/media/rodrigo/c1d7e9c9-c8cb-402e-b241-9090925389b3/IA_Challenger/save_log'
# MODEL_DIR = os.path.join(MODEL_DIR, "logs")
# Path to COCO trained weights
COCO_MODEL_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
# Path to MPII data set
# TODO:
#MPII_DATA_PATH = '/media/rodrigo/c1d7e9c9-c8cb-402e-b241-9090925389b3/IA_Challenger'
MPII_DATA_PATH = os.path.join(ROOT_DIR, 'dataset')
class mpii_data_config(Config):
"""Configuration for training on the dataset.
Derives from the base Config class and overrides values specific
to training dataset.
"""
# Give the configuration a recognizable name
NAME = "MPII"
# Train on 1 GPU and 1 images per GPU.
GPU_COUNT = 1
IMAGES_PER_GPU = 1
# Number of classes (including background)
NUM_CLASSES = 1 + 1 # background + 1 mask
RPN_TRAIN_ANCHORS_PER_IMAGE = 150
# Number of validation steps to run at the end of every training epoch.
# A bigger number improves accuracy of validation stats, but slows
# down the training.
VALIDATION_STPES = 100
STEPS_PER_EPOCH = 5
MINI_MASK_SHAPE = (56, 56)
# Pooled ROIs
POOL_SIZE = 7
MASK_POOL_SIZE = 14
MASK_SHAPE = [28, 28]
# Maximum number of ground truth instances to use in one image
MAX_GT_INSTANCES = 128
config = mpii_data_config()
# GPU for training.
DEVICE = "/gpu:0" # /cpu:0 or /gpu:0
TEST_MODE = "train"
# Training dataset
dataset_train = mpii_data_set()
dataset_train.load_mpii(MPII_DATA_PATH, TEST_MODE)
dataset_train.prepare()
# Validation dataset
dataset_val = mpii_data_set()
dataset_val.load_mpii(MPII_DATA_PATH, 'val')
dataset_val.prepare()
print("Classes: {}.\n".format(dataset_train.class_names))
print("Train Images: {}.\n".format(len(dataset_train.image_ids)))
print("Valid Images: {}".format(len(dataset_val.image_ids)))
image_id = np.random.choice(dataset_val.image_ids)
image, image_meta, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_val, config, image_id, use_mini_mask=False, augment=False)
"""
def plot_mask_points(dataset, config, model, filter=True, image_id=None):
if not image_id:
image_id = random.choice(dataset.image_ids)
original_image, image_meta, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
mrcnn = model.run_graph([original_image], [
("detections", model.keras_model.get_layer("mrcnn_detection").output),
("masks", model.keras_model.get_layer("mrcnn_mask").output),
("mask_classes",
model.keras_model.get_layer("mrcnn_class_mask").output),
])
det_ix = mrcnn['detections'][0, :, 4]
det_count = np.where(det_ix == 0)[0][0]
det_masks = mrcnn['masks'][0, :det_count, :, :, :]
det_boxes = mrcnn['detections'][0, :det_count, :]
det_mask_classes = np.argmax(mrcnn['mask_classes'][0, :det_count, :, :],
axis=2)
det_mask_classes = np.where(det_mask_classes == 0,
np.ones_like(det_mask_classes),
np.zeros_like(det_mask_classes))
visualize.draw_mpii_boxes(original_image,
refined_boxes=det_boxes[:, :4],
bp=mpii_class_names_)
_, ax = plt.subplots(3, 5)
for i in range(5):
ax[0, i].set_title(mpii_class_names_[i])
if filter:
m = np.where(det_masks[0, :, :, i] > 0.8, det_masks[0, :, :, i], 0)
m = np.where(m == m.max(), 1, 0)
m = m * det_mask_classes[0, i]
else:
m = det_masks[0, :, :, i] * det_mask_classes[0, i]
ax[0, i].imshow(m, interpolation='none')
for i in range(5):
ax[1, i].set_title(mpii_class_names_[5 + i])
if filter:
m = np.where(det_masks[0, :, :, 5 + i] > 0.8, det_masks[0, :, :,
5 + i], 0)
m = np.where(m == m.max(), 1, 0)
m = m * det_mask_classes[0, 5 + i]
else:
m = det_masks[0, :, :, 5 + i] * det_mask_classes[0, 5 + i]
ax[1, i].imshow(m, interpolation='none')
for i in range(4):
ax[2, i].set_title(mpii_class_names_[10 + i])
if filter:
m = np.where(det_masks[0, :, :, 10 + i] > 0.8,
det_masks[0, :, :, 10 + i], 0)
m = np.where(m == m.max(), 1, 0)
m = m * det_mask_classes[0, 10 + i]
else:
m = det_masks[0, :, :, 10 + i] * det_mask_classes[0, 10 + i]
ax[2, i].imshow(m, interpolation='none')
ax[2, 4].set_title('Real image')
visualize.draw_mpii_boxes(original_image,
refined_boxes=det_boxes[:1, :4],
ax=ax[2, 4],
bp=mpii_class_names_)
# Plot the gt mask points
_, axx = plt.subplots(3, 5)
axx[2, 4].set_title('Real image')
visualize.draw_mpii_boxes(original_image,
refined_boxes=gt_bbox[:1, :4],
masks=gt_mask,
ax=axx[2, 4],
bp=mpii_class_names_)
original_image, image_meta, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=True)
for i in range(5):
axx[0, i].set_title(mpii_class_names_[i])
axx[0, i].imshow(gt_mask[0, :, :, i], interpolation='none')
for i in range(5):
axx[1, i].set_title(mpii_class_names_[5 + i])
axx[1, i].imshow(gt_mask[0, :, :, 5 + i], interpolation='none')
for i in range(4):
axx[2, i].set_title(mpii_class_names_[10 + i])
axx[2, i].imshow(gt_mask[0, :, :, 10 + i], interpolation='none')
#plt.show()
plt.imsave('output.png')
def test(model, config, limit=None, savefiledir=None):
dataset_test = S3DDataset()
dataset_test.load_s3d('test')
dataset_test.prepare()
print("Start inferencing on %d images..." % len(dataset_test.image_info))
APs1, APs2 = [], []
subset = {}
with open(os.path.join(meta_path, 'subset.txt'), 'r') as fin:
for l in fin:
subset[int(l)] = 1
import tqdm
for i in tqdm.tqdm(range(len(dataset_test.image_info))):
if limit is not None:
if i >= int(limit):
continue
if i not in subset:
continue
image, meta, gt_class_ids, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_test, config, i)
result = model.detect([image], verbose=0)[0]
bbox = result['rois']
mask = result['masks']
class_ids = result['class_ids']
scores = result['scores']
second_class_ids = result['second_class_ids']
second_scores = result['second_scores']
probs = result['probs'][0]
# print(class_ids)
# print(second_class_ids)
# print(scores)
# print(second_scores)
# print(bbox.shape, gt_bbox.shape)
# print(mask.shape, gt_mask.shape)
# print(class_ids.shape, gt_class_ids.shape)
# print(scores.shape)
def savefig():
visualize.display_instances(image,
gt_bbox,
gt_mask,
gt_class_ids, [
categories.category2name(i)
for i in range(categories.cate_cnt)
],
savefilename=os.path.join(
save_visual_path,
'%05d_gt.jpg' % i))
visualize.display_instances(image,
bbox,
mask,
class_ids, [
categories.category2name(i)
for i in range(categories.cate_cnt)
],
savefilename=os.path.join(
save_visual_path,
'%05d_pred.jpg' % i))
# @timer
def secondClassResults():
# 基础的结果
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
delta = 0.0
# 计入概率次高分类之后的结果
for i in range(len(class_ids)):
ori = class_ids[i]
class_ids[i] = second_class_ids[i]
mAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores,
mask)
class_ids[i] = ori
if mAP - basemAP > 0:
delta += mAP - basemAP
if basemAP >= 0 and basemAP <= 1:
APs2.append(basemAP + delta)
# @timer
def basicResults():
basemAP, precisions, recalls, overlaps = utils.compute_ap(
gt_bbox, gt_class_ids, gt_mask, bbox, class_ids, scores, mask)
if basemAP >= 0 and basemAP <= 1:
APs1.append(basemAP)
# visualize.display_instances(image, gt_bbox, gt_mask, gt_class_ids, [categories.category2name(i) for i in range(categories.cate_cnt)], savefilename=os.path.join(savefiledir, 'visual', '%05d_A.jpg' % i))
# visualize.display_instances_second_class(image, bbox, mask, class_ids, second_class_ids, [categories.category2name(i) for i in range(categories.cate_cnt)], scores, second_scores, savefilename=os.path.join(savefiledir, 'visual', '%05d_B.jpg' % i))
basicResults()
print('%.3f' % np.mean(APs1))
def predictOne(model, dataset, config, id, plot=True):
image_id = dataset.image_ids[id]
scaled_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config,
image_id, use_mini_mask=True, augment=False)
file_id = dataset.image_info[image_id]['id']
original_shape = (image_meta[1], image_meta[2])
#print('\n{}\t{}\t'.format(file_id, original_shape))
results = model.detect([scaled_image], verbose=0)
r = results[0]
score_pick = filter_lowscores(r['scores'], 0.8)
r = pick_by_idx(r, score_pick)
boxes = r['rois']
masks = r['masks']
pick = non_max_suppression_mask_fast(r['masks'], 0.2)
pick_rois = r['rois'][pick]
pick_masks = r['masks'][:, :, pick]
pick_class_ids = r['class_ids'][pick]
pick_scores = r['scores'][pick]
# if plot:
# print('non_max_suppression [{}] to [{}]'.format(boxes.shape[0], len(pick)))
# if boxes.shape[0] != len(pick):
# ax = get_ax(rows=1, cols=2, size=5)
# visualize.display_instances(scaled_image, r['rois'], r['masks'], r['class_ids'],
# dataset.class_names, r['scores'], ax=ax[0])
# visualize.display_instances(scaled_image, pick_rois, pick_masks, pick_class_ids,
# dataset.class_names, pick_scores, ax=ax[1])
# plt.show()
# else:
# visualize.display_instances(scaled_image, pick_rois, pick_masks, pick_class_ids,
# dataset.class_names, pick_scores, ax=get_ax())
# plt.show()
r['rois'] = pick_rois
r['masks'] = pick_masks
r['class_ids'] = pick_class_ids
r['scores'] = pick_scores
#from scipy.misc import imresize
from skimage.transform import resize
rles = []
window = image_meta[4:8]
original_img = scaled_image[window[0]:window[2], window[1]:window[3], :]
original_img = resize(original_img, original_shape)
if min(r['masks'].shape) > 0:
submasks = np.zeros(
(image_meta[1], image_meta[2], r['masks'].shape[2]))
mask_record = np.ones(original_shape)
for i in range(r['masks'].shape[2]):
submask = r['masks'][window[0]:window[2], window[1]:window[3], i]
area = np.sum(submask)
submask = resize(submask, original_shape)
submask = np.logical_and(submask, mask_record)
if submask.shape != original_shape:
print(submask.shape)
mask_record = np.logical_and(mask_record, np.logical_not(submask))
#plt.figure()
#plt.imshow(mask_record)
submasks[:, :, i] = submask
rle = rle_encoding(submask)
if len(rle) > 0:
rles.append(rle)
else:
print('error')
else:
rles = [[]]
return file_id, rles
# Get path to saved weights
# Either set a specific path or find last trained weights
# model_path = os.path.join(ROOT_DIR, ".h5 file name here")
model_path = model.find_last()[1]
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
# In[12]:
# Test on a random image
image_id = random.choice(dataset_val.image_ids)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset_val, inference_config, image_id, use_mini_mask=False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
visualize.display_instances(original_image,
gt_bbox,
gt_mask,
gt_class_id,
dataset_train.class_names,
figsize=(8, 8))
# In[13]:
all_nums[line] = 1
imageCount = 0
accSum = 0
avgAccuracy = 0
for i in range(0, 200):
##Phase masks together##
if (all_nums[i] == 1):
fileName = str(i).zfill(4)
label_filepath = "/home/default/.keras/datasets/weedspic/broccoli/label/%s.png" % (
dataset_val.image_reference(i)["id"])
phased_mask = np.zeros((512, 512))
image_id = i
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
results = model.detect([original_image], verbose=0)
r = results[0]
msk = r['masks']
if (msk.size != 0):
for m in np.rollaxis(msk, 2):
for i in range(0, 512):
for k in range(0, 512):
if (m[i][k] == 1):
phased_mask[i][k] = 1
label = Image.open(label_filepath)
label = img_to_array(label.resize((512, 512),
Image.NEAREST)).astype(np.uint8)
label[np.where(label == 255)] = 1
def segment_inference(**kwargs):
data_base_dir = kwargs['data_base_dir']
dataset_type = kwargs['dataset_type']
image_id = kwargs['image_id']
epochs = kwargs['epochs']
use_edgelist = kwargs['use_edgelist']
outputs_base_dir = 'outputs'
vis_result_save_dir = os.path.join(outputs_base_dir, 'visual_result',
dataset_type)
trained_model_dir = os.path.join(outputs_base_dir, 'snapshot')
edgelist_result_dir = os.path.join(outputs_base_dir, 'edgelist')
model_path = os.path.join(trained_model_dir,
'mask_rcnn_sketchyscene_' + epochs + '.h5')
os.makedirs(vis_result_save_dir, exist_ok=True)
config = SketchInferConfig()
config.display()
# val/test dataset
dataset_infer = SketchDataset(data_base_dir)
dataset_infer.load_sketches(dataset_type)
dataset_infer.prepare()
# Recreate the model in inference mode
model = modellib.MaskRCNN(mode="inference",
config=config,
model_dir='',
log_dir='')
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
original_image, _, gt_class_id, gt_bbox, gt_mask, _ = \
modellib.load_image_gt(dataset_infer, config, image_id - 1, use_mini_mask=False)
log("original_image", original_image)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
gt_seg_path = os.path.join(vis_result_save_dir, str(image_id) + '_gt.png')
visualize.display_instances(original_image,
gt_bbox,
gt_mask,
gt_class_id,
dataset_infer.class_names,
title='Ground-Truth',
save_path=gt_seg_path,
fix_color=True)
## inference
results = model.detect([original_image], verbose=1)
r = results[0]
pred_boxes = r["rois"] # (nRoIs, (y1, x1, y2, x2))
pred_class_ids = r["class_ids"] # (nRoIs)
pred_scores = r["scores"] # (nRoIs)
pred_masks = r["masks"] # (768, 768, nRoIs)
log("pred_boxes", pred_boxes)
log("pred_masks", pred_masks)
if config.IGNORE_BG:
# Use original_image(768, 768, 3) {0, 255} to filter pred_masks
pred_masks = np.transpose(pred_masks, (2, 0, 1)) # (nRoIs, 768, 768)
bin_input = original_image[:, :, 0] == 255
pred_masks[:, bin_input[:, :]] = 0 # (nRoIs, 768, 768)
pred_masks = np.transpose(pred_masks, (1, 2, 0)) # (768, 768, nRoIs)
# refine pred_masks(768, 768, nRoIs) with edge-list
if use_edgelist:
refined_pred_masks = \
refine_mask_with_edgelist(image_id, dataset_type, data_base_dir, edgelist_result_dir,
pred_masks.copy(), pred_boxes)
# caculating AP
iou_thresholds = np.linspace(.5,
0.95,
np.round((0.95 - .5) / .05) + 1,
endpoint=True)
APs = np.zeros([len(iou_thresholds)], dtype=np.float32)
APs_edg = np.zeros([len(iou_thresholds)], dtype=np.float32)
for i in range(len(iou_thresholds)):
iouThr = iou_thresholds[i]
AP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
pred_boxes, pred_class_ids, pred_scores, pred_masks,
iou_threshold=iouThr)
APs[i] = AP
if use_edgelist:
AP_edg, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
pred_boxes, pred_class_ids, pred_scores, refined_pred_masks,
iou_threshold=iouThr)
APs_edg[i] = AP_edg
mAP = np.mean(APs)
mAP_edg = np.mean(APs_edg)
print('APs', APs)
print('mAP', mAP)
print('APs_edg', APs_edg)
print('mAP_edg', mAP_edg)
# save visual results
visual_seg_path = os.path.join(vis_result_save_dir,
str(image_id) + '_seg.png')
visualize.display_instances(original_image,
pred_boxes,
pred_masks,
pred_class_ids,
dataset_infer.class_names,
pred_scores,
title='Normal result',
save_path=visual_seg_path,
fix_color=True)
if use_edgelist:
visual_seg_edg_path = os.path.join(vis_result_save_dir,
str(image_id) + '_seg_edgelist.png')
visualize.display_instances(original_image,
pred_boxes,
refined_pred_masks,
pred_class_ids,
dataset_infer.class_names,
pred_scores,
title='Result with edgelist',
save_path=visual_seg_edg_path,
fix_color=True)
