def evaluate_model(model, config):
"""
Evaluate the loaded model on the target dataset
:param model: the architecture model, with loaded weights
:param config: the configuration class for this dataset
"""
# Automatically discriminate the dataset according to the config file
if isinstance(config, configurations.TabletopConfigInference):
# Load the validation dataset
dataset_val = datasets.TabletopDataset()
elif isinstance(config, configurations.YCBVideoConfigInference):
dataset_val = datasets.YCBVideoDataset()
dataset_val.load_dataset(args.dataset, "val")
dataset_val.prepare()
# Compute COCO-Style mAP @ IoU=0.5-0.95 in 0.05 increments
# Running on all images
#image_ids = np.random.choice(dataset_val.image_ids, 200)
image_ids = dataset_val.image_ids
APs = []
AP50s = []
AP75s = []
image_batch_vector = []
image_batch_eval_data = []
img_batch_count = 0
import time
t_inference = 0
class image_eval_data:
def __init__(self, image_id, gt_class_id, gt_bbox, gt_mask):
self.IMAGE_ID = image_id
self.GT_CLASS_ID = gt_class_id
self.GT_BBOX = gt_bbox
self.GT_MASK = gt_mask
self.DETECTION_RESULTS = None
print("Evaluating model...")
progbar = Progbar(target=len(image_ids))
for idx, image_id in enumerate(image_ids):
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, config,
image_id, use_mini_mask=False)
# Compose a vector of images and data
image_batch_vector.append(image)
image_batch_eval_data.append(
image_eval_data(image_id, gt_class_id, gt_bbox, gt_mask))
img_batch_count += 1
# If a batch is ready, go on to detection
# The last few images in the dataset will not be used if batch size > 1
if img_batch_count < config.BATCH_SIZE:
continue
# Run object detection
t_start = time.time()
results = model.detect(image_batch_vector, verbose=0)
t_inference += (time.time() - t_start)
assert len(image_batch_eval_data) == len(results)
for eval_data, detection_results in zip(image_batch_eval_data,
results):
eval_data.DETECTION_RESULTS = detection_results
# Compute mAP at different IoU (as msCOCO mAP is computed)
AP = utils.compute_ap_range(
eval_data.GT_BBOX,
eval_data.GT_CLASS_ID,
eval_data.GT_MASK,
eval_data.DETECTION_RESULTS["rois"],
eval_data.DETECTION_RESULTS["class_ids"],
eval_data.DETECTION_RESULTS["scores"],
eval_data.DETECTION_RESULTS['masks'],
verbose=0)
AP50, _, _, _ = utils.compute_ap(
eval_data.GT_BBOX,
eval_data.GT_CLASS_ID,
eval_data.GT_MASK,
eval_data.DETECTION_RESULTS["rois"],
eval_data.DETECTION_RESULTS["class_ids"],
eval_data.DETECTION_RESULTS["scores"],
eval_data.DETECTION_RESULTS['masks'],
iou_threshold=0.5)
AP75, _, _, _ = utils.compute_ap(
eval_data.GT_BBOX,
eval_data.GT_CLASS_ID,
eval_data.GT_MASK,
eval_data.DETECTION_RESULTS["rois"],
eval_data.DETECTION_RESULTS["class_ids"],
eval_data.DETECTION_RESULTS["scores"],
eval_data.DETECTION_RESULTS['masks'],
iou_threshold=0.75)
APs.append(AP)
AP50s.append(AP50)
AP75s.append(AP75)
# Reset the batch info
image_batch_vector = []
image_batch_eval_data = []
img_batch_count = 0
progbar.update(idx + 1)
print("\nmAP[0.5::0.05::0.95]: ", np.mean(APs))
print("mAP[0.5]: ", np.mean(AP50s))
print("mAP[0.75]: ", np.mean(AP75s))
print("Inference time for", len(image_ids), "images: ", t_inference,
"s \tAverage FPS: ",
len(image_ids) / t_inference)
return APs
model.load_weights(weights_path, by_name=True)
folder = '/home/nel/Code/NEL_LAB/Mask_RCNN/result/'+os.path.split(os.path.split(weights_path)[0])[-1]+'/'+mode+'/'
try:
os.makedirs(folder)
print('make folder')
except:
print('already exist')
# %% Run Detection
run_single_detection = False
if run_single_detection:
image_id = random.choice(dataset.image_ids)
image_id = 0
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id,
dataset.image_reference(image_id)))
# Run object detection
results = model.detect([image], verbose=1)
# Display results
ax = get_ax(1)
r = results[0]
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
dataset.class_names, r['scores'], ax=ax,
title="Predictions")
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
def recall(model, class_names):
class_dict = {}
label_dict = ['background']
if args.label:
label_file = open(args.label)
label_lines = label_file.readlines()
label_id = 1
for label_line in label_lines:
label_line = label_line.replace('\n', '')
class_dict[label_line] = label_id
label_dict.append(label_line)
label_id = label_id + 1
# Validation dataset
dataset_val = MyDataset()
dataset_val.load_my(args.dataset, "val", class_dict)
dataset_val.prepare()
pre_correct_dict = {}
pre_total_dict = {}
pre_iou_dict = {}
pre_scores_dict = {}
gt_total_dict = {}
for i in range(1, len(class_dict) + 1):
pre_correct_dict[i] = 0
pre_total_dict[i] = 0
pre_iou_dict[i] = 0.0
pre_scores_dict[i] = 0.0
gt_total_dict[i] = 0
backbone_shapes = modellib.compute_backbone_shapes(config, [768, 1280, 3])
anchor_boxes = utils.generate_pyramid_anchors(config.RPN_ANCHOR_SCALES,
config.RPN_ANCHOR_RATIOS,
backbone_shapes,
config.BACKBONE_STRIDES,
config.RPN_ANCHOR_STRIDE)
#utils.generate_anchors(300, config.RPN_ANCHOR_RATIOS, [40,40], 32, config.RPN_ANCHOR_STRIDE)
#print(anchor_boxes)
rois = []
obj_groups = []
# {image_file, [gt_class_id], [gt_box, (y1,x1,y2,x2)], [gt_bbox_area], [gt_wh_ratio], [gt_mask_area], [gt_mask_ratio], [gt_size], }
for image_id in dataset_val.image_ids:
image, image_meta, gt_class_id, gt_box, gt_mask = modellib.load_image_gt(
dataset_val, config, image_id, use_mini_mask=False)
#print(image.shape)
gt_detects = {}
gt_detects['image'] = dataset_val.image_reference(image_id)
gt_detects['gt_class_id'] = gt_class_id
gt_detects['gt_bbox'] = gt_box
gt_detects['gt_bbox_area'] = []
gt_detects['gt_wh_ratio'] = []
gt_detects['gt_mask_area'] = []
gt_detects['gt_mask_ratio'] = []
gt_detects['gt_size'] = []
for i in range(0, len(gt_class_id)):
gt_total_dict[gt_class_id[i]] = gt_total_dict[gt_class_id[i]] + 1
wh_ratio, box_size, box_area, square_box = toSquareBox(gt_box[i])
mask_area = np.sum(gt_mask[:, :, i] == True)
mask_ratio = mask_area / box_area
gt_detects['gt_bbox_area'].append(box_area)
gt_detects['gt_wh_ratio'].append(wh_ratio)
gt_detects['gt_mask_area'].append(mask_area)
gt_detects['gt_mask_ratio'].append(mask_ratio)
gt_detects['gt_size'].append(box_size)
molded_image = modellib.mold_image(image, config)
#print(molded_image.shape)
# Anchors
"""
anchors = model.get_anchors(molded_image.shape)
# Duplicate across the batch dimension because Keras requires it
# TODO: can this be optimized to avoid duplicating the anchors?
anchors = np.broadcast_to(anchors, (config.BATCH_SIZE,) + anchors.shape)
print(anchors)
# Run object detection
detections, mrcnn_class, mrcnn_bbox, mrcnn_mask, rpn_rois, rpn_class, rpn_bbox =\
model.keras_model.predict([np.expand_dims(molded_image, 0), np.expand_dims(image_meta, 0), anchors], verbose=0)
print(detections[0])
print(mrcnn_class[0])
print(rpn_class[0])
"""
#skimage.io.imsave("test.jpg", image)
start_time = time.time()
results = model.detect_molded(np.expand_dims(molded_image, 0),
np.expand_dims(image_meta, 0),
verbose=0)
end_time = time.time()
#print("Time: %s" % str(end_time - start_time))
#print(results)
r = results[0]
pre_class_ids = r['class_ids']
for i in range(0, len(pre_class_ids)):
pre_total_dict[
pre_class_ids[i]] = pre_total_dict[pre_class_ids[i]] + 1
pre_scores = r['scores']
#print(r['rois'])
for roi in r['rois']:
whr, bsize, _, _ = toSquareBox(roi)
rois.append([bsize, whr])
#print(gt_detects['gt_size'])
#overlaps = utils.compute_iou(roi, gt_detects['gt_bbox'], roi_area, gt_detects['gt_bbox_area'])
#print(overlaps)
gt_match, pred_match, overlap = display_differences(
image,
gt_box,
gt_class_id,
gt_mask,
r['rois'],
pre_class_ids,
pre_scores,
r['masks'],
class_names,
title="",
ax=None,
show_mask=True,
show_box=True,
iou_threshold=0.1,
score_threshold=0.1)
gt_detects['rois'] = r['rois']
gt_detects['gt_match'] = gt_match
gt_detects['pred_match'] = pred_match
#print(gt_match)
"""
visualize.display_differences(image,
gt_box, gt_class_id, gt_mask,
r['rois'], pre_class_ids, pre_scores, r['masks'],
class_names, title="", ax=None,
show_mask=True, show_box=True,
iou_threshold=0.1, score_threshold=0.1)
"""
for i in range(0, len(pred_match)):
if pred_match[i] > -1.0:
#print(r['rois'][i])
pre_correct_dict[
pre_class_ids[i]] = pre_correct_dict[pre_class_ids[i]] + 1
pre_iou_dict[pre_class_ids[i]] = pre_iou_dict[
pre_class_ids[i]] + overlap[i, int(pred_match[i])]
pre_scores_dict[pre_class_ids[i]] = pre_scores_dict[
pre_class_ids[i]] + pre_scores[i]
obj_groups.append(gt_detects)
#print(rois)
print("图片,类别,标注框,标注宽高比,标注尺寸,检测框,检测宽高比,检测尺寸,最大IOU")
for det in obj_groups:
for i in range(0, len(det['gt_class_id'])):
overlaped = utils.compute_overlaps(
anchor_boxes, np.reshape(det['gt_bbox'][i], (1, 4)))
omax = max(overlaped)
#if det['gt_size'][i] > 150 and det['gt_size'][i] < 367:
if omax[0] > 0.0:
print(det['image'], end='')
print(",",
label_dict[det['gt_class_id'][i]],
",",
det['gt_bbox'][i],
",",
det['gt_wh_ratio'][i],
",",
det['gt_size'][i],
end="")
if det['gt_match'][i] > -1.0:
idx = int(det['gt_match'][i])
#print(idx, det['rois'])
whr, bsize, _, _ = toSquareBox(det['rois'][idx])
print(",", det['rois'][idx], ",", whr, ",", bsize, ",",
omax[0])
else:
print(",", 0, ",", 0, ",", 0, ",", omax[0])
tol_pre_correct_dict = 0
tol_pre_total_dict = 0
tol_pre_iou_dict = 0
tol_pre_scores_dict = 0
tol_gt_total_dict = 0
lines = []
tile_line = 'Type,Number,Correct,Proposals,Total,Rps/img,Avg IOU,Avg score,Recall,Precision\n'
lines.append(tile_line)
for key in class_dict:
tol_pre_correct_dict = tol_pre_correct_dict + pre_correct_dict[
class_dict[key]]
tol_pre_total_dict = pre_total_dict[
class_dict[key]] + tol_pre_total_dict
tol_pre_iou_dict = pre_iou_dict[class_dict[key]] + tol_pre_iou_dict
tol_pre_scores_dict = pre_scores_dict[
class_dict[key]] + tol_pre_scores_dict
tol_gt_total_dict = gt_total_dict[class_dict[key]] + tol_gt_total_dict
type_rps_img = pre_total_dict[class_dict[key]] / len(
dataset_val.image_ids)
if pre_correct_dict[class_dict[key]] > 0:
type_avg_iou = pre_iou_dict[class_dict[key]] / pre_correct_dict[
class_dict[key]]
type_avg_score = pre_scores_dict[
class_dict[key]] / pre_correct_dict[class_dict[key]]
else:
type_avg_iou = 0
type_avg_score = 0
if gt_total_dict[class_dict[key]] > 0:
type_recall = pre_total_dict[class_dict[key]] / gt_total_dict[
class_dict[key]]
else:
type_recall = 0
if pre_total_dict[class_dict[key]] > 0:
type_precision = pre_correct_dict[
class_dict[key]] / pre_total_dict[class_dict[key]]
else:
type_precision = 0
line = '{:s},{:d},{:d},{:d},{:d},{:.2f},{:.2f}%,{:.2f},{:.2f}%,{:.2f}%\n'.format(
key, len(dataset_val.image_ids), pre_correct_dict[class_dict[key]],
pre_total_dict[class_dict[key]], gt_total_dict[class_dict[key]],
type_rps_img, type_avg_iou * 100, type_avg_score,
type_recall * 100, type_precision * 100)
lines.append(line)
print(line)
tol_rps_img = tol_pre_total_dict / len(dataset_val.image_ids)
if tol_pre_correct_dict > 0:
tol_avg_iou = tol_pre_iou_dict / tol_pre_correct_dict
tol_avg_score = tol_pre_scores_dict / tol_pre_correct_dict
else:
tol_avg_iou = 0
tol_avg_score = 0
if tol_gt_total_dict > 0:
tol_recall = tol_pre_total_dict / tol_gt_total_dict
else:
tol_recall = 0
if tol_pre_total_dict > 0:
tol_precision = tol_pre_correct_dict / tol_pre_total_dict
else:
tol_precision = 0
totle_line = '{:s},{:d},{:d},{:d},{:d},{:.2f},{:.2f}%,{:.2f},{:.2f}%,{:.2f}%\n'.format(
'Total', len(dataset_val.image_ids), tol_pre_correct_dict,
tol_pre_total_dict, tol_gt_total_dict, type_rps_img, tol_avg_iou * 100,
tol_avg_score, tol_recall * 100, tol_precision * 100)
print(totle_line)
lines.append(totle_line)
result_file_name = "result_{:%Y%m%dT%H%M%S}.csv".format(datetime.now())
result_file = open(result_file_name, 'w+')
result_file.writelines(lines)
result_file.close()
def display_output(model_path, fold, backbone):
with open('config.json', 'r') as f:
cfg = json.load(f)
TRAIN_PATH = cfg['TRAIN_PATH']
ORIG_SZ = cfg['ORIG_SZ']
inference_config = InferenceConfig()
inference_config.BACKBONE = backbone
# Recreate the model in inference mode
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir=os.path.dirname(model_path))
# Load trained weights (fill in path to trained weights here)
assert model_path != "", "Provide path to trained weights"
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
df_val = pd.read_csv("folds/cv5val{}.csv".format(fold))
val_fns = df_val[df_val.EncodedPixels.notnull()].ImageId.unique().tolist()
val_fns = [f for f in val_fns if val_fns not in exclude_list]
dataset_val = DetectorDataset(val_fns, df_val, TRAIN_PATH, ORIG_SZ,
ORIG_SZ)
dataset_val.prepare()
dataset = dataset_val
fig, axes = plt.subplots(2, 2, figsize=(10, 40))
fig.tight_layout()
for i in range(2):
image_id = random.choice(dataset.image_ids)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
# print(original_image.shape)
visualize.display_instances(
original_image,
gt_bbox,
gt_mask,
gt_class_id,
dataset.class_names,
colors=get_colors_for_class_ids(gt_class_id),
ax=axes[i, 0])
# print(original_image.shape, original_image.dtype, original_image.max(), original_image.min())
# (384, 384, 3) uint8 254 3
results = model.detect([original_image]) #, verbose=1)
r = results[0]
# if len(r['scores']) > 0:
# print(r['rois'].shape,r['rois'].dtype, r['rois'].max(), r['rois'].min())
# print(r['masks'].shape, r['masks'].dtype, r['masks'].max(), r['masks'].min())
# print(r['class_ids'].shape, r['class_ids'].dtype, r['class_ids'].max(), r['class_ids'].min())
# print(r['scores'].shape, r['scores'].dtype, r['scores'].max(), r['scores'].min())
visualize.display_instances(original_image,
r['rois'],
r['masks'],
r['class_ids'],
dataset.class_names,
r['scores'],
colors=get_colors_for_class_ids(
r['class_ids']),
ax=axes[i, 1])
axes[0, 0].set_title('Ground True')
axes[0, 1].set_title('Preditions')
DATA_SAVE_DIR = os.path.join(args.savedir, 'Eval_on_test')
if not os.path.exists(DATA_SAVE_DIR):
os.makedirs(DATA_SAVE_DIR)
atol = 20 #15
IoUs = np.arange(0.5, 1, 0.05)
lis = []
mAPs = []
F1 = []
sub = 0
captions = list(range(100000))
image_ids = dataset_eval.image_ids
for image_id in image_ids:
begin = len(lis)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_eval, YeastInferenceConfig(),
image_id, use_mini_mask=False)
name = list(dataset_eval.image_info[image_id].values())[0]
visualize.display_instances(original_image,
gt_bbox,
gt_mask,
gt_class_id,
dataset_eval.class_names,
SAVE_DIR=os.path.join(DATA_SAVE_DIR, name))
##########################################################################################################################################
### Detect on random image of random Images
##########################################################################################################################################
results = model.detect([original_image], verbose=1)
r = results[0]
weights_path = model.find_last()
# Load weights
print("Loading weights ", weights_path)
model.load_weights(weights_path, by_name=True)
print(dataset.image_ids)
print(type(dataset.image_ids))
print(len(dataset.image_ids))
# change the starting file index depending on the training or validation dataset
sav_dir_str_index = 0
for i in range(len(dataset.image_ids)):
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, dataset.image_ids[i],
use_mini_mask=False)
info = dataset.image_info[dataset.image_ids[i]]
print(info['id'])
# Run object detection
results = model.detect([image], verbose=1)
ax = get_ax(1)
r = results[0]
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
dataset.class_names,
r['scores'],
#
# To improve training speed, we optimize masks by:
# * We store mask pixels that are inside the object bounding box,
# rather than a mask of the full image. Most objects are small compared to
# the image size, so we save space by not storing a lot of zeros around the object.
# * We resize the mask to a smaller size (e.g. 56x56).
# For objects that are larger than the selected size we lose a bit of accuracy.
# But most object annotations are not very accuracy to begin with,
# so this loss is negligable for most practical purposes.
# Thie size of the mini_mask can be set in the config class.
#
# To visualize the effect of mask resizing, and to verify the code correctness,
# we visualize some examples.
image_id = np.random.choice(dataset.image_ids, 1)[0]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
log("image", image)
log("image_meta", image_meta)
log("class_ids", class_ids)
log("bbox", bbox)
log("mask", mask)
display_images([image] +
[mask[:, :, i] for i in range(min(mask.shape[-1], 7))])
visualize.display_instances(image, bbox, mask, class_ids, dataset.class_names)
# Add augmentation and mask resizing.
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, augment=True, use_mini_mask=True)
r = results[0]
visualize.display_instances(original_image, r['rois'], r['masks'], r['class_ids'],
dataset_val.class_names, r['scores'])
# Compute mAP
#APs_05: IoU = 0.5
#APs_all: IoU from 0.5-0.95 with increments of 0.05
image_ids = np.random.choice(dataset_val.image_ids, 5)
APs_05 = []
APs_all = []
for i, image_id in enumerate(image_ids):
# Load images and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, modelI,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, inference_config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
AP_05, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs_05.append(AP_05)
AP_all = \
utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
APs_all.append(AP_all)
print("image " + str(i) + ": AP_05 = " + str(AP_05) + ", AP_all = " + str(AP_all))
def watershed(self, image_id):
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
ret, thresh = cv2.threshold(gray, 0, 255,
cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
#model.load_weights(WEIGHT_DIR, by_name=True, exclude=[
# "mrcnn_class_logits", "mrcnn_bbox_fc",
# "mrcnn_bbox", "mrcnn_mask"])
########################################################
##################################################################################### results wont predict with this code
weights_path = model.find_last()
########## Load weights
print("Loading weights ", weights_path)
model.load_weights(weights_path, by_name=True)
########################################################################################
print("Validation image")
image_id = random.choice(dataset_val.image_ids)
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, config, image_id, use_mini_mask=False)
info = dataset_val.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id,
dataset_val.image_reference(image_id)))
# Run object detection
results = model.detect([image], verbose=1)
# Display results
ax = get_ax(1)
r = results[0]
visualize.display_instances(image,
r['rois'],
r['masks'],
r['class_ids'],
dataset_val.class_names,
# Create output directory
if not os.path.exists(base): os.makedirs(base)
# Load model and weights
model = modellib.MaskRCNN(mode="inference",
config=config,
model_dir=config.MODEL_DIR)
model.load_weights(weights, by_name=True)
print('Loaded model weights ' + weights.split('/')[-1])
# Deploy in batches
for id_ix in tqdm(range(0, len(dataset.image_ids), config.BATCH_SIZE)):
ids = [
dataset.image_ids[i] for i in range(id_ix, id_ix + config.BATCH_SIZE)
]
imgs = [modellib.load_image_gt(dataset, config, i)[0] for i in ids]
results = model.detect(imgs, verbose=0)
for rix, r in enumerate(results):
img_name = join(base, dataset.image_reference(ids[rix]))
if len(sys.argv) > 3:
if sys.argv[3] == 'p':
plot_instances(imgs[rix], r['rois'], r['masks'],
r['class_ids'], img_name[:-4] + '_plt.png')
else: # Save instance mask
ints = np.random.choice(np.arange(1, 256),
size=r['masks'].shape[-1],
replace=False)
mask = np.zeros((256, 256))
for inst_ix in range(r['masks'].shape[-1]):
instance = r['masks'][:, :, inst_ix]
def compute_batch_ap(dataset, image_ids, verbose=1):
APs_general = []
APs_ring = []
APs_crack = []
# loop throgh all val dataset
for image_id in image_ids:
# Load image ground truths
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
print(image_id)
# Run object detection
results = model.detect([image], verbose=0)
# Compute AP over range 0.5 to 0.95
r = results[0]
ap = utils.compute_ap_range(gt_bbox,
gt_class_id,
gt_mask,
r['rois'],
r['class_ids'],
r['scores'],
r['masks'],
verbose=0)
APs_general.append(ap)
#print('APs_general', APs_general)
# get AP values for ring and crack separately
AP_loop = []
for i in [1, 2]:
#print("LOOP START", i)
if gt_mask[:, :, gt_class_id == i].shape[-1] > 0:
ap = utils.compute_ap_range(
gt_bbox[gt_class_id == i],
gt_class_id[gt_class_id == i],
gt_mask[:, :, gt_class_id == i],
r['rois'][r['class_ids'] == i],
r['class_ids'][r['class_ids'] == i],
r['scores'][r['class_ids'] == i],
r['masks'][:, :, r['class_ids'] == i],
verbose=0)
AP_loop.append(ap)
#print(ap)
else:
ap = np.nan
AP_loop.append(ap)
#print(ap)
#print('AP_loop', AP_loop)
APs_ring.append(AP_loop[0])
APs_crack.append(AP_loop[1])
if verbose:
info = dataset.image_info[image_id]
meta = modellib.parse_image_meta(image_meta[np.newaxis, ...])
#print("{:3} {} AP: {:.2f}".format(
#meta["image_id"][0], meta["original_image_shape"][0], ap))
mAP_general = np.nanmean(APs_general)
mAP_ring = np.nanmean(APs_ring)
mAP_crack = np.nanmean(APs_crack)
return mAP_general, mAP_ring, mAP_crack #this outputs mAPs per image. May be usefull to keep it like that
def compute_batch_ap(dataset, image_ids, verbose=1):
"""
# Load validation dataset if you need to use this function.
dataset = slum.slumDataset()
dataset.load_slum(folder_path,fol)
"""
APs = []
IOUs = []
for image_id in image_ids:
# Load image
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config,
image_id, use_mini_mask=False)
# Run object detection
results = model.detect_molded(image[np.newaxis],
image_meta[np.newaxis],
verbose=0)
# Compute AP over range 0.5 to 0.95
r = results[0]
#merge_masks.
gt_merge_mask = np.zeros((gt_mask.shape[:2]))
for i in range(gt_mask.shape[2]):
gt_merge_mask = np.logical_or(gt_merge_mask, gt_mask[:, :, i])
pred_merge_mask = np.zeros((r['masks'].shape[:2]))
for i in range(r['masks'].shape[2]):
pred_merge_mask = np.logical_or(pred_merge_mask, r['masks'][:, :,
i])
pred_merge_mask = np.expand_dims(pred_merge_mask, 2)
#print(pred_merge_mask.shape)
pred_merge_mask, wind, scale, pad, crop = utils.resize_image(
pred_merge_mask, 1024, 1024)
#print(pred_merge_mask.shape,gt_merge_mask.shape)
iou = jaccard_similarity_score(np.squeeze(pred_merge_mask),
gt_merge_mask)
#mAP at 50
print("mAP at 50")
ap = utils.compute_ap_range(gt_bbox,
gt_class_id,
gt_mask,
r['rois'],
r['class_ids'],
r['scores'],
r['masks'],
np.arange(0.5, 1.0),
verbose=0)
#Make sure ap doesnt go above 1 !
if ap > 1.0:
ap = 1.0
APs.append(ap)
IOUs.append(iou)
if verbose:
info = dataset.image_info[image_id]
meta = modellib.parse_image_meta(image_meta[np.newaxis, ...])
print("{:3} {} AP: {:.2f} Image_id: {}, IOU: {}".format(
meta["image_id"][0], meta["original_image_shape"][0], ap,
image_id, iou))
return APs, IOUs
def score_model(model_path, model, dataset, config):
model.load_weights(model_path, by_name=True)
img_num = 0
tile_num = 0
infos = []
size_and_infos_per_img = []
for info in dataset.image_info:
nums = [int(s) for s in re.findall(r'\d+', info['mask_path'])]
if nums[0] != img_num:
if tile_num == 98:
size = (2028, 2704)
elif tile_num == 76:
size = (1538, 2704)
else:
raise ValueError
size_and_infos_per_img.append((size, infos.copy()))
img_num = nums[0]
infos.clear()
infos.append(info)
tile_num = nums[1]
size_and_infos_per_img.append((size, infos.copy()))
aps = []
for size, infos in size_and_infos_per_img:
inst_idx = 1
imgs_list = []
gt_masks_list = []
masks_list = []
for info in infos:
# print(info)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, info['id'], use_mini_mask=False)
imgs_list.append((original_image, ))
gt_arr = np.load(info['mask_path'])['arr_0']
gt_arr = np.split(gt_arr, gt_arr.shape[-1], axis=-1)
gt_masks_list.append(gt_arr)
results = model.detect([original_image], verbose=0)
r = results[0]
masks = r['masks']
if masks.shape[-1] != 0:
scores = r['scores']
instances = [
np.squeeze(x)
for x in np.split(masks, masks.shape[-1], axis=-1)
]
instances = [x for x, s in zip(instances, scores) if s > 0.9]
tmp = []
for inst in instances:
tmp.append(np.int32(inst) * inst_idx)
inst_idx += 1
masks_list.append(tmp)
else:
masks_list.append([np.zeros(original_image.shape[0:2])])
if len(masks_list) == 0:
print('No instances detected.')
stitched_img, stitched_gt_masks, stitched_masks = image_and_labelling(
imgs_list, masks_list, gt_masks_list, size)
if stitched_masks is None:
aps.append(0.0)
print(0.0)
else:
sgm_filter = np.uint8(stitched_gt_masks > 0)
sm_filter = np.uint8(stitched_masks > 0)
futil.filter_components([sgm_filter, sm_filter],
mode='MIN',
size=100)
stitched_gt_masks = np.int32(stitched_gt_masks * sgm_filter)
stitched_masks = np.int32(stitched_masks * sm_filter)
mean_ap = ap.ap_dsb2018([stitched_masks], [stitched_gt_masks],
compute_mean=True)
aps.append(mean_ap[-1])
# print(mean_ap)
# futil.plot_imgs([stitched_img, stitched_gt_masks, stitched_masks])
# blend_pred = futil.blend_images(stitched_img, futil.mask_to_color_image(np.uint8(stitched_masks > 0)),
# factor=0.3)
# blend_gt = futil.blend_images(stitched_img, futil.mask_to_color_image(np.uint8(stitched_gt_masks > 0)),
# factor=0.3)
# futil.plot_imgs([blend_pred, blend_gt])
return sum(aps) / len(aps)
def evaluation(model, dataset_dir, subset):
# Create directory
if not os.path.exists(RESULTS_DIR):
os.makedirs(RESULTS_DIR)
submit_dir = "submit_{:%Y%m%dT%H%M%S}".format(datetime.datetime.now())
submit_dir = os.path.join(RESULTS_DIR, submit_dir)
os.makedirs(submit_dir)
# Read dataset
dataset_val = vehDetectionDataset()
dataset_val.load_evaluation(dataset_dir, subset)
dataset_val.prepare()
numImages_eval = dataset_val.image_ids.size
# Prepare for saving as matlab file
images_eval = []
mAP_all = []
mAP_all_range = []
precisions_all = []
recalls_all = []
overlaps_all = []
for image_id in range(0,numImages_eval):
print(image_id)
source_id = dataset_val.image_info[image_id]["id"]
print(source_id)
# Load image and ground truth data
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
modellib.load_image_gt(dataset_val, config,
image_id, use_mini_mask=False)
molded_images = np.expand_dims(modellib.mold_image(image, config), 0)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
# Compute AP
mAP, precisions, recalls, overlaps = \
utils.compute_ap(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'])
# Compute AP range (0.5:0.05:0.95)
mAP_range = \
utils.compute_ap_range(gt_bbox, gt_class_id, gt_mask,
r["rois"], r["class_ids"], r["scores"], r['masks'], iou_thresholds = None, verbose = 1)
# Append results from image
mAP_all.append(mAP)
mAP_all_range.append(mAP_range)
images_eval.append(source_id)
precisions_all.append(precisions)
recalls_all.append(recalls)
overlaps_all.append(overlaps)
# Save image with shape polygon around vehicles. Bbox and mask can be activated, s. below
visualize.display_instances(
image, r['rois'], r['masks'], r['class_ids'],
dataset_val.class_names, scores=None,# r['scores'],
show_bbox=False, show_mask=False,
colors=None,
figsize=(19.20,10.80)) # can also add title="Predictions"
plt.box(on=None) # plt.box(False)
plt.savefig(os.path.join(submit_dir, dataset_val.image_info[image_id]["id"]))
plt.close() # plt.clf()
print("Evaluation Process with Mask-RCNN Done. Files saved to ", submit_dir)
print("mAP: ", np.mean(mAP_all))
# FK: save to Matlab files
saveMatFileName = submit_dir + "\evaluation_maskrcnn_output.mat"
savemat(saveMatFileName,
{"mAP_all": mAP_all, "mAP_all_range": mAP_all_range, "images_eval": images_eval, "precisions_all": precisions_all, "recalls_all": recalls_all, "overlaps_all": overlaps_all})
#print('.....................', os.walk(MODEL_DIR))
#model_path = model.find_last()
# Load trained weights
#print("Loading weights from ", model_path)
weights_path = '/gluster/home/sdevaramani/Thesis/refactor/versions/binary_logs/for_10k_data/ycb20210109T1609/mask_rcnn_ycb_0093.h5'
model.load_weights(weights_path, by_name=True)
#for i in range(5):
image_id = random.choice(dataset_val.image_ids)
#image_id = 176
print('Image id..........', image_id)
original_image, _, gt_class_ids, gt_bbox, gt_r_mask, gt_g_mask, gt_b_mask = modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
#if 19 in gt_class_ids:
# image_id = im_id
#print(gt_bbox)
#print('gt ids....', gt_class_ids)
gt_data = {'rois': gt_bbox, 'class_ids':gt_class_ids, 'gt_r_masks': gt_r_mask, 'gt_g_masks': gt_g_mask, 'gt_b_masks': gt_b_mask}
cv2.imwrite('image.png', original_image)
results = model.detect([original_image], verbose=1)
r = results[0]
data = {'rois': r['rois'], 'class_ids': r['class_ids'], 'scores': r['scores'], 'r_masks': r['r_masks'], 'g_masks': r['g_masks'], 'b_masks': r['b_masks']}
#print('results ....', r['rois'])
#print(r['class_ids'])
"""Return a Matplotlib Axes array to be used in
all visualizations in the notebook. Provide a
central point to control graph sizes.
Change the default size attribute to control the size
of rendered images
"""
_, ax = plt.subplots(rows, cols, figsize=(size * cols, size * rows))
return ax
image_id = random.choice(valid_dataset.image_ids)
print(image_id)
print(valid_dataset.image_info[image_id])
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(valid_dataset, inference_config,
image_id, use_mini_mask=False)
if gt_mask.shape[-1] != 0:
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
image = visualize.display_instances(original_image,
gt_bbox,
gt_mask,
gt_class_id,
train_dataset.class_names,
figsize=(8, 8))
results = model_pred.detect([original_image], verbose=1)
def train(args_,
model,
config,
is_mlflow=False,
run_name=None,
overfit=False,
train_strategy='A'):
dataset_train = ApprantiDataset()
dataset_train.load_appranti(dataset_dir=args_.dataset,
subset="train",
year=args_.year,
class_ids=None,
config=config,
limit_number_images=(1 if overfit else None))
dataset_train.prepare()
dataset_val = ApprantiDataset()
dataset_val.load_appranti(dataset_dir=args_.dataset,
subset="val",
year=args_.year,
class_ids=None,
config=config)
dataset_val.prepare()
if args_.eyes_of_the_net is not None:
for dataset, name in zip((dataset_train, dataset_val),
('train', 'val')):
output_dir = args_.eyes_of_the_net + '/' + name + '/'
os.makedirs(output_dir, exist_ok=True)
for img_info in dataset.image_info:
source_id = str(img_info['id'])
image_id = dataset.image_from_source_map["coco.{}".format(
source_id)]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
an_image = visualize.display_instances(image,
bbox,
mask,
class_ids,
dataset.class_names,
show_bbox=False,
only_return_image=True)
cv2.imwrite(output_dir + str(image_id) + '.jpg', an_image)
config.STEPS_PER_EPOCH = 2 * (
int(len(dataset_train.image_ids) / config.IMAGES_PER_GPU) + 1)
assert config.STEPS_PER_EPOCH * config.IMAGES_PER_GPU >= len(
dataset_train.image_ids)
config.VALIDATION_STEPS = int(
len(dataset_val.image_ids) / config.IMAGES_PER_GPU) + 1
assert config.VALIDATION_STEPS * config.IMAGES_PER_GPU >= len(
dataset_val.image_ids)
custom_callbacks = [
# keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=7, verbose=1, mode='auto', min_delta=0.01, cooldown=0, min_lr=10e-7),
# ['val_loss', 'val_rpn_class_loss', 'val_rpn_bbox_loss', 'val_mrcnn_class_loss', 'val_mrcnn_bbox_loss', 'val_mrcnn_mask_loss', 'loss', 'rpn_class_loss', 'rpn_bbox_loss',
# 'mrcnn_class_loss', 'mrcnn_bbox_loss', 'mrcnn_mask_loss'])
# keras.callbacks.LambdaCallback(on_epoch_end=lambda epoch, logs: logger.debug(logs.keys())),
]
if is_mlflow:
mlflow.start_run(run_name=run_name)
mlflow.log_params({
'model_dir':
str(model.log_dir),
'n_train':
str(len(dataset_train.image_ids)),
'n_val':
str(len(dataset_val.image_ids)),
'os':
os.name,
'dataset':
args_.dataset,
'CUDA_VISIBLE_DEVICES':
os.environ["CUDA_VISIBLE_DEVICES"],
'class_names':
str(dataset_train.class_names),
'init_with_model':
str(args_.model)
})
mlflow.log_params(config.get_parameters())
def my_log_func(epoch, logs):
logger.debug(logs)
mlflow.log_metrics(logs)
mlflow.log_metrics({'epoch': float(epoch)})
custom_callbacks.append(
keras.callbacks.LambdaCallback(
on_epoch_end=lambda epoch, logs: my_log_func(epoch, logs)))
model_check_point = False if overfit else True
if is_mlflow:
mlflow.log_param('train_strategy', train_strategy)
if 'A' == train_strategy:
# Strategy A
logger.debug("Training network heads")
epoch = 40
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=epoch,
layers='heads',
augmentation=None,
custom_callbacks=custom_callbacks,
model_check_point=model_check_point)
logger.debug("Fine tune Resnet stage 4 and up")
epoch = 120
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=epoch,
layers='4+',
augmentation=None,
custom_callbacks=custom_callbacks,
model_check_point=model_check_point)
logger.debug("Fine tune all layers")
epoch = 160 + 250
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE / 10,
epochs=epoch,
layers='all',
augmentation=None,
custom_callbacks=custom_callbacks,
model_check_point=model_check_point)
if 'B' == train_strategy:
epoch = 350
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=epoch,
layers='all',
augmentation=None,
custom_callbacks=custom_callbacks,
model_check_point=model_check_point)
# Testing dataset
dataset = BDD100KDataset()
dataset.load_bdd100k(args.dataset_dir, "val", load_images_flag=False)
dataset.prepare()
# uuids = ["c1f8d9b3-81ee1c2d", "b2db41a2-721e0f4e", "b222c329-5dc8dbf7", "bb8e2033-6c418fc7", "c0625a26-cefa81e9",
# "b6d0b9d1-d643d86a", "c18feebb-3e10acea"]
uuids = ["c927d51b-92852659"]
# uuids = ["b1d0a191-06deb55d"]
ids = get_ids_from_uuids(dataset, uuids)
# ids = [random.choice(dataset.image_ids)]
# ids = [1536]
for image_id in ids:
image, _, gt_class_id, gt_bbox = modellib.load_image_gt(dataset, config, image_id)
info = dataset.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id,
dataset.image_reference(image_id)))
# Run object detection
results = model.detect([image], verbose=1, gpi_type=config.GPI_TYPE)
# Display results
ax = get_ax(1)
r = results[0]
# image = dataset.load_image(image_id)
visualize.save_instances(image, r['rois'], gt_bbox, r['class_ids'], gt_class_id, dataset.class_names,
r['scores'],
ax=ax, title="Predictions_{}".format(info["id"]),
path="{}/{}_{}.jpg".format(args.save_path, args.model.split('/')[-2], info["id"]),
show_mask=False)
def prediction_generator(model,
dataset,
augmentation=None,
image_ids=None,
config=None):
"""Generate predictions paired with ground truth information
Args:
model: Mask RCNN Model instance
dataset: Mask RCNN Dataset
augmentation: Optional imgaug augmentation object to be applied to each ground truth image as it is loaded;
when provided, augmentation will be applied to ground-truth images and masks and predictions will
be provided based on augmented version as well
image_ids: List of image ids within dataset to process; For example:
- To process all images, leave as None
- To process the same image with different augmentations, specify the same image_id N times
and set augmentation parameter
- To process a pre-selected subset, specify only those image ids
config: Mask RCNN configuration; if not provided the configuration already associated with the model
instance will be used
Returns:
A generator of Prediction instances (see inference.Prediction)
"""
if config is None:
config = model.config
if config.BATCH_SIZE != 1:
raise ValueError(
'Configuration batch size must = 1 for this inference method')
if image_ids is None:
image_ids = dataset.image_ids
for image_id in image_ids:
image_meta, gt_class_id, gt_class_names, gt_bbox, gt_mask = [None] * 5
try:
image, image_meta, gt_class_id, gt_bbox, gt_mask = \
mrcnn_model.load_image_gt(
dataset, config, image_id,
use_mini_mask=False, augmentation=augmentation
)
gt_class_names = np.array(
[dataset.class_names[i] for i in gt_class_id])
except FileNotFoundError:
# Catch this on failure to read annotation files and
# just load the original image instead
image = dataset.load_image(image_id)
detection = model.detect([image], verbose=0)[0]
if gt_class_id is not None and gt_class_id.ndim != 1:
raise ValueError(
'Expecting true class ids array to have ndim == 1 (shape = {})'
.format(gt_class_id.shape))
if gt_mask is not None and gt_mask.ndim != 3:
raise ValueError(
'Expecting true masks array to have ndim == 3 (shape = {})'.
format(gt_mask.shape))
if detection['class_ids'].ndim != 1:
raise ValueError(
'Expecting pred class ids array to have ndim == 1 (shape = {})'
.format(detection['class_ids'].shape))
if detection['masks'].ndim != 3:
raise ValueError(
'Expecting pred masks array to have ndim == 3 (shape = {})'.
format(detection['masks'].shape))
yield Prediction(image=image,
image_id=image_id,
image_info=dataset.image_reference(image_id),
pred_class_ids=detection['class_ids'],
pred_class_names=np.array([
dataset.class_names[i]
for i in detection['class_ids']
]),
pred_masks=detection['masks'],
pred_rois=detection['rois'],
pred_scores=detection['scores'],
true_class_ids=gt_class_id,
true_class_names=gt_class_names,
true_rois=gt_bbox,
true_masks=gt_mask)
def evaluate(model):
# Load validation dataset
dataset = FiguresDataset()
dataset.load_figures(args.validation_dataset, "val_annotations.json")
dataset.prepare()
gen = AnnotationsGenerator("Dataset validation results")
gen.add_categories(dataset.class_names)
for image_id in dataset.image_ids:
if config.ORIENTATION:
image, image_meta, gt_class_id, gt_bbox, gt_mask, gt_orientations = modellib.load_image_gt_or(
dataset,
config,
image_id,
use_mini_mask=False,
orientation=True)
else:
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(
dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(
info["source"], info["id"], image_id,
dataset.image_reference(image_id)))
# Run object detection
results = model.detect([image], config, verbose=1)
r = results[0]
# Save image as pred_x.png and save annotations in COCO format
gen.add_image(info['path'], image.shape[0], image.shape[1])
image_id = gen.get_image_id(info['path'])
filename = info["source"] + "_" + str(image_id)
if config.ORIENTATION:
show_results(image,
filename,
r['rois'],
r['masks'],
r['orientations'],
r['class_ids'],
r['scores'],
dataset.class_names,
gen,
image_id,
gt_bbox,
gt_orientations,
save_dir="../results/val",
mode=3)
else:
show_results(image,
filename,
r['rois'],
r['masks'],
None,
r['class_ids'],
r['scores'],
dataset.class_names,
gen,
image_id,
gt_bbox,
None,
save_dir="../results/val",
mode=0)
# Save the results in an annotation file following the COCO dataset structure
gen.save_json("../results/val" + "/evaluation_annotations.json",
pretty=True)
def detect(run_dir, inference_config, model, dataset, bin_mask_dir=False, overlap_thresh=0.5):
"""
Given a run directory, a MaskRCNN config object, a MaskRCNN model object,
and a Dataset object,
- Loads and processes ground-truth masks, saving them to a new directory
for annotation
- Makes predictions on images
- Saves prediction masks in a certain directory
- Saves other prediction info (scores, bboxes) in a separate directory
Returns paths to directories for prediction masks, prediction info, and
modified GT masks.
If bin_mask_dir is specified, then we will be checking predictions against
the "bin-vs-no bin" mask for the test case.
For each predicted instance, if less than overlap_thresh of the mask actually
consists of non-bin pixels, we will toss out the mask.
"""
# Create subdirectory for prediction masks
pred_dir = os.path.join(run_dir, 'pred_masks')
utils.mkdir_if_missing(pred_dir)
# Create subdirectory for prediction scores & bboxes
pred_info_dir = os.path.join(run_dir, 'pred_info')
utils.mkdir_if_missing(pred_info_dir)
# Create subdirectory for transformed GT segmasks
resized_segmask_dir = os.path.join(run_dir, 'modal_segmasks_processed')
utils.mkdir_if_missing(resized_segmask_dir)
# Feed images into model one by one. For each image, predict and save.
image_ids = dataset.image_ids
indices = dataset.indices
times = []
print('MAKING PREDICTIONS')
for image_id in tqdm(image_ids):
# Load image and ground truth data and resize for net
image, _, _, _, gt_mask =\
modellib.load_image_gt(dataset, inference_config, image_id,
use_mini_mask=False)
# Run object detection
results = model.detect([image], verbose=0)
r = results[0]
times.append(r['time'])
# If we choose to mask out bin pixels, load the bin masks and
# transform them properly.
# Then, delete the mask, score, class id, and bbox corresponding
# to each mask that is entirely bin pixels.
if bin_mask_dir:
name = 'image_{:06d}.png'.format(indices[image_id])
bin_mask = io.imread(os.path.join(bin_mask_dir, name))[:,:,np.newaxis]
bin_mask, _, _, _, _ = utilslib.resize_image(
bin_mask,
max_dim=inference_config.IMAGE_MAX_DIM,
min_dim=inference_config.IMAGE_MIN_DIM,
mode=inference_config.IMAGE_RESIZE_MODE
)
bin_mask = bin_mask.squeeze()
deleted_masks = [] # which segmasks are gonna be tossed?
num_detects = r['masks'].shape[2]
for k in range(num_detects):
# compute the area of the overlap.
inter = np.logical_and(bin_mask, r['masks'][:,:,k])
frac_overlap = np.sum(inter) / np.sum(r['masks'][:,:,k])
if frac_overlap <= overlap_thresh:
deleted_masks.append(k)
r['masks'] = [r['masks'][:,:,k] for k in range(num_detects) if k not in deleted_masks]
r['masks'] = np.stack(r['masks'], axis=2) if r['masks'] else np.array([])
r['rois'] = [r['rois'][k,:] for k in range(num_detects) if k not in deleted_masks]
r['rois'] = np.stack(r['rois'], axis=0) if r['rois'] else np.array([])
r['class_ids'] = np.array([r['class_ids'][k] for k in range(num_detects)
if k not in deleted_masks])
r['scores'] = np.array([r['scores'][k] for k in range(num_detects)
if k not in deleted_masks])
# Save copy of transformed GT segmasks to disk in preparation for annotations
mask_name = 'image_{:06d}'.format(image_id)
mask_path = os.path.join(resized_segmask_dir, mask_name)
# save the transpose so it's (n, h, w) instead of (h, w, n)
np.save(mask_path, gt_mask.transpose(2, 0, 1))
# Save masks
save_masks = np.stack([r['masks'][:,:,i] for i in range(r['masks'].shape[2])]) if np.any(r['masks']) else np.array([])
save_masks_path = os.path.join(pred_dir, 'image_{:06d}.npy'.format(image_id))
np.save(save_masks_path, save_masks)
# Save info
r_info = {
'rois': r['rois'],
'scores': r['scores'],
'class_ids': r['class_ids']
}
r_info_path = os.path.join(pred_info_dir, 'image_{:06d}.npy'.format(image_id))
np.save(r_info_path, r_info)
print('Took {} s'.format(sum(times)))
print('Saved prediction masks to:\t {}'.format(pred_dir))
print('Saved prediction info (bboxes, scores, classes) to:\t {}'.format(pred_info_dir))
print('Saved transformed GT segmasks to:\t {}'.format(resized_segmask_dir))
return pred_dir, pred_info_dir, resized_segmask_dir
print('\n --------------- Samples ---------------')
num_images = 4
# get random image
### image_ids = np.random.choice(dataset.image_ids, size=len(dataset.image_ids))
image_ids = np.random.choice(len(dataset.image_ids), size=num_images)
image_id = image_ids[0]
print("image_id:", image_id)
print("image_file:", dataset.image_reference(image_id))
# Load the image multiple times to show augmentations
limit = num_images
ax = get_ax(rows=int(np.sqrt(limit)), cols=int(np.sqrt(limit)))
for i in range(limit):
image_idx = image_ids[i]
image, image_meta, class_ids, bbox, mask = modellib.load_image_gt(
dataset, config, image_idx, use_mini_mask=False)
visualize.display_instances(image,
bbox,
mask,
class_ids,
dataset.class_names,
ax=ax[i // int(np.sqrt(limit)),
i % int(np.sqrt(limit))],
captions=captions[class_ids].tolist())
# log("molded_image", image)
# log("mask", mask)
# log("class_ids", class_ids)
### print("captions", np.array(dataset.class_names)[class_ids].tolist())
plt.savefig(os.getcwd() + save_to_folder +
"gt_affordance_labels/gt_affordance_labels_" +
np.str(idx_samples) + ".png",
def main(args):
parser = argparse.ArgumentParser(
description='Train Mask R-CNN to detect your Own Dataset.')
parser.add_argument('--device',
help='cpu or gpu device ex: /cpu:0 ex: /gpu:0')
parser.add_argument('--weights',
help="Path to folder contains weights .h5 file and num_classes.txt. Default: last trained weights")
parser.add_argument('--dataset',
help="Path to folder contains test images and annotations")
args = parser.parse_args()
config = own_dataset_train.OwnDatasetConfig()
model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR,
config=config)
if args.weights:
weights_path = args.weights
else:
try:
weights_path = model.find_last()
except FileNotFoundError as fne:
print('FileNotFoundError: [Errno 2] Could not find weight files in directory')
print('! Please please run a successful train OR give --weights path \nOR delete last old unsuccessful log folders from mrcnn/logs(reccomended)')
sys.exit()
if args.device:
DEVICE = args.device
else:
DEVICE = "/cpu:0" # /cpu:0 or /gpu:0
if args.dataset:
DATA_DIR = args.dataset
else:
DATA_DIR = os.path.abspath(os.path.join(ROOT_DIR,'../','dataset'))
num_classes_txt_path = os.path.join(weights_path,'../', 'num_classes.txt')
try:
f = open(num_classes_txt_path, "r")
num_classes_from_dataset = int(f.read()) +1 # +1 is for Back Ground Class
f.close()
config.NUM_CLASSES = num_classes_from_dataset
config.refresh()
config.display()
except Exception as ex:
print('Please run build_dataset to create num_classes.txt')
print('ex:', ex)
sys.exit()
dataset = own_dataset_train.OwnDataset()
dataset.load_dataset(DATA_DIR, "test")
# Must call before using the dataset
dataset.prepare()
print("Images: {}\nClasses: {}".format(len(dataset.image_ids), dataset.class_names))
# Device to load the neural network on.
# Useful if you're training a model on the same
# machine, in which case use CPU and leave the
# GPU for training.
# Inspect the model in training or inference modes
# values: 'inference' or 'training'
# TODO: code for 'training' test mode not ready yet
TEST_MODE = "inference"
# Create model in inference mode
with tf.device(DEVICE):
model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR,
config=config)
if args.weights:
weights_path = args.weights
else:
weights_path = model.find_last()
# Load weights
print("Loading weights ", weights_path)
model.load_weights(weights_path, by_name=True)
for image_id in dataset.image_ids:
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
info = dataset.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(info["source"], info["id"], image_id,
dataset.image_reference(image_id)))
# Run object detection
results = model.detect([image], verbose=1)
# Display results
ax = get_ax(1)
r = results[0]
visualize.display_instances(image, r['rois'], r['masks'], r['class_ids'],
dataset.class_names, r['scores'], ax=ax,
title="Predictions")
fig = plt.gcf()
save_path = os.path.abspath(os.path.join(ROOT_DIR,'outputs', (str(image_id) + '.png')))
fig.savefig(save_path)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
# Get path to saved weights
# Either set a specific path or find last trained weights
# model_path = os.path.join(ROOT_DIR, ".h5 file name here")
model_path = model.find_last()
print(model_path)
# Load trained weights
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
# Test on 2 random images
for i in range(10):
image_id = random.choice(dataset_val.image_ids)
original_image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_val, inference_config,
image_id, use_mini_mask=False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
# visualize.display_instances(original_image, gt_bbox, gt_mask, gt_class_id,
# dataset_train.class_names, figsize=(8, 8))
results = model.detect([original_image], verbose=1)
r = results[0]
visualize.display_instances(original_image,
r['rois'],
# Either set a specific path or find last trained weights
# model_path = os.path.join(ROOT_DIR, ".h5 file name here")
model_path = model.find_last()
# Load trained weights
print("Loading weights from ", model_path)
model.load_weights(model_path, by_name=True)
# In[34]:
# Test on a random image
# image_id = random.choice(eddy_val.image_ids)
image_id = 60
original_image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(eddy_val, inference_config,
image_id, use_mini_mask=False)
log("original_image", original_image)
log("image_meta", image_meta)
log("gt_class_id", gt_class_id)
log("gt_bbox", gt_bbox)
log("gt_mask", gt_mask)
print(original_image.shape)
print(gt_bbox.shape)
print(gt_mask.shape)
print(gt_class_id.shape)
original_image = eddy_val.load_orig_image(image_id)
original_image1 = ssh_convert_to_image(original_image)
#visualize.display_instances(original_image1, gt_bbox, gt_mask, gt_class_id,
#---------------------------------------------------------------------
# Create model in inference mode
MODEL_DIR = "./logs_VF"
with tf.device(DEVICE):
model = modellib.MaskRCNN(mode="inference",
model_dir=MODEL_DIR,
config=config)
# SELECT IMAGE
#-----------------------------------------------------------------------------------
#image_id = random.choice(dataset_train.image_ids)
image_id = 18
image, image_meta, gt_class_id, gt_bbox, gt_mask =\
modellib.load_image_gt(dataset_train, config, image_id, use_mini_mask=False)
info = dataset_train.image_info[image_id]
print("image ID: {}.{} ({}) {}".format(
info["source"], info["id"], image_id,
dataset_train.image_reference(image_id)))
# CREATE SUBPLOTS
ax = get_ax()
# GROUND TRUTH
#-----------------------------------------------------------------------------------
visualize.display_instances(image,
gt_bbox,
gt_mask,
def generatecsv(csvpath, datasetfolder):
# Load validation dataset
dataset = pneumothorax.SiimDataset()
dataset.load_siim(SIIM_DIR, datasetfolder)
# Must call before using the dataset
dataset.prepare()
print("Images: {}\nClasses: {}".format(len(dataset.image_ids), dataset.class_names))
# ## Load Model
# In[8]:
# Create model in inference mode
with tf.device(DEVICE):
model = modellib.MaskRCNN(mode="inference", model_dir=MODEL_DIR,
config=config)
# In[9]:
# Set path to balloon weights file
# Download file from the Releases page and set its path
# https://github.com/matterport/Mask_RCNN/releases
weights_path = "/home/sa-279/Mask_RCNN/logs/siim20190813T1933/mask_rcnn_siim_0029.h5"
# Or, load the last model you trained
#weights_path = model.find_last()
# Load weights
print("Loading weights ", weights_path)
model.load_weights(weights_path, by_name=True)
# ## TEST
# In[26]:
test_images = dataset.image_ids
#print(len(test_images))
test_set = np.empty((0,2))
total_images = len(test_images)
for row, image_id in enumerate(test_images):
info = dataset.image_info[image_id]
img_id = info["id"] # image id as in csv file
print("processing {} :image {} of {}".format(img_id, row+1, total_images))
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset, config, image_id, use_mini_mask=False)
#print(image.shape)
result = model.detect([image], verbose=0)
cols = result[0]
mask = cols['masks']
mask = np.array(mask)
if (np.any(mask)):
mask = mask.astype(np.uint64)
mask = mask * 255
mask = np.squeeze(mask, axis = -1)
width = mask.shape[1]
height = mask.shape[0]
mask = mask.T
rle = mask2rle(mask,width, height)
#print(rle)
else:
rle = -1
test_set = np.append(test_set,[[img_id, rle]], axis = 0)
#print(len(test_set))
# Run object detection
#print(test_set.shape)
csvfile = os.path.join(SIIM_DIR, csvpath)
np.savetxt(csvfile,test_set, delimiter=',', fmt='%s')
return image_ids
image_ids = getImageIdsFromFileNames(image_files)
# print("image ids", image_ids)
# ============
# Interference
# ============
APs = []
metrics = dict()
for image_id in image_ids:
image_name = os.path.basename(dataset_val.image_reference(image_id))
print("===============================")
print("Image reference: ", image_name)
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt(dataset_val, inference_config, image_id, use_mini_mask=False)
results = infer_model.detect([image], verbose=1)
r = results[0]
visualize.save_image(image, image_name[:-4] + "-detected", r['rois'], r['masks'], r['class_ids'], r['scores'], ["ship", "Ship", "ship"])
AP, precisions, recalls, overlaps = utils.compute_ap(gt_bbox, gt_class_id, gt_mask, r["rois"], r["class_ids"], r["scores"], r['masks'])
APs.append(AP)
image_metrics = dict()
image_metrics["AP"] = AP
image_metrics["precisions"] = precisions.tolist()
image_metrics["recalls"] = recalls.tolist()
image_metrics["overlaps"] = overlaps.tolist()
# print(json.dumps(image_metrics))
metrics[image_name] = image_metrics
def process( self, args ):
"""
Run inference with trained model
"""
# load validation / test dataset
test_ds = ShipDataset( (768,768) )
test_ds.load_info( os.path.join( args.data_path, 'test' ) )
test_ds.prepare()
# Load weights
print("Loading weights ", args.model_pathname)
self._model.load_weights(args.model_pathname, by_name=True)
for sample in range ( 10 ):
# pick random image
image_id = random.choice( test_ds.image_ids )
image, image_meta, gt_class_id, gt_bbox, gt_mask = modellib.load_image_gt( test_ds,
self._config,
image_id,
use_mini_mask=False )
# get image info
info = test_ds.image_info[ image_id ]
print("image ID: {}.{} ({}) {}".format( info["source"],
info["id"],
image_id,
test_ds.image_reference(image_id) ) )
# run object detection
results = self._model.detect([image], verbose=1)
r = results[0]
# compute AP over range 0.5 to 0.95 and print it
utils.compute_ap_range( gt_bbox,
gt_class_id,
gt_mask,
r['rois'],
r['class_ids'],
r['scores'],
r['masks'],
verbose=1)
# display results
visualize.display_instances( image,
r['rois'],
r['masks'],
r['class_ids'],
test_ds.class_names,
r['scores'],
title="Predictions",
show_bbox=False)
# display actual vs predicted differences
visualize.display_differences( image,
gt_bbox,
gt_class_id,
gt_mask,
r['rois'],
r['class_ids'],
r['scores'],
r['masks'],
test_ds.class_names,
title="Actual vs Predict Difference" )
return
