def show_image_debug(draw, boxes, scores, labels, masks, classes):
from keras_retinanet.utils.visualization import draw_box, draw_caption
from keras_maskrcnn.utils.visualization import draw_mask
from keras_retinanet.utils.colors import label_color
# visualize detections
limit_conf = 0.2
for box, score, label, mask in zip(boxes, scores, labels, masks):
# scores are sorted so we can break
if score < limit_conf:
break
color = label_color(label)
color_mask = (255, 0, 0)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :]
draw_mask(draw, b, mask, color=color_mask)
caption = "{} {:.3f}".format(classes[label], score)
print(caption)
draw_caption(draw, b, caption)
draw = cv2.cvtColor(draw, cv2.COLOR_RGB2BGR)
show_image(draw)
def get_detections(generator, model, out_dir, score_threshold=0.5):
all_results = [None for _ in range(generator.num_images)]
start = time()
print("Processing %i images." % generator.num_images)
for i, path, image, draw, scale in generator.load_images():
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
boxes /= scale
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < score_threshold:
break
b = box.astype(int)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
out_name = os.path.join(out_dir, path + '.png')
cv2.imwrite(out_name, draw)
rles = _masks_to_rles(masks, threshold=score_threshold)
all_results[i] = (path, boxes, rles)
end = time()
print("Processed images in %is" % (end - start))
return all_results
def annotate_image(self,
path_to_image,
num_gpus=1,
min_image_dimension=800,
max_image_dimension=1024,
steps_per_epoch=100,
validation_steps=70):
labels_to_names = self._load_labels()
model = models.load_model(self._path_to_model,
backbone_name='resnet50')
# load image
image = read_image_bgr(path_to_image)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(draw)
plt.show()
"""config = ImageMonkeyConfig(len(labels), num_gpus, min_image_dimension, max_image_dimension, steps_per_epoch, validation_steps)
def retinamask(self):
backbone = models.backbone('resnet50')
batch_size = 1
train_generator, validation_generator = Retinamask.create_generators(
batch_size, self.annotations, self.classes)
freeze_backbone = 'store_true'
weights = self.Input_weights_path
print('Creating model, this may take a second...')
model, training_model, prediction_model = Retinamask.create_models(
backbone_retinanet=backbone.maskrcnn,
num_classes=train_generator.num_classes(),
weights=weights,
freeze_backbone=freeze_backbone)
#print(model.summary())
training_model.fit_generator(generator=train_generator,
steps_per_epoch=1000,
epochs=self.epoch,
verbose=1,
max_queue_size=1)
training_model.save(self.trained_weights_path + 'retinamask.h5')
#Testing
model_path = self.trained_weights_path + 'retinamask.h5'
model = models.load_model(model_path, backbone_name='resnet50')
labels_to_names = {0: 'ship'}
# load image
image = read_image_bgr(test_image_path)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
plt.imsave(self.output_image_path + 'output.jpg', draw)
def apply_mask(model,
image,
draw=None,
threshold_score=0.5,
labels_to_names=None,
image_segments=True):
''' Process image numpy matrix using model and return the annotations
use draw if you want to draw over the image. the draw parameter is the image in RGB (image is in BGR instead) '''
from keras_retinanet.utils.image import preprocess_image, resize_image
from keras_retinanet.utils.colors import label_color
from keras_maskrcnn.utils.visualization import draw_mask
from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations
# import miscellaneous modules
import numpy as np
import time
if not labels_to_names:
# load label to names mapping for visualization purposes
labels_to_names = {
0: 'bag',
1: 'belt',
2: 'boots',
3: 'footwear',
4: 'outer',
5: 'dress',
6: 'sunglasses',
7: 'pants',
8: 'top',
9: 'shorts',
10: 'skirt',
11: 'headwear',
12: 'scarf/tie'
}
if not draw.any():
# copy to draw on
draw = image.copy()
#switching to RGB from BGR
draw[:, :, 2] = image[:, :, 0]
draw[:, :, 0] = image[:, :, 2]
draw_segment = draw.copy()
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
annotations = [{
'bbox': None,
'score': None,
'category': None,
'segment': None
} for i in range(
len([score for score in scores if score >= threshold_score]))]
i = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < threshold_score:
break
color = label_color(label)
drawclone = np.copy(draw_segment)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask_only(drawclone, b, mask, color=label_color(label))
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
annotations[i]['bbox'] = [b[0], b[1], b[2] - b[0], b[3] - b[1]]
annotations[i]['score'] = score
annotations[i]['category'] = labels_to_names[label]
if image_segments:
annotations[i][
'segment'] = drawclone # only the object inside the mask is shown, the rest is black
i += 1
return annotations
def main(proc_img_path=None,
proc_img_url=None,
all_set=True,
save_path=None,
model_path=None,
segments=False,
annotations=False,
threshold_score=0.5,
limit=None,
model=None,
labels_to_names=None):
# import keras
import keras
# import keras_retinanet
from keras_maskrcnn import models
from keras_maskrcnn.utils.visualization import draw_mask
from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations
from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image
from keras_retinanet.utils.colors import label_color
# import miscellaneous modules
import matplotlib.pyplot as plt
import cv2
import numpy as np
import time
# set tf backend to allow memory to grow, instead of claiming everything
import tensorflow as tf
# use this environment flag to change which GPU to use
#os.environ["CUDA_VISIBLE_DEVICES"] = "1"
with open(
os.path.expanduser('~') + '/.maskrcnn-modanet/' +
'savedvars.json') as f:
savedvars = json.load(f)
path = savedvars['datapath']
img_path = path + "datasets/coco/images/"
if not model:
# set the modified tf session as backend in keras
keras.backend.tensorflow_backend.set_session(get_session())
# adjust this to point to your trained model
# get all models names in the results folder
modelnames = [
f for f in os.listdir(snp_path)
if os.path.isfile(os.path.join(snp_path, f))
]
import re
def extract_number(f):
s = re.findall("\d+$", f)
return (int(s[0]) if s else -1, f)
# get the model name with the highest epoch
print(max(modelnames, key=extract_number))
model_path = os.path.join(snp_path, max(modelnames,
key=extract_number))
# load retinanet model
model = models.load_model(model_path, backbone_name='resnet50')
if not labels_to_names:
# load label to names mapping for visualization purposes
labels_to_names = {
0: 'bag',
1: 'belt',
2: 'boots',
3: 'footwear',
4: 'outer',
5: 'dress',
6: 'sunglasses',
7: 'pants',
8: 'top',
9: 'shorts',
10: 'skirt',
11: 'headwear',
12: 'scarf/tie'
}
default_save_path = False
if save_path == 'default':
# set path to default
save_path = path + 'results/processedimages/' #images/1.jpg'
if not annotations:
save_path += 'images/'
elif annotations:
save_path += 'annotations/'
default_save_path = True
SAVE_PATH = save_path # used for multiple images
if annotations:
# if save_path: save_path = path + 'results/processedimages/annotations/1.json'
annotations = [{
'bbox': None,
'score': None,
'category': None,
'part': None
}]
if all_set:
# load images
with open(ann_path + 'instances_val.json') as f:
instances = json.load(f)
images = instances['images']
for img in images:
img['file_name'] = img_path + img['file_name']
elif proc_img_path:
# just draw the image selected
images = [{
'file_name':
img_path + proc_img_path if
os.path.abspath(proc_img_path) != proc_img_path else proc_img_path
}]
elif proc_img_url:
# just draw the image selected
images = [{'file_name': proc_img_url}]
try:
#for each image in the dataset
for i, img in enumerate(images):
print(i, end=' ')
if limit and i >= limit:
break
if all_set:
image = read_image_bgr(img['file_name'])
elif proc_img_path:
image = read_image_bgr(img['file_name'])
elif proc_img_url:
import requests
from io import BytesIO
r = requests.get(img['file_name'], allow_redirects=True)
image = read_image_bgr(BytesIO(r.content))
if save_path:
if proc_img_path or all_set:
img_file_name = img['file_name'].split("/")[-1]
elif proc_img_url:
img_file_name = 'urlimg.jpg'
if not annotations:
save_path += img_file_name
elif annotations:
save_path += img_file_name.split('.')[0] + '.json'
if save_path and segments and not annotations:
#remove the extension
save_path = save_path.split('.')[0]
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ",
time.time() - start, "\t(Ctrl+c and close image to exit)")
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
if annotations:
annotations = [{
'bbox': None,
'score': None,
'category': None,
'part': None
} for i in range(
len([
score for score in scores if score >= threshold_score
]))]
segment_id = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < threshold_score:
break
color = label_color(label)
if not segments:
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
elif segments:
drawclone = np.copy(draw)
b = box.astype(int)
# draw_box(drawclone, b, color=color)
mask = mask[:, :, label]
draw_mask_only(drawclone,
b,
mask,
color=label_color(label))
if not annotations:
caption = "{} {:.3f}".format(labels_to_names[label],
score)
draw_caption(drawclone, b, caption)
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(drawclone)
if not save_path:
plt.show()
elif save_path:
segment_path = '_segment_' + segment_id + '.jpg'
save_path_segment = save_path + segment_path
print(save_path_segment)
plt.savefig(save_path_segment)
plt.close()
elif annotations:
annotations[segment_id]['bbox'] = b
annotations[segment_id]['score'] = score
annotations[segment_id]['category'] = label
annotations[segment_id][
'part'] = drawclone # only the object inside the mask is shown, the rest is black
segment_id += 1
if not segments:
if not save_path:
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(draw)
if not proc_img_url:
print(img['file_name'])
plt.show()
elif save_path:
processed_image = Image.fromarray(draw, 'RGB')
processed_image.save(save_path)
del processed_image
print(save_path)
# plt.savefig(save_path)
# plt.close()
elif segments:
if annotations:
if save_path:
print(save_path)
with open(save_path, 'w') as outfile:
json.dump(annotations, outfile)
else:
return annotations
save_path = SAVE_PATH # restore path for next image
except KeyboardInterrupt:
pass
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
#draw_caption(draw, b, caption)
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.savefig("output.png")
plt.imshow(draw)
plt.show()
def segmentation(imgpath, score_threshold=0.5, binarize_threshold=0.5):
# adjust this to point to your downloaded/trained model
model_path = "resnet50_weights/resnet50_coco_v0.2.0.h5"
# load retinanet model
model = models.load_model(model_path, backbone_name='resnet50')
# load label to names mapping for visualization purposes
labels_to_names = {
0: 'person',
1: 'bicycle',
2: 'car',
3: 'motorcycle',
4: 'airplane',
5: 'bus',
6: 'train',
7: 'truck',
8: 'boat',
9: 'traffic light',
10: 'fire hydrant',
11: 'stop sign',
12: 'parking meter',
13: 'bench',
14: 'bird',
15: 'cat',
16: 'dog',
17: 'horse',
18: 'sheep',
19: 'cow',
20: 'elephant',
21: 'bear',
22: 'zebra',
23: 'giraffe',
24: 'backpack',
25: 'umbrella',
26: 'handbag',
27: 'tie',
28: 'suitcase',
29: 'frisbee',
30: 'skis',
31: 'snowboard',
32: 'sports ball',
33: 'kite',
34: 'baseball bat',
35: 'baseball glove',
36: 'skateboard',
37: 'surfboard',
38: 'tennis racket',
39: 'bottle',
40: 'wine glass',
41: 'cup',
42: 'fork',
43: 'knife',
44: 'spoon',
45: 'bowl',
46: 'banana',
47: 'apple',
48: 'sandwich',
49: 'orange',
50: 'broccoli',
51: 'carrot',
52: 'hot dog',
53: 'pizza',
54: 'donut',
55: 'cake',
56: 'chair',
57: 'couch',
58: 'potted plant',
59: 'bed',
60: 'dining table',
61: 'toilet',
62: 'tv',
63: 'laptop',
64: 'mouse',
65: 'remote',
66: 'keyboard',
67: 'cell phone',
68: 'microwave',
69: 'oven',
70: 'toaster',
71: 'sink',
72: 'refrigerator',
73: 'book',
74: 'clock',
75: 'vase',
76: 'scissors',
77: 'teddy bear',
78: 'hair drier',
79: 'toothbrush'
}
# load image
image = read_image_bgr(imgpath)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("Segmentation took", round(time.time() - start, 2), "seconds.")
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
mask_list = []
box_list = []
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
# save box coordinates in list
box = box.astype(np.int16)
box_list.append(box[:])
# resize to fit the box
mask = mask.astype(np.float32)
mask = cv2.resize(mask, (box[2] - box[0], box[3] - box[1]))
# binarize the mask
mask = (mask > binarize_threshold).astype(np.uint8)
mask = cv2.normalize(src=mask,
dst=None,
alpha=0,
beta=255,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_8U)
mask_list.append(mask[:, :, label])
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
return mask_list, box_list, draw
def cloth_identifier_extract_text(input_imagefile):
# load image
image = read_image_bgr(input_imagefile)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
outputs1 = model.predict_on_batch(np.expand_dims(image, axis=0))
boxes = outputs1[-4][0]
scores = outputs1[-3][0]
labels = outputs1[-2][0]
masks = outputs1[-1][0]
# correct for image scale
boxes /= scale
masks_dic = {}
boxes_dic = {}
labels_dic = {}
counter = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
masks_dic[str(counter)] = mask
boxes_dic[str(counter)] = box
labels_dic[str(counter)] = label
counter += 1
items_dic = {}
counter = 0
masks = []
for box, mask, label in zip(boxes_dic.values(), masks_dic.values(),
labels_dic.values()):
b = box.astype(int)
# resize to fit the box
mask = mask.astype(np.float32)
mask = cv2.resize(mask, (b[2] - b[0], b[3] - b[1]))
# binarize the mask
mask = (mask > 0.5).astype(np.uint8)
# draw the mask in the image
mask_image = np.zeros((draw.shape[0], draw.shape[1]), np.uint8)
mask_image[b[1]:b[3], b[0]:b[2]] = mask
mask = mask_image
mask = (np.stack([mask] * 3, axis=2)) * draw
items_dic[str(counter)] = mask
counter += 1
return mask, label
def cloth(input_imagefile):
# load image
image = read_image_bgr(input_imagefile)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
masks_dic = {}
boxes_dic = {}
labels_dic = {}
counter = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
masks_dic[str(counter)] = mask
boxes_dic[str(counter)] = box
labels_dic[str(counter)] = label
counter += 1
image = read_image_bgr(input_imagefile)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# visualize detections
items_dic = {}
counter = 0
for box, mask2, label2 in zip(boxes_dic.values(), masks_dic.values(),
labels_dic.values()):
b = box.astype(int)
# resize to fit the box
mask2 = mask2.astype(np.float32)
mask2 = cv2.resize(mask2, (b[2] - b[0], b[3] - b[1]))
# binarize the mask1
mask2 = (mask2 > 0.5).astype(np.uint8)
# draw the mask2 in the image
mask2_image = np.zeros((draw.shape[0], draw.shape[1]), np.uint8)
mask2_image[b[1]:b[3], b[0]:b[2]] = mask2
mask2 = mask2_image
mask2 = (np.stack([mask2] * 3, axis=2)) * draw
items_dic[str(counter)] = mask2
counter += 1
# newfileneame=input_imagefile.split("/")[4].split('.')[0]
# plt.ioff()
# plt.figure(figsize=(15, 15))
# plt.axis('off')
# plt.imshow(mask2)
#plt.savefig('/home/ec2-user/SageMaker/'+str(newfileneame)+'-masked-'+str(label2)+'.jpg',bbox_inches='tight', pad_inches=0)
#plt.show()
plt.close('all')
return mask2, label2
