def process_frame(frame):
height, width, channels = frame.shape
frame_area = width * height
draw = frame.copy()
frame = preprocess_image(frame)
frame, scale = resize_image(frame)
boxes, scores, labels, = MODEL.predict_on_batch(np.expand_dims(frame, axis=0))
boxes /= scale
box_json_array = []
for box, score, label in zip(boxes[0], scores[0], labels[0]):
b = box.astype(int)
box_area = np.abs(b[2] - b[0]) * np.abs(b[3] - b[1])
if score < CONFIDENCE_THRESHOLD or box_area > (frame_area * BOX_AREA_RATIO_TO_IGNORE) or label > MAX_CLASS_ID:
break
color = label_color(label)
draw_box(draw, b, color=color)
caption = '{} {:.3f}'.format(LABELS_TO_NAMES[label], score)
draw_caption(draw, b, caption)
box_json_array.append({
'label': caption,
'topLeft': [int(b[0]), int(b[1])],
'bottomRight': [int(b[2]), int(b[3])]
})
return draw, box_json_array
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
image_paths = list()
with open(args.csv_path, 'w', newline='') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',')
tree = ET.parse(args.gt_path)
# get root element
root = tree.getroot()
annotation_items = root.findall('./annotation')
for annotation_item in annotation_items:
filename_item = annotation_item.find('./filename')
text = filename_item.text
if 'FigureSeparationTraining2016' in args.gt_path:
image_path = os.path.join('/datasets/ImageCLEF/2016/training/FigureSeparationTraining2016/', text+'.jpg')
elif 'FigureSeparationTest2016GT' in args.gt_path:
image_path = os.path.join('/datasets/ImageCLEF/2016/test/FigureSeparationTest2016/', text+'.jpg')
else:
raise Exception('Error {0}'.format(args.gt_path))
image_paths.append(image_path+"\n")
image = read_image_bgr(image_path)
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
csv_path_individual = os.path.join('csv/', text + '.csv')
jpg_path_individual = os.path.join('preview/', text + '.jpg')
with open(csv_path_individual, 'w', newline='') as csvfile_individual:
csv_writer_individual = csv.writer(csvfile_individual, delimiter=',')
object_items = annotation_item.findall('./object')
for idx, object_item in enumerate(object_items):
point_items = object_item.findall('./point')
x1 = point_items[0].get('x')
y1 = point_items[0].get('y')
x2 = point_items[3].get('x')
y2 = point_items[3].get('y')
if int(x1) >= int(x2) or int(y1) >= int(y2):
continue
csv_writer.writerow([image_path, x1, y1, x2, y2, 'panel'])
csv_writer_individual.writerow([image_path, x1, y1, x2, y2, 'panel'])
color = label_color(idx)
box = [int(x1), int(y1), int(x2), int(y2)]
draw_box(draw, box, color)
cv2.imwrite(jpg_path_individual, draw)
with open(args.list_path, "w") as text_file:
text_file.writelines(image_paths)
boxes /= scale
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# if label == 0:
# scores are sorted so we can break
if score < 0.75:
break
value = (image_location, int(box[0]), int(box[1]), int(box[2]),
int(box[3]), str(labels_to_names[label]), score)
xmlList.append(value)
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
caption = "{} {:.3f}".format(labels_to_names[label], score)
# caption = ""
draw_caption(draw, b, caption)
# plt.figure(figsize=(15, 15))
# plt.axis('off')
# plt.imshow(draw)
# plt.show()
xmlDF = pd.DataFrame(xmlList)
cv2.imwrite("./output/" + name + "vestFrame_{}.jpg".format(count), draw)
img_array.append(draw)
# added one more read to effectively skip every second frame...
# vidcap.read()
success, image = vidcap.read()
# correct for image scale
boxes /= scale
pre_passed_boxes = []
pre_passed_scores = []
for box, score, label in zip(boxes[0], scores[0], labels[0]):
if score >= 0.2:
pre_passed_boxes.append(box.tolist())
pre_passed_scores.append(score.tolist())
passed_boxes, passed_scores = filter_boxes(
in_boxes=pre_passed_boxes,
in_scores=pre_passed_scores,
_passed_boxes=[],
_passed_scores=[]) # These are necessary
print("found " + str(len(passed_boxes)) + " cells in " + str(infile) +
" frame " + str(i))
for b, score in zip(passed_boxes, passed_scores):
# b = box.astype(int)
draw_box(draw, b, color=(0, 255, 0), thickness=1)
caption = "{:.3f}".format(score)
draw_caption(draw, b, caption)
#cv2.imwrite('./tracking_demo_image/stack/out' + str(i) + '.png', draw)
writer.writeFrame(draw)
writer.close()
def main(args=None):
# parse arguments
if args is None:
args = sys.argv[1:]
args = parse_args(args)
image_paths = list()
with open(args.csv_path, 'w', newline='') as csvfile:
csv_writer = csv.writer(csvfile, delimiter=',')
tree = ET.parse(args.gt_path)
# get root element
root = tree.getroot()
annotation_items = root.findall('./annotation')
for annotation_item in annotation_items:
filename_item = annotation_item.find('./filename')
text = filename_item.text
if 'FigureSeparationTraining2016' in args.gt_path:
image_path = os.path.join(
'/datasets/ImageCLEF/2016/training/FigureSeparationTraining2016/',
text + '.jpg')
elif 'FigureSeparationTest2016GT' in args.gt_path:
image_path = os.path.join(
'/datasets/ImageCLEF/2016/test/FigureSeparationTest2016/',
text + '.jpg')
else:
raise Exception('Error {0}'.format(args.gt_path))
image_paths.append(image_path + "\n")
image = read_image_bgr(image_path)
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
csv_path_individual = os.path.join('csv/', text + '.csv')
jpg_path_individual = os.path.join('preview/', text + '.jpg')
with open(csv_path_individual, 'w',
newline='') as csvfile_individual:
csv_writer_individual = csv.writer(csvfile_individual,
delimiter=',')
object_items = annotation_item.findall('./object')
for idx, object_item in enumerate(object_items):
point_items = object_item.findall('./point')
x1 = point_items[0].get('x')
y1 = point_items[0].get('y')
x2 = point_items[3].get('x')
y2 = point_items[3].get('y')
if int(x1) >= int(x2) or int(y1) >= int(y2):
continue
csv_writer.writerow([image_path, x1, y1, x2, y2, 'panel'])
csv_writer_individual.writerow(
[image_path, x1, y1, x2, y2, 'panel'])
color = label_color(idx)
box = [int(x1), int(y1), int(x2), int(y2)]
draw_box(draw, box, color)
cv2.imwrite(jpg_path_individual, draw)
with open(args.list_path, "w") as text_file:
text_file.writelines(image_paths)
def detect(self, image):
self.time2store = self.gen_datetime()
# self.time2store = datetime.now()
self.cen_x = image.shape[1] // 2
self.cen_y = image.shape[0] // 2
# copy to draw on
draw = image.copy()
# preprocess image for network
image = preprocess_image(image)
# cv2.imshow('ss22', image)
image, scale = resize_image(image,
min_side=self.min_side4train,
max_side=self.max_side4train)
# time_ = self.time2store
time_ = datetime.now()
eventid = time_.strftime('%Y%m-%d%H-%M%S-') + str(uuid4())
# process image
start = time.time()
boxes, scores, labels = self.model.predict_on_batch(
np.expand_dims(image, axis=0))
processing_time = time.time() - start
# print("processing time: ", processing_time)
img4elas, scale4elas = resize_image(draw,
min_side=self.min_side4elas,
max_side=self.max_side4elas)
# correct for image scale
# boxes /= scale
box4turret = boxes / scale
found_ = {}
main_body = {'eventid': eventid, 'time_': time_}
self.__data4turret = {
"box": box4turret[0][0],
"score": scores[0][0],
"label": labels[0][0],
"scale": scale,
"raw_image-shape": draw.shape
}
# visualize detections
temp_data = []
index = 1
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
# print(index, box, score, self.labels_to_names[label])
if score < self.confThreshold:
break
color = label_color(label)
b = box.astype(int)
draw_box(img4elas, b, color=color)
caption = "{} {:.3f}".format(self.labels_to_names[label], score)
# print(caption)
draw_caption(img4elas, b, caption)
temp_data.append(
[self.labels_to_names[label], score, b, processing_time])
return temp_data
with open(ROOTDIR + '/predictions.txt', 'a') as the_file:
the_file.write(
str(score) + ',' + '1' + ',' + str(iou) + ',' +
str(LABELS_TO_NAMES.values().index(row['class'])) +
'\n')
else:
print("Not correct with IoU " + str(iou) + " and score " +
str(score))
# Store prediction information
with open(ROOTDIR + '/predictions.txt', 'a') as the_file:
the_file.write(
str(score) + ',' + '0' + ',' + str(iou) + ',' +
str(LABELS_TO_NAMES.values().index(row['class'])) +
'\n')
if SHOW_IMAGES:
color = label_color(label)
color_gr = (0, 255, 255)
draw_box(draw, b, color=color)
draw_box(draw, b_gt, color=color_gr)
caption = "{} {:.3f}".format(LABELS_TO_NAMES[label], score)
print("Score: " + str(score))
draw_caption(draw, b, caption)
if SHOW_IMAGES:
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(draw)
plt.show()
def run(image_path):
detector = models.load_model(inference_model_save_path,
backbone_name='resnet50')
# detector.summary()
# ### 加载检测 类别名称 detector_class_name
df = pd.read_csv("../data/baijiu/class.csv", header=None)
# load label to names mapping for visualization purposes
detector_class_name = df[0].values.tolist()
# ### 加载分类模型 ResNet
classifier = tf.keras.models.load_model(
"../models/ResNet50/ResNet50_classified.h5")
# classifier.summary()
# ### 加载类别名称文件
with open("../models/ResNet50/ResNet50_classified_class.txt", "r+") as f:
classifier_class_name = eval(f.read())
classifier_class_name[:10]
# ## 检测+分类
def load_image(img_path):
img = image.load_img(img_path, target_size=(224, 224))
img_tensor = image.img_to_array(img)
img_tensor = np.expand_dims(img_tensor, axis=0)
img_tensor /= 255.
plt.imshow(img_tensor[0])
plt.axis('off')
plt.show()
return img_tensor
# test_image_path = 'test_1.jpg'
test_dir = "test/"
# 读取图像
input_image = read_image_bgr(image_path)
# 预处理
input_image = preprocess_image(input_image)
input_image, scale = resize_image(input_image)
# 计时开始
start = time.time()
# 商品检测
boxes, scores, labels = detector.predict_on_batch(
np.expand_dims(input_image, axis=0))
# 图像尺寸矫正
boxes /= scale
# 分类结果画图，重新读取原图
draw = read_image_bgr(image_path).copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# 记录分类结果
idx = 0
predict_list = []
# 商品识别和可视化
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# 过滤小于0.5检测框
if score < 0.5:
break
# 分类框颜色（复用RetinaNet可视化模块，颜色类别可能不够用）
color = label_color(label)
b = box.astype(int)
# 检测框中的SKU小图
cropped = draw[b[1]:b[3], b[0]:b[2]]
if cropped is not None:
sku_image_path = f"test/{idx}.jpg"
# 保存SKU小图结果
cv2.imwrite(sku_image_path, cropped)
# 商品识别
img_tensor = load_image(sku_image_path)
# 商品识别
pred = classifier.predict(img_tensor)
sku_id = np.argmax(pred)
sku_name = classifier_class_name[sku_id]
sku_score = np.max(pred)
predict_list.append(sku_name)
# 打印识别结果
print(f"{sku_id} {sku_name} {sku_score}")
# 在货架图中可视化分类结果
caption = "{} {:.3f}".format(sku_id, sku_score)
draw_caption(draw, b, caption)
draw_box(draw, b, color=label_color(sku_id))
idx += 1
print("processing time: ", time.time() - start)
plt.figure(figsize=(25, 45))
plt.axis('off')
plt.imshow(draw)
plt.show()
return predict_list
def start_proceed(img_path):
def get_session():
# for tf2.0
# config = tf.compat.v1.ConfigProto()
# for tf 1.x
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# tf2.0
# return tf.compat.v1.Session(config=config)
# tf 1.x
return tf.Session(config=config)
# use this environment flag to change which GPU to use
# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
# set the modified tf session as backend in keras
keras.backend.tensorflow_backend.set_session(get_session())
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
filename = 'keras-retinanet/path/classes.csv'
# df = pd.read_csv(filename, header=None)
model_path2 = ('keras-retinanet/snapshots/converted_restnet50_model.h5')
# load retinanet model
model = models.load_model(model_path2, backbone_name='resnet50')
# if the model is not converted to an inference model, use the line below
# see: https://github.com/fizyr/keras-retinanet#converting-a-training-model-to-inference-model
# model = models.convert_model(model)
# load label to names mapping for visualization purposes
# labels_to_names = {0: 'person', 1: 'bicycle', 2: 'car', 3: 'motorcycle', 4: 'airplane', 5: 'bus', 6: 'train',
# 7: 'truck', 8: 'boat', 9: 'traffic light', 10: 'fire hydrant', 11: 'stop sign', 12: 'parking meter',
# 13: 'bench', 14: 'bird', 15: 'cat', 16: 'dog', 17: 'horse', 18: 'sheep', 19: 'cow', 20: 'elephant',
# 21: 'bear', 22: 'zebra', 23: 'giraffe', 24: 'backpack', 25: 'umbrella', 26: 'handbag', 27: 'tie',
# 28: 'suitcase', 29: 'frisbee', 30: 'skis', 31: 'snowboard', 32: 'sports ball', 33: 'kite',
# 34: 'baseball bat', 35: 'baseball glove', 36: 'skateboard', 37: 'surfboard', 38: 'tennis racket',
# 39: 'bottle', 40: 'wine glass', 41: 'cup', 42: 'fork', 43: 'knife', 44: 'spoon', 45: 'bowl',
# 46: 'banana', 47: 'apple', 48: 'sandwich', 49: 'orange', 50: 'broccoli', 51: 'carrot', 52: 'hot dog',
# 53: 'pizza', 54: 'donut', 55: 'cake', 56: 'chair', 57: 'couch', 58: 'potted plant', 59: 'bed',
# 60: 'dining table', 61: 'toilet', 62: 'tv', 63: 'laptop', 64: 'mouse', 65: 'remote', 66: 'keyboard',
# 67: 'cell phone', 68: 'microwave', 69: 'oven', 70: 'toaster', 71: 'sink', 72: 'refrigerator',
# 73: 'book', 74: 'clock', 75: 'vase', 76: 'scissors', 77: 'teddy bear', 78: 'hair drier',
# 79: 'toothbrush'}
labels_to_names = {0: 'AM General Hummer SUV 2000', 1: 'Acura RL Sedan 2012', 2: 'Acura TL Sedan 2012',
3: 'Acura TL Type-S 2008', 4: 'Acura TSX Sedan 2012', 5: 'Acura Integra Type R 2001',
6: 'Acura ZDX Hatchback 2012', 7: 'Aston Martin V8 Vantage Convertible 2012',
8: 'Aston Martin V8 Vantage Coupe 2012', 9: 'Aston Martin Virage Convertible 2012',
10: 'Aston Martin Virage Coupe 2012', 11: 'Audi RS 4 Convertible 2008', 12: 'Audi A5 Coupe 2012',
13: 'Audi TTS Coupe 2012', 14: 'Audi R8 Coupe 2012', 15: 'Audi V8 Sedan 1994',
16: 'Audi 100 Sedan 1994', 17: 'Audi 100 Wagon 1994', 18: 'Audi TT Hatchback 2011',
19: 'Audi S6 Sedan 2011', 20: 'Audi S5 Convertible 2012', 21: 'Audi S5 Coupe 2012',
22: 'Audi S4 Sedan 2012', 23: 'Audi S4 Sedan 2007', 24: 'Audi TT RS Coupe 2012',
25: 'BMW ActiveHybrid 5 Sedan 2012', 26: 'BMW 1 Series Convertible 2012',
27: 'BMW 1 Series Coupe 2012', 28: 'BMW 3 Series Sedan 2012', 29: 'BMW 3 Series Wagon 2012',
30: 'BMW 6 Series Convertible 2007', 31: 'BMW X5 SUV 2007', 32: 'BMW X6 SUV 2012',
33: 'BMW M3 Coupe 2012', 34: 'BMW M5 Sedan 2010', 35: 'BMW M6 Convertible 2010',
36: 'BMW X3 SUV 2012', 37: 'BMW Z4 Convertible 2012',
38: 'Bentley Continental Supersports Conv. Convertible 2012', 39: 'Bentley Arnage Sedan 2009',
40: 'Bentley Mulsanne Sedan 2011', 41: 'Bentley Continental GT Coupe 2012',
42: 'Bentley Continental GT Coupe 2007', 43: 'Bentley Continental Flying Spur Sedan 2007',
44: 'Bugatti Veyron 16.4 Convertible 2009', 45: 'Bugatti Veyron 16.4 Coupe 2009',
46: 'Buick Regal GS 2012', 47: 'Buick Rainier SUV 2007', 48: 'Buick Verano Sedan 2012',
49: 'Buick Enclave SUV 2012', 50: 'Cadillac CTS-V Sedan 2012', 51: 'Cadillac SRX SUV 2012',
52: 'Cadillac Escalade EXT Crew Cab 2007', 53: 'Chevrolet Silverado 1500 Hybrid Crew Cab 2012',
54: 'Chevrolet Corvette Convertible 2012', 55: 'Chevrolet Corvette ZR1 2012',
56: 'Chevrolet Corvette Ron Fellows Edition Z06 2007', 57: 'Chevrolet Traverse SUV 2012',
58: 'Chevrolet Camaro Convertible 2012', 59: 'Chevrolet HHR SS 2010',
60: 'Chevrolet Impala Sedan 2007', 61: 'Chevrolet Tahoe Hybrid SUV 2012',
62: 'Chevrolet Sonic Sedan 2012', 63: 'Chevrolet Express Cargo Van 2007',
64: 'Chevrolet Avalanche Crew Cab 2012', 65: 'Chevrolet Cobalt SS 2010',
66: 'Chevrolet Malibu Hybrid Sedan 2010', 67: 'Chevrolet TrailBlazer SS 2009',
68: 'Chevrolet Silverado 2500HD Regular Cab 2012',
69: 'Chevrolet Silverado 1500 Classic Extended Cab 2007', 70: 'Chevrolet Express Van 2007',
71: 'Chevrolet Monte Carlo Coupe 2007', 72: 'Chevrolet Malibu Sedan 2007',
73: 'Chevrolet Silverado 1500 Extended Cab 2012', 74: 'Chevrolet Silverado 1500 Regular Cab 2012',
75: 'Chrysler Aspen SUV 2009', 76: 'Chrysler Sebring Convertible 2010',
77: 'Chrysler Town and Country Minivan 2012', 78: 'Chrysler 300 SRT-8 2010',
79: 'Chrysler Crossfire Convertible 2008', 80: 'Chrysler PT Cruiser Convertible 2008',
81: 'Daewoo Nubira Wagon 2002', 82: 'Dodge Caliber Wagon 2012', 83: 'Dodge Caliber Wagon 2007',
84: 'Dodge Caravan Minivan 1997', 85: 'Dodge Ram Pickup 3500 Crew Cab 2010',
86: 'Dodge Ram Pickup 3500 Quad Cab 2009', 87: 'Dodge Sprinter Cargo Van 2009',
88: 'Dodge Journey SUV 2012', 89: 'Dodge Dakota Crew Cab 2010', 90: 'Dodge Dakota Club Cab 2007',
91: 'Dodge Magnum Wagon 2008', 92: 'Dodge Challenger SRT8 2011', 93: 'Dodge Durango SUV 2012',
94: 'Dodge Durango SUV 2007', 95: 'Dodge Charger Sedan 2012', 96: 'Dodge Charger SRT-8 2009',
97: 'Eagle Talon Hatchback 1998', 98: 'FIAT 500 Abarth 2012', 99: 'FIAT 500 Convertible 2012',
100: 'Ferrari FF Coupe 2012', 101: 'Ferrari California Convertible 2012',
102: 'Ferrari 458 Italia Convertible 2012', 103: 'Ferrari 458 Italia Coupe 2012',
104: 'Fisker Karma Sedan 2012', 105: 'Ford F-450 Super Duty Crew Cab 2012',
106: 'Ford Mustang Convertible 2007', 107: 'Ford Freestar Minivan 2007',
108: 'Ford Expedition EL SUV 2009', 109: 'Ford Edge SUV 2012', 110: 'Ford Ranger SuperCab 2011',
111: 'Ford GT Coupe 2006', 112: 'Ford F-150 Regular Cab 2012', 113: 'Ford F-150 Regular Cab 2007',
114: 'Ford Focus Sedan 2007', 115: 'Ford E-Series Wagon Van 2012', 116: 'Ford Fiesta Sedan 2012',
117: 'GMC Terrain SUV 2012', 118: 'GMC Savana Van 2012', 119: 'GMC Yukon Hybrid SUV 2012',
120: 'GMC Acadia SUV 2012', 121: 'GMC Canyon Extended Cab 2012', 122: 'Geo Metro Convertible 1993',
123: 'HUMMER H3T Crew Cab 2010', 124: 'HUMMER H2 SUT Crew Cab 2009',
125: 'Honda Odyssey Minivan 2012', 126: 'Honda Odyssey Minivan 2007', 127: 'Honda Accord Coupe 2012',
128: 'Honda Accord Sedan 2012', 129: 'Hyundai Veloster Hatchback 2012',
130: 'Hyundai Santa Fe SUV 2012', 131: 'Hyundai Tucson SUV 2012', 132: 'Hyundai Veracruz SUV 2012',
133: 'Hyundai Sonata Hybrid Sedan 2012', 134: 'Hyundai Elantra Sedan 2007',
135: 'Hyundai Accent Sedan 2012', 136: 'Hyundai Genesis Sedan 2012',
137: 'Hyundai Sonata Sedan 2012', 138: 'Hyundai Elantra Touring Hatchback 2012',
139: 'Hyundai Azera Sedan 2012', 140: 'Infiniti G Coupe IPL 2012', 141: 'Infiniti QX56 SUV 2011',
142: 'Isuzu Ascender SUV 2008', 143: 'Jaguar XK XKR 2012', 144: 'Jeep Patriot SUV 2012',
145: 'Jeep Wrangler SUV 2012', 146: 'Jeep Liberty SUV 2012', 147: 'Jeep Grand Cherokee SUV 2012',
148: 'Jeep Compass SUV 2012', 149: 'Lamborghini Reventon Coupe 2008',
150: 'Lamborghini Aventador Coupe 2012', 151: 'Lamborghini Gallardo LP 570-4 Superleggera 2012',
152: 'Lamborghini Diablo Coupe 2001', 153: 'Land Rover Range Rover SUV 2012',
154: 'Land Rover LR2 SUV 2012', 155: 'Lincoln Town Car Sedan 2011',
156: 'MINI Cooper Roadster Convertible 2012', 157: 'Maybach Landaulet Convertible 2012',
158: 'Mazda Tribute SUV 2011', 159: 'McLaren MP4-12C Coupe 2012',
160: 'Mercedes-Benz 300-Class Convertible 1993', 161: 'Mercedes-Benz C-Class Sedan 2012',
162: 'Mercedes-Benz SL-Class Coupe 2009', 163: 'Mercedes-Benz E-Class Sedan 2012',
164: 'Mercedes-Benz S-Class Sedan 2012', 165: 'Mercedes-Benz Sprinter Van 2012',
166: 'Mitsubishi Lancer Sedan 2012', 167: 'Nissan Leaf Hatchback 2012',
168: 'Nissan NV Passenger Van 2012', 169: 'Nissan Juke Hatchback 2012',
170: 'Nissan 240SX Coupe 1998', 171: 'Plymouth Neon Coupe 1999', 172: 'Porsche Panamera Sedan 2012',
173: 'Ram C/V Cargo Van Minivan 2012', 174: 'Rolls-Royce Phantom Drophead Coupe Convertible 2012',
175: 'Rolls-Royce Ghost Sedan 2012', 176: 'Rolls-Royce Phantom Sedan 2012',
177: 'Scion xD Hatchback 2012', 178: 'Spyker C8 Convertible 2009', 179: 'Spyker C8 Coupe 2009',
180: 'Suzuki Aerio Sedan 2007', 181: 'Suzuki Kizashi Sedan 2012', 182: 'Suzuki SX4 Hatchback 2012',
183: 'Suzuki SX4 Sedan 2012', 184: 'Tesla Model S Sedan 2012', 185: 'Toyota Sequoia SUV 2012',
186: 'Toyota Camry Sedan 2012', 187: 'Toyota Corolla Sedan 2012', 188: 'Toyota 4Runner SUV 2012',
189: 'Volkswagen Golf Hatchback 2012', 190: 'Volkswagen Golf Hatchback 1991',
191: 'Volkswagen Beetle Hatchback 2012', 192: 'Volvo C30 Hatchback 2012',
193: 'Volvo 240 Sedan 1993', 194: 'Volvo XC90 SUV 2007', 195: 'smart fortwo Convertible 2012'}
# model.summary()
# df_test = pd.read_csv("keras-retinanet/path/annotations_test.csv", header=None)
# load image
# image = read_image_bgr('/Users/joey/Nutstore Files/我的坚果云/vehicle/00003.jpg')
image = read_image_bgr(img_path)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))
process_time=time.time()-start
# print("processing time: ", time.time() - start)
# print(labels)
# correct for image scale
boxes /= scale
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
# print(labels_to_names[label])
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
plt.figure(figsize=(10, 8))
plt.axis('off')
plt.imshow(draw)
plt.savefig('output.jpg')
# plt.show()
# print(caption)
return caption,process_time
def make_predictions(model_path,
q,
detection_threshold=0.5,
overlap_threshold=0.4):
'''
Grab image from queue, make predictions and overlay bounding boxes onto it.
Assumes that image is preprocessed into a form ingestible by predict_on_batch()
Args:
model:
queue::Queue Contains images as numpy.array (batch_size, height, width); batch_size is usually 1
detection_threshold::float
overlap_threshold::float
Modifies 'image' in-place, draws bounding boxes (if any) on top of
existing array.
'''
# Load model into memory.
model = load_trained_model(model_path, backbone_name='resnet50')
labels_to_names = load_label_map()
while True:
if not q.empty():
image = q.get_nowait()
q.task_done()
# process image
boxes, scores, labels = model.predict_on_batch(image)
# Filter out non-detections, using the classification probability
# Note that we want to keep the last dimension of boxes, which contains the bounding box cooordinates
score_mask = (scores > detection_threshold)
scores = scores[score_mask]
labels = labels[score_mask]
boxes = boxes[0, score_mask[0], :]
## Apply non-max suppression
for iA, (boxA, scoreA) in enumerate(zip(boxes, scores)):
if scoreA > 0.0:
try:
# Search forward for overlapping boxes
suppression_candidates = [(iA, scoreA)]
for iB, (boxB,
scoreB) in enumerate(zip(boxes[iA + 1:, :],
scores[iA + 1:]),
start=iA + 1):
if intersection_over_union(boxA,
boxB) > overlap_threshold:
suppression_candidates.append((iB, scoreB))
# Non-max suppression by setting score to 0.0
if len(suppression_candidates) > 1:
suppression_candidates.sort(key=lambda x: x[-1])
for target, _ in suppression_candidates[:-1]:
scores[target] = 0.0
except IndexError:
# catch iA+1 when it goes out of bound
# Allegedly try-except blocks are ubiquitous for control-flow in Python
pass
## Make use of bounding boxes and predicted classes
## e.g. Draw bounding boxes
p_screen = cv2.cvtColor(image[0], cv2.COLOR_RGB2BGR)
for box, score, label in zip(boxes, scores, labels):
if (score <= detection_threshold): continue
color = label_color(label)
b = box.astype(int)
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_box(p_screen, b, color=color, thickness=2)
draw_caption(p_screen, b, caption)
cv2.imshow('Object detector', p_screen)
#cv2.imshow('window',cv2.cvtColor(np.array(p_screen),cv2.COLOR_BGR2RGB))
if cv2.waitKey(25) & 0xFF == ord('q'):
cv2.destroyAllWindows()
break
def parse_result(result_future):
"""Callback function.
Calculates the statistics for the prediction result.
Args:
result_future: Result future of the RPC.
For the example image the results would come like using retinanet pre-processing
'in image shape', (800, 1067, 3))
('Input shape=', (1, 800, 1067, 3))
('in tf shape', (2, 800, 1067, 3))
('result no', 0)
('boxes output', (1, 300, 4))
('scores output', (1, 300))
('labels output', (1, 300))
('Label', 'person', ' at ', array([409, 167, 728, 603]), ' Score ', 0.9681119)
('Label', 'person', ' at ', array([ 0, 426, 512, 785]), ' Score ', 0.8355836)
('Label', 'person', ' at ', array([ 723, 475, 1067, 791]), ' Score ', 0.72344124)
('Label', 'tie', ' at ', array([527, 335, 569, 505]), ' Score ', 0.525432)
('Time for ', 1, ' is ', 0.7898709774017334) in 1070 8Gb
"""
global _counter
global _start
global _response_awaiting
global _draw
boxes = result_future.\
outputs['filtered_detections/map/TensorArrayStack/TensorArrayGatherV3:0']
scores = result_future.\
outputs['filtered_detections/map/TensorArrayStack_1/TensorArrayGatherV3:0']
labels = result_future.\
outputs['filtered_detections/map/TensorArrayStack_2/TensorArrayGatherV3:0']
boxes = tf.make_ndarray(boxes)
scores = tf.make_ndarray(scores)
labels = tf.make_ndarray(labels)
print("result no", _counter)
print("boxes output", (boxes).shape)
print("scores output", (scores).shape)
print("labels output", (labels).shape)
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < 0.3:
break
b = box.astype(int)
print("Label", labels_to_names[label], " at ", b, " Score ", score)
# draw the image and write out
color = label_color(label)
draw_box(_draw, b, color=color)
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(_draw, b, caption)
_counter += 1
#if( (_counter % 1) ==0):#print every 100
# print("[", _counter,"] From Callback Predicted Result is ", prediction,"confidence= ",response[prediction])
if (_counter == FLAGS.num_tests):
end = time.time()
print("Time for ", FLAGS.num_tests, " is ", end - _start)
_response_awaiting = False
cv2.imwrite("/etc/object-detection/model/out.jpg", _draw)
def cloth(input_imagefile):
# load image
image = read_image_bgr(input_imagefile)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
masks_dic = {}
boxes_dic = {}
labels_dic = {}
counter = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
masks_dic[str(counter)] = mask
boxes_dic[str(counter)] = box
labels_dic[str(counter)] = label
counter += 1
image = read_image_bgr(input_imagefile)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# visualize detections
items_dic = {}
counter = 0
for box, mask2, label2 in zip(boxes_dic.values(), masks_dic.values(),
labels_dic.values()):
b = box.astype(int)
# resize to fit the box
mask2 = mask2.astype(np.float32)
mask2 = cv2.resize(mask2, (b[2] - b[0], b[3] - b[1]))
# binarize the mask1
mask2 = (mask2 > 0.5).astype(np.uint8)
# draw the mask2 in the image
mask2_image = np.zeros((draw.shape[0], draw.shape[1]), np.uint8)
mask2_image[b[1]:b[3], b[0]:b[2]] = mask2
mask2 = mask2_image
mask2 = (np.stack([mask2] * 3, axis=2)) * draw
items_dic[str(counter)] = mask2
counter += 1
# newfileneame=input_imagefile.split("/")[4].split('.')[0]
# plt.ioff()
# plt.figure(figsize=(15, 15))
# plt.axis('off')
# plt.imshow(mask2)
#plt.savefig('/home/ec2-user/SageMaker/'+str(newfileneame)+'-masked-'+str(label2)+'.jpg',bbox_inches='tight', pad_inches=0)
#plt.show()
plt.close('all')
return mask2, label2
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
boxes, scores, labels = model.predict(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes /= scale
# print(scores)
# if np.max(scores[0]) < 0.95:
# no_hat_list.append(img)
# # visualize detections
# else:
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < 0.95:
break
b = box.astype(int)
draw_box(draw, b, color=[0, 128, 0])
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
cv2.imwrite('../DATASET/safe_hat/output/' + img,
cv2.cvtColor(draw, cv2.COLOR_RGB2BGR))
def execute(inp, queu=None):
if queu is None:
queu = []
try:
video_inp = inp[1]
video = inp[2]
queue = inp[3]
print(video_inp)
model_path = 'model1.h5'
print("#################")
model = models.load_model(model_path, backbone_name='resnet50')
#model=models.convert_model(model)
font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCornerOfText = (30, 30)
fontScale = 1
fontColor1 = (255, 0, 0)
fontColor2 = (0, 255, 0)
lineType = 2
video_reader = cv2.VideoCapture(video_inp)
if video_reader is None or not video_reader.isOpened():
ps.pusherrrorininput("Q2")
else:
nb_frames = int(video_reader.get(cv2.CAP_PROP_FRAME_COUNT))
#nb_fps= int(video_reader.get(cv2.CAP_PROP_FPS))
start = 0
FRS = 150
time_, people_ = [], []
i = video
k = queue
print("Queueu is {}".format(queue))
processid = os.getpid()
for j in range(0, nb_frames, FRS):
video_reader.set(1, j)
time_sec = int(video_reader.get(0))
print("Current time in the Video {}".format((time_sec / 1000)))
print("Memory allocated is {}".format(
psutil.Process(processid).memory_info()[0]))
res, image = video_reader.read()
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
if i == "V1":
print("QUEUUE LENTH....{}".format(i))
crop_img_g = draw[200:600, 100:272].copy()
elif i == "V2":
print("QUEUUE LENTH....{}".format(i))
crop_img_g = draw[200:600, 100:300].copy()
elif i == "V3":
if k == "Q3":
crop_img_g = draw[100:600, 200:320].copy()
elif k == "Q4":
crop_img_g = draw[100:600, 400:550].copy()
else:
crop_img_g = draw[100:600, 200:320].copy()
elif i == "V4":
print("QUEUUE LENTH....{}".format(i))
if k == "Q5":
crop_img_g = draw[100:600, 100:270].copy()
elif k == "Q6":
crop_img_g = draw[100:600, 250:350].copy()
else:
crop_img_g = draw[100:600, 80:250].copy()
#crop_img_g=draw[100:600,80:250].copy()
elif i == "V6":
print("QUEUUE LENTH....{}".format(i))
crop_img_g = draw[100:600, 150:330].copy()
else:
print("QUEUUE LENTH....{}".format(i))
crop_img_g = draw[100:600, 150:330].copy()
crop_img_gg = preprocess_image(crop_img_g)
crop_img_gg, scale = resize_image(crop_img_gg)
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(crop_img_gg, axis=0))
count1 = sum((scores[0] > 0.5) * (labels[0] == 0))
time_.append(time_sec)
people_.append(count1)
db.connection(count1, queue)
db.mysql_connection(count1, queue)
print(count1)
queu.append(count1)
ps.pushser(count1, queue)
start = start + 1
time.sleep(1)
boxes /= scale
for box, score, label in zip(boxes[0], scores[0], labels[0]):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
if label == 0:
draw_box(crop_img_g, b, color=color)
if count1 > 5:
fontColor = fontColor1
else:
fontColor = fontColor2
cv2.putText(crop_img_g, str(count1), bottomLeftCornerOfText,
font, fontScale, fontColor, lineType)
cv2.waitKey(1)
time.sleep(1)
cv2.destroyAllWindows()
except KeyboardInterrupt:
print("Queue")
except IndexError as er:
print("The Exception is {}".format(er))
ps.pushinputerror("Q3")
def detect(score_threshold):
# model_path = os.path.join('.', 'snapshots', 'model_save', 'resnet152_' + str(cross_id) + '_pascal'+'.h5')
model_path = "/home/ustc/jql/x-ray/keras-retinanet/snapshots/model_save/resnet152_5_pascal.h5"
model = models.load_model(model_path,
backbone_name='resnet152',
convert=True)
test_img_file = '/home/ustc/jql/JSRT/normal.txt'
test_img_path = '/home/ustc/jql/JSRT/JPEGImages/'
test_list = []
with open(test_img_file, 'r') as f:
for line in f:
name = line.split('.')[0]
test_list.append(name)
result_path = '/home/ustc/jql/x-ray/' + 'jsrt_normal' + str(
score_threshold)
if not os.path.exists(result_path):
os.makedirs(result_path)
count = 0
for img_name in test_list:
img_path = test_img_path + img_name + '.jpg'
img = read_image_bgr(img_path)
print(img_name)
# copy to draw on
draw = img.copy()
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
start = time.time()
# preprocess image for network
img = preprocess_image(img)
img, scale = resize_image(img)
# process image
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(img, axis=0))
# correct for image scale
boxes /= scale
nodule_boxes = []
color = (0, 0, 0)
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < score_threshold:
break
nodule_boxes.append(box)
nodule_boxes = np.asarray(nodule_boxes)
if nodule_boxes.shape[0] > 0:
nodule_boxes = nms(nodule_boxes, 0.1)
count += len(nodule_boxes)
for box in nodule_boxes:
b = box.astype(int)
draw_box(draw, b, color=color, thickness=10)
print("processing time: ", time.time() - start)
cv2.imwrite(result_path + '/' + img_name + '.jpg', draw)
return count
def detect(self, image):
self.time2store = self.gen_datetime()
# self.time2store = datetime.now()
self.cen_x = image.shape[1]//2
self.cen_y = image.shape[0]//2
# copy to draw on
draw = image.copy()
# preprocess image for network
image = preprocess_image(image)
# cv2.imshow('ss22', image)
image, scale = resize_image(
image, min_side=self.min_side4train, max_side=self.max_side4train)
time_ = self.time2store
# time_ = datetime.now()
image_id = time_.strftime('%Y%m-%d%H-%M%S-') + str(uuid4())
# process image
start = time.time()
boxes, scores, labels = self.model.predict_on_batch(
np.expand_dims(image, axis=0))
processing_time = time.time() - start
# print("processing time: ", processing_time)
img4elas, scale4elas = resize_image(
draw, min_side=self.min_side4elas, max_side=self.max_side4elas)
# correct for image scale
boxes /= scale
found_ = {}
main_body = {'image_id': image_id, 'time_': time_}
# visualize detections
# print('es_mode: {}\nstatus: {}'.format(self.es_mode, self.es_status))
temp_data = []
index = 1
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
# print(index, box, score, self.labels_to_names[label])
if score < self.confThreshold:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
caption = "{} {:.3f}".format(
self.labels_to_names[label], score)
# print(caption)
draw_caption(draw, b, caption)
temp_data.append([self.labels_to_names[label],
score, b, processing_time])
box = [np.ushort(x).item() for x in box]
# cv2.imshow(str(self.model_is), draw )
# cv2.waitKey()
# if self.es_mode and self.es_status:
# self.es.elas_record(label=label, score=np.float32(score).item(), box=box, image_id=image_id, time_=time_)
if self.es_mode and self.es_status and self.labels_to_names[label] == 'pigeon':
# print('{tag}\n\n{data}\n\n{tag}'.format(
# tag='#'*20,
# data='label: {label}\nscore: {score}'.format(
# label=self.labels_to_names[label], score=score)
# ))
self.es.elas_record(label=label, score=np.float32(
score).item(), box=box, **main_body)
index += 1
try:
found_[self.labels_to_names[label]] += 1
except:
found_[self.labels_to_names[label]] = 1
# os.makedirs("data4eval/non-pigeon/result/{}".format(self.model_is), exist_ok=True)
# cv2.imwrite("data4eval/non-pigeon/result/{}/{}-{}.jpg".format(self.model_is, self.model_is, datetime.now(), image_id), draw)
try:
if self.es_mode and self.es_status and found_['pigeon'] > 0:
print('{tag}\n\n{data}\n\n{tag}'.format(
tag='#'*50,
data='id: {id}\nbird count: {found}'.format(
id=image_id, found=found_)))
self.es.elas_image(image=img4elas, scale=scale, found_=found_,
processing_time=processing_time, **main_body)
# self.es.elas_date(**main_body)
except Exception as e:
print(e)
return processing_time
def run_analysis(filename):
"""
Run analysis on the provided file
Parameters
---------
filename: String
The wall placement strategies the user has provided
"""
parts = filename.split(".")
model = models.load_model('resnet50_coco_best_v2.1.0.h5',
backbone_name='resnet50')
cap = cv2.VideoCapture(filename)
camera_cut = CameraCut()
team_hists = np.array([[
0.00200573715878, 0.008174931463, 0.01083576065, 0.010703199637,
0.005602396121, 0.003398788058, 0.003046975742, 0.002051472299,
0.001472585509, 0.0004638160026
],
[
0.000803887693714, 0.006247766973138,
0.008450147937586, 0.01134667238931,
0.004603156519655, 0.003117572623929,
0.003535995531379, 0.003621568075172,
0.00310307416, 0.001896991398214
]])
cut_counter = 1
counter = 0
while (cap.isOpened()):
print(counter)
ret, frame = cap.read()
if camera_cut.new_frame(frame):
# If this frame is a camera cut, lose the existing VideoWriter and
# open a new one
if cut_counter > 1:
out.release()
fourcc = cv2.VideoWriter_fourcc('F', 'M', 'P', '4')
out = cv2.VideoWriter('{}_{}.mp4'.format(parts[0],
cut_counter), fourcc,
50.0, (int(cap.get(3)), int(cap.get(4))))
cut_counter += 1
counter = 0
draw = np.array(frame)
if counter % 5 == 0:
# Every 5 frames use retinanet to find players in the frame
processed_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
processed_frame = preprocess_image(processed_frame)
processed_frame, scale = resize_image(processed_frame)
boxes, scores, _ = model.predict_on_batch(
np.expand_dims(processed_frame, axis=0))
boxes /= scale
players = []
confident_boxes = []
teams = []
for box, score in zip(boxes[0], scores[0]):
if score < 0.5:
break
box = box.astype(int)
player_image = draw[box[1]:box[3], box[0]:box[2], :]
confident_boxes.append(box)
player_image = cv2.cvtColor(player_image, cv2.COLOR_RGB2GRAY)
player_hist = np.histogram(player_image, density=True)[0]
dist = np.sum(np.power((team_hists - player_hist), 2), axis=1)
min_index = np.argmin(dist)
teams.append([0, 255, 0])
if min_index == 0 and dist[0] < 2e-05:
teams[-1] = [255, 0, 0]
elif dist[1] < 2e-05:
teams[-1] = [0, 0, 255]
# Plot the most recent detections on the current frame
for box, team in zip(confident_boxes, teams):
draw_box(draw, box, color=team)
# Write the current frame with annotations to the VideoWriter
# import matplotlib.pyplot as plt
# plt.imshow(draw)
# plt.show()
out.write(draw)
counter += 1
cap.release()
out.release()
cv2.destroyAllWindows()
def predict(model, demo_image_folder):
id = np.random.randint(low=0, high=len(test_list))
label_name = test_list[id]
image_name = ''
if (os.path.splitext(label_name)[1] == '.txt'):
image_name = label_name.replace('AllSamples/Label',
'AllSamples/Image').replace(
'.txt', '.jpg')
image_name = image_name.replace('Caffe/Labels', 'Caffe/Images')
else:
image_name = label_name.replace('Labels',
'Images').replace('.json', '.jpg')
labels_to_names = {0: 'bg', 1: 'ped', 2: 'rb', 3: 'train'}
# load image
image = read_image_bgr(image_name)
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
image = preprocess_image(image)
image, scale = resize_image(image)
model = models.convert_model(model,
nms=True,
class_specific_filter=False,
anchor_params=None)
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))
# correct for image scale
boxes /= scale
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < 0.2:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
#plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(draw)
#plt.show()
time_str = strftime("%Y%m%d_%H:%M:%S", gmtime())
#plt.savefig(demo_image_folder+time_str+".jpg")
scipy.misc.imsave(demo_image_folder + time_str + ".jpg", draw)
output = io.BytesIO()
plt.savefig(output, format='png')
image_string = output.getvalue()
output.close()
plt.close()
img_width, img_height = draw.shape[0], draw.shape[1]
tf_im = tf.Summary.Image(height=img_height,
width=img_width,
colorspace=3,
encoded_image_string=image_string)
return tf_im
def segmentation(imgpath, score_threshold=0.5, binarize_threshold=0.5):
# adjust this to point to your downloaded/trained model
model_path = "resnet50_weights/resnet50_coco_v0.2.0.h5"
# load retinanet model
model = models.load_model(model_path, backbone_name='resnet50')
# load label to names mapping for visualization purposes
labels_to_names = {
0: 'person',
1: 'bicycle',
2: 'car',
3: 'motorcycle',
4: 'airplane',
5: 'bus',
6: 'train',
7: 'truck',
8: 'boat',
9: 'traffic light',
10: 'fire hydrant',
11: 'stop sign',
12: 'parking meter',
13: 'bench',
14: 'bird',
15: 'cat',
16: 'dog',
17: 'horse',
18: 'sheep',
19: 'cow',
20: 'elephant',
21: 'bear',
22: 'zebra',
23: 'giraffe',
24: 'backpack',
25: 'umbrella',
26: 'handbag',
27: 'tie',
28: 'suitcase',
29: 'frisbee',
30: 'skis',
31: 'snowboard',
32: 'sports ball',
33: 'kite',
34: 'baseball bat',
35: 'baseball glove',
36: 'skateboard',
37: 'surfboard',
38: 'tennis racket',
39: 'bottle',
40: 'wine glass',
41: 'cup',
42: 'fork',
43: 'knife',
44: 'spoon',
45: 'bowl',
46: 'banana',
47: 'apple',
48: 'sandwich',
49: 'orange',
50: 'broccoli',
51: 'carrot',
52: 'hot dog',
53: 'pizza',
54: 'donut',
55: 'cake',
56: 'chair',
57: 'couch',
58: 'potted plant',
59: 'bed',
60: 'dining table',
61: 'toilet',
62: 'tv',
63: 'laptop',
64: 'mouse',
65: 'remote',
66: 'keyboard',
67: 'cell phone',
68: 'microwave',
69: 'oven',
70: 'toaster',
71: 'sink',
72: 'refrigerator',
73: 'book',
74: 'clock',
75: 'vase',
76: 'scissors',
77: 'teddy bear',
78: 'hair drier',
79: 'toothbrush'
}
# load image
image = read_image_bgr(imgpath)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("Segmentation took", round(time.time() - start, 2), "seconds.")
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
mask_list = []
box_list = []
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
# save box coordinates in list
box = box.astype(np.int16)
box_list.append(box[:])
# resize to fit the box
mask = mask.astype(np.float32)
mask = cv2.resize(mask, (box[2] - box[0], box[3] - box[1]))
# binarize the mask
mask = (mask > binarize_threshold).astype(np.uint8)
mask = cv2.normalize(src=mask,
dst=None,
alpha=0,
beta=255,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_8U)
mask_list.append(mask[:, :, label])
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
return mask_list, box_list, draw
def cloth_identifier_extract_text(input_imagefile):
# load image
image = read_image_bgr(input_imagefile)
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
outputs1 = model.predict_on_batch(np.expand_dims(image, axis=0))
boxes = outputs1[-4][0]
scores = outputs1[-3][0]
labels = outputs1[-2][0]
masks = outputs1[-1][0]
# correct for image scale
boxes /= scale
masks_dic = {}
boxes_dic = {}
labels_dic = {}
counter = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < 0.5:
break
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
masks_dic[str(counter)] = mask
boxes_dic[str(counter)] = box
labels_dic[str(counter)] = label
counter += 1
items_dic = {}
counter = 0
masks = []
for box, mask, label in zip(boxes_dic.values(), masks_dic.values(),
labels_dic.values()):
b = box.astype(int)
# resize to fit the box
mask = mask.astype(np.float32)
mask = cv2.resize(mask, (b[2] - b[0], b[3] - b[1]))
# binarize the mask
mask = (mask > 0.5).astype(np.uint8)
# draw the mask in the image
mask_image = np.zeros((draw.shape[0], draw.shape[1]), np.uint8)
mask_image[b[1]:b[3], b[0]:b[2]] = mask
mask = mask_image
mask = (np.stack([mask] * 3, axis=2)) * draw
items_dic[str(counter)] = mask
counter += 1
return mask, label
def main():
print('image_lisnter')
rospy.init_node('image_listener')
# Define your image topic
image_topic = "/usb_cam/image_raw"
# Set up your subscriber and define its callback
rospy.Subscriber(image_topic, Image, image_callback, queue_size=1)
image_pub = rospy.Publisher("/output/image_raw", Image)
label_pub = rospy.Publisher("/detection/label", label_frame, queue_size=1)
label_sequence = 0
model = models.load_model(
'/home/colson/catkin_ws/src/ros_keras_retinanet/model/resnet50_coco_best_v2.1.0.h5',
backbone_name='resnet50')
#model = models.load_model('/home/colson/catkin_ws/src/ros-keras-retinanet/model/resnet_50_pascal_12_inference.h5', backbone_name='resnet50')
#model = models.load_model('/home/colson/catkin_ws/src/ros-keras-retinanet/model/mobilenet128_1.0_pascal_06_inference.h5', backbone_name='mobilenet128')
#model = models.load_model('/home/colson/catkin_ws/src/ros-keras-retinanet/model/vgg19_pascal_14_inference.h5', backbone_name='vgg19')
#model = models.load_model('/home/colson/catkin_ws/src/ros_keras_retinanet/model/resnet50_pascal_14_inference.h5', backbone_name='resnet50')
labels_to_names = {
0: 'person',
1: 'bicycle',
2: 'car',
3: 'motorcycle',
4: 'airplane',
5: 'bus',
6: 'train',
7: 'truck',
8: 'boat',
9: 'traffic light',
10: 'fire hydrant',
11: 'stop sign',
12: 'parking meter',
13: 'bench',
14: 'bird',
15: 'cat',
16: 'dog',
17: 'horse',
18: 'sheep',
19: 'cow',
20: 'elephant',
21: 'bear',
22: 'zebra',
23: 'giraffe',
24: 'backpack',
25: 'umbrella',
26: 'handbag',
27: 'tie',
28: 'suitcase',
29: 'frisbee',
30: 'skis',
31: 'snowboard',
32: 'sports ball',
33: 'kite',
34: 'baseball bat',
35: 'baseball glove',
36: 'skateboard',
37: 'surfboard',
38: 'tennis racket',
39: 'bottle',
40: 'wine glass',
41: 'cup',
42: 'fork',
43: 'knife',
44: 'spoon',
45: 'bowl',
46: 'banana',
47: 'apple',
48: 'sandwich',
49: 'orange',
50: 'broccoli',
51: 'carrot',
52: 'hot dog',
53: 'pizza',
54: 'donut',
55: 'cake',
56: 'chair',
57: 'couch',
58: 'potted plant',
59: 'bed',
60: 'dining table',
61: 'toilet',
62: 'tv',
63: 'laptop',
64: 'mouse',
65: 'remote',
66: 'keyboard',
67: 'cell phone',
68: 'microwave',
69: 'oven',
70: 'toaster',
71: 'sink',
72: 'refrigerator',
73: 'book',
74: 'clock',
75: 'vase',
76: 'scissors',
77: 'teddy bear',
78: 'hair drier',
79: 'toothbrush'
}
# Spin until ctrl + c
#rospy.spin()
while not rospy.is_shutdown():
global is_running
global image
if is_running == 1:
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
image = preprocess_image(image)
#image, scale = resize_image(image)
scale = 1
# process image
start = time.time()
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
# correct for image scale
boxes /= scale
box_data = label_frame()
label_sequence = label_sequence + 1
box_data.seq = label_sequence
box_data.x_axis = image.shape[1]
box_data.y_axis = image.shape[0]
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < 0.3:
break
color = label_color(label)
b = box.astype(float)
x1 = b[0]
y1 = b[1]
x2 = b[2]
y2 = b[3]
d = [0] * 4
d[0] = x1
d[1] = y1
d[2] = x2
d[3] = y2
box_data.data.extend(d)
box_data.label_num = box_data.label_num + 1
print(box_data.data)
draw_box(draw, b, color=color)
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
# print(draw)
# print(draw.size)
# print(draw.shape)
label_pub.publish(box_data)
draw = draw.astype(np.uint8)
msg = CvBridge().cv2_to_imgmsg(draw, encoding="rgb8")
image_pub.publish(msg)
# draw = cv2.cvtColor(draw, f
# plt.figure(figsize=(15, 15))
# plt.axis('off')
# plt.imshow(draw)
# plt.show()
is_running = 0
def process_video(current_file):
with open("./train_annotations/{}.json".format(current_file)) as f:
train_json = json.load(f)
with open("./1_baseline/src/{}.preds.json".format(current_file)) as f:
train_pred_json = json.load(f)
cap = cv2.VideoCapture("./train_videos/{}.mp4".format(current_file))
w = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
h = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
fps = cap.get(cv2.CAP_PROP_FPS)
print("fps = {}".format(fps))
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
out = cv2.VideoWriter(
"./train_videos/{}_tracking_output.mp4".format(current_file), fourcc,
fps, (int(w), int(h)))
# out.set(cv2.CAP_PROP_FPS, fps)
# Check if camera opened successfully
if (cap.isOpened() == False):
print("Error opening video stream or file")
frame_count = 0
id_cache = {}
# Read until video is completed
while (cap.isOpened()):
if frame_count % 100 == 0:
print("Frames processed {}".format(frame_count))
# Capture frame-by-frame
ret, image = cap.read()
if ret == True:
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# # visualize ground truth
# if 'Pedestrian' in train_json['sequence'][frame_count].keys():
# pedestrians_list = train_json['sequence'][frame_count]['Pedestrian']
# for box in pedestrians_list:
# b = box['box2d']
# area = (b[2]-b[0])*(b[3]-b[1])
# if (area >= 1024):
# b = list(map(int, b))
# draw_box(image, b, color=(0, 0, 0))
# #draw_caption(image, b, str(box["id"]))
# if 'Car' in train_json['sequence'][frame_count].keys():
# car_list = train_json['sequence'][frame_count]['Car']
# for box in car_list:
# b = box['box2d']
# b = list(map(int, b))
# area = (b[2]-b[0])*(b[3]-b[1])
# if (area >= 1024):
# draw_box(image, b, color=(0, 0, 0))
# #draw_caption(image, b, str(box["id"]))
# visualize predictions
if 'Pedestrian' in train_pred_json['{}.mp4'.format(
current_file)][frame_count].keys():
pedestrians_list = train_pred_json['{}.mp4'.format(
current_file)][frame_count]['Pedestrian']
for box in pedestrians_list:
b = box['box2d']
b = list(map(int, b))
if (box['id'] not in id_cache.keys()):
col = random_color()
id_cache[box['id']] = col
else:
col = id_cache[box['id']]
draw_box(image, b, color=col)
draw_caption(image, b, str(box["id"]))
if 'Car' in train_pred_json['{}.mp4'.format(
current_file)][frame_count].keys():
car_list = train_pred_json['{}.mp4'.format(
current_file)][frame_count]['Car']
for box in car_list:
b = box['box2d']
b = list(map(int, b))
if (box['id'] not in id_cache.keys()):
col = random_color()
id_cache[box['id']] = col
else:
col = id_cache[box['id']]
draw_box(image, b, color=col)
draw_caption(image, b, str(box["id"]))
frame_count = frame_count + 1
# write the resulting frame
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
out.write(image)
# Break the loop
else:
break
# When everything done, release the video capture object
cap.release()
# Closes all the frames
cv2.destroyAllWindows()
# write the file
out.release()
def model_predict(self, simg):
img = read_image_bgr(simg.imgpil)
timg = img.copy()
drawimg = cv2.cvtColor(img.copy(), cv2.COLOR_BGR2RGB)
# preprocess
img = preprocess_image(img)
# Json for predictions
data = {"source": simg.imgpath}
# cropped images
cimg = []
# process image
start = time.time()
boxes, scores, labels = self.model.predict_on_batch(pandas.np.expand_dims(img, axis=0))
# count animals
count = 0
zebra_count = 0
wildebeest_count = 0
gazelle_count = 0
# Visualize Detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted
if score < THRESHOLD_SCORE:
break
lcolor = label_color(label)
b = box.astype(int)
draw_box(drawimg, b, color=lcolor)
#crop image
cropped = timg[b[1]:b[3], b[0]:b[2]]
# Get the correct label and save
# Get the correct label
if label == 0:
animal = 'wildebeest'
wildebeest_count = wildebeest_count + 1
cfilename = './static/uploads/detections/wildebeest/' + simg.imgname + animal + str(count) + '.jpg'
cv2.imwrite(cfilename, cropped)
elif label == 1:
animal = 'zebra'
zebra_count = zebra_count + 1
cfilename = './static/uploads/detections/zebra/' + simg.imgname + animal + str(count) + '.jpg'
cv2.imwrite(cfilename, cropped)
elif label == 2:
animal = 'gazelleThomsons'
gazelle_count = gazelle_count + 1
cfilename = './static/uploads/detections/gazelleThomsons/' + simg.imgname + animal + str(count) + '.jpg'
cv2.imwrite(cfilename, cropped)
else:
animal = label
cfilename = './static/uploads/detections/' + simg.imgname + animal + str(count) + '.jpg'
cv2.imwrite(cfilename, cropped)
count = count + 1
caption = "{} {:.3f}".format(self.labelstonames[label], score)
draw_caption(drawimg, b, caption)
# create list of detected animals
cimg.append({
"img_src": cfilename,
"label": animal
})
data["wcount"] = wildebeest_count
data["zcount"] = zebra_count
data["gcount"] = gazelle_count
data["prediction"] = cimg
print(data)
detected_img = Image.fromarray(drawimg.astype('uint8'), 'RGB')
detected_img.save("./static/uploads/source/" + simg.imgname +"detected.jpg")
data["detected"] = './static/uploads/source/' + simg.imgname +'detected.jpg'
return data
# process image
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))
# correct for image scale
boxes /= scale
num_org = 0
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < threshold:
break
num_org = num_org + 1
b = box.astype(int)
draw_box(draw, b, color=(255, 0, 255))
st.image(draw, use_column_width=True)
st.write("Image name:", imagelist[sample_image])
st.write("Number of organoids detected:", num_org)
# Batch process from here on
st.sidebar.subheader('Batch Processing')
if st.sidebar.button("Process All"):
progress_bar = st.sidebar.progress(0)
for i, filename in enumerate(imagelist):
try:
#IMAGE_PATH = os.path.join(root,filename)
IMAGE_PATH = filename
# load image
def predict_tile(self,
raster_path=None,
numpy_image=None,
patch_size=400,
patch_overlap=0.15,
iou_threshold=0.15,
return_plot=False):
"""
For images too large to input into the model, predict_tile cuts the image into overlapping windows, predicts trees on each window and reassambles into a single array.
Args:
raster_path: Path to image on disk
numpy_image (array): Numpy image array in BGR channel order following openCV convention
iou_threshold: Minimum iou overlap among predictions between windows to be suppressed. Defaults to 0.5. Lower values suppress more boxes at edges.
return_plot: Should the image be returned with the predictions drawn?
Returns:
boxes (array): if return_plot, an image. Otherwise a numpy array of predicted bounding boxes, scores and labels
"""
if numpy_image:
pass
else:
#Load raster as image
raster = Image.open(raster_path)
numpy_image = np.array(raster)
#Compute sliding window index
windows = preprocess.compute_windows(numpy_image, patch_size,
patch_overlap)
#Save images to tmpdir
predicted_boxes = []
for index, window in enumerate(windows):
#Crop window and predict
crop = numpy_image[windows[index].indices()]
#Crop is RGB channel order, change to BGR
crop = crop[..., ::-1]
boxes = self.predict_image(
numpy_image=crop,
return_plot=False,
score_threshold=self.config["score_threshold"])
#transform coordinates to original system
xmin, ymin, xmax, ymax = windows[index].getRect()
boxes.xmin = boxes.xmin + xmin
boxes.xmax = boxes.xmax + xmin
boxes.ymin = boxes.ymin + ymin
boxes.ymax = boxes.ymax + ymin
predicted_boxes.append(boxes)
predicted_boxes = pd.concat(predicted_boxes)
#Non-max supression for overlapping boxes among window
if patch_overlap == 0:
mosaic_df = predicted_boxes
else:
with tf.Session() as sess:
print(
"{} predictions in overlapping windows, applying non-max supression"
.format(predicted_boxes.shape[0]))
new_boxes, new_scores, new_labels = predict.non_max_suppression(
sess,
predicted_boxes[["xmin", "ymin", "xmax", "ymax"]].values,
predicted_boxes.score.values,
predicted_boxes.label.values,
max_output_size=predicted_boxes.shape[0],
iou_threshold=iou_threshold)
#Recreate box dataframe
image_detections = np.concatenate([
new_boxes,
np.expand_dims(new_scores, axis=1),
np.expand_dims(new_labels, axis=1)
],
axis=1)
mosaic_df = pd.DataFrame(
image_detections,
columns=["xmin", "ymin", "xmax", "ymax", "score", "label"])
mosaic_df.label = mosaic_df.label.str.decode("utf-8")
print("{} predictions kept after non-max suppression".format(
mosaic_df.shape[0]))
if return_plot:
#Draw predictions
for box in mosaic_df[["xmin", "ymin", "xmax", "ymax"]].values:
draw_box(numpy_image, box, [0, 0, 255])
#Mantain consistancy with predict_image
return numpy_image
else:
return mosaic_df
def main(proc_img_path=None, proc_img_url=None, all_set=True, save_path=None, model_path=None,
segments=False, annotations=False, threshold_score=0.5, limit=None, model=None, labels_to_names=None):
# import keras
import keras
# import keras_retinanet
from keras_maskrcnn import models
from keras_maskrcnn.utils.visualization import draw_mask
from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations
from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image
from keras_retinanet.utils.colors import label_color
# import miscellaneous modules
import matplotlib.pyplot as plt
import cv2
import numpy as np
import time
# set tf backend to allow memory to grow, instead of claiming everything
import tensorflow as tf
# use this environment flag to change which GPU to use
#os.environ["CUDA_VISIBLE_DEVICES"] = "1"
with open(os.path.expanduser('~')+ '/.maskrcnn-modanet/' + 'savedvars.json') as f:
savedvars = json.load(f)
path = savedvars['datapath']
img_path = path + "datasets/coco/images/"
if not model:
# set the modified tf session as backend in keras
keras.backend.tensorflow_backend.set_session(get_session())
# adjust this to point to your trained model
# get all models names in the results folder
modelnames = [f for f in os.listdir(snp_path) if os.path.isfile(os.path.join(snp_path, f))]
import re
def extract_number(f):
s = re.findall("\d+$",f)
return (int(s[0]) if s else -1,f)
# get the model name with the highest epoch
print(max(modelnames,key=extract_number))
model_path = os.path.join(snp_path, max(modelnames,key=extract_number))
# load retinanet model
model = models.load_model(model_path, backbone_name='resnet50')
if not labels_to_names:
# load label to names mapping for visualization purposes
labels_to_names = {0: 'bag', 1: 'belt', 2: 'boots', 3: 'footwear', 4: 'outer', 5: 'dress', 6: 'sunglasses', 7: 'pants', 8: 'top', 9: 'shorts', 10: 'skirt', 11: 'headwear', 12: 'scarf/tie'}
default_save_path = False
if save_path == 'default':
# set path to default
save_path = path + 'results/processedimages/'#images/1.jpg'
if not annotations:
save_path += 'images/'
elif annotations:
save_path += 'annotations/'
default_save_path = True
SAVE_PATH = save_path # used for multiple images
if annotations:
# if save_path: save_path = path + 'results/processedimages/annotations/1.json'
annotations = [{
'bbox': None,
'score': None,
'category': None,
'part' : None
}]
if all_set:
# load images
with open(ann_path + 'instances_val.json') as f:
instances = json.load(f)
images = instances['images']
for img in images:
img['file_name'] = img_path + img['file_name']
elif proc_img_path:
# just draw the image selected
images = [{'file_name': img_path + proc_img_path if os.path.abspath(proc_img_path) != proc_img_path else proc_img_path}]
elif proc_img_url:
# just draw the image selected
images = [{'file_name': proc_img_url}]
try:
#for each image in the dataset
for i, img in enumerate(images):
print(i, end=' ')
if limit and i >= limit:
break
if all_set:
image = read_image_bgr(img['file_name'])
elif proc_img_path:
image = read_image_bgr(img['file_name'])
elif proc_img_url:
import requests
from io import BytesIO
r = requests.get(img['file_name'], allow_redirects=True)
image = read_image_bgr(BytesIO(r.content))
if save_path:
if proc_img_path or all_set:
img_file_name = img['file_name'].split("/")[-1]
elif proc_img_url:
img_file_name = 'urlimg.jpg'
if not annotations:
save_path += img_file_name
elif annotations:
save_path += img_file_name.split('.')[0] + '.json'
if save_path and segments and not annotations:
#remove the extension
save_path = save_path.split('.')[0]
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start, "\t(Ctrl+c and close image to exit)")
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
if annotations:
annotations = [{
'bbox': None,
'score': None,
'category': None,
'part' : None
} for i in range(len([score for score in scores if score >= threshold_score]))]
segment_id = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < threshold_score:
break
color = label_color(label)
if not segments:
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
elif segments:
drawclone = np.copy(draw)
b = box.astype(int)
# draw_box(drawclone, b, color=color)
mask = mask[:, :, label]
draw_mask_only(drawclone, b, mask, color=label_color(label))
if not annotations:
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(drawclone, b, caption)
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(drawclone)
if not save_path:
plt.show()
elif save_path:
segment_path = '_segment_' + segment_id + '.jpg'
save_path_segment = save_path + segment_path
print(save_path_segment)
plt.savefig(save_path_segment)
plt.close()
elif annotations:
annotations[segment_id]['bbox'] = b
annotations[segment_id]['score'] = score
annotations[segment_id]['category'] = label
annotations[segment_id]['part'] = drawclone # only the object inside the mask is shown, the rest is black
segment_id += 1
if not segments:
if not save_path:
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(draw)
if not proc_img_url:
print(img['file_name'])
plt.show()
elif save_path:
processed_image = Image.fromarray(draw, 'RGB')
processed_image.save(save_path)
del processed_image
print(save_path)
# plt.savefig(save_path)
# plt.close()
elif segments:
if annotations:
if save_path:
print(save_path)
with open(save_path, 'w') as outfile:
json.dump(annotations, outfile)
else:
return annotations
save_path = SAVE_PATH # restore path for next image
except KeyboardInterrupt:
pass
def detect(self):
global i
i = str(self.camara.currentText())
print(i)
i = int(i)
import numpy as np
import pandas as pd
import seaborn as sns
from pylab import rcParams
import matplotlib.pyplot as plt
from matplotlib import rc
from pandas.plotting import register_matplotlib_converters
from sklearn.model_selection import train_test_split
import urllib
import os
import csv
import cv2
import time
from PIL import Image
from keras_retinanet import models
from keras_retinanet.utils.image import read_image_bgr, preprocess_image, resize_image
from keras_retinanet.utils.visualization import draw_box, draw_caption
from keras_retinanet.utils.colors import label_color
# Cargamos el modelo
# debe estar en la carpeta /snapshots/
from keras.models import load_model
from keras_retinanet import models
model_path = os.path.join(
'snapshots',
sorted(os.listdir('snapshots'), reverse=True)[0])
print(model_path)
model = models.load_model(model_path, backbone_name='resnet50')
model = models.convert_model(model)
labels_to_names = pd.read_csv('classes.csv',
header=None).T.loc[0].to_dict()
# Función que realizaz la predicción
import skimage.io as io
def predict(image):
image = preprocess_image(image.copy())
image, scale = resize_image(image)
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))
boxes /= scale
return boxes, scores, labels
# Captura de Video
camera = cv2.VideoCapture(i)
camera_height = 500
while (True):
_, frame = camera.read()
frame = cv2.flip(frame, 1)
boxes, scores, labels = predict(frame)
draw = frame.copy()
for box, score, label in zip(boxes[0], scores[0], labels[0]):
if score > 0.8:
print(box)
b = box.astype(int)
color = label_color(label)
draw_box(draw, b, color=color)
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
if label == 0:
self.resultado.setText(
'se ha detectado : rasberry pi4')
if label == 1:
self.resultado.setText('se ha detectado :protoboar')
# show the frame
cv2.imshow("Test out", draw)
detec = label
key = cv2.waitKey(1)
# quit camera if 'q' key is pressed
if key & 0xFF == ord("q"):
break
camera.release()
cv2.destroyAllWindows()
def apply_mask(model, image, draw=None, threshold_score=0.5, labels_to_names=None, image_segments=True):
''' Process image numpy matrix using model and return the annotations
use draw if you want to draw over the image. the draw parameter is the image in RGB (image is in BGR instead) '''
from keras_retinanet.utils.image import preprocess_image, resize_image
from keras_retinanet.utils.colors import label_color
from keras_maskrcnn.utils.visualization import draw_mask
from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations
# import miscellaneous modules
import numpy as np
import time
if not labels_to_names:
# load label to names mapping for visualization purposes
labels_to_names = {0: 'bag', 1: 'belt', 2: 'boots', 3: 'footwear', 4: 'outer', 5: 'dress', 6: 'sunglasses', 7: 'pants', 8: 'top', 9: 'shorts', 10: 'skirt', 11: 'headwear', 12: 'scarf/tie'}
if not draw.any():
# copy to draw on
draw = image.copy()
#switching to RGB from BGR
draw[:, :, 2] = image[:, :, 0]
draw[:, :, 0] = image[:, :, 2]
draw_segment = draw.copy()
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
start = time.time()
outputs = model.predict_on_batch(np.expand_dims(image, axis=0))
print("processing time: ", time.time() - start)
boxes = outputs[-4][0]
scores = outputs[-3][0]
labels = outputs[-2][0]
masks = outputs[-1][0]
# correct for image scale
boxes /= scale
annotations = [{
'bbox': None,
'score': None,
'category': None,
'segment' : None
} for i in range(len([score for score in scores if score >= threshold_score]))]
i = 0
# visualize detections
for box, score, label, mask in zip(boxes, scores, labels, masks):
if score < threshold_score:
break
color = label_color(label)
drawclone = np.copy(draw_segment)
b = box.astype(int)
draw_box(draw, b, color=color)
mask = mask[:, :, label]
draw_mask_only(drawclone, b, mask, color=label_color(label))
draw_mask(draw, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
annotations[i]['bbox'] = [b[0],b[1],b[2]-b[0],b[3]-b[1]]
annotations[i]['score'] = score
annotations[i]['category'] = labels_to_names[label]
if image_segments:
annotations[i]['segment'] = drawclone # only the object inside the mask is shown, the rest is black
i += 1
return annotations
if labels_to_names[label] == 'goodshoes':
label = 3
else:
label = 4
foots.append([box, label, score])
for person in people:
# if ispass(person, heads, foots) == {'head': True, 'foot': True}:
if ispass(person, heads, foots)['head'] > 0 and ispass(person, heads, foots)['foot'] > 0:
color = (0, 255, 0)
caption = 'pass'
else:
color = (255, 0, 0)
caption = 'not pass'
b = person.astype(int)
draw_box(draw, b, color=color)
draw_caption(draw, b, caption)
for instance in (*heads, *foots):
color = [(0, 255, 128), (255, 0, 128), (128, 255, 0), (255, 128, 0), ][int(instance[1]) - 1]
caption = ['goodhelmet', 'badhelmet', 'goodshoes', 'badshoes', ][int(instance[1]) - 1]
b = instance[0].astype(int)
draw_box(draw, b, color=color)
draw_caption(draw, b, caption)
a = 0
for plate in plates:
x1, y1, x2, y2 = plate.astype(int)
caption = 'LP'
if x1 < x2 and y1 < y2:
boxes = [
list(
map(int, (line.split()[3], line.split()[2], line.split()[5],
line.split()[4])))
for line in open('C:\\Users\\Pawan\\Music\\Paku5\\' + id +
'.txt', 'r').readlines()
]
scores = [
float(line.split()[1])
for line in open('C:\\Users\\Pawan\\Music\\Paku5\\' + id +
'.txt', 'r').readlines()
]
labels = [
int(line.split()[0]) - 1
for line in open('C:\\Users\\Pawan\\Music\\Paku5\\' + id +
'.txt', 'r').readlines()
]
print("scores is :::::", scores)
for box, score, label in zip(boxes, scores, labels):
if score < score_thres:
break
color = label_color(label)
draw_box(draw, box, color=color, thickness=1)
caption = "{:.3f}".format(score)
draw_caption(draw, box, caption)
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(draw)
plt.show()
break
def predict_first():
global frames, frames_ptr, prev_frames_ptr, frameNumber
if vidcap.isOpened():
# Acquire frame and expand frame dimensions to have shape: [1, None, None, 3]
# i.e. a single-column array, where each item in the column has the pixel RGB value
success, image = vidcap.read()
if success:
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
# preprocess image for network
image = preprocess_image(image)
image, scale = resize_image(image)
# process image
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))
# correct for image scale
boxes /= scale
# init SI containers for this frame
frame = {0:[], 1:[], 2:[], 3:[], 4:[], 5:[], 6:[], 7:[], 8:[], 9:[], 10:[], 11:[], 12:[]}
# visualize detections
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted so we can break
if score < min_SI_scores[label]:
break
i_box = np.array([int(box[0]), int(box[1]), int(box[2]), int(box[3])])
i_center = box_center(i_box)
i_score = int(score * 100)
frame[label].append({'id': -1, 'trk_cnt': 0, 'link_id': -1, 'link_cnt': 0, 'box':i_box,
'scr':i_score, 'cen':i_center, 'cen_pred':[float('nan'), float('nan')],
'vel':[0, 0], 'accel':[0, 0], 'adj_glove': -1, 'adj_obs': -1, 'in': '-', 'cov':0})
color = label_color(label)
b = box.astype(int)
draw_box(draw, b, color=color)
caption = "{} {:.3f}".format(labels_to_names[label], score)
draw_caption(draw, b, caption)
#resized = cv2.resize(draw, (1024, 1024))
# All the results have been drawn on the frame, so it's time to display it.
#cv2.imshow('Object detector', resized)
# add SIs to the frames
if frames_ptr == 10:
frames_ptr = 0
prev_frames_ptr = 9
else:
prev_frames_ptr = frames_ptr - 1
frames[frames_ptr] = frame
# assign id/link id: correlate SIs from consecutive frames, link enclosing SIs (glove & forceps)
track_SIs()
pred_info = {'si':'', 'id':'', 'box':'', 'scr':'', 'cen':'', 'vel':'', 'misc':''}
for si in frames[frames_ptr]:
for si_item in frames[frames_ptr][si]:
pred_info['si'] += "{} {}/{}\n".format(labels_to_names[si], si_item['in'], si_item['cov'])
pred_info['id'] += "{}/{}\n".format(si_item['id'], si_item['trk_cnt'])
pred_info['box'] += "{:4d},{:4d},{:4d},{:4d}\n".format(si_item['box'][0],
si_item['box'][1], si_item['box'][2], si_item['box'][3])
pred_info['scr'] += "{:3d}\n".format(si_item['scr'])
pred_info['cen'] += "{},{}\n".format(si_item['cen'][0], si_item['cen'][1])
pred_info['vel'] += "{:.0f},{:.0f}/{:.0f},{:.0f}\n".format(si_item['vel'][0], si_item['vel'][1],
si_item['accel'][0], si_item['accel'][1])
pred_info['misc'] += "{},{}/{}/{}/{}/{}\n".format(si_item['cen_pred'][0], si_item['cen_pred'][1],
si_item['link_id'], si_item['link_cnt'], si_item['adj_glove'], si_item['adj_obs'])
frames_ptr += 1
frameNumber += 1
return draw, pred_info, frameNumber
def viewImages(img_path,
segments,
all_set,
save_path=None,
limit=None,
original=False,
begin_from=0,
anns_path=None):
score = 1
limit = 100
import json
import os
with open(
os.path.expanduser('~') + '/.maskrcnn-modanet/' +
'savedvars.json') as f:
savedvars = json.load(f)
path = savedvars['datapath']
images_path = path + "datasets/coco/images/"
ann_path = path + "datasets/coco/annotations/"
ann_orig_path = path + 'datasets/modanet/annotations/'
snp_path = path + "results/snapshots"
from keras_maskrcnn.utils.visualization import draw_mask
from keras_retinanet.utils.visualization import draw_box, draw_caption, draw_annotations
from keras_retinanet.utils.image import read_image_bgr
from keras_retinanet.utils.colors import label_color
from pycocotools import mask as maskUtils
# from . import MyCOCO
# import miscellaneous modules
import matplotlib.pyplot as plt
import cv2
import numpy as np
import time
coco = MyCOCO(ann_path + 'instances_all.json')
# load annotations
if original:
with open(ann_orig_path + 'modanet2018_instances_train.json') as f:
instances = json.load(f)
title = ann_orig_path + 'modanet2018_instances_train.json'
elif anns_path:
with open(anns_path) as f:
instances = json.load(f)
title = anns_path
else:
with open(ann_path + 'instances_all.json') as f:
instances = json.load(f)
title = ann_path + 'instances_all.json'
titleshort = title.split("/")[-1]
images_ids = {'file_name': 'id'}
#images_filenames = [None] * 1115985
for img in instances['images']:
images_ids[img['file_name']] = img['id']
#images_filenames[img['id']] = img['file_name']
# images_anns contains all the annotations for each image_id.
# the key is the image_id,
# the value is a list of the annotations for that id
images_anns = [[] for i in range(1115985)]
# load label to names mapping for visualization purposes
labels_to_names = {
0: 'bag',
1: 'belt',
2: 'boots',
3: 'footwear',
4: 'outer',
5: 'dress',
6: 'sunglasses',
7: 'pants',
8: 'top',
9: 'shorts',
10: 'skirt',
11: 'headwear',
12: 'scarf/tie'
}
default_save_path = False
if save_path == 'default':
# set path to default
save_path = path + 'results/processedimages/' #images/1.jpg'
if not annotations:
save_path += 'images/'
elif annotations:
save_path += 'annotations/'
default_save_path = True
SAVE_PATH = save_path # used for multiple images
# now the juicy part: loading the image/s.
# if img_path, ask again at the end to save time if you want to see multiple files quickly.
while True:
if all_set:
# load images
images = instances['images']
for ann in instances['annotations']:
images_anns[ann['image_id']].append(ann)
elif img_path:
# just draw the image selected
img_id = images_ids[img_path]
for img in instances['images']:
if img['file_name'] == img_path:
images = [img]
for ann in instances['annotations']:
if ann['image_id'] == img_id:
images_anns[img_id].append(ann)
try:
#for each image in the dataset
for i, img in enumerate(images[begin_from:]):
print(i, end=' ')
if limit and i >= limit:
break
image = read_image_bgr(images_path + img['file_name'])
# if default_save_path:
# if img_path or all_set:
# img_file_name = img['file_name'].split("/")[-1]
# save_path += img_file_name
# if save_path and segments:
# #remove the extension
# save_path = save_path.split('.')[0]
# copy to draw on
draw = image.copy()
draw = cv2.cvtColor(draw, cv2.COLOR_BGR2RGB)
img_id = img['id']
plt.figure(num=img['file_name'] + ' – ' + titleshort)
plt.imshow(draw)
plt.axis('off')
plt.tight_layout()
coco.showAnns(images_anns[img_id])
plt.show()
segment_id = 0
# visualize detections
for ann in images_anns[img_id]:
break
box = ann['bbox']
label = ann[
'category_id'] - 1 # they start from 1 in the annotations
segmentation = ann['segmentation']
box[2] += box[0]
box[3] += box[1]
color = label_color(label)
#mask = mask_utils.decode(np.asfortranarray(segmentation))
if not segments:
b = np.array(box)
draw_box(draw, b, color=color)
#draw_mask(draw, b, mask, color=label_color(label))
caption = "{}".format(labels_to_names[label])
draw_caption(draw, b, caption)
elif segments:
drawclone = np.copy(draw)
b = np.array(box)
draw_box(drawclone, b, color=color)
#draw_mask_only(drawclone, b, mask, color=label_color(label))
caption = "{} {:.3f}".format(labels_to_names[label],
score)
draw_caption(drawclone, b, caption)
plt.figure(figsize=(15, 15))
plt.axis('off')
plt.imshow(drawclone)
if not save_path:
plt.show()
elif save_path:
segment_path = '_segment_' + segment_id + '.jpg'
save_path_segment = save_path + segment_path
print(save_path_segment)
plt.savefig(save_path_segment)
plt.close()
segment_id += 1
# if not segments:
# plt.figure(figsize=(15, 15))
# plt.axis('off')
# plt.imshow(draw)
# if not save_path:
# print(img['file_name'])
# plt.show()
# elif save_path:
# print(save_path)
# plt.savefig(save_path)
# plt.close()
save_path = SAVE_PATH # restore path for next image
except KeyboardInterrupt:
pass
if img_path:
img_path = input(
'Enter another image to view (press enter to exit):\n')
else:
break
if not img_path:
break
