class TestConfig(Config):
#NAME = "test"
#NAME ="kangaroo_cfg"
NAME = "man_mov"
GPU_COUNT = 1
IMAGES_PER_GPU = 1
NUM_CLASSES = 1 + 11
#NUM_CLASSES = 1 + 80
global rcnn
global class_id
# define the model
rcnn = MaskRCNN(mode='inference', model_dir='./', config=TestConfig())
# load coco model weights
rcnn.load_weights('mask_rcnn_ocr_0013.h5', by_name=True)
# load class
class_id = ["BG", "0", "1", "2", "3", "4", "5", "6", "7", "8", "A", "C"]
rcnn.keras_model._make_predict_function()
def drawBBox(boxes_list, score, idx):
boxWithCharector = []
for i, box in enumerate(boxes_list):
if score[i] > .80:
boxList = list(box)
boxList.append(class_id[idx[i]])
class PersonSegmentation:
"""Interface for interacting with
pretrained on COCO dataset Mask R-CNN model."""
def __init__(self,
mode="inference",
model_dir="logs",
config=MaskRCNNConfig()):
self.model = MaskRCNN(mode=mode, model_dir=model_dir, config=config)
self.model.load_weights(DATASET_FILE, by_name=True)
self.CLASS_NAMES = [
'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
'train', 'truck', 'boat', 'traffic light', 'fire hydrant',
'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog',
'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat',
'baseball glove', 'skateboard', 'surfboard', 'tennis racket',
'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop',
'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
self.detections = None
self.person_masks = None
self.person_boxes = None
def detect(self, image):
"""Perform instance segmentation on image.
In OpenCV images are stored in BGR mode, so we have to convert it to RGB."""
self.detections = self.model.detect([image[:, :, ::-1]], verbose=1)[0]
self.person_masks = np.array(
self.detections['masks'])[:, :, self.detections['class_ids'] == 1]
self.person_boxes = np.array(
self.detections['rois'])[self.detections['class_ids'] == 1]
self.num_person = self.person_masks.shape[2]
def visualize_detections(self, image, person_only=True):
"""Draw bounding boxes and masks for detected objects on image.
If person_only is true, draw only for "person" objects.
Returns new image. """
COLORS = mrcnn.visualize.random_colors(len(self.CLASS_NAMES))
masks = self.detections['masks']
boxes = self.detections['rois']
class_ids = self.detections['class_ids']
scores = self.detections['scores']
for i in range(boxes.shape[0]):
class_id = class_ids[i]
if person_only and class_id != 1:
continue
y1, x1, y2, x2 = boxes[i]
label = self.CLASS_NAMES[class_id]
font = cv.FONT_HERSHEY_DUPLEX
color = tuple([int(c) for c in np.array(COLORS[class_id]) * 255])
text = "{}: {:.3f}".format(label, scores[i])
size = 0.5
width = 2
mask = masks[:, :, i]
image = mrcnn.visualize.apply_mask(image, mask, color, alpha=0.6)
cv.rectangle(image, (x1, y1), (x2, y2), color, width)
cv.putText(image, text, (x1, y1 - 20), font, size, color, width)
return image
def load_model():
model = MaskRCNN(mode="inference",
config=SimpleConfig(),
model_dir=f'{PATH}/model_logs')
model.load_weights(WEIGHTS_FILE, by_name=True)
return model
def main():
argv = sys.argv
if "--" not in argv:
argv = [] # as if no args are passed
else:
argv = argv[argv.index("--") + 1:] # get all args after "--"
# When --help or no args are given, print this help
usage_text = (
"python synth_test.py -- [options]"
)
parser = argparse.ArgumentParser(description=usage_text)
parser.add_argument(
"-isynth", "--input_syndir", dest="synth_path", type=str, required=True,
help="Input the synthetic image directory",
)
parser.add_argument(
"-ijson", "--input_jsonfile", dest="json_path", type=str, required=True,
help="Input the annotation file",
)
parser.add_argument(
"-iweight", "--input_weightfile", dest="weight_path", type=str, required=True,
help="Input the weight file",
)
parser.add_argument(
"-ianota", "--input_anotafile", dest="anota_path", type=str, required=True,
help="Input the annotation file",
)
args = parser.parse_args(argv)
if not argv:
parser.print_help()
return
if (not args.synth_path or
not arg.json_path or
not args.weight_path or
not args.anota_path):
print("Error: argument not given, aborting.")
parser.print_help()
return
#change model path here
rcnn = MaskRCNN(mode='inference', model_dir='./', config=TestConfig())
#load weight to the model
rcnn.load_weights(args.weight_path, by_name=True)
for r,d,f in os.walk(args.synth_path):
for file in f:
annotations = json.load(open(os.path.join(args.json_path, "via_region_data.json")))
filename = file
filesize = os.stat(os.path.join(args.synth_path, filename)).st_size
key = filename + str(filesize)
the_file = open(args.anota_path, 'r')
reader = csv.reader(the_file)
N = int(filename.split('.')[0])
line = next((x for i, x in enumerate(reader) if i == N), None)
the_file.close()
image = cv2.imread(os.path.join(args.synth_path, filename))
#get coordinates of signs for ground truth txt preparation
marked_image, coord_list = draw_rect(image, args.anota_path, filename)
ground_truth_signs = []
for region_num in sorted(annotations[key]['regions']):
#print("Sign_name: %s, Id: %d" %
# (annotations[key]['regions'][region_num]['region_attributes']['name'],
# SIGN_DICT[annotations[key]['regions'][region_num]['region_attributes']['name']]))
ground_truth_signs.append(annotations[key]['regions'][region_num]['region_attributes']['name'])
# load photograph
img = load_img(os.path.join(args.synth_path, filename))
img = img_to_array(img)
# make prediction
results = rcnn.detect([img], verbose=0)
# visualize the results
r = results[0]
#draw_image_with_boxes(os.path.join(args.synth_path, filename), r['rois'], r['class_ids'], r['scores'], ground_truth_signs, coord_list, filename)
file_name = filename.split('.')[0]
#generate detected data txt files for mAP calculation
generate_predicted_txt(file_name, r['rois'], r['class_ids'], r['scores'])
#generate ground truth txt files for mAP calculation
generate_ground_truth_txt(file_name, ground_truth_signs, coord_list)
###################################################################################
# for image_id in train_set.image_ids:
# # load image info
# info = train_set.image_info[image_id]
# # display on the console
# print(info)
#####################################################################
########## to display your image with display instane################
#####################################################################
# image_id = 0
# # load the image
# image = train_set.load_image(image_id)
# # load the masks and the class ids
# mask, class_ids = train_set.load_mask(image_id)
# # extract bounding boxes from the masks
# bbox = extract_bboxes(mask)
# # display image with masks and bounding boxes
# display_instances(image, bbox, mask, class_ids, train_set.class_names)
config = KangarooConfig()
config.display()
## using model
model = MaskRCNN(mode = 'training', model_dir = './',config = config)
#load weightss
model.load_weights('mask_rcnn_coco.h5',by_name = True,exclude = ["mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox", "mrcnn_mask"])
## training starts
model.train(train_set, test_set, learning_rate=config.LEARNING_RATE, epochs=5, layers='heads')
if args.command == "train":
config = BalloonConfig()
else:
class InferenceConfig(BalloonConfig):
# Set batch size to 1 since we'll be running inference on
# one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
GPU_COUNT = 1
IMAGES_PER_GPU = 1
config = InferenceConfig()
config.display()
# Create model
if args.command == "train":
model = MaskRCNN(
mode="training",
config=config,
model_dir=args.logs
)
else:
model = MaskRCNN(
mode="inference",
config=config,
model_dir=args.logs
)
# Select weights file to load
if args.weights.lower() == "coco":
weights_path = COCO_WEIGHTS_PATH
# Download weights file
if not os.path.exists(weights_path):
utils.download_trained_weights(weights_path)
if not os.path.exists(COCO_MODEL_PATH):
mrcnn.utils.download_trained_weights(COCO_MODEL_PATH)
# working dir
IMAGE_DIR = os.path.join(ROOT_DIR, "images")
VIDEO_DIR = "test_images/parking.mp4"
# twilio config
twilio_account_sid = 'TWILIO_SID'
twilio_auth_token = 'TWILIO_AUTH_TOKEN'
twilio_phone_number = 'TWILIO_SOURCE_PHONE_NUMBER'
destination_phone_number = 'PHONE_NUMBER_TO_TEXT'
client = Client(twilio_account_sid, twilio_auth_token)
# create a Mask-RCNN model in inference mode
model = MaskRCNN(mode="inference", config=MaskRCNNConfig(), model_dir=MODEL_DIR)
# load pre-trained
model.load_weights(COCO_MODEL_PATH, by_name=True)
parked_veh_boxes = None
free_space_frames = 0 # count how many frames a potential parking lot is open in a row in the video
sms_sent = False
# load video
video = cv2.VideoCapture(VIDEO_DIR)
while video.isOpened():
success, frame = video.read()
if not success:
break
# calculate width and height of the box
width, height = x2 - x1, y2 - y1
# create the shape
rect = Rectangle((x1, y1), width, height, fill=False, color='red')
# draw the box
ax.add_patch(rect)
# show the figure
pyplot.show()
# load the train dataset
train_set = ParkingDataset()
train_set.load_dataset('parking', is_train=True)
train_set.prepare()
print('Train: %d' % len(train_set.image_ids))
# load the test dataset
test_set = ParkingDataset()
test_set.load_dataset('parking', is_train=False)
test_set.prepare()
print('Test: %d' % len(test_set.image_ids))
# create config
cfg = PredictionConfig()
# define the model
model = MaskRCNN(mode='inference', model_dir='./', config=cfg)
# load model weights
model_path = 'mask_rcnn_parking_cfg_0005.h5'
model.load_weights(model_path, by_name=True)
# plot predictions for train dataset
plot_actual_vs_predicted(train_set, model, cfg)
# plot predictions for test dataset
plot_actual_vs_predicted(test_set, model, cfg)
# load the train dataset
train_set = KangarooDataset()
train_set.load_dataset('kangaroo', is_train=True)
train_set.prepare()
print('Train: %d' % len(train_set.image_ids))
# load the test dataset
test_set = KangarooDataset()
test_set.load_dataset('kangaroo', is_train=False)
test_set.prepare()
print('Test: %d' % len(test_set.image_ids))
# create config
cfg = PredictionConfig()
# define the model
model = MaskRCNN(mode='inference', model_dir='./', config=cfg)
# load model weights
model_path = 'mask_rcnn_kangaroo_cfg_0005.h5'
model.load_weights(model_path, by_name=True)
# ------------------------------------------------------
# Compare actual and predict
# # plot predictions for train dataset
# plot_actual_vs_predicted(train_set, model, cfg)
# # plot predictions for test dataset
# plot_actual_vs_predicted(test_set, model, cfg)
# ----------------------------------------------------------
# load photograph
img = load_img('kangaroo117.jpg')
img = img_to_array(img)
# make prediction
# prepare train set
train_set = PlaneDataset()
train_set.load_dataset('dataset', is_train=True)
train_set.prepare()
print('Train: %d' % len(train_set.image_ids))
# prepare test/val set
test_set = PlaneDataset()
test_set.load_dataset('dataset', is_train=False)
test_set.prepare()
print('Test: %d' % len(test_set.image_ids))
# prepare config
config = PlaneConfig()
config.display()
# define the model
model = MaskRCNN(mode='training', model_dir='./logs/', config=config)
# load weights (mscoco) and exclude the output layers
# Select weights file to load
if args.model.lower() == "coco":
model_path = 'mask_rcnn_plane.h5'
elif args.model.lower() == "last":
# Find last trained weights
model_path = model.find_last()
else:
model_path = args.model
print("Training on: " + model_path)
model.load_weights(model_path,
by_name=True,
exclude=[
type=str,
required=True,
help="Image to perform object recognition on.")
ap.add_argument("-m",
"--model",
default="data/mask_rcnn_coco.h5",
type=str,
help="Model weights for Mask R-CNN model.")
ap.add_argument("-o",
"--object-detection",
action="store_true",
help="Perform object detection using Mask R-CNN model.")
args = vars(ap.parse_args())
# Define and load model
rcnn = MaskRCNN(mode='inference', model_dir='./', config=TestConfig())
rcnn.load_weights(args["model"], by_name=True)
img = load_img(args["image"])
img_pixels = img_to_array(img)
results = rcnn.detect([img_pixels], verbose=0)
r = results[0]
if args["object_detection"]:
print("[INFO] Performing object detection using display_instances...")
# define 81 classes that the coco model knowns about
class_names = load_coco_classes('data/coco_classes.txt')
NAME = "custom_object"
IMAGES_PER_GPU = 1
GPU_COUNT = 1
NUM_CLASSES = 1 + 1 # custom object + background class
# Root directory of the project
ROOT_DIR = Path(".")
MODEL_DIR = ROOT_DIR / "training_logs"
# Local path to trained weights file (make sure you update this)
TRAINED_MODEL_PATH = MODEL_DIR / "custom_object20180817T1201" / "mask_rcnn_custom_object_0030.h5"
# Create a Mask-RCNN model in inference mode
model = MaskRCNN(mode="inference",
model_dir=MODEL_DIR,
config=ObjectDetectorConfig())
# Load pre-trained model
model.load_weights(str(TRAINED_MODEL_PATH), by_name=True)
# COCO Class names
class_names = ['BG', 'custom_object']
# Load the image we want to run detection on
image_path = "./training_images/validation_set/20180816_133618.jpg"
image = cv2.imread(image_path)
# Convert the image from BGR color (which OpenCV uses) to RGB color
rgb_image = image[:, :, ::-1]
img = cv2.cvtColor(Image, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(img, (5, 5), 0)
ret3, img = cv2.threshold(blur, 0, 255,
cv2.THRESH_BINARY + cv2.THRESH_OTSU)
#convert image to 3 channels
Image = gg(img)
scaled_image = mold_image(Image, cfg)
sample = expand_dims(scaled_image, 0)
yhat = model.detect(sample, verbose=0)
return yhat, Image
cfg = PredictionConfig()
# define the model
model = MaskRCNN(mode='inference',
model_dir='/content/drive/My Drive/',
config=cfg)
model.load_weights(
'/content/drive/My Drive/handwritting_models/handwrittingconfig20200830T0341/mask_rcnn_handwrittingconfig_0011.h5',
by_name=True)
train_mAP = evaluate_model(train_set, model, cfg)
print("Train mAP: %.3f" % train_mAP)
test_mAP = evaluate_model(test_set, model, cfg)
print(test_mAP)
import os
import cv2 as cv
Y = "/content/drive/My Drive/Dooooo/"
k = os.listdir(Y)
print(k)
"""Perform prediction for all images in The mentioned directory"""
#poor code for plotting tho
class Mask(object):
"""
Mask R-CNN
"""
CLASS_NAMES: List[str] = []
COLORS: List[str] = []
model: MaskRCNN = None
def __init__(self):
self.CLASS_NAMES = classes
self.COLORS = extra.getRandomColors(self.CLASS_NAMES)
model = getMaskConfig(float(config.detectionMinConfidence))
self.model = MaskRCNN(mode="inference",
model_dir=config.LOGS_DIR,
config=model)
self.model.load_weights(config.DATASET_DIR, by_name=True)
@timeChecker.checkElapsedTime(3, 2, 1, "Mask detecting")
def pipeline(self, inputPath: str, outputPath: str = None):
"""
almost main
"""
if outputPath:
dirs.createDirs(os.path.split(outputPath)[0])
filename = os.path.split(outputPath)[1]
else:
filename = os.path.split(inputPath)[1]
cameraId = filename.split('_')[0]
img = Image(inputPath, int(cameraId), outputPath=outputPath)
binaryImage = img.read()
rowDetections = self._detectByMaskCNN(img)
detections = _parseR(self._humanizeTypes(rowDetections))
img.addDetections(detections)
signedImg = self._visualize_detections(img,
rowDetections['masks'],
drawMask=False)
img.write(outputPath, signedImg)
return img
def _visualize_detections(self,
image: Image,
masks,
drawMask=False) -> np.ndarray:
"""
input: the original image, the full object from the mask cnn neural network, and the object ID, if it came out to get it
output: an object indicating the objects found in the image, and the image itself, with selected objects and captions
"""
bgr_image = image.read()
font = cv2.FONT_HERSHEY_DUPLEX
fontScale = 0.8
thickness = 2
for i, currentObject in enumerate(image.objects):
if currentObject.type not in config.availableObjects:
continue
y1, x1, y2, x2 = currentObject.coordinates
lineInClassName = self.CLASS_NAMES.index(currentObject.type)
color = [
int(c) for c in np.array(self.COLORS[lineInClassName]) * 255
]
text = "{}: {:.1f}".format(currentObject.type,
currentObject.scores * 100)
if (drawMask):
mask = masks[:, :, i] # берем срез
bgr_image = mrcnn.visualize.apply_mask(
bgr_image, mask, color, alpha=0.6) # рисование маски
cv2.rectangle(bgr_image, (x1, y1), (x2, y2), color, thickness)
cv2.putText(bgr_image, text, (x1, y1 - 20), font, fontScale, color,
thickness)
return bgr_image.astype(np.uint8)
def _detectByMaskCNN(self, image: Image):
"""
input: image - the result of cv2.imread (<filename>)
output: r - dictionary of objects found (r ['masks'], r ['rois'], r ['class_ids'], r ['scores']), detailed help somewhere else
"""
rgbImage = image.getRGBImage()
r = self.model.detect([rgbImage], verbose=1)[0] # тут вся магия
# проверить что будет если сюда подать НЕ ОДНО ИЗОБРАЖЕНИЕ, А ПОТОК
return r
def _humanizeTypes(self, r: dict) -> dict:
typesList = [
self.CLASS_NAMES[objectClass] for objectClass in r['class_ids']
]
r.update({'class_ids': typesList})
return r
def Modelmain():
test_set = ClownDataset()
test_set.load_dataset(
'/home/ubuntu/FobiaPhilter/ActionFiles/FramesFromVideo',
is_train=False)
test_set.prepare()
cfg = PredictionConfig()
model_path = '/home/ubuntu/FobiaPhilter/ActionFiles/model/mask_rcnn_clown_cfg_0025.h5'
model = MaskRCNN(mode='inference', model_dir='./', config=cfg)
model.load_weights(model_path, by_name=True)
cfg2 = InferenceConfig()
cfg3 = InferenceConfigOrig()
weights_path = '/home/ubuntu/FobiaPhilter/ActionFiles/model/mask_rcnn_coco.h5'
model2 = MaskRCNN(mode='inference', model_dir='./', config=cfg2)
model2.load_weights(weights_path, by_name=True)
model3 = MaskRCNN(mode='inference', model_dir='./', config=cfg3)
model3.load_weights(weights_path, by_name=True)
if phobia == 'clown':
plot_predicted_new(test_set, model, model2, cfg, cfg2, class_names,
class_names2, n_images)
#Export imageID vs. original Filename
files = []
for m in test_set.image_from_source_map:
files.append(m)
df = pd.DataFrame({'Original_files': files})
df['Index_outputFile'] = df.index
df['Original_files'] = df['Original_files'].str.replace(
'dataset.image', '').astype('int64')
df = df.sort_values(by=['Original_files'])
df.to_csv(
'/home/ubuntu/FobiaPhilter/ActionFiles/TestSampleImageID.txt')
elif phobia != 'clown':
plot_predicted_coco(test_set, model3, cfg3, class_names2, n_images,
phobia)
#Export imageID vs. original Filename
files = []
for m in test_set.image_from_source_map:
files.append(m)
df = pd.DataFrame({'Original_files': files})
df['Index_outputFile'] = df.index
df['Original_files'] = df['Original_files'].str.replace(
'dataset.image', '').astype('int64')
df = df.sort_values(by=['Original_files'])
df.to_csv(
'/home/ubuntu/FobiaPhilter/ActionFiles/TestSampleImageID.txt')
else:
pass
builder.add_meta_graph_and_variables(export_session,
[saved_model.tag_constants.SERVING],
signature_def_map = signature)
builder.save()
if __name__ == '__main__':
# Get model config
model_config = get_model_config()
export_dir = os.path.join(ExportConfig.EXPORT_DIR, ExportConfig.MODEL_NAME)
if not os.path.exists(export_dir):
os.mkdir(export_dir)
# Load maask rcnn keras model and the pretrained weights
model = MaskRCNN(mode = "inference", model_dir = ExportConfig.KERAS_MODEL_DIR, config = model_config)
model.load_weights(ExportConfig.KERAS_WEIGHTS_PATH, by_name = True)
with K.get_session() as master_session:
graph_def = freeze_model(model.keras_model, transforms = ExportConfig.TRANSFORMS)
with tf.Session(graph = tf.Graph()) as export_session:
tf.import_graph_def(graph_def, name = "")
export_saved_model(export_dir, ExportConfig.VERSION_NUMBER)
# Print the size of the tf-serving model
print("*" * 80)
get_model_size(export_dir, ExportConfig.VERSION_NUMBER)
print("*" * 80)
print("COMPLETED")
#the following is to train the model
# prepare the config
config = SproutsConfig()
config.display()
# display the model
model = MaskRCNN(mode='training', model_dir='./', config=config)
# load weights (mscoco) and exclude the output layers
model.load_weights('mask_rcnn_coco.h5', by_name=True, exclude=["mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox", "mrcnn_mask"])
# train weights (output layers or 'heads')
model.train(train_set, test_set, learning_rate=config.LEARNING_RATE, epochs=5, layers='heads')
'''
# create config
cfg = PredictionConfig()
# define the model
model = MaskRCNN(mode='inference', model_dir='./', config=cfg)
# load model weights
model.load_weights('mask_rcnn_sprouts_cfg_0005.h5', by_name=True)
'''
# evaluate model on training dataset
train_mAP = evaluate_model(train_set, model, cfg)
print("Train mAP: %.3f" % train_mAP)
#evaluate model on test dataset
test_mAP = evaluate_model(test_set, model, cfg)
print("Test mAP: %.3f" % test_mAP)
'''
# plot predictions for train dataset
#plot_actual_vs_predicted(train_set, model, cfg)
# plot predictions for test dataset
#plot_actual_vs_predicted(test_set, model, cfg)
plot_prediction(new_set, model, cfg)
def worker4():
# Video file or camera to process - set this to 0 to use your webcam instead of a video file
VIDEO_SOURCE = "http://109.236.111.203:90/mjpg/video.mjpg"
# Physical capacity of area
TOTAL_PARKING_CAPACITY = 120
# Create a Mask-RCNN model in inference mode
model = MaskRCNN(mode="inference",
model_dir=MODEL_DIR,
config=MaskRCNNConfig())
# Load pre-trained model
model.load_weights(COCO_MODEL_PATH, by_name=True)
# Location of parking spaces
parked_car_boxes = None
# Load the video file we want to run detection on
video_capture = cv2.VideoCapture(VIDEO_SOURCE)
cv2.waitKey(33)
frame_counter = 1
has_space = False
# Loop over each frame of video
while video_capture.isOpened():
success, frame = video_capture.read()
if not success:
break
parking_areas = np.array([[390, 713, 444, 800], [371, 658, 421, 761],
[366, 393, 421, 520], [278, 538, 314, 605],
[352, 630, 392, 718], [260, 518, 293, 573],
[262, 260, 300, 344], [405, 261, 450, 420],
[323, 270, 371, 375], [299, 572, 338, 667],
[295, 269, 341, 389], [234, 173, 266, 209],
[413, 412, 448, 520], [274, 353, 308, 445],
[241, 345, 272, 413], [205, 437, 225, 484],
[213, 458, 235, 503], [335, 617, 376, 685],
[196, 444, 210, 487], [138, 192, 161, 219],
[222, 475, 259, 553], [169, 399, 187, 434],
[357, 373, 395, 466], [158, 382, 169, 410],
[182, 417, 200, 456], [162, 406, 179, 442],
[151, 382, 164, 414], [320, 616, 340, 669],
[227, 328, 258, 403], [189, 433, 205, 477],
[174, 470, 193, 529], [328, 367, 373, 494],
[120, 447, 131, 466], [305, 572, 332, 598],
[291, 356, 333, 469], [272, 558, 285, 600],
[389, 260, 422, 384], [145, 377, 158, 407],
[185, 413, 198, 433], [125, 446, 137, 465]])
overlaps = parking_areas
if (frame_counter % 100 == 0):
print("----------------------------------------------")
print("Start detecting cars ....")
# Capture frame-by-frame
start_time = time.time()
# Convert the image from BGR color (which OpenCV uses) to RGB color
rgb_image = frame[:, :, ::-1]
# Run the image through the Mask R-CNN model to get results.
results = model.detect([rgb_image], verbose=0)
# Mask R-CNN assumes we are running detection on multiple images.
# We only passed in one image to detect, so only grab the first result.
r = results[0]
# The r variable will now have the results of detection:
# - r['rois'] are the bounding box of each detected object
# - r['class_ids'] are the class id (type) of each detected object
# - r['scores'] are the confidence scores for each detection
# - r['masks'] are the object masks for each detected object (which gives you the object outline)
# Filter the results to only grab the car / truck bounding boxes
car_boxes = get_car_boxes(r['rois'], r['class_ids'])
print("car_boxes")
# print(car_boxes)
print("Cars found in frame of video:")
# Draw each box on the frame
i = 1
for box in car_boxes:
print("Car ", i, ":", box)
y1, x1, y2, x2 = box
# Draw the box
cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 1)
cv2.putText(frame, "%s" % str(i), (x1, y1), cv2.LINE_AA, 1,
(0, 0, 255))
cv2.circle(frame, (x1, y1), 5, (0, 0, 255), -1)
i = i + 1
# See how much cars overlap with the known parking spaces
print("parking_areas")
print(parking_areas)
overlaps = mrcnn.utils.compute_overlaps(
car_boxes, parking_areas) # parking_areas
print(len(overlaps.tolist()))
print("Checking overlaps .... frame %d" % frame_counter)
print(overlaps)
# print(overlaps)
print(overlaps < 0.5)
result = space_Violation(overlaps)
if result < TOTAL_PARKING_CAPACITY:
print("Free Parking Spaces")
has_space = True
cv2.putText(
frame, "Parking Spaces Available : %s" %
str(TOTAL_PARKING_CAPACITY - result), (10, 50),
cv2.LINE_AA, 1, (100, 255, 0))
# Add Time Stamp
time_stamp = time.strftime("%Y/%m/%d %H:%M:%S %Z",
time.localtime())
cv2.putText(frame, time_stamp, (950, 710),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0, 0), 1,
cv2.LINE_AA)
else:
has_space = False
cv2.putText(frame, "Don't Have Parking Spaces", (10, 50),
cv2.LINE_AA, 1, (0, 0, 255))
# Add Time Stamp
time_stamp = time.strftime("%Y/%m/%d %H:%M:%S %Z",
time.localtime())
cv2.putText(frame, time_stamp, (950, 710),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 255, 0, 0), 1,
cv2.LINE_AA)
#cv2.imwrite("analyze/output/frame%d.jpg" % frame_counter, frame), for debug
cv2.imwrite("analyze/output/frame-analyzed-4.jpg", frame)
# Show the frame of video on the screen, for debug
# cv2.imshow('Video', frame)
#add a sleep for demo
# time.sleep(5)
feed2 = Settings()
feed2.device.state.update(
{'Russia': TOTAL_PARKING_CAPACITY - result})
print("-----STATE--------")
print(feed2.device.state)
if has_space:
# print("Free Parking Spaces")
# TODO - Push to DB
cv2.putText(frame, "Free Parking Spaces", (10, 50), cv2.LINE_AA, 1,
(0, 255, 0))
else:
# TODO - Push to DB
cv2.putText(frame, "Don't Have Free Parking Spaces", (10, 50),
cv2.LINE_AA, 1, (0, 0, 255))
# Show the frame of video on the screen, for debug
# cv2.imshow('Video', frame)
frame_counter = frame_counter + 1
#Hit 'q' to quit
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# Clean up everything when finished
video_capture.release()
cv2.destroyAllWindows()
for i, box in enumerate(boxes):
# If the detected object isn't a car / truck, skip it
if class_ids[i] in [3, 8, 6]:
car_boxes.append(box)
return np.array(car_boxes)
# Download COCO trained weights from Releases if needed
if not os.path.exists(COCO_MODEL_PATH):
mrcnn.utils.download_trained_weights(COCO_MODEL_PATH)
# Create a Mask-RCNN model in inference mode
model = MaskRCNN(mode="inference",
model_dir=MODEL_DIR,
config=MaskRCNNConfig())
# Load pre-trained model
model.load_weights(COCO_MODEL_PATH, by_name=True)
# Load the video file we want to run detection on
video_capture = cv2.VideoCapture(RTSP_SOURCE)
"""SKIP FRAME ACTUATE"""
# video_capture.set(cv2.CAP_PROP_POS_FRAMES, SKIP_FRAMES)
# Save the detection result video
"""ACTUATE IMAGE SIZE BY CV2 TOOLS"""
# video_writer = cv2.VideoWriter(OUTPUT_SOURCE, -1, 20.0, FRAME_SIZE)
# Loop over each frame of video
# while iter < max_frame:
dataset_validate = SeverstalDataset(dataframe=validate)
dataset_validate.load_dataset()
dataset_validate.prepare()
### curl -LO https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5
# configuration
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# session stuff
session = tf.Session(config=config)
session.run(tf.global_variables_initializer())
session.run(tf.local_variables_initializer())
# initialiazing model
model = MaskRCNN(mode='training',
config=severstal_config,
model_dir='modeldir')
# we will retrain starting with the coco weights
model.load_weights('mask_rcnn_coco.h5',
by_name=True,
exclude=[
'mrcnn_bbox_fc', 'mrcnn_class_logits', 'mrcnn_mask',
'mrcnn_bbox'
])
### %time
# ignore UserWarnongs
import warnings
warnings.filterwarnings('ignore', category=UserWarning)
'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant',
'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote',
'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink',
'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear',
'hair drier', 'toothbrush'
]
class TestConfig(Config):
NAME = "test"
GPU_COUNT = 1
IMAGES_PER_GPU = 1
NUM_CLASSES = 1 + 80
rcnn = MaskRCNN(mode='inference', model_dir='./', config=TestConfig())
rcnn.load_weights('mask_rcnn_coco.h5', by_name=True)
cap = cv2.VideoCapture(0)
while True:
ret, frame = cap.read()
results = rcnn.detect([frame], verbose=0)
r = results[0]
masked_img = get_masked_image(frame, r['rois'], r['masks'], r['class_ids'],
class_names, r['scores'])
# masked_img = display_instances(frame, r['rois'], r['masks'], r['class_ids'], class_names, r['scores'])
cv2.imshow('', masked_img)
if cv2.waitKey(1) & 0xFF == ord('q'):
class RCNNDetector(IObjectDetection):
def __init__(self, dataset_path, dataset_name):
super(RCNNDetector, self).__init__(dataset_path, dataset_name)
self.train_set = ClassDataset()
self.test_set = ClassDataset()
# self.train_set = KangarooDataset()
# self.test_set = KangarooDataset()
self.model = "rcnn"
self.modelWeights = None
self.config = Config()
def transform(self):
# fn.organizeDataset(self.DATASET_NAME, self.OUTPUT_PATH, self.DATASET)
self.train_set.load_dataset(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME), True)
# self.train_set.load_dataset(dataset_path, True)
self.train_set.prepare()
#self.test_set.load_dataset(os.path.join(self.OUTPUT_PATH,self.DATASET_NAME), False)
# self.test_set.load_dataset(dataset_path, False)
#self.test_set.prepare()
# def organize(self, train_percentage):
# super(RCNNDetector, self).organize( train_percentage)
def createModel(self):
# En este caso tambien debe ser output por que ya se ha hecho la division y se ha guardado
classes_file = os.path.join(self.OUTPUT_PATH, self.DATASET_NAME,
"classes.names")
file = open(os.path.join(classes_file))
classes = []
for line in file:
classes.append(line)
n_classes = fn.count_classes(classes)
n_images = len(
glob.glob(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME,
"train/JPEGImages/*.jpg")))
ClassConfig.NUM_CLASSES += n_classes
ClassConfig.NAME = self.DATASET_NAME
ClassConfig.N_IMAGES = n_images
ClassConfig.STEPS_PER_EPOCH = n_images // (ClassConfig.GPU_COUNT *
ClassConfig.IMAGES_PER_GPU)
self.config = ClassConfig()
# Por lo mismo de antes. El dataset ya esta procesado y guardado ahi. Es donde se tiene que trabajar con el en este caso
# self.modelWeights = MaskRCNN(mode='training', model_dir=os.path.join(self.OUTPUT_PATH,"model"), config=self.config)
if not os.path.exists(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "models")):
os.mkdir(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "models"))
self.modelWeights = MaskRCNN(mode='training',
model_dir=os.path.join(
self.OUTPUT_PATH, self.DATASET_NAME,
"models"),
config=self.config)
if not os.path.exists(
'objectDetectors/RCNNObjectDetector/mask_rcnn_coco.h5'):
wget.download(
"https://www.dropbox.com/s/12ou730jt730qvu/mask_rcnn_coco.h5?dl=1",
'objectDetectors/RCNNObjectDetector/mask_rcnn_coco.h5')
self.modelWeights.load_weights(
'objectDetectors/RCNNObjectDetector/mask_rcnn_coco.h5',
by_name=True,
exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox",
"mrcnn_mask"
])
def train(self, framework_path=None, n_gpus=1):
ClassConfig.GPU_COUNT = n_gpus
# self.model.train(self.TRAIN_SET, self.TEST_SET, learning_rate=self.CONFIG.LEARNING_RATE, epochs=5, layers='heads')
self.modelWeights.train(self.train_set,
self.train_set,
learning_rate=self.config.LEARNING_RATE,
epochs=5,
layers='heads')
results = []
# Path(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "models")).rglob(".h5")
for r in glob.glob(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "models",
"**", "*5.h5")):
results.append(r)
# results = [p for p in os.listdir(os.path.join(self.OUTPUT_PATH, self.DATASET_NAME,"models")) if p.endswith(".h5") and "mask_rcnn_" + self.DATASET_NAME + "_0005" in p]
shutil.copy2(
results[0],
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "models",
"mask_rcnn_" + self.DATASET_NAME + "_0005.h5"))
def evaluate(self):
rcnnPredict = RCNNPredict(
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "models",
"mask_rcnn_" + self.DATASET_NAME.lower() +
"_0005.h5"),
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME, "classes.names"))
map = Map(rcnnPredict, self.DATASET_NAME,
os.path.join(self.OUTPUT_PATH, self.DATASET_NAME),
self.model)
map.evaluate()
def main():
# train set
train_set = AnimalDataset()
train_set.load_dataset('kangaroo', is_train=True)
train_set.prepare()
print('Train: %d' % len(train_set.image_ids))
# test/val set
test_set = AnimalDataset()
test_set.load_dataset('kangaroo', is_train=False)
test_set.prepare()
print('Test: %d' % len(test_set.image_ids))
# load an image
image_id = 0
image = train_set.load_image(image_id)
print(image.shape)
# load image mask
mask, class_ids = train_set.load_mask(image_id)
print(mask.shape)
# plot image
pyplot.imshow(image)
# plot mask
pyplot.imshow(mask[:, :, 0], cmap='gray', alpha=0.5)
pyplot.show()
# define image id
image_id = 1
# load the image
image = train_set.load_image(image_id)
# load the masks and the class ids
mask, class_ids = train_set.load_mask(image_id)
# extract bounding boxes from the masks
bbox = extract_bboxes(mask)
# display image with masks and bounding boxes
display_instances(image, bbox, mask, class_ids, train_set.class_names)
# prepare config
config = AnimalConfig()
config.display()
# define the model
model = MaskRCNN(mode='training', model_dir='./', config=config)
# load weights (mscoco) and exclude the output layers
model.load_weights('mask_rcnn_coco.h5', by_name=True, exclude=["mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox", "mrcnn_mask"])
# train weights (output layers or 'heads')
model.train(train_set, test_set, learning_rate=config.LEARNING_RATE, epochs=1, layers='heads')
# create config
cfg = AnimalConfig()
# define the model
model = MaskRCNN(mode='inference', model_dir='./', config=cfg)
# load model weights
model.load_weights('mask_rcnn_kangaroo_cfg_0002.h5', by_name=True)
# evaluate model on training dataset
train_mAP = evaluate_model(train_set, model, cfg)
print("Train mAP: %.3f" % train_mAP)
# evaluate model on test dataset
test_mAP = evaluate_model(test_set, model, cfg)
print("Test mAP: %.3f" % test_mAP)
#
## load model weights
#model_path = 'mask_rcnn_kangaroo_cfg_0005.h5'
#model.load_weights(model_path, by_name=True)
# plot predictions for train dataset
plot_actual_vs_predicted('train_actual_vs_pred.png',train_set, model, cfg)
# plot predictions for test dataset
plot_actual_vs_predicted('test_actual_vs_pred.png',test_set, model, cfg)
def get_mrcnn(self):
model = MaskRCNN(mode="inference",
model_dir=self.log_dir,
config=MaskRCNNConfig())
model.load_weights(self.h5weights_path, by_name=True)
return model
shape = [d.size for d in t.tensor_shape.dim]
if t.dtype == types_pb2.DT_FLOAT:
return np.array(t.float_val).reshape(shape)
else:
raise NotImplementedError(f"Tensor type: {t.dtype}")
class MyConfig(Config):
NAME = "maskrcnn"
IMAGES_PER_GPU = 1
BACKBONE = "resnet50"
if __name__ == "__main__":
model = MaskRCNN(mode="inference", model_dir="None", config=MyConfig())
# Raise the maximum message size to 100MB
max_message_size = 100 * 1024 * 1024
options = [
("grpc.max_message_length", max_message_size),
("grpc.max_receive_message_length", max_message_size),
]
channel = grpc.insecure_channel("127.0.0.1:8500", options=options)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
image_data = imageio.imread("data/155806.jpg")
molded_images, image_metas, windows = model.mold_inputs([image_data])
image_shape = molded_images[0].shape
anchors = model.get_anchors(image_shape)
anchors = np.broadcast_to(anchors,
class MaskRCNNDetectObject:
def __init__(self):
self.fetch_resources()
self.model_init = False
self.user_config = self.get_operator_config()
# define special param here
self.config = MaskRCNNConfig()
self.label = [
'BG', 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus',
'train', 'truck', 'boat', 'traffic light', 'fire hydrant',
'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog',
'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe',
'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee',
'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat',
'baseball glove', 'skateboard', 'surfboard', 'tennis racket',
'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl',
'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot',
'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch',
'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop',
'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven',
'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase',
'scissors', 'teddy bear', 'hair drier', 'toothbrush'
]
self.model_path = os.path.join(temp_directory(), COCO_MODEL_PATH)
# initialize model
try:
self.graph = tf.Graph()
with self.graph.as_default():
with tf.device(self.device_str):
self.session = tf.Session(config=self.user_config)
KTF.set_session(self.session)
self.rcnn = MaskRCNN(mode="inference",
model_dir=MODEL_DIR,
config=self.config)
self.graph = KTF.get_graph()
self.session = KTF.get_session()
with self.session.as_default():
self.bulk_execute(np.zeros((1, 300, 300, 3)))
except Exception as e:
logging.error("unexpected error happen during build graph",
exc_info=True)
raise e
def get_operator_config(self):
try:
self.device_str = os.environ.get("device_id", "/cpu:0")
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
gpu_mem_limit = float(os.environ.get("gpu_mem_limit", 0.3))
config.gpu_options.per_process_gpu_memory_fraction = gpu_mem_limit
# for device debug info print
if os.environ.get("log_device_placement", False):
config.log_device_placement = True
logging.info("device id %s, gpu memory limit: %f", self.device_str,
gpu_mem_limit)
except Exception as e:
logging.error("unexpected error happen during read config",
exc_info=True)
raise e
logging.info("Model device str: %s, session config: %s",
self.device_str, config)
return config
def fetch_resources(self):
# download_temp_file(COCO_MODEL_URL, COCO_MODEL_PATH)
pass
def load_model(self):
self.rcnn.load_weights(self.model_path, by_name=True)
self.model_init = True
def get_bboxes(self, boxes, scores, classes):
bboxes = [[
BoundingBox(x1=box[1],
y1=box[0],
x2=box[3],
y2=box[2],
score=score,
label=self.label[int(cls)])
for (box, score,
cls) in zip(boxes.tolist(), scores.tolist(), classes.tolist())
]]
return bboxes
@staticmethod
def get_obj_image(images, bboxes):
obj_images = []
for i, frame_bboxes in enumerate(bboxes):
frame_object = []
for j, bbox in enumerate(frame_bboxes):
tmp = images[i][int(bbox.y1):int(bbox.y2),
int(bbox.x1):int(bbox.x2)]
frame_object.append(cv2base64(tmp))
obj_images.append(frame_object)
return obj_images
def execute(self, image):
with self.graph.as_default():
with self.session.as_default():
if not self.model_init:
self.load_model()
results = self.rcnn.detect([image])
bboxes = self.get_bboxes(results[0]["rois"],
results[0]["scores"],
results[0]["class_ids"])
bboxes[0].sort(key=lambda x: -x.score)
objects_image = self.get_obj_image([image], bboxes)
return objects_image[0]
def bulk_execute(self, images):
objs = []
for image in images:
objs.append(self.execute(image))
return objs
@property
def name(self):
return "mask_rcnn"
@property
def type(self):
return "processor"
@property
def input(self):
return "image"
@property
def output(self):
return "images"
@property
def dimension(self):
return "-1"
@property
def metric_type(self):
return "-1"
# number of training steps per epoch
# <- MODIFY -> Replace 178 with the number of images of your dataset
STEPS_PER_EPOCH = 178 // (GPU_COUNT * IMAGES_PER_GPU)
# prepare train set
train_set = TableBankDataset()
# <- MODIFY -> Replace mydataset with the name of the folder containing your images and annotations
train_set.load_dataset('mydataset', is_train=True)
train_set.prepare()
print('Train: %d' % len(train_set.image_ids))
# prepare config
config = TableBankConfig()
config.display()
# define the model
model = MaskRCNN(mode='training', model_dir='./', config=config)
# load weights (mscoco) and exclude the output layers
model.load_weights('mask_rcnn_tablebank_cfg_0002.h5',
by_name=True,
exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc", "mrcnn_bbox",
"mrcnn_mask"
])
# train weights (output layers or 'heads')
model.train(train_set,
None,
learning_rate=config.LEARNING_RATE,
epochs=2,
layers='heads')
# unfreeze the body of the network and train *all* layers
model.train(train_set,
sess = tf.Session(config=tf_config)
graph = tf.get_default_graph()
K.set_session(sess)
os.makedirs(os.path.join(MODELS_ROOT, 'trained-models'), exist_ok=True)
model_path = "https://github.com/Qbrayan/food_detection/releases/download/0.1.0/mask_rcnn_foodmodel_0030.h5"
q = requests.get(model_path)
with open(MODEL_DIR, 'wb') as f:
f.write(q.content)
# Recreate the model in inference mode
model = MaskRCNN(mode="inference",
config=inference_config,
model_dir=MODEL_DIR)
model.load_weights(MODEL_DIR, by_name=True)
class_names = ['BG', 'Chicken', 'Eba', 'Fish', 'Rice', 'Bread']
#for image_path in image_paths:
# img = skimage.io.imread(IMG_DIR)
# img_arr = np.array(img)
# results = model.detect([img_arr], verbose=1)
# r = results[0]
#visualize.display_instances(img, r['rois'], r['masks'], r['class_ids'],
# class_names, r['scores'], figsize=(5,5))
#ans = visualize.apply_mask(img, r['masks'], color=None, alpha=0.5)
# define the name of the configuration
NAME = "part_cfg"
# number of classes (background + kangaroo)
NUM_CLASSES = 1 + 1
# simplify GPU config
GPU_COUNT = 1
IMAGES_PER_GPU = 1
# Load the model for inference
# create config
cfg = PredictionConfig()
# define the model
model_dir = os.path.join(os.getcwd(), "models")
model = MaskRCNN(mode='inference', model_dir=model_dir, config=cfg)
# load model weights
weights_dir = os.path.join(os.getcwd(), "models",
'mask_rcnn_part_cfg_0010.h5')
model.load_weights(weights_dir, by_name=True)
while status:
print(np.shape(color_image))
try:
# Predict for each frame -> One by one
# Convert pixel values
scaled_color_img = mold_image(color_image, cfg)
COCO_MODEL_PATH = "mask_rcnn_peruvian_bill_0005.h5"
class InferenceConfig(Config):
NAME = "PERUVIAN_BILL"
NUM_CLASSES = 1 + 4
GPU_COUNT = 1
IMAGES_PER_GPU = 1
STEPS_PER_EPOCH = 100
LEARNING_RATE = 0.0006
config = InferenceConfig()
config.display()
model = MaskRCNN(mode="inference", model_dir=MODEL_DIR, config=config)
model.load_weights(COCO_MODEL_PATH, by_name=True)
class_names = ['BG', 'B10', 'B100', 'B20', 'B50']
def random_colors(N):
np.random.seed(1)
colors = [tuple(255 * np.random.rand(3)) for _ in range(N)]
return colors
colors = random_colors(len(class_names))
class_dict = {name: color for name, color in zip(class_names, colors)}
def apply_mask(image, mask, color, alpha=0.5):
