def test_external_delegate_options_wrong_logging_level(delegate_dir):
with pytest.raises(ValueError):
tflite.load_delegate(delegate_dir,
options={"logging-severity": "wrong"})
import numpy as np
import tflite_runtime.interpreter as tflite
interpreter = tflite.Interpreter('converted_model.tflite', experimental_delegates=[tflite.load_delegate('libedgetpu.so.1')])
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
data = [1, 1, 1]
input_data = np.float32([data])
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
print(interpreter.get_tensor(output_details[0]['index']).tolist()[0])
def __init__(self,
weights='yolov5s.pt',
device=torch.device('cpu'),
dnn=False,
data=None,
fp16=False):
# Usage:
# PyTorch: weights = *.pt
# TorchScript: *.torchscript
# ONNX Runtime: *.onnx
# ONNX OpenCV DNN: *.onnx with --dnn
# OpenVINO: *.xml
# CoreML: *.mlmodel
# TensorRT: *.engine
# TensorFlow SavedModel: *_saved_model
# TensorFlow GraphDef: *.pb
# TensorFlow Lite: *.tflite
# TensorFlow Edge TPU: *_edgetpu.tflite
from models.experimental import attempt_download, attempt_load # scoped to avoid circular import
super().__init__()
w = str(weights[0] if isinstance(weights, list) else weights)
pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs = self.model_type(
w) # get backend
stride, names = 64, [f'class{i}'
for i in range(1000)] # assign defaults
w = attempt_download(w) # download if not local
fp16 &= (pt or jit or onnx or engine) and device.type != 'cpu' # FP16
if data: # data.yaml path (optional)
with open(data, errors='ignore') as f:
names = yaml.safe_load(f)['names'] # class names
if pt: # PyTorch
model = attempt_load(weights if isinstance(weights, list) else w,
map_location=device)
stride = max(int(model.stride.max()), 32) # model stride
names = model.module.names if hasattr(
model, 'module') else model.names # get class names
model.half() if fp16 else model.float()
self.model = model # explicitly assign for to(), cpu(), cuda(), half()
elif jit: # TorchScript
LOGGER.info(f'Loading {w} for TorchScript inference...')
extra_files = {'config.txt': ''} # model metadata
model = torch.jit.load(w, _extra_files=extra_files)
model.half() if fp16 else model.float()
if extra_files['config.txt']:
d = json.loads(extra_files['config.txt']) # extra_files dict
stride, names = int(d['stride']), d['names']
elif dnn: # ONNX OpenCV DNN
LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')
check_requirements(('opencv-python>=4.5.4', ))
net = cv2.dnn.readNetFromONNX(w)
elif onnx: # ONNX Runtime
LOGGER.info(f'Loading {w} for ONNX Runtime inference...')
cuda = torch.cuda.is_available()
check_requirements(
('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))
import onnxruntime
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'
] if cuda else ['CPUExecutionProvider']
session = onnxruntime.InferenceSession(w, providers=providers)
elif xml: # OpenVINO
LOGGER.info(f'Loading {w} for OpenVINO inference...')
check_requirements(
('openvino-dev', )
) # requires openvino-dev: https://pypi.org/project/openvino-dev/
import openvino.inference_engine as ie
core = ie.IECore()
if not Path(w).is_file(): # if not *.xml
w = next(Path(w).glob(
'*.xml')) # get *.xml file from *_openvino_model dir
network = core.read_network(
model=w,
weights=Path(w).with_suffix('.bin')) # *.xml, *.bin paths
executable_network = core.load_network(network,
device_name='CPU',
num_requests=1)
elif engine: # TensorRT
LOGGER.info(f'Loading {w} for TensorRT inference...')
import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download
check_version(trt.__version__, '7.0.0',
hard=True) # require tensorrt>=7.0.0
Binding = namedtuple('Binding',
('name', 'dtype', 'shape', 'data', 'ptr'))
logger = trt.Logger(trt.Logger.INFO)
with open(w, 'rb') as f, trt.Runtime(logger) as runtime:
model = runtime.deserialize_cuda_engine(f.read())
bindings = OrderedDict()
fp16 = False # default updated below
for index in range(model.num_bindings):
name = model.get_binding_name(index)
dtype = trt.nptype(model.get_binding_dtype(index))
shape = tuple(model.get_binding_shape(index))
data = torch.from_numpy(np.empty(
shape, dtype=np.dtype(dtype))).to(device)
bindings[name] = Binding(name, dtype, shape, data,
int(data.data_ptr()))
if model.binding_is_input(index) and dtype == np.float16:
fp16 = True
binding_addrs = OrderedDict(
(n, d.ptr) for n, d in bindings.items())
context = model.create_execution_context()
batch_size = bindings['images'].shape[0]
elif coreml: # CoreML
LOGGER.info(f'Loading {w} for CoreML inference...')
import coremltools as ct
model = ct.models.MLModel(w)
else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)
if saved_model: # SavedModel
LOGGER.info(
f'Loading {w} for TensorFlow SavedModel inference...')
import tensorflow as tf
keras = False # assume TF1 saved_model
model = tf.keras.models.load_model(
w) if keras else tf.saved_model.load(w)
elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt
LOGGER.info(
f'Loading {w} for TensorFlow GraphDef inference...')
import tensorflow as tf
def wrap_frozen_graph(gd, inputs, outputs):
x = tf.compat.v1.wrap_function(
lambda: tf.compat.v1.import_graph_def(gd, name=""),
[]) # wrapped
ge = x.graph.as_graph_element
return x.prune(tf.nest.map_structure(ge, inputs),
tf.nest.map_structure(ge, outputs))
gd = tf.Graph().as_graph_def() # graph_def
gd.ParseFromString(open(w, 'rb').read())
frozen_func = wrap_frozen_graph(gd,
inputs="x:0",
outputs="Identity:0")
elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python
try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu
from tflite_runtime.interpreter import Interpreter, load_delegate
except ImportError:
import tensorflow as tf
Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,
if edgetpu: # Edge TPU https://coral.ai/software/#edgetpu-runtime
LOGGER.info(
f'Loading {w} for TensorFlow Lite Edge TPU inference...'
)
delegate = {
'Linux': 'libedgetpu.so.1',
'Darwin': 'libedgetpu.1.dylib',
'Windows': 'edgetpu.dll'
}[platform.system()]
interpreter = Interpreter(
model_path=w,
experimental_delegates=[load_delegate(delegate)])
else: # Lite
LOGGER.info(
f'Loading {w} for TensorFlow Lite inference...')
interpreter = Interpreter(
model_path=w) # load TFLite model
interpreter.allocate_tensors() # allocate
input_details = interpreter.get_input_details() # inputs
output_details = interpreter.get_output_details() # outputs
elif tfjs:
raise Exception(
'ERROR: YOLOv5 TF.js inference is not supported')
self.__dict__.update(locals()) # assign all variables to self
def main(lidar_data_queue):
global perceptron_network
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--model',
help='File path of .tflite file.',
required=True)
parser.add_argument('--labels',
help='File path of labels file.',
required=True)
parser.add_argument('--threshold',
help='Score threshold for detected objects.',
required=False,
type=float,
default=0.6)
args = parser.parse_args()
labels = load_labels(args.labels)
interpreter = Interpreter(
args.model,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
interpreter.allocate_tensors()
_, input_height, input_width, _ = interpreter.get_input_details(
)[0]['shape']
print(input_height, input_width)
count = 0
with picamera.PiCamera(resolution=(CAMERA_WIDTH, CAMERA_HEIGHT)) as camera:
camera.rotation = 180
camera.start_preview()
try:
stream = io.BytesIO()
annotator = Annotator(camera)
for _ in camera.capture_continuous(stream,
format='jpeg',
use_video_port=True):
start_time = time.monotonic()
stream.seek(0)
image = Image.open(stream).convert('RGB').resize(
(input_width, input_height), Image.ANTIALIAS)
# start_time = time.monotonic()
results = detect_objects(interpreter, image, args.threshold)
#elapsed_ms = (time.monotonic() - start_time) * 1000
annotator.clear()
middle_xy = annotate_objects(annotator, results, labels)
#annotator.text([5, 0], '%.1fms' % (elapsed_ms))
# annotator.update()
if perceptron_network.person_detected == True: # this only changes the first time a person is detected
data = []
if lidar_data_queue.empty() is False:
data = lidar_data_queue.get()
else:
data = [0]
if middle_xy[
0] != 0: # or ((middle_xy[0] < 300 and middle_xy != 0) and perceptron_network.first_arc_turn == True) or (middle_xy[0] > 20 and perceptron_network.first_arc_turn == True):
perceptron_network.person_detected_queue.insert(
0, True) # keep track og frames with person in it
perceptron_network.person_detected_queue.pop()
object_width = middle_xy[2]
print('object width: ', object_width)
if perceptron_network.first_arc_turn == True:
object_width = 80
distance_to_target = perceptron_network.getDistanceToPerson(
object_width)
print('distance = ', distance_to_target)
new_motor_speeds = perceptron_network.followTarget(
middle_xy, distance_to_target, data)
motorSpeedLeft(
1,
round(perceptron_network.motor_speed_total_left))
motorSpeedRight(
1,
round(perceptron_network.motor_speed_total_right))
print(
"Left motor: ",
round(perceptron_network.motor_speed_total_left),
" Right motor: ",
round(perceptron_network.motor_speed_total_right))
print('..........................................')
elif perceptron_network.first_arc_turn == True or (
perceptron_network.first_reflex_turn == True):
arc_motor_speeds = perceptron_network.makeArcTurn(
perceptron_network.reflex_avoiding_obstacle_dist +
400)
perceptron_network.motor_speed_total_left = arc_motor_speeds[
0]
perceptron_network.motor_speed_total_right = arc_motor_speeds[
1]
motorSpeedLeft(1, arc_motor_speeds[0])
motorSpeedRight(1, arc_motor_speeds[1])
print('second',
perceptron_network.motor_speed_total_left)
elif perceptron_network.first_arc_turn == True or (
perceptron_network.making_avoidance_turn == True):
arc_motor_speeds = perceptron_network.makeArcTurn(
perceptron_network.start_avoiding_obstacle_dist +
300)
perceptron_network.motor_speed_total_left = arc_motor_speeds[
0]
perceptron_network.motor_speed_total_right = arc_motor_speeds[
1]
motorSpeedLeft(1, arc_motor_speeds[0])
motorSpeedRight(1, arc_motor_speeds[1])
print('second',
perceptron_network.motor_speed_total_left)
else:
perceptron_network.person_detected_queue.insert(
0, False)
perceptron_network.person_detected_queue.pop()
# Is all the last 15 frames was without a person
if any(perceptron_network.person_detected_queue
) == False:
perceptron_network.motor_speed_total_left = 0
perceptron_network.motor_speed_total_right = 0
perceptron_network.motor_speed_distance = 0
print("Locating target....")
perceptron_network.robot_is_stationary = True
if perceptron_network.side_left_person_last_detected == True:
motorSpeedLeft(0, 19)
motorSpeedRight(1, 19)
elif perceptron_network.side_left_person_last_detected == False:
motorSpeedLeft(1, 19)
motorSpeedRight(0, 19)
# For calibrating the focal length
# focal = perceptron_network.getPercievedFocal(object_width, 2000, 500)
# print('focal = ', focal)
elapsed_ms = (time.monotonic() - start_time) * 1000
annotator.text([5, 0], '%.1fms' % (elapsed_ms))
annotator.update()
frame_times_for_coral_test.append(elapsed_ms)
#print(perceptron_network.getPercievedFocal(object_height, distance_test, person_height))
stream.seek(0)
stream.truncate()
except KeyboardInterrupt:
print('Saving distance data and shutting down')
motorSpeedLeft(1, 0)
motorSpeedRight(1, 0)
toggleLED({})
frame_average = sum(frame_times_for_coral_test) / len(
frame_times_for_coral_test)
#perceptron_network.save_test1()
#perceptron_network.save_test2()
#perceptron_network.save_test3()
#perceptron_network.save_test4()
perceptron_network.saveWeights()
# makePlots(perceptron_network.percep_l_wheel.weights_for_test, perceptron_network.percep_r_wheel.weights_for_test, perceptron_network.percep_far_distance.weights_for_test, perceptron_network.distances_for_test)
# file = open('distances.csv', 'w')
# file.truncate()
# with file:
# writer = csv.writer(file)
# writer.writerow(perceptron_network.distances_for_test)
# writer.writerow(perceptron_network.percep_l_wheel.weights_for_test)
# writer.writerow(perceptron_network.percep_r_wheel.weights_for_test)
# writer.writerow(perceptron_network.percep_distance.weights_for_test)
finally:
camera.stop_preview()
def __init__(self, weights='yolov5s.pt', device=None, dnn=False):
# Usage:
# PyTorch: weights = *.pt
# TorchScript: *.torchscript
# CoreML: *.mlmodel
# TensorFlow: *_saved_model
# TensorFlow: *.pb
# TensorFlow Lite: *.tflite
# ONNX Runtime: *.onnx
# OpenCV DNN: *.onnx with dnn=True
# TensorRT: *.engine
from models.experimental import attempt_download, attempt_load # scoped to avoid circular import
super().__init__()
w = str(weights[0] if isinstance(weights, list) else weights)
suffix = Path(w).suffix.lower()
suffixes = [
'.pt', '.torchscript', '.onnx', '.engine', '.tflite', '.pb', '',
'.mlmodel'
]
check_suffix(w, suffixes) # check weights have acceptable suffix
pt, jit, onnx, engine, tflite, pb, saved_model, coreml = (
suffix == x for x in suffixes) # backend booleans
stride, names = 64, [f'class{i}'
for i in range(1000)] # assign defaults
attempt_download(w) # download if not local
if jit: # TorchScript
LOGGER.info(f'Loading {w} for TorchScript inference...')
extra_files = {'config.txt': ''} # model metadata
model = torch.jit.load(w, _extra_files=extra_files)
if extra_files['config.txt']:
d = json.loads(extra_files['config.txt']) # extra_files dict
stride, names = int(d['stride']), d['names']
elif pt: # PyTorch
model = attempt_load(weights, map_location=device)
stride = int(model.stride.max()) # model stride
names = model.module.names if hasattr(
model, 'module') else model.names # get class names
self.model = model # explicitly assign for to(), cpu(), cuda(), half()
elif coreml: # CoreML
LOGGER.info(f'Loading {w} for CoreML inference...')
import coremltools as ct
model = ct.models.MLModel(w)
elif dnn: # ONNX OpenCV DNN
LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')
check_requirements(('opencv-python>=4.5.4', ))
net = cv2.dnn.readNetFromONNX(w)
elif onnx: # ONNX Runtime
LOGGER.info(f'Loading {w} for ONNX Runtime inference...')
check_requirements(('onnx', 'onnxruntime-gpu' if
torch.cuda.is_available() else 'onnxruntime'))
import onnxruntime
session = onnxruntime.InferenceSession(w, None)
elif engine: # TensorRT
LOGGER.info(f'Loading {w} for TensorRT inference...')
import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download
Binding = namedtuple('Binding',
('name', 'dtype', 'shape', 'data', 'ptr'))
logger = trt.Logger(trt.Logger.INFO)
with open(w, 'rb') as f, trt.Runtime(logger) as runtime:
model = runtime.deserialize_cuda_engine(f.read())
bindings = OrderedDict()
for index in range(model.num_bindings):
name = model.get_binding_name(index)
dtype = trt.nptype(model.get_binding_dtype(index))
shape = tuple(model.get_binding_shape(index))
data = torch.from_numpy(np.empty(
shape, dtype=np.dtype(dtype))).to(device)
bindings[name] = Binding(name, dtype, shape, data,
int(data.data_ptr()))
binding_addrs = OrderedDict(
(n, d.ptr) for n, d in bindings.items())
context = model.create_execution_context()
batch_size = bindings['images'].shape[0]
else: # TensorFlow model (TFLite, pb, saved_model)
if pb: # https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt
LOGGER.info(f'Loading {w} for TensorFlow *.pb inference...')
import tensorflow as tf
def wrap_frozen_graph(gd, inputs, outputs):
x = tf.compat.v1.wrap_function(
lambda: tf.compat.v1.import_graph_def(gd, name=""),
[]) # wrapped
return x.prune(
tf.nest.map_structure(x.graph.as_graph_element,
inputs),
tf.nest.map_structure(x.graph.as_graph_element,
outputs))
graph_def = tf.Graph().as_graph_def()
graph_def.ParseFromString(open(w, 'rb').read())
frozen_func = wrap_frozen_graph(gd=graph_def,
inputs="x:0",
outputs="Identity:0")
elif saved_model:
LOGGER.info(
f'Loading {w} for TensorFlow saved_model inference...')
import tensorflow as tf
model = tf.keras.models.load_model(w)
elif tflite: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python
if 'edgetpu' in w.lower():
LOGGER.info(
f'Loading {w} for TensorFlow Lite Edge TPU inference...'
)
import tflite_runtime.interpreter as tfli
delegate = {
'Linux':
'libedgetpu.so.1', # install https://coral.ai/software/#edgetpu-runtime
'Darwin': 'libedgetpu.1.dylib',
'Windows': 'edgetpu.dll'
}[platform.system()]
interpreter = tfli.Interpreter(
model_path=w,
experimental_delegates=[tfli.load_delegate(delegate)])
else:
LOGGER.info(
f'Loading {w} for TensorFlow Lite inference...')
import tensorflow as tf
interpreter = tf.lite.Interpreter(
model_path=w) # load TFLite model
interpreter.allocate_tensors() # allocate
input_details = interpreter.get_input_details() # inputs
output_details = interpreter.get_output_details() # outputs
self.__dict__.update(locals()) # assign all variables to self
def make_interpreter(model_file):
EDGETPU_SHARED_LIB = '/usr/lib/aarch64-linux-gnu/libedgetpu.so.1'
return tflite.Interpreter(
model_path=model_file,
experimental_delegates=[
tflite.load_delegate(EDGETPU_SHARED_LIB)])
def open_dialog_box(self):
img = QFileDialog.getOpenFileName()
img_elec = img[0]
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir',
help='Folder the .tflite file is located in',
default='medbox')
parser.add_argument(
'--graph',
help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument(
'--labels',
help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument(
'--threshold',
help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument(
'--image',
help=
'Name of the single image to perform detection on. To run detection on multiple images, use --imagedir',
default=None)
parser.add_argument(
'--imagedir',
help=
'Name of the folder containing images to perform detection on. Folder must contain only images.',
default=None)
parser.add_argument(
'--edgetpu',
help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
use_TPU = args.edgetpu
# Parse input image name and directory.
IM_NAME = args.image
IM_DIR = args.imagedir
# If both an image AND a folder are specified, throw an error
if (IM_NAME and IM_DIR):
print(
'Error! Please only use the --image argument or the --imagedir argument, not both. Issue "python TFLite_detection_image.py -h" for help.'
)
sys.exit()
# If neither an image or a folder are specified, default to using 'test1.jpg' for image name
if (not IM_NAME and not IM_DIR):
IM_NAME = img_elec
# Import TensorFlow libraries
# If tensorflow is not installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tensorflow')
if pkg is None:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Define path to images and grab all image filenames
if IM_DIR:
PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_DIR)
images = glob.glob(PATH_TO_IMAGES + '/*')
elif IM_NAME:
PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_NAME)
images = glob.glob(PATH_TO_IMAGES)
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
start_time = time.time()
# Loop over every image and perform detection
for image_path in images:
# Load image and resize to expected shape [1xHxWx3]
image = cv2.imread(image_path)
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
imH, imW, _ = image.shape
image_resized = cv2.resize(image_rgb, (width, height))
input_data = np.expand_dims(image_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[
0] # Bounding box coordinates of detected objects
classes = interpreter.get_tensor(output_details[1]['index'])[
0] # Class index of detected objects
scores = interpreter.get_tensor(output_details[2]['index'])[
0] # Confidence of detected objects
#num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed)
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
ymin = int(max(1, (boxes[i][0] * imH)))
xmin = int(max(1, (boxes[i][1] * imW)))
ymax = int(min(imH, (boxes[i][2] * imH)))
xmax = int(min(imW, (boxes[i][3] * imW)))
# Draw label
object_name = labels[int(
classes[i]
)] # Look up object name from "labels" array using class index
label = '%s' % (object_name) # Example: 'person'
labelSize, baseLine = cv2.getTextSize(
label, cv2.FONT_HERSHEY_SIMPLEX, 0.7,
2) # Get font size
label_ymin = max(
ymin, labelSize[1] + 10
) # Make sure not to draw label too close to top of window
cv2.rectangle(
image, (xmin, label_ymin - labelSize[1] - 10),
(xmin + labelSize[0], label_ymin + baseLine - 10),
(255, 255, 255),
cv2.FILLED) # Draw white box to put label text in
cv2.putText(image, label, (xmin, label_ymin - 7),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
2) # Draw label text
# All the results have been drawn on the image, now display the image
cv2.imshow('Q=quit', image)
print("PROCESSING TIME: %s seconds" % (time.time() - start_time))
# Press any key to continue to next image, or press 'q' to quit
if cv2.waitKey(0) == ord('q'):
break
# Clean up
cv2.destroyAllWindows()
def init(model_dir,
show_images=False,
graph='detect.tflite',
labelmap_name='labelmap.txt',
threshold=0.5,
edgetpu=False):
# globals
global CWD_PATH
global width
global height
global floating_model
global interpreter
global input_details
global output_details
global SHOW_IMAGES
MODEL_NAME = model_dir
SHOW_IMAGES = show_images
GRAPH_NAME = graph
LABELMAP_NAME = labelmap_name
min_conf_threshold = float(threshold)
use_TPU = edgetpu
# Import TensorFlow libraries
# If tensorflow is not installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tensorflow')
if pkg is None:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
EDGETPU_SHARED_LIB = {
'Linux': 'libedgetpu.so.1',
'Darwin': 'libedgetpu.1.dylib',
'Windows': 'edgetpu.dll'
}[platform.system()]
if len(sys.argv) < 3:
print('Usage:', sys.argv[0], '<model_path> <test_image_dir>')
exit()
model_path = str(sys.argv[1])
# Creates tflite interpreter
if 'edgetpu' in model_path:
interpreter = Interpreter(model_path, experimental_delegates=[
load_delegate(EDGETPU_SHARED_LIB)])
else:
interpreter = Interpreter(model_path)
interpreter.allocate_tensors()
interpreter.invoke() # warmup
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
width = input_details[0]['shape'][2]
height = input_details[0]['shape'][1]
def run_inference(interpreter, image):
interpreter.set_tensor(input_details[0]['index'], image)
interpreter.invoke()
boxes = interpreter.get_tensor(output_details[0]['index'])[0]
def load_edgetpu_delegate(options=None):
"""Loads the Edge TPU delegate with the given options."""
return tflite.load_delegate(_EDGETPU_SHARED_LIB, options or {})
def main():
import sys, getopt
checknum = 0
while True:
try:
# face recognizing code
print('face camera ')
args, video_src = getopt.getopt(sys.argv[1:2], '',
['cascade=', 'nested-cascade='])
try:
video_src = video_src[0]
except:
video_src = 0
args = dict(args)
cascade_fn = args.get(
'--cascade',
"data/haarcascades/haarcascade_frontalface_alt.xml")
nested_fn = args.get('--nested-cascade',
"data/haarcascades/haarcascade_eye.xml")
cascade = cv.CascadeClassifier(cv.samples.findFile(cascade_fn))
nested = cv.CascadeClassifier(cv.samples.findFile(nested_fn))
cam = create_capture(
video_src,
fallback='synth:bg={}:noise=0.05'.format(
cv.samples.findFile('samples/data/lena.jpg')))
while True:
ret, img = cam.read()
gray = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
gray = cv.equalizeHist(gray)
rects = detect(gray, cascade)
vis = img.copy()
if len(rects):
if not nested.empty():
print('into nested') # 사람이 들어왔을 때
for x1, y1, x2, y2 in rects:
roi = gray[y1:y2, x1:x2]
vis_roi = vis[y1:y2, x1:x2]
print('findrects')
subrects = detect(roi.copy(), nested)
if subrects != '[]':
faceok = 'faceok.wav'
fa = sa.WaveObject.from_wave_file(faceok)
face = fa.play()
face.wait_done()
print('detect!!')
break
cam.release() # face recognition camera off
print("helmet camera")
# helmet detectecting code
filename = 'helmet.wav'
wave_obj = sa.WaveObject.from_wave_file(filename)
helmetok = 'helmetok.wav'
wave = sa.WaveObject.from_wave_file(helmetok)
labels = "labels.txt"
model = "model_edgetpu.tflite"
interpreter = Interpreter(
model,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
interpreter.allocate_tensors()
_, height, width, _ = interpreter.get_input_details()[0]['shape']
# helmet detect camera on
with picamera.PiCamera(resolution=(640, 480),
framerate=30) as camera:
camera.start_preview()
try:
stream = io.BytesIO()
for _ in camera.capture_continuous(stream,
format='jpeg',
use_video_port=True):
stream.seek(0)
image = Image.open(stream).convert('RGB').resize(
(width, height), Image.ANTIALIAS)
results = classify_image(interpreter, image)
print("result:")
print(results)
stream.seek(0)
stream.truncate()
# 헬멧 착용여부 판단
if results == 0:
play_obj = wave_obj.play()
play_obj.wait_done()
checknum += 1
if checknum == 3:
checknum = 0
break
else:
helm = wave.play()
helm.wait_done()
print('GoodBoy')
break
finally:
camera.stop_preview()
except KeyboardInterrupt:
break
#!pip3 install https://dl.google.com/coral/python/tflite_runtime-2.1.0.post1-cp36-cp36m-linux_x86_64.whl
import os
import numpy as np
from tflite_runtime.interpreter import Interpreter
from tflite_runtime.interpreter import load_delegate
from PIL import Image
from PIL import ImageDraw
detect_path='./test_img'
save_path='./saved_detect'
#interpreter = Interpreter('./model_float32.tflite')
interpreter = Interpreter('./model_float32.tflite', experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
interpreter.allocate_tensors()
interpreter.invoke() # warmup
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
#print("input_details:", str(input_details))
#print("output_details:", str(output_details))
width = input_details[0]['shape'][2]
height = input_details[0]['shape'][1]
def run_inference(interpreter, image):
interpreter.set_tensor(input_details[0]['index'], image)
interpreter.invoke()
boxes = interpreter.get_tensor(output_details[0]['index'])[0]
classes = interpreter.get_tensor(output_details[1]['index'])[0]
scores = interpreter.get_tensor(output_details[2]['index'])[0]
# num_detections = interpreter.get_tensor(output_details[3]['index'])[0]
return boxes, classes, scores
def predict():
MODEL_NAME = "model"
GRAPH_NAME = 'glyphs.tflite'
LABELMAP_NAME = "labelmap.txt"
min_conf_threshold = float(0.3)
use_TPU = False
IM_NAME = None
IM_DIR = "images"
if (IM_NAME and IM_DIR):
print(
'Error! Please only use the --image argument or the --imagedir argument, not both. Issue "python TFLite_detection_image.py -h" for help.'
)
sys.exit()
if (not IM_NAME and not IM_DIR):
IM_NAME = 'test1.jpg'
pkg = importlib.util.find_spec('tensorflow')
if pkg is None:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
if use_TPU:
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
CWD_PATH = os.getcwd()
if IM_DIR:
PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_DIR)
images = glob.glob(PATH_TO_IMAGES + '/*')
elif IM_NAME:
PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_NAME)
images = glob.glob(PATH_TO_IMAGES)
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
if labels[0] == '???':
del (labels[0])
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
results = {}
for image_path in images:
image = cv2.imread(image_path)
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
imH, imW, _ = image.shape
image_resized = cv2.resize(image_rgb, (width, height))
input_data = np.expand_dims(image_resized, axis=0)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
boxes = interpreter.get_tensor(output_details[0]['index'])[0]
classes = interpreter.get_tensor(output_details[1]['index'])[0]
scores = interpreter.get_tensor(output_details[2]['index'])[0]
num = interpreter.get_tensor(output_details[3]['index'])[0]
results['boxes'] = boxes
results['classes'] = classes
results['scores'] = scores
results['num'] = num
return results
cv2.destroyAllWindows()
def init(model_dir,
show_images=False,
resolution='640x480',
graph='model.tflite',
labelmap_name='labelmap.txt',
threshold=0.5,
edgetpu=False):
# globals
global CWD_PATH
global width
global height
global floating_model
global interpreter
global input_details
global output_details
global frame_rate_calc
global freq
global videostream
global t1
global t2
global time1
global imageWidth
global imageHeight
global min_conf_threshold
global frame_rate_calc
global SHOW_IMAGES
global LabelsTF
global FOCALLENGTH
# in mm
FOCALLENGTH = 4.8
# in mm
global CELLPHONEXSIZE
global TARGETYSIZE
CELLPHONEXSIZE = 68.2
TARGETYSIZE = 145.6
graph_def_file = model_dir + "/saved_model/saved_model.pb"
input_arrays = ["Input"]
output_arrays = ["output"]
converter = tf.contrib.lite.TocoConverter.from_frozen_graph(
graph_def_file, input_arrays, output_arrays)
tflite_model = converter.convert()
open("converted_model.tflite", "wb").write(tflite_model)
MODEL_NAME = 'model.tflite'
SHOW_IMAGES = show_images
GRAPH_NAME = graph
LABELMAP_NAME = labelmap_name
min_conf_threshold = float(threshold)
use_TPU = edgetpu
resW, resH = resolution.split('x')
imageWidth, imageHeight = int(resW), int(resH)
# Import TensorFlow libraries
# If tensorflow is not installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tensorflow')
if pkg is None:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
LabelsTF = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if LabelsTF[0] == '???':
del (LabelsTF[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
# Initialize video stream
videostream = VideoStream(resolution=(imageWidth, imageHeight),
framerate=30).start()
time.sleep(1)
time_stop.append(time.time())
cores.append(psutil.cpu_percent(interval=None, percpu=True))
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# ImRecKerasRead.py
# 124 images from COCO val2017 dataset
print("ImRecKerasRead")
path_image = "./images/KerasRead/"
extention = ".jpg"
imagenet_labels = np.array(open(path_image+"ImageNetLabels.txt").read().splitlines())
# Load TFLite model and allocate tensors.
interpreter = tflite.Interpreter(model_path="./SavedNN/ImRecKerasModel/ImRecKerasModel.tflite",
experimental_delegates=[tflite.load_delegate('libedgetpu.so.1')])
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
floating_model = input_details[0]['dtype'] == np.float32
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
input_data2 = []
with open("./images/KerasRead/labels.txt", mode='r') as label_file:
for label in label_file:
path = path_image + label.rstrip() + extention
img = Image.open(path).resize((width, height))
def make_interpreter():
return tflite.Interpreter(
model_path=MODELS,
experimental_delegates=[tflite.load_delegate(EDGETPU_SHARED_LIB)])
converter = tf.compat.v1.lite.TFLiteConverter.from_keras_model_file(
save_file)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.representative_dataset = representative_dataset_gen
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.target_spec.supported_types = [tf.int8]
converter.inference_input_type = tf.uint8
converter.inference_output_type = tf.uint8
model = converter.convert()
save_file += ".tflite"
open(save_file, "wb").write(model)
else:
interpreter = tflite.Interpreter(
model_path=save_file,
experimental_delegates=[tflite.load_delegate(EDGETPU_SHARED_LIB, {})])
interpreter.allocate_tensors()
for i in range(10):
input_index = interpreter.get_input_details()[0]["index"]
output_index = interpreter.get_output_details()[0]["index"]
# Pre-processing: add batch dimension and convert to float32 to match with the model's input data format.
test_image = np.expand_dims(train_data_gen[0][0][0],
axis=0).astype(np.uint8)
interpreter.set_tensor(input_index, test_image)
start = time.time()
interpreter.invoke() # Run inference.
# Post-processing: remove batch dimension and find the digit with highest probability.
output = interpreter.tensor(output_index)
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
PATH_TO_CKPT, *device = PATH_TO_CKPT.split('@')
interpreter = tflite.Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[
tflite.load_delegate(EDGETPU_SHARED_LIB,
{'device': device[0]} if device else {})
])
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Open video file
def make_interpreter(model_file):
return tflite.Interpreter(
model_path=model_file,
experimental_delegates=[tflite.load_delegate(EDGETPU_SHARED_LIB, {})])
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--model',
help='File path of .tflite file.',
required=True)
parser.add_argument('--labels',
help='File path of labels file.',
required=True)
parser.add_argument('--threshold',
help='Score threshold for detected objects.',
required=False,
type=float,
default=0.4)
args = parser.parse_args()
labels = load_labels(args.labels)
interpreter = Interpreter(
args.model,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')]) #coral
interpreter.allocate_tensors()
_, input_height, input_width, _ = interpreter.get_input_details(
)[0]['shape']
# initialize variables to calculate FPS
instantaneous_frame_rates = []
# initialize variable for tracker use
t = None
test_time_all = []
counter = 0
test_start_time = time.monotonic()
with picamera.PiCamera(resolution=(CAMERA_WIDTH, CAMERA_HEIGHT),
framerate=30) as camera:
camera.start_preview() #alpha = 200
start_time = time.monotonic()
try:
stream = io.BytesIO()
annotator = Annotator(camera)
for _ in camera.capture_continuous(stream,
format='jpeg',
use_video_port=True):
test_time = (time.monotonic() - test_start_time)
test_time_all.append(test_time)
print(
str(sum(test_time_all) / len(test_time_all)) + ", FPS: " +
str(1 / (sum(test_time_all) / len(test_time_all))))
stream.seek(0)
counter += 1
image = Image.open(stream).convert('RGB')
cv_img = np.asarray(image)
annotator.clear()
# if there are no trackes, first must try to detect objects
if t == None:
image = image.resize((input_width, input_height),
Image.ANTIALIAS)
results = detect_objects(interpreter, image,
args.threshold)
rects = get_rects(results)
for i in np.arange(0, len(results)):
#format bounding box coordinates
print("new tracker")
box = np.array(rects[i])
(startY, startX, endY, endX) = box.astype("int")
cv_rect = (startX, startY, endX - startX,
endY - startY)
t = cv2.TrackerMOSSE_create()
t.init(cv_img, cv_rect)
annotator.bounding_box([startX, startY, endX, endY])
#annotate_objects(annotator, results, labels)
else:
(success, box) = t.update(cv_img)
if success:
annotator.bounding_box(
[box[0], box[1], box[0] + box[2], box[1] + box[3]])
#cv2.rectangle(cv_img, (int(box[0]), int(box[1])), (int(box[0] + box[2]), int(box [1] + box[3])),(0, 255, 0), 2)
#if (counter % 40) == 0:
#t = None
#elapsed_ms = (time.monotonic() - start_time) * 1000
#annotator.text([5, 0], '%.1f ms' % (elapsed_ms))
#frame_rate = 1/ ((time.monotonic() - start_time))
#start_time = time.monotonic()
#print(frame_rate)
#calculate average FPS
#instantaneous_frame_rates.append(frame_rate)
#avg_frame_rate = sum(instantaneous_frame_rates)/len(instantaneous_frame_rates)
#print("FPS: " + str(avg_frame_rate))
#annotator.text([5, 15], '%.1f FPS' % (avg_frame_rate))
#annotator.clear()
annotator.update()
stream.seek(0)
stream.truncate()
test_start_time = time.monotonic()
finally:
camera.stop_preview()
params = json.load(readFile)
img = cv2.imread(input_img)
resized_img = letter_box_image(img, (params["input_w"], params["input_h"]),
128)
img = cv2.cvtColor(resized_img, cv2.COLOR_BGR2RGB)
img = img.astype(np.float32)
img = np.expand_dims(img, axis=0)
classes = load_coco_names(class_names)
colors = [(random.randint(0, 255), random.randint(0, 255),
random.randint(0, 255)) for _ in range(len(classes))]
# Load TFLite model and allocate tensors.
interpreter = tflite.Interpreter(
model_path=tflite_model_path,
experimental_delegates=[load_delegate("libedgetpu.so.1.0")])
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Dequantization
scale, zero_point = input_details[0]['quantization']
img = np.uint8(img / scale + zero_point)
# Test model on random input data.
interpreter.set_tensor(input_details[0]['index'], img)
interpreter.invoke()
def initDetector(model_path):
interpreter = tflite.Interpreter(model_path,
experimental_delegates=[tflite.load_delegate('libedgetpu.so.1')])
interpreter.allocate_tensors()
return interpreter
def app_args():
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir',
help='Folder the .tflite file is located in',
default='coco_ssd_mobilenet_v1')
parser.add_argument(
'--graph',
help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument(
'--labels',
help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument(
'--threshold',
help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument(
'--resolution',
help=
'Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
default='1280x720')
parser.add_argument(
'--edgetpu',
help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
global MODEL_NAME, GRAPH_NAME, LABELMAP_NAME, min_conf_threshold, resW, resH, imW, imH, use_TPU
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
resW, resH = args.resolution.split('x')
imW, imH = int(resW), int(resH)
use_TPU = args.edgetpu
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
global labels
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
global input_details, output_details, height, width, floating_model
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
return interpreter
def __init__(
self,
model_path: str,
options: ObjectDetectorOptions = ObjectDetectorOptions()
) -> None:
"""Initialize a TFLite object detection model.
Args:
model_path: Path to the TFLite model.
options: The config to initialize an object detector. (Optional)
Raises:
ValueError: If the TFLite model is invalid.
OSError: If the current OS isn't supported by EdgeTPU.
"""
# Load metadata from model.
displayer = metadata.MetadataDisplayer.with_model_file(model_path)
# Save model metadata for preprocessing later.
model_metadata = json.loads(displayer.get_metadata_json())
process_units = model_metadata['subgraph_metadata'][0][
'input_tensor_metadata'][0]['process_units']
mean = 127.5
std = 127.5
for option in process_units:
if option['options_type'] == 'NormalizationOptions':
mean = option['options']['mean'][0]
std = option['options']['std'][0]
self._mean = mean
self._std = std
# Load label list from metadata.
file_name = displayer.get_packed_associated_file_list()[0]
label_map_file = displayer.get_associated_file_buffer(file_name).decode()
label_list = list(filter(len, label_map_file.splitlines()))
self._label_list = label_list
# Initialize TFLite model.
if options.enable_edgetpu:
if edgetpu_lib_name() is None:
raise OSError("The current OS isn't supported by Coral EdgeTPU.")
interpreter = Interpreter(
model_path=model_path,
experimental_delegates=[load_delegate(edgetpu_lib_name())],
num_threads=options.num_threads)
else:
interpreter = Interpreter(
model_path=model_path, num_threads=options.num_threads)
interpreter.allocate_tensors()
input_detail = interpreter.get_input_details()[0]
# From TensorFlow 2.6, the order of the outputs become undefined.
# Therefore we need to sort the tensor indices of TFLite outputs and to know
# exactly the meaning of each output tensor. For example, if
# output indices are [601, 599, 598, 600], tensor names and indices aligned
# are:
# - location: 598
# - category: 599
# - score: 600
# - detection_count: 601
# because of the op's ports of TFLITE_DETECTION_POST_PROCESS
# (https://github.com/tensorflow/tensorflow/blob/a4fe268ea084e7d323133ed7b986e0ae259a2bc7/tensorflow/lite/kernels/detection_postprocess.cc#L47-L50).
sorted_output_indices = sorted(
[output['index'] for output in interpreter.get_output_details()])
self._output_indices = {
self._OUTPUT_LOCATION_NAME: sorted_output_indices[0],
self._OUTPUT_CATEGORY_NAME: sorted_output_indices[1],
self._OUTPUT_SCORE_NAME: sorted_output_indices[2],
self._OUTPUT_NUMBER_NAME: sorted_output_indices[3],
}
l = input_detail['shape'].tolist()
self._input_size = l[2], l[1]
self._is_quantized_input = input_detail['dtype'] == np.uint8
self._interpreter = interpreter
self._options = options
def __init__(
self,
model_path: str,
options: ImageSegmenterOptions = ImageSegmenterOptions()
) -> None:
"""Initialize a image segmentation model.
Args:
model_path: Name of the TFLite image segmentation model.
options: The config to initialize an image segmenter. (Optional)
Raises:
ValueError: If the TFLite model is invalid.
OSError: If the current OS isn't supported by EdgeTPU.
"""
# Load metadata from model.
displayer = metadata.MetadataDisplayer.with_model_file(model_path)
# Save model metadata for preprocessing later.
model_metadata = json.loads(displayer.get_metadata_json())
process_units = model_metadata['subgraph_metadata'][0][
'input_tensor_metadata'][0]['process_units']
mean = 127.5
std = 127.5
for option in process_units:
if option['options_type'] == 'NormalizationOptions':
mean = option['options']['mean'][0]
std = option['options']['std'][0]
self._mean = mean
self._std = std
# Load label list from metadata.
file_name = displayer.get_packed_associated_file_list()[0]
label_map_file = displayer.get_associated_file_buffer(
file_name).decode()
label_list = list(filter(len, label_map_file.splitlines()))
self._label_list = label_list
# Initialize TFLite model.
if options.enable_edgetpu:
if edgetpu_lib_name() is None:
raise OSError(
"The current OS isn't supported by Coral EdgeTPU.")
interpreter = Interpreter(
model_path=model_path,
experimental_delegates=[load_delegate(edgetpu_lib_name())],
num_threads=options.num_threads)
else:
interpreter = Interpreter(model_path=model_path,
num_threads=options.num_threads)
interpreter.allocate_tensors()
self._options = options
self._input_index = interpreter.get_input_details()[0]['index']
self._output_index = interpreter.get_output_details()[0]['index']
self._input_height = interpreter.get_input_details()[0]['shape'][1]
self._input_width = interpreter.get_input_details()[0]['shape'][2]
self._is_quantized_input = interpreter.get_input_details(
)[0]['dtype'] == np.uint8
self._interpreter = interpreter
def capture_and_detect(center_x, center_y, use_TPU):
MODEL_NAME = '/home/pi/rpi_detect_track/rpi/Sample_TFLite_model'
GRAPH_NAME = 'detect.tflite'
LABELMAP_NAME = 'labelmap.txt'
min_conf_threshold = float(0.5)
resW, resH = 1280, 720
imW, imH = int(resW), int(resH)
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del(labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
# Initialize video stream
videostream = VideoStream(resolution=(imW,imH), framerate=30).start()
time.sleep(1)
while True:
# Start timer (for calculating frame rate)
t1 = cv2.getTickCount()
# Grab frame from video stream
frame1 = videostream.read()
# Acquire frame and resize to expected shape [1xHxWx3]
frame = frame1.copy()
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb, (width, height))
input_data = np.expand_dims(frame_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[0]
classes = interpreter.get_tensor(output_details[1]['index'])[0]
scores = interpreter.get_tensor(output_details[2]['index'])[0]
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0) and labels[int(classes[i])] == 'person'):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions,
# need to force them to be within image using max() and min()
ymin = int(max(1, (boxes[i][0] * imH)))
xmin = int(max(1, (boxes[i][1] * imW)))
ymax = int(min(imH, (boxes[i][2] * imH)))
xmax = int(min(imW, (boxes[i][3] * imW)))
center_x.value = imW - (xmin + ((xmax - xmin) / 2))
center_y.value = imH - (ymin + ((ymax - ymin) / 2))
cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)
# Draw label
label = 'person: %d%%' % (int(scores[i] * 100))
labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2)
label_ymin = max(ymin, labelSize[1] + 10)
cv2.rectangle(frame, (xmin, label_ymin - labelSize[1] - 10), (xmin + labelSize[0], label_ymin + baseLine - 10), (255, 255, 255), cv2.FILLED)
cv2.putText(frame, label, (xmin, label_ymin - 7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2)
break
# Draw framerate in corner of frame
cv2.putText(frame, 'FPS: {0:.2f}'.format(frame_rate_calc), (30,50), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,0), 2, cv2.LINE_AA)
# All the results have been drawn on the frame, so it's time to display it.
cv2.imshow('Object detector', frame)
# Calculate framerate
t2 = cv2.getTickCount()
time1 = (t2 - t1) / freq
frame_rate_calc = 1 / time1
# Press 'q' to quit
if cv2.waitKey(1) == ord('q'):
break
# Clean up
cv2.destroyAllWindows()
videostream.stop()
def Neural_Network(queue):
class VideoStream:
"""Camera object that controls video streaming from the Picamera"""
def __init__(self,resolution=(640,480),framerate=30):
# Initialize the PiCamera and the camera image stream
self.stream = cv2.VideoCapture(0)
ret = self.stream.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
ret = self.stream.set(3,resolution[0])
ret = self.stream.set(4,resolution[1])
# Read first frame from the stream
(self.grabbed, self.frame) = self.stream.read()
# Variable to control when the camera is stopped
self.stopped = False
def start(self):
# Start the thread that reads frames from the video stream
Thread(target=self.update,args=()).start()
return self
def update(self):
# Keep looping indefinitely until the thread is stopped
while True:
# If the camera is stopped, stop the thread
if self.stopped:
# Close camera resources
self.stream.release()
return
# Otherwise, grab the next frame from the stream
(self.grabbed, self.frame) = self.stream.read()
def read(self):
# Return the most recent frame
return self.frame
def stop(self):
# Indicate that the camera and thread should be stopped
self.stopped = True
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir', help='Folder the .tflite file is located in', default='Sample_TFLite_model')
parser.add_argument('--graph', help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument('--labels', help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument('--threshold', help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument('--resolution', help='Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
default='1280x720')
parser.add_argument('--edgetpu', help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
resW, resH = args.resolution.split('x')
imW, imH = int(resW), int(resH)
use_TPU = args.edgetpu
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del(labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
# Initialize video stream
videostream = VideoStream(resolution=(imW,imH),framerate=30).start()
time.sleep(1)
#for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
while True:
# Start timer (for calculating frame rate)
t1 = cv2.getTickCount()
# Grab frame from video stream
frame1 = videostream.read()
# Acquire frame and resize to expected shape [1xHxWx3]
frame = frame1.copy()
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb, (width, height))
input_data = np.expand_dims(frame_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'],input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[0] # Bounding box coordinates of detected objects
classes = interpreter.get_tensor(output_details[1]['index'])[0] # Class index of detected objects
scores = interpreter.get_tensor(output_details[2]['index'])[0] # Confidence of detected objects
#num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed)
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
ymin = int(max(1,(boxes[i][0] * imH)))
xmin = int(max(1,(boxes[i][1] * imW)))
ymax = int(min(imH,(boxes[i][2] * imH)))
xmax = int(min(imW,(boxes[i][3] * imW)))
cv2.rectangle(frame, (xmin,ymin), (xmax,ymax), (10, 255, 0), 2)
# Draw label
object_name = labels[int(classes[i])] # Look up object name from "labels" array using class index
label = '%s: %d%%' % (object_name, int(scores[i]*100)) # Example: 'person: 72%'
labelSize, baseLine = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2) # Get font size
label_ymin = max(ymin, labelSize[1] + 10) # Make sure not to draw label too close to top of window
cv2.rectangle(frame, (xmin, label_ymin-labelSize[1]-10), (xmin+labelSize[0], label_ymin+baseLine-10), (255, 255, 255), cv2.FILLED) # Draw white box to put label text in
cv2.putText(frame, label, (xmin, label_ymin-7), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0), 2) # Draw label text
##################
if object_name == 'person':
queue.put(1)
print("NN has detected person")
##################
# Draw framerate in corner of frame
cv2.putText(frame,'FPS: {0:.2f}'.format(frame_rate_calc),(30,50),cv2.FONT_HERSHEY_SIMPLEX,1,(255,255,0),2,cv2.LINE_AA)
# All the results have been drawn on the frame, so it's time to display it.
#cv2.imshow('Object detector', frame)
# Calculate framerate
t2 = cv2.getTickCount()
time1 = (t2-t1)/freq
frame_rate_calc= 1/time1
# Press 'q' to quit
if cv2.waitKey(1) == ord('q'):
break
def main():
tracemalloc.start()
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir',
help='Folder the .tflite file is located in',
default='model')
parser.add_argument(
'--graph',
help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument(
'--labels',
help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument(
'--threshold',
help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument(
'--resolution',
help=
'Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
default='640x480')
parser.add_argument(
'--edgetpu',
help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
parser.add_argument('--camera', help='Choose camera input', default=0)
args = parser.parse_args()
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
resW, resH = args.resolution.split('x')
imW, imH = int(resW), int(resH)
camera = int(args.camera)
use_TPU = args.edgetpu
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Initialize frame rate calculation
frame_rate_calc = 1
freq = cv2.getTickFrequency()
# Line detection code setup
#image_file = 'line.jpg'
field_file = 'field.jpg'
point_list = []
field_point = []
cap = cv2.VideoCapture(0)
cap.set(3, imW)
cap.set(4, imH)
test_img = cap.read() # Initial image
bigW = 1920
bigH = 950
while True:
ret, test_img = cap.read()
test_img = cv2.resize(test_img, (imW, imH))
imgGray = cv2.cvtColor(test_img, cv2.COLOR_BGR2GRAY)
cv2.imshow("image", test_img)
if cv2.waitKey(1) & 0xFF == ord('c'):
break
cv2.imshow("image", test_img)
# cv2.waitKey(0)
#Here, you need to change the image name and it's path according to your directory
# test_img = cv2.imread(image_file)
# cv2.imshow("image", test_img)
#calling the mouse click event
while len(point_list) < 3:
cv2.setMouseCallback("image", cc.click_event, [test_img, point_list])
cv2.waitKey(1)
field_img = cv2.imread(field_file)
field_img = cv2.resize(field_img, (imW, imH))
cv2.imshow("field_image", field_img)
#calling the mouse click event
while len(field_point) < 3:
cv2.setMouseCallback("field_image", cc.click_event_2,
[field_img, field_point])
cv2.waitKey(1)
cv2.destroyAllWindows()
A_matrix, translation_vector = cc.affineMatrix(point_list, field_point)
#print(point_list, field_point)
# print("Transform new point:")
# test_point = [625, 200]
# image_p = np.dot(A_matrix, test_point) + translation_vector
# print(test_point, " mapped to: ", image_p)
# image = cv2.circle(field_img, (int(image_p[0]), int(image_p[1])), 2, (0, 0, 255), 10)
cv2.destroyAllWindows()
#print("Here 3")
cap.release()
time.sleep(5)
# Initialize video stream
videostream = VideoStream(resolution=(imW, imH),
framerate=10,
camera=camera).start()
time.sleep(1)
flag_in = 1
eventType = "TestEvent"
#for frame1 in camera.capture_continuous(rawCapture, format="bgr",use_video_port=True):
while True:
# first = tracemalloc.take_snapshot()
videostream.update()
if videostream.postReplayCount > 0:
videostream.postReplayCount -= 1
if videostream.postReplayCount == 0:
now = datetime.now()
timestamp = now.strftime("%H-%M-%S")
videostream.saveStream(eventType, timestamp)
#replayThread = threading.Thread(target=videostream.saveStream(eventType, timestamp))
#replayThread.daemon = True
#replayThread.start()
# Start timer (for calculating frame rate)
t1 = cv2.getTickCount()
# Grab frame from video stream
frame = videostream.read()
# Acquire frame and resize to expected shape [1xHxWx3]
#frame = frame1.copy()
frame = cv2.resize(frame, (imW, imH))
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame_resized = cv2.resize(frame_rgb, (width, height))
input_data = np.expand_dims(frame_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[
0] # Bounding box coordinates of detected objects
classes = interpreter.get_tensor(
output_details[1]['index'])[0] # Class index of detected objects
scores = interpreter.get_tensor(
output_details[2]['index'])[0] # Confidence of detected objects
#num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed)
# Initialise lists to store labels and centerpoints
centerpoints = []
adjustedpoints = []
detection_labels = []
f_img = cv2.imread(field_file)
f_img = cv2.resize(f_img, (imW, imH))
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
ymin = int(max(1, (boxes[i][0] * imH)))
xmin = int(max(1, (boxes[i][1] * imW)))
ymax = int(min(imH, (boxes[i][2] * imH)))
xmax = int(min(imW, (boxes[i][3] * imW)))
xcen = int(xmin + (xmax - xmin) / 2)
#ycen = int(ymin + (ymax-ymin)/2)
ycen = int(ymax)
centerpoints.append(
(xcen, ycen)
) # Append centerpoint to list of centerpoints to be sent to database
# Mark centerpoint on frame to make sure we have correct image centerpoint coords
frame[ycen - 5:ycen + 5, xcen - 5:xcen + 5] = (0, 0, 255)
# Aplly affine matrix
image_p = np.dot(A_matrix, (xcen, ycen)) + translation_vector
adjustedpoints.append(image_p)
#-------------EVENT DETECTION---------------------------------------
max_x_out = imW * (1195 / 1280)
max_y_out = imH * (688 / 720)
min_x_out = imW * (85 / 1280)
min_y_out = imH * (31 / 720)
goal_y_min = imH * (409 / 950)
goal_y_max = imH * (543 / 950)
flag_out = 0
flag_goal = 0
cx = int(image_p[0])
cy = int(image_p[1])
#
#person is 0 on list
#sports ball is 36
currentDetect = int(classes[i])
if currentDetect == 0:
if (cx < min_x_out) or (cx > max_x_out):
if (cy > goal_y_min) and (cy < goal_y_max):
flag_goal = 1
else:
flag_out = 1
if (cy < min_y_out) or (cy > max_y_out):
flag_out = 1
#Plotting dots for visualization
if flag_goal == 1:
field_image_with_point = cv2.circle(
f_img, (int(image_p[0]), int(image_p[1])), 2,
(255, 255), 10)
elif flag_out == 1:
field_image_with_point = cv2.circle(
f_img, (int(image_p[0]), int(image_p[1])), 2,
(255, 255, 255), 10)
else:
field_image_with_point = cv2.circle(
f_img, (int(image_p[0]), int(image_p[1])), 2,
(0, 255, 255), 10)
#Sending Message Should only occur once unless ball is back in
if flag_goal == 1 and flag_in == 1:
print("goalllll")
eventType = "Goal"
videostream.postReplayCount = 150
flag_in = 0
now = datetime.now()
timestamp = now.strftime("%H-%M-%S")
sio.emit('message',
f"{socket.gethostname()}: Goal {timestamp}")
elif flag_out == 1 and flag_in == 1:
print("outtttt")
eventType = "OutOfBounds"
flag_in = 0
now = datetime.now()
timestamp = now.strftime("%H-%M-%S")
sio.emit('message',
f"{socket.gethostname()}: Out {timestamp}")
elif flag_out == 0 and flag_goal == 0:
flag_in = 1
elif currentDetect == 36:
field_image_with_point = cv2.circle(
f_img, (int(image_p[0]), int(image_p[1])), 2,
(0, 0, 255), 10)
#if we want to see other detections on the map
else:
field_image_with_point = cv2.circle(
f_img, (int(image_p[0]), int(image_p[1])), 2,
(0, 0, 0), 10)
cv2.imshow("adjusted_image", field_image_with_point)
# Draw Bounding Box
#COMMMENT THIS OUT WHEN WE DON"T NEED IT
cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), (10, 255, 0),
2)
# Draw label
object_name = labels[int(
classes[i]
)] # Look up object name from "labels" array using class index
detection_labels.append(
object_name
) # Append object name to list of object names to be sent to database
label = '%s: %d%%' % (object_name, int(scores[i] * 100)
) # Example: 'person: 72%'
labelSize, baseLine = cv2.getTextSize(label,
cv2.FONT_HERSHEY_SIMPLEX,
0.7, 2) # Get font size
label_ymin = max(
ymin, labelSize[1] + 10
) # Make sure not to draw label too close to top of window
cv2.rectangle(
frame, (xmin, label_ymin - labelSize[1] - 10),
(xmin + labelSize[0], label_ymin + baseLine - 10),
(255, 255, 255),
cv2.FILLED) # Draw white box to put label text in
cv2.putText(frame, label, (xmin, label_ymin - 7),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
2) # Draw label text
# Send results to database
# print(centerpoints, ', '.join(detection_labels)) # Debug print
# db.update('right hand corner', centerpoints, ', '.join(detection_labels), db.engine) # Use join() to send labels as single string
# Draw framerate in corner of frame
cv2.putText(frame, 'FPS: {0:.2f}'.format(frame_rate_calc), (30, 50),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 2, cv2.LINE_AA)
# All the results have been drawn on the frame, so it's time to display it.
cv2.imshow('Object detector', frame)
#print("Frame rate:", frame_rate_calc)
# Calculate framerate
t2 = cv2.getTickCount()
time1 = (t2 - t1) / freq
frame_rate_calc = 1 / time1
# Press 'q' to quit
key = cv2.waitKey(1)
if key == ord('q'):
print("Quitting")
break
elif key == ord('s'):
print("Saving")
eventType = "Request"
# Begin count for post-event frames
videostream.postReplayCount = 150
# videostream.saveStream()
# Instead of checking keypress, query database to see if request has been made for video file,
# if yes, execute saveStream(), then post the video to server
global replay_flag
try:
if replay_requested() and replay_flag == 1:
print("Saving Video File to Send To Server")
# clear_replay()
videostream.postReplayCount = 50
replay_flag = 0
# set flag to 0
# videostream.saveStream()
except:
pass
#print("[TFLITE]: Error! ")
# snap = tracemalloc.take_snapshot()
# stats = snap.compare_to(first, 'lineno')
# print(stats[0])
# print(stats[0].traceback.format())
#cv2.waitKey(0)
# Clean up
cv2.destroyAllWindows()
sio.disconnect()
videostream.stop()
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
def test_external_delegate_options_multiple_backends(delegate_dir):
tflite.load_delegate(delegate_dir,
options={"backends": "GpuAcc,CpuAcc,CpuRef,Unknown"})