def inference(rec):
global wakeWordOn
global stopInference
# Provjerava se da li je varijabla stopInference false te ako je znači da se trenutno ne vrši nikakva inferencija
# i da ju mi možemo vršiti, ako je true izlazimo iz petlje i čekamo na novu iteraciju
if (stopInference == False):
# wakeWordOn služi kao mehanizam da sustav zna šta treba u ovome trenutku slušati, ako je wake word true znači
# da smo u prijašnjoj iteraciji prepoznali da je govornik rekao "hej kućo" i sada narednih 3 sekunde slušamo za druge naredbe
# ako je wake word false znači da trenutno ne očekujemo nikakvu naredbu te da samo slušamo i čekamo prvo na wake word "hej kućo"
if (wakeWordOn):
# Promjena oblika snimke i downsample
rec = rec.transpose()
rec = np.squeeze(rec)
rec = decimate(rec, sample_rate, resample_rate)
# Prozor za provjeru snimke u bufferu
window[:len(window) // 2] = window[len(window) // 2:]
window[len(window) // 2:] = rec
# Dretva za izvlačenje mfcc podataka za vršenje inference-a
y_thread_result = queue.Queue()
y = threading.Thread(target=extract_features,
args=(window, y_thread_result))
y.start()
y.join()
inferenceWav1 = y_thread_result.get()
inferenceWav1 = np.reshape(
inferenceWav1,
(1, inferenceWav1.shape[0], inferenceWav1.shape[1], 1))
# Putanja modela i labela
model = 'ConvModel.tflite'
labels = 'Labels.txt'
# Dretva i ostali dio koda za vršenje inference-a
labels = load_labels(labels)
interpreter = Interpreter(model)
interpreter.allocate_tensors()
z_thread_result = queue.Queue()
z = threading.Thread(target=classify_wav,
args=(interpreter, inferenceWav1,
z_thread_result))
start = timeit.default_timer()
z.start()
z.join()
results = z_thread_result.get()
elapsed_ms = (timeit.default_timer() - start)
label_id, prob = results[0]
print(
str(labels[label_id]) + ";%.2f" % (prob) + '%' + ";%.2f" %
(elapsed_ms) + 'ms')
print("\n")
# Prepoznavanje naredbe nakon što je uključena ključna riječ,
# ako se komanda prepozna vrši se neka od mapiranih funkcija preko GPIO
# Vjerojatnost mora biti 60% ili veća da bi se rezultat okarakterizirao kao uspješan
# Ako nije jednostavno se izlazi iz petlje i čeka na novu snimku
if (prob >= 0.6):
# Provjerava se ako prepoznati zvuk nije wake word "hej kućo" ili pozadinski šum
# U slučaju da jesu u ovome dijelu se oni ignoriraju
if (label_id != 1 and label_id != 0):
if (label_id == 2):
stopInference = True
napraviKavuLED.on()
t = threading.Timer(2, ugasiNapraviKavu)
t.start()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
if (label_id == 3):
stopInference = True
prozorLED.on()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
if (label_id == 4):
stopInference = True
vrataLED.on()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
if (label_id == 5):
stopInference = True
tvLED.off()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
if (label_id == 6):
stopInference = True
tvLED.on()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
if (label_id == 7):
stopInference = True
prozorLED.off()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
if (label_id == 8):
stopInference = True
vrataLED.off()
pp = threading.Thread(target=povratnaPoruka,
args=([label_id]))
pp.start()
pp.join()
print("Prepoznata naredba <--> " + str(labels[label_id]))
print("\n")
else:
pass
else:
# Promjena oblika snimke i downsample
rec = rec.transpose()
rec = np.squeeze(rec)
rec = decimate(rec, sample_rate, resample_rate)
# Prozor za provjeru snimke u bufferu
window[:len(window) // 2] = window[len(window) // 2:]
window[len(window) // 2:] = rec
# Dretva za izvlačenje mfcc podataka za vršenje inference-a
y_thread_result = queue.Queue()
y = threading.Thread(target=extract_features,
args=(window, y_thread_result))
y.start()
y.join()
inferenceWav1 = y_thread_result.get()
inferenceWav1 = np.reshape(
inferenceWav1,
(1, inferenceWav1.shape[0], inferenceWav1.shape[1], 1))
# Putanja modela i labela
model = 'ConvModel.tflite'
labels = 'Labels.txt'
# Dretva i ostali dio koda za vršenje inference-a
labels = load_labels(labels)
interpreter = Interpreter(model)
interpreter.allocate_tensors()
z_thread_result = queue.Queue()
z = threading.Thread(target=classify_wav,
args=(interpreter, inferenceWav1,
z_thread_result))
start = timeit.default_timer()
z.start()
z.join()
results = z_thread_result.get()
elapsed_ms = (timeit.default_timer() - start)
label_id, prob = results[0]
print(
str(labels[label_id]) + ";%.2f" % (prob) + '%' + ";%.2f" %
(elapsed_ms) + 'ms')
print("\n")
# Ako wake word nije bio uključen i sada se prepozna uključuje se timer koji sljedeće
# 3 sekunde sluša za naredbe i nakon toga se isključuje i opet sluša za wake word
# Vjerojatnost mora biti bar 60% da se prihvati kao ispravan rezultat prepoznavanja
if (prob >= 0.6):
if (label_id == 1):
wakeWordOn = True
t = threading.Timer(3, timeout)
t.start()
else:
pass
else:
pass
GRAPH_NAME = 'detect.tflite'
LABELMAP_NAME = 'labelmap.txt'
CWD_PATH = os.getcwd() # get current working directory
# importing the labels on which our model ssdmobilenet is trained
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
if labels[0] == '???': #deleting first label if '???' is present
del (labels[0])
#print
# importing the tflite model file which contains the weights
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors() #allocating all tensors to tflite interpreter
# detecting the dimensions for the input image, to convert our image to same dimension
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
last_epoch = 0
email_update_interval = 50000
#starting camera to record video
video = cv2.VideoCapture(0)
'IoT sensor node.')
parser.add_argument('-p',
action='store',
dest='port',
type=int,
default=DEFUALT_PORT,
help='Port number for server')
args = parser.parse_args()
port = args.port
# Print versions
print('Numpy ' + np.__version__)
print('SciPy ' + sp.__version__)
# Load model
interpreter = Interpreter(join(MODELS_PATH, TFLITE_MODEL_FILE) + '.tflite')
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
print(input_details)
# Create server
handler = partial(SimpleHTTPRequestHandler)
server = HTTPServer(('', port), handler)
server_addr = server.socket.getsockname()
print('Server running at: ' + str(server_addr[0]) + ':' + str(server_addr[1]))
# Create thread running server
server_thread = ServerThread(name='server_daemon', target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
class Inception:
def __init__(self):
self._model = None
self._labels = None
self._width = None
self._height = None
self._input_mean = None
self._input_std = None
self._floating_model = None
self._input_details = None
self._output_details = None
@property
def model(self):
if self._model is None:
self._model = Interpreter(Resource.inception)
self._model.allocate_tensors()
return self._model
@property
def labels(self):
if self._labels is None:
with open(Resource.inception_labels, 'r') as f:
self._labels = [line.strip() for line in f.readlines()]
return self._labels
@property
def input_details(self):
if self._input_details is None:
self._input_details = self.model.get_input_details()
return self._input_details
@property
def output_details(self):
if self._output_details is None:
self._output_details = self.model.get_output_details()
return self._output_details
@property
def floating_model(self):
if self._floating_model is None:
self._floating_model = self.input_details[0]['dtype'] == np.float32
return self._floating_model
@property
def width_height(self):
if self._width is None:
self._width = self.input_details[0]['shape'][2]
if self._height is None:
self._height = self.input_details[0]['shape'][1]
return self._width, self._height
def _process_image(self, img):
width, height = self.width_height
img = img.resize((width, height))
# add N dim
input_data = np.expand_dims(img, axis=0)
if self.floating_model:
input_data = (np.float32(input_data) -
self._input_mean) / self._input_std
return input_data
def _process_output(self):
output_data = self.model.get_tensor(self.output_details[0]['index'])
results = np.squeeze(output_data)
top_k = results.argsort()[-5:][::-1]
for i in top_k:
if self.floating_model:
yield Prediction(probability=float(results[i]),
label=self.labels[i])
else:
yield Prediction(probability=float(results[i] / 255.0),
label=self.labels[i])
def predict(self, img) -> List[Prediction]:
input_data = self._process_image(img)
self.model.set_tensor(self.input_details[0]['index'], input_data)
start_time = time.time()
self.model.invoke()
stop_time = time.time()
print('Inception Predict Time: {:.3f}ms'.format(
(stop_time - start_time) * 1000))
return list(self._process_output())
def __init__(
self,
model_path: str,
options: ImageSegmenterOptions = ImageSegmenterOptions()
) -> None:
"""Initialize a image segmentation model.
Args:
model_path: Name of the TFLite image segmentation model.
options: The config to initialize an image segmenter. (Optional)
Raises:
ValueError: If the TFLite model is invalid.
OSError: If the current OS isn't supported by EdgeTPU.
"""
# Load metadata from model.
displayer = metadata.MetadataDisplayer.with_model_file(model_path)
# Save model metadata for preprocessing later.
model_metadata = json.loads(displayer.get_metadata_json())
process_units = model_metadata['subgraph_metadata'][0][
'input_tensor_metadata'][0]['process_units']
mean = 127.5
std = 127.5
for option in process_units:
if option['options_type'] == 'NormalizationOptions':
mean = option['options']['mean'][0]
std = option['options']['std'][0]
self._mean = mean
self._std = std
# Load label list from metadata.
file_name = displayer.get_packed_associated_file_list()[0]
label_map_file = displayer.get_associated_file_buffer(
file_name).decode()
label_list = list(filter(len, label_map_file.splitlines()))
self._label_list = label_list
# Initialize TFLite model.
if options.enable_edgetpu:
if edgetpu_lib_name() is None:
raise OSError(
"The current OS isn't supported by Coral EdgeTPU.")
interpreter = Interpreter(
model_path=model_path,
experimental_delegates=[load_delegate(edgetpu_lib_name())],
num_threads=options.num_threads)
else:
interpreter = Interpreter(model_path=model_path,
num_threads=options.num_threads)
interpreter.allocate_tensors()
self._options = options
self._input_index = interpreter.get_input_details()[0]['index']
self._output_index = interpreter.get_output_details()[0]['index']
self._input_height = interpreter.get_input_details()[0]['shape'][1]
self._input_width = interpreter.get_input_details()[0]['shape'][2]
self._is_quantized_input = interpreter.get_input_details(
)[0]['dtype'] == np.uint8
self._interpreter = interpreter
def __init__(self,
model=None,
labels=None,
confidence_threshold=0.8,
top_k=10,
**kwargs):
"""Create an instance of Tensorflow inference engine.
:Parameters:
----------
model: dict
{
'tflite': path,
'edgetpu': path,
}
Where path is of type string and points to the
location of frozen graph file (AI model).
labels : string
Location of file with model labels.
confidence_threshold : float
Inference confidence threshold.
top_k : type
Inference top-k threshold.
"""
assert model
assert model["tflite"], "TFLite AI model path required."
model_tflite = model["tflite"]
assert os.path.isfile(
model_tflite
), f"TFLite AI model file does not exist: {model_tflite}"
self._model_tflite_path = model_tflite
model_edgetpu = model.get("edgetpu", None)
if model_edgetpu:
assert os.path.isfile(
model_edgetpu
), f"EdgeTPU AI model file does not exist: {model_edgetpu}"
self._model_edgetpu_path = model_edgetpu
assert labels, "AI model labels path required."
assert os.path.isfile(
labels), "AI model labels file does not exist: {}".format(labels)
self._model_labels_path = labels
self._confidence_threshold = confidence_threshold
self._top_k = top_k
log.info(
"Loading AI model:\n"
"TFLite graph: %r\n"
"EdgeTPU graph: %r\n"
"Labels %r."
"Condidence threshod: %.0f%%"
"top-k: %d",
model_tflite,
model_edgetpu,
labels,
confidence_threshold * 100,
top_k,
)
# EdgeTPU is not available in testing and other environments
# load dynamically as needed
# edgetpu_class = 'DetectionEngine'
# module_object = import_module('edgetpu.detection.engine',
# packaage=edgetpu_class)
# target_class = getattr(module_object, edgetpu_class)
self._tf_interpreter = _get_edgetpu_interpreter(model=model_edgetpu)
if not self._tf_interpreter:
log.debug("EdgeTPU not available. Will use TFLite CPU runtime.")
self._tf_interpreter = Interpreter(model_path=model_tflite)
assert self._tf_interpreter
self._tf_interpreter.allocate_tensors()
# check the type of the input tensor
self._tf_input_details = self._tf_interpreter.get_input_details()
self._tf_output_details = self._tf_interpreter.get_output_details()
self._tf_is_quantized_model = self.input_details[0][
"dtype"] != np.float32
PATH_TO_CKPT = os.path.join(CWD_PATH,MODEL_NAME,GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
if labels[0] == '???':
del(labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
def predict(self, data):
try:
data = np.array(data, np.float32)
data = np.expand_dims(data, axis = 0)
data = signal.resample(data, self.sample_rate, axis = 1)
assert data.shape == (1, 16000)
# Normalize short ints to floats in range [-1..1).
#data = data / float(np.max(np.absolute(data)))
data = np.array(data, np.float32) / 32768.0
# prepare TFLite interpreter
with open(os.path.join(self.path_model, self.name_model), 'rb') as f:
model_content = f.read()
interpreter = Interpreter(model_content=model_content)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
padded_input = np.zeros((1, 16000), dtype=np.float32)
padded_input[:, :data.shape[1]] = data
# set input audio data (by default data at index 0)
interpreter.set_tensor(input_details[0]['index'], padded_input.astype(np.float32))
# run inference
interpreter.invoke()
# get output: classification
out_tflite = interpreter.get_tensor(output_details[0]['index'])
out_tflite_argmax = np.argmax(out_tflite)
return out_tflite_argmax
except(AssertionError):
self.stream = False
return -1
def open_dialog_box(self):
img = QFileDialog.getOpenFileName()
img_elec = img[0]
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir',
help='Folder the .tflite file is located in',
default='medbox')
parser.add_argument(
'--graph',
help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument(
'--labels',
help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument(
'--threshold',
help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument(
'--image',
help=
'Name of the single image to perform detection on. To run detection on multiple images, use --imagedir',
default=None)
parser.add_argument(
'--imagedir',
help=
'Name of the folder containing images to perform detection on. Folder must contain only images.',
default=None)
parser.add_argument(
'--edgetpu',
help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
use_TPU = args.edgetpu
# Parse input image name and directory.
IM_NAME = args.image
IM_DIR = args.imagedir
# If both an image AND a folder are specified, throw an error
if (IM_NAME and IM_DIR):
print(
'Error! Please only use the --image argument or the --imagedir argument, not both. Issue "python TFLite_detection_image.py -h" for help.'
)
sys.exit()
# If neither an image or a folder are specified, default to using 'test1.jpg' for image name
if (not IM_NAME and not IM_DIR):
IM_NAME = img_elec
# Import TensorFlow libraries
# If tensorflow is not installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tensorflow')
if pkg is None:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Define path to images and grab all image filenames
if IM_DIR:
PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_DIR)
images = glob.glob(PATH_TO_IMAGES + '/*')
elif IM_NAME:
PATH_TO_IMAGES = os.path.join(CWD_PATH, IM_NAME)
images = glob.glob(PATH_TO_IMAGES)
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
start_time = time.time()
# Loop over every image and perform detection
for image_path in images:
# Load image and resize to expected shape [1xHxWx3]
image = cv2.imread(image_path)
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
imH, imW, _ = image.shape
image_resized = cv2.resize(image_rgb, (width, height))
input_data = np.expand_dims(image_resized, axis=0)
# Normalize pixel values if using a floating model (i.e. if model is non-quantized)
if floating_model:
input_data = (np.float32(input_data) - input_mean) / input_std
# Perform the actual detection by running the model with the image as input
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
# Retrieve detection results
boxes = interpreter.get_tensor(output_details[0]['index'])[
0] # Bounding box coordinates of detected objects
classes = interpreter.get_tensor(output_details[1]['index'])[
0] # Class index of detected objects
scores = interpreter.get_tensor(output_details[2]['index'])[
0] # Confidence of detected objects
#num = interpreter.get_tensor(output_details[3]['index'])[0] # Total number of detected objects (inaccurate and not needed)
# Loop over all detections and draw detection box if confidence is above minimum threshold
for i in range(len(scores)):
if ((scores[i] > min_conf_threshold) and (scores[i] <= 1.0)):
# Get bounding box coordinates and draw box
# Interpreter can return coordinates that are outside of image dimensions, need to force them to be within image using max() and min()
ymin = int(max(1, (boxes[i][0] * imH)))
xmin = int(max(1, (boxes[i][1] * imW)))
ymax = int(min(imH, (boxes[i][2] * imH)))
xmax = int(min(imW, (boxes[i][3] * imW)))
# Draw label
object_name = labels[int(
classes[i]
)] # Look up object name from "labels" array using class index
label = '%s' % (object_name) # Example: 'person'
labelSize, baseLine = cv2.getTextSize(
label, cv2.FONT_HERSHEY_SIMPLEX, 0.7,
2) # Get font size
label_ymin = max(
ymin, labelSize[1] + 10
) # Make sure not to draw label too close to top of window
cv2.rectangle(
image, (xmin, label_ymin - labelSize[1] - 10),
(xmin + labelSize[0], label_ymin + baseLine - 10),
(255, 255, 255),
cv2.FILLED) # Draw white box to put label text in
cv2.putText(image, label, (xmin, label_ymin - 7),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 0),
2) # Draw label text
# All the results have been drawn on the image, now display the image
cv2.imshow('Q=quit', image)
print("PROCESSING TIME: %s seconds" % (time.time() - start_time))
# Press any key to continue to next image, or press 'q' to quit
if cv2.waitKey(0) == ord('q'):
break
# Clean up
cv2.destroyAllWindows()
from PIL import Image
import numpy as np
import argparse
parser = argparse.ArgumentParser(description='Cat vs Dog')
parser.add_argument('--filename', type=str, help='Specify the filename', required=True)
parser.add_argument('--model_path', type=str, help='Specify the model path', required=True)
args = parser.parse_args()
filename = args.filename
model_path = args.model_path
# Load TFLite model and allocate tensors
interpreter = Interpreter(model_path=model_path)
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Read image
img = Image.open(filename).convert('RGB')
# Get input size
input_shape = input_details[0]['shape']
size = input_shape[:2] if len(input_shape) == 3 else input_shape[1:3]
# Preprocess image
img = img.resize(size)
# # If the model is quantized (uint8 data), then dequantize the results
# if output_details['dtype'] == np.uint8:
# scale, zero_point = output_details['quantization']
# tensor = scale * (tensor - zero_point)
return tensor
def set_input_tensor(image):
tensor_index = interpreter.get_input_details()[0]['index']
input_tensor = interpreter.tensor(tensor_index)()[0]
input_tensor[:, :] = image
interpreter = Interpreter(model_path="model.tflite")
interpreter.allocate_tensors()
_, height, width, _ = interpreter.get_input_details()[0]['shape']
def name(id):
if id == 0:
print('pet small')
elif id == 1:
print('pet large')
elif id == 2:
print('aluminum')
elif id == 3:
print('detergante')
class HouserBoon:
args = {
"labels": "/tmp/coco_labels.txt",
"model": "/tmp/detect.tflite",
"threshold": .4
}
def load_labels(self, path):
with open(path, 'r', encoding='utf-8') as f:
lines = f.readlines()
labels = {}
for row_number, content in enumerate(lines):
pair = re.split(r'[:\s]+', content.strip(), maxsplit=1)
if len(pair) == 2 and pair[0].strip().isdigit():
labels[int(pair[0])] = pair[1].strip()
else:
labels[row_number] = pair[0].strip()
return labels
def __init__(self):
self.labels = self.load_labels(self.args["labels"])
self.interpreter = Interpreter(self.args["model"])
self.interpreter.allocate_tensors()
_, self.input_height, self.input_width, _ = self.interpreter.get_input_details()[0]['shape']
def set_input_tensor(self, interpreter, image):
"""Sets the input tensor."""
tensor_index = interpreter.get_input_details()[0]['index']
input_tensor = interpreter.tensor(tensor_index)()[0]
input_tensor[:, :] = image
def get_output_tensor(self, interpreter, index):
"""Returns the output tensor at the given index."""
output_details = interpreter.get_output_details()[index]
tensor = np.squeeze(interpreter.get_tensor(output_details['index']))
return tensor
def detect_objects(self, interpreter, image, threshold):
"""Returns a list of detection results, each a dictionary of object info."""
self.set_input_tensor(interpreter, image)
self.interpreter.invoke()
# Get all output details
boxes = self.get_output_tensor(self.interpreter, 0)
classes = self.get_output_tensor(self.interpreter, 1)
scores = self.get_output_tensor(self.interpreter, 2)
count = int(self.get_output_tensor(self.interpreter, 3))
results = []
for i in range(count):
if scores[i] >= threshold:
result = {
'bounding_box': boxes[i],
'class_id': classes[i],
'score': scores[i]
}
results.append(result)
return results
def show_us_the_bolt(self):
with picamera.PiCamera(resolution=(CAMERA_WIDTH, CAMERA_HEIGHT), framerate=30) as camera:
camera.start_preview()
try:
stream = io.BytesIO()
camera.capture(stream, format = "jpeg")
stream.seek(0)
image = Image.open(stream).convert('RGB').resize(
(self.input_width, self.input_height), Image.ANTIALIAS)
results = self.detect_objects(self.interpreter, image, self.args["threshold"])
for obj in results:
if self.labels[obj['class_id']] == "person" or self.labels[obj['class_id']] == "stop sign":
return True
stream.seek(0)
stream.truncate()
finally:
camera.stop_preview()
return False
def __init__(self):
self.labels = self.load_labels(self.args["labels"])
self.interpreter = Interpreter(self.args["model"])
self.interpreter.allocate_tensors()
_, self.input_height, self.input_width, _ = self.interpreter.get_input_details()[0]['shape']
def __init__(self,
weights='yolov5s.pt',
device=torch.device('cpu'),
dnn=False,
data=None,
fp16=False):
# Usage:
# PyTorch: weights = *.pt
# TorchScript: *.torchscript
# ONNX Runtime: *.onnx
# ONNX OpenCV DNN: *.onnx with --dnn
# OpenVINO: *.xml
# CoreML: *.mlmodel
# TensorRT: *.engine
# TensorFlow SavedModel: *_saved_model
# TensorFlow GraphDef: *.pb
# TensorFlow Lite: *.tflite
# TensorFlow Edge TPU: *_edgetpu.tflite
from models.experimental import attempt_download, attempt_load # scoped to avoid circular import
super().__init__()
w = str(weights[0] if isinstance(weights, list) else weights)
pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs = self.model_type(
w) # get backend
w = attempt_download(w) # download if not local
fp16 &= (pt or jit or onnx or engine) and device.type != 'cpu' # FP16
stride, names = 32, [f'class{i}'
for i in range(1000)] # assign defaults
if data: # assign class names (optional)
with open(data, errors='ignore') as f:
names = yaml.safe_load(f)['names']
if pt: # PyTorch
model = attempt_load(weights if isinstance(weights, list) else w,
device=device)
stride = max(int(model.stride.max()), 32) # model stride
names = model.module.names if hasattr(
model, 'module') else model.names # get class names
model.half() if fp16 else model.float()
self.model = model # explicitly assign for to(), cpu(), cuda(), half()
elif jit: # TorchScript
LOGGER.info(f'Loading {w} for TorchScript inference...')
extra_files = {'config.txt': ''} # model metadata
model = torch.jit.load(w, _extra_files=extra_files)
model.half() if fp16 else model.float()
if extra_files['config.txt']:
d = json.loads(extra_files['config.txt']) # extra_files dict
stride, names = int(d['stride']), d['names']
elif dnn: # ONNX OpenCV DNN
LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...')
check_requirements(('opencv-python>=4.5.4', ))
net = cv2.dnn.readNetFromONNX(w)
elif onnx: # ONNX Runtime
LOGGER.info(f'Loading {w} for ONNX Runtime inference...')
cuda = torch.cuda.is_available()
check_requirements(
('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime'))
import onnxruntime
providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'
] if cuda else ['CPUExecutionProvider']
session = onnxruntime.InferenceSession(w, providers=providers)
meta = session.get_modelmeta().custom_metadata_map # metadata
if 'stride' in meta:
stride, names = int(meta['stride']), eval(meta['names'])
elif xml: # OpenVINO
LOGGER.info(f'Loading {w} for OpenVINO inference...')
check_requirements(
('openvino', )
) # requires openvino-dev: https://pypi.org/project/openvino-dev/
from openvino.runtime import Core
ie = Core()
if not Path(w).is_file(): # if not *.xml
w = next(Path(w).glob(
'*.xml')) # get *.xml file from *_openvino_model dir
network = ie.read_model(model=w,
weights=Path(w).with_suffix('.bin'))
executable_network = ie.compile_model(model=network,
device_name="CPU")
output_layer = next(iter(executable_network.outputs))
meta = Path(w).with_suffix('.yaml')
if meta.exists():
stride, names = self._load_metadata(meta) # load metadata
elif engine: # TensorRT
LOGGER.info(f'Loading {w} for TensorRT inference...')
import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download
check_version(trt.__version__, '7.0.0',
hard=True) # require tensorrt>=7.0.0
Binding = namedtuple('Binding',
('name', 'dtype', 'shape', 'data', 'ptr'))
logger = trt.Logger(trt.Logger.INFO)
with open(w, 'rb') as f, trt.Runtime(logger) as runtime:
model = runtime.deserialize_cuda_engine(f.read())
bindings = OrderedDict()
fp16 = False # default updated below
for index in range(model.num_bindings):
name = model.get_binding_name(index)
dtype = trt.nptype(model.get_binding_dtype(index))
shape = tuple(model.get_binding_shape(index))
data = torch.from_numpy(np.empty(
shape, dtype=np.dtype(dtype))).to(device)
bindings[name] = Binding(name, dtype, shape, data,
int(data.data_ptr()))
if model.binding_is_input(index) and dtype == np.float16:
fp16 = True
binding_addrs = OrderedDict(
(n, d.ptr) for n, d in bindings.items())
context = model.create_execution_context()
batch_size = bindings['images'].shape[0]
elif coreml: # CoreML
LOGGER.info(f'Loading {w} for CoreML inference...')
import coremltools as ct
model = ct.models.MLModel(w)
else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU)
if saved_model: # SavedModel
LOGGER.info(
f'Loading {w} for TensorFlow SavedModel inference...')
import tensorflow as tf
keras = False # assume TF1 saved_model
model = tf.keras.models.load_model(
w) if keras else tf.saved_model.load(w)
elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt
LOGGER.info(
f'Loading {w} for TensorFlow GraphDef inference...')
import tensorflow as tf
def wrap_frozen_graph(gd, inputs, outputs):
x = tf.compat.v1.wrap_function(
lambda: tf.compat.v1.import_graph_def(gd, name=""),
[]) # wrapped
ge = x.graph.as_graph_element
return x.prune(tf.nest.map_structure(ge, inputs),
tf.nest.map_structure(ge, outputs))
gd = tf.Graph().as_graph_def() # graph_def
with open(w, 'rb') as f:
gd.ParseFromString(f.read())
frozen_func = wrap_frozen_graph(gd,
inputs="x:0",
outputs="Identity:0")
elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python
try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu
from tflite_runtime.interpreter import Interpreter, load_delegate
except ImportError:
import tensorflow as tf
Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate,
if edgetpu: # Edge TPU https://coral.ai/software/#edgetpu-runtime
LOGGER.info(
f'Loading {w} for TensorFlow Lite Edge TPU inference...'
)
delegate = {
'Linux': 'libedgetpu.so.1',
'Darwin': 'libedgetpu.1.dylib',
'Windows': 'edgetpu.dll'
}[platform.system()]
interpreter = Interpreter(
model_path=w,
experimental_delegates=[load_delegate(delegate)])
else: # Lite
LOGGER.info(
f'Loading {w} for TensorFlow Lite inference...')
interpreter = Interpreter(
model_path=w) # load TFLite model
interpreter.allocate_tensors() # allocate
input_details = interpreter.get_input_details() # inputs
output_details = interpreter.get_output_details() # outputs
elif tfjs:
raise Exception(
'ERROR: YOLOv5 TF.js inference is not supported')
self.__dict__.update(locals()) # assign all variables to self
def __init__(self, model_path='detect.tflite'):
self.interpreter = Interpreter(model_path=model_path)
self.interpreter.set_num_threads(4)
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()
self.output_details = self.interpreter.get_output_details()
import re
from PIL import Image
from tflite_runtime.interpreter import Interpreter
import numpy as np
from object_detection_utils import draw_boxes
import telegram_send
PATTERN_TO_WATCH = "/var/lib/motion/*.jpg"
SAVED_MODEL_PATH = "./model_mobilenet_bee.tflite"
SAVED_MODEL_PATH = "./trailertruck.tflite"
ALTERED_IMAGE_OUTPUT = "/home/pi/Pictures/wildlife/"
TRAINED_MODEL_IMAGE_SIZE = (384, 384)
OBSERVED_LABELS = ["bigtrailer"]
MIN_SCORE = 0.6
interpreter = Interpreter(SAVED_MODEL_PATH)
interpreter.allocate_tensors()
previous_file = None
coco_labels = {}
with open("./coco_labels.txt") as fread:
lines = fread.read().split('\n')
for line in lines:
number = re.split("\s+", line)[0]
text = ' '.join(re.split("\s+", line)[1:])
coco_labels[int(number)] = text
def interpret_results_coco(output_details, im_resized, filepath):
class TFInferenceEngine:
"""Thin wrapper around TFLite Interpreter.
The official TFLite API is moving fast and still changes frequently.
This class intends to abstract out underlying TF changes to some extend.
It dynamically detects if EdgeTPU is available and uses it.
Otherwise falls back to TFLite Runtime.
"""
def __init__(self,
model=None,
labels=None,
confidence_threshold=0.8,
top_k=10,
**kwargs):
"""Create an instance of Tensorflow inference engine.
:Parameters:
----------
model: dict
{
'tflite': path,
'edgetpu': path,
}
Where path is of type string and points to the
location of frozen graph file (AI model).
labels : string
Location of file with model labels.
confidence_threshold : float
Inference confidence threshold.
top_k : type
Inference top-k threshold.
"""
assert model
assert model["tflite"], "TFLite AI model path required."
model_tflite = model["tflite"]
assert os.path.isfile(
model_tflite
), f"TFLite AI model file does not exist: {model_tflite}"
self._model_tflite_path = model_tflite
model_edgetpu = model.get("edgetpu", None)
if model_edgetpu:
assert os.path.isfile(
model_edgetpu
), f"EdgeTPU AI model file does not exist: {model_edgetpu}"
self._model_edgetpu_path = model_edgetpu
assert labels, "AI model labels path required."
assert os.path.isfile(
labels), "AI model labels file does not exist: {}".format(labels)
self._model_labels_path = labels
self._confidence_threshold = confidence_threshold
self._top_k = top_k
log.info(
"Loading AI model:\n"
"TFLite graph: %r\n"
"EdgeTPU graph: %r\n"
"Labels %r."
"Condidence threshod: %.0f%%"
"top-k: %d",
model_tflite,
model_edgetpu,
labels,
confidence_threshold * 100,
top_k,
)
# EdgeTPU is not available in testing and other environments
# load dynamically as needed
# edgetpu_class = 'DetectionEngine'
# module_object = import_module('edgetpu.detection.engine',
# packaage=edgetpu_class)
# target_class = getattr(module_object, edgetpu_class)
self._tf_interpreter = _get_edgetpu_interpreter(model=model_edgetpu)
if not self._tf_interpreter:
log.debug("EdgeTPU not available. Will use TFLite CPU runtime.")
self._tf_interpreter = Interpreter(model_path=model_tflite)
assert self._tf_interpreter
self._tf_interpreter.allocate_tensors()
# check the type of the input tensor
self._tf_input_details = self._tf_interpreter.get_input_details()
self._tf_output_details = self._tf_interpreter.get_output_details()
self._tf_is_quantized_model = self.input_details[0][
"dtype"] != np.float32
@property
def input_details(self):
return self._tf_input_details
@property
def output_details(self):
return self._tf_output_details
@property
def is_quantized(self):
return self._tf_is_quantized_model
@property
def labels_path(self):
"""
Location of labels file.
:Returns:
-------
string
Path to AI model labels.
"""
return self._model_labels_path
@property
def confidence_threshold(self):
"""
Inference confidence threshold.
:Returns:
-------
float
Confidence threshold for inference results.
Only results at or above
this threshold should be returned by each engine inference.
"""
return self._confidence_threshold
@property
def top_k(self):
"""
Inference top-k threshold.
:Returns:
-------
int
Max number of results to be returned by each inference.
Ordered by confidence score.
"""
return self._top_k
def infer(self):
"""Invoke model inference on current input tensor."""
return self._tf_interpreter.invoke()
def set_tensor(self, index=None, tensor_data=None):
"""Set tensor data at given reference index."""
assert isinstance(index, int)
self._tf_interpreter.set_tensor(index, tensor_data)
def get_tensor(self, index=None):
"""Return tensor data at given reference index."""
assert isinstance(index, int)
return self._tf_interpreter.get_tensor(index)
def main(lidar_data_queue):
global perceptron_network
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--model',
help='File path of .tflite file.',
required=True)
parser.add_argument('--labels',
help='File path of labels file.',
required=True)
parser.add_argument('--threshold',
help='Score threshold for detected objects.',
required=False,
type=float,
default=0.6)
args = parser.parse_args()
labels = load_labels(args.labels)
interpreter = Interpreter(
args.model,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
interpreter.allocate_tensors()
_, input_height, input_width, _ = interpreter.get_input_details(
)[0]['shape']
print(input_height, input_width)
count = 0
with picamera.PiCamera(resolution=(CAMERA_WIDTH, CAMERA_HEIGHT)) as camera:
camera.rotation = 180
camera.start_preview()
try:
stream = io.BytesIO()
annotator = Annotator(camera)
for _ in camera.capture_continuous(stream,
format='jpeg',
use_video_port=True):
start_time = time.monotonic()
stream.seek(0)
image = Image.open(stream).convert('RGB').resize(
(input_width, input_height), Image.ANTIALIAS)
# start_time = time.monotonic()
results = detect_objects(interpreter, image, args.threshold)
#elapsed_ms = (time.monotonic() - start_time) * 1000
annotator.clear()
middle_xy = annotate_objects(annotator, results, labels)
#annotator.text([5, 0], '%.1fms' % (elapsed_ms))
# annotator.update()
if perceptron_network.person_detected == True: # this only changes the first time a person is detected
data = []
if lidar_data_queue.empty() is False:
data = lidar_data_queue.get()
else:
data = [0]
if middle_xy[
0] != 0: # or ((middle_xy[0] < 300 and middle_xy != 0) and perceptron_network.first_arc_turn == True) or (middle_xy[0] > 20 and perceptron_network.first_arc_turn == True):
perceptron_network.person_detected_queue.insert(
0, True) # keep track og frames with person in it
perceptron_network.person_detected_queue.pop()
object_width = middle_xy[2]
print('object width: ', object_width)
if perceptron_network.first_arc_turn == True:
object_width = 80
distance_to_target = perceptron_network.getDistanceToPerson(
object_width)
print('distance = ', distance_to_target)
new_motor_speeds = perceptron_network.followTarget(
middle_xy, distance_to_target, data)
motorSpeedLeft(
1,
round(perceptron_network.motor_speed_total_left))
motorSpeedRight(
1,
round(perceptron_network.motor_speed_total_right))
print(
"Left motor: ",
round(perceptron_network.motor_speed_total_left),
" Right motor: ",
round(perceptron_network.motor_speed_total_right))
print('..........................................')
elif perceptron_network.first_arc_turn == True or (
perceptron_network.first_reflex_turn == True):
arc_motor_speeds = perceptron_network.makeArcTurn(
perceptron_network.reflex_avoiding_obstacle_dist +
400)
perceptron_network.motor_speed_total_left = arc_motor_speeds[
0]
perceptron_network.motor_speed_total_right = arc_motor_speeds[
1]
motorSpeedLeft(1, arc_motor_speeds[0])
motorSpeedRight(1, arc_motor_speeds[1])
print('second',
perceptron_network.motor_speed_total_left)
elif perceptron_network.first_arc_turn == True or (
perceptron_network.making_avoidance_turn == True):
arc_motor_speeds = perceptron_network.makeArcTurn(
perceptron_network.start_avoiding_obstacle_dist +
300)
perceptron_network.motor_speed_total_left = arc_motor_speeds[
0]
perceptron_network.motor_speed_total_right = arc_motor_speeds[
1]
motorSpeedLeft(1, arc_motor_speeds[0])
motorSpeedRight(1, arc_motor_speeds[1])
print('second',
perceptron_network.motor_speed_total_left)
else:
perceptron_network.person_detected_queue.insert(
0, False)
perceptron_network.person_detected_queue.pop()
# Is all the last 15 frames was without a person
if any(perceptron_network.person_detected_queue
) == False:
perceptron_network.motor_speed_total_left = 0
perceptron_network.motor_speed_total_right = 0
perceptron_network.motor_speed_distance = 0
print("Locating target....")
perceptron_network.robot_is_stationary = True
if perceptron_network.side_left_person_last_detected == True:
motorSpeedLeft(0, 19)
motorSpeedRight(1, 19)
elif perceptron_network.side_left_person_last_detected == False:
motorSpeedLeft(1, 19)
motorSpeedRight(0, 19)
# For calibrating the focal length
# focal = perceptron_network.getPercievedFocal(object_width, 2000, 500)
# print('focal = ', focal)
elapsed_ms = (time.monotonic() - start_time) * 1000
annotator.text([5, 0], '%.1fms' % (elapsed_ms))
annotator.update()
frame_times_for_coral_test.append(elapsed_ms)
#print(perceptron_network.getPercievedFocal(object_height, distance_test, person_height))
stream.seek(0)
stream.truncate()
except KeyboardInterrupt:
print('Saving distance data and shutting down')
motorSpeedLeft(1, 0)
motorSpeedRight(1, 0)
toggleLED({})
frame_average = sum(frame_times_for_coral_test) / len(
frame_times_for_coral_test)
#perceptron_network.save_test1()
#perceptron_network.save_test2()
#perceptron_network.save_test3()
#perceptron_network.save_test4()
perceptron_network.saveWeights()
# makePlots(perceptron_network.percep_l_wheel.weights_for_test, perceptron_network.percep_r_wheel.weights_for_test, perceptron_network.percep_far_distance.weights_for_test, perceptron_network.distances_for_test)
# file = open('distances.csv', 'w')
# file.truncate()
# with file:
# writer = csv.writer(file)
# writer.writerow(perceptron_network.distances_for_test)
# writer.writerow(perceptron_network.percep_l_wheel.weights_for_test)
# writer.writerow(perceptron_network.percep_r_wheel.weights_for_test)
# writer.writerow(perceptron_network.percep_distance.weights_for_test)
finally:
camera.stop_preview()
def app_args():
# Define and parse input arguments
parser = argparse.ArgumentParser()
parser.add_argument('--modeldir',
help='Folder the .tflite file is located in',
default='coco_ssd_mobilenet_v1')
parser.add_argument(
'--graph',
help='Name of the .tflite file, if different than detect.tflite',
default='detect.tflite')
parser.add_argument(
'--labels',
help='Name of the labelmap file, if different than labelmap.txt',
default='labelmap.txt')
parser.add_argument(
'--threshold',
help='Minimum confidence threshold for displaying detected objects',
default=0.5)
parser.add_argument(
'--resolution',
help=
'Desired webcam resolution in WxH. If the webcam does not support the resolution entered, errors may occur.',
default='1280x720')
parser.add_argument(
'--edgetpu',
help='Use Coral Edge TPU Accelerator to speed up detection',
action='store_true')
args = parser.parse_args()
global MODEL_NAME, GRAPH_NAME, LABELMAP_NAME, min_conf_threshold, resW, resH, imW, imH, use_TPU
MODEL_NAME = args.modeldir
GRAPH_NAME = args.graph
LABELMAP_NAME = args.labels
min_conf_threshold = float(args.threshold)
resW, resH = args.resolution.split('x')
imW, imH = int(resW), int(resH)
use_TPU = args.edgetpu
# Import TensorFlow libraries
# If tflite_runtime is installed, import interpreter from tflite_runtime, else import from regular tensorflow
# If using Coral Edge TPU, import the load_delegate library
pkg = importlib.util.find_spec('tflite_runtime')
if pkg:
from tflite_runtime.interpreter import Interpreter
if use_TPU:
from tflite_runtime.interpreter import load_delegate
else:
from tensorflow.lite.python.interpreter import Interpreter
if use_TPU:
from tensorflow.lite.python.interpreter import load_delegate
# If using Edge TPU, assign filename for Edge TPU model
if use_TPU:
# If user has specified the name of the .tflite file, use that name, otherwise use default 'edgetpu.tflite'
if (GRAPH_NAME == 'detect.tflite'):
GRAPH_NAME = 'edgetpu.tflite'
# Get path to current working directory
CWD_PATH = os.getcwd()
# Path to .tflite file, which contains the model that is used for object detection
PATH_TO_CKPT = os.path.join(CWD_PATH, MODEL_NAME, GRAPH_NAME)
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
global labels
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(
model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
global input_details, output_details, height, width, floating_model
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
return interpreter
model_dir = './mobilenet_v1_0.5_128/'
model_name = 'mobilenet_v1_0.5_128.tflite'
repeat = 10
model_dir = download_model_zoo(model_dir, model_name)
tflite_model_file = os.path.join(model_dir, model_name)
tflite_model_buf = open(tflite_model_file, "rb").read()
try:
import tflite
tflite_model = tflite.Model.GetRootAsModel(tflite_model_buf, 0)
except AttributeError:
import tflite.Model
tflite_model = tflite.Model.Model.GetRootAsModel(tflite_model_buf, 0)
interpreter = Interpreter(tflite_model_file, num_threads=get_cpu_count())
interpreter.allocate_tensors()
_, height, width, _ = interpreter.get_input_details()[0]['shape']
image = load_test_image('float32', height, width)
numpy_time = np.zeros(repeat)
for i in range(0, repeat):
start_time = time.time()
results = classify_image(interpreter, image)
elapsed_ms = (time.time() - start_time) * 1000
numpy_time[i] = elapsed_ms
print("tflite %-20s %-19s (%s)" % (model_name, "%.2f ms" % np.mean(numpy_time),
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH,MODEL_NAME,LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
if labels[0] == '???':
del(labels[0])
# Load the Tensorflow Lite model.
# If using Edge TPU, use special load_delegate argument
if use_TPU:
interpreter = Interpreter(model_path=PATH_TO_CKPT,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
print(PATH_TO_CKPT)
else:
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
class Detector:
"""
Perform object detection with the given model. The model is a quantized tflite
file which if the detector can not find it at the path it will download it
from neuralet repository automatically.
:param config: Is a ConfigEngine instance which provides necessary parameters.
"""
def __init__(self, config):
self.config = config
# Get the model name from the config
self.model_name = self.config.get_section_dict('Detector')['Name']
# Frames Per Second
self.fps = None
self.model_file = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
self.model_path = '/repo/data/edgetpu/' + self.model_file
# Get the model .tflite file path from the config.
# If there is no .tflite file in the path it will be downloaded automatically from base_url
user_model_path = self.config.get_section_dict('Detector')['ModelPath']
if len(user_model_path) > 0:
print('using %s as model' % user_model_path)
self.model_path = user_model_path
else:
base_url = 'https://media.githubusercontent.com/media/neuralet/neuralet-models/master/edge-tpu/'
url = base_url + self.model_name + '/' + self.model_file
if not os.path.isfile(self.model_path):
print('model does not exist under: ', self.model_path,
'downloading from ', url)
wget.download(url, self.model_path)
# Load TFLite model and allocate tensors
self.interpreter = Interpreter(
self.model_path,
experimental_delegates=[load_delegate("libedgetpu.so.1")])
self.interpreter.allocate_tensors()
# Get the model input and output tensor details
self.input_details = self.interpreter.get_input_details()
self.output_details = self.interpreter.get_output_details()
# Get class id from config
self.class_id = int(
self.config.get_section_dict('Detector')['ClassID'])
self.score_threshold = float(
self.config.get_section_dict('Detector')['MinScore'])
def inference(self, resized_rgb_image):
"""
inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding detection output which is used for creating result
Args:
resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)
Returns:
result: a dictionary contains of [{"id": 0, "bbox": [x1, y1, x2, y2], "score":s%}, {...}, {...}, ...]
"""
input_image = np.expand_dims(resized_rgb_image, axis=0)
# Fill input tensor with input_image
self.interpreter.set_tensor(self.input_details[0]["index"],
input_image)
t_begin = time.perf_counter()
self.interpreter.invoke()
inference_time = time.perf_counter() - t_begin # Second
self.fps = convert_infr_time_to_fps(inference_time)
# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
boxes = self.interpreter.get_tensor(self.output_details[0]['index'])
labels = self.interpreter.get_tensor(self.output_details[1]['index'])
scores = self.interpreter.get_tensor(self.output_details[2]['index'])
# TODO: will be used for getting number of objects
# num = self.interpreter.get_tensor(self.output_details[3]['index'])
result = []
for i in range(boxes.shape[1]): # number of boxes
if labels[0,
i] == self.class_id and scores[0,
i] > self.score_threshold:
result.append({
"id": str(self.class_id) + '-' + str(i),
"bbox": boxes[0, i, :],
"score": scores[0, i]
})
return result
def model(self):
if self._model is None:
self._model = Interpreter(Resource.inception)
self._model.allocate_tensors()
return self._model
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter2 = Interpreter("./model/vwmodelv3.tflite")
interpreter.allocate_tensors()
interpreter2.allocate_tensors()
_, height2, width2, _ = interpreter2.get_input_details()[0]['shape']
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
# Path to label map file
PATH_TO_LABELS = os.path.join(CWD_PATH, MODEL_NAME, LABELMAP_NAME)
# Load the label map
with open(PATH_TO_LABELS, 'r') as f:
labels = [line.strip() for line in f.readlines()]
# Have to do a weird fix for label map if using the COCO "starter model" from
# https://www.tensorflow.org/lite/models/object_detection/overview
# First label is '???', which has to be removed.
if labels[0] == '???':
del (labels[0])
# Load the Tensorflow Lite model.
interpreter = Interpreter(model_path=PATH_TO_CKPT)
interpreter.allocate_tensors()
# Get model details
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
floating_model = (input_details[0]['dtype'] == np.float32)
input_mean = 127.5
input_std = 127.5
# Initialize frame rate calculation
def main():
interpreter = Interpreter(
model_path, experimental_delegates=[load_delegate("libedgetpu.so.1")])
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
input_queue = input_manager.get_input_queue()
output_queue = output_manager.get_output_queue()
print(
'------------------------------------------------------------------------------------------'
)
print(
'Started Inference server, waiting for incoming requests ... (send \'stop\' to kill server)'
)
print(
'------------------------------------------------------------------------------------------'
)
while True:
data = input_queue.get()
print('recieved data with type ', type(data))
if type(data) == str and data == "stop": break
if type(data) == np.ndarray:
input_image = np.expand_dims(data, axis=0)
interpreter.set_tensor(input_details[0]["index"], input_image)
t_begin = time.perf_counter()
interpreter.invoke()
inference_time = time.perf_counter() - t_begin
net_output = interpreter.get_tensor(output_details[0]["index"])
print('inference output: ', net_output, ', done in ',
inference_time, ' seconds')
output_queue.put(net_output)
# End while
input_manager.shutdown()
output_manager.shutdown()
def main(args):
interpreter = Interpreter(model_path=args.model, num_threads=4)
interpreter.allocate_tensors()
inblobs = interpreter.get_input_details()
outblobs = interpreter.get_output_details()
# print('input:', inblobs[0]['shape'])
# print('output0:', outblobs[0]['shape'])
# print('output1:', outblobs[1]['shape'])
# print('output2:', outblobs[2]['shape'])
# input : [1, 256, 256, 3]
# output: [1, 64, 64, 10], [1, 64, 64, 4], [1, 64, 64, 1]
lm_idx = 0
box_idx = 1
hm_idx = 2
fps = ""
detectfps = ""
framecount = 0
detectframecount = 0
time1 = 0
mean = np.array([0.408, 0.447, 0.47], dtype="float32")
std = np.array([0.289, 0.274, 0.278], dtype="float32")
if args.input == 'cam':
cap = cv2.VideoCapture(0)
while True:
start_time = time.perf_counter()
if args.input == 'cam':
ret, image = cap.read()
else:
image = cv2.imread(args.input)
scale_w = image.shape[1] / inblobs[0]['shape'][2]
scale_h = image.shape[0] / inblobs[0]['shape'][1]
img = cv2.resize(image,
(inblobs[0]['shape'][2], inblobs[0]['shape'][1]))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img = img.astype(np.float32)
img = ((img / 255.0 - mean) / std).astype(np.float32)
img = img[np.newaxis, :, :, :]
interpreter.set_tensor(inblobs[0]['index'], img)
interpreter.invoke()
lm = interpreter.get_tensor(
outblobs[lm_idx]['index'])[0][np.newaxis, :, :, :] # 1,h,w,10
box = interpreter.get_tensor(
outblobs[box_idx]['index'])[0][np.newaxis, :, :, :] # 1,h,w,4
hm = interpreter.get_tensor(
outblobs[hm_idx]['index'])[0][np.newaxis, :, :, :].transpose(
(0, 3, 1, 2)) # 1,1,h,w
objs = detect(hm=hm, box=box, landmark=lm, threshold=0.4, nms_iou=0.5)
for obj in objs:
drawBBox(image, obj, scale_w, scale_h)
cv2.putText(image, fps, (image.shape[1] - 170, 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (38, 0, 255), 1,
cv2.LINE_AA)
cv2.imshow('output', image)
if args.input == 'cam':
if cv2.waitKey(1) == 27: # ESC key
return
else:
cv2.waitKey(0)
cv2.imwrite('output.jpg', image)
print('"output.jpg" is generated')
return
# FPS calculation
framecount += 1
if framecount >= 10:
fps = "(Playback) {:.1f} FPS".format(time1 / 10)
framecount = 0
time1 = 0
end_time = time.perf_counter()
elapsedTime = end_time - start_time
time1 += 1 / elapsedTime
class ObjectDetectorLite():
def __init__(self, model_path='detect.tflite'):
self.interpreter = Interpreter(model_path=model_path)
self.interpreter.set_num_threads(4)
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()
self.output_details = self.interpreter.get_output_details()
def _boxes_coordinates(self,
image,
boxes,
classes,
scores,
max_boxes_to_draw=20,
min_score_thresh=.5):
if not max_boxes_to_draw:
max_boxes_to_draw = boxes.shape[0]
number_boxes = min(max_boxes_to_draw, boxes.shape[0])
person_boxes = []
for i in range(number_boxes):
if scores is None or scores[i] > min_score_thresh:
box = tuple(boxes[i].tolist())
ymin, xmin, ymax, xmax = box
im_height, im_width, _ = image.shape
left, right, top, bottom = [
int(z) for z in (xmin * im_width, xmax * im_width,
ymin * im_height, ymax * im_height)
]
person_boxes.append([(left, top), (right, bottom), scores[i],
LABELS[classes[i]]])
return person_boxes
def detect(self, image, threshold=0.1):
# Resize and normalize image for network input
frame = cv2.resize(image, (300, 300))
frame = np.expand_dims(frame, axis=0)
frame = frame.astype('uint8')
# run model
self.interpreter.set_tensor(self.input_details[0]['index'], frame)
start_time = time.time()
self.interpreter.invoke()
stop_time = time.time()
print("time: ", stop_time - start_time)
# get results
boxes = self.interpreter.get_tensor(self.output_details[0]['index'])
classes = self.interpreter.get_tensor(self.output_details[1]['index'])
scores = self.interpreter.get_tensor(self.output_details[2]['index'])
num = self.interpreter.get_tensor(self.output_details[3]['index'])
# Find detected boxes coordinates
return self._boxes_coordinates(image,
np.squeeze(boxes[0]),
np.squeeze(classes[0] + 1).astype(
np.int32),
np.squeeze(scores[0]),
min_score_thresh=threshold)
class EdgeTpuDetector(BaseDetector):
def load_model(self, model_path=None, label_map=None):
"""
Loads model with specified model_path, if no model_path provided, the COCO model will be downloaded
and saved under 'detectors/data/'.
Args:
model_path: path to the edge tpu tflite model.
"""
if not model_path:
logging.info(
"you didn't specify the model file so the COCO pretrained model will be used"
)
base_url = 'https://media.githubusercontent.com/media/neuralet/neuralet-models/master/edge-tpu/mobilenet_ssd_v2/'
model_file = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
base_dir = "detectors/data"
model_path = os.path.join(base_dir, model_file)
if not os.path.isfile(model_path):
logging.info(
'model does not exist under: {}, downloading from {}'.
format(str(model_path), base_url + model_file))
os.makedirs(base_dir, exist_ok=True)
wget.download(base_url + model_file, model_path)
self.model = Interpreter(
model_path,
experimental_delegates=[load_delegate("libedgetpu.so.1")])
self.model.allocate_tensors()
# Get the model input and output tensor details
self.input_details = self.model.get_input_details()
self.output_details = self.model.get_output_details()
self.classes = list(label_map.keys())
self.label_map = label_map
def preprocess(self, raw_image):
"""
preprocess function prepares the raw input for inference.
Args:
raw_image: A BGR numpy array with shape (img_height, img_width, channels)
Returns:
rgb_resized_image: A numpy array which contains preprocessed verison of input
"""
resized_image = cv.resize(raw_image, (self.width, self.height))
rgb_resized_image = cv.cvtColor(resized_image, cv.COLOR_BGR2RGB)
return rgb_resized_image
def inference(self, preprocessed_image):
"""
Inference function sets input tensor to input image and gets the output.
The interpreter instance provides corresponding detection output which is used for creating result
Args:
resized_rgb_image: uint8 numpy array with shape (img_height, img_width, channels)
Returns:
result: A Frame protobuf massages
"""
if not self.model:
raise RuntimeError(
"first load the model with 'load_model()' method then call inferece()"
)
input_image = np.expand_dims(preprocessed_image, axis=0)
# Fill input tensor with input_image
self.model.set_tensor(self.input_details[0]["index"], input_image)
t_begin = time.perf_counter()
self.model.invoke()
inference_time = time.perf_counter() - t_begin # Second
self.fps = convert_infr_time_to_fps(inference_time)
# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
boxes = self.model.get_tensor(self.output_details[0]['index'])
labels = self.model.get_tensor(self.output_details[1]['index'])
scores = self.model.get_tensor(self.output_details[2]['index'])
frame = Frame(width=self.width, height=self.height, fps=self.fps)
for i in range(boxes.shape[1]): # number of boxes
label = labels[0, i] + 1
if label in self.classes and scores[0, i] > self.thresh:
left = boxes[0, i, 1]
top = boxes[0, i, 0]
right = boxes[0, i, 3]
bottom = boxes[0, i, 2]
score = scores[0, i]
frame.objects.append(
Instance(id=str(int(label)) + '-' + str(i),
category=self.label_map[int(label)]["name"],
bbox=Bbox(left=left,
top=top,
right=right,
bottom=bottom,
score=score)))
return frame
#-----initialise the Model and Load into interpreter-------------------------
#specify the path of Model and Label file
model_path = "mobilenet_v1_1.0_224_quant_edgetpu.tflite"
label_path = "labels_mobilenet_quant_v1_224.txt"
top_k_results = 2
with open(label_path, 'r') as f:
labels = list(map(str.strip, f.readlines()))
# Load TFLite model and allocate tensors
interpreter = Interpreter(
model_path=model_path,
experimental_delegates=[load_delegate('libedgetpu.so.1.0')])
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
## Get input size
input_shape = input_details[0]['shape']
#print(input_shape)
size = input_shape[:2] if len(input_shape) == 3 else input_shape[1:3]
#print(size)
#prediction threshold for triggering actions
threshold = 0.5
