def load_tflite(self, tflite_path):
if self.tpu:
self.interpreter = tflite.Interpreter(
model_path=tflite_path,
experimental_delegates=[
tflite.load_delegate("libedgetpu.so.1")
],
)
else:
self.interpreter = tflite.Interpreter(model_path=tflite_path)
self.interpreter.allocate_tensors()
input_details = self.interpreter.get_input_details()[0]
self.input_size = input_details["shape"][1]
self.input_index = input_details["index"]
output_details = self.interpreter.get_output_details()
self.output_index = [details["index"] for details in output_details]
if self.tpu:
# sig, raw, sig, raw, ...
self.output_size = [
output_details[2 * i]["shape"][1]
for i in range(len(output_details) // 2)
]
self.grid_coord = []
for _size in self.output_size:
xy_grid = np.meshgrid(np.arange(_size), np.arange(_size))
xy_grid = np.stack(xy_grid, axis=-1)
xy_grid = xy_grid[np.newaxis, ...]
self.grid_coord.append(xy_grid)
def process_folder(model, folder_name, new_folder_name):
"""
Function to find .wav files in the folder and subfolders of "folder_name",
process each .wav file with an algorithm and write it back to disk in the
folder "new_folder_name". The structure of the original directory is
preserved. The processed files will be saved with the same name as the
original file.
Parameters
----------
model : STRING
Name of TF-Lite model.
folder_name : STRING
Input folder with .wav files.
new_folder_name : STRING
Target folder for the processed files.
"""
# create interpreters
interpreter_1 = tflite.Interpreter(model_path=model + "_1.tflite")
interpreter_1.allocate_tensors()
interpreter_2 = tflite.Interpreter(model_path=model + "_2.tflite")
interpreter_2.allocate_tensors()
# empty list for file and folder names
file_names = []
directories = []
new_directories = []
# walk through the directory
for root, dirs, files in os.walk(folder_name):
for file in files:
# look for .wav files
if file.endswith("mic.wav"):
# write paths and filenames to lists
file_names.append(file)
directories.append(root)
# create new directory names
new_directories.append(
root.replace(folder_name, new_folder_name))
# check if the new directory already exists, if not create it
if not os.path.exists(
root.replace(folder_name, new_folder_name)):
os.makedirs(root.replace(folder_name, new_folder_name))
# iterate over all .wav files
for idx in range(len(file_names)):
# process each file with the mode
process_file(
interpreter_1,
interpreter_2,
os.path.join(directories[idx], file_names[idx]),
os.path.join(new_directories[idx], file_names[idx]),
)
print(file_names[idx] + " processed successfully!")
def preload_model(self,):
"""Preload tflite model"""
name_tflite = [name for name in os.listdir(self.dir_model) if name.endswith(".tflite")][0]
model_path = Path(f"{self.dir_model}/{name_tflite}")
if self.NUM_THREADS > 0:
self.interpreter = lite.Interpreter(model_path=str(model_path), num_threads=self.NUM_THREADS)
else:
self.interpreter = lite.Interpreter(model_path=str(model_path))
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()
self.output_details = self.interpreter.get_output_details()
def load_tflite(self, tflite_path):
if self.tpu:
self.interpreter = tflite.Interpreter(
model_path=tflite_path,
experimental_delegates=[
tflite.load_delegate("libedgetpu.so.1")
],
)
else:
self.interpreter = tflite.Interpreter(model_path=tflite_path)
self.interpreter.allocate_tensors()
input_details = self.interpreter.get_input_details()[0]
self.input_size = input_details["shape"][1]
self.input_index = input_details["index"]
output_details = self.interpreter.get_output_details()
self.output_index = [details["index"] for details in output_details]
def __init__(self, model_file: str, label_file: str, use_edge_tpu: bool = False) -> None:
log.info("Loading model: %s", model_file)
self._labels = _load_labels(label_file)
experimental_delegates = [tflite.load_delegate('libedgetpu.so.1')] if use_edge_tpu else None
self._interpreter = tflite.Interpreter(model_path=model_file, experimental_delegates=experimental_delegates)
self._interpreter.allocate_tensors()
# Prepare input metadata
# NxHxWxC, H:1, W:2
input_details = self._interpreter.get_input_details()
shape = input_details[0]['shape']
height = shape[1]
width = shape[2]
self._input_size = (width, height)
self._input_index = input_details[0]['index']
# Prepare output data
output_details = self._interpreter.get_output_details()
self._output_boxes_index = output_details[0]['index']
self._output_labels_index = output_details[1]['index']
self._output_confidences_index = output_details[2]['index']
self._output_num_of_boxes_index = output_details[3]['index']
# Initialize a color mapping
cm = zip(self._labels, COLORS)
self._color_mapping = {v[0]: v[1] for v in cm}
log.info("Color mapping: %s", self._color_mapping)
def loadModel(model_file, config_file):
global INPUT_LAYER_INDEX
global OUTPUT_LAYER_INDEX
log.p('LOADING TF LITE MODEL...', new_line=False)
# Load TFLite model and allocate tensors.
interpreter = tflite.Interpreter(model_path=model_file)
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Get input tensor index
INPUT_LAYER_INDEX = input_details[0]['index']
OUTPUT_LAYER_INDEX = output_details[0]['index']
# Load model-specific config
cfg['LOAD'](config_file, [
'CLASSES', 'SPEC_TYPE', 'MAGNITUDE_SCALE', 'WIN_LEN', 'SAMPLE_RATE',
'SPEC_FMIN', 'SPEC_FMAX', 'SPEC_LENGTH', 'INPUT_TYPE', 'INPUT_SHAPE'
])
log.p('DONE!')
log.p(('INPUT LAYER INDEX:', INPUT_LAYER_INDEX))
log.p(('OUTPUT LAYER INDEX:', OUTPUT_LAYER_INDEX))
return interpreter
def test_tflite_model(tflite_model):
# Load TFLite model and allocate tensors.
interpreter = lite.Interpreter(model_path=tflite_model)
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Print input and output 'name' and 'shape'
print(
f'Model-Input > Name: {input_details[0]["name"]}, Shape: {input_details[0]["shape"]}'
)
print(
f'Model-Output > Name: {output_details[0]["name"]}, Shape: {output_details[0]["shape"]}\n'
)
# Test model on random input data.
input_shape = input_details[0]['shape']
input_data = np.array(np.random.random_sample(input_shape),
dtype=np.float32)
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
output_data = interpreter.get_tensor(output_details[0]['index'])
print(output_data)
def run_tflite_model(tflite_model_buf, input_data):
""" Generic function to execute TFLite """
try:
from tensorflow import lite as interpreter_wrapper
except ImportError:
from tensorflow.contrib import lite as interpreter_wrapper
input_data = input_data if isinstance(input_data, list) else [input_data]
interpreter = interpreter_wrapper.Interpreter(model_content=tflite_model_buf)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# set input
assert len(input_data) == len(input_details)
for i in range(len(input_details)):
interpreter.set_tensor(input_details[i]['index'], input_data[i])
# Run
interpreter.invoke()
# get output
tflite_output = list()
for i in range(len(output_details)):
tflite_output.append(interpreter.get_tensor(output_details[i]['index']))
return tflite_output
def __init__(self, frame, model_path, image_size):
"""
Method for initializing Inference Engine object
Args:
frame (object)
model_path (str)
image_size (int)
Attibutes:
frame (object) - Video capture object
model_path (str) - absolute path of tflite model file
image_size (int) - desired frame shape in tuple form
interpreter (Tensorflow object) - Tensorflow lite runtime object
input_details (array) - input characteristics of tflite model
output_details (array) - output characteristics of tflite model
"""
self.image_size = image_size
self.model_path = model_path
self.frame = frame
self.interpreter = tflite.Interpreter(model_path=model_path)
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()[0]["index"]
self.output_details = self.interpreter.get_output_details()[0]["index"]
def get_median_measure_tf_lite_python(model, number_of_measures,
tmp_keras_file, tmp_tflite_file):
"""given a model, loads that model in tf_lite and benchmarks the time needed for a prediction in python
:return: the median of number_of_measures trials"""
measures = np.zeros(number_of_measures)
model.compile(optimizer=SGD(), loss='binary_crossentropy')
save_model(model, tmp_keras_file)
# Convert to TensorFlow Lite model.
converter = lite.TFLiteConverter.from_keras_model_file(tmp_keras_file)
tflite_model = converter.convert()
with open(tmp_tflite_file, "wb") as file:
file.write(tflite_model)
# Load TFLite model and get measures
interpreter = lite.Interpreter(model_path=tmp_tflite_file)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
for k in range(number_of_measures):
# Test model on random input data.
input_shape = input_details[0]['shape']
input_data = np.array(np.random.random_sample(input_shape),
dtype=np.float32)
interpreter.set_tensor(input_details[0]['index'], input_data)
begin = time.perf_counter()
interpreter.invoke()
measures[k] = time.perf_counter() - begin
return np.median(measures)
def __init__(self):
big_model_path = "./big.tflite"
self.interpreter = tflite.Interpreter(model_path=big_model_path)
self.interpreter.allocate_tensors()
self.input_details = self.interpreter.get_input_details()
self.output_details = self.interpreter.get_output_details()
self.sampling_rate = 16000
self.frame_length = 640
self.frame_step = 320
self.lower_frequency = 20
self.upper_frequency = 4000
self.num_mel_bins = 40
self.num_coefficients = 10
num_spectrogram_bins = (self.frame_length) // 2 + 1
self.num_frames = (self.sampling_rate -
self.frame_length) // self.frame_step + 1
self.linear_to_mel_weight_matrix = tf.signal.linear_to_mel_weight_matrix(
self.num_mel_bins, num_spectrogram_bins, self.sampling_rate,
self.lower_frequency, self.upper_frequency)
def __init__(self, signature: Signature):
super(TFLiteModel, self).__init__(signature=signature)
model_path = "{}/{}".format(signature.model_path, signature.filename)
self.interpreter = tflite.Interpreter(model_path=model_path)
self.interpreter.allocate_tensors()
# Combine the information about the inputs and outputs from the signature.json file
# with the Interpreter runtime details
input_details = {
detail.get("name"): detail
for detail in self.interpreter.get_input_details()
}
self.model_inputs = {
key: {
**sig,
**input_details.get(sig.get(TENSOR_NAME))
}
for key, sig in self.signature.inputs.items()
}
output_details = {
detail.get("name"): detail
for detail in self.interpreter.get_output_details()
}
self.model_outputs = {
key: {
**sig,
**output_details.get(sig.get(TENSOR_NAME))
}
for key, sig in self.signature.outputs.items()
}
self.lock = Lock()
def run_tflite_graph(tflite_model_buf, input_data):
""" Generic function to execute TFLite """
input_data = convert_to_list(input_data)
interpreter = interpreter_wrapper.Interpreter(
model_content=tflite_model_buf)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# set input
assert len(input_data) == len(input_details)
for i in range(len(input_details)):
interpreter.set_tensor(input_details[i]['index'], input_data[i])
# Run
interpreter.invoke()
# get output
tflite_output = list()
for i in range(len(output_details)):
tflite_output.append(interpreter.get_tensor(
output_details[i]['index']))
return tflite_output
def __init__(self, model_dir_path, embeddings_type):
self._model_dir_path = model_dir_path
#-----------------------Using TFLite------------------------
self._model = tfl.Interpreter(
os.path.join(model_dir_path, 'neural_model.tflite'))
self._model.allocate_tensors()
self._input_idxs = {
x['name']: x['index']
for x in self._model.get_input_details()
}
self._output_idx = self._model.get_output_details()[0]['index']
#-----------------------------------------------------------
self._embeddings_type = embeddings_type
self._embeddings = self._load_embeddings()
with open(os.path.join(model_dir_path, 'feature_encoder.pckl'),
'rb') as f:
self._categorical_encoder = pickle.load(f)
with open(os.path.join(model_dir_path, 'label_encoder.pckl'),
'rb') as f:
self._roles = pickle.load(f)
with open(os.path.join(model_dir_path, 'feature_model.pckl'),
'rb') as f:
self._feature_model = pickle.load(f)
def load_lite_model(lite_model=None, tflite_load='Loaded', filename=None):
print(filename)
# Load TFLite model
if tflite_load == 'Loaded':
interpreter = tflite.Interpreter(model_content=lite_model)
elif tflite_load == 'File':
interpreter = tflite.Interpreter(model_path=filename + ".tflite")
# allocate tensors
interpreter.allocate_tensors()
# Get input and output tensors.
input_index = interpreter.get_input_details()[0]["index"]
output_index = interpreter.get_output_details()[0]["index"]
return interpreter, input_index, output_index
def make_interpreter(model_file):
model_file, *device = model_file.split('@')
try:
if backend._USING_EDGE_TPU:
interpreter = tflite.Interpreter(
model_path=model_file,
experimental_delegates=[
tflite.load_delegate(
backend._EDGETPU_SHARED_LIB,
{'device': device[0]} if device else {})
])
else:
interpreter = tflite.Interpreter(model_path=model_file)
except ValueError:
interpreter = tflite.Interpreter(model_path=model_file)
return interpreter
def init(tflite_model_file):
global interpreter, input_index, output_index
interpreter = tflite.Interpreter(tflite_model_file)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
input_index = input_details[0]["index"]
output_index = output_details[0]["index"]
def __init__(self, landmarks_path, model_path, verbose=False):
self.interpreter = tflite.Interpreter(model_path=str(model_path))
self.detector = dlib.get_frontal_face_detector()
self.predictor = dlib.shape_predictor(str(landmarks_path))
self.face_aligner = FaceAligner(self.predictor, desiredFaceWidth=256)
self.verbose = verbose
self.total_frames = None
def load_model_interpreter(model_path):
interpreter = tfl.Interpreter(model_path)
interpreter.allocate_tensors()
_, input_height, input_width, _ = interpreter.get_input_details(
)[0]['shape']
return interpreter, input_height, input_width
def init_model(tf_path, cls_path):
interpreter = tflite.Interpreter(model_path=tf_path)
interpreter.allocate_tensors()
with open(cls_path, 'rb') as infile:
(cls_model, class_names) = pickle.load(infile)
return interpreter, cls_model, class_names
def get_model(framework, model_variant):
"""
Load the desired EfficientPose model variant using the requested deep learning framework.
Args:
framework: string
Deep learning framework to use (Keras, TensorFlow, TensorFlow Lite or PyTorch)
model_variant: string
EfficientPose model to utilize (RT, I, II, III, IV, RT_Lite, I_Lite or II_Lite)
Returns:
Initialized EfficientPose model and corresponding resolution.
"""
# Keras
if framework in ['keras', 'k']:
from tensorflow.keras.backend import set_learning_phase
from tensorflow.keras.models import load_model
set_learning_phase(0)
model = load_model(join('models', 'keras', 'EfficientPose{0}.h5'.format(model_variant.upper())), custom_objects={'BilinearWeights': helpers.keras_BilinearWeights, 'Swish': helpers.Swish(helpers.eswish), 'eswish': helpers.eswish, 'swish1': helpers.swish1})
# TensorFlow
elif framework in ['tensorflow', 'tf']:
from tensorflow.python.platform.gfile import FastGFile
from tensorflow.compat.v1 import GraphDef
from tensorflow.compat.v1.keras.backend import get_session
from tensorflow import import_graph_def
f = FastGFile(join('models', 'tensorflow', 'EfficientPose{0}.pb'.format(model_variant.upper())), 'rb')
graph_def = GraphDef()
graph_def.ParseFromString(f.read())
f.close()
model = get_session()
model.graph.as_default()
import_graph_def(graph_def)
# TensorFlow Lite
elif framework in ['tensorflowlite', 'tflite']:
from tensorflow import lite
model = lite.Interpreter(model_path=join('models', 'tflite', 'EfficientPose{0}.tflite'.format(model_variant.upper())))
model.allocate_tensors()
# PyTorch
elif framework in ['pytorch', 'torch']:
from imp import load_source
from torch import load, quantization, backends
try:
MainModel = load_source('MainModel', join('models', 'pytorch', 'EfficientPose{0}.py'.format(model_variant.upper())))
except:
print('\n##########################################################################################################')
print('Desired model "EfficientPose{0}Lite" not available in PyTorch. Please select among "RT", "I", "II", "III" or "IV".'.format(model_variant.split('lite')[0].upper()))
print('##########################################################################################################\n')
return False, False
model = load(join('models', 'pytorch', 'EfficientPose{0}'.format(model_variant.upper())))
model.eval()
qconfig = quantization.get_default_qconfig('qnnpack')
backends.quantized.engine = 'qnnpack'
return model, {'rt': 224, 'i': 256, 'ii': 368, 'iii': 480, 'iv': 600, 'rt_lite': 224, 'i_lite': 256, 'ii_lite': 368}[model_variant]
def __init__(self, clientID, model_path): self.clientID = clientID self.myMqttClient = MyMQTT(self.clientID, "mqtt.eclipseprojects.io", 1883, self) #initialize the interpreter self.interpreter = tflite.Interpreter(model_path=model_path) self.interpreter.allocate_tensors() self.input_details = self.interpreter.get_input_details() self.output_details = self.interpreter.get_output_details()
def _load_tensorflowLite_model(self, model_path: str):
""" Load a TensorflowLite compressed flatbuffer model file and return predict function
"""
self.model = lite.Interpreter(model_path=model_path)
self.model.allocate_tensors()
self.input_details = self.model.get_input_details()
self.output_details = self.model.get_output_details()
self.input_shape = self.input_details[0]['shape']
return lambda x: self._tflitePredict(x)
def __init__(self, path):
self._interpreter = lite.Interpreter(path)
self._interpreter.allocate_tensors()
self._input_details = self._interpreter.get_input_details()
self._output_details = self._interpreter.get_output_details()
self._state_input = list(
filter(lambda x: x['name'] == 'state', self._input_details))[0]
self._portf_input = list(
filter(lambda x: x['name'] == 'portf', self._input_details))[0]
def emotion(file):
# load lite model
interpreter = lite.Interpreter(model_path="model.tflite")
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# prevents openCL usage and unnecessary logging messages
cv2.ocl.setUseOpenCL(False)
# dictionary which assigns each label an emotion (alphabetical order)
emotion_dict = {0: "Angry", 1: "Disgusted", 2: "Fearful", 3: "Happy", 4: "Neutral", 5: "Sad", 6: "Surprised"}
# load default face cascade
faceCascade = cv2.CascadeClassifier('cascade.xml')
cap = cv2.VideoCapture(file)
while(cap.isOpened()):
# Find haar cascade to draw bounding box around face
ret, frame = cap.read()
if not ret: break
frame = cv2.resize(frame, (200, 120), interpolation = cv2.INTER_CUBIC)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faces = faceCascade.detectMultiScale(gray, scaleFactor=1.3, minNeighbors=5)
for (x, y, w, h) in faces:
try:
cv2.rectangle(frame, (x, y - 10), (x + w, y + h + 10), (255, 0, 0), 1)
roi_gray = gray[y:y + h, x:x + w]
cropped_img = np.expand_dims(np.expand_dims(cv2.resize(roi_gray, (48, 48)), -1), 0)
try:
input_data = np.array(cropped_img, dtype=np.float32)
interpreter.set_tensor(input_details[0]['index'], input_data)
except:
input_data = np.array(cropped_img, dtype=np.int8)
interpreter.set_tensor(input_details[0]['index'], input_data)
interpreter.invoke()
prediction = interpreter.get_tensor(output_details[0]['index'])
maxindex = int(np.argmax(prediction))
frame = cv2.resize(frame, (800, 480), interpolation = cv2.INTER_CUBIC)
cv2.putText(frame, emotion_dict[maxindex], (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.LINE_AA)
except: pass
frame = cv2.resize(frame, (800, 480), interpolation = cv2.INTER_CUBIC)
cv2.imshow('Video', frame)
if cv2.waitKey(1) & 0xFF == ord('q'): break
cap.release()
cv2.destroyAllWindows()
return True
def load(path):
global interpreter
global input_details
global output_details
# Load the TFLite model and allocate tensors.
interpreter = tflite.Interpreter(model_path=path.format('model.tflite'))
interpreter.allocate_tensors()
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
def __init__(self, model_path: str):
self._interpreter = lite.Interpreter(path.join(model_path, 'model.tflite'))
_assert_is_classification_model(self._interpreter)
self._input_details = self._interpreter.get_input_details()[0]
self._output_details = self._interpreter.get_output_details()[0]
self._classifications = _read_text_list(path.join(model_path, 'dict.txt'))
self._meta_data = _read_meta_data(path.join(model_path, 'tflite_metadata.json'))
self._interpreter.allocate_tensors()
def __init__(self, tflite_file):
if not tflite_file.endswith(".tflite"):
raise ValueError("Not a TFLite file: {}".format(tflite_file))
if not os.path.exists(tflite_file):
raise ValueError(
"TFLite file {} doesn't exist".format(tflite_file))
logging.info("Loading TFLite file: {}".format(tflite_file))
self._intrp = lite.Interpreter(model_path=tflite_file)
self._intrp.allocate_tensors()
self._input_tensor_dict, self._output_tensor_dict = _get_input_and_output_names(
self._intrp)
def __init__(self, model):
self.interpreter = tflite.Interpreter(model_path=model)
self.interpreter.allocate_tensors()
self.input_detail = self.interpreter.get_input_details()
self.output_detail = self.interpreter.get_output_details()
[input_h, input_w] = self.input_detail[0]["shape"][1:3]
[output_h, output_w] = self.output_detail[0]["shape"][1:3]
self.InputSize = (input_w, input_h)
self.Stride = ((input_w / output_w), (input_h / output_h))
def __call__(self, img_url, print_func):
import tensorflow.lite as tflite
import numpy as np
print_func("Auto solving CAPTCHA")
interpreter = tflite.Interpreter(model_content=self.model_content)
u = requests.get(img_url)
raw_data = u.content
img = Image.open(BytesIO(raw_data))
img = np.asarray(img)
# normalize to [0...1]
img = (img / 255).astype(np.float32)
# convert to grayscale
r, g, b = img[:, :, 0], img[:, :, 1], img[:, :, 2]
input = 0.299 * r + 0.587 * g + 0.114 * b
# input has nowof shape (70, 175)
# we modify dimensions to match model's input
input = np.expand_dims(input, 0)
input = np.expand_dims(input, -1)
# input is now of shape (batch_size, 70, 175, 1)
# output will have shape (batch_size, 4, 26)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
interpreter.set_tensor(input_details[0]['index'], input)
interpreter.invoke()
# predict and get the output
output = interpreter.get_tensor(output_details[0]['index'])
# now get labels
labels_indices = np.argmax(output, axis=2)
available_chars = "abcdefghijklmnopqrstuvwxyz"
def decode(li):
result = []
for char in li:
result.append(available_chars[char])
return "".join(result)
decoded_label = [decode(x) for x in labels_indices][0]
print_func(f"CAPTCHA auto solved as '{decoded_label}'")
return decoded_label
