def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model', help='File path of Tflite model.', required=True)
parser.add_argument('--label', help='File path of label file.', required=True)
parser.add_argument('--image', help='File path of the image to be recognized.', required=True)
args = parser.parse_args()
try:
# Prepare labels.
labels = dataset_utils.read_label_file(args.label)
except:
print("Error loading labels")
exit(1)
try:
# Initialize engine.
engine = ClassificationEngine(args.model)
except:
print("Error loading model")
exit(1)
# Run inference.
img = Image.open(args.image)
for result in engine.classify_with_image(img, top_k=3):
print('---------------------------')
print(labels[result[0]])
print('Score : ', result[1])
exit(0)
class ImageClassifierHandler():
def __init__(self, model):
self.engine = ClassificationEngine(model)
def infer(self, frame):
"""Infer class of image.
Parameters
----------
frame : numpy.ndarray
Image to perform inference on
Returns
-------
results :
top_1 classification
inference_time : float
Time taken to perform inference in milliseconds
"""
frame = imutils.resize(frame, width=500)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame = Image.fromarray(frame)
start = time.time()
results = self.engine.classify_with_image(frame, top_k=1)
end = time.time()
inference_time = (end-start) * 1000.0
return results, inference_time
def _test_classify_cat(self, model_name, expected):
labels = test_utils.read_label_file(test_utils.test_data_path('imagenet_labels.txt'))
engine = ClassificationEngine(test_utils.test_data_path(model_name))
with test_utils.test_image('cat.bmp') as img:
ret = engine.classify_with_image(img, top_k=1)
self.assertEqual(len(ret), 1)
# Some models recognize it as egyptian cat while others recognize it as
# tabby cat.
self.assertTrue(labels[ret[0][0]] == 'tabby, tabby cat' or
labels[ret[0][0]] == 'Egyptian cat')
ret = engine.classify_with_image(img, top_k=3)
self.assertEqual(len(expected), len(ret))
for i in range(len(expected)):
# Check label.
self.assertEqual(labels[ret[i][0]], expected[i][0])
# Check score.
self.assertGreater(ret[i][1], expected[i][1])
class Classifier:
def __init__(self, using_model: str, label_file: str):
# Prepare labels.
self.labels = dataset_utils.read_label_file(label_file)
# Initialize engine.
self.engine = ClassificationEngine(using_model)
def classify(self, image: Image, top_k=3):
return self.engine.classify_with_image(image, top_k=top_k)
def run_benchmark(model, image):
"""Returns average inference time in ms on specified model and image."""
print('Benchmark for [%s] on %s' % (model, image))
engine = ClassificationEngine(test_utils.test_data_path(model))
iterations = 200 if 'edgetpu' in model else 10
with test_utils.test_image(image) as img:
result = 1000 * timeit.timeit(
lambda: engine.classify_with_image(img, threshold=0.4, top_k=10),
number=iterations) / iterations
print('%.2f ms (iterations = %d)' % (result, iterations))
return result
class image_classification:
MODEL_EFFICIENT_S = 'models/efficientnet-edgetpu-S_quant_edgetpu.tflite'
MODEL_EFFICIENT_M = 'models/efficientnet-edgetpu-M_quant_edgetpu.tflite'
MODEL_EFFICIENT_L = 'models/efficientnet-edgetpu-L_quant_edgetpu.tflite'
MODEL_MOBILENET_V1 = 'models/mobilenet_v1_1.0_224_quant_edgetpu.tflite'
MODEL_MOBILENET_V2 = 'models/mobilenet_v2_1.0_224_quant_edgetpu.tflite'
MODEL_INCEPTION_V1 = 'models/inception_v1_224_quant_edgetpu.tflite'
MODEL_INCEPTION_V2 = 'models/inception_v2_224_quant_edgetpu.tflite'
MODEL_INCEPTION_V3 = 'models/inception_v3_299_quant_edgetpu.tflite'
MODEL_INCEPTION_V4 = 'models/inception_v4_299_quant_edgetpu.tflite'
LABELS = 'models/imagenet_labels.txt'
def __init__(self,
threshold=0.5,
num_results=10,
model=MODEL_EFFICIENT_S,
labels=LABELS):
self.engine = ClassificationEngine(model)
self.model_labels = read_label_file(labels)
self.objs = None
self.scores = None
self.labels = None
self.threshold = threshold
self.num_results = num_results
def set_threshold(self, num):
self.threshold = num
def set_max_results(self, num):
self.num_results = num
def classify(self, img):
img = Image.fromarray(img)
self.objs = self.engine.classify_with_image(img,
threshold=self.threshold,
top_k=self.num_results)
self.scores = [obj[1] for obj in self.objs]
self.labels = [self.model_labels[obj[0]] for obj in self.objs]
return self.objs
def get_scores(self):
return self.scores
def get_labels(self):
return self.labels
def classification_task(num_inferences):
tid = threading.get_ident()
print('Thread: %d, %d inferences for classification task' %
(tid, num_inferences))
labels = test_utils.read_label_file(
test_utils.test_data_path('imagenet_labels.txt'))
model_name = 'mobilenet_v1_1.0_224_quant_edgetpu.tflite'
engine = ClassificationEngine(
test_utils.test_data_path(model_name))
print('Thread: %d, using device %s' % (tid, engine.device_path()))
with test_utils.test_image('cat.bmp') as img:
for _ in range(num_inferences):
ret = engine.classify_with_image(img, top_k=1)
self.assertEqual(len(ret), 1)
self.assertEqual(labels[ret[0][0]], 'Egyptian cat')
print('Thread: %d, done classification task' % tid)
def get_frame():
cap = cv2.VideoCapture(0)
engine = ClassificationEngine(modelPath)
prevTime = 0
while True:
_, frame = cap.read()
curTime = time.time()
sec = curTime - prevTime
prevTime = curTime
fps = 1 / (sec)
fpsText = "FPS : {:.2f}".format(fps)
frameRGB = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
framePIL = Image.fromarray(frameRGB)
classify = engine.classify_with_image(framePIL)
label = classify[0][0]
if label == 0:
labelText = "rock"
elif label == 1:
labelText = "paper"
elif label == 2:
labelText = "scissors"
score = round(classify[0][1], 3)
print(labelText, score)
cv2.putText(frame, labelText + " " + str(score), (0, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.putText(frame, fpsText, (0, 60), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 0, 255), 2)
imgencode = cv2.imencode('.jpg', frame)[1]
stringData = imgencode.tostring()
yield (b'--frame\r\n'
b'Content-Type: text/plain\r\n\r\n' + stringData + b'\r\n')
del (cap)
def _run_benchmark_for_model(model_name, image):
"""Benchmarks model with given image.
Args:
model_name: string, file name of the model.
image: string, name of the image used for test.
Returns:
float, average inference time.
"""
print('Benchmark for [', model_name, '] on ', image)
engine = ClassificationEngine(test_utils.test_data_path(model_name))
iterations = 200 if ('edgetpu' in model_name) else 10
with test_utils.test_image(image) as img_obj:
benchmark_time = timeit.timeit(
lambda: engine.classify_with_image(img_obj, threshold=0.4, top_k=10),
number=iterations)
time_per_inference = (benchmark_time / iterations) * 1000
print(time_per_inference, 'ms (iterations = ', iterations, ')')
return time_per_inference
def _test_model(self,
model_name,
expected_top_1=None,
expected_top_5=None):
engine = ClassificationEngine(test_utils.test_data_path(model_name))
with open(test_utils.test_data_path('imagenet/val.txt'),
'r') as gt_file:
gt = [line.strip().split(' ') for line in gt_file.readlines()]
top_1_count = 0
top_5_count = 0
print('Running inference for model %s...' % model_name)
for i in range(50000):
label = int(gt[i][1]) + 1
image_name = 'imagenet/ILSVRC2012_val_%s.JPEG' % str(i +
1).zfill(8)
with test_utils.test_image(image_name) as image:
image = self._crop_image(image.convert('RGB'))
prediction = engine.classify_with_image(image,
threshold=0.0,
top_k=5)
if prediction[0][0] == label:
top_1_count += 1
top_5_count += 1
else:
for j in range(1, len(prediction)):
if prediction[j][0] == label:
top_5_count += 1
top_1_accuracy = top_1_count / 50000.0
top_5_accuracy = top_5_count / 50000.0
print('Top 1 accuracy: %.2f%%' % (top_1_accuracy * 100))
print('Top 5 accuracy: %.2f%%' % (top_5_accuracy * 100))
if expected_top_1 is not None:
self.assertAlmostEqual(top_1_accuracy, expected_top_1, places=4)
if expected_top_5 is not None:
self.assertAlmostEqual(top_5_accuracy, expected_top_5, places=4)
try:
p = True
camera.start_preview()
while p:
if GPIO.input(23) == GPIO.LOW:
os.system("./jsay.sh \"かしゃ\"")
#camera.capture('./image.jpg')
name = "./image/image" + str(cntr) + ".jpg"
camera.capture(name)
cntr = cntr + 1
img = Image.open(name)
img2 = img.crop((280, 0, 720, 720))
start = time.time()
inf = engine.classify_with_image(img2, top_k=3)
eltime = time.time() - start
iftime = engine.get_inference_time()
print('num i = ', len(inf))
if (len(inf) == 0):
lab2 = u"よくわかりません"
else:
for result in inf:
print('---------------------------')
print(labels[result[0]])
print('Score : ', result[1])
print('elTime : ', eltime)
print('ifTime : ', iftime)
result = inf[0]
if result[1] >= 0.4:
lab2 = u"これは" + labels[result[0]] + "です"
def _transfer_learn_and_evaluate(self, model_path, keep_classes,
dataset_path, test_ratio, top_k_range):
"""Transfer-learns with given params and returns the evaluatoin result.
Args:
model_path: string, path of the base model.
keep_classes: bool, whether to keep base model classes.
dataset_path: string, path to the directory of dataset. The images
should be put under sub-directory named by category.
test_ratio: float, the ratio of images used for test.
top_k_range: int, top_k range to be evaluated. The function will return
accuracy from top 1 to top k.
Returns:
list of float numbers.
"""
print('--------------- Parsing dataset ----------------')
print('Dataset path:', dataset_path)
# train in fixed order to ensure the same evaluation result.
train_set, test_set = test_utils.prepare_data_set_from_directory(
dataset_path, test_ratio, True)
print('Image list successfully parsed! Number of Categories = ',
len(train_set))
input_shape = self._get_input_tensor_shape(model_path)
required_image_shape = (input_shape[2], input_shape[1]
) # (width, height)
print('--------------- Processing training data ----------------')
print('This process may take more than 30 seconds.')
num_classes = self._get_output_number_classes(
model_path) if keep_classes else 0
train_input = []
labels_map = {}
for class_id, (category, image_list) in enumerate(train_set.items()):
print('Processing {} ({} images)'.format(category,
len(image_list)))
train_input.append(
test_utils.prepare_images(image_list,
os.path.join(dataset_path, category),
required_image_shape))
labels_map[num_classes + class_id] = category
# train
print('---------------- Start training -----------------')
imprinting_engine = ImprintingEngine(model_path, keep_classes)
imprinting_engine.train_all(train_input)
print('---------------- Training finished -----------------')
output_model_path = tempfile.NamedTemporaryFile(suffix='.tflite')
imprinting_engine.save_model(output_model_path.name)
# Evaluate
print('---------------- Start evaluating -----------------')
classification_engine = ClassificationEngine(output_model_path.name)
# top[i] represents number of top (i+1) correct inference.
top_k_correct_count = [0] * top_k_range
image_num = 0
for category, image_list in test_set.items():
n = len(image_list)
print('Evaluating {} ({} images)'.format(category, n))
for image_name in image_list:
with test_image(
os.path.join(dataset_path, category,
image_name)) as raw_image:
# Set threshold as a negative number to ensure we get top k candidates
# even if its score is 0.
candidates = classification_engine.classify_with_image(
raw_image, threshold=-0.1, top_k=top_k_range)
for i in range(len(candidates)):
if candidates[i][0] in labels_map and labels_map[
candidates[i][0]] == category:
top_k_correct_count[i] += 1
break
image_num += n
for i in range(1, top_k_range):
top_k_correct_count[i] += top_k_correct_count[i - 1]
return [top_k_correct_count[i] / image_num for i in range(top_k_range)]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--taskSessionId',
help='taskSessionId number.',
required=True)
parser.add_argument('--epochtime',
help='time sample captured.',
required=True)
args = parser.parse_args()
# Get labels file
#os.chdir(os.getcwd())
os.chdir('/home/pi/ThinkBioT/ClassProcess/CModel')
for label_file in glob.glob("*.txt"):
print("LabelFile: " + label_file)
# Prepare labels.
labels = ReadLabelFile(label_file)
# Get model file
for model_file in glob.glob("*.tflite"):
print("Model File: " + model_file)
# Initialize engine.
time.sleep(5)
engine = ClassificationEngine(model_file)
# Prepare database connector & cursor
try:
conn = sqlite3.connect('/home/pi/tbt_database')
except Error:
print(Error)
c = conn.cursor()
# Run inference.
pathlist = Path('/home/pi/ThinkBioT/ClassProcess/CSpectrograms').glob(
'**/*.png')
for path in pathlist:
# because path is object not string
path_in_str = str(path)
print(path_in_str)
img = Image.open(path_in_str)
#for result in engine.classify_with_image(img, top_k=10):
for result in engine.classify_with_image(img,
threshold=0.001,
top_k=1,
resample=0):
print('---------------------------')
print(labels[result[0]])
print('Score : ', result[1])
c.execute(
"INSERT INTO ClassTasks(ClassTaskTime, ClassTaskSourceFile, ClassTaskResult, ClassTaskPercent, SessionID) VALUES(?,?,?,?,?)",
(args.epochtime, path_in_str, labels[result[0]], str(
result[1]), args.taskSessionId))
conn.commit()
#Remove processed file
os.remove(path_in_str)
conn.close()
#Record Completion in Log
f = open("/home/pi/ThinkBioT/tbt_log.txt", "a+")
f.write("auto_classify_spect Completed Classification at : " +
str(datetime.datetime.now()) + "\n")
f.close()
#Update to next process
os.system('sh /home/pi/ThinkBioT/tbt_update.sh')
def main():
"""set variables"""
video_number = 2
label_path = 'coco_labels.txt'
model_path_for_object = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
model_path_for_poopee = 'poopee_edgetpu.tflite'
threshold = 0.3
prevTime = 0 # initializing for calculating fps
box_colors = {} # initializing for setting color
pad_coordinate = {}
json_path = 'poopee_data.json'
"""set variables from json"""
json_data = read_json(json_path)
serial_num, user_id, ip_addr, image_name = json_data['serial_num'], json_data['user_id'], json_data['ip_addr'], json_data['image_name']
HOST, PORT = json_data['bluetooth']['HOST'], json_data['bluetooth']['PORT']
"""set variables to draw the bounding box and label for the pad"""
# pad_color = [int(j) for j in np.random.randint(0,255, 3)]
"""load class"""
poopee = Poopee(user_id, serial_num, ip_addr, image_name)
"""log in ppcam"""
response = poopee.ppcam_login()
if str(type(response)) == "<class 'dict'>":
token = response['device_access_token']
ppcam_id = response['ppcam_id']
pet_id = response['pet_id']
# print(token, ppcam_id, pet_id)
else:
return response # if login fails, the program is terminate
"""load labels for detect object"""
labels = load_labels(label_path)
"""load engine for detect object"""
engine_for_object = DetectionEngine(model_path_for_object)
"""load engine for predict poopee"""
engine_for_predict = ClassificationEngine(model_path_for_poopee)
"""load video"""
cap = cv2.VideoCapture(video_number)
# for checking the sequences
que_size = 20
queue = [2]*que_size
p_flag = False
isOnpad = False
while True:
ret, frame = cap.read()
if not ret:
print('cannot read frame')
break
img = frame[:, :, ::-1].copy() # BGR to RGB
img = Image.fromarray(img) # NumPy ndarray to PIL.Image
"""draw the bounding box and label for the pad"""
# json_data = read_json(json_path)
# pad_coordinate = json_data['pad']
# annotate_pad(frame, pad_coordinate, pad_color)
"""detect object"""
candidates = engine_for_object.detect_with_image(img, threshold=threshold, top_k=len(labels), keep_aspect_ratio=True, relative_coord=False, resample=0)
if candidates:
for obj in candidates:
"""set color for drawing"""
if obj.label_id in box_colors:
box_color = box_colors[obj.label_id] # the same color for the same object
else:
box_color = [int(j) for j in np.random.randint(0,255, 3)] # random color for new object
box_colors[obj.label_id] = box_color
coordinate = tuple(map(int, obj.bounding_box.ravel()))
accuracy = int(obj.score * 100)
# label_text = labels[obj.label_id] + ' (' + str(accuracy) + '%)'
"""draws the bounding box and label"""
# annotate_objects(frame, coordinate, label_text, box_color)
if obj.label_id == 17 or obj.label_id == 16 : # id 17 is dog & 16 is cat
"""crop the image"""
dog_image = crop_image(img, obj.bounding_box.ravel())
"""predict poopee"""
classify = engine_for_predict.classify_with_image(dog_image, top_k=1)
result = classify[0][0]
accuracy = classify[0][1] * 100
"""
predict poopee
0 --> poo
1 --> pee
2 --> nothing
"""
# print("dog's coordinate is", coordinate, end=' ')
# if result == 0:
# print('and dog poop', end=' ')
# elif result == 1:
# print('and dog pees', end=' ')
# else:
# print('and dog is nothing', end=' ')
# print('with', accuracy, 'percent accuracy.')
if ((result == 0 or 1) and isOnpad == False) :
"""send a signal to the snack bar if the dog defecates on the pad"""
if (accuracy > 70) :
dog_to_send = img
temp_key, temp_value = ('lux', 'luy', 'rdx', 'rdy'), coordinate
dog_coordinate = dict(zip(temp_key, temp_value))
json_data = read_json(json_path)
pad_coordinate = json_data['pad']
if ((pad_coordinate["rdx"] < dog_coordinate["lux"]) or (pad_coordinate["lux"] > dog_coordinate["rdx"]) or (pad_coordinate["luy"] > dog_coordinate["rdy"]) or (pad_coordinate["rdy"] < dog_coordinate["luy"])) :
isOnpad = False
else :
# dog area
dog_wid = dog_coordinate["rdx"] - dog_coordinate["lux"]
dog_hei = dog_coordinate["rdy"] - dog_coordinate["luy"]
dog_area = dog_wid * dog_hei
# overlapped area
lx = max(pad_coordinate["lux"], dog_coordinate["lux"])
rx = min(pad_coordinate["rdx"], dog_coordinate["rdx"])
dy = max(pad_coordinate["rdy"], dog_coordinate["rdy"])
uy = min(pad_coordinate["luy"], dog_coordinate["luy"])
co_wid = rx - lx
co_hei = dy - uy
co_area = co_wid * co_hei
# Decide whether the dog is on the pad
if (co_area / dog_area >= 0.4) :
isOnpad = True
for r in range(1,que_size) :
queue[r-1] = queue[r]
queue[que_size-1] = result
# Sequential decision
Q = np.array(queue)
counte = collections.Counter(Q)
c_0 = counte[0]
c_1 = counte[1]
c_2 = counte[2]
x = np.array([c_0, c_1, c_2])
Q_res = x.argmax()
# print(counte)
# print(Q_res)
if (Q_res == 0 or Q_res == 1) :
p_flag = True
# print("poo&pee flag up")
else :
if (p_flag == True) :
# Success
if (isOnpad == True) :
json_data = read_json(json_path)
feedback = json_data['feedback']
rnd = np.random.randint(1,10)
if (rnd <= feedback*10) :
send_feeding_signal(HOST, PORT)
response, token = send_result(poopee, dog_to_send, pet_id, token, 'SUCCESS', image_name)
print('Send SUCCESS signal!')
# defecates on wrong place
else :
response, token = send_result(poopee, dog_to_send, pet_id, token, 'FAIL', image_name)
print('Send FAIL signal!')
p_flag = False
isOnpad = False
"""calculating and drawing fps"""
currTime = time.time()
fps = 1/ (currTime - prevTime)
prevTime = currTime
# print('fps is', fps)
# cv2.putText(frame, "fps:%.1f"%fps, (10,30), cv2.FONT_HERSHEY_PLAIN, 2, (0,255,0), 2)
"""show video"""
# cv2.imshow('goodpp', frame)
# if cv2.waitKey(1)&0xFF == ord('q'):
# break # press q to break
"""release video"""
cap.release()
vs = VideoStream(src=0).start()
time.sleep(2.0)
print("starting_videostream")
# loop over the frames from the video stream
while True:
# grab the next frame and handle if we are reading from either
# VideoCapture or VideoStream
frame = vs.read()
frame = imutils.resize(frame, width=500)
orig = frame.copy()
# prepare the image for classification by converting from a NumPy
# array to PIL image format
frame = Image.fromarray(frame)
# make predictions on the input frame
results = model.classify_with_image(frame, top_k=1)
# ensure at least one result was found
if len(results) > 0:
# draw the predicted class label and probability on the
# output frame
(classID, score) = results[0]
text = "{}: {:.2f}% ({:.4f} sec)".format(classNames[classID],
score * 100, model.get_inference_time() / 1000)
cv2.putText(orig, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX,
0.5, (0, 0, 255), 2)
# show the output frame and wait for a key press
cv2.imshow("Frame", orig)
key = cv2.waitKey(1) & 0xFF
def facial_recogntion(props, smartLock):
"""
Processing camera pictures with multiple AI's and executing background jobs.
This is the main loop for the Door Lock.
AI stages: Movement detection -> Face detection -> Face classification
:param props: Necessery predefined properties
:type props: Dictonary
:param smartLock: object used to access unlocking functions and door status
:type smartLock: SmartLock object
"""
cap = cv2.VideoCapture(0)
#get labels
with open(props['classification']['labels']) as f:
labels_map = json.load(f)
#loading detection model
detection = DetectionEngine(props['detection'])
classification = ClassificationEngine(props['classification']['path'])
while cap.isOpened():
ret, cv2_im = cap.read()
if not ret:
break
# flip picture
cv2_im = cv2.flip(cv2_im, -1)
# Skip classification if door is open
if not smartLock.is_door_closed():
#shuffle all the pixels and make the picture unrecoginzable
cv2_im = cv2.randShuffle(cv2_im)
#TODO Indicate that door is open
cv2.imshow('FIDL', cv2_im)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
continue
pil_im = Image.fromarray(cv2_im)
# TODO check first if the there is movement in the picture before utilizing edge tpu (power saving)
#searching for faces in the picture
faces = detection.detect_with_image(pil_im,
threshold=0.5,
top_k=3,
keep_aspect_ratio=True,
relative_coord=True)
#check each face and append results to frame
height, width, _ = cv2_im.shape
for face in faces:
x0, y0, x1, y1 = face.bounding_box.flatten().tolist()
x0, y0, x1, y1 = int(x0 * width), int(y0 * height), int(
x1 * width), int(y1 * height)
#crop out face from camera picture
face_im = Image.fromarray(cv2_im[y0:y1, x0:x1])
#classify face
results = classification.classify_with_image(face_im,
threshold=0.1,
top_k=3)
#annotate frame
best_result = (0, 0) # index, score
text_lines = []
for index, score in results:
if score > best_result[1]:
best_result = (index, score)
text_lines.append(
'score=%.2f: %s' % (score, labels_map[str(index)])
) #TODO decide to show all scores or only best score
access_granted = props['user'][labels_map[str(
best_result[0])]]['access'] and score >= 0.9
#open the door
if access_granted and smartLock.unlocking == False:
threading.Thread(target=smartLock.unlock, daemon=True).start()
# TODO indicate unlocking
#Coloring green if access was granted
cv2_im = cv2.rectangle(cv2_im, (x0, y0), (x1, y1),
(0, 255, 0) if access_granted else
(0, 0, 255), 2)
for y, line in enumerate(text_lines):
cv2.putText(cv2_im,
line, (x0 + 10, y0 + y * 20 + 20),
fontFace=cv2.FONT_HERSHEY_SIMPLEX,
fontScale=0.5,
color=(255, 255, 255))
cv2.imshow('FIDL', cv2_im)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
class image_feature:
def __init__(self, path):
'''Initialize ros publisher, ros subscriber'''
# topic where we publish
self.font_path = "/home/pi/python/cascadia_font/CascadiaCode-Regular-VTT.ttf"
self.font = ImageFont.truetype(self.font_path, 15)
self.engine = ClassificationEngine(path +
'/retrained_model_edgetpu.tflite')
self.labels = dataset_utils.read_label_file(path + '/label_map.txt')
self.image_pub = rospy.Publisher("/output/image_classified/compressed",
CompressedImage,
queue_size=1)
# self.bridge = CvBridge()
self.tpu_objects_pub = rospy.Publisher("/tpu_objects",
tpu_objects,
queue_size=1)
# subscribed Topic
self.subscriber = rospy.Subscriber("/output/image_raw/compressed",
CompressedImage,
self.callback,
queue_size=1)
self.velocity_publisher = rospy.Publisher('/cmd_vel',
Twist,
queue_size=1)
self.vel_msg = Twist()
rospy.init_node('image_class', anonymous=True)
def callback(self, ros_data):
'''Callback function of subscribed topic.
Here images get converted and features detected'''
np_arr = np.frombuffer(ros_data.data, np.uint8)
image_np = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) # OpenCV >= 3.0:
prepimg = image_np[:, :, ::-1].copy()
prepimg = Image.fromarray(prepimg)
draw = ImageDraw.Draw(prepimg)
t1 = time.time()
out = self.engine.classify_with_image(prepimg, top_k=3)
# Initialize engine.
tpu_objects_msg = tpu_objects()
#print(out)
ii = 1
if out:
for obj in out:
#print ('-----------------------------------------')
if self.labels:
vbal = f"{self.labels[obj[0]]} {obj[1]:0.2f}"
draw.text((10, 20 * ii),
vbal,
font=self.font,
fill='green')
tpu_object_m = tpu_object()
tpu_object_m.cx = obj[1]
tpu_object_m.cy = obj[1]
tpu_object_m.width = 0
tpu_object_m.height = 0
tpu_object_m.label = self.labels[obj[0]]
tpu_objects_msg.tpu_objects.append(tpu_object_m)
#draw.text((box[0] + (box[2]-box[0]), box[1]), self.labels[obj.label_id] , fill='green')
ii = ii + 1
t2 = time.time()
fps = 1 / (t2 - t1)
fps_str = 'FPS = %.2f' % fps
draw.text((10, 220), fps_str, fill='green')
#### Create CompressedIamge ####
msg = CompressedImage()
msg.header.stamp = rospy.Time.now()
msg.format = "jpeg"
#prepimg.save(fileIO,'jpeg')
#msg.data = np.array(fileIO.getvalue()).tostring()
open_cv_image = np.array(prepimg)
open_cv_image = open_cv_image[:, :, ::-1].copy()
msg.data = np.array(cv2.imencode('.jpg', open_cv_image)[1]).tostring()
#msg.data = np.array(cv2.imencode('.jpg', image_np)[1]).tostring()
# Publish new image
self.image_pub.publish(msg)
self.tpu_objects_pub.publish(tpu_objects_msg)
for split in ['train', 'validation', 'test']:
for label in ['positive', 'negative']:
data_paths = glob.glob('data/' + split + '/' + label + '/*.png')
predictions['path'].extend(data_paths)
predictions['set'].extend([split] * len(data_paths))
predictions['truth'].extend([int(label == 'positive')] * len(data_paths))
inference_times = {}
model_paths = glob.glob('models/*/model_edgetpu.tflite')
for model_path in model_paths:
classifier = ClassificationEngine(model_path)
model_name = model_path.split('/')[1]
input_shape = [int(dim) for dim in model_name.split('_')[:3]]
predictions[model_name] = []
inference_times[model_name] = []
for path in predictions['path']:
image = Image.open(path)
# Set threshold to smaller than 0 to receive each prediction in range [0, 1]
prediction = classifier.classify_with_image(image, threshold=-1)
inference_time = classifier.get_inference_time()
# Predictions are returned as [(label_id, confidence_score)]
predictions[model_name].append(prediction[0][1].astype(float))
inference_times[model_name].append(inference_time)
with open('results/predictions_edgetpu.json', 'w') as fp:
json.dump(predictions, fp)
with open('results/inference_times_edgetpu.json', 'w') as fp:
json.dump(inference_times, fp)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--model',
help='File path of Tflite model.',
default=os.path.join(
'all_models',
'mobilenet_v2_1.0_224_inat_bird_quant_edgetpu.tflite'))
parser.add_argument('--label',
help='File path of label file.',
default=os.path.join('all_models',
'inat_bird_labels.txt'))
parser.add_argument('--image',
help='File path of the image to be recognized.',
required=False)
parser.add_argument('--dir',
help='File path of the dir to be recognized.',
required=False)
parser.add_argument('--dryrun',
help='Whether to actually move files or not.',
required=False,
default=False)
args = parser.parse_args()
# Prepare labels.
labels = dataset_utils.read_label_file(args.label)
# Initialize engine.
engine = ClassificationEngine(args.model)
# Run inference.
if args.image:
img = Image.open(args.image)
for result in engine.classify_with_image(img, top_k=3):
print('---------------------------')
print(labels[result[0]])
print('Score : ', result[1])
if args.dir:
f = []
for (dirpath, dirnames, filenames) in os.walk(args.dir):
for filename in filenames:
try:
filepath = "{}/{}".format(dirpath, filename)
if "boxed" in filename:
print("attempting to classify {}".format(filepath))
img = Image.open(filepath)
for result in engine.classify_with_image(img, top_k=3):
label = labels[result[0]]
percent = int(100 * result[1])
if label != "background":
print('dirpath', dirpath)
path_sections = dirpath.split("/")
new_dir = "/var/www/html/classified/"
if len(path_sections) == 4:
date = path_sections[2]
visitation_id = path_sections[3]
new_dir = "/var/www/html/classified/{}/{}".format(
date, visitation_id)
newname = filename.replace(
".png", "_{}_{}.png".format(
label.replace(" ", "-"), percent))
newpath = "{}/{}".format(new_dir, newname)
print('move {} -> {}'.format(
filepath, newpath))
print('dryrun', args.dryrun)
if args.dryrun == False:
if not os.path.exists(new_dir):
os.makedirs(new_dir)
shutil.move(os.path.abspath(filepath),
os.path.abspath(newpath))
if "full" in filename:
new_dir = get_new_dir(dirpath)
print('new full image dir {}'.format(new_dir))
new_path = "{}/{}".format(new_dir, filename)
if os.path.exists(new_dir):
print('full image move {} -> {}'.format(
os.path.abspath(filepath),
os.path.abspath(new_path)))
if args.dryrun == False:
shutil.move(os.path.abspath(filepath),
os.path.abspath(new_path))
else:
print('full image new directory doesnt exist')
except:
print("failed to classify ")
