def inference(feature_list, task, img):
x = -1
name = ""
dist = 0
for x in [0, 80, 160]:
for y in [0, 60, 120]:
print(x, y)
gc.collect()
#img2 = img.copy((x,y,160,120))
img2 = img.copy((0, 0, 320, 240))
fmap = kpu.forward(task, img2)
#p = np.array(fmap[:])
kpu.fmap_free(fmap)
'''
# get nearest target
name,dist,v = get_nearest(feature_list, p)
print("name:" + name + " dist:" + str(dist) + " v:" + str(v) + " mem:" + str(gc.mem_free()))
if dist < 200:
break
if dist < 200:
break
'''
#img2 = img.copy((x,y,160,120))
fmap = kpu.forward(task, img)
p = np.array(fmap[:])
kpu.fmap_free(fmap)
# get nearest target
name, dist, v = get_nearest(feature_list, p)
print("name:" + name + " dist:" + str(dist) + " v:" + str(v) + " mem:" +
str(gc.mem_free()))
print(x)
return name, dist, x
def main(labels=None,
model_addr="/sd/m.kmodel",
sensor_window=(224, 224),
lcd_rotation=0,
sensor_hmirror=False,
sensor_vflip=False):
sensor.reset()
sensor.set_pixformat(sensor.RGB565)
sensor.set_framesize(sensor.QVGA)
sensor.set_windowing(sensor_window)
sensor.set_hmirror(sensor_hmirror)
sensor.set_vflip(sensor_vflip)
sensor.run(1)
lcd.init(type=1)
lcd.rotation(lcd_rotation)
lcd.clear(lcd.WHITE)
if not labels:
with open('labels.txt', 'r') as f:
exec(f.read())
if not labels:
print("no labels.txt")
img = image.Image(size=(320, 240))
img.draw_string(90, 110, "no labels.txt", color=(255, 0, 0), scale=2)
lcd.display(img)
return 1
try:
img = image.Image("startup.jpg")
lcd.display(img)
except Exception:
img = image.Image(size=(320, 240))
img.draw_string(90,
110,
"loading model...",
color=(255, 255, 255),
scale=2)
lcd.display(img)
task = kpu.load(model_addr)
try:
while (True):
img = sensor.snapshot()
t = time.ticks_ms()
fmap = kpu.forward(task, img)
t = time.ticks_ms() - t
plist = fmap[:]
pmax = max(plist)
max_index = plist.index(pmax)
img.draw_string(0,
0,
"%.2f : %s" % (pmax, labels[max_index].strip()),
scale=2)
img.draw_string(0, 200, "t:%dms" % (t), scale=2)
lcd.display(img)
except Exception as e:
raise e
finally:
kpu.deinit(task)
def detect_landmarks(self, threshold, return_img=False):
img, roi = self.FaceDetector.detect_objects(threshold, True)
self.kpts = [[-1, -1] for x in range(5)]
if roi:
face_cut = img.cut(*roi)
face_cut_128 = face_cut.resize(128, 128)
a = face_cut_128.pix_to_ai()
fmap = kpu.forward(self.landmark_task, face_cut_128)
plist = fmap[:]
le = (roi[0] + int(plist[0] * roi[2] + 5),
roi[1] + int(plist[1] * roi[3] + 5))
re = (roi[0] + int(plist[2] * roi[2]),
roi[1] + int(plist[3] * roi[3] + 5))
n = (roi[0] + int(plist[4] * roi[2]),
roi[1] + int(plist[5] * roi[3]))
lm = (roi[0] + int(plist[6] * roi[2]),
roi[1] + int(plist[7] * roi[3]))
rm = (roi[0] + int(plist[8] * roi[2]),
roi[1] + int(plist[9] * roi[3]))
self.kpts = [le, re, n, lm, rm]
del (face_cut)
del (face_cut_128)
if return_img:
return img, self.kpts, roi
a = img.draw_cross(le[0],
le[1],
color=(0, 255, 0),
size=5,
thickness=3)
a = img.draw_cross(re[0],
re[1],
color=(0, 255, 0),
size=5,
thickness=3)
a = img.draw_cross(n[0],
n[1],
color=(0, 255, 0),
size=5,
thickness=3)
a = img.draw_cross(lm[0],
lm[1],
color=(0, 255, 0),
size=5,
thickness=3)
a = img.draw_cross(rm[0],
rm[1],
color=(0, 255, 0),
size=5,
thickness=3)
a = lcd.display(img)
return img, None, None
def mnist_run(img, dx, dy, dis, x00=0, y00=80, nnn=2):
if nnn == 4:
x00 = x00
dy = dy
img0 = img.copy((x00 + dis * nnn, y00 + nnn * 0, dx, dy))
#img0.mean(2, threshold=True, offset=1, invert=True) #A
img0.median(2, percentile=0.3, threshold=True, offset=-3, invert=True)
#img0.midpoint(2, bias=0.3, threshold=True, offset=0, invert=True)
#img0.mode(2, threshold=True, offset=0, invert=True) #B
#img0.binary([(110,255)], invert = True)
for dx0 in range(dx):
for dy0 in range(dy):
a0 = img0.get_pixel(dx0, dy0)
img.set_pixel(x00 + dis * nnn + dx0, y00 + nnn * 0 + dy0, a0)
#img1 = img0.copy((1,1, dx-1, dy-1))
img1 = img0
img1 = img1.resize(28, 28)
img1 = img1.to_grayscale(1)
img1.pix_to_ai()
fmap = kpu.forward(task, img1)
plist = fmap[:]
pmax = max(plist)
max_index = plist.index(pmax)
kpu.fmap_free(fmap)
return max_index, pmax
def compute_features(self, register=False):
img, src_point, self.roi = self.LandmarkDetector.detect_landmarks(
0.4, return_img=True)
self.max_score = 0
self.id = -1
if src_point:
# align face to standard position
a = img.pix_to_ai()
T = image.get_affine_transform(src_point, self.dst_point)
a = image.warp_affine_ai(img, self.img_face, T)
a = self.img_face.ai_to_pix()
if register:
reg_img = image.Image('logo.jpg')
a = reg_img.draw_image(
self.img_face,
(lcd.width() // 2 - 68, lcd.height() // 2 - 20))
a = reg_img.draw_string(30,
lcd.height() // 2 - 48,
"Registring face",
color=(0, 255, 0),
scale=2,
mono_space=False)
a = lcd.display(reg_img)
del (reg_img)
time.sleep(2)
a = self.img_face.pix_to_ai()
# calculate face feature vector
fmap = kpu.forward(self.fe_task, self.img_face)
#print(fmap[:])
vector = list(map(lambda x: x / 256, fmap[:]))
self.feature = kpu.face_encode(vector)
for id in self.db.keys():
entry = _unhexlify(self.db[id]['vector'])
score = kpu.face_compare(entry, self.feature)
if score > self.max_score:
self.max_score = score
name = self.db[id]['name']
self.id = id
if not self.max_score > self.threshold:
name = 'Unknown'
a = img.draw_rectangle(self.roi,
color=(0x1c, 0xa2, 0xff),
thickness=2)
a = img.draw_string(self.roi[0],
self.roi[1] - 14,
("%s %%: %.2f" % (name, self.max_score)),
color=(255, 255, 255),
scale=1.5,
mono_space=False)
a = lcd.display(img)
del (img)
def get_feature(task):
img = sensor.snapshot()
img.draw_rectangle(1, 46, 222, 132, color=(255, 0, 0), thickness=3)
lcd.display(img)
feature = kpu.forward(task, img)
print('get_feature')
gc.collect()
return np.array(feature[:])
def updateKpu():
global g_cFiler
global g_selCnt
global g_cWav
global g_task
global g_powArr
info = g_cFiler.getInfoList()[g_selCnt]
if (g_task == None):
g_task = _resetTask()
if (g_task == None):
g_cWav.play('/sd/snd/sys_ng.wav')
g_cWav.wait()
img = sensor.snapshot()
if (info.modelType == 'yolo2'):
plist = []
for id in range(0, len(info.classList)):
plist.append(0.0)
code_obj = kpu.run_yolo2(g_task, img)
if code_obj: # object detected
for i in code_obj:
rect_size = i.w() * i.h()
if rect_size > 10:
print(len(plist))
print(i.classid())
plist[i.classid()] = 0.95
else:
fmap = kpu.forward(g_task, img, False)
plist = fmap[:]
colArr = [(255, 0, 0), (0, 255, 0), (0, 0, 255), (5, 5, 5), (0, 255, 255),
(255, 255, 0), (128, 128, 128), (50, 200, 50)]
for id in range(0, len(plist)):
if (plist[id] > 0.9):
g_powArr[id] = min((g_powArr[id] + plist[id] - 0.9) * 5.0, 100.0)
else:
g_powArr[id] -= 10.0
g_powArr[id] = max(g_powArr[id], 0.0)
img.draw_rectangle((10, 50 + 10 * id, int(g_powArr[id] * 1), 8),
colArr[id & 7], 10, True)
if (g_powArr[id] >= 100.0):
g_powArr[id] = 0.0
info = g_cFiler.getInfoList()[g_selCnt]
labels = info.classList
wavPath = info.dirName + '/' + labels[id] + '.wav'
lcd.draw_string(0, 20, wavPath)
g_cWav.play('/sd/models/' + wavPath)
g_cWav.wait()
a = lcd.display(img)
g_cButton.update()
def get_feature(task, img):
print("--- 1")
free()
print("--- 4")
feature = kpu.forward(task,img)
print("--- 5")
free()
print("--- 6")
ar = np.array(feature[:])
print("--- 7")
return ar
def get_feature(task):
count = 20
for i in range(3):
for j in range(count):
img = sensor.snapshot()
if j < (count>>1):
img.draw_rectangle(1,46,222,132,color=br.get_color(255,0,0),thickness=3)
lcd.display(img)
time.sleep(1.0)
feature = kpu.forward(task,img)
return feature
def run_kpu():
while True:
print("KPU : fetch data from Camera")
img = sensor.snapshot()
kpu_img = img.copy((0, 60, 160, 60))
print("KPU : run kpu")
a = kpu.forward(task, kpu_img)
print("KPU : fetch data from kpu")
output_steer = kpu.get_output(task, 0)
output_throttle = kpu.get_output(task, 1)
print("KPU : Data", output_steer, output_throttle)
time.sleep_ms(1)
def get_feature(task):
count = 20
for i in range(3):
for j in range(count):
img = sensor.snapshot()
if j < (count>>1):
br.set_led(1,0,0)
img.draw_rectangle(1,46,222,132,color=br.get_color(255,0,0),thickness=3)
else:
br.set_led(0,0,0)
br.show_image(img)
time.sleep(1.0)
feature = kpu.forward(task,img)
return np.array(feature[:])
def measure_fps(model_file):
task = kpu.load(model_file)
kpu.set_outputs(task, 0, 1, 1, 2)
clock = time.clock()
fps_ = []
for i in range(20):
img = sensor.snapshot()
clock.tick()
fmap = kpu.forward(task, img)
lcd.display(img, oft=(0, 0))
fps_.append(clock.fps())
average_fps = sum(fps_) / len(fps_)
print(average_fps)
global fps_result
fps_result = average_fps
_ = kpu.deinit(task)
def inference(model_file):
task = kpu.load(model_file)
kpu.set_outputs(task, 0, 1, 1, 2)
clock = time.clock()
while (True):
img = sensor.snapshot()
clock.tick()
fmap = kpu.forward(task, img)
fps = clock.fps()
plist = fmap[:]
pmax = max(plist)
max_index = plist.index(pmax)
a = lcd.display(img, oft=(0, 0))
lcd.draw_string(
0, 128, "%.2f:%s " %
(pmax, labels[max_index].strip()))
_ = kpu.deinit(task)
def measure_latency(model_file):
task = kpu.load(model_file)
kpu.set_outputs(task, 0, 1, 1, 2)
clock = time.clock()
latency_ = []
for i in range(20):
img = sensor.snapshot()
clock.tick()
t1 = time.ticks_us()
fmap = kpu.forward(task, img)
t2 = time.ticks_diff(time.ticks_us(), t1) / 1000
lcd.display(img, oft=(0, 0))
latency_.append(t2)
average_latency = sum(latency_) / len(latency_)
print(average_latency)
global latency_result
latency_result = average_latency
_ = kpu.deinit(task)
def work(img):
tmp = img.cut(48, 8, 224, 224)
#tmp = img.resize(224, 224)
img.draw_rectangle(48, 8, 224, 224, thickness=2, color=(0, 255, 0))
tmp.pix_to_ai()
MoblieNet.things = kpu.forward(MoblieNet.task, tmp)
del tmp
#print(MoblieNet.task, tmp)
if MoblieNet.things:
#print(MoblieNet.things)
plist = MoblieNet.things[:]
pmax = max(plist)
max_index = plist.index(pmax)
MoblieNet.score = pmax
MoblieNet.result = MoblieNet.labels[max_index]
return img
def autopilot_loop(self):
img = sensor.snapshot()
img = img.resize(224, 224)
#img.draw_rectangle(0, 0, 224, 93, color=(0,0,0), fill=True)
#img.draw_rectangle(0, 0, 224, 224, color=(255,0,0), fill=False)
#img.draw_rectangle(0, 0, 112, 112, color=(0,255,0), fill=False)
img.pix_to_ai()
fmap = kpu.forward(self._task, img)
plist = fmap[:]
if True:
# categorical model
pmax = max(plist)
max_index = plist.index(pmax)
print(" ".join(["%2d" % (p * 10) for p in plist]))
print((" " * (3 * max_index)) + "**")
siz = len(plist) // 2
steering = 45 * ((max_index - siz) / siz)
throttle = 45
print("throttle=", throttle, " steering=", steering)
#left, right = self._twoWheelSteeringThrottle.run(throttle, steering)
#left = int(left * 100)
#right = int(right * 100)
#s = "auto v_ms=%d left=%d right=%d " % (time.ticks_ms(), left, right)
s = "auto v_ms=%d throttle=%d steering=%d " % (time.ticks_ms(),
throttle, steering)
self.sendToC(s + "\n")
print("V: " + s)
if False:
# liner model
print("plist=", plist)
steering = plist[0] - 100
throttle = 40
#print("throttle=", throttle, " steering=", steering)
s = "auto v_ms=%d throttle=%d steering=%d " % (time.ticks_ms(),
throttle, steering)
self.sendToC(s + "\n")
print("V: " + s)
lcd.display(img)
img = None
gc.collect()
def classify_image(self, threshold):
img = snapshot()
img_copy = img.resize(224, 224)
a = img_copy.pix_to_ai()
fmap = kpu.forward(self.image_objects_task, img_copy)
plist = fmap[:]
pmax = max(plist)
self.percent = pmax
if self.percent >= threshold:
max_index = plist.index(pmax)
a = img.draw_string(8,
rows[self.row],
("Result: %s %%: %.2f" %
(self.labels[max_index].strip(), pmax)),
color=(255, 255, 255),
scale=1.5,
mono_space=False)
self.image_class = self.labels[max_index].strip()
a = lcd.display(img)
del (img)
del (img_copy)
def main(labels = None, model_addr="/sd/m.kmodel", lcd_rotation=0, sensor_hmirror=False, sensor_vflip=False):
gc.collect()
sensor.reset()
sensor.set_pixformat(sensor.RGB565)
sensor.set_framesize(sensor.QVGA)
sensor.set_windowing((224, 224))
sensor.set_hmirror(sensor_hmirror)
sensor.set_vflip(sensor_vflip)
sensor.run(1)
lcd.init(type=1)
lcd.rotation(lcd_rotation)
lcd.clear(lcd.WHITE)
if not labels:
raise Exception("no labels.txt")
task = kpu.load(model_addr)
try:
while(True):
img = sensor.snapshot()
t = time.ticks_ms()
fmap = kpu.forward(task, img)
t = time.ticks_ms() - t
plist=fmap[:]
pmax=max(plist)
max_index=plist.index(pmax)
img.draw_string(0,0, "%.2f\n%s" %(pmax, labels[max_index].strip()), scale=2, color=(255, 0, 0))
img.draw_string(0, 200, "t:%dms" %(t), scale=2, color=(255, 0, 0))
lcd.display(img)
except Exception as e:
sys.print_exception(e)
finally:
kpu.deinit(task)
def get_feature(task):
count = 20
print('get_feature 1')
for i in range(3):
print('get_feature 2-' + str(3 - i))
br.play_sound("/sd/voice/" + str(3 - i) + "s.wav", vol)
for j in range(count):
img = sensor.snapshot()
if j < (count >> 1):
img.draw_rectangle(1,
46,
222,
132,
color=br.get_color(255, 0, 0),
thickness=3)
lcd.display(img)
# print('get_feature 2')
# time.sleep(1.0)
print('get_feature 3')
feature = kpu.forward(task, img)
print('get_feature 4')
return np.array(feature[:])
print('[info]: Exit by user operation')
sys.exit()
initialize_camera()
task = kpu.load("/sd/model/mbnet751.kmodel")
print('[info]: Started.')
but_stu = 1
fore_color = rgb888_to_rgb565(119, 48, 48)
back_color = rgb888_to_rgb565(250, 205, 137)
border_color = (back_color >> 8) | ((back_color & 0xff) << 8)
img = sensor.snapshot()
a = kpu.set_layers(task, 29)
firstmap = kpu.forward(task, img)
firstdata = firstmap[:]
try:
while (True):
img = sensor.snapshot()
fmap = kpu.forward(task, img)
plist = fmap[:]
dist = 0
for i in range(768):
dist = dist + (plist[i] - firstdata[i]) * (plist[i] - firstdata[i])
if dist < 200:
img.draw_rectangle(1, 46, 222, 132, color=31, thickness=3)
img.draw_string(2, 47, "%.2f " % (dist))
lcd.display(img)
dist_int = int(dist)
print('loading face detect model')
task_detect_face = kpu.load(0x300000) # Charge face detect model into KPU
print('loading face expresion classify model')
task_classify_face = kpu.load(
0x500000) # Charge face classification model into KPU
a = kpu.set_outputs(task_classify_face, 0, 1, 1, 2)
anchor = (1.889, 2.5245, 2.9465, 3.94056, 3.99987, 5.3658, 5.155437, 6.92275,
6.718375, 9.01025)
a = kpu.init_yolo2(task_detect_face, 0.5, 0.3, 5, anchor)
labels = ['happy', 'sad'] # Facial expression labels
print('configuration complete')
while (True):
clock.tick() # Update the FPS clock.
img = sensor.snapshot() # Take a picture and return the image.
detected_face = kpu.run_yolo2(task_detect_face, img)
if detected_face:
for i in detected_face:
face = img.cut(i.x(), i.y(), i.w(), i.h())
face_128 = face.resize(128, 128)
a = face_128.pix_to_ai()
fmap = kpu.forward(task_classify_face, face_128)
plist = fmap[:]
pmax = max(plist)
print("%s: %s" % (labels[plist.index(pmax)], pmax))
a = kpu.init_yolo2(task, 0.5, 0.3, 5, anchor)
while (True):
img = sensor.snapshot()
code = kpu.run_yolo2(task, img)
if code:
for i in code:
xx = i.x() - 10
yy = i.y() - 10
ww = i.w() + 15
hh = i.h() + 10
face_cut = img.cut(xx, yy, ww, hh)
face_cut = face_cut.resize(128, 128)
a = face_cut.pix_to_ai()
a = kpu.set_outputs(tid, 0, 1, 1, 2)
fmap = kpu.forward(tid, face_cut)
plist = fmap[:]
pmax = max(plist)
max_index = plist.index(pmax)
print(plist)
if plist[0] >= 0.94:
a = img.draw_rectangle(xx,
yy,
ww,
hh,
color=(255, 0, 0),
thickness=4)
elif plist[1] >= 0.98:
a = img.draw_rectangle(xx,
yy,
faces = kpu.run_yolo2(task_face_detect, img)
if faces:
#--- check face size ---#
x1 = faces[0].x() - 10
y1 = faces[0].y() - 10
w = faces[0].w() + 20
h = faces[0].h() + 10
if w > 80 and h > 80:
#--- crop target face ---#
face = img.cut(x1,y1,w,h)
face = face.resize(112,112)
a = img.draw_rectangle(x1,y1,w,h,color = (255,0,0), thickness=2)
a = face.pix_to_ai()
#--- encode face ---#
fmap = kpu.forward(task_face_encode,face)
encoded = fmap[:]
#--- save new face ---#
if dude.IsBootPressed() : # User pressed BOOT button
time.sleep_ms(500)
lcd.clear((0,255,0))
time.sleep_ms(2000)
append_dataset(dataset_filename,encoded)
print("saved")
if face_dataset:
#--- find match ---#
score, pid = match(encoded,face_dataset)
print(score)
print("Match ID %d score = %.3f" % (pid,score))
if score < face_threshold : # recognized
ACCURACY = 85
while (1):
img = sensor.snapshot()
clock.tick()
code = kpu.run_yolo2(task_fd, img)
if code:
for i in code:
# Cut face and resize to 128x128
a = img.draw_rectangle(i.rect())
face_cut = img.cut(i.x(), i.y(), i.w(), i.h())
face_cut_128 = face_cut.resize(128, 128)
a = face_cut_128.pix_to_ai()
# a = img.draw_image(face_cut_128, (0,0))
# Landmark for face 5 points
fmap = kpu.forward(task_ld, face_cut_128)
plist = fmap[:]
le = (i.x() + int(plist[0] * i.w() - 10),
i.y() + int(plist[1] * i.h()))
re = (i.x() + int(plist[2] * i.w()), i.y() + int(plist[3] * i.h()))
nose = (i.x() + int(plist[4] * i.w()),
i.y() + int(plist[5] * i.h()))
lm = (i.x() + int(plist[6] * i.w()), i.y() + int(plist[7] * i.h()))
rm = (i.x() + int(plist[8] * i.w()), i.y() + int(plist[9] * i.h()))
a = img.draw_circle(le[0], le[1], 4)
a = img.draw_circle(re[0], re[1], 4)
a = img.draw_circle(nose[0], nose[1], 4)
a = img.draw_circle(lm[0], lm[1], 4)
a = img.draw_circle(rm[0], rm[1], 4)
# align face to standard position
src_point = [le, re, nose, lm, rm]
def run(self,
on_detect,
on_img,
on_clear,
on_people=None,
always_show_img=False):
img = sensor.snapshot()
try:
code = kpu.run_yolo2(self._m_fd, img)
except Exception:
return
if code:
for i in code:
face_cut = img.cut(i.x(), i.y(), i.w(), i.h())
face_cut_128 = face_cut.resize(128, 128)
a = face_cut_128.pix_to_ai()
#a = img.draw_image(face_cut_128, (0,0))
# Landmark for face 5 points
try:
fmap = kpu.forward(self._m_ld, face_cut_128)
except Exception:
continue
plist = fmap[:]
le = (i.x() + int(plist[0] * i.w() - 10),
i.y() + int(plist[1] * i.h()))
re = (i.x() + int(plist[2] * i.w()),
i.y() + int(plist[3] * i.h()))
nose = (i.x() + int(plist[4] * i.w()),
i.y() + int(plist[5] * i.h()))
lm = (i.x() + int(plist[6] * i.w()),
i.y() + int(plist[7] * i.h()))
rm = (i.x() + int(plist[8] * i.w()),
i.y() + int(plist[9] * i.h()))
a = img.draw_circle(le[0], le[1], 4)
a = img.draw_circle(re[0], re[1], 4)
a = img.draw_circle(nose[0], nose[1], 4)
a = img.draw_circle(lm[0], lm[1], 4)
a = img.draw_circle(rm[0], rm[1], 4)
# align face to standard position
src_point = [le, re, nose, lm, rm]
T = image.get_affine_transform(src_point, self._dst_point)
a = image.warp_affine_ai(img, self.img_face, T)
a = self.img_face.ai_to_pix()
#a = img.draw_image(img_face, (128,0))
del (face_cut_128)
# calculate face feature vector
try:
fmap = kpu.forward(self._m_fe, self.img_face)
except Exception:
continue
feature = kpu.face_encode(fmap[:])
scores = []
for j in range(len(self.features)):
score = kpu.face_compare(self.features[j], feature)
scores.append(score)
max_score = 0
index = 0
for k in range(len(scores)):
if max_score < scores[k]:
max_score = scores[k]
index = k
if max_score > 85:
a = img.draw_rectangle(i.rect(), color=(0, 255, 0))
a = img.draw_string(
i.x(),
i.y(), ("%s :%2.1f" % (self.names[index], max_score)),
color=(0, 255, 0),
scale=2)
on_detect(self.names[index], feature, max_score, img)
else:
a = img.draw_rectangle(i.rect(), color=(255, 0, 0))
# a = img.draw_string(i.x(),i.y(), ("X :%2.1f" % (max_score)), color=(255,0,0),scale=2)
on_img(img)
if on_people:
on_people(feature, img)
self._show_img_t = time.ticks_ms() / 1000.0
else:
if always_show_img:
on_img(img)
else:
if time.ticks_ms(
) - self._show_img_t * 1000 < self.show_img_timeout * 1000:
on_img(img)
else:
on_clear()
send_data(str(temp), 1)
buzz(100)
sleep_ms(SLEEP_TIME // 2)
# measure the user's DISTANCE from the CAMERA
distance_cam = get_distance(echo_cam, trigger_cam)
if distance_cam <= DISTANCE_THRESHOLD_CAM:
sleep_ms(100)
# take a picture and display it on the lcd
img = sensor.snapshot()
last_image = sensor.snapshot()
lcd.display(img)
# run the image through the model to get classification and send it to the server
img = img.resize(IMG_SIZE, IMG_SIZE)
img.pix_to_ai()
res = kpu.forward(task, img)[:]
res_max = max(res)
max_index = res.index(res_max)
send_data("{:.2f}".format(res_max * 100), 4)
send_data(CLASSES[max_index], 2)
buzz(100)
sleep_ms(SLEEP_TIME)
# image doesn't get garbage collected automatically when pix_to_ai() is called
del img
else:
# take a picture and display it on the lcd
img = sensor.snapshot()
lcd.display(img)
# ask for data
send_data("", 0)
feature = get_feature(task)
feature_list.append([name,feature[:]])
if name=='0':
feature_0.append([name,feature[:]])
if name=='100':
feature_100.append([name,feature[:]])
save(feature_file, feature_list)
br.play_sound("/sd/set.wav")
gc.collect()
# print(gc.mem_free())
kpu.fmap_free(feature)
print("[QR]: " + name)
continue
# inference
fmap = kpu.forward(task, img)
plist=fmap[:]
clock.tick()
if len(feature_0)>0 and len(feature_100)>0:
p = plist
f0 = feature_0[0]
f100 = feature_100[0]
dist = 100.0 * get_dist(f0[1],f100[1],p)
dist_str = "%.1f"%(dist)
print("[DISTANCE]: " + dist_str)
img.draw_string(2,47, dist_str,scale=3)
lcd.display(img)
continue
name,dist = get_nearest(feature_list,plist)
#print(clock.fps())
if dist < 50 and name != "exclude": #50 is modified from original value 200
import sensor, lcd, image
import KPU as kpu
lcd.init()
sensor.reset()
sensor.set_pixformat(sensor.RGB565)
sensor.set_framesize(sensor.QVGA)
sensor.set_windowing((224, 224)) #set to 224x224 input
sensor.set_hmirror(0) #flip camera
task = kpu.load(0x200000) #load model from flash address 0x200000
sensor.run(1)
while True:
img = sensor.snapshot()
lcd.display(img, oft=(0, 0)) #display large picture
img = img.to_grayscale(1) #convert to gray
img = img.resize(28, 28) #resize to mnist input 28x28
a = img.invert() #invert picture as mnist need
a = img.strech_char(1) #preprocessing pictures, eliminate dark corner
lcd.display(img, oft=(224, 32)) #display small 28x28 picture
a = img.pix_to_ai()
#generate data for ai
fmap = kpu.forward(task, img) #run neural network model
plist = fmap[:] #get result (10 digit's probability)
pmax = max(plist) #get max probability
max_index = plist.index(pmax) #get the digit
lcd.draw_string(224, 0, "%d: %.3f" % (max_index, pmax), lcd.WHITE,
lcd.BLACK) #show result
code = kpu.run_yolo2(task_fd, img) # 运行人脸检测模型，获取人脸坐标位置
read_data = uart.read()
if read_data:
with open("/sd/recordftr3.txt", "a") as f:
f.write(read_data[2])
if code: # 如果检测到人脸
for i in code: # 迭代坐标框
# Cut face and resize to 128x128
a = img.draw_rectangle(i.rect()) # 在屏幕显示人脸方框
face_cut = img.cut(i.x(), i.y(), i.w(),
i.h()) # 裁剪人脸部分图片到 face_cut
face_cut_128 = face_cut.resize(128, 128) # 将裁出的人脸图片 缩放到128 * 128像素
a = face_cut_128.pix_to_ai() # 将猜出图片转换为kpu接受的格式
#a = img.draw_image(face_cut_128, (0,0))
# Landmark for face 5 points
fmap = kpu.forward(task_ld, face_cut_128) # 运行人脸5点关键点检测模型
plist = fmap[:] # 获取关键点预测结果
le = (i.x() + int(plist[0] * i.w() - 10),
i.y() + int(plist[1] * i.h())
) # 计算左眼位置， 这里在w方向-10 用来补偿模型转换带来的精度损失
re = (i.x() + int(plist[2] * i.w()), i.y() + int(plist[3] * i.h())
) # 计算右眼位置
nose = (i.x() + int(plist[4] * i.w()),
i.y() + int(plist[5] * i.h())) #计算鼻子位置
lm = (i.x() + int(plist[6] * i.w()), i.y() + int(plist[7] * i.h())
) #计算左嘴角位置
rm = (i.x() + int(plist[8] * i.w()), i.y() + int(plist[9] * i.h())
) #右嘴角位置
a = img.draw_circle(le[0], le[1], 4)
a = img.draw_circle(re[0], re[1], 4)
a = img.draw_circle(nose[0], nose[1], 4)
def work(img):
#tmp = image.Image(size=(160, 160))
#tmp = tmp.to_grayscale()
ui.canvas = ui.canvas.to_grayscale()
bak = img.cut(100, 60, 120, 120).to_grayscale()
#img.draw_image(tmp, (0, 0))
ui.canvas.draw_image(bak, (320 + 20, 0 + 20))
#img1 = tmp
#img1 = img1.to_grayscale(1)
#tmp = img.resize(224, 224)
#img1 = img.to_grayscale(1) # convert to gray
#tmp = img1.cut(20, 20, 120, 120) # resize to Minist input 28x28
#ui.canvas.draw_image(tmp, (320 + 20, 160 + 20))
a = bak.invert() # invert picture as Minist need
a = bak.strech_char(1) # preprocessing pictures, eliminate dark corner
img.draw_image(bak.resize(80, 80),
(0, 0)) # display small 28x28 picture # alpha=25
bak = bak.resize(28, 28)
a = bak.pix_to_ai() # generate data for ai
fmap = kpu.forward(Minist.task, bak) # run neural network model
plist = fmap[:] # get result (10 digit's probability)
pmax = max(plist) # get max probability
try:
max_index = plist.index(pmax) # get the digit
except Exception as e:
print(e)
Minist.score = pmax
Minist.result = max_index
#ui.canvas.draw_string(0, 300, "%d: %.3f" % (max_index, pmax), scale=2) # show result
img.draw_rectangle(80, 40, 160, 160, thickness=2, color=(0, 255, 0))
img.draw_rectangle(80 + 20,
40 + 20,
120,
120,
thickness=2,
color=(255, 0, 0))
if ui.weight > 240:
size = 28 * 2
x, y = 5, int(240) + 5 - size
#watch conv0
a = kpu.set_layers(Minist.task, 1)
fmap = kpu.forward(Minist.task, bak) # run neural network model
for i in range(0, 16):
tmp = kpu.fmap(fmap, i)
tmpx2 = tmp.resize(14 * 2, 14 * 2) # scale to display
a = ui.canvas.draw_image(tmpx2, (i % 8) * 14 * 2 + x,
y + size + 14 * 2 * int(i / 8))
x, y = x + 10 + size * 4, y + 5 - size
#watch conv1
a = kpu.set_layers(Minist.task, 2)
fmap = kpu.forward(Minist.task, bak) # run neural network model
for i in range(0, 32):
tmp = kpu.fmap(fmap, i)
tmpx2 = tmp.resize(7 * 2, 7 * 2) # scale to display
a = ui.canvas.draw_image(
tmpx2, (i % 16) * 7 * 2 + x,
y + size + 14 * 2 * 2 + 7 * 2 * int(i / 16))
x, y = x + 15, y + 10
#watch conv2
a = kpu.set_layers(Minist.task, 8)
fmap = kpu.forward(Minist.task, bak) # run neural network model
for i in range(0, 10):
tmp = kpu.fmap(fmap, i)
tmpx2 = tmp.resize(4 * 4, 4 * 4) # scale to display
a = ui.canvas.draw_image(tmpx2, i * 4 * 4 + x + i * 5,
y + size + 14 * 2 * 2 + 7 * 2 * 2)
#watch softmax
a = kpu.set_layers(Minist.task, 11)
fmap = kpu.forward(Minist.task, bak)
plist = fmap[:]
for i in range(0, 10):
cc = int(plist[i] * 256)
ui.canvas.draw_string(i * 4 * 4 + x + 3 + i * 5,
y + 20 + size + 14 * 2 * 2 + 7 * 2 * 2,
'%.0f' % (plist[i] * 100),
scale=1,
color=(255, 255, 255))
#print(i, cc)
del bak
return img