def labelling(client, name):
try:
os.mkdir("./data/" + name)
except:
pass
print("label ", name)
a, img_work = utils.take_workspace_img(client, 1.5)
mask = utils.objs_mask(img_work)
debug = concat_imgs([img_work, mask], 1)
if __name__ == '__main__':
show_img("robot view", debug, wait_ms=1)
objs = utils.extract_objs(img_work, mask)
if len(objs) != 0:
print(str(len(objs)) + " object detected")
objs[0].img = resize_img(objs[0].img, width=64, height=64)
if __name__ == '__main__':
show_img("robot view2", img_work, wait_ms=50)
print("saved", name)
cv2.imwrite("./data/" + name + "/" + str(time.time()) + ".jpg",
img_work)
else:
print(str(len(objs)) + " object detected")
return img_work
def get_all_objs():
a, img_work = utils.take_workspace_img(client)
img_work = utils.standardize_img(img_work)
if not a:
a, img_work = debug_markers(img_work)
return img_work, None
mask = utils.objs_mask(img_work)
objs = utils.extract_objs(img_work, mask)
if len(objs) == 0:
return img_work, []
imgs = []
if model is None:
return img_work, objs
for x in range(len(objs)):
imgs.append(cv2.resize(objs[x].img, (64, 64)))
imgs = np.array(imgs)
predictions = model.predict(imgs)
for x in range(len(predictions)):
obj = objs[x]
obj.type = predictions[x].argmax()
# graphical debug
cv2.drawContours(img_work, [obj.box], 0, (0, 0, 255), 2)
pos = [obj.x + 20, obj.y]
# Text position
img_work = cv2.putText(img_work, objects_names[obj.type], tuple(pos), font, 0.5, (255, 0, 0), 1, cv2.LINE_AA)
pos[0] += img_work.shape[0]
img_work = cv2.putText(img_work, objects_names[obj.type], tuple(pos), font, 0.5, (0, 255, 0), 1, cv2.LINE_AA)
return img_work, objs
def background():
if model:
img_work, ok = get_all_objs()
if ok is None:
ok = False
draw_background(img_work)
else:
ok, img = utils.take_workspace_img(client)
draw_background(img)
if not ok:
client.move_pose(*observation_pose.to_list())
pass
# Connecting to robot
client = NiryoOneClient()
client.connect(robot_ip_address)
try:
client.calibrate(CalibrateMode.AUTO)
client.change_tool(RobotTool.VACUUM_PUMP_1)
except:
print("calibration failed")
name = input("object name :")
client.move_pose(*observation_pose_wkshop.to_list())
try:
os.mkdir("./data")
except:
pass
try:
os.mkdir("./data/" + name)
except:
pass
a, img_work = utils.take_workspace_img(client, 1.5)
show_img("robot view", img_work, wait_ms=50)
cv2.createTrackbar("threshold", "robot view", 130, 256, Nothing)
while "user doesn't quit":
n = input("press enter to take picture")
if n == 'q':
break
labelling(client, name)
client.set_learning_mode(True)
client.quit()
def init_all():
global main_menu
global labbeling_menu
global training_menu
global select_menu
global select_menu_height
global settings_menu
global objects_names
global model
select_menu = []
# test observation_pose
client.move_pose(*observation_pose.to_list())
a = False
for i in range(10): # to be sure at least 1 image was correctly
a, img = utils.take_workspace_img(client)
if a:
break
if not a:
change_observation_pose()
# test data folder and model folder
try:
print(os.getcwd())
os.mkdir("./data")
except FileExistsError:
pass
try:
objects_names = os.listdir("data/")
except FileNotFoundError:
objects_names = []
if len(objects_names) == 0:
objects_names = [" "]
try:
model = tf.keras.models.load_model('model')
if len(model.predict(np.zeros((1, 64, 64, 3), np.float32))[0]) != len(objects_names):
model = None
except Exception:
pass
# labbeling menu
labbeling_menu = pygame_menu.Menu('Labelling', *menu_size,
theme=niryo_theme,
position=menu_pos,
onclose=pygame_menu.events.BACK)
objects_names_plus = copy.deepcopy(objects_names)
n = 1
while ("obj_" + str(n)) in objects_names:
n += 1
objects_names_plus.append("obj_" + str(n))
objects_names_ziped = list(zip(objects_names_plus, [labbeling_menu] * (len(objects_names) + 1)))
labbeling_menu.add.button(' ', None, font_size=font_size, selection_effect=None)
labbeling_menu.add.selector("name: ", objects_names_ziped, onchange=event_change_objname,
selector_id="objinputname", font_size=font_size)
widget_text = labbeling_menu.add.text_input("name: ", default='obj_name', textinput_id='objname', onchange=None,
font_size=font_size)
labbeling_menu.add.button("add img", labelling_event, client, widget_text, font_size=font_size)
labbeling_menu.add.button("", None, font_size=font_size, selection_effect=None)
labbeling_menu.add.button("Back", menu_back_1, labbeling_menu, font_size=font_size)
# training menu
training_menu = pygame_menu.Menu('Training', *menu_size,
theme=niryo_theme,
position=menu_pos,
onclose=pygame_menu.events.BACK)
training_menu.add.button(' ', None, font_size=font_size, selection_effect=None)
training_menu.add.button("Full training", training_event, font_size=font_size)
training_menu.add.button("Lite training", lite_training_event, font_size=font_size)
training_menu.add.button("", None, font_size=font_size, selection_effect=None)
training_menu.add.button("Back", menu_back_1, training_menu, font_size=font_size)
# settings_menu
settings_menu = pygame_menu.Menu('Settings', *menu_size,
theme=niryo_theme,
position=menu_pos,
onclose=pygame_menu.events.BACK)
settings_menu.add.button("", None, font_size=font_size, selection_effect=None)
settings_menu.add.button('Observation Pose', change_observation_pose, font_size=font_size)
settings_menu.add.button("Drop pose", change_droping_pos, font_size=font_size, )
settings_menu.add.button(labbeling_menu.get_title(), labbeling_menu_event, font_size=font_size)
settings_menu.add.button(training_menu.get_title(), training_menu_event, font_size=font_size)
settings_menu.add.button("Update", init_all, font_size=font_size)
settings_menu.add.button("", None, font_size=font_size, selection_effect=None)
settings_menu.add.button("Back", menu_back_1, settings_menu, font_size=font_size)
# select_menus
try:
os.mkdir("./logo")
except FileExistsError:
pass
print('len', int(len(objects_names) / select_menu_height + 1), len(objects_names))
select_menu_nbs = int(len(objects_names) / select_menu_height + 1)
for menu_id in range(int(len(objects_names) / select_menu_height + 1)):
select_menu.append(pygame_menu.Menu('Play', *menu_size,
theme=niryo_theme,
position=menu_pos,
columns=2,
rows=select_menu_height + 5,
onclose=pygame_menu.events.BACK
))
select_menu_x = select_menu[-1]
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
select_menu_x.add.vertical_margin(logo_big.get_size()[1])
if 0 != menu_id:
select_menu_x.add.button("<==", menu_back_1, select_menu_x, font_size=font_size)
else:
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
for name in objects_names:
add_logo(name)
all_button = []
for x in range(select_menu_height):
x += select_menu_height * menu_id
if x >= len(objects_names):
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
else:
button = select_menu_x.add.button(objects_names[x], pick_by_name, objects_names[x], font_size=font_size)
all_button.append(button)
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
select_menu_x.add.button("Back", event_quit_select_menu, font_size=font_size)
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
select_menu_x.add.vertical_margin(logo_big.get_size()[1])
if menu_id != select_menu_nbs - 1:
select_menu_x.add.button("==>", select_menu_event, menu_id + 1, font_size=font_size)
else:
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
for x in range(select_menu_height):
xx = x + menu_id * select_menu_height
if xx >= len(objects_names):
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
else:
img_logo = cv2.imread("./logo/" + objects_names[xx] + ".jpg")
scale = all_button[x].get_rect().height / img_logo.shape[1]
widget_img = select_menu_x.add.image("./logo/" + objects_names[xx] + ".jpg", scale=(scale, scale),
scale_smooth=True)
img = cv2.imread("./logo/" + objects_names[xx] + ".jpg")
img = cv2.resize(img, widget_img._image._surface.get_size())
color_hsl = [[0, 0, 0], [256, 20, 256]]
mask = cv2.bitwise_not(utils.threshold_hls(img, *color_hsl))
img_masked = cv2.bitwise_and(img, img, mask=mask)
img_masked = np.flip(img_masked[:][:]) # BGR to RGB
img_masked = np.rot90(img_masked, 1, (1, 0))
img_masked = np.flip(img_masked, 0)
surf = pygame.surfarray.make_surface(img_masked)
widget_img._image._surface.blit(surf, (0, 0))
widget_img._image._surface.set_colorkey((0, 0, 0))
widget_img._image.checkpoint()
select_menu_x.add.button(str(menu_id + 1) + "/" + str(select_menu_nbs), None, font_size=font_size)
select_menu_x.add.button("", None, font_size=font_size, selection_effect=None)
# Main menu
main_menu = pygame_menu.Menu('Welcome', *menu_size,
theme=niryo_theme,
position=menu_pos,
onclose=close_aplication)
main_menu.add.button('', None, font_size=font_size, selection_effect=None)
main_menu.add.button(select_menu[0].get_title(), select_menu_event, 0, font_size=font_size)
main_menu.add.button(settings_menu.get_title(), settings_menu_event, font_size=font_size)
main_menu.add.button('Quit', close_aplication, font_size=font_size)
main_menu.mainloop(surface, background)