async def do_handle_create(self, x,y,w,h,theta): try: var_storage = self.device_manager.get_device_subscription("variable_storage").GetDefaultClient() b = RRN.NewStructure("com.robotraconteur.geometry.BoundingBox2D", var_storage) center = RRN.NewStructure("com.robotraconteur.geometry.NamedPose2D", var_storage) pose2d_dtype = RRN.GetNamedArrayDType("com.robotraconteur.geometry.Pose2D", var_storage) size2d_dtype = RRN.GetNamedArrayDType("com.robotraconteur.geometry.Size2D", var_storage) center.pose = np.zeros((1,),dtype=pose2d_dtype) center.pose[0]["position"]["x"] = x center.pose[0]["position"]["y"] = y center.pose[0]["orientation"] = theta b.center = center size = np.zeros((1,),dtype=size2d_dtype) size[0]["width"] = w size[0]["height"] = h b.size = size var_consts = RRN.GetConstants('tech.pyri.variable_storage', var_storage) variable_persistence = var_consts["VariablePersistence"] variable_protection_level = var_consts["VariableProtectionLevel"] await var_storage.async_add_variable2("globals", self.new_name ,"com.robotraconteur.geometry.BoundingBox2D", \ RR.VarValue(b,"com.robotraconteur.geometry.BoundingBox2D"), ["image_roi"], {}, \ variable_persistence["const"], None, variable_protection_level["read_write"], \ [], "Image region of interest", False, None) except: js.alert(f"Save image roi failed:\n\n{traceback.format_exc()}")
async def do_device_remove(self, dev_name): try: dev_manager = self.device_manager.device_manager.GetDefaultClient() await dev_manager.async_remove_device(dev_name, None) except: traceback.print_exc() js.alert(f"Device remove failed:\n\n{traceback.format_exc()}")
async def set_image(self, image_global_name): try: db = self.device_manager.get_device_subscription( "variable_storage").GetDefaultClient() img1 = await db.async_getf_variable_value("globals", image_global_name, None) img = img1.data d_encoded = str(base64.b64encode(img.data))[2:-1] if np.array_equal(img.data[0:3], np.array([0xFF, 0xD8, 0xFF], dtype=np.uint8)): img_src = "data:image/jpeg;base64," + d_encoded elif np.array_equal(img.data[0:3], np.array([0x89, 0x50, 0x4E], dtype=np.uint8)): img_src = "data:image/png;base64," + d_encoded else: js.alert("Invalid image magic") raise Exception("Invalid image magic") if self.cropper is None: img_el = js.document.getElementById("training_image") img_el.src = img_src self.cropper = js.Cropper.new(img_el, js.python_to_js({"viewMode": 2})) else: self.cropper.reset() self.cropper.replace(img_src) except: traceback.print_exc()
async def do_handle_create(self, img_np, w, h): try: var_storage = self.device_manager.get_device_subscription( "variable_storage").GetDefaultClient() compressed_img = RRN.NewStructure( "com.robotraconteur.image.CompressedImage", var_storage) compressed_img_info = RRN.NewStructure( "com.robotraconteur.image.ImageInfo", var_storage) compressed_img_info.width = w compressed_img_info.height = h compressed_img_info.encoding = 0x6000 compressed_img.image_info = compressed_img_info compressed_img.data = img_np var_consts = RRN.GetConstants('tech.pyri.variable_storage', var_storage) variable_persistence = var_consts["VariablePersistence"] variable_protection_level = var_consts["VariableProtectionLevel"] await var_storage.async_add_variable2("globals", self.new_name ,"com.robotraconteur.image.CompressedImage", \ RR.VarValue(compressed_img,"com.robotraconteur.image.CompressedImage"), ["image_template"], {}, \ variable_persistence["const"], None, variable_protection_level["read_write"], \ [], "Image matching template", False, None) except: js.alert( f"Save image template failed:\n\n{traceback.format_exc()}")
async def do_refresh_device_add(self): try: dev_manager = self.device_manager.device_manager.GetDefaultClient() detected_devices = await dev_manager.async_getf_detected_devices( None) vue_devs = [] ident_util = IdentifierUtil(client_obj=dev_manager) for d in detected_devices: d2 = { "device": ident_util.IdentifierToString(d.device), "parent_device": ident_util.IdentifierToString(d.parent_device), "manufacturer": ident_util.IdentifierToString(d.manufacturer), "model": ident_util.IdentifierToString(d.model), "serial_number": d.serial_number, "user_description": d.user_description, "service_name": d.service_name, "urls": "<br>".join(d.urls), "root_object_type": d.root_object_type, "root_object_implements": "<br>".join(d.root_object_implements) } vue_devs.append(d2) self.vue["$data"]["detected_devices"] = js.python_to_js(vue_devs) except: traceback.print_exc() js.alert(f"Refresh add device failed:\n\n{traceback.format_exc()}")
async def do_device_info(self, dev_name): try: dev_manager = self.device_manager.device_manager.GetDefaultClient() d = await dev_manager.async_getf_device_info(dev_name, None) vue_devs = [] ident_util = IdentifierUtil(client_obj=dev_manager) d2 = { "device": ident_util.IdentifierToString(d.device), "parent_device": ident_util.IdentifierToString(d.parent_device), "manufacturer": ident_util.IdentifierToString(d.manufacturer), "model": ident_util.IdentifierToString(d.model), "serial_number": d.serial_number, "user_description": d.user_description, "service_name": d.service_name, "urls": "<br>".join(d.urls), "root_object_type": d.root_object_type, "root_object_implements": "<br>".join(d.root_object_implements) } vue_devs.append(d2) self.vue["$data"]["selected_device_info"] = js.python_to_js( vue_devs) self.vue["$bvModal"].show("device-info-modal") except: traceback.print_exc() js.alert(f"Get device info failed:\n\n{traceback.format_exc()}")
async def do_sequence_capture_cancel(self): try: await self._sequence_gen.AsyncAbort(None) self._sequence_gen = None except: self._sequence_gen = None js.alert(f"Save image sequence failed:\n\n{traceback.format_exc()}") traceback.print_exc()
async def do_sequence_capture_next(self): try: # TODO: AsyncNext should only have one argument await self._sequence_gen.AsyncNext(None,None) except: self._sequence_gen = None js.alert(f"Save image sequence failed:\n\n{traceback.format_exc()}") traceback.print_exc()
async def do_add_device_selected(self, selected_device, local_dev_name): try: dev_manager = self.device_manager.device_manager.GetDefaultClient() ident_util = IdentifierUtil(client_obj=dev_manager) dev = ident_util.StringToIdentifier(selected_device) await dev_manager.async_add_device(dev, local_dev_name, [], None) except: traceback.print_exc() js.alert(f"Add device failed:\n\n{traceback.format_exc()}")
async def do_save_sequence_to_globals(self, global_name, save_state): try: viewer_service = self.device_manager.get_device_subscription("vision_camera_viewer").GetDefaultClient() viewer_obj = await viewer_service.async_get_camera_viewer(self.camera_local_name,None) self._sequence_gen = await viewer_obj.async_save_sequence_to_globals(global_name, save_state, None) except: self._sequence_gen = None js.alert(f"Save image sequence failed:\n\n{traceback.format_exc()}") traceback.print_exc()
async def do_show_new_image_template_dialog(new_name: str, variable_type: str, variable_tags: str, core: "PyriWebUIBrowser"): try: db = core.device_manager.get_device_subscription( "variable_storage").GetDefaultClient() img_var_names = await db.async_filter_variables( "globals", "", ["image"], None) if len(img_var_names) <= 0: js.alert("No source images available!") return dialog_html = importlib_resources.read_text( __package__, "new_image_template_dialog.html") dialog_obj = NewImageTemplateDialog(new_name, core, core.device_manager) el = js.document.createElement('div') el.id = "new_image_template_dialog_wrapper" js.document.getElementById("wrapper").appendChild(el) dialog = js.Vue.new( js.python_to_js({ "el": "#new_image_template_dialog_wrapper", "template": dialog_html, "data": { "image_selected": "", "image_select_options": [], }, "methods": { "handle_create": dialog_obj.handle_create, "handle_hidden": dialog_obj.handle_hidden, "image_select_changed": dialog_obj.image_select_changed, "image_reset": dialog_obj.image_reset, "image_rot_m5": dialog_obj.image_rot_m5, "image_rot_p5": dialog_obj.image_rot_p5 } })) dialog_obj.init_vue(dialog) img_vars = [] for v in img_var_names: img_vars.append({"value": v, "text": v}) dialog["$data"].image_select_options = js.python_to_js(img_vars) if len(img_vars) > 0: dialog["$data"].image_selected = img_vars[0]["value"] core.create_task(dialog_obj.set_image(img_var_names[0])) except: js.alert( f"Image template creating failed:\n\n{traceback.format_exc()}") dialog["$bvModal"].show("new_image_template")
async def do_calibration(camera_local_device_name, intrins_calib_global_name, image_global_name, calib_target, new_name, core): try: camera_calib = core.device_manager.get_device_subscription("vision_camera_calibration").GetDefaultClient() calib_res = await camera_calib.async_calibrate_camera_extrinsic(camera_local_device_name, intrins_calib_global_name, image_global_name, calib_target, new_name, None) except: js.alert(f"Calibration failed:\n\n{traceback.format_exc()}") return try: do_show_new_camera_calibration_extrinsic_dialog2(new_name, calib_res.camera_pose, calib_res.display_image, core) except: traceback.print_exc()
def handle_create(self, *args): try: cropped_canvas = self.cropper.getCroppedCanvas( js.python_to_js({"imageSmoothingEnabled": False})) img_b64 = cropped_canvas.toDataURL('image/png') img_b64 = img_b64.replace("data:image/png;base64,", "") img_bytes = base64.b64decode(img_b64) img_np = np.frombuffer(img_bytes, np.uint8) self.core.create_task( self.do_handle_create(img_np, cropped_canvas.width, cropped_canvas.height)) except: js.alert( f"Save image template failed:\n\n{traceback.format_exc()}")
def handle_create(self, *args): try: crop_data = self.cropper.getData() cropper_x = float(crop_data["x"]) cropper_y = float(crop_data["y"]) cropper_w = float(crop_data["width"]) cropper_h = float(crop_data["height"]) cropper_r = np.deg2rad(float(crop_data["rotate"])) img_data = self.cropper.getImageData() cropper_img_w = float(img_data["naturalWidth"]) cropper_img_h = float(img_data["naturalHeight"]) c = np.cos(cropper_r) s = np.sin(cropper_r) corner1 = np.array([c*cropper_img_w, s*cropper_img_w]) corner3 = np.array([-s*cropper_img_h,c*cropper_img_h]) corner2 = corner1 + corner3 img_x_min = np.min([0.,corner1[0],corner2[0],corner3[0]]) #img_x_max = np.max([0.,corner1[0],corner2[0],corner3[0]]) img_y_min = np.min([0.,corner1[1],corner2[1],corner3[1]]) #img_y_max = np.max([0.,corner1[1],corner2[1],corner3[1]]) center_x_unrotated1 = cropper_x + img_x_min + cropper_w/2. center_y_unrotated1 = cropper_y + img_y_min + cropper_h/2. R = np.array([[c,s],[-s,c]]) center_unrotated1 = np.array([center_x_unrotated1,center_y_unrotated1],dtype=np.float64) center = R@center_unrotated1 x = center[0] y = center[1] w = cropper_w h = cropper_h theta = -cropper_r #print(f"x: {x}, y: {y}, w: {w}, h: {h}, theta: {theta}") self.core.create_task(self.do_handle_create(x,y,w,h,theta)) except: js.alert(f"Save image roi failed:\n\n{traceback.format_exc()}")
async def do_calibration(robot_local_device_name, intrinsic_calib, extrinsic_calib, \ image_sequence_global_name, aruco_dict, aruco_tag_id, aruco_tag_size, marker_pose, new_name, core): try: robot_calib = core.device_manager.get_device_subscription( "vision_robot_calibration").GetDefaultClient() calib_res = await robot_calib.async_calibrate_robot_origin(robot_local_device_name, intrinsic_calib, \ extrinsic_calib, image_sequence_global_name, aruco_dict, aruco_tag_id, aruco_tag_size, \ marker_pose, new_name, None) except: js.alert(f"Calibration failed:\n\n{traceback.format_exc()}") return try: do_show_new_robot_origin_calibration_dialog2(new_name, calib_res.robot_pose, calib_res.display_images, core) except: traceback.print_exc()
async def do_show_new_camera_calibration_intrinsic_dialog( new_name: str, variable_type: str, variable_tags: str, core: "PyriWebUIBrowser"): try: core.device_manager.connect_device("vision_camera_calibration") dialog_html = importlib_resources.read_text( __package__, "new_calibrate_intrinsic_dialog.html") dialog_obj = NewCameraIntrinsicCalibrationDialog( new_name, core, core.device_manager) el = js.document.createElement('div') el.id = "new_calibrate_intrinsic_dialog_wrapper" js.document.getElementById("wrapper").appendChild(el) dialog = js.Vue.new( js.python_to_js({ "el": "#new_calibrate_intrinsic_dialog_wrapper", "template": dialog_html, "data": { "camera_selected": "", "camera_select_options": [], "image_sequence_selected": "", "image_sequence_select_options": [], "calibration_target_selected": "", "calibration_target_select_options": [] }, "methods": { "handle_create": dialog_obj.handle_create, "handle_hidden": dialog_obj.handle_hidden } })) dialog_obj.init_vue(dialog) cameras = [] camera_names = util.get_devices_with_type( core, "com.robotraconteur.imaging.Camera") cameras = util.device_names_to_dropdown_options(camera_names) dialog["$data"].camera_select_options = js.python_to_js(cameras) if len(cameras) > 0: dialog["$data"].camera_selected = cameras[0]["value"] db = core.device_manager.get_device_subscription( "variable_storage").GetDefaultClient() seq_var_names = await db.async_filter_variables( "globals", "", ["image_sequence"], None) seq_vars = [] for v in seq_var_names: seq_vars.append({"value": v, "text": v}) dialog["$data"].image_sequence_select_options = js.python_to_js( seq_vars) if len(seq_vars) > 0: dialog["$data"].image_sequence_selected = seq_vars[0]["value"] cal = [{"value": "chessboard", "text": "chessboard"}] dialog["$data"].calibration_target_select_options = js.python_to_js( cal) dialog["$data"].calibration_target_selected = "chessboard" dialog["$bvModal"].show("new_vision_camera_calibrate_intrinsic") except: js.alert(f"Calibration failed:\n\n{traceback.format_exc()}")
async def do_show_new_robot_origin_calibration_dialog( new_name: str, variable_type: str, variable_tags: str, core: "PyriWebUIBrowser"): try: core.device_manager.connect_device("vision_robot_calibration") dialog_html = importlib_resources.read_text( __package__, "new_calibrate_robot_origin_dialog.html") dialog_obj = NewRobotOriginCalibrationDialog(new_name, core, core.device_manager) el = js.document.createElement('div') el.id = "new_calibrate_robot_origin_dialog_wrapper" js.document.getElementById("wrapper").appendChild(el) dialog = js.Vue.new( js.python_to_js({ "el": "#new_calibrate_robot_origin_dialog_wrapper", "template": dialog_html, "data": { "robot_selected": "", "robot_select_options": [], "camera_intrinsic_selected": "", "camera_intrinsic_select_options": [], "camera_extrinsic_selected": "", "camera_extrinsic_select_options": [], "image_sequence_selected": "", "image_sequence_select_options": [], "aruco_dict_selected": "", "aruco_dict_select_options": [], "aruco_tag_id": "120", "aruco_tag_size": "0.06", "marker_pose_x": "0", "marker_pose_y": "0", "marker_pose_z": "0", "marker_pose_r_r": "0", "marker_pose_r_p": "0", "marker_pose_r_y": "0", }, "methods": { "handle_create": dialog_obj.handle_create, "handle_hidden": dialog_obj.handle_hidden } })) dialog_obj.init_vue(dialog) robots = [] robot_names = util.get_devices_with_type( core, "com.robotraconteur.robotics.robot.Robot") robots = util.device_names_to_dropdown_options(robot_names) dialog["$data"].robot_select_options = js.python_to_js(robots) if len(robots) > 0: dialog["$data"].robot_selected = robots[0]["value"] db = core.device_manager.get_device_subscription( "variable_storage").GetDefaultClient() intrins_var_names = await db.async_filter_variables( "globals", "", ["camera_calibration_intrinsic"], None) intrins_vars = [] for v in intrins_var_names: intrins_vars.append({"value": v, "text": v}) dialog["$data"].camera_intrinsic_select_options = js.python_to_js( intrins_vars) if len(intrins_vars) > 0: dialog["$data"].camera_intrinsic_selected = intrins_vars[0][ "value"] extrins_var_names = await db.async_filter_variables( "globals", "", ["camera_calibration_extrinsic"], None) extrins_vars = [] for v in extrins_var_names: extrins_vars.append({"value": v, "text": v}) dialog["$data"].camera_extrinsic_select_options = js.python_to_js( extrins_vars) if len(extrins_vars) > 0: dialog["$data"].camera_extrinsic_selected = extrins_vars[0][ "value"] seq_var_names = await db.async_filter_variables( "globals", "", ["image_sequence"], None) seq_vars = [] for v in seq_var_names: seq_vars.append({"value": v, "text": v}) dialog["$data"].image_sequence_select_options = js.python_to_js( seq_vars) if len(seq_vars) > 0: dialog["$data"].image_sequence_selected = seq_vars[0]["value"] aruco_dicts = ['DICT_4X4_100', 'DICT_4X4_1000', 'DICT_4X4_250', \ 'DICT_4X4_50', 'DICT_5X5_100', 'DICT_5X5_1000', 'DICT_5X5_250', \ 'DICT_5X5_50', 'DICT_6X6_100', 'DICT_6X6_1000', 'DICT_6X6_250', \ 'DICT_6X6_50', 'DICT_7X7_100', 'DICT_7X7_1000', 'DICT_7X7_250', \ 'DICT_7X7_50', 'DICT_APRILTAG_16H5', 'DICT_APRILTAG_16h5', 'DICT_APRILTAG_25H9', \ 'DICT_APRILTAG_25h9', 'DICT_APRILTAG_36H10', 'DICT_APRILTAG_36H11', 'DICT_APRILTAG_36h10', \ 'DICT_APRILTAG_36h11', 'DICT_ARUCO_ORIGINAL'] aruco_opts = [{"value": v, "text": v} for v in aruco_dicts] dialog["$data"].aruco_dict_select_options = js.python_to_js(aruco_opts) dialog["$data"].aruco_dict_selected = 'DICT_6X6_250' dialog["$bvModal"].show("new_vision_camera_calibrate_robot_origin") except: js.alert(f"Calibration failed:\n\n{traceback.format_exc()}")