async def do_handle_create(self, x,y,w,h,theta):
try:
var_storage = self.device_manager.get_device_subscription("variable_storage").GetDefaultClient()
b = RRN.NewStructure("com.robotraconteur.geometry.BoundingBox2D", var_storage)
center = RRN.NewStructure("com.robotraconteur.geometry.NamedPose2D", var_storage)
pose2d_dtype = RRN.GetNamedArrayDType("com.robotraconteur.geometry.Pose2D", var_storage)
size2d_dtype = RRN.GetNamedArrayDType("com.robotraconteur.geometry.Size2D", var_storage)
center.pose = np.zeros((1,),dtype=pose2d_dtype)
center.pose[0]["position"]["x"] = x
center.pose[0]["position"]["y"] = y
center.pose[0]["orientation"] = theta
b.center = center
size = np.zeros((1,),dtype=size2d_dtype)
size[0]["width"] = w
size[0]["height"] = h
b.size = size
var_consts = RRN.GetConstants('tech.pyri.variable_storage', var_storage)
variable_persistence = var_consts["VariablePersistence"]
variable_protection_level = var_consts["VariableProtectionLevel"]
await var_storage.async_add_variable2("globals", self.new_name ,"com.robotraconteur.geometry.BoundingBox2D", \
RR.VarValue(b,"com.robotraconteur.geometry.BoundingBox2D"), ["image_roi"], {}, \
variable_persistence["const"], None, variable_protection_level["read_write"], \
[], "Image region of interest", False, None)
except:
js.alert(f"Save image roi failed:\n\n{traceback.format_exc()}")
async def do_device_remove(self, dev_name):
try:
dev_manager = self.device_manager.device_manager.GetDefaultClient()
await dev_manager.async_remove_device(dev_name, None)
except:
traceback.print_exc()
js.alert(f"Device remove failed:\n\n{traceback.format_exc()}")
async def set_image(self, image_global_name):
try:
db = self.device_manager.get_device_subscription(
"variable_storage").GetDefaultClient()
img1 = await db.async_getf_variable_value("globals",
image_global_name, None)
img = img1.data
d_encoded = str(base64.b64encode(img.data))[2:-1]
if np.array_equal(img.data[0:3],
np.array([0xFF, 0xD8, 0xFF], dtype=np.uint8)):
img_src = "data:image/jpeg;base64," + d_encoded
elif np.array_equal(img.data[0:3],
np.array([0x89, 0x50, 0x4E], dtype=np.uint8)):
img_src = "data:image/png;base64," + d_encoded
else:
js.alert("Invalid image magic")
raise Exception("Invalid image magic")
if self.cropper is None:
img_el = js.document.getElementById("training_image")
img_el.src = img_src
self.cropper = js.Cropper.new(img_el,
js.python_to_js({"viewMode": 2}))
else:
self.cropper.reset()
self.cropper.replace(img_src)
except:
traceback.print_exc()
async def do_handle_create(self, img_np, w, h):
try:
var_storage = self.device_manager.get_device_subscription(
"variable_storage").GetDefaultClient()
compressed_img = RRN.NewStructure(
"com.robotraconteur.image.CompressedImage", var_storage)
compressed_img_info = RRN.NewStructure(
"com.robotraconteur.image.ImageInfo", var_storage)
compressed_img_info.width = w
compressed_img_info.height = h
compressed_img_info.encoding = 0x6000
compressed_img.image_info = compressed_img_info
compressed_img.data = img_np
var_consts = RRN.GetConstants('tech.pyri.variable_storage',
var_storage)
variable_persistence = var_consts["VariablePersistence"]
variable_protection_level = var_consts["VariableProtectionLevel"]
await var_storage.async_add_variable2("globals", self.new_name ,"com.robotraconteur.image.CompressedImage", \
RR.VarValue(compressed_img,"com.robotraconteur.image.CompressedImage"), ["image_template"], {}, \
variable_persistence["const"], None, variable_protection_level["read_write"], \
[], "Image matching template", False, None)
except:
js.alert(
f"Save image template failed:\n\n{traceback.format_exc()}")
async def do_refresh_device_add(self):
try:
dev_manager = self.device_manager.device_manager.GetDefaultClient()
detected_devices = await dev_manager.async_getf_detected_devices(
None)
vue_devs = []
ident_util = IdentifierUtil(client_obj=dev_manager)
for d in detected_devices:
d2 = {
"device": ident_util.IdentifierToString(d.device),
"parent_device":
ident_util.IdentifierToString(d.parent_device),
"manufacturer":
ident_util.IdentifierToString(d.manufacturer),
"model": ident_util.IdentifierToString(d.model),
"serial_number": d.serial_number,
"user_description": d.user_description,
"service_name": d.service_name,
"urls": "<br>".join(d.urls),
"root_object_type": d.root_object_type,
"root_object_implements":
"<br>".join(d.root_object_implements)
}
vue_devs.append(d2)
self.vue["$data"]["detected_devices"] = js.python_to_js(vue_devs)
except:
traceback.print_exc()
js.alert(f"Refresh add device failed:\n\n{traceback.format_exc()}")
async def do_device_info(self, dev_name):
try:
dev_manager = self.device_manager.device_manager.GetDefaultClient()
d = await dev_manager.async_getf_device_info(dev_name, None)
vue_devs = []
ident_util = IdentifierUtil(client_obj=dev_manager)
d2 = {
"device": ident_util.IdentifierToString(d.device),
"parent_device":
ident_util.IdentifierToString(d.parent_device),
"manufacturer": ident_util.IdentifierToString(d.manufacturer),
"model": ident_util.IdentifierToString(d.model),
"serial_number": d.serial_number,
"user_description": d.user_description,
"service_name": d.service_name,
"urls": "<br>".join(d.urls),
"root_object_type": d.root_object_type,
"root_object_implements": "<br>".join(d.root_object_implements)
}
vue_devs.append(d2)
self.vue["$data"]["selected_device_info"] = js.python_to_js(
vue_devs)
self.vue["$bvModal"].show("device-info-modal")
except:
traceback.print_exc()
js.alert(f"Get device info failed:\n\n{traceback.format_exc()}")
async def do_sequence_capture_cancel(self):
try:
await self._sequence_gen.AsyncAbort(None)
self._sequence_gen = None
except:
self._sequence_gen = None
js.alert(f"Save image sequence failed:\n\n{traceback.format_exc()}")
traceback.print_exc()
async def do_sequence_capture_next(self):
try:
# TODO: AsyncNext should only have one argument
await self._sequence_gen.AsyncNext(None,None)
except:
self._sequence_gen = None
js.alert(f"Save image sequence failed:\n\n{traceback.format_exc()}")
traceback.print_exc()
async def do_add_device_selected(self, selected_device, local_dev_name):
try:
dev_manager = self.device_manager.device_manager.GetDefaultClient()
ident_util = IdentifierUtil(client_obj=dev_manager)
dev = ident_util.StringToIdentifier(selected_device)
await dev_manager.async_add_device(dev, local_dev_name, [], None)
except:
traceback.print_exc()
js.alert(f"Add device failed:\n\n{traceback.format_exc()}")
async def do_save_sequence_to_globals(self, global_name, save_state):
try:
viewer_service = self.device_manager.get_device_subscription("vision_camera_viewer").GetDefaultClient()
viewer_obj = await viewer_service.async_get_camera_viewer(self.camera_local_name,None)
self._sequence_gen = await viewer_obj.async_save_sequence_to_globals(global_name, save_state, None)
except:
self._sequence_gen = None
js.alert(f"Save image sequence failed:\n\n{traceback.format_exc()}")
traceback.print_exc()
async def do_show_new_image_template_dialog(new_name: str, variable_type: str,
variable_tags: str,
core: "PyriWebUIBrowser"):
try:
db = core.device_manager.get_device_subscription(
"variable_storage").GetDefaultClient()
img_var_names = await db.async_filter_variables(
"globals", "", ["image"], None)
if len(img_var_names) <= 0:
js.alert("No source images available!")
return
dialog_html = importlib_resources.read_text(
__package__, "new_image_template_dialog.html")
dialog_obj = NewImageTemplateDialog(new_name, core,
core.device_manager)
el = js.document.createElement('div')
el.id = "new_image_template_dialog_wrapper"
js.document.getElementById("wrapper").appendChild(el)
dialog = js.Vue.new(
js.python_to_js({
"el": "#new_image_template_dialog_wrapper",
"template": dialog_html,
"data": {
"image_selected": "",
"image_select_options": [],
},
"methods": {
"handle_create": dialog_obj.handle_create,
"handle_hidden": dialog_obj.handle_hidden,
"image_select_changed": dialog_obj.image_select_changed,
"image_reset": dialog_obj.image_reset,
"image_rot_m5": dialog_obj.image_rot_m5,
"image_rot_p5": dialog_obj.image_rot_p5
}
}))
dialog_obj.init_vue(dialog)
img_vars = []
for v in img_var_names:
img_vars.append({"value": v, "text": v})
dialog["$data"].image_select_options = js.python_to_js(img_vars)
if len(img_vars) > 0:
dialog["$data"].image_selected = img_vars[0]["value"]
core.create_task(dialog_obj.set_image(img_var_names[0]))
except:
js.alert(
f"Image template creating failed:\n\n{traceback.format_exc()}")
dialog["$bvModal"].show("new_image_template")
async def do_calibration(camera_local_device_name, intrins_calib_global_name, image_global_name, calib_target, new_name, core):
try:
camera_calib = core.device_manager.get_device_subscription("vision_camera_calibration").GetDefaultClient()
calib_res = await camera_calib.async_calibrate_camera_extrinsic(camera_local_device_name, intrins_calib_global_name, image_global_name, calib_target, new_name, None)
except:
js.alert(f"Calibration failed:\n\n{traceback.format_exc()}")
return
try:
do_show_new_camera_calibration_extrinsic_dialog2(new_name, calib_res.camera_pose, calib_res.display_image, core)
except:
traceback.print_exc()
def handle_create(self, *args):
try:
cropped_canvas = self.cropper.getCroppedCanvas(
js.python_to_js({"imageSmoothingEnabled": False}))
img_b64 = cropped_canvas.toDataURL('image/png')
img_b64 = img_b64.replace("data:image/png;base64,", "")
img_bytes = base64.b64decode(img_b64)
img_np = np.frombuffer(img_bytes, np.uint8)
self.core.create_task(
self.do_handle_create(img_np, cropped_canvas.width,
cropped_canvas.height))
except:
js.alert(
f"Save image template failed:\n\n{traceback.format_exc()}")
def handle_create(self, *args):
try:
crop_data = self.cropper.getData()
cropper_x = float(crop_data["x"])
cropper_y = float(crop_data["y"])
cropper_w = float(crop_data["width"])
cropper_h = float(crop_data["height"])
cropper_r = np.deg2rad(float(crop_data["rotate"]))
img_data = self.cropper.getImageData()
cropper_img_w = float(img_data["naturalWidth"])
cropper_img_h = float(img_data["naturalHeight"])
c = np.cos(cropper_r)
s = np.sin(cropper_r)
corner1 = np.array([c*cropper_img_w, s*cropper_img_w])
corner3 = np.array([-s*cropper_img_h,c*cropper_img_h])
corner2 = corner1 + corner3
img_x_min = np.min([0.,corner1[0],corner2[0],corner3[0]])
#img_x_max = np.max([0.,corner1[0],corner2[0],corner3[0]])
img_y_min = np.min([0.,corner1[1],corner2[1],corner3[1]])
#img_y_max = np.max([0.,corner1[1],corner2[1],corner3[1]])
center_x_unrotated1 = cropper_x + img_x_min + cropper_w/2.
center_y_unrotated1 = cropper_y + img_y_min + cropper_h/2.
R = np.array([[c,s],[-s,c]])
center_unrotated1 = np.array([center_x_unrotated1,center_y_unrotated1],dtype=np.float64)
center = R@center_unrotated1
x = center[0]
y = center[1]
w = cropper_w
h = cropper_h
theta = -cropper_r
#print(f"x: {x}, y: {y}, w: {w}, h: {h}, theta: {theta}")
self.core.create_task(self.do_handle_create(x,y,w,h,theta))
except:
js.alert(f"Save image roi failed:\n\n{traceback.format_exc()}")
async def do_calibration(robot_local_device_name, intrinsic_calib, extrinsic_calib, \
image_sequence_global_name, aruco_dict, aruco_tag_id, aruco_tag_size, marker_pose, new_name, core):
try:
robot_calib = core.device_manager.get_device_subscription(
"vision_robot_calibration").GetDefaultClient()
calib_res = await robot_calib.async_calibrate_robot_origin(robot_local_device_name, intrinsic_calib, \
extrinsic_calib, image_sequence_global_name, aruco_dict, aruco_tag_id, aruco_tag_size, \
marker_pose, new_name, None)
except:
js.alert(f"Calibration failed:\n\n{traceback.format_exc()}")
return
try:
do_show_new_robot_origin_calibration_dialog2(new_name,
calib_res.robot_pose,
calib_res.display_images,
core)
except:
traceback.print_exc()
async def do_show_new_camera_calibration_intrinsic_dialog(
new_name: str, variable_type: str, variable_tags: str,
core: "PyriWebUIBrowser"):
try:
core.device_manager.connect_device("vision_camera_calibration")
dialog_html = importlib_resources.read_text(
__package__, "new_calibrate_intrinsic_dialog.html")
dialog_obj = NewCameraIntrinsicCalibrationDialog(
new_name, core, core.device_manager)
el = js.document.createElement('div')
el.id = "new_calibrate_intrinsic_dialog_wrapper"
js.document.getElementById("wrapper").appendChild(el)
dialog = js.Vue.new(
js.python_to_js({
"el": "#new_calibrate_intrinsic_dialog_wrapper",
"template": dialog_html,
"data": {
"camera_selected": "",
"camera_select_options": [],
"image_sequence_selected": "",
"image_sequence_select_options": [],
"calibration_target_selected": "",
"calibration_target_select_options": []
},
"methods": {
"handle_create": dialog_obj.handle_create,
"handle_hidden": dialog_obj.handle_hidden
}
}))
dialog_obj.init_vue(dialog)
cameras = []
camera_names = util.get_devices_with_type(
core, "com.robotraconteur.imaging.Camera")
cameras = util.device_names_to_dropdown_options(camera_names)
dialog["$data"].camera_select_options = js.python_to_js(cameras)
if len(cameras) > 0:
dialog["$data"].camera_selected = cameras[0]["value"]
db = core.device_manager.get_device_subscription(
"variable_storage").GetDefaultClient()
seq_var_names = await db.async_filter_variables(
"globals", "", ["image_sequence"], None)
seq_vars = []
for v in seq_var_names:
seq_vars.append({"value": v, "text": v})
dialog["$data"].image_sequence_select_options = js.python_to_js(
seq_vars)
if len(seq_vars) > 0:
dialog["$data"].image_sequence_selected = seq_vars[0]["value"]
cal = [{"value": "chessboard", "text": "chessboard"}]
dialog["$data"].calibration_target_select_options = js.python_to_js(
cal)
dialog["$data"].calibration_target_selected = "chessboard"
dialog["$bvModal"].show("new_vision_camera_calibrate_intrinsic")
except:
js.alert(f"Calibration failed:\n\n{traceback.format_exc()}")
async def do_show_new_robot_origin_calibration_dialog(
new_name: str, variable_type: str, variable_tags: str,
core: "PyriWebUIBrowser"):
try:
core.device_manager.connect_device("vision_robot_calibration")
dialog_html = importlib_resources.read_text(
__package__, "new_calibrate_robot_origin_dialog.html")
dialog_obj = NewRobotOriginCalibrationDialog(new_name, core,
core.device_manager)
el = js.document.createElement('div')
el.id = "new_calibrate_robot_origin_dialog_wrapper"
js.document.getElementById("wrapper").appendChild(el)
dialog = js.Vue.new(
js.python_to_js({
"el": "#new_calibrate_robot_origin_dialog_wrapper",
"template": dialog_html,
"data": {
"robot_selected": "",
"robot_select_options": [],
"camera_intrinsic_selected": "",
"camera_intrinsic_select_options": [],
"camera_extrinsic_selected": "",
"camera_extrinsic_select_options": [],
"image_sequence_selected": "",
"image_sequence_select_options": [],
"aruco_dict_selected": "",
"aruco_dict_select_options": [],
"aruco_tag_id": "120",
"aruco_tag_size": "0.06",
"marker_pose_x": "0",
"marker_pose_y": "0",
"marker_pose_z": "0",
"marker_pose_r_r": "0",
"marker_pose_r_p": "0",
"marker_pose_r_y": "0",
},
"methods": {
"handle_create": dialog_obj.handle_create,
"handle_hidden": dialog_obj.handle_hidden
}
}))
dialog_obj.init_vue(dialog)
robots = []
robot_names = util.get_devices_with_type(
core, "com.robotraconteur.robotics.robot.Robot")
robots = util.device_names_to_dropdown_options(robot_names)
dialog["$data"].robot_select_options = js.python_to_js(robots)
if len(robots) > 0:
dialog["$data"].robot_selected = robots[0]["value"]
db = core.device_manager.get_device_subscription(
"variable_storage").GetDefaultClient()
intrins_var_names = await db.async_filter_variables(
"globals", "", ["camera_calibration_intrinsic"], None)
intrins_vars = []
for v in intrins_var_names:
intrins_vars.append({"value": v, "text": v})
dialog["$data"].camera_intrinsic_select_options = js.python_to_js(
intrins_vars)
if len(intrins_vars) > 0:
dialog["$data"].camera_intrinsic_selected = intrins_vars[0][
"value"]
extrins_var_names = await db.async_filter_variables(
"globals", "", ["camera_calibration_extrinsic"], None)
extrins_vars = []
for v in extrins_var_names:
extrins_vars.append({"value": v, "text": v})
dialog["$data"].camera_extrinsic_select_options = js.python_to_js(
extrins_vars)
if len(extrins_vars) > 0:
dialog["$data"].camera_extrinsic_selected = extrins_vars[0][
"value"]
seq_var_names = await db.async_filter_variables(
"globals", "", ["image_sequence"], None)
seq_vars = []
for v in seq_var_names:
seq_vars.append({"value": v, "text": v})
dialog["$data"].image_sequence_select_options = js.python_to_js(
seq_vars)
if len(seq_vars) > 0:
dialog["$data"].image_sequence_selected = seq_vars[0]["value"]
aruco_dicts = ['DICT_4X4_100', 'DICT_4X4_1000', 'DICT_4X4_250', \
'DICT_4X4_50', 'DICT_5X5_100', 'DICT_5X5_1000', 'DICT_5X5_250', \
'DICT_5X5_50', 'DICT_6X6_100', 'DICT_6X6_1000', 'DICT_6X6_250', \
'DICT_6X6_50', 'DICT_7X7_100', 'DICT_7X7_1000', 'DICT_7X7_250', \
'DICT_7X7_50', 'DICT_APRILTAG_16H5', 'DICT_APRILTAG_16h5', 'DICT_APRILTAG_25H9', \
'DICT_APRILTAG_25h9', 'DICT_APRILTAG_36H10', 'DICT_APRILTAG_36H11', 'DICT_APRILTAG_36h10', \
'DICT_APRILTAG_36h11', 'DICT_ARUCO_ORIGINAL']
aruco_opts = [{"value": v, "text": v} for v in aruco_dicts]
dialog["$data"].aruco_dict_select_options = js.python_to_js(aruco_opts)
dialog["$data"].aruco_dict_selected = 'DICT_6X6_250'
dialog["$bvModal"].show("new_vision_camera_calibrate_robot_origin")
except:
js.alert(f"Calibration failed:\n\n{traceback.format_exc()}")