def _request_data(self, request, network_compute_bridge_metadata,
store_helper):
"""Make the RPC to the network compute bridge worker.
Args:
request (bosdyn.api.AcquirePluginDataRequest): Plugin request.
network_compute_bridge_metadata (google.protobuf.Struct): Metadata containing
information needed for the request
store_helper (bosdyn.client.DataAcquisitionStoreHelper): Helper used to manage storage
of objects in data acquisition store service.
Returns:
The response from the compute request or None if error occurs
"""
server_config = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=self._worker_name)
try:
image_service = network_compute_bridge_metadata["image_service"]
image_source = network_compute_bridge_metadata["image_source"]
except ValueError:
errMsg = "Unable to get image service and source info."
data_id = self._get_data_id(request, kCapabilityImage)
store_helper.state.add_errors([make_error(data_id, errMsg)])
_LOGGER.error(errMsg)
service_source = network_compute_bridge_pb2.ImageSourceAndService(
image_service=image_service, image_source=image_source)
try:
model_name = network_compute_bridge_metadata["model_name"]
min_confidence = network_compute_bridge_metadata["min_confidence"]
except ValueError:
errMsg = "Unable to get model name or confidence value."
data_id = self._get_data_id(request, kCapabilityObjectInImage)
store_helper.state.add_errors([make_error(data_id, errMsg)])
_LOGGER.error(errMsg)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=service_source,
model_name=model_name,
min_confidence=min_confidence)
network_compute_request = network_compute_bridge_pb2.NetworkComputeRequest(
server_config=server_config, input_data=input_data)
response = self._network_compute_bridge_client.network_compute_bridge_command(
network_compute_request)
return response
def get_obj_and_img(network_compute_client, server, model, confidence,
image_sources, label):
for source in image_sources:
# Build a network compute request for this image source.
image_source_and_service = network_compute_bridge_pb2.ImageSourceAndService(
image_source=source)
# Input data:
# model name
# minimum confidence (between 0 and 1)
# if we should automatically rotate the image
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=image_source_and_service,
model_name=model,
min_confidence=confidence,
rotate_image=network_compute_bridge_pb2.NetworkComputeInputData.
ROTATE_IMAGE_ALIGN_HORIZONTAL)
# Server data: the service name
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=server)
# Pack and send the request.
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
resp = network_compute_client.network_compute_bridge_command(
process_img_req)
best_obj = None
highest_conf = 0.0
best_vision_tform_obj = None
img = get_bounding_box_image(resp)
image_full = resp.image_response
# Show the image
cv2.imshow("Fetch", img)
cv2.waitKey(15)
if len(resp.object_in_image) > 0:
for obj in resp.object_in_image:
# Get the label
obj_label = obj.name.split('_label_')[-1]
if obj_label != label:
continue
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
conf = conf_msg.value
try:
vision_tform_obj = frame_helpers.get_a_tform_b(
obj.transforms_snapshot,
frame_helpers.VISION_FRAME_NAME,
obj.image_properties.frame_name_image_coordinates)
except bosdyn.client.frame_helpers.ValidateFrameTreeError:
# No depth data available.
vision_tform_obj = None
if conf > highest_conf and vision_tform_obj is not None:
highest_conf = conf
best_obj = obj
best_vision_tform_obj = vision_tform_obj
if best_obj is not None:
return best_obj, image_full, best_vision_tform_obj
return None, None, None
def main(argv):
"""An example using the API to list and get specific objects."""
parser = argparse.ArgumentParser()
bosdyn.client.util.add_common_arguments(parser)
parser.add_argument('-i', '--image-source', help='Image source on the robot to use.')
parser.add_argument(
'-q', '--image-source-service', help=
'Image *service* for the image source to use. Defaults to the main image service if not provided.',
default='')
parser.add_argument('-s', '--service',
help='Service name of external machine learning server in the directory.',
required=False)
parser.add_argument('-m', '--model', help='Model file on the server')
parser.add_argument('-c', '--confidence', help='Minimum confidence to return an object.',
default=0.5, type=float)
parser.add_argument('-j', '--input-image',
help='Path to an image to use instead of an image source.')
parser.add_argument(
'-l', '--model-list',
help='List all available network compute servers and their provided models.',
action='store_true')
parser.add_argument('-r', '--disable-rotation',
help='Disable rotation of images (to align with horizontal)',
action='store_true')
options = parser.parse_args(argv)
if options.image_source is not None and options.input_image is not None:
print('Error: cannot provide both an input image and an image source.')
sys.exit(1)
if options.model_list and (options.image_source is not None or options.input_image is not None):
print('Error: cannot list models with input image source or input image.')
sys.exit(1)
if options.image_source is None and options.input_image is None and options.model_list == False:
default_image_source = 'frontleft_fisheye_image'
print('No image source provided so defaulting to "' + default_image_source + '".')
options.image_source = default_image_source
# Create robot object with a world object client
sdk = bosdyn.client.create_standard_sdk('IdentifyObjectClient')
robot = sdk.create_robot(options.hostname)
robot.authenticate(options.username, options.password)
#Time sync is necessary so that time-based filter requests can be converted
robot.time_sync.wait_for_sync()
#Create the network compute client
network_compute_client = robot.ensure_client(NetworkComputeBridgeClient.default_service_name)
directory_client = robot.ensure_client(
bosdyn.client.directory.DirectoryClient.default_service_name)
robot_state_client = robot.ensure_client(RobotStateClient.default_service_name)
robot_command_client = robot.ensure_client(RobotCommandClient.default_service_name)
robot.time_sync.wait_for_sync()
if options.model_list:
server_service_names = get_all_network_compute_services(directory_client)
print('Found ' + str(len(server_service_names)) +
' available service(s). Listing their models:')
print('------------------------------------')
for service in server_service_names:
print(' ' + service)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=service)
list_req = network_compute_bridge_pb2.ListAvailableModelsRequest(
server_config=server_data)
response = network_compute_client.list_available_models_command(list_req)
if response.header.error.message:
print(' Error message: {}'.format(response.header.error.message))
else:
for model in response.available_models:
print(' ' + model)
sys.exit(0)
# A service name must be provided if not doing a directory list.
if options.service is None or len(options.service) == 0:
print('Error: --service must be provided for operations other than --model-list')
sys.exit(1)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=options.service)
if options.image_source is not None:
if options.model is None:
print('Error: you must provide a model.')
sys.exit(1)
img_source_and_service = network_compute_bridge_pb2.ImageSourceAndService(
image_source=options.image_source, image_service=options.image_source_service)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=img_source_and_service, model_name=options.model,
min_confidence=options.confidence)
else:
# Read the input image.
image_in = cv2.imread(options.input_image)
if image_in is None:
print('Error: failed to read "' + options.input_image + '". Does the file exist?')
sys.exit(1)
rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB)
success, im_buffer = cv2.imencode(".jpg", rgb)
if not success:
print('Error: failed to encode input image as a jpg. Abort.')
sys.exit(1)
height = image_in.shape[0]
width = image_in.shape[1]
image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height,
data=im_buffer.tobytes(),
pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image=image_proto, model_name=options.model, min_confidence=options.confidence)
if options.disable_rotation:
input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_NO_ROTATION
else:
input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_ALIGN_HORIZONTAL
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
response = network_compute_client.network_compute_bridge_command(process_img_req)
if len(response.object_in_image) <= 0:
print('No objects found')
else:
print('Got ' + str(len(response.object_in_image)) + ' objects.')
if options.image_source is not None:
# We asked for an image to be taken, so the return proto should have an image in it.
dtype = np.uint8
img = np.frombuffer(response.image_response.shot.image.data, dtype=dtype)
if response.image_response.shot.image.format == image_pb2.Image.FORMAT_RAW:
img = img.reshape(response.image_response.shot.image.rows,
response.image_response.shot.image.cols)
else:
img = cv2.imdecode(img, -1)
else:
# To save bandwidth, the network_compute_bridge service won't re-send us back our own
# image.
img = image_in
# The image always comes back in the raw orientation. Rotate it to horizontal so that we can
# visualize it the same as it was processed.
img, rotmat = rotate_image_nocrop(img, response.image_rotation_angle)
# Convert to color for nicer drawing
if len(img.shape) < 3:
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# Draw bounding boxes in the image for all the detections.
for obj in response.object_in_image:
print(obj)
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
confidence = conf_msg.value
polygon = []
min_x = float('inf')
min_y = float('inf')
for v in obj.image_properties.coordinates.vertexes:
# If we are rotating the output image, make sure to rotate the bounding box points
# as well
x, y = rotate_point(v.x, v.y, rotmat)
polygon.append([x, y])
min_x = min(min_x, x)
min_y = min(min_y, y)
polygon = np.array(polygon, np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(img, [polygon], True, (0, 255, 0), 2)
caption = "{} {:.3f}".format(obj.name, confidence)
cv2.putText(img, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
cv2.imwrite('identify_object_output.jpg', img)