def main(argv):
"""An example using the API to list and get specific objects."""
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--server', help='IP address of external machine learning server.',
required=True)
parser.add_argument('-m', '--model', help='Model file on the server', default='coco_inference')
parser.add_argument('-c', '--confidence', help='Minimum confidence to return an object.',
default=0.7, type=float)
parser.add_argument('-j', '--input-image-dir', help='Path to a directory of images.')
parser.add_argument('-n', '--num-runs', help='Number of runs of the image directory to perform',
default=1, type=int)
parser.add_argument('-l', '--model-list', help='List of models to be used.',
action='store_true')
parser.add_argument('-v', '--verbose', help='Print verbose output', action='store_true')
options = parser.parse_args(argv)
if options.input_image_dir is None and not options.model_list:
print('Error: must provide an input image.')
sys.exit(1)
if options.model_list and options.input_image_dir is not None:
print('Error: cannot list models with input image.')
sys.exit(1)
if options.model_list:
channel = grpc.insecure_channel(options.server)
stub = network_compute_bridge_service_pb2_grpc.NetworkComputeBridgeWorkerStub(channel)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name='test')
list_req = network_compute_bridge_pb2.ListAvailableModelsRequest(server_config=server_data)
response = stub.ListAvailableModels(list_req)
print('Available models on server at ' + options.server + ' are:')
for model in response.available_models:
print(' ' + model)
sys.exit(0)
image_paths = []
for entry in os.scandir(options.input_image_dir):
if (entry.path.endswith(".jpg") or entry.path.endswith(".png")) and entry.is_file():
image_paths.append(entry.path)
image_paths = image_paths * options.num_runs
start_time = time.perf_counter()
with mp.Pool() as p:
latencies = p.starmap(
_send_request,
[(options.server, path, options.model, options.confidence, options.verbose)
for path in image_paths])
latency_sum = sum(latencies)
count = len(latencies)
end_time = time.perf_counter()
total_time = end_time - start_time
print(f'Total time: {total_time} seconds.')
avg_latency = latency_sum / count
print(f'avg latency: {avg_latency * 1000} ms, fps: {count / total_time} fps')
def _request_data(self, request, network_compute_bridge_metadata,
store_helper):
"""Make the RPC to the network compute bridge worker.
Args:
request (bosdyn.api.AcquirePluginDataRequest): Plugin request.
network_compute_bridge_metadata (google.protobuf.Struct): Metadata containing
information needed for the request
store_helper (bosdyn.client.DataAcquisitionStoreHelper): Helper used to manage storage
of objects in data acquisition store service.
Returns:
The response from the compute request or None if error occurs
"""
server_config = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=self._worker_name)
try:
image_service = network_compute_bridge_metadata["image_service"]
image_source = network_compute_bridge_metadata["image_source"]
except ValueError:
errMsg = "Unable to get image service and source info."
data_id = self._get_data_id(request, kCapabilityImage)
store_helper.state.add_errors([make_error(data_id, errMsg)])
_LOGGER.error(errMsg)
service_source = network_compute_bridge_pb2.ImageSourceAndService(
image_service=image_service, image_source=image_source)
try:
model_name = network_compute_bridge_metadata["model_name"]
min_confidence = network_compute_bridge_metadata["min_confidence"]
except ValueError:
errMsg = "Unable to get model name or confidence value."
data_id = self._get_data_id(request, kCapabilityObjectInImage)
store_helper.state.add_errors([make_error(data_id, errMsg)])
_LOGGER.error(errMsg)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=service_source,
model_name=model_name,
min_confidence=min_confidence)
network_compute_request = network_compute_bridge_pb2.NetworkComputeRequest(
server_config=server_config, input_data=input_data)
response = self._network_compute_bridge_client.network_compute_bridge_command(
network_compute_request)
return response
def get_capabilities(self):
"""Get list of available data capture options for the network compute bridge worker.
Returns:
Array with list of Capabilities corresponding to the available data capture options.
"""
# Try to get a list of models available from the worker to see if this service is alive.
while True:
try:
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=self._worker_name)
list_req = network_compute_bridge_pb2.ListAvailableModelsRequest(
server_config=server_data)
response = self._network_compute_bridge_client.list_available_models_command(
list_req)
break
except (ExternalServiceNotFoundError, ExternalServerError):
_LOGGER.exception(
'Network compute bridge worker is still unavailable:\n')
time.sleep(2)
if response.header.error.message:
_LOGGER.error(
"List available models from %s returned with error: %s",
self._worker_name, response.header.error.message)
else:
_LOGGER.info('Available models from %s:', self._worker_name)
for model in response.available_models:
_LOGGER.info(' %s', model)
# Compose the list of data capture options.
capabilities = []
_LOGGER.info('Available data capture options:')
for name in kCapabilityNames:
_LOGGER.info(' %s', name)
capabilities.append(
Capability(name=name,
description="Processed {} from {}.".format(
name, self._worker_name),
channel_name="{}--{}".format(
self._worker_name, name)))
return capabilities
def get_obj_and_img(network_compute_client, server, model, confidence,
image_sources, label):
for source in image_sources:
# Build a network compute request for this image source.
image_source_and_service = network_compute_bridge_pb2.ImageSourceAndService(
image_source=source)
# Input data:
# model name
# minimum confidence (between 0 and 1)
# if we should automatically rotate the image
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=image_source_and_service,
model_name=model,
min_confidence=confidence,
rotate_image=network_compute_bridge_pb2.NetworkComputeInputData.
ROTATE_IMAGE_ALIGN_HORIZONTAL)
# Server data: the service name
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=server)
# Pack and send the request.
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
resp = network_compute_client.network_compute_bridge_command(
process_img_req)
best_obj = None
highest_conf = 0.0
best_vision_tform_obj = None
img = get_bounding_box_image(resp)
image_full = resp.image_response
# Show the image
cv2.imshow("Fetch", img)
cv2.waitKey(15)
if len(resp.object_in_image) > 0:
for obj in resp.object_in_image:
# Get the label
obj_label = obj.name.split('_label_')[-1]
if obj_label != label:
continue
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
conf = conf_msg.value
try:
vision_tform_obj = frame_helpers.get_a_tform_b(
obj.transforms_snapshot,
frame_helpers.VISION_FRAME_NAME,
obj.image_properties.frame_name_image_coordinates)
except bosdyn.client.frame_helpers.ValidateFrameTreeError:
# No depth data available.
vision_tform_obj = None
if conf > highest_conf and vision_tform_obj is not None:
highest_conf = conf
best_obj = obj
best_vision_tform_obj = vision_tform_obj
if best_obj is not None:
return best_obj, image_full, best_vision_tform_obj
return None, None, None
def main(argv):
"""An example using the API to list and get specific objects."""
parser = argparse.ArgumentParser()
bosdyn.client.util.add_common_arguments(parser)
parser.add_argument('-i', '--image-source', help='Image source on the robot to use.')
parser.add_argument(
'-q', '--image-source-service', help=
'Image *service* for the image source to use. Defaults to the main image service if not provided.',
default='')
parser.add_argument('-s', '--service',
help='Service name of external machine learning server in the directory.',
required=False)
parser.add_argument('-m', '--model', help='Model file on the server')
parser.add_argument('-c', '--confidence', help='Minimum confidence to return an object.',
default=0.5, type=float)
parser.add_argument('-j', '--input-image',
help='Path to an image to use instead of an image source.')
parser.add_argument(
'-l', '--model-list',
help='List all available network compute servers and their provided models.',
action='store_true')
parser.add_argument('-r', '--disable-rotation',
help='Disable rotation of images (to align with horizontal)',
action='store_true')
options = parser.parse_args(argv)
if options.image_source is not None and options.input_image is not None:
print('Error: cannot provide both an input image and an image source.')
sys.exit(1)
if options.model_list and (options.image_source is not None or options.input_image is not None):
print('Error: cannot list models with input image source or input image.')
sys.exit(1)
if options.image_source is None and options.input_image is None and options.model_list == False:
default_image_source = 'frontleft_fisheye_image'
print('No image source provided so defaulting to "' + default_image_source + '".')
options.image_source = default_image_source
# Create robot object with a world object client
sdk = bosdyn.client.create_standard_sdk('IdentifyObjectClient')
robot = sdk.create_robot(options.hostname)
robot.authenticate(options.username, options.password)
#Time sync is necessary so that time-based filter requests can be converted
robot.time_sync.wait_for_sync()
#Create the network compute client
network_compute_client = robot.ensure_client(NetworkComputeBridgeClient.default_service_name)
directory_client = robot.ensure_client(
bosdyn.client.directory.DirectoryClient.default_service_name)
robot_state_client = robot.ensure_client(RobotStateClient.default_service_name)
robot_command_client = robot.ensure_client(RobotCommandClient.default_service_name)
robot.time_sync.wait_for_sync()
if options.model_list:
server_service_names = get_all_network_compute_services(directory_client)
print('Found ' + str(len(server_service_names)) +
' available service(s). Listing their models:')
print('------------------------------------')
for service in server_service_names:
print(' ' + service)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=service)
list_req = network_compute_bridge_pb2.ListAvailableModelsRequest(
server_config=server_data)
response = network_compute_client.list_available_models_command(list_req)
if response.header.error.message:
print(' Error message: {}'.format(response.header.error.message))
else:
for model in response.available_models:
print(' ' + model)
sys.exit(0)
# A service name must be provided if not doing a directory list.
if options.service is None or len(options.service) == 0:
print('Error: --service must be provided for operations other than --model-list')
sys.exit(1)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=options.service)
if options.image_source is not None:
if options.model is None:
print('Error: you must provide a model.')
sys.exit(1)
img_source_and_service = network_compute_bridge_pb2.ImageSourceAndService(
image_source=options.image_source, image_service=options.image_source_service)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=img_source_and_service, model_name=options.model,
min_confidence=options.confidence)
else:
# Read the input image.
image_in = cv2.imread(options.input_image)
if image_in is None:
print('Error: failed to read "' + options.input_image + '". Does the file exist?')
sys.exit(1)
rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB)
success, im_buffer = cv2.imencode(".jpg", rgb)
if not success:
print('Error: failed to encode input image as a jpg. Abort.')
sys.exit(1)
height = image_in.shape[0]
width = image_in.shape[1]
image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height,
data=im_buffer.tobytes(),
pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image=image_proto, model_name=options.model, min_confidence=options.confidence)
if options.disable_rotation:
input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_NO_ROTATION
else:
input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_ALIGN_HORIZONTAL
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
response = network_compute_client.network_compute_bridge_command(process_img_req)
if len(response.object_in_image) <= 0:
print('No objects found')
else:
print('Got ' + str(len(response.object_in_image)) + ' objects.')
if options.image_source is not None:
# We asked for an image to be taken, so the return proto should have an image in it.
dtype = np.uint8
img = np.frombuffer(response.image_response.shot.image.data, dtype=dtype)
if response.image_response.shot.image.format == image_pb2.Image.FORMAT_RAW:
img = img.reshape(response.image_response.shot.image.rows,
response.image_response.shot.image.cols)
else:
img = cv2.imdecode(img, -1)
else:
# To save bandwidth, the network_compute_bridge service won't re-send us back our own
# image.
img = image_in
# The image always comes back in the raw orientation. Rotate it to horizontal so that we can
# visualize it the same as it was processed.
img, rotmat = rotate_image_nocrop(img, response.image_rotation_angle)
# Convert to color for nicer drawing
if len(img.shape) < 3:
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# Draw bounding boxes in the image for all the detections.
for obj in response.object_in_image:
print(obj)
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
confidence = conf_msg.value
polygon = []
min_x = float('inf')
min_y = float('inf')
for v in obj.image_properties.coordinates.vertexes:
# If we are rotating the output image, make sure to rotate the bounding box points
# as well
x, y = rotate_point(v.x, v.y, rotmat)
polygon.append([x, y])
min_x = min(min_x, x)
min_y = min(min_y, y)
polygon = np.array(polygon, np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(img, [polygon], True, (0, 255, 0), 2)
caption = "{} {:.3f}".format(obj.name, confidence)
cv2.putText(img, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
cv2.imwrite('identify_object_output.jpg', img)
def _send_request(server, image_path, model, confidence, verbose=False):
start = time.time()
channel = grpc.insecure_channel(server)
stub = network_compute_bridge_service_pb2_grpc.NetworkComputeBridgeWorkerStub(channel)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(service_name='test')
# Read the input image.
image_in = cv2.imread(image_path)
if image_in is None:
print('Error: failed to read "' + image_path + '". Does the file exist?')
sys.exit(1)
rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB)
success, im_buffer = cv2.imencode(".jpg", rgb)
if not success:
print('Error: failed to encode input image as a jpg. Abort.')
sys.exit(1)
height = image_in.shape[0]
width = image_in.shape[1]
image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height,
data=im_buffer.tobytes(),
pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(image=image_proto,
model_name=model,
min_confidence=confidence)
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
response = stub.NetworkCompute(process_img_req)
end = time.time()
latency = end - start
print(f'latency: {latency * 1000} ms')
if verbose:
if len(response.object_in_image) <= 0:
print('No objects found')
else:
print('Got ' + str(len(response.object_in_image)) + ' objects.')
# Draw bounding boxes in the image for all the detections.
for obj in response.object_in_image:
print(obj)
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
confidence = conf_msg.value
polygon = []
min_x = float('inf')
min_y = float('inf')
for v in obj.image_properties.coordinates.vertexes:
polygon.append([v.x, v.y])
min_x = min(min_x, v.x)
min_y = min(min_y, v.y)
polygon = np.array(polygon, np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(image_in, [polygon], True, (0, 255, 0), 2)
caption = "{} {:.3f}".format(obj.name, confidence)
cv2.putText(image_in, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
cv2.imwrite(append_str_to_filename(image_path, 'detections'), image_in)
return latency