def test_field_mask_different_level_diffs():
original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=2.0),
red=1.0)
assert sorted(protobuf_helpers.field_mask(original, modified).paths) == [
"alpha",
"red",
]
def get_bounding_box_image(response):
dtype = np.uint8
img = np.fromstring(response.image_response.shot.image.data, dtype=dtype)
if response.image_response.shot.image.format == image_pb2.Image.FORMAT_RAW:
img = img.reshape(response.image_response.shot.image.rows,
response.image_response.shot.image.cols)
else:
img = cv2.imdecode(img, -1)
# Convert to BGR so we can draw colors
img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
# Draw bounding boxes in the image for all the detections.
for obj in response.object_in_image:
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
confidence = conf_msg.value
polygon = []
min_x = float('inf')
min_y = float('inf')
for v in obj.image_properties.coordinates.vertexes:
polygon.append([v.x, v.y])
min_x = min(min_x, v.x)
min_y = min(min_y, v.y)
polygon = np.array(polygon, np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(img, [polygon], True, (0, 255, 0), 2)
caption = "{} {:.3f}".format(obj.name, confidence)
cv2.putText(img, caption, (int(min_x), int(min_y)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
return img
def pod_to_pb_any(value):
if isinstance(value, bool):
v = wrappers_pb2.BoolValue(value=value)
elif isinstance(value, int):
v = wrappers_pb2.Int32Value(value=value)
elif isinstance(value, float):
v = wrappers_pb2.FloatValue(value=value)
elif isinstance(value, str):
v = wrappers_pb2.StringValue(value=value)
else:
raise ValueError("not supported cell data type: %s" % type(value))
return v
def test_field_mask_equal_values():
assert protobuf_helpers.field_mask(None, None).paths == []
original = struct_pb2.Value(number_value=1.0)
modified = struct_pb2.Value(number_value=1.0)
assert protobuf_helpers.field_mask(original, modified).paths == []
original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
assert protobuf_helpers.field_mask(original, modified).paths == []
original = struct_pb2.ListValue(
values=[struct_pb2.Value(number_value=1.0)])
modified = struct_pb2.ListValue(
values=[struct_pb2.Value(number_value=1.0)])
assert protobuf_helpers.field_mask(original, modified).paths == []
original = struct_pb2.Struct(
fields={'bar': struct_pb2.Value(number_value=1.0)})
modified = struct_pb2.Struct(
fields={'bar': struct_pb2.Value(number_value=1.0)})
assert protobuf_helpers.field_mask(original, modified).paths == []
def test_field_mask_wrapper_type_diffs():
original = color_pb2.Color()
modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
assert protobuf_helpers.field_mask(original, modified).paths == ['alpha']
original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
modified = color_pb2.Color()
assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha'])
original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=2.0))
assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha'])
original = None
modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=2.0))
assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha'])
original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0))
modified = None
assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha'])
def get_obj_and_img(network_compute_client, server, model, confidence,
image_sources, label):
for source in image_sources:
# Build a network compute request for this image source.
image_source_and_service = network_compute_bridge_pb2.ImageSourceAndService(
image_source=source)
# Input data:
# model name
# minimum confidence (between 0 and 1)
# if we should automatically rotate the image
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=image_source_and_service,
model_name=model,
min_confidence=confidence,
rotate_image=network_compute_bridge_pb2.NetworkComputeInputData.
ROTATE_IMAGE_ALIGN_HORIZONTAL)
# Server data: the service name
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=server)
# Pack and send the request.
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
resp = network_compute_client.network_compute_bridge_command(
process_img_req)
best_obj = None
highest_conf = 0.0
best_vision_tform_obj = None
img = get_bounding_box_image(resp)
image_full = resp.image_response
# Show the image
cv2.imshow("Fetch", img)
cv2.waitKey(15)
if len(resp.object_in_image) > 0:
for obj in resp.object_in_image:
# Get the label
obj_label = obj.name.split('_label_')[-1]
if obj_label != label:
continue
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
conf = conf_msg.value
try:
vision_tform_obj = frame_helpers.get_a_tform_b(
obj.transforms_snapshot,
frame_helpers.VISION_FRAME_NAME,
obj.image_properties.frame_name_image_coordinates)
except bosdyn.client.frame_helpers.ValidateFrameTreeError:
# No depth data available.
vision_tform_obj = None
if conf > highest_conf and vision_tform_obj is not None:
highest_conf = conf
best_obj = obj
best_vision_tform_obj = vision_tform_obj
if best_obj is not None:
return best_obj, image_full, best_vision_tform_obj
return None, None, None
def process_thread(args, request_queue, response_queue):
# Load the model(s)
models = {}
for model in args.model:
this_model = TensorFlowObjectDetectionModel(model[0], model[1])
models[this_model.name] = this_model
print('')
print('Service ' + args.name + ' running on port: ' + str(args.port))
print('Loaded models:')
for model_name in models:
print(' ' + model_name)
while True:
request = request_queue.get()
if isinstance(request,
network_compute_bridge_pb2.ListAvailableModelsRequest):
out_proto = network_compute_bridge_pb2.ListAvailableModelsResponse(
)
for model_name in models:
out_proto.available_models.append(model_name)
response_queue.put(out_proto)
continue
else:
out_proto = network_compute_bridge_pb2.NetworkComputeResponse()
# Find the model
if request.input_data.model_name not in models:
err_str = 'Cannot find model "' + request.input_data.model_name + '" in loaded models.'
print(err_str)
# Set the error in the header.
out_proto.header.error.code = header_pb2.CommonError.CODE_INVALID_REQUEST
out_proto.header.error.message = err_str
response_queue.put(out_proto)
continue
model = models[request.input_data.model_name]
# Unpack the incoming image.
if request.input_data.image.format == image_pb2.Image.FORMAT_RAW:
pil_image = Image.open(io.BytesIO(request.input_data.image.data))
if request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_GREYSCALE_U8:
# If the input image is grayscale, convert it to RGB.
image = cv2.cvtColor(pil_image, cv2.COLOR_GRAY2RGB)
elif request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_RGB_U8:
# Already an RGB image.
image = pil_image
else:
print('Error: image input in unsupported pixel format: ',
request.input_data.image.pixel_format)
response_queue.put(out_proto)
continue
elif request.input_data.image.format == image_pb2.Image.FORMAT_JPEG:
dtype = np.uint8
jpg = np.frombuffer(request.input_data.image.data, dtype=dtype)
image = cv2.imdecode(jpg, -1)
if len(image.shape) < 3:
# If the input image is grayscale, convert it to RGB.
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
image_width = image.shape[0]
image_height = image.shape[1]
detections = model.predict(image)
num_objects = 0
# All outputs are batches of tensors.
# Convert to numpy arrays, and take index [0] to remove the batch dimension.
# We're only interested in the first num_detections.
num_detections = int(detections.pop('num_detections'))
detections = {
key: value[0, :num_detections].numpy()
for key, value in detections.items()
}
boxes = detections['detection_boxes']
classes = detections['detection_classes']
scores = detections['detection_scores']
for i in range(boxes.shape[0]):
if scores[i] < request.input_data.min_confidence:
continue
box = tuple(boxes[i].tolist())
# Boxes come in with normalized coordinates. Convert to pixel values.
box = [
box[0] * image_width, box[1] * image_height,
box[2] * image_width, box[3] * image_height
]
score = scores[i]
if classes[i] in model.category_index.keys():
label = model.category_index[classes[i]]['name']
else:
label = 'N/A'
num_objects += 1
print('Found object with label: "' + label + '" and score: ' +
str(score))
point1 = np.array([box[1], box[0]])
point2 = np.array([box[3], box[0]])
point3 = np.array([box[3], box[2]])
point4 = np.array([box[1], box[2]])
# Add data to the output proto.
out_obj = out_proto.object_in_image.add()
out_obj.name = "obj" + str(num_objects) + "_label_" + label
vertex1 = out_obj.image_properties.coordinates.vertexes.add()
vertex1.x = point1[0]
vertex1.y = point1[1]
vertex2 = out_obj.image_properties.coordinates.vertexes.add()
vertex2.x = point2[0]
vertex2.y = point2[1]
vertex3 = out_obj.image_properties.coordinates.vertexes.add()
vertex3.x = point3[0]
vertex3.y = point3[1]
vertex4 = out_obj.image_properties.coordinates.vertexes.add()
vertex4.x = point4[0]
vertex4.y = point4[1]
# Pack the confidence value.
confidence = wrappers_pb2.FloatValue(value=score)
out_obj.additional_properties.Pack(confidence)
if not args.no_debug:
polygon = np.array([point1, point2, point3, point4], np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(image, [polygon], True, (0, 255, 0), 2)
caption = "{}: {:.3f}".format(label, score)
left_x = min(point1[0],
min(point2[0], min(point3[0], point4[0])))
top_y = min(point1[1], min(point2[1],
min(point3[1], point4[1])))
cv2.putText(image, caption, (int(left_x), int(top_y)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
print('Found ' + str(num_objects) + ' object(s)')
if not args.no_debug:
debug_image_filename = 'network_compute_server_output.jpg'
cv2.imwrite(debug_image_filename, image)
print('Wrote debug image output to: "' + debug_image_filename +
'"')
response_queue.put(out_proto)
def main(argv):
"""An example using the API to list and get specific objects."""
parser = argparse.ArgumentParser()
bosdyn.client.util.add_common_arguments(parser)
parser.add_argument('-i', '--image-source', help='Image source on the robot to use.')
parser.add_argument(
'-q', '--image-source-service', help=
'Image *service* for the image source to use. Defaults to the main image service if not provided.',
default='')
parser.add_argument('-s', '--service',
help='Service name of external machine learning server in the directory.',
required=False)
parser.add_argument('-m', '--model', help='Model file on the server')
parser.add_argument('-c', '--confidence', help='Minimum confidence to return an object.',
default=0.5, type=float)
parser.add_argument('-j', '--input-image',
help='Path to an image to use instead of an image source.')
parser.add_argument(
'-l', '--model-list',
help='List all available network compute servers and their provided models.',
action='store_true')
parser.add_argument('-r', '--disable-rotation',
help='Disable rotation of images (to align with horizontal)',
action='store_true')
options = parser.parse_args(argv)
if options.image_source is not None and options.input_image is not None:
print('Error: cannot provide both an input image and an image source.')
sys.exit(1)
if options.model_list and (options.image_source is not None or options.input_image is not None):
print('Error: cannot list models with input image source or input image.')
sys.exit(1)
if options.image_source is None and options.input_image is None and options.model_list == False:
default_image_source = 'frontleft_fisheye_image'
print('No image source provided so defaulting to "' + default_image_source + '".')
options.image_source = default_image_source
# Create robot object with a world object client
sdk = bosdyn.client.create_standard_sdk('IdentifyObjectClient')
robot = sdk.create_robot(options.hostname)
robot.authenticate(options.username, options.password)
#Time sync is necessary so that time-based filter requests can be converted
robot.time_sync.wait_for_sync()
#Create the network compute client
network_compute_client = robot.ensure_client(NetworkComputeBridgeClient.default_service_name)
directory_client = robot.ensure_client(
bosdyn.client.directory.DirectoryClient.default_service_name)
robot_state_client = robot.ensure_client(RobotStateClient.default_service_name)
robot_command_client = robot.ensure_client(RobotCommandClient.default_service_name)
robot.time_sync.wait_for_sync()
if options.model_list:
server_service_names = get_all_network_compute_services(directory_client)
print('Found ' + str(len(server_service_names)) +
' available service(s). Listing their models:')
print('------------------------------------')
for service in server_service_names:
print(' ' + service)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=service)
list_req = network_compute_bridge_pb2.ListAvailableModelsRequest(
server_config=server_data)
response = network_compute_client.list_available_models_command(list_req)
if response.header.error.message:
print(' Error message: {}'.format(response.header.error.message))
else:
for model in response.available_models:
print(' ' + model)
sys.exit(0)
# A service name must be provided if not doing a directory list.
if options.service is None or len(options.service) == 0:
print('Error: --service must be provided for operations other than --model-list')
sys.exit(1)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(
service_name=options.service)
if options.image_source is not None:
if options.model is None:
print('Error: you must provide a model.')
sys.exit(1)
img_source_and_service = network_compute_bridge_pb2.ImageSourceAndService(
image_source=options.image_source, image_service=options.image_source_service)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image_source_and_service=img_source_and_service, model_name=options.model,
min_confidence=options.confidence)
else:
# Read the input image.
image_in = cv2.imread(options.input_image)
if image_in is None:
print('Error: failed to read "' + options.input_image + '". Does the file exist?')
sys.exit(1)
rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB)
success, im_buffer = cv2.imencode(".jpg", rgb)
if not success:
print('Error: failed to encode input image as a jpg. Abort.')
sys.exit(1)
height = image_in.shape[0]
width = image_in.shape[1]
image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height,
data=im_buffer.tobytes(),
pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(
image=image_proto, model_name=options.model, min_confidence=options.confidence)
if options.disable_rotation:
input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_NO_ROTATION
else:
input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_ALIGN_HORIZONTAL
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
response = network_compute_client.network_compute_bridge_command(process_img_req)
if len(response.object_in_image) <= 0:
print('No objects found')
else:
print('Got ' + str(len(response.object_in_image)) + ' objects.')
if options.image_source is not None:
# We asked for an image to be taken, so the return proto should have an image in it.
dtype = np.uint8
img = np.frombuffer(response.image_response.shot.image.data, dtype=dtype)
if response.image_response.shot.image.format == image_pb2.Image.FORMAT_RAW:
img = img.reshape(response.image_response.shot.image.rows,
response.image_response.shot.image.cols)
else:
img = cv2.imdecode(img, -1)
else:
# To save bandwidth, the network_compute_bridge service won't re-send us back our own
# image.
img = image_in
# The image always comes back in the raw orientation. Rotate it to horizontal so that we can
# visualize it the same as it was processed.
img, rotmat = rotate_image_nocrop(img, response.image_rotation_angle)
# Convert to color for nicer drawing
if len(img.shape) < 3:
img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB)
# Draw bounding boxes in the image for all the detections.
for obj in response.object_in_image:
print(obj)
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
confidence = conf_msg.value
polygon = []
min_x = float('inf')
min_y = float('inf')
for v in obj.image_properties.coordinates.vertexes:
# If we are rotating the output image, make sure to rotate the bounding box points
# as well
x, y = rotate_point(v.x, v.y, rotmat)
polygon.append([x, y])
min_x = min(min_x, x)
min_y = min(min_y, y)
polygon = np.array(polygon, np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(img, [polygon], True, (0, 255, 0), 2)
caption = "{} {:.3f}".format(obj.name, confidence)
cv2.putText(img, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
cv2.imwrite('identify_object_output.jpg', img)
def run_model(self, request, this_model):
# Define the out proto
out_proto = network_compute_bridge_pb2.NetworkComputeResponse()
if request.input_data.image.format == image_pb2.Image.FORMAT_RAW:
pil_image = Image.open(io.BytesIO(request.input_data.image.data))
pil_image = ndimage.rotate(pil_image, 0)
if request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_GREYSCALE_U8:
image = cv2.cvtColor(pil_image, cv2.COLOR_GRAY2RGB) # Converted to RGB for Tensorflow
elif request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_RGB_U8:
# Already in the correct format
image = pil_image
else:
print('Error: image input in unsupported pixel format: ', request.input_data.image.pixel_format)
RESPONSE_QUEUE.put(out_proto)
return
elif request.input_data.image.format == image_pb2.Image.FORMAT_JPEG:
dtype = np.uint8
jpg = np.frombuffer(request.input_data.image.data, dtype=dtype)
image = cv2.imdecode(jpg, -1)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if len(image.shape) < 3:
# Single channel image, convert to RGB.
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
print('')
print('Starting model eval...')
# Run the model on the image
keras_image = preprocess_image(image)
(keras_image, scale) = resize_image(keras_image)
keras_image = np.expand_dims(keras_image, axis=0)
start = timer()
boxes, scores, classes = this_model.predict(keras_image)
end = timer()
print("Model eval took " + str(end - start) + ' seconds')
boxes /= scale
# Package detections into the output proto
num_objects = 0
boxes = boxes[0]
scores = scores[0]
classes = classes[0]
# Only use the detection w/ the highest confidence for Sensor pointing features
max_conf = max(scores)
print(f"Max conf was {max_conf}")
for i in range(len(boxes)):
# Skip if not the Fire Extinguisher class
if classes[i] != 7: continue
label = this_model.labels[(classes[i])]
box = boxes[i]
score = scores[i]
if score < max_conf:
# scores are sorted so we can break
continue
num_objects += 1
#draw_box(draw, b, color=color)
print('Found object with label: "' + label + '" and score: ' + str(score))
print(f"Box is {box}")
point1 = np.array([box[0], box[1]])
point2 = np.array([box[0], box[3]])
point3 = np.array([box[2], box[3]])
point4 = np.array([box[2], box[1]])
# Add data to the output proto.
out_obj = out_proto.object_in_image.add()
out_obj.name = label
vertex1 = out_obj.image_properties.coordinates.vertexes.add()
vertex1.x = point1[0]
vertex1.y = point1[1]
vertex2 = out_obj.image_properties.coordinates.vertexes.add()
vertex2.x = point2[0]
vertex2.y = point2[1]
vertex3 = out_obj.image_properties.coordinates.vertexes.add()
vertex3.x = point3[0]
vertex3.y = point3[1]
vertex4 = out_obj.image_properties.coordinates.vertexes.add()
vertex4.x = point4[0]
vertex4.y = point4[1]
# Pack the confidence value.
confidence = wrappers_pb2.FloatValue(value=score)
out_obj.additional_properties.Pack(confidence)
if not self.options.no_debug:
polygon = np.array([point1, point2, point3, point4], np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(image, [polygon], True, (0, 255, 0), 2)
caption = "{}: {:.3f}".format(label, score)
left_x = min(point1[0], min(point2[0], min(point3[0], point4[0])))
top_y = min(point1[1], min(point2[1], min(point3[1], point4[1])))
cv2.putText(image, caption, (int(left_x), int(top_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
print('Found ' + str(num_objects) + ' object(s)')
if not self.options.no_debug:
debug_image_filename = 'retinanet_server_output.jpg'
cv2.imwrite(debug_image_filename, cv2.cvtColor(image, cv2.COLOR_RGB2BGR))
print('Wrote debug image output to: "' + debug_image_filename + '"')
# Pack all the outputs up and send them back.
self.out_queue.put(out_proto)
def process_images(options, model_extension):
"""Starts Tensorflow and detects objects in the incoming images.
"""
models = {}
for f in os.listdir(options.model_dir):
if f in models:
print('Warning: duplicate model name of "' + f +
'", ignoring second model.')
continue
path = os.path.join(options.model_dir, f)
if os.path.isfile(path) and path.endswith(model_extension):
model_name = ''.join(f.rsplit(model_extension,
1)) # remove the extension
# reverse replace the extension
labels_file = '.csv'.join(f.rsplit(model_extension, 1))
labels_path = os.path.join(options.model_dir, labels_file)
if not os.path.isfile(labels_path):
labels_path = None
models[model_name] = (TensorflowModel(path, labels_path))
# Tensorflow prints out a bunch of stuff, so print out useful data here after a space.
print('')
print('Running on port: ' + str(options.port))
print('Loaded models:')
for model_name in models:
if models[model_name].labels is not None:
labels_str = 'yes'
else:
labels_str = 'no'
print(' ' + model_name + ' (loaded labels: ' + labels_str + ')')
while True:
request = REQUEST_QUEUE.get()
if isinstance(request,
network_compute_bridge_pb2.ListAvailableModelsRequest):
out_proto = network_compute_bridge_pb2.ListAvailableModelsResponse(
)
for f in models:
out_proto.available_models.append(f)
RESPONSE_QUEUE.put(out_proto)
continue
else:
out_proto = network_compute_bridge_pb2.NetworkComputeResponse()
# Find the model
if request.input_data.model_name not in models:
err_str = 'Cannot find model "' + request.input_data.model_name + '" in loaded models.'
print(err_str)
# Set the error in the header.
out_proto.header.error.code = header_pb2.CommonError.CODE_INVALID_REQUEST
out_proto.header.error.message = err_str
RESPONSE_QUEUE.put(out_proto)
continue
model = models[request.input_data.model_name]
if request.input_data.image.format == image_pb2.Image.FORMAT_RAW:
pil_image = Image.open(io.BytesIO(request.input_data.image.data))
pil_image = ndimage.rotate(pil_image, 0)
if request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_GREYSCALE_U8:
image = cv2.cvtColor(
pil_image,
cv2.COLOR_GRAY2RGB) # Converted to RGB for Tensorflow
elif request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_RGB_U8:
# Already in the correct format
image = pil_image
else:
err_str = 'Error: image input in unsupported pixel format: ', request.input_data.image.pixel_format
print(err_str)
# Set the error in the header.
out_proto.header.error.code = header_pb2.CommonError.CODE_INVALID_REQUEST
out_proto.header.error.message = err_str
RESPONSE_QUEUE.put(out_proto)
continue
elif request.input_data.image.format == image_pb2.Image.FORMAT_JPEG:
dtype = np.uint8
jpg = np.frombuffer(request.input_data.image.data, dtype=dtype)
image = cv2.imdecode(jpg, -1)
if len(image.shape) < 3:
# Single channel image, convert to RGB.
image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
boxes, scores, classes, _ = model.predict(image)
num_objects = 0
for i in range(len(boxes)):
label = str(classes[i])
box = boxes[i]
score = scores[i]
if score < request.input_data.min_confidence:
# scores are sorted so we can break
break
num_objects += 1
#draw_box(draw, b, color=color)
print('Found object with label: "' + label + '" and score: ' +
str(score))
point1 = np.array([box[1], box[0]])
point2 = np.array([box[3], box[0]])
point3 = np.array([box[3], box[2]])
point4 = np.array([box[1], box[2]])
# Add data to the output proto.
out_obj = out_proto.object_in_image.add()
out_obj.name = "obj" + str(num_objects) + "_label_" + label
vertex1 = out_obj.image_properties.coordinates.vertexes.add()
vertex1.x = point1[0]
vertex1.y = point1[1]
vertex2 = out_obj.image_properties.coordinates.vertexes.add()
vertex2.x = point2[0]
vertex2.y = point2[1]
vertex3 = out_obj.image_properties.coordinates.vertexes.add()
vertex3.x = point3[0]
vertex3.y = point3[1]
vertex4 = out_obj.image_properties.coordinates.vertexes.add()
vertex4.x = point4[0]
vertex4.y = point4[1]
# Pack the confidence value.
confidence = wrappers_pb2.FloatValue(value=score)
out_obj.additional_properties.Pack(confidence)
if not options.no_debug:
polygon = np.array([point1, point2, point3, point4], np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(image, [polygon], True, (0, 255, 0), 2)
caption = "{}: {:.3f}".format(label, score)
left_x = min(point1[0],
min(point2[0], min(point3[0], point4[0])))
top_y = min(point1[1], min(point2[1],
min(point3[1], point4[1])))
cv2.putText(image, caption, (int(left_x), int(top_y)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
print('Found ' + str(num_objects) + ' object(s)')
if not options.no_debug:
debug_image_filename = 'tensorflow_server_output.jpg'
cv2.imwrite(debug_image_filename, image)
print('Wrote debug image output to: "' + debug_image_filename +
'"')
RESPONSE_QUEUE.put(out_proto)
def _send_request(server, image_path, model, confidence, verbose=False):
start = time.time()
channel = grpc.insecure_channel(server)
stub = network_compute_bridge_service_pb2_grpc.NetworkComputeBridgeWorkerStub(channel)
server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(service_name='test')
# Read the input image.
image_in = cv2.imread(image_path)
if image_in is None:
print('Error: failed to read "' + image_path + '". Does the file exist?')
sys.exit(1)
rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB)
success, im_buffer = cv2.imencode(".jpg", rgb)
if not success:
print('Error: failed to encode input image as a jpg. Abort.')
sys.exit(1)
height = image_in.shape[0]
width = image_in.shape[1]
image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height,
data=im_buffer.tobytes(),
pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8)
input_data = network_compute_bridge_pb2.NetworkComputeInputData(image=image_proto,
model_name=model,
min_confidence=confidence)
process_img_req = network_compute_bridge_pb2.NetworkComputeRequest(
input_data=input_data, server_config=server_data)
response = stub.NetworkCompute(process_img_req)
end = time.time()
latency = end - start
print(f'latency: {latency * 1000} ms')
if verbose:
if len(response.object_in_image) <= 0:
print('No objects found')
else:
print('Got ' + str(len(response.object_in_image)) + ' objects.')
# Draw bounding boxes in the image for all the detections.
for obj in response.object_in_image:
print(obj)
conf_msg = wrappers_pb2.FloatValue()
obj.additional_properties.Unpack(conf_msg)
confidence = conf_msg.value
polygon = []
min_x = float('inf')
min_y = float('inf')
for v in obj.image_properties.coordinates.vertexes:
polygon.append([v.x, v.y])
min_x = min(min_x, v.x)
min_y = min(min_y, v.y)
polygon = np.array(polygon, np.int32)
polygon = polygon.reshape((-1, 1, 2))
cv2.polylines(image_in, [polygon], True, (0, 255, 0), 2)
caption = "{} {:.3f}".format(obj.name, confidence)
cv2.putText(image_in, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 255, 0), 2)
cv2.imwrite(append_str_to_filename(image_path, 'detections'), image_in)
return latency
def walk_to_object(config):
"""Get an image and command the robot to walk up to a selected object.
We'll walk "up to" the object, not on top of it. The idea is that you
want to interact or manipulate the object."""
# See hello_spot.py for an explanation of these lines.
bosdyn.client.util.setup_logging(config.verbose)
sdk = bosdyn.client.create_standard_sdk('WalkToObjectClient')
robot = sdk.create_robot(config.hostname)
robot.authenticate(config.username, config.password)
robot.time_sync.wait_for_sync()
assert robot.has_arm(), "Robot requires an arm to run this example."
# Verify the robot is not estopped and that an external application has registered and holds
# an estop endpoint.
assert not robot.is_estopped(), "Robot is estopped. Please use an external E-Stop client, " \
"such as the estop SDK example, to configure E-Stop."
lease_client = robot.ensure_client(
bosdyn.client.lease.LeaseClient.default_service_name)
image_client = robot.ensure_client(ImageClient.default_service_name)
manipulation_api_client = robot.ensure_client(
ManipulationApiClient.default_service_name)
lease = lease_client.acquire()
try:
with bosdyn.client.lease.LeaseKeepAlive(lease_client):
# Now, we are ready to power on the robot. This call will block until the power
# is on. Commands would fail if this did not happen. We can also check that the robot is
# powered at any point.
robot.logger.info(
"Powering on robot... This may take a several seconds.")
robot.power_on(timeout_sec=20)
assert robot.is_powered_on(), "Robot power on failed."
robot.logger.info("Robot powered on.")
# Tell the robot to stand up. The command service is used to issue commands to a robot.
# The set of valid commands for a robot depends on hardware configuration. See
# SpotCommandHelper for more detailed examples on command building. The robot
# command service requires timesync between the robot and the client.
robot.logger.info("Commanding robot to stand...")
command_client = robot.ensure_client(
RobotCommandClient.default_service_name)
blocking_stand(command_client, timeout_sec=10)
robot.logger.info("Robot standing.")
time.sleep(2.0)
# Take a picture with a camera
robot.logger.info('Getting an image from: ' + config.image_source)
image_responses = image_client.get_image_from_sources(
[config.image_source])
if len(image_responses) != 1:
print('Got invalid number of images: ' +
str(len(image_responses)))
print(image_responses)
assert False
image = image_responses[0]
if image.shot.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_DEPTH_U16:
dtype = np.uint16
else:
dtype = np.uint8
img = np.fromstring(image.shot.image.data, dtype=dtype)
if image.shot.image.format == image_pb2.Image.FORMAT_RAW:
img = img.reshape(image.shot.image.rows, image.shot.image.cols)
else:
img = cv2.imdecode(img, -1)
# Show the image to the user and wait for them to click on a pixel
robot.logger.info('Click on an object to walk up to...')
image_title = 'Click to walk up to something'
cv2.namedWindow(image_title)
cv2.setMouseCallback(image_title, cv_mouse_callback)
global g_image_click, g_image_display
g_image_display = img
cv2.imshow(image_title, g_image_display)
while g_image_click is None:
key = cv2.waitKey(1) & 0xFF
if key == ord('q') or key == ord('Q'):
# Quit
print('"q" pressed, exiting.')
exit(0)
robot.logger.info('Walking to object at image location (' +
str(g_image_click[0]) + ', ' +
str(g_image_click[1]) + ')')
walk_vec = geometry_pb2.Vec2(x=g_image_click[0],
y=g_image_click[1])
# Optionally populate the offset distance parameter.
if config.distance is None:
offset_distance = None
else:
offset_distance = wrappers_pb2.FloatValue(
value=config.distance)
# Build the proto
walk_to = manipulation_api_pb2.WalkToObjectInImage(
pixel_xy=walk_vec,
transforms_snapshot_for_camera=image.shot.transforms_snapshot,
frame_name_image_sensor=image.shot.frame_name_image_sensor,
camera_model=image.source.pinhole,
offset_distance=offset_distance)
# Ask the robot to pick up the object
walk_to_request = manipulation_api_pb2.ManipulationApiRequest(
walk_to_object_in_image=walk_to)
# Send the request
cmd_response = manipulation_api_client.manipulation_api_command(
manipulation_api_request=walk_to_request)
# Get feedback from the robot
while True:
time.sleep(0.25)
feedback_request = manipulation_api_pb2.ManipulationApiFeedbackRequest(
manipulation_cmd_id=cmd_response.manipulation_cmd_id)
# Send the request
response = manipulation_api_client.manipulation_api_feedback_command(
manipulation_api_feedback_request=feedback_request)
print(
'Current state: ',
manipulation_api_pb2.ManipulationFeedbackState.Name(
response.current_state))
if response.current_state == manipulation_api_pb2.MANIP_STATE_DONE:
break
robot.logger.info('Finished.')
time.sleep(4.0)
robot.logger.info('Sitting down and turning off.')
# Power the robot off. By specifying "cut_immediately=False", a safe power off command
# is issued to the robot. This will attempt to sit the robot before powering off.
robot.power_off(cut_immediately=False, timeout_sec=20)
assert not robot.is_powered_on(), "Robot power off failed."
robot.logger.info("Robot safely powered off.")
finally:
# If we successfully acquired a lease, return it.
lease_client.return_lease(lease)
