def test_field_mask_different_level_diffs(): original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=2.0), red=1.0) assert sorted(protobuf_helpers.field_mask(original, modified).paths) == [ "alpha", "red", ]
def get_bounding_box_image(response): dtype = np.uint8 img = np.fromstring(response.image_response.shot.image.data, dtype=dtype) if response.image_response.shot.image.format == image_pb2.Image.FORMAT_RAW: img = img.reshape(response.image_response.shot.image.rows, response.image_response.shot.image.cols) else: img = cv2.imdecode(img, -1) # Convert to BGR so we can draw colors img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) # Draw bounding boxes in the image for all the detections. for obj in response.object_in_image: conf_msg = wrappers_pb2.FloatValue() obj.additional_properties.Unpack(conf_msg) confidence = conf_msg.value polygon = [] min_x = float('inf') min_y = float('inf') for v in obj.image_properties.coordinates.vertexes: polygon.append([v.x, v.y]) min_x = min(min_x, v.x) min_y = min(min_y, v.y) polygon = np.array(polygon, np.int32) polygon = polygon.reshape((-1, 1, 2)) cv2.polylines(img, [polygon], True, (0, 255, 0), 2) caption = "{} {:.3f}".format(obj.name, confidence) cv2.putText(img, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) return img
def pod_to_pb_any(value): if isinstance(value, bool): v = wrappers_pb2.BoolValue(value=value) elif isinstance(value, int): v = wrappers_pb2.Int32Value(value=value) elif isinstance(value, float): v = wrappers_pb2.FloatValue(value=value) elif isinstance(value, str): v = wrappers_pb2.StringValue(value=value) else: raise ValueError("not supported cell data type: %s" % type(value)) return v
def test_field_mask_equal_values(): assert protobuf_helpers.field_mask(None, None).paths == [] original = struct_pb2.Value(number_value=1.0) modified = struct_pb2.Value(number_value=1.0) assert protobuf_helpers.field_mask(original, modified).paths == [] original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) assert protobuf_helpers.field_mask(original, modified).paths == [] original = struct_pb2.ListValue( values=[struct_pb2.Value(number_value=1.0)]) modified = struct_pb2.ListValue( values=[struct_pb2.Value(number_value=1.0)]) assert protobuf_helpers.field_mask(original, modified).paths == [] original = struct_pb2.Struct( fields={'bar': struct_pb2.Value(number_value=1.0)}) modified = struct_pb2.Struct( fields={'bar': struct_pb2.Value(number_value=1.0)}) assert protobuf_helpers.field_mask(original, modified).paths == []
def test_field_mask_wrapper_type_diffs(): original = color_pb2.Color() modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) assert protobuf_helpers.field_mask(original, modified).paths == ['alpha'] original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) modified = color_pb2.Color() assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha']) original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=2.0)) assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha']) original = None modified = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=2.0)) assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha']) original = color_pb2.Color(alpha=wrappers_pb2.FloatValue(value=1.0)) modified = None assert (protobuf_helpers.field_mask(original, modified).paths == ['alpha'])
def get_obj_and_img(network_compute_client, server, model, confidence, image_sources, label): for source in image_sources: # Build a network compute request for this image source. image_source_and_service = network_compute_bridge_pb2.ImageSourceAndService( image_source=source) # Input data: # model name # minimum confidence (between 0 and 1) # if we should automatically rotate the image input_data = network_compute_bridge_pb2.NetworkComputeInputData( image_source_and_service=image_source_and_service, model_name=model, min_confidence=confidence, rotate_image=network_compute_bridge_pb2.NetworkComputeInputData. ROTATE_IMAGE_ALIGN_HORIZONTAL) # Server data: the service name server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration( service_name=server) # Pack and send the request. process_img_req = network_compute_bridge_pb2.NetworkComputeRequest( input_data=input_data, server_config=server_data) resp = network_compute_client.network_compute_bridge_command( process_img_req) best_obj = None highest_conf = 0.0 best_vision_tform_obj = None img = get_bounding_box_image(resp) image_full = resp.image_response # Show the image cv2.imshow("Fetch", img) cv2.waitKey(15) if len(resp.object_in_image) > 0: for obj in resp.object_in_image: # Get the label obj_label = obj.name.split('_label_')[-1] if obj_label != label: continue conf_msg = wrappers_pb2.FloatValue() obj.additional_properties.Unpack(conf_msg) conf = conf_msg.value try: vision_tform_obj = frame_helpers.get_a_tform_b( obj.transforms_snapshot, frame_helpers.VISION_FRAME_NAME, obj.image_properties.frame_name_image_coordinates) except bosdyn.client.frame_helpers.ValidateFrameTreeError: # No depth data available. vision_tform_obj = None if conf > highest_conf and vision_tform_obj is not None: highest_conf = conf best_obj = obj best_vision_tform_obj = vision_tform_obj if best_obj is not None: return best_obj, image_full, best_vision_tform_obj return None, None, None
def process_thread(args, request_queue, response_queue): # Load the model(s) models = {} for model in args.model: this_model = TensorFlowObjectDetectionModel(model[0], model[1]) models[this_model.name] = this_model print('') print('Service ' + args.name + ' running on port: ' + str(args.port)) print('Loaded models:') for model_name in models: print(' ' + model_name) while True: request = request_queue.get() if isinstance(request, network_compute_bridge_pb2.ListAvailableModelsRequest): out_proto = network_compute_bridge_pb2.ListAvailableModelsResponse( ) for model_name in models: out_proto.available_models.append(model_name) response_queue.put(out_proto) continue else: out_proto = network_compute_bridge_pb2.NetworkComputeResponse() # Find the model if request.input_data.model_name not in models: err_str = 'Cannot find model "' + request.input_data.model_name + '" in loaded models.' print(err_str) # Set the error in the header. out_proto.header.error.code = header_pb2.CommonError.CODE_INVALID_REQUEST out_proto.header.error.message = err_str response_queue.put(out_proto) continue model = models[request.input_data.model_name] # Unpack the incoming image. if request.input_data.image.format == image_pb2.Image.FORMAT_RAW: pil_image = Image.open(io.BytesIO(request.input_data.image.data)) if request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_GREYSCALE_U8: # If the input image is grayscale, convert it to RGB. image = cv2.cvtColor(pil_image, cv2.COLOR_GRAY2RGB) elif request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_RGB_U8: # Already an RGB image. image = pil_image else: print('Error: image input in unsupported pixel format: ', request.input_data.image.pixel_format) response_queue.put(out_proto) continue elif request.input_data.image.format == image_pb2.Image.FORMAT_JPEG: dtype = np.uint8 jpg = np.frombuffer(request.input_data.image.data, dtype=dtype) image = cv2.imdecode(jpg, -1) if len(image.shape) < 3: # If the input image is grayscale, convert it to RGB. image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) image_width = image.shape[0] image_height = image.shape[1] detections = model.predict(image) num_objects = 0 # All outputs are batches of tensors. # Convert to numpy arrays, and take index [0] to remove the batch dimension. # We're only interested in the first num_detections. num_detections = int(detections.pop('num_detections')) detections = { key: value[0, :num_detections].numpy() for key, value in detections.items() } boxes = detections['detection_boxes'] classes = detections['detection_classes'] scores = detections['detection_scores'] for i in range(boxes.shape[0]): if scores[i] < request.input_data.min_confidence: continue box = tuple(boxes[i].tolist()) # Boxes come in with normalized coordinates. Convert to pixel values. box = [ box[0] * image_width, box[1] * image_height, box[2] * image_width, box[3] * image_height ] score = scores[i] if classes[i] in model.category_index.keys(): label = model.category_index[classes[i]]['name'] else: label = 'N/A' num_objects += 1 print('Found object with label: "' + label + '" and score: ' + str(score)) point1 = np.array([box[1], box[0]]) point2 = np.array([box[3], box[0]]) point3 = np.array([box[3], box[2]]) point4 = np.array([box[1], box[2]]) # Add data to the output proto. out_obj = out_proto.object_in_image.add() out_obj.name = "obj" + str(num_objects) + "_label_" + label vertex1 = out_obj.image_properties.coordinates.vertexes.add() vertex1.x = point1[0] vertex1.y = point1[1] vertex2 = out_obj.image_properties.coordinates.vertexes.add() vertex2.x = point2[0] vertex2.y = point2[1] vertex3 = out_obj.image_properties.coordinates.vertexes.add() vertex3.x = point3[0] vertex3.y = point3[1] vertex4 = out_obj.image_properties.coordinates.vertexes.add() vertex4.x = point4[0] vertex4.y = point4[1] # Pack the confidence value. confidence = wrappers_pb2.FloatValue(value=score) out_obj.additional_properties.Pack(confidence) if not args.no_debug: polygon = np.array([point1, point2, point3, point4], np.int32) polygon = polygon.reshape((-1, 1, 2)) cv2.polylines(image, [polygon], True, (0, 255, 0), 2) caption = "{}: {:.3f}".format(label, score) left_x = min(point1[0], min(point2[0], min(point3[0], point4[0]))) top_y = min(point1[1], min(point2[1], min(point3[1], point4[1]))) cv2.putText(image, caption, (int(left_x), int(top_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) print('Found ' + str(num_objects) + ' object(s)') if not args.no_debug: debug_image_filename = 'network_compute_server_output.jpg' cv2.imwrite(debug_image_filename, image) print('Wrote debug image output to: "' + debug_image_filename + '"') response_queue.put(out_proto)
def main(argv): """An example using the API to list and get specific objects.""" parser = argparse.ArgumentParser() bosdyn.client.util.add_common_arguments(parser) parser.add_argument('-i', '--image-source', help='Image source on the robot to use.') parser.add_argument( '-q', '--image-source-service', help= 'Image *service* for the image source to use. Defaults to the main image service if not provided.', default='') parser.add_argument('-s', '--service', help='Service name of external machine learning server in the directory.', required=False) parser.add_argument('-m', '--model', help='Model file on the server') parser.add_argument('-c', '--confidence', help='Minimum confidence to return an object.', default=0.5, type=float) parser.add_argument('-j', '--input-image', help='Path to an image to use instead of an image source.') parser.add_argument( '-l', '--model-list', help='List all available network compute servers and their provided models.', action='store_true') parser.add_argument('-r', '--disable-rotation', help='Disable rotation of images (to align with horizontal)', action='store_true') options = parser.parse_args(argv) if options.image_source is not None and options.input_image is not None: print('Error: cannot provide both an input image and an image source.') sys.exit(1) if options.model_list and (options.image_source is not None or options.input_image is not None): print('Error: cannot list models with input image source or input image.') sys.exit(1) if options.image_source is None and options.input_image is None and options.model_list == False: default_image_source = 'frontleft_fisheye_image' print('No image source provided so defaulting to "' + default_image_source + '".') options.image_source = default_image_source # Create robot object with a world object client sdk = bosdyn.client.create_standard_sdk('IdentifyObjectClient') robot = sdk.create_robot(options.hostname) robot.authenticate(options.username, options.password) #Time sync is necessary so that time-based filter requests can be converted robot.time_sync.wait_for_sync() #Create the network compute client network_compute_client = robot.ensure_client(NetworkComputeBridgeClient.default_service_name) directory_client = robot.ensure_client( bosdyn.client.directory.DirectoryClient.default_service_name) robot_state_client = robot.ensure_client(RobotStateClient.default_service_name) robot_command_client = robot.ensure_client(RobotCommandClient.default_service_name) robot.time_sync.wait_for_sync() if options.model_list: server_service_names = get_all_network_compute_services(directory_client) print('Found ' + str(len(server_service_names)) + ' available service(s). Listing their models:') print('------------------------------------') for service in server_service_names: print(' ' + service) server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration( service_name=service) list_req = network_compute_bridge_pb2.ListAvailableModelsRequest( server_config=server_data) response = network_compute_client.list_available_models_command(list_req) if response.header.error.message: print(' Error message: {}'.format(response.header.error.message)) else: for model in response.available_models: print(' ' + model) sys.exit(0) # A service name must be provided if not doing a directory list. if options.service is None or len(options.service) == 0: print('Error: --service must be provided for operations other than --model-list') sys.exit(1) server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration( service_name=options.service) if options.image_source is not None: if options.model is None: print('Error: you must provide a model.') sys.exit(1) img_source_and_service = network_compute_bridge_pb2.ImageSourceAndService( image_source=options.image_source, image_service=options.image_source_service) input_data = network_compute_bridge_pb2.NetworkComputeInputData( image_source_and_service=img_source_and_service, model_name=options.model, min_confidence=options.confidence) else: # Read the input image. image_in = cv2.imread(options.input_image) if image_in is None: print('Error: failed to read "' + options.input_image + '". Does the file exist?') sys.exit(1) rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB) success, im_buffer = cv2.imencode(".jpg", rgb) if not success: print('Error: failed to encode input image as a jpg. Abort.') sys.exit(1) height = image_in.shape[0] width = image_in.shape[1] image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height, data=im_buffer.tobytes(), pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8) input_data = network_compute_bridge_pb2.NetworkComputeInputData( image=image_proto, model_name=options.model, min_confidence=options.confidence) if options.disable_rotation: input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_NO_ROTATION else: input_data.rotate_image = network_compute_bridge_pb2.NetworkComputeInputData.ROTATE_IMAGE_ALIGN_HORIZONTAL process_img_req = network_compute_bridge_pb2.NetworkComputeRequest( input_data=input_data, server_config=server_data) response = network_compute_client.network_compute_bridge_command(process_img_req) if len(response.object_in_image) <= 0: print('No objects found') else: print('Got ' + str(len(response.object_in_image)) + ' objects.') if options.image_source is not None: # We asked for an image to be taken, so the return proto should have an image in it. dtype = np.uint8 img = np.frombuffer(response.image_response.shot.image.data, dtype=dtype) if response.image_response.shot.image.format == image_pb2.Image.FORMAT_RAW: img = img.reshape(response.image_response.shot.image.rows, response.image_response.shot.image.cols) else: img = cv2.imdecode(img, -1) else: # To save bandwidth, the network_compute_bridge service won't re-send us back our own # image. img = image_in # The image always comes back in the raw orientation. Rotate it to horizontal so that we can # visualize it the same as it was processed. img, rotmat = rotate_image_nocrop(img, response.image_rotation_angle) # Convert to color for nicer drawing if len(img.shape) < 3: img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) # Draw bounding boxes in the image for all the detections. for obj in response.object_in_image: print(obj) conf_msg = wrappers_pb2.FloatValue() obj.additional_properties.Unpack(conf_msg) confidence = conf_msg.value polygon = [] min_x = float('inf') min_y = float('inf') for v in obj.image_properties.coordinates.vertexes: # If we are rotating the output image, make sure to rotate the bounding box points # as well x, y = rotate_point(v.x, v.y, rotmat) polygon.append([x, y]) min_x = min(min_x, x) min_y = min(min_y, y) polygon = np.array(polygon, np.int32) polygon = polygon.reshape((-1, 1, 2)) cv2.polylines(img, [polygon], True, (0, 255, 0), 2) caption = "{} {:.3f}".format(obj.name, confidence) cv2.putText(img, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) cv2.imwrite('identify_object_output.jpg', img)
def run_model(self, request, this_model): # Define the out proto out_proto = network_compute_bridge_pb2.NetworkComputeResponse() if request.input_data.image.format == image_pb2.Image.FORMAT_RAW: pil_image = Image.open(io.BytesIO(request.input_data.image.data)) pil_image = ndimage.rotate(pil_image, 0) if request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_GREYSCALE_U8: image = cv2.cvtColor(pil_image, cv2.COLOR_GRAY2RGB) # Converted to RGB for Tensorflow elif request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_RGB_U8: # Already in the correct format image = pil_image else: print('Error: image input in unsupported pixel format: ', request.input_data.image.pixel_format) RESPONSE_QUEUE.put(out_proto) return elif request.input_data.image.format == image_pb2.Image.FORMAT_JPEG: dtype = np.uint8 jpg = np.frombuffer(request.input_data.image.data, dtype=dtype) image = cv2.imdecode(jpg, -1) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) if len(image.shape) < 3: # Single channel image, convert to RGB. image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) print('') print('Starting model eval...') # Run the model on the image keras_image = preprocess_image(image) (keras_image, scale) = resize_image(keras_image) keras_image = np.expand_dims(keras_image, axis=0) start = timer() boxes, scores, classes = this_model.predict(keras_image) end = timer() print("Model eval took " + str(end - start) + ' seconds') boxes /= scale # Package detections into the output proto num_objects = 0 boxes = boxes[0] scores = scores[0] classes = classes[0] # Only use the detection w/ the highest confidence for Sensor pointing features max_conf = max(scores) print(f"Max conf was {max_conf}") for i in range(len(boxes)): # Skip if not the Fire Extinguisher class if classes[i] != 7: continue label = this_model.labels[(classes[i])] box = boxes[i] score = scores[i] if score < max_conf: # scores are sorted so we can break continue num_objects += 1 #draw_box(draw, b, color=color) print('Found object with label: "' + label + '" and score: ' + str(score)) print(f"Box is {box}") point1 = np.array([box[0], box[1]]) point2 = np.array([box[0], box[3]]) point3 = np.array([box[2], box[3]]) point4 = np.array([box[2], box[1]]) # Add data to the output proto. out_obj = out_proto.object_in_image.add() out_obj.name = label vertex1 = out_obj.image_properties.coordinates.vertexes.add() vertex1.x = point1[0] vertex1.y = point1[1] vertex2 = out_obj.image_properties.coordinates.vertexes.add() vertex2.x = point2[0] vertex2.y = point2[1] vertex3 = out_obj.image_properties.coordinates.vertexes.add() vertex3.x = point3[0] vertex3.y = point3[1] vertex4 = out_obj.image_properties.coordinates.vertexes.add() vertex4.x = point4[0] vertex4.y = point4[1] # Pack the confidence value. confidence = wrappers_pb2.FloatValue(value=score) out_obj.additional_properties.Pack(confidence) if not self.options.no_debug: polygon = np.array([point1, point2, point3, point4], np.int32) polygon = polygon.reshape((-1, 1, 2)) cv2.polylines(image, [polygon], True, (0, 255, 0), 2) caption = "{}: {:.3f}".format(label, score) left_x = min(point1[0], min(point2[0], min(point3[0], point4[0]))) top_y = min(point1[1], min(point2[1], min(point3[1], point4[1]))) cv2.putText(image, caption, (int(left_x), int(top_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) print('Found ' + str(num_objects) + ' object(s)') if not self.options.no_debug: debug_image_filename = 'retinanet_server_output.jpg' cv2.imwrite(debug_image_filename, cv2.cvtColor(image, cv2.COLOR_RGB2BGR)) print('Wrote debug image output to: "' + debug_image_filename + '"') # Pack all the outputs up and send them back. self.out_queue.put(out_proto)
def process_images(options, model_extension): """Starts Tensorflow and detects objects in the incoming images. """ models = {} for f in os.listdir(options.model_dir): if f in models: print('Warning: duplicate model name of "' + f + '", ignoring second model.') continue path = os.path.join(options.model_dir, f) if os.path.isfile(path) and path.endswith(model_extension): model_name = ''.join(f.rsplit(model_extension, 1)) # remove the extension # reverse replace the extension labels_file = '.csv'.join(f.rsplit(model_extension, 1)) labels_path = os.path.join(options.model_dir, labels_file) if not os.path.isfile(labels_path): labels_path = None models[model_name] = (TensorflowModel(path, labels_path)) # Tensorflow prints out a bunch of stuff, so print out useful data here after a space. print('') print('Running on port: ' + str(options.port)) print('Loaded models:') for model_name in models: if models[model_name].labels is not None: labels_str = 'yes' else: labels_str = 'no' print(' ' + model_name + ' (loaded labels: ' + labels_str + ')') while True: request = REQUEST_QUEUE.get() if isinstance(request, network_compute_bridge_pb2.ListAvailableModelsRequest): out_proto = network_compute_bridge_pb2.ListAvailableModelsResponse( ) for f in models: out_proto.available_models.append(f) RESPONSE_QUEUE.put(out_proto) continue else: out_proto = network_compute_bridge_pb2.NetworkComputeResponse() # Find the model if request.input_data.model_name not in models: err_str = 'Cannot find model "' + request.input_data.model_name + '" in loaded models.' print(err_str) # Set the error in the header. out_proto.header.error.code = header_pb2.CommonError.CODE_INVALID_REQUEST out_proto.header.error.message = err_str RESPONSE_QUEUE.put(out_proto) continue model = models[request.input_data.model_name] if request.input_data.image.format == image_pb2.Image.FORMAT_RAW: pil_image = Image.open(io.BytesIO(request.input_data.image.data)) pil_image = ndimage.rotate(pil_image, 0) if request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_GREYSCALE_U8: image = cv2.cvtColor( pil_image, cv2.COLOR_GRAY2RGB) # Converted to RGB for Tensorflow elif request.input_data.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_RGB_U8: # Already in the correct format image = pil_image else: err_str = 'Error: image input in unsupported pixel format: ', request.input_data.image.pixel_format print(err_str) # Set the error in the header. out_proto.header.error.code = header_pb2.CommonError.CODE_INVALID_REQUEST out_proto.header.error.message = err_str RESPONSE_QUEUE.put(out_proto) continue elif request.input_data.image.format == image_pb2.Image.FORMAT_JPEG: dtype = np.uint8 jpg = np.frombuffer(request.input_data.image.data, dtype=dtype) image = cv2.imdecode(jpg, -1) if len(image.shape) < 3: # Single channel image, convert to RGB. image = cv2.cvtColor(image, cv2.COLOR_GRAY2RGB) boxes, scores, classes, _ = model.predict(image) num_objects = 0 for i in range(len(boxes)): label = str(classes[i]) box = boxes[i] score = scores[i] if score < request.input_data.min_confidence: # scores are sorted so we can break break num_objects += 1 #draw_box(draw, b, color=color) print('Found object with label: "' + label + '" and score: ' + str(score)) point1 = np.array([box[1], box[0]]) point2 = np.array([box[3], box[0]]) point3 = np.array([box[3], box[2]]) point4 = np.array([box[1], box[2]]) # Add data to the output proto. out_obj = out_proto.object_in_image.add() out_obj.name = "obj" + str(num_objects) + "_label_" + label vertex1 = out_obj.image_properties.coordinates.vertexes.add() vertex1.x = point1[0] vertex1.y = point1[1] vertex2 = out_obj.image_properties.coordinates.vertexes.add() vertex2.x = point2[0] vertex2.y = point2[1] vertex3 = out_obj.image_properties.coordinates.vertexes.add() vertex3.x = point3[0] vertex3.y = point3[1] vertex4 = out_obj.image_properties.coordinates.vertexes.add() vertex4.x = point4[0] vertex4.y = point4[1] # Pack the confidence value. confidence = wrappers_pb2.FloatValue(value=score) out_obj.additional_properties.Pack(confidence) if not options.no_debug: polygon = np.array([point1, point2, point3, point4], np.int32) polygon = polygon.reshape((-1, 1, 2)) cv2.polylines(image, [polygon], True, (0, 255, 0), 2) caption = "{}: {:.3f}".format(label, score) left_x = min(point1[0], min(point2[0], min(point3[0], point4[0]))) top_y = min(point1[1], min(point2[1], min(point3[1], point4[1]))) cv2.putText(image, caption, (int(left_x), int(top_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) print('Found ' + str(num_objects) + ' object(s)') if not options.no_debug: debug_image_filename = 'tensorflow_server_output.jpg' cv2.imwrite(debug_image_filename, image) print('Wrote debug image output to: "' + debug_image_filename + '"') RESPONSE_QUEUE.put(out_proto)
def _send_request(server, image_path, model, confidence, verbose=False): start = time.time() channel = grpc.insecure_channel(server) stub = network_compute_bridge_service_pb2_grpc.NetworkComputeBridgeWorkerStub(channel) server_data = network_compute_bridge_pb2.NetworkComputeServerConfiguration(service_name='test') # Read the input image. image_in = cv2.imread(image_path) if image_in is None: print('Error: failed to read "' + image_path + '". Does the file exist?') sys.exit(1) rgb = cv2.cvtColor(image_in, cv2.COLOR_BGR2RGB) success, im_buffer = cv2.imencode(".jpg", rgb) if not success: print('Error: failed to encode input image as a jpg. Abort.') sys.exit(1) height = image_in.shape[0] width = image_in.shape[1] image_proto = image_pb2.Image(format=image_pb2.Image.FORMAT_JPEG, cols=width, rows=height, data=im_buffer.tobytes(), pixel_format=image_pb2.Image.PIXEL_FORMAT_RGB_U8) input_data = network_compute_bridge_pb2.NetworkComputeInputData(image=image_proto, model_name=model, min_confidence=confidence) process_img_req = network_compute_bridge_pb2.NetworkComputeRequest( input_data=input_data, server_config=server_data) response = stub.NetworkCompute(process_img_req) end = time.time() latency = end - start print(f'latency: {latency * 1000} ms') if verbose: if len(response.object_in_image) <= 0: print('No objects found') else: print('Got ' + str(len(response.object_in_image)) + ' objects.') # Draw bounding boxes in the image for all the detections. for obj in response.object_in_image: print(obj) conf_msg = wrappers_pb2.FloatValue() obj.additional_properties.Unpack(conf_msg) confidence = conf_msg.value polygon = [] min_x = float('inf') min_y = float('inf') for v in obj.image_properties.coordinates.vertexes: polygon.append([v.x, v.y]) min_x = min(min_x, v.x) min_y = min(min_y, v.y) polygon = np.array(polygon, np.int32) polygon = polygon.reshape((-1, 1, 2)) cv2.polylines(image_in, [polygon], True, (0, 255, 0), 2) caption = "{} {:.3f}".format(obj.name, confidence) cv2.putText(image_in, caption, (int(min_x), int(min_y)), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) cv2.imwrite(append_str_to_filename(image_path, 'detections'), image_in) return latency
def walk_to_object(config): """Get an image and command the robot to walk up to a selected object. We'll walk "up to" the object, not on top of it. The idea is that you want to interact or manipulate the object.""" # See hello_spot.py for an explanation of these lines. bosdyn.client.util.setup_logging(config.verbose) sdk = bosdyn.client.create_standard_sdk('WalkToObjectClient') robot = sdk.create_robot(config.hostname) robot.authenticate(config.username, config.password) robot.time_sync.wait_for_sync() assert robot.has_arm(), "Robot requires an arm to run this example." # Verify the robot is not estopped and that an external application has registered and holds # an estop endpoint. assert not robot.is_estopped(), "Robot is estopped. Please use an external E-Stop client, " \ "such as the estop SDK example, to configure E-Stop." lease_client = robot.ensure_client( bosdyn.client.lease.LeaseClient.default_service_name) image_client = robot.ensure_client(ImageClient.default_service_name) manipulation_api_client = robot.ensure_client( ManipulationApiClient.default_service_name) lease = lease_client.acquire() try: with bosdyn.client.lease.LeaseKeepAlive(lease_client): # Now, we are ready to power on the robot. This call will block until the power # is on. Commands would fail if this did not happen. We can also check that the robot is # powered at any point. robot.logger.info( "Powering on robot... This may take a several seconds.") robot.power_on(timeout_sec=20) assert robot.is_powered_on(), "Robot power on failed." robot.logger.info("Robot powered on.") # Tell the robot to stand up. The command service is used to issue commands to a robot. # The set of valid commands for a robot depends on hardware configuration. See # SpotCommandHelper for more detailed examples on command building. The robot # command service requires timesync between the robot and the client. robot.logger.info("Commanding robot to stand...") command_client = robot.ensure_client( RobotCommandClient.default_service_name) blocking_stand(command_client, timeout_sec=10) robot.logger.info("Robot standing.") time.sleep(2.0) # Take a picture with a camera robot.logger.info('Getting an image from: ' + config.image_source) image_responses = image_client.get_image_from_sources( [config.image_source]) if len(image_responses) != 1: print('Got invalid number of images: ' + str(len(image_responses))) print(image_responses) assert False image = image_responses[0] if image.shot.image.pixel_format == image_pb2.Image.PIXEL_FORMAT_DEPTH_U16: dtype = np.uint16 else: dtype = np.uint8 img = np.fromstring(image.shot.image.data, dtype=dtype) if image.shot.image.format == image_pb2.Image.FORMAT_RAW: img = img.reshape(image.shot.image.rows, image.shot.image.cols) else: img = cv2.imdecode(img, -1) # Show the image to the user and wait for them to click on a pixel robot.logger.info('Click on an object to walk up to...') image_title = 'Click to walk up to something' cv2.namedWindow(image_title) cv2.setMouseCallback(image_title, cv_mouse_callback) global g_image_click, g_image_display g_image_display = img cv2.imshow(image_title, g_image_display) while g_image_click is None: key = cv2.waitKey(1) & 0xFF if key == ord('q') or key == ord('Q'): # Quit print('"q" pressed, exiting.') exit(0) robot.logger.info('Walking to object at image location (' + str(g_image_click[0]) + ', ' + str(g_image_click[1]) + ')') walk_vec = geometry_pb2.Vec2(x=g_image_click[0], y=g_image_click[1]) # Optionally populate the offset distance parameter. if config.distance is None: offset_distance = None else: offset_distance = wrappers_pb2.FloatValue( value=config.distance) # Build the proto walk_to = manipulation_api_pb2.WalkToObjectInImage( pixel_xy=walk_vec, transforms_snapshot_for_camera=image.shot.transforms_snapshot, frame_name_image_sensor=image.shot.frame_name_image_sensor, camera_model=image.source.pinhole, offset_distance=offset_distance) # Ask the robot to pick up the object walk_to_request = manipulation_api_pb2.ManipulationApiRequest( walk_to_object_in_image=walk_to) # Send the request cmd_response = manipulation_api_client.manipulation_api_command( manipulation_api_request=walk_to_request) # Get feedback from the robot while True: time.sleep(0.25) feedback_request = manipulation_api_pb2.ManipulationApiFeedbackRequest( manipulation_cmd_id=cmd_response.manipulation_cmd_id) # Send the request response = manipulation_api_client.manipulation_api_feedback_command( manipulation_api_feedback_request=feedback_request) print( 'Current state: ', manipulation_api_pb2.ManipulationFeedbackState.Name( response.current_state)) if response.current_state == manipulation_api_pb2.MANIP_STATE_DONE: break robot.logger.info('Finished.') time.sleep(4.0) robot.logger.info('Sitting down and turning off.') # Power the robot off. By specifying "cut_immediately=False", a safe power off command # is issued to the robot. This will attempt to sit the robot before powering off. robot.power_off(cut_immediately=False, timeout_sec=20) assert not robot.is_powered_on(), "Robot power off failed." robot.logger.info("Robot safely powered off.") finally: # If we successfully acquired a lease, return it. lease_client.return_lease(lease)