def route_cmd(self, command, context): stub = None try: route = self._routes[command.destination] stub = self._children_stubs[route] logging.debug("Routing for node %d through child %d", command.destination, route) except KeyError: logging.debug("Routing for node %d through parent", command.destination) stub = self._parent_stub if stub is None: logging.info("Bad stub for %d from %d", command.destination, self._vmid) return agent_pb2.RouteMessageResult( source=command.destination, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.GenericError, description='No route for vm{}'.format( command.destination))) try: # Route the command by executing a RPC on the next # hop. Cancel that RPC if the Route RPC from our client is # cancelled or timeouts future = stub.route_command.future(command, context.time_remaining()) context.add_callback(future.cancel) return future.result() except grpc.RpcError as e: if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED: # Timeouts should be coherent along a route so the parent should timeout # concurrently but we still return something just in case return agent_pb2.RouteMessageResult( source=command.destination, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.Timeout, description="Agent did not answer before time limit")) else: return agent_pb2.RouteMessageResult( source=command.destination, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.GenericError, description=str(e))) except grpc.FutureCancelledError as e: # Again should not be necessary as the parent should be # cancelled too but just in case return agent_pb2.RouteMessageResult( source=command.destination, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.Cancelled, description="Route request cancelled"))
def route_command(self, request, context): if not self._ready.wait(60): logging.error("Timeout while establishing tree relay") return agent_pb2.RouteMessageResult(error=agent_pb2.GenericError( kind=agent_pb2.GenericError.TimeoutError, description="Timeout while establishing tree relay")) if request.destination == self._vmid: resp = self._process_local(request, context) else: resp = self._relay.route_cmd(request, context) return resp
def route_stream(self, rng, init_cmd, stream_cmd, msg_iterator, cancel_cb=None): def route_iterator(): cmd = init_cmd for msg in msg_iterator: grpc_message = agent_pb2.McastMessage(destinations=str(rng), name=cmd) grpc_message.args.Pack(msg) yield grpc_message cmd = stream_cmd try: res = self._stub.route_stream(route_iterator()) if cancel_cb: res.add_callback(cancel_cb) def result_unpacker(): try: for r in res: try: logging.debug( "Stream client: unpacking a result %s", r) yield r.source, self._handle_route_result( init_cmd, r) except PcoccError as e: yield r.source, e except grpc.RpcError as e: logging.error( "Stream client interrupted due to GRPC error") yield -1, self._handle_route_result( init_cmd, self._handle_grpc_error(e, -1)) logging.debug("Stream client: No more results to unpack") return result_unpacker(), res except Exception as e: #FIXME: we should probably generate a more informative error return [ self._handle_route_result( init_cmd, agent_pb2.RouteMessageResult( source=-1, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.GenericError, description="Unable to establish stream: {}". format(e)))) ]
def _handle_grpc_error(e, source): if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED: ex = agent_pb2.RouteMessageResult( source=source, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.Timeout, description="Timeout while waiting for agent to answer")) elif e.code() == grpc.StatusCode.CANCELLED: ex = agent_pb2.RouteMessageResult( source=source, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.Cancelled, description="RPC was cancelled")) else: logging.warning("RPC request failed with: %s", e.details()) ex = agent_pb2.RouteMessageResult( source=source, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.GenericError, description="Transport error while " "relaying command: {}".format(e.details()))) return ex
def command(self, dest, cmd, data, timeout): logging.info("sending %s to %d", cmd, dest) try: grpc_message = agent_pb2.RouteMessage(destination=dest, name=cmd) grpc_message.args.Pack(data) except Exception as e: return self._handle_route_result( cmd, agent_pb2.RouteMessageResult( source=dest, error=agent_pb2.GenericError( kind=agent_pb2.GenericError.PayloadError, description="Unable to create message " "with payload: {}".format(e)))) try: res = self._stub.route_command(grpc_message, timeout=timeout) except grpc.RpcError as e: res = self._handle_grpc_error(e, dest) return self._handle_route_result(cmd, res)