def __init__( self, address: Address, host: str, port: int, api_spec_path: Optional[str], connection_id: PublicId, restricted_to_protocols: Set[PublicId], timeout_window: float = 5.0, logger: logging.Logger = _default_logger, ): """ Initialize a channel and process the initial API specification from the file path (if given). :param address: the address of the agent. :param host: RESTful API hostname / IP address :param port: RESTful API port number :param api_spec_path: Directory API path and filename of the API spec YAML source file. :param connection_id: public id of connection using this chanel. :param restricted_to_protocols: set of restricted protocols :param timeout_window: the timeout (in seconds) for a request to be handled. """ super().__init__(address=address, connection_id=connection_id) self.host = host self.port = port self.server_address = "http://{}:{}".format(self.host, self.port) self.restricted_to_protocols = restricted_to_protocols self._api_spec = APISpec(api_spec_path, self.server_address, logger) self.timeout_window = timeout_window self.http_server: Optional[web.TCPSite] = None self.pending_requests: Dict[RequestId, Future] = {} self._dialogues = HttpDialogues(self.address) self.logger = logger
def setup_client(self): """Set up client connection.""" self.client_agent_address = "client_agent_address" self.client_agent_identity = Identity( "agent running client", address=self.client_agent_address) configuration = ConnectionConfig( host="localost", port="8888", # TODO: remove host/port for client? connection_id=HTTPClientConnection.connection_id, ) self.client = HTTPClientConnection(configuration=configuration, identity=self.client_agent_identity) self.loop.run_until_complete(self.client.connect()) # skill side dialogues def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ return HttpDialogue.Role.CLIENT self._client_dialogues = HttpDialogues( self.client_agent_address, role_from_first_message=role_from_first_message)
def setup(self): """Initialise the test case.""" self.identity = Identity("name", address="my_key") self.agent_address = self.identity.address self.host = get_host() self.port = get_unused_tcp_port() self.api_spec_path = os.path.join( ROOT_DIR, "tests", "data", "petstore_sim.yaml" ) self.connection_id = HTTPServerConnection.connection_id self.protocol_id = PublicId.from_str("fetchai/http:0.4.0") self.configuration = ConnectionConfig( host=self.host, port=self.port, api_spec_path=self.api_spec_path, connection_id=HTTPServerConnection.connection_id, restricted_to_protocols=set([self.protocol_id]), ) self.http_connection = HTTPServerConnection( configuration=self.configuration, identity=self.identity, ) self.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.http_connection.connect()) self.connection_address = str(HTTPServerConnection.connection_id) self._dialogues = HttpDialogues(self.connection_address) self.original_timeout = self.http_connection.channel.RESPONSE_TIMEOUT
def __init__(self, self_address: Address, **kwargs: Any) -> None: """ Initialize dialogues. :return: None """ def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address ) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ # The server connection maintains the dialogue on behalf of the client return HttpDialogue.Role.CLIENT BaseHttpDialogues.__init__( self, self_address=self_address, role_from_first_message=role_from_first_message, **kwargs, )
def __init__(self) -> None: """ Initialize dialogues. :return: None """ def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address ) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ # The client connection maintains the dialogue on behalf of the server return HttpDialogue.Role.SERVER BaseHttpDialogues.__init__( self, self_address=str(HTTPClientConnection.connection_id), role_from_first_message=role_from_first_message, dialogue_class=HttpDialogue, )
def setup_server(self): """Set up server connection.""" self.server_agent_address = "server_agent_address" self.server_agent_identity = Identity( "agent running server", address=self.server_agent_address ) self.host = get_host() self.port = get_unused_tcp_port() self.connection_id = HTTPServerConnection.connection_id self.protocol_id = PublicId.from_str("fetchai/http:0.4.0") self.configuration = ConnectionConfig( host=self.host, port=self.port, api_spec_path=None, # do not filter on API spec connection_id=HTTPServerConnection.connection_id, restricted_to_protocols=set([self.protocol_id]), ) self.server = HTTPServerConnection( configuration=self.configuration, identity=self.server_agent_identity, ) self.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.server.connect()) # skill side dialogues self._server_dialogues = HttpDialogues(self.server_agent_address)
def __init__( self, agent_address: Address, webhook_address: Address, webhook_port: int, webhook_url_path: str, connection_id: PublicId, logger: logging.Logger = _default_logger, ): """ Initialize a webhook channel. :param agent_address: the address of the agent :param webhook_address: webhook hostname / IP address :param webhook_port: webhook port number :param webhook_url_path: the url path to receive webhooks from :param connection_id: the connection id """ self.agent_address = agent_address self.webhook_address = webhook_address self.webhook_port = webhook_port self.webhook_url_path = webhook_url_path self.webhook_site = None # type: Optional[web.TCPSite] self.runner = None # type: Optional[web.AppRunner] self.app = None # type: Optional[web.Application] self.is_stopped = True self.connection_id = connection_id self.in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover self.logger = logger self.logger.info("Initialised a webhook channel") self._dialogues = HttpDialogues(str(WebhookConnection.connection_id))
def __init__(self, **kwargs: Any) -> None: """ Initialize dialogues. :param agent_address: the address of the agent for whom dialogues are maintained :return: None """ Model.__init__(self, **kwargs) def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent in this dialogue """ if (message.performative == HttpMessage.Performative.REQUEST and message.sender != receiver_address ) or (message.performative == HttpMessage.Performative.RESPONSE and message.sender == receiver_address): return BaseHttpDialogue.Role.SERVER return BaseHttpDialogue.Role.CLIENT BaseHttpDialogues.__init__( self, self_address=str(self.skill_id), role_from_first_message=role_from_first_message, )
def __init__(self, **kwargs: Any) -> None: """ Initialize dialogues. :param agent_address: the address of the agent for whom dialogues are maintained :return: None """ Model.__init__(self, **kwargs) def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ return BaseHttpDialogue.Role.CLIENT BaseHttpDialogues.__init__( self, self_address=self.context.agent_address, role_from_first_message=role_from_first_message, )
def __init__(self, **kwargs) -> None: """ Initialize dialogues. :return: None """ Model.__init__(self, **kwargs) BaseHttpDialogues.__init__(self, self.context.agent_address)
def __init__(self, **kwargs) -> None: """ Initialize dialogues. :param agent_address: the address of the agent for whom dialogues are maintained :return: None """ Model.__init__(self, **kwargs) BaseHttpDialogues.__init__(self, self.context.agent_address)
def __init__(self): """Set dialogues.""" # pylint: disable=unused-argument def role(m: Message, addr: Address) -> Dialogue.Role: return HttpDialogue.Role.CLIENT self.addr = self.random_string self.dialogues = HttpDialogues(self.addr, role_from_first_message=role)
def setup(self) -> None: """Noop setup.""" # pylint: disable=attribute-defined-outside-init, unused-argument self.count: int = 0 self.rtt_total_time: float = 0.0 self.rtt_count: int = 0 self.latency_total_time: float = 0.0 self.latency_count: int = 0 def role(m: Message, addr: Address) -> Dialogue.Role: return HttpDialogue.Role.CLIENT self.dialogues = HttpDialogues(self.context.agent_address, role_from_first_message=role)
def setup(self): """Initialise the class.""" self.address = get_host() self.port = get_unused_tcp_port() self.agent_identity = Identity("name", address="some string") self.agent_address = self.agent_identity.address configuration = ConnectionConfig( host=self.address, port=self.port, connection_id=HTTPClientConnection.connection_id, ) self.http_client_connection = HTTPClientConnection( configuration=configuration, identity=self.agent_identity) self.http_client_connection.loop = asyncio.get_event_loop() self.connection_address = str(HTTPClientConnection.connection_id) self.http_dialogs = HttpDialogues(self.connection_address)
def setup_client(self): """Set up client connection.""" self.client_agent_address = "client_agent_address" self.client_agent_identity = Identity( "agent running client", address=self.client_agent_address ) configuration = ConnectionConfig( host="localost", port="8888", # TODO: remove host/port for client? connection_id=HTTPClientConnection.connection_id, ) self.client = HTTPClientConnection( configuration=configuration, identity=self.client_agent_identity ) self.client.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.client.connect()) # skill side dialogues self._client_dialogues = HttpDialogues(self.client_agent_address)
class DialogueHandler: """Generate messages and process with dialogues.""" def __init__(self): """Set dialogues.""" # pylint: disable=unused-argument def role(m: Message, addr: Address) -> Dialogue.Role: return HttpDialogue.Role.CLIENT self.addr = self.random_string self.dialogues = HttpDialogues(self.addr, role_from_first_message=role) @property def random_string(self) -> str: """Get random string on every access.""" return uuid.uuid4().hex def process_message(self) -> None: """Process a message with dialogues.""" message = self.create() dialogue = self.update(message) self.reply(dialogue, message) def update(self, message: HttpMessage) -> HttpDialogue: """Update dialogues with message.""" return cast(HttpDialogue, self.dialogues.update(message)) @staticmethod def reply(dialogue: HttpDialogue, message: HttpMessage): """Construct and send a response for message received.""" return dialogue.reply( target_message=message, performative=HttpMessage.Performative.RESPONSE, version=message.version, headers="", status_code=200, status_text="Success", body=message.body, ) def create(self) -> HttpMessage: """Make initial http request.""" message = HttpMessage( dialogue_reference=HttpDialogues. new_self_initiated_dialogue_reference(), performative=HttpMessage.Performative.REQUEST, method="get", url="some url", headers="", version="", body=b"", ) message.sender = self.random_string message.to = self.addr return message
def setup(self): """Initialise the class.""" self.host = get_host() self.port = get_unused_tcp_port() self.identity = Identity("", address="some string") self.path = "/webhooks/topic/{topic}/" self.loop = asyncio.get_event_loop() configuration = ConnectionConfig( webhook_address=self.host, webhook_port=self.port, webhook_url_path=self.path, connection_id=WebhookConnection.connection_id, ) self.webhook_connection = WebhookConnection( configuration=configuration, identity=self.identity, ) self.webhook_connection.loop = self.loop self.dialogues = HttpDialogues(self.identity.address)
def to_envelope_and_set_id( self, connection_id: PublicId, agent_address: str, dialogues: HttpDialogues, ) -> Envelope: """ Process incoming API request by packaging into Envelope and sending it in-queue. :param connection_id: id of the connection :param agent_address: agent's address :param dialogue_reference: new dialog refernece for envelope :return: envelope """ url = (self.full_url_pattern if self.parameters.query == {} else self.full_url_pattern + "?" + urlencode(self.parameters.query)) uri = URI(self.full_url_pattern) context = EnvelopeContext(connection_id=connection_id, uri=uri) http_message = HttpMessage( dialogue_reference=dialogues.new_self_initiated_dialogue_reference( ), performative=HttpMessage.Performative.REQUEST, method=self.method, url=url, headers=self.parameters.header, bodyy=self.body if self.body is not None else b"", version="", ) http_message.counterparty = agent_address dialogue = cast(Optional[HttpDialogue], dialogues.update(http_message)) assert dialogue is not None, "Could not create dialogue for message={}".format( http_message) self.id = dialogue.incomplete_dialogue_label envelope = Envelope( to=agent_address, sender=str(connection_id), protocol_id=http_message.protocol_id, context=context, message=http_message, ) return envelope
def __init__(self, self_address: Address, **kwargs) -> None: """ Initialize dialogues. :return: None """ def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> Dialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ return HttpDialogue.Role.CLIENT BaseHttpDialogues.__init__( self, self_address=self_address, role_from_first_message=role_from_first_message, )
def setup_server(self): """Set up server connection.""" self.server_agent_address = "server_agent_address" self.server_agent_identity = Identity( "agent running server", address=self.server_agent_address) self.host = get_host() self.port = get_unused_tcp_port() self.connection_id = HTTPServerConnection.connection_id self.protocol_id = HttpMessage.protocol_id self.configuration = ConnectionConfig( host=self.host, port=self.port, api_spec_path=None, # do not filter on API spec connection_id=HTTPServerConnection.connection_id, restricted_to_protocols=set([self.protocol_id]), ) self.server = HTTPServerConnection( configuration=self.configuration, identity=self.server_agent_identity, ) self.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.server.connect()) # skill side dialogues def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ return HttpDialogue.Role.SERVER self._server_dialogues = HttpDialogues( self.server_agent_address, role_from_first_message=role_from_first_message)
def __init__( self, agent_address: Address, address: str, port: int, connection_id: PublicId, excluded_protocols: Optional[Set[PublicId]] = None, restricted_to_protocols: Optional[Set[PublicId]] = None, ): """ Initialize an http client channel. :param agent_address: the address of the agent. :param address: server hostname / IP address :param port: server port number :param excluded_protocols: this connection cannot handle messages adhering to any of the protocols in this set :param restricted_to_protocols: this connection can only handle messages adhering to protocols in this set """ self.agent_address = agent_address self.address = address self.port = port self.connection_id = connection_id self.restricted_to_protocols = restricted_to_protocols self._dialogues = HttpDialogues(str( HTTPClientConnection.connection_id)) self._in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover self._loop = ( None ) # type: Optional[asyncio.AbstractEventLoop] # pragma: no cover self.excluded_protocols = excluded_protocols self.is_stopped = True self._tasks: Set[Task] = set() self.logger = logger self.logger.info("Initialised the HTTP client channel")
class TestHTTPClientConnect: """Tests the http client connection's 'connect' functionality.""" def setup(self): """Initialise the class.""" self.address = get_host() self.port = get_unused_tcp_port() self.agent_identity = Identity("name", address="some string") self.agent_address = self.agent_identity.address configuration = ConnectionConfig( host=self.address, port=self.port, connection_id=HTTPClientConnection.connection_id, ) self.http_client_connection = HTTPClientConnection( configuration=configuration, identity=self.agent_identity) self.http_client_connection.loop = asyncio.get_event_loop() self.connection_address = str(HTTPClientConnection.connection_id) self.http_dialogs = HttpDialogues(self.connection_address) @pytest.mark.asyncio async def test_initialization(self): """Test the initialisation of the class.""" assert self.http_client_connection.address == self.agent_identity.address @pytest.mark.asyncio async def test_connection(self): """Test the connect functionality of the http client connection.""" await self.http_client_connection.connect() assert self.http_client_connection.is_connected is True @pytest.mark.asyncio async def test_disconnect(self): """Test the disconnect functionality of the http client connection.""" await self.http_client_connection.connect() assert self.http_client_connection.is_connected is True await self.http_client_connection.disconnect() assert self.http_client_connection.is_connected is False @pytest.mark.asyncio async def test_http_send_error(self): """Test request fails and send back result with code 600.""" await self.http_client_connection.connect() request_http_message = HttpMessage( dialogue_reference=self.http_dialogs. new_self_initiated_dialogue_reference(), performative=HttpMessage.Performative.REQUEST, method="get", url="bad url", headers="", version="", bodyy=b"", ) request_http_message.counterparty = self.connection_address sending_dialogue = self.http_dialogs.update(request_http_message) assert sending_dialogue is not None request_envelope = Envelope( to=self.connection_address, sender=self.agent_address, protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID, message=request_http_message, ) connection_response_mock = Mock() connection_response_mock.status_code = 200 await self.http_client_connection.send(envelope=request_envelope) # TODO: Consider returning the response from the server in order to be able to assert that the message send! envelope = await asyncio.wait_for( self.http_client_connection.receive(), timeout=10) assert envelope assert envelope.message.status_code == 600 await self.http_client_connection.disconnect() @pytest.mark.asyncio async def test_http_client_send_not_connected_error(self): """Test connection.send error if not conencted.""" with pytest.raises(ConnectionError): await self.http_client_connection.send(Mock()) @pytest.mark.asyncio async def test_http_channel_send_not_connected_error(self): """Test channel.send error if not conencted.""" with pytest.raises(ValueError): self.http_client_connection.channel.send(Mock()) @pytest.mark.asyncio async def test_send_envelope_excluded_protocol_fail(self): """Test send error if protocol not supported.""" request_http_message = HttpMessage( dialogue_reference=self.http_dialogs. new_self_initiated_dialogue_reference(), performative=HttpMessage.Performative.REQUEST, method="get", url="bad url", headers="", version="", bodyy=b"", ) request_http_message.counterparty = self.connection_address sending_dialogue = self.http_dialogs.update(request_http_message) assert sending_dialogue is not None request_envelope = Envelope( to=self.connection_address, sender=self.agent_address, protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID, message=request_http_message, ) await self.http_client_connection.connect() with patch.object( self.http_client_connection.channel, "excluded_protocols", new=[UNKNOWN_PROTOCOL_PUBLIC_ID], ): with pytest.raises(ValueError): await self.http_client_connection.send(request_envelope) @pytest.mark.asyncio async def test_send_empty_envelope_skip(self): """Test skip on empty envelope request sent.""" await self.http_client_connection.connect() with patch.object(self.http_client_connection.channel, "_http_request_task") as mock: await self.http_client_connection.send(None) mock.assert_not_called() @pytest.mark.asyncio async def test_channel_get_message_not_connected(self): """Test errro on message get if not connected.""" with pytest.raises(ValueError): await self.http_client_connection.channel.get_message() @pytest.mark.asyncio async def test_channel_cancel_tasks_on_disconnect(self): """Test requests tasks cancelled on disconnect.""" await self.http_client_connection.connect() request_http_message = HttpMessage( dialogue_reference=self.http_dialogs. new_self_initiated_dialogue_reference(), performative=HttpMessage.Performative.REQUEST, method="get", url="https://not-a-google.com", headers="", version="", bodyy=b"", ) request_http_message.counterparty = self.connection_address sending_dialogue = self.http_dialogs.update(request_http_message) assert sending_dialogue is not None request_envelope = Envelope( to=self.connection_address, sender=self.agent_address, protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID, message=request_http_message, ) connection_response_mock = Mock() connection_response_mock.status_code = 200 response_mock = Mock() response_mock.status = 200 response_mock.headers = {"headers": "some header"} response_mock.reason = "OK" response_mock._body = b"Some content" response_mock.read.return_value = asyncio.Future() with patch.object( aiohttp.ClientSession, "request", return_value=_MockRequest(response_mock), ): await self.http_client_connection.send(envelope=request_envelope) assert self.http_client_connection.channel._tasks task = list(self.http_client_connection.channel._tasks)[0] assert not task.done() await self.http_client_connection.disconnect() assert not self.http_client_connection.channel._tasks assert task.done() with pytest.raises(CancelledError): await task @pytest.mark.asyncio async def test_http_send_ok(self): """Test request is ok cause mocked.""" await self.http_client_connection.connect() request_http_message = HttpMessage( dialogue_reference=self.http_dialogs. new_self_initiated_dialogue_reference(), performative=HttpMessage.Performative.REQUEST, method="get", url="https://not-a-google.com", headers="", version="", bodyy=b"", ) request_http_message.counterparty = self.connection_address sending_dialogue = self.http_dialogs.update(request_http_message) assert sending_dialogue is not None request_envelope = Envelope( to=self.connection_address, sender=self.agent_address, protocol_id=request_http_message.protocol_id, message=request_http_message, ) connection_response_mock = Mock() connection_response_mock.status_code = 200 response_mock = Mock() response_mock.status = 200 response_mock.headers = {"headers": "some header"} response_mock.reason = "OK" response_mock._body = b"Some content" response_mock.read.return_value = asyncio.Future() response_mock.read.return_value.set_result("") with patch.object( aiohttp.ClientSession, "request", return_value=_MockRequest(response_mock), ): await self.http_client_connection.send(envelope=request_envelope) # TODO: Consider returning the response from the server in order to be able to assert that the message send! envelope = await asyncio.wait_for( self.http_client_connection.receive(), timeout=10) assert envelope is not None and envelope.message is not None response = copy.copy(envelope.message) response.is_incoming = True response.counterparty = envelope.message.sender response_dialogue = self.http_dialogs.update(response) assert response.status_code == response_mock.status, response.bodyy.decode( "utf-8") assert sending_dialogue == response_dialogue await self.http_client_connection.disconnect() @pytest.mark.asyncio async def test_http_dialogue_construct_fail(self): """Test dialogue not properly constructed.""" await self.http_client_connection.connect() http_message = HttpMessage( dialogue_reference=self.http_dialogs. new_self_initiated_dialogue_reference(), performative=HttpMessage.Performative.RESPONSE, status_code=500, headers="", status_text="", bodyy=b"", version="", ) http_message.counterparty = self.connection_address http_dialogue = self.http_dialogs.update(http_message) http_message.sender = self.agent_address assert http_dialogue is None envelope = Envelope( to=http_message.counterparty, sender=http_message.sender, protocol_id=http_message.protocol_id, message=http_message, ) with patch.object(self.http_client_connection.channel.logger, "warning") as mock_logger: await self.http_client_connection.channel._http_request_task( envelope) mock_logger.assert_any_call( AnyStringWith("Could not create dialogue for message="))
class TestClientServer: """Client-Server end-to-end test.""" def setup_server(self): """Set up server connection.""" self.server_agent_address = "server_agent_address" self.server_agent_identity = Identity( "agent running server", address=self.server_agent_address ) self.host = get_host() self.port = get_unused_tcp_port() self.connection_id = HTTPServerConnection.connection_id self.protocol_id = PublicId.from_str("fetchai/http:0.4.0") self.configuration = ConnectionConfig( host=self.host, port=self.port, api_spec_path=None, # do not filter on API spec connection_id=HTTPServerConnection.connection_id, restricted_to_protocols=set([self.protocol_id]), ) self.server = HTTPServerConnection( configuration=self.configuration, identity=self.server_agent_identity, ) self.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.server.connect()) # skill side dialogues self._server_dialogues = HttpDialogues(self.server_agent_address) def setup_client(self): """Set up client connection.""" self.client_agent_address = "client_agent_address" self.client_agent_identity = Identity( "agent running client", address=self.client_agent_address ) configuration = ConnectionConfig( host="localost", port="8888", # TODO: remove host/port for client? connection_id=HTTPClientConnection.connection_id, ) self.client = HTTPClientConnection( configuration=configuration, identity=self.client_agent_identity ) self.client.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.client.connect()) # skill side dialogues self._client_dialogues = HttpDialogues(self.client_agent_address) def setup(self): """Set up test case.""" self.setup_server() self.setup_client() def _make_request( self, path: str, method: str = "get", headers: str = "", bodyy: bytes = b"" ) -> Envelope: """Make request envelope.""" request_http_message = HttpMessage( dialogue_reference=self._client_dialogues.new_self_initiated_dialogue_reference(), target=0, message_id=1, performative=HttpMessage.Performative.REQUEST, method=method, url=f"http://{self.host}:{self.port}{path}", headers="", version="", bodyy=b"", ) request_http_message.counterparty = str(HTTPClientConnection.connection_id) assert self._client_dialogues.update(request_http_message) is not None request_envelope = Envelope( to=request_http_message.counterparty, sender=request_http_message.sender, protocol_id=request_http_message.protocol_id, message=request_http_message, ) return request_envelope def _make_response( self, request_envelope: Envelope, status_code: int = 200, status_text: str = "" ) -> Envelope: """Make response envelope.""" incoming_message = cast(HttpMessage, request_envelope.message) incoming_message.is_incoming = True incoming_message.counterparty = str(HTTPServerConnection.connection_id) dialogue = self._server_dialogues.update(incoming_message) assert dialogue is not None message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, dialogue_reference=dialogue.dialogue_label.dialogue_reference, target=incoming_message.message_id, message_id=incoming_message.message_id + 1, version=incoming_message.version, headers=incoming_message.headers, status_code=status_code, status_text=status_text, bodyy=incoming_message.bodyy, ) message.counterparty = incoming_message.counterparty assert dialogue.update(message) is not None response_envelope = Envelope( to=message.counterparty, sender=message.sender, protocol_id=message.protocol_id, context=request_envelope.context, message=message, ) return response_envelope @pytest.mark.asyncio async def test_post_with_payload(self): """Test client and server with post request.""" initial_request = self._make_request("/test", "POST", bodyy=b"1234567890") await self.client.send(initial_request) request = await asyncio.wait_for(self.server.receive(), timeout=5) # this is "inside" the server agent initial_response = self._make_response(request) await self.server.send(initial_response) response = await asyncio.wait_for(self.client.receive(), timeout=5) assert ( cast(HttpMessage, initial_request.message).bodyy == cast(HttpMessage, response.message).bodyy ) assert ( initial_request.message.dialogue_reference[0] == response.message.dialogue_reference[0] ) def teardown(self): """Tear down testcase.""" self.loop.run_until_complete(self.client.disconnect()) self.loop.run_until_complete(self.server.disconnect())
class WebhookChannel: """A wrapper for a Webhook.""" def __init__( self, agent_address: Address, webhook_address: Address, webhook_port: int, webhook_url_path: str, connection_id: PublicId, logger: logging.Logger = _default_logger, ): """ Initialize a webhook channel. :param agent_address: the address of the agent :param webhook_address: webhook hostname / IP address :param webhook_port: webhook port number :param webhook_url_path: the url path to receive webhooks from :param connection_id: the connection id """ self.agent_address = agent_address self.webhook_address = webhook_address self.webhook_port = webhook_port self.webhook_url_path = webhook_url_path self.webhook_site = None # type: Optional[web.TCPSite] self.runner = None # type: Optional[web.AppRunner] self.app = None # type: Optional[web.Application] self.is_stopped = True self.connection_id = connection_id self.in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover self.logger = logger self.logger.info("Initialised a webhook channel") self._dialogues = HttpDialogues(str(WebhookConnection.connection_id)) async def connect(self) -> None: """ Connect the webhook. Connects the webhook via the webhook_address and webhook_port parameters :return: None """ if self.is_stopped: self.app = web.Application() self.app.add_routes( [web.post(self.webhook_url_path, self._receive_webhook)]) self.runner = web.AppRunner(self.app) await self.runner.setup() self.webhook_site = web.TCPSite(self.runner, self.webhook_address, self.webhook_port) await self.webhook_site.start() self.is_stopped = False async def disconnect(self) -> None: """ Disconnect. Shut-off and cleanup the webhook site, the runner and the web app, then stop the channel. :return: None """ assert (self.webhook_site is not None and self.runner is not None and self.app is not None), "Application not connected, call connect first!" if not self.is_stopped: await self.webhook_site.stop() await self.runner.shutdown() await self.runner.cleanup() await self.app.shutdown() await self.app.cleanup() self.logger.info("Webhook app is shutdown.") self.is_stopped = True async def _receive_webhook(self, request: web.Request) -> web.Response: """ Receive a webhook request. Get webhook request, turn it to envelop and send it to the agent to be picked up. :param request: the webhook request :return: Http response with a 200 code """ webhook_envelop = await self.to_envelope(request) self.in_queue.put_nowait(webhook_envelop) # type: ignore return web.Response(status=200) async def send(self, envelope: Envelope) -> None: """ Send an envelope. Sending envelopes via the webhook is not possible! :param envelope: the envelope """ self.logger.warning( "Dropping envelope={} as sending via the webhook is not possible!". format(envelope)) async def to_envelope(self, request: web.Request) -> Envelope: """ Convert a webhook request object into an Envelope containing an HttpMessage `from the 'http' Protocol`. :param request: the webhook request :return: The envelop representing the webhook request """ payload_bytes = await request.read() version = str(request.version[0]) + "." + str(request.version[1]) context = EnvelopeContext(uri=URI("aea/mail/base.py")) http_message = HttpMessage( performative=HttpMessage.Performative.REQUEST, method=request.method, url=str(request.url), version=version, headers=json.dumps(dict(request.headers)), bodyy=payload_bytes if payload_bytes is not None else b"", dialogue_reference=self._dialogues. new_self_initiated_dialogue_reference(), ) http_message.counterparty = self.agent_address http_dialogue = self._dialogues.update(http_message) assert http_dialogue is not None, "Could not create dialogue." envelope = Envelope( to=http_message.counterparty, sender=http_message.sender, protocol_id=http_message.protocol_id, context=context, message=http_message, ) return envelope
class HTTPChannel(BaseAsyncChannel): """A wrapper for an RESTful API with an internal HTTPServer.""" RESPONSE_TIMEOUT = 300 def __init__( self, address: Address, host: str, port: int, api_spec_path: Optional[str], connection_id: PublicId, restricted_to_protocols: Set[PublicId], timeout_window: float = 5.0, logger: logging.Logger = _default_logger, ): """ Initialize a channel and process the initial API specification from the file path (if given). :param address: the address of the agent. :param host: RESTful API hostname / IP address :param port: RESTful API port number :param api_spec_path: Directory API path and filename of the API spec YAML source file. :param connection_id: public id of connection using this chanel. :param restricted_to_protocols: set of restricted protocols :param timeout_window: the timeout (in seconds) for a request to be handled. """ super().__init__(address=address, connection_id=connection_id) self.host = host self.port = port self.server_address = "http://{}:{}".format(self.host, self.port) self.restricted_to_protocols = restricted_to_protocols self._api_spec = APISpec(api_spec_path, self.server_address, logger) self.timeout_window = timeout_window self.http_server: Optional[web.TCPSite] = None self.pending_requests: Dict[RequestId, Future] = {} self._dialogues = HttpDialogues(self.address) self.logger = logger @property def api_spec(self) -> APISpec: """Get the api spec.""" return self._api_spec async def connect(self, loop: AbstractEventLoop) -> None: """ Connect. Upon HTTP Channel connection, kickstart the HTTP Server in its own thread. :param loop: asyncio event loop :return: None """ if self.is_stopped: await super().connect(loop) try: await self._start_http_server() self.logger.info( "HTTP Server has connected to port: {}.".format(self.port)) except Exception: # pragma: nocover # pylint: disable=broad-except self.is_stopped = True self._in_queue = None self.logger.exception( "Failed to start server on {}:{}.".format( self.host, self.port)) async def _http_handler(self, http_request: BaseRequest) -> Response: """ Verify the request then send the request to Agent as an envelope. :param request: the request object :return: a tuple of response code and response description """ request = await Request.create(http_request) assert self._in_queue is not None, "Channel not connected!" is_valid_request = self.api_spec.verify(request) if not is_valid_request: self.logger.warning(f"request is not valid: {request}") return Response(status=NOT_FOUND, reason="Request Not Found") try: # turn request into envelope envelope = request.to_envelope_and_set_id( self.connection_id, self.address, dialogues=self._dialogues, ) self.pending_requests[request.id] = Future() # send the envelope to the agent's inbox (via self.in_queue) await self._in_queue.put(envelope) # wait for response envelope within given timeout window (self.timeout_window) to appear in dispatch_ready_envelopes response_message = await asyncio.wait_for( self.pending_requests[request.id], timeout=self.RESPONSE_TIMEOUT, ) return Response.from_message(response_message) except asyncio.TimeoutError: return Response(status=REQUEST_TIMEOUT, reason="Request Timeout") except BaseException: # pragma: nocover # pylint: disable=broad-except self.logger.exception("Error during handling incoming request") return Response(status=SERVER_ERROR, reason="Server Error", text=format_exc()) finally: if request.is_id_set: self.pending_requests.pop(request.id, None) async def _start_http_server(self) -> None: """Start http server.""" server = web.Server(self._http_handler) runner = web.ServerRunner(server) await runner.setup() self.http_server = web.TCPSite(runner, self.host, self.port) await self.http_server.start() def send(self, envelope: Envelope) -> None: """ Send the envelope in_queue. :param envelope: the envelope :return: None """ assert self.http_server is not None, "Server not connected, call connect first!" if envelope.protocol_id not in self.restricted_to_protocols: self.logger.error( "This envelope cannot be sent with the http connection: protocol_id={}" .format(envelope.protocol_id)) raise ValueError("Cannot send message.") http_message = cast(HttpMessage, envelope.message) message = copy.copy( http_message ) # TODO: fix; need to copy atm to avoid overwriting "is_incoming" message.is_incoming = True # TODO: fix; should be done by framework message.counterparty = envelope.sender # TODO: fix; should be done by framework dialogue = self._dialogues.update(message) if dialogue is None: self.logger.warning( "Could not create dialogue for message={}".format(message)) return future = self.pending_requests.pop(dialogue.incomplete_dialogue_label, None) if not future: self.logger.warning( "Dropping message={} for incomplete_dialogue_label={} which has timed out." .format(message, dialogue.incomplete_dialogue_label)) else: future.set_result(message) async def disconnect(self) -> None: """ Disconnect. Shut-off the HTTP Server. """ assert self.http_server is not None, "Server not connected, call connect first!" if not self.is_stopped: await self.http_server.stop() self.logger.info("HTTP Server has shutdown on port: {}.".format( self.port)) self.is_stopped = True self._in_queue = None
class TestHTTPServer: """Tests for HTTPServer connection.""" async def request(self, method: str, path: str, **kwargs) -> ClientResponse: """ Make a http request. :param method: HTTP method: GET, POST etc :param path: path to request on server. full url constructed automatically :return: http response """ try: url = f"http://{self.host}:{self.port}{path}" async with aiohttp.ClientSession() as session: async with session.request(method, url, **kwargs) as resp: await resp.read() return resp except Exception: print_exc() raise def setup(self): """Initialise the test case.""" self.identity = Identity("name", address="my_key") self.agent_address = self.identity.address self.host = get_host() self.port = get_unused_tcp_port() self.api_spec_path = os.path.join( ROOT_DIR, "tests", "data", "petstore_sim.yaml" ) self.connection_id = HTTPServerConnection.connection_id self.protocol_id = PublicId.from_str("fetchai/http:0.4.0") self.configuration = ConnectionConfig( host=self.host, port=self.port, api_spec_path=self.api_spec_path, connection_id=HTTPServerConnection.connection_id, restricted_to_protocols=set([self.protocol_id]), ) self.http_connection = HTTPServerConnection( configuration=self.configuration, identity=self.identity, ) self.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.http_connection.connect()) self.connection_address = str(HTTPServerConnection.connection_id) self._dialogues = HttpDialogues(self.connection_address) self.original_timeout = self.http_connection.channel.RESPONSE_TIMEOUT @pytest.mark.asyncio async def test_http_connection_disconnect_channel(self): """Test the disconnect.""" await self.http_connection.channel.disconnect() assert self.http_connection.channel.is_stopped def _get_message_and_dialogue( self, envelope: Envelope ) -> Tuple[HttpMessage, HttpDialogue]: message = cast(HttpMessage, envelope.message) message = copy.copy( message ) # TODO: fix; need to copy atm to avoid overwriting "is_incoming" message.is_incoming = True # TODO: fix; should be done by framework message.counterparty = envelope.sender # TODO: fix; should be done by framework dialogue = cast(HttpDialogue, self._dialogues.update(message)) assert dialogue is not None return message, dialogue @pytest.mark.asyncio async def test_get_200(self): """Test send get request w/ 200 response.""" request_task = self.loop.create_task(self.request("get", "/pets")) envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=20) assert envelope incoming_message, dialogue = self._get_message_and_dialogue(envelope) message = HttpMessage( dialogue_reference=dialogue.dialogue_label.dialogue_reference, performative=HttpMessage.Performative.RESPONSE, version=incoming_message.version, headers=incoming_message.headers, message_id=incoming_message.message_id + 1, target=incoming_message.message_id, status_code=200, status_text="Success", bodyy=b"Response body", ) message.counterparty = incoming_message.counterparty assert dialogue.update(message) response_envelope = Envelope( to=envelope.sender, sender=envelope.to, protocol_id=envelope.protocol_id, context=envelope.context, message=message, ) await self.http_connection.send(response_envelope) response = await asyncio.wait_for(request_task, timeout=20,) assert ( response.status == 200 and response.reason == "Success" and await response.text() == "Response body" ) @pytest.mark.asyncio async def test_bad_performative_get_timeout_error(self): """Test send get request w/ 200 response.""" self.http_connection.channel.RESPONSE_TIMEOUT = 3 request_task = self.loop.create_task(self.request("get", "/pets")) envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=10) assert envelope incoming_message, dialogue = self._get_message_and_dialogue(envelope) message = HttpMessage( performative=HttpMessage.Performative.REQUEST, dialogue_reference=dialogue.dialogue_label.dialogue_reference, target=incoming_message.message_id, message_id=incoming_message.message_id + 1, method="post", url="/pets", version=incoming_message.version, headers=incoming_message.headers, bodyy=b"Request body", ) message.counterparty = incoming_message.counterparty assert not dialogue.update(message) response_envelope = Envelope( to=envelope.sender, sender=envelope.to, protocol_id=envelope.protocol_id, context=envelope.context, message=message, ) with patch.object(self.http_connection.logger, "warning") as mock_logger: await self.http_connection.send(response_envelope) mock_logger.assert_any_call( f"Could not create dialogue for message={message}" ) response = await asyncio.wait_for(request_task, timeout=10) assert ( response.status == 408 and response.reason == "Request Timeout" and await response.text() == "" ) @pytest.mark.asyncio async def test_late_message_get_timeout_error(self): """Test send get request w/ 200 response.""" self.http_connection.channel.RESPONSE_TIMEOUT = 1 request_task = self.loop.create_task(self.request("get", "/pets")) envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=10) assert envelope incoming_message, dialogue = self._get_message_and_dialogue(envelope) message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, dialogue_reference=dialogue.dialogue_label.dialogue_reference, target=incoming_message.message_id, message_id=incoming_message.message_id + 1, version=incoming_message.version, headers=incoming_message.headers, status_code=200, status_text="Success", bodyy=b"Response body", ) message.counterparty = incoming_message.counterparty assert dialogue.update(message) response_envelope = Envelope( to=envelope.sender, sender=envelope.to, protocol_id=envelope.protocol_id, context=envelope.context, message=message, ) await asyncio.sleep(1.5) with patch.object(self.http_connection.logger, "warning") as mock_logger: await self.http_connection.send(response_envelope) mock_logger.assert_any_call( RegexComparator( "Dropping message=.* for incomplete_dialogue_label=.* which has timed out." ) ) response = await asyncio.wait_for(request_task, timeout=10) assert ( response.status == 408 and response.reason == "Request Timeout" and await response.text() == "" ) @pytest.mark.asyncio async def test_post_201(self): """Test send get request w/ 200 response.""" request_task = self.loop.create_task(self.request("post", "/pets",)) envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=20) assert envelope incoming_message, dialogue = self._get_message_and_dialogue(envelope) message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, dialogue_reference=dialogue.dialogue_label.dialogue_reference, target=incoming_message.message_id, message_id=incoming_message.message_id + 1, version=incoming_message.version, headers=incoming_message.headers, status_code=201, status_text="Created", bodyy=b"Response body", ) message.counterparty = incoming_message.counterparty assert dialogue.update(message) response_envelope = Envelope( to=envelope.sender, sender=envelope.to, protocol_id=envelope.protocol_id, context=envelope.context, message=message, ) await self.http_connection.send(response_envelope) response = await asyncio.wait_for(request_task, timeout=20,) assert ( response.status == 201 and response.reason == "Created" and await response.text() == "Response body" ) @pytest.mark.asyncio async def test_get_404(self): """Test send post request w/ 404 response.""" response = await self.request("get", "/url-non-exists") assert ( response.status == 404 and response.reason == "Request Not Found" and await response.text() == "" ) @pytest.mark.asyncio async def test_post_404(self): """Test send post request w/ 404 response.""" response = await self.request("get", "/url-non-exists", data="some data") assert ( response.status == 404 and response.reason == "Request Not Found" and await response.text() == "" ) @pytest.mark.asyncio async def test_get_408(self): """Test send post request w/ 404 response.""" await self.http_connection.connect() self.http_connection.channel.RESPONSE_TIMEOUT = 0.1 response = await self.request("get", "/pets") assert ( response.status == 408 and response.reason == "Request Timeout" and await response.text() == "" ) @pytest.mark.asyncio async def test_post_408(self): """Test send post request w/ 404 response.""" self.http_connection.channel.RESPONSE_TIMEOUT = 0.1 response = await self.request("post", "/pets", data="somedata") assert ( response.status == 408 and response.reason == "Request Timeout" and await response.text() == "" ) @pytest.mark.asyncio async def test_send_connection_drop(self): """Test unexpected response.""" message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, dialogue_reference=("", ""), target=1, message_id=2, headers="", version="", status_code=200, status_text="Success", bodyy=b"", ) message.counterparty = "to_key" message.sender = "from_key" envelope = Envelope( to=message.counterparty, sender=message.sender, protocol_id=message.protocol_id, message=message, ) await self.http_connection.send(envelope) @pytest.mark.asyncio async def test_get_message_channel_not_connected(self): """Test error on channel get message if not connected.""" await self.http_connection.disconnect() with pytest.raises(ValueError): await self.http_connection.channel.get_message() @pytest.mark.asyncio async def test_fail_connect(self): """Test error on server connection.""" await self.http_connection.disconnect() with patch.object( self.http_connection.channel, "_start_http_server", side_effect=Exception("expected"), ): await self.http_connection.connect() assert not self.http_connection.is_connected @pytest.mark.asyncio async def test_server_error_on_send_response(self): """Test exception raised on response sending to the client.""" request_task = self.loop.create_task(self.request("post", "/pets",)) envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=20) assert envelope incoming_message, dialogue = self._get_message_and_dialogue(envelope) message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, dialogue_reference=dialogue.dialogue_label.dialogue_reference, target=incoming_message.message_id, message_id=incoming_message.message_id + 1, version=incoming_message.version, headers=incoming_message.headers, status_code=201, status_text="Created", bodyy=b"Response body", ) message.counterparty = incoming_message.counterparty assert dialogue.update(message) response_envelope = Envelope( to=envelope.sender, sender=envelope.to, protocol_id=envelope.protocol_id, context=envelope.context, message=message, ) with patch.object(Response, "from_message", side_effect=Exception("expected")): await self.http_connection.send(response_envelope) response = await asyncio.wait_for(request_task, timeout=20,) assert response and response.status == 500 and response.reason == "Server Error" @pytest.mark.asyncio async def test_send_envelope_restricted_to_protocols_fail(self): """Test fail on send if envelope protocol not supported.""" message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, dialogue_reference=("", ""), target=1, message_id=2, version="1.0", headers="", status_code=200, status_text="Success", bodyy=b"Response body", ) envelope = Envelope( to="receiver", sender="sender", protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID, message=message, ) with patch.object( self.http_connection.channel, "restricted_to_protocols", new=[HTTP_PROTOCOL_PUBLIC_ID], ): with pytest.raises(ValueError): await self.http_connection.send(envelope) def teardown(self): """Teardown the test case.""" self.loop.run_until_complete(self.http_connection.disconnect()) self.http_connection.channel.RESPONSE_TIMEOUT = self.original_timeout
class HTTPClientAsyncChannel: """A wrapper for a HTTPClient.""" DEFAULT_TIMEOUT = 300 # default timeout in seconds DEFAULT_EXCEPTION_CODE = ( 600 # custom code to indicate there was exception during request ) def __init__( self, agent_address: Address, address: str, port: int, connection_id: PublicId, excluded_protocols: Optional[Set[PublicId]] = None, restricted_to_protocols: Optional[Set[PublicId]] = None, ): """ Initialize an http client channel. :param agent_address: the address of the agent. :param address: server hostname / IP address :param port: server port number :param excluded_protocols: this connection cannot handle messages adhering to any of the protocols in this set :param restricted_to_protocols: this connection can only handle messages adhering to protocols in this set """ self.agent_address = agent_address self.address = address self.port = port self.connection_id = connection_id self.restricted_to_protocols = restricted_to_protocols self._dialogues = HttpDialogues(str( HTTPClientConnection.connection_id)) self._in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover self._loop = ( None ) # type: Optional[asyncio.AbstractEventLoop] # pragma: no cover self.excluded_protocols = excluded_protocols self.is_stopped = True self._tasks: Set[Task] = set() self.logger = logger self.logger.info("Initialised the HTTP client channel") async def connect(self, loop: AbstractEventLoop) -> None: """ Connect channel using loop. :param loop: asyncio event loop to use :return: None """ self._loop = loop self._in_queue = asyncio.Queue() self.is_stopped = False def _get_message_and_dialogue( self, envelope: Envelope) -> Tuple[HttpMessage, Optional[HttpDialogue]]: """ Get a message copy and dialogue related to this message. :param envelope: incoming envelope :return: Tuple[MEssage, Optional[Dialogue]] """ orig_message = cast(HttpMessage, envelope.message) message = copy.copy( orig_message ) # TODO: fix; need to copy atm to avoid overwriting "is_incoming" message.is_incoming = True # TODO: fix; should be done by framework message.counterparty = (orig_message.sender ) # TODO: fix; should be done by framework dialogue = cast(HttpDialogue, self._dialogues.update(message)) return message, dialogue async def _http_request_task(self, request_envelope: Envelope) -> None: """ Perform http request and send back response. :param request_http_message: HttpMessage with http request constructed. :return: None """ if not self._loop: # pragma: nocover raise ValueError("Channel is not connected") request_http_message, dialogue = self._get_message_and_dialogue( request_envelope) if not dialogue: self.logger.warning( "Could not create dialogue for message={}".format( request_http_message)) return try: resp = await asyncio.wait_for( self._perform_http_request(request_http_message), timeout=self.DEFAULT_TIMEOUT, ) envelope = self.to_envelope( self.connection_id, request_http_message, status_code=resp.status, headers=resp.headers, status_text=resp.reason, bodyy=resp._body # pylint: disable=protected-access if resp._body is not None # pylint: disable=protected-access else b"", dialogue=dialogue, ) except Exception: # pragma: nocover # pylint: disable=broad-except envelope = self.to_envelope( self.connection_id, request_http_message, status_code=self.DEFAULT_EXCEPTION_CODE, headers={}, status_text="HTTPConnection request error.", bodyy=format_exc().encode("utf-8"), dialogue=dialogue, ) if self._in_queue is not None: await self._in_queue.put(envelope) async def _perform_http_request( self, request_http_message: HttpMessage) -> ClientResponse: """ Perform http request and return response. :param request_http_message: HttpMessage with http request constructed. :return: aiohttp.ClientResponse """ try: async with aiohttp.ClientSession() as session: async with session.request( method=request_http_message.method, url=request_http_message.url, headers=request_http_message.headers, data=request_http_message.bodyy, ) as resp: await resp.read() return resp except Exception: # pragma: nocover # pylint: disable=broad-except self.logger.exception( f"Exception raised during http call: {request_http_message.method} {request_http_message.url}" ) raise def send(self, request_envelope: Envelope) -> None: """ Send an envelope with http request data to request. Convert an http envelope into an http request. Send the http request Wait for and receive its response Translate the response into a response envelop. Send the response envelope to the in-queue. :param request_envelope: the envelope containing an http request :return: None """ if self._loop is None or self.is_stopped: raise ValueError("Can not send a message! Channel is not started!") if request_envelope is None: return if request_envelope.protocol_id in (self.excluded_protocols or []): self.logger.error( "This envelope cannot be sent with the http client connection: protocol_id={}" .format(request_envelope.protocol_id)) raise ValueError("Cannot send message.") assert isinstance(request_envelope.message, HttpMessage), "Message not of type HttpMessage" request_http_message = cast(HttpMessage, request_envelope.message) if (request_http_message.performative != HttpMessage.Performative.REQUEST): # pragma: nocover self.logger.warning( "The HTTPMessage performative must be a REQUEST. Envelop dropped." ) return task = self._loop.create_task( self._http_request_task(request_envelope)) task.add_done_callback(self._task_done_callback) self._tasks.add(task) def _task_done_callback(self, task: Task) -> None: """ Handle http request task completed. Removes tasks from _tasks. :param task: Task completed. :return: None """ self._tasks.remove(task) self.logger.debug(f"Task completed: {task}") async def get_message(self) -> Union["Envelope", None]: """ Get http response from in-queue. :return: None or envelope with http response. """ if self._in_queue is None: raise ValueError("Looks like channel is not connected!") try: return await self._in_queue.get() except CancelledError: # pragma: nocover return None def to_envelope( self, connection_id: PublicId, http_request_message: HttpMessage, status_code: int, headers: dict, status_text: Optional[Any], bodyy: bytes, dialogue: HttpDialogue, ) -> Envelope: """ Convert an HTTP response object (from the 'requests' library) into an Envelope containing an HttpMessage (from the 'http' Protocol). :param connection_id: the connection id :param http_request_message: the message of the http request envelop :param status_code: the http status code, int :param headers: dict of http response headers :param status_text: the http status_text, str :param bodyy: bytes of http response content :return: Envelope with http response data. """ context = EnvelopeContext(connection_id=connection_id) http_message = HttpMessage( performative=HttpMessage.Performative.RESPONSE, status_code=status_code, headers=json.dumps(dict(headers.items())), status_text=status_text, bodyy=bodyy, version="", dialogue_reference=dialogue.dialogue_label.dialogue_reference, target=http_request_message.message_id, message_id=http_request_message.message_id + 1, ) http_message.counterparty = http_request_message.counterparty assert dialogue.update(http_message) envelope = Envelope( to=self.agent_address, sender="HTTP Server", protocol_id=PublicId.from_str("fetchai/http:0.4.0"), context=context, message=http_message, ) return envelope async def _cancel_tasks(self) -> None: """Cancel all requests tasks pending.""" for task in list(self._tasks): if task.done(): # pragma: nocover continue task.cancel() for task in list(self._tasks): try: await task except KeyboardInterrupt: # pragma: nocover raise except BaseException: # pragma: nocover # pylint: disable=broad-except pass # nosec async def disconnect(self) -> None: """Disconnect.""" if not self.is_stopped: self.logger.info("HTTP Client has shutdown on port: {}.".format( self.port)) self.is_stopped = True await self._cancel_tasks()
class TestWebhookConnection: """Tests the webhook connection's 'connect' functionality.""" def setup(self): """Initialise the class.""" self.host = get_host() self.port = get_unused_tcp_port() self.identity = Identity("", address="some string") self.path = "/webhooks/topic/{topic}/" self.loop = asyncio.get_event_loop() configuration = ConnectionConfig( webhook_address=self.host, webhook_port=self.port, webhook_url_path=self.path, connection_id=WebhookConnection.connection_id, ) self.webhook_connection = WebhookConnection( configuration=configuration, identity=self.identity, ) self.webhook_connection.loop = self.loop self.dialogues = HttpDialogues(self.identity.address) async def test_initialization(self): """Test the initialisation of the class.""" assert self.webhook_connection.address == self.identity.address @pytest.mark.asyncio async def test_connection(self): """Test the connect functionality of the webhook connection.""" await self.webhook_connection.connect() assert self.webhook_connection.is_connected is True @pytest.mark.asyncio async def test_disconnect(self): """Test the disconnect functionality of the webhook connection.""" await self.webhook_connection.connect() assert self.webhook_connection.is_connected is True await self.webhook_connection.disconnect() assert self.webhook_connection.is_connected is False def teardown(self): """Close connection after testing.""" try: self.loop.run_until_complete(self.webhook_connection.disconnect()) except Exception: print_exc() raise @pytest.mark.asyncio async def test_receive_post_ok(self): """Test the connect functionality of the webhook connection.""" await self.webhook_connection.connect() assert self.webhook_connection.is_connected is True payload = {"hello": "world"} call_task = self.loop.create_task(self.call_webhook("test_topic", json=payload)) envelope = await asyncio.wait_for(self.webhook_connection.receive(), timeout=10) assert envelope orig_message = cast(HttpMessage, envelope.message) message = copy.copy(orig_message) message.counterparty = orig_message.sender message.is_incoming = True dialogue = self.dialogues.update(message) assert dialogue is not None assert message.method.upper() == "POST" assert message.bodyy.decode("utf-8") == json.dumps(payload) await call_task @pytest.mark.asyncio async def test_send(self): """Test the connect functionality of the webhook connection.""" await self.webhook_connection.connect() assert self.webhook_connection.is_connected is True http_message = HttpMessage( dialogue_reference=("", ""), target=0, message_id=1, performative=HttpMessage.Performative.REQUEST, method="get", url="/", headers="", bodyy="", version="", ) envelope = Envelope( to="addr", sender="my_id", protocol_id=PublicId.from_str("fetchai/http:0.4.0"), message=http_message, ) with patch.object(self.webhook_connection.logger, "warning") as mock_logger: await self.webhook_connection.send(envelope) await asyncio.sleep(0.01) mock_logger.assert_any_call( RegexComparator( "Dropping envelope=.* as sending via the webhook is not possible!" ) ) async def call_webhook(self, topic: str, **kwargs) -> ClientResponse: """ Make a http request to a webhook. :param topic: topic to use :params **kwargs: data or json for payload :return: http response """ path = self.path.format(topic=topic) method = kwargs.get("method", "post") url = f"http://{self.host}:{self.port}{path}" try: async with aiohttp.ClientSession() as session: async with session.request(method, url, **kwargs) as resp: await resp.read() return resp except Exception: print_exc() raise
class HttpPingPongHandler(Handler): """Dummy handler to handle messages.""" SUPPORTED_PROTOCOL = HttpMessage.protocol_id def setup(self) -> None: """Noop setup.""" # pylint: disable=attribute-defined-outside-init, unused-argument self.count: int = 0 self.rtt_total_time: float = 0.0 self.rtt_count: int = 0 self.latency_total_time: float = 0.0 self.latency_count: int = 0 def role(m: Message, addr: Address) -> Dialogue.Role: return HttpDialogue.Role.CLIENT self.dialogues = HttpDialogues(self.context.agent_address, role_from_first_message=role) def teardown(self) -> None: """Noop teardown.""" def handle(self, message: Message) -> None: """Handle incoming message.""" self.count += 1 message = cast(HttpMessage, message) dialogue = self.dialogues.update(message) if not dialogue: raise Exception("something goes wrong") rtt_ts, latency_ts = struct.unpack("dd", message.body) # type: ignore if message.performative == HttpMessage.Performative.REQUEST: self.latency_total_time += time.time() - latency_ts self.latency_count += 1 self.make_response(cast(HttpDialogue, dialogue), message) elif message.performative == HttpMessage.Performative.RESPONSE: self.rtt_total_time += time.time() - rtt_ts self.rtt_count += 1 # got response, make another requet to the same agent self.make_request(message.sender) def make_response(self, dialogue: HttpDialogue, message: HttpMessage) -> None: """Construct and send a response for message received.""" response_message = dialogue.reply( target_message=message, performative=HttpMessage.Performative.RESPONSE, version=message.version, headers="", status_code=200, status_text="Success", body=message.body, ) self.context.outbox.put_message(response_message) def make_request(self, recipient_addr: str) -> None: """Make initial http request.""" message, _ = self.dialogues.create( recipient_addr, performative=HttpMessage.Performative.REQUEST, method="get", url="some url", headers="", version="", body=struct.pack("dd", time.time(), time.time()), ) self.context.outbox.put_message(message)
class TestClientServer: """Client-Server end-to-end test.""" def setup_server(self): """Set up server connection.""" self.server_agent_address = "server_agent_address" self.server_agent_identity = Identity( "agent running server", address=self.server_agent_address) self.host = get_host() self.port = get_unused_tcp_port() self.connection_id = HTTPServerConnection.connection_id self.protocol_id = HttpMessage.protocol_id self.configuration = ConnectionConfig( host=self.host, port=self.port, api_spec_path=None, # do not filter on API spec connection_id=HTTPServerConnection.connection_id, restricted_to_protocols=set([self.protocol_id]), ) self.server = HTTPServerConnection( configuration=self.configuration, identity=self.server_agent_identity, ) self.loop = asyncio.get_event_loop() self.loop.run_until_complete(self.server.connect()) # skill side dialogues def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ return HttpDialogue.Role.SERVER self._server_dialogues = HttpDialogues( self.server_agent_address, role_from_first_message=role_from_first_message) def setup_client(self): """Set up client connection.""" self.client_agent_address = "client_agent_address" self.client_agent_identity = Identity( "agent running client", address=self.client_agent_address) configuration = ConnectionConfig( host="localost", port="8888", # TODO: remove host/port for client? connection_id=HTTPClientConnection.connection_id, ) self.client = HTTPClientConnection(configuration=configuration, identity=self.client_agent_identity) self.loop.run_until_complete(self.client.connect()) # skill side dialogues def role_from_first_message( # pylint: disable=unused-argument message: Message, receiver_address: Address) -> BaseDialogue.Role: """Infer the role of the agent from an incoming/outgoing first message :param message: an incoming/outgoing first message :param receiver_address: the address of the receiving agent :return: The role of the agent """ return HttpDialogue.Role.CLIENT self._client_dialogues = HttpDialogues( self.client_agent_address, role_from_first_message=role_from_first_message) def setup(self): """Set up test case.""" self.setup_server() self.setup_client() def _make_request(self, path: str, method: str = "get", headers: str = "", body: bytes = b"") -> Envelope: """Make request envelope.""" request_http_message, _ = self._client_dialogues.create( counterparty=str(HTTPClientConnection.connection_id), performative=HttpMessage.Performative.REQUEST, method=method, url=f"http://{self.host}:{self.port}{path}", headers="", version="", body=b"", ) request_envelope = Envelope( to=request_http_message.to, sender=request_http_message.sender, protocol_id=request_http_message.protocol_id, message=request_http_message, ) return request_envelope def _make_response(self, request_envelope: Envelope, status_code: int = 200, status_text: str = "") -> Envelope: """Make response envelope.""" incoming_message = cast(HttpMessage, request_envelope.message) dialogue = self._server_dialogues.update(incoming_message) assert dialogue is not None message = dialogue.reply( target_message=incoming_message, performative=HttpMessage.Performative.RESPONSE, version=incoming_message.version, headers=incoming_message.headers, status_code=status_code, status_text=status_text, body=incoming_message.body, ) response_envelope = Envelope( to=message.to, sender=message.sender, protocol_id=message.protocol_id, context=request_envelope.context, message=message, ) return response_envelope @pytest.mark.asyncio async def test_post_with_payload(self): """Test client and server with post request.""" initial_request = self._make_request("/test", "POST", body=b"1234567890") await self.client.send(initial_request) request = await asyncio.wait_for(self.server.receive(), timeout=5) # this is "inside" the server agent initial_response = self._make_response(request) await self.server.send(initial_response) response = await asyncio.wait_for(self.client.receive(), timeout=5) assert (cast(HttpMessage, initial_request.message).body == cast( HttpMessage, response.message).body) assert (initial_request.message.dialogue_reference[0] == response.message.dialogue_reference[0]) @pytest.mark.asyncio async def test_get_with_query(self): """Test client and server with url query.""" query = {"key": "value"} path = "/test?{}".format(urllib.parse.urlencode(query)) initial_request = self._make_request(path, "GET") await self.client.send(initial_request) request = await asyncio.wait_for(self.server.receive(), timeout=5) # this is "inside" the server agent parsed_query = dict( urllib.parse.parse_qsl( urllib.parse.splitquery( cast(HttpMessage, request.message).url)[1])) assert parsed_query == query initial_response = self._make_response(request) await self.server.send(initial_response) response = await asyncio.wait_for(self.client.receive(), timeout=5) assert (initial_request.message.dialogue_reference[0] == response.message.dialogue_reference[0]) def teardown(self): """Tear down testcase.""" self.loop.run_until_complete(self.client.disconnect()) self.loop.run_until_complete(self.server.disconnect())