def __init__(
self,
address: Address,
host: str,
port: int,
api_spec_path: Optional[str],
connection_id: PublicId,
restricted_to_protocols: Set[PublicId],
timeout_window: float = 5.0,
logger: logging.Logger = _default_logger,
):
"""
Initialize a channel and process the initial API specification from the file path (if given).
:param address: the address of the agent.
:param host: RESTful API hostname / IP address
:param port: RESTful API port number
:param api_spec_path: Directory API path and filename of the API spec YAML source file.
:param connection_id: public id of connection using this chanel.
:param restricted_to_protocols: set of restricted protocols
:param timeout_window: the timeout (in seconds) for a request to be handled.
"""
super().__init__(address=address, connection_id=connection_id)
self.host = host
self.port = port
self.server_address = "http://{}:{}".format(self.host, self.port)
self.restricted_to_protocols = restricted_to_protocols
self._api_spec = APISpec(api_spec_path, self.server_address, logger)
self.timeout_window = timeout_window
self.http_server: Optional[web.TCPSite] = None
self.pending_requests: Dict[RequestId, Future] = {}
self._dialogues = HttpDialogues(self.address)
self.logger = logger
def setup_client(self):
"""Set up client connection."""
self.client_agent_address = "client_agent_address"
self.client_agent_identity = Identity(
"agent running client", address=self.client_agent_address)
configuration = ConnectionConfig(
host="localost",
port="8888", # TODO: remove host/port for client?
connection_id=HTTPClientConnection.connection_id,
)
self.client = HTTPClientConnection(configuration=configuration,
identity=self.client_agent_identity)
self.loop.run_until_complete(self.client.connect())
# skill side dialogues
def role_from_first_message( # pylint: disable=unused-argument
message: Message,
receiver_address: Address) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
return HttpDialogue.Role.CLIENT
self._client_dialogues = HttpDialogues(
self.client_agent_address,
role_from_first_message=role_from_first_message)
def setup(self):
"""Initialise the test case."""
self.identity = Identity("name", address="my_key")
self.agent_address = self.identity.address
self.host = get_host()
self.port = get_unused_tcp_port()
self.api_spec_path = os.path.join(
ROOT_DIR, "tests", "data", "petstore_sim.yaml"
)
self.connection_id = HTTPServerConnection.connection_id
self.protocol_id = PublicId.from_str("fetchai/http:0.4.0")
self.configuration = ConnectionConfig(
host=self.host,
port=self.port,
api_spec_path=self.api_spec_path,
connection_id=HTTPServerConnection.connection_id,
restricted_to_protocols=set([self.protocol_id]),
)
self.http_connection = HTTPServerConnection(
configuration=self.configuration, identity=self.identity,
)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.http_connection.connect())
self.connection_address = str(HTTPServerConnection.connection_id)
self._dialogues = HttpDialogues(self.connection_address)
self.original_timeout = self.http_connection.channel.RESPONSE_TIMEOUT
def __init__(self, self_address: Address, **kwargs: Any) -> None:
"""
Initialize dialogues.
:return: None
"""
def role_from_first_message( # pylint: disable=unused-argument
message: Message, receiver_address: Address
) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
# The server connection maintains the dialogue on behalf of the client
return HttpDialogue.Role.CLIENT
BaseHttpDialogues.__init__(
self,
self_address=self_address,
role_from_first_message=role_from_first_message,
**kwargs,
)
def __init__(self) -> None:
"""
Initialize dialogues.
:return: None
"""
def role_from_first_message( # pylint: disable=unused-argument
message: Message, receiver_address: Address
) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
# The client connection maintains the dialogue on behalf of the server
return HttpDialogue.Role.SERVER
BaseHttpDialogues.__init__(
self,
self_address=str(HTTPClientConnection.connection_id),
role_from_first_message=role_from_first_message,
dialogue_class=HttpDialogue,
)
def setup_server(self):
"""Set up server connection."""
self.server_agent_address = "server_agent_address"
self.server_agent_identity = Identity(
"agent running server", address=self.server_agent_address
)
self.host = get_host()
self.port = get_unused_tcp_port()
self.connection_id = HTTPServerConnection.connection_id
self.protocol_id = PublicId.from_str("fetchai/http:0.4.0")
self.configuration = ConnectionConfig(
host=self.host,
port=self.port,
api_spec_path=None, # do not filter on API spec
connection_id=HTTPServerConnection.connection_id,
restricted_to_protocols=set([self.protocol_id]),
)
self.server = HTTPServerConnection(
configuration=self.configuration, identity=self.server_agent_identity,
)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.server.connect())
# skill side dialogues
self._server_dialogues = HttpDialogues(self.server_agent_address)
def __init__(
self,
agent_address: Address,
webhook_address: Address,
webhook_port: int,
webhook_url_path: str,
connection_id: PublicId,
logger: logging.Logger = _default_logger,
):
"""
Initialize a webhook channel.
:param agent_address: the address of the agent
:param webhook_address: webhook hostname / IP address
:param webhook_port: webhook port number
:param webhook_url_path: the url path to receive webhooks from
:param connection_id: the connection id
"""
self.agent_address = agent_address
self.webhook_address = webhook_address
self.webhook_port = webhook_port
self.webhook_url_path = webhook_url_path
self.webhook_site = None # type: Optional[web.TCPSite]
self.runner = None # type: Optional[web.AppRunner]
self.app = None # type: Optional[web.Application]
self.is_stopped = True
self.connection_id = connection_id
self.in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover
self.logger = logger
self.logger.info("Initialised a webhook channel")
self._dialogues = HttpDialogues(str(WebhookConnection.connection_id))
def __init__(self, **kwargs: Any) -> None:
"""
Initialize dialogues.
:param agent_address: the address of the agent for whom dialogues are maintained
:return: None
"""
Model.__init__(self, **kwargs)
def role_from_first_message( # pylint: disable=unused-argument
message: Message,
receiver_address: Address) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent in this dialogue
"""
if (message.performative == HttpMessage.Performative.REQUEST
and message.sender != receiver_address
) or (message.performative == HttpMessage.Performative.RESPONSE
and message.sender == receiver_address):
return BaseHttpDialogue.Role.SERVER
return BaseHttpDialogue.Role.CLIENT
BaseHttpDialogues.__init__(
self,
self_address=str(self.skill_id),
role_from_first_message=role_from_first_message,
)
def __init__(self, **kwargs: Any) -> None:
"""
Initialize dialogues.
:param agent_address: the address of the agent for whom dialogues are maintained
:return: None
"""
Model.__init__(self, **kwargs)
def role_from_first_message( # pylint: disable=unused-argument
message: Message,
receiver_address: Address) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
return BaseHttpDialogue.Role.CLIENT
BaseHttpDialogues.__init__(
self,
self_address=self.context.agent_address,
role_from_first_message=role_from_first_message,
)
def __init__(self, **kwargs) -> None:
"""
Initialize dialogues.
:return: None
"""
Model.__init__(self, **kwargs)
BaseHttpDialogues.__init__(self, self.context.agent_address)
def __init__(self, **kwargs) -> None:
"""
Initialize dialogues.
:param agent_address: the address of the agent for whom dialogues are maintained
:return: None
"""
Model.__init__(self, **kwargs)
BaseHttpDialogues.__init__(self, self.context.agent_address)
def __init__(self):
"""Set dialogues."""
# pylint: disable=unused-argument
def role(m: Message, addr: Address) -> Dialogue.Role:
return HttpDialogue.Role.CLIENT
self.addr = self.random_string
self.dialogues = HttpDialogues(self.addr, role_from_first_message=role)
def setup(self) -> None:
"""Noop setup."""
# pylint: disable=attribute-defined-outside-init, unused-argument
self.count: int = 0
self.rtt_total_time: float = 0.0
self.rtt_count: int = 0
self.latency_total_time: float = 0.0
self.latency_count: int = 0
def role(m: Message, addr: Address) -> Dialogue.Role:
return HttpDialogue.Role.CLIENT
self.dialogues = HttpDialogues(self.context.agent_address,
role_from_first_message=role)
def setup(self):
"""Initialise the class."""
self.address = get_host()
self.port = get_unused_tcp_port()
self.agent_identity = Identity("name", address="some string")
self.agent_address = self.agent_identity.address
configuration = ConnectionConfig(
host=self.address,
port=self.port,
connection_id=HTTPClientConnection.connection_id,
)
self.http_client_connection = HTTPClientConnection(
configuration=configuration, identity=self.agent_identity)
self.http_client_connection.loop = asyncio.get_event_loop()
self.connection_address = str(HTTPClientConnection.connection_id)
self.http_dialogs = HttpDialogues(self.connection_address)
def setup_client(self):
"""Set up client connection."""
self.client_agent_address = "client_agent_address"
self.client_agent_identity = Identity(
"agent running client", address=self.client_agent_address
)
configuration = ConnectionConfig(
host="localost",
port="8888", # TODO: remove host/port for client?
connection_id=HTTPClientConnection.connection_id,
)
self.client = HTTPClientConnection(
configuration=configuration, identity=self.client_agent_identity
)
self.client.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.client.connect())
# skill side dialogues
self._client_dialogues = HttpDialogues(self.client_agent_address)
class DialogueHandler:
"""Generate messages and process with dialogues."""
def __init__(self):
"""Set dialogues."""
# pylint: disable=unused-argument
def role(m: Message, addr: Address) -> Dialogue.Role:
return HttpDialogue.Role.CLIENT
self.addr = self.random_string
self.dialogues = HttpDialogues(self.addr, role_from_first_message=role)
@property
def random_string(self) -> str:
"""Get random string on every access."""
return uuid.uuid4().hex
def process_message(self) -> None:
"""Process a message with dialogues."""
message = self.create()
dialogue = self.update(message)
self.reply(dialogue, message)
def update(self, message: HttpMessage) -> HttpDialogue:
"""Update dialogues with message."""
return cast(HttpDialogue, self.dialogues.update(message))
@staticmethod
def reply(dialogue: HttpDialogue, message: HttpMessage):
"""Construct and send a response for message received."""
return dialogue.reply(
target_message=message,
performative=HttpMessage.Performative.RESPONSE,
version=message.version,
headers="",
status_code=200,
status_text="Success",
body=message.body,
)
def create(self) -> HttpMessage:
"""Make initial http request."""
message = HttpMessage(
dialogue_reference=HttpDialogues.
new_self_initiated_dialogue_reference(),
performative=HttpMessage.Performative.REQUEST,
method="get",
url="some url",
headers="",
version="",
body=b"",
)
message.sender = self.random_string
message.to = self.addr
return message
def setup(self):
"""Initialise the class."""
self.host = get_host()
self.port = get_unused_tcp_port()
self.identity = Identity("", address="some string")
self.path = "/webhooks/topic/{topic}/"
self.loop = asyncio.get_event_loop()
configuration = ConnectionConfig(
webhook_address=self.host,
webhook_port=self.port,
webhook_url_path=self.path,
connection_id=WebhookConnection.connection_id,
)
self.webhook_connection = WebhookConnection(
configuration=configuration, identity=self.identity,
)
self.webhook_connection.loop = self.loop
self.dialogues = HttpDialogues(self.identity.address)
def to_envelope_and_set_id(
self,
connection_id: PublicId,
agent_address: str,
dialogues: HttpDialogues,
) -> Envelope:
"""
Process incoming API request by packaging into Envelope and sending it in-queue.
:param connection_id: id of the connection
:param agent_address: agent's address
:param dialogue_reference: new dialog refernece for envelope
:return: envelope
"""
url = (self.full_url_pattern if self.parameters.query == {} else
self.full_url_pattern + "?" + urlencode(self.parameters.query))
uri = URI(self.full_url_pattern)
context = EnvelopeContext(connection_id=connection_id, uri=uri)
http_message = HttpMessage(
dialogue_reference=dialogues.new_self_initiated_dialogue_reference(
),
performative=HttpMessage.Performative.REQUEST,
method=self.method,
url=url,
headers=self.parameters.header,
bodyy=self.body if self.body is not None else b"",
version="",
)
http_message.counterparty = agent_address
dialogue = cast(Optional[HttpDialogue], dialogues.update(http_message))
assert dialogue is not None, "Could not create dialogue for message={}".format(
http_message)
self.id = dialogue.incomplete_dialogue_label
envelope = Envelope(
to=agent_address,
sender=str(connection_id),
protocol_id=http_message.protocol_id,
context=context,
message=http_message,
)
return envelope
def __init__(self, self_address: Address, **kwargs) -> None:
"""
Initialize dialogues.
:return: None
"""
def role_from_first_message( # pylint: disable=unused-argument
message: Message, receiver_address: Address) -> Dialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
return HttpDialogue.Role.CLIENT
BaseHttpDialogues.__init__(
self,
self_address=self_address,
role_from_first_message=role_from_first_message,
)
def setup_server(self):
"""Set up server connection."""
self.server_agent_address = "server_agent_address"
self.server_agent_identity = Identity(
"agent running server", address=self.server_agent_address)
self.host = get_host()
self.port = get_unused_tcp_port()
self.connection_id = HTTPServerConnection.connection_id
self.protocol_id = HttpMessage.protocol_id
self.configuration = ConnectionConfig(
host=self.host,
port=self.port,
api_spec_path=None, # do not filter on API spec
connection_id=HTTPServerConnection.connection_id,
restricted_to_protocols=set([self.protocol_id]),
)
self.server = HTTPServerConnection(
configuration=self.configuration,
identity=self.server_agent_identity,
)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.server.connect())
# skill side dialogues
def role_from_first_message( # pylint: disable=unused-argument
message: Message,
receiver_address: Address) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
return HttpDialogue.Role.SERVER
self._server_dialogues = HttpDialogues(
self.server_agent_address,
role_from_first_message=role_from_first_message)
def __init__(
self,
agent_address: Address,
address: str,
port: int,
connection_id: PublicId,
excluded_protocols: Optional[Set[PublicId]] = None,
restricted_to_protocols: Optional[Set[PublicId]] = None,
):
"""
Initialize an http client channel.
:param agent_address: the address of the agent.
:param address: server hostname / IP address
:param port: server port number
:param excluded_protocols: this connection cannot handle messages adhering to any of the protocols in this set
:param restricted_to_protocols: this connection can only handle messages adhering to protocols in this set
"""
self.agent_address = agent_address
self.address = address
self.port = port
self.connection_id = connection_id
self.restricted_to_protocols = restricted_to_protocols
self._dialogues = HttpDialogues(str(
HTTPClientConnection.connection_id))
self._in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover
self._loop = (
None
) # type: Optional[asyncio.AbstractEventLoop] # pragma: no cover
self.excluded_protocols = excluded_protocols
self.is_stopped = True
self._tasks: Set[Task] = set()
self.logger = logger
self.logger.info("Initialised the HTTP client channel")
class TestHTTPClientConnect:
"""Tests the http client connection's 'connect' functionality."""
def setup(self):
"""Initialise the class."""
self.address = get_host()
self.port = get_unused_tcp_port()
self.agent_identity = Identity("name", address="some string")
self.agent_address = self.agent_identity.address
configuration = ConnectionConfig(
host=self.address,
port=self.port,
connection_id=HTTPClientConnection.connection_id,
)
self.http_client_connection = HTTPClientConnection(
configuration=configuration, identity=self.agent_identity)
self.http_client_connection.loop = asyncio.get_event_loop()
self.connection_address = str(HTTPClientConnection.connection_id)
self.http_dialogs = HttpDialogues(self.connection_address)
@pytest.mark.asyncio
async def test_initialization(self):
"""Test the initialisation of the class."""
assert self.http_client_connection.address == self.agent_identity.address
@pytest.mark.asyncio
async def test_connection(self):
"""Test the connect functionality of the http client connection."""
await self.http_client_connection.connect()
assert self.http_client_connection.is_connected is True
@pytest.mark.asyncio
async def test_disconnect(self):
"""Test the disconnect functionality of the http client connection."""
await self.http_client_connection.connect()
assert self.http_client_connection.is_connected is True
await self.http_client_connection.disconnect()
assert self.http_client_connection.is_connected is False
@pytest.mark.asyncio
async def test_http_send_error(self):
"""Test request fails and send back result with code 600."""
await self.http_client_connection.connect()
request_http_message = HttpMessage(
dialogue_reference=self.http_dialogs.
new_self_initiated_dialogue_reference(),
performative=HttpMessage.Performative.REQUEST,
method="get",
url="bad url",
headers="",
version="",
bodyy=b"",
)
request_http_message.counterparty = self.connection_address
sending_dialogue = self.http_dialogs.update(request_http_message)
assert sending_dialogue is not None
request_envelope = Envelope(
to=self.connection_address,
sender=self.agent_address,
protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID,
message=request_http_message,
)
connection_response_mock = Mock()
connection_response_mock.status_code = 200
await self.http_client_connection.send(envelope=request_envelope)
# TODO: Consider returning the response from the server in order to be able to assert that the message send!
envelope = await asyncio.wait_for(
self.http_client_connection.receive(), timeout=10)
assert envelope
assert envelope.message.status_code == 600
await self.http_client_connection.disconnect()
@pytest.mark.asyncio
async def test_http_client_send_not_connected_error(self):
"""Test connection.send error if not conencted."""
with pytest.raises(ConnectionError):
await self.http_client_connection.send(Mock())
@pytest.mark.asyncio
async def test_http_channel_send_not_connected_error(self):
"""Test channel.send error if not conencted."""
with pytest.raises(ValueError):
self.http_client_connection.channel.send(Mock())
@pytest.mark.asyncio
async def test_send_envelope_excluded_protocol_fail(self):
"""Test send error if protocol not supported."""
request_http_message = HttpMessage(
dialogue_reference=self.http_dialogs.
new_self_initiated_dialogue_reference(),
performative=HttpMessage.Performative.REQUEST,
method="get",
url="bad url",
headers="",
version="",
bodyy=b"",
)
request_http_message.counterparty = self.connection_address
sending_dialogue = self.http_dialogs.update(request_http_message)
assert sending_dialogue is not None
request_envelope = Envelope(
to=self.connection_address,
sender=self.agent_address,
protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID,
message=request_http_message,
)
await self.http_client_connection.connect()
with patch.object(
self.http_client_connection.channel,
"excluded_protocols",
new=[UNKNOWN_PROTOCOL_PUBLIC_ID],
):
with pytest.raises(ValueError):
await self.http_client_connection.send(request_envelope)
@pytest.mark.asyncio
async def test_send_empty_envelope_skip(self):
"""Test skip on empty envelope request sent."""
await self.http_client_connection.connect()
with patch.object(self.http_client_connection.channel,
"_http_request_task") as mock:
await self.http_client_connection.send(None)
mock.assert_not_called()
@pytest.mark.asyncio
async def test_channel_get_message_not_connected(self):
"""Test errro on message get if not connected."""
with pytest.raises(ValueError):
await self.http_client_connection.channel.get_message()
@pytest.mark.asyncio
async def test_channel_cancel_tasks_on_disconnect(self):
"""Test requests tasks cancelled on disconnect."""
await self.http_client_connection.connect()
request_http_message = HttpMessage(
dialogue_reference=self.http_dialogs.
new_self_initiated_dialogue_reference(),
performative=HttpMessage.Performative.REQUEST,
method="get",
url="https://not-a-google.com",
headers="",
version="",
bodyy=b"",
)
request_http_message.counterparty = self.connection_address
sending_dialogue = self.http_dialogs.update(request_http_message)
assert sending_dialogue is not None
request_envelope = Envelope(
to=self.connection_address,
sender=self.agent_address,
protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID,
message=request_http_message,
)
connection_response_mock = Mock()
connection_response_mock.status_code = 200
response_mock = Mock()
response_mock.status = 200
response_mock.headers = {"headers": "some header"}
response_mock.reason = "OK"
response_mock._body = b"Some content"
response_mock.read.return_value = asyncio.Future()
with patch.object(
aiohttp.ClientSession,
"request",
return_value=_MockRequest(response_mock),
):
await self.http_client_connection.send(envelope=request_envelope)
assert self.http_client_connection.channel._tasks
task = list(self.http_client_connection.channel._tasks)[0]
assert not task.done()
await self.http_client_connection.disconnect()
assert not self.http_client_connection.channel._tasks
assert task.done()
with pytest.raises(CancelledError):
await task
@pytest.mark.asyncio
async def test_http_send_ok(self):
"""Test request is ok cause mocked."""
await self.http_client_connection.connect()
request_http_message = HttpMessage(
dialogue_reference=self.http_dialogs.
new_self_initiated_dialogue_reference(),
performative=HttpMessage.Performative.REQUEST,
method="get",
url="https://not-a-google.com",
headers="",
version="",
bodyy=b"",
)
request_http_message.counterparty = self.connection_address
sending_dialogue = self.http_dialogs.update(request_http_message)
assert sending_dialogue is not None
request_envelope = Envelope(
to=self.connection_address,
sender=self.agent_address,
protocol_id=request_http_message.protocol_id,
message=request_http_message,
)
connection_response_mock = Mock()
connection_response_mock.status_code = 200
response_mock = Mock()
response_mock.status = 200
response_mock.headers = {"headers": "some header"}
response_mock.reason = "OK"
response_mock._body = b"Some content"
response_mock.read.return_value = asyncio.Future()
response_mock.read.return_value.set_result("")
with patch.object(
aiohttp.ClientSession,
"request",
return_value=_MockRequest(response_mock),
):
await self.http_client_connection.send(envelope=request_envelope)
# TODO: Consider returning the response from the server in order to be able to assert that the message send!
envelope = await asyncio.wait_for(
self.http_client_connection.receive(), timeout=10)
assert envelope is not None and envelope.message is not None
response = copy.copy(envelope.message)
response.is_incoming = True
response.counterparty = envelope.message.sender
response_dialogue = self.http_dialogs.update(response)
assert response.status_code == response_mock.status, response.bodyy.decode(
"utf-8")
assert sending_dialogue == response_dialogue
await self.http_client_connection.disconnect()
@pytest.mark.asyncio
async def test_http_dialogue_construct_fail(self):
"""Test dialogue not properly constructed."""
await self.http_client_connection.connect()
http_message = HttpMessage(
dialogue_reference=self.http_dialogs.
new_self_initiated_dialogue_reference(),
performative=HttpMessage.Performative.RESPONSE,
status_code=500,
headers="",
status_text="",
bodyy=b"",
version="",
)
http_message.counterparty = self.connection_address
http_dialogue = self.http_dialogs.update(http_message)
http_message.sender = self.agent_address
assert http_dialogue is None
envelope = Envelope(
to=http_message.counterparty,
sender=http_message.sender,
protocol_id=http_message.protocol_id,
message=http_message,
)
with patch.object(self.http_client_connection.channel.logger,
"warning") as mock_logger:
await self.http_client_connection.channel._http_request_task(
envelope)
mock_logger.assert_any_call(
AnyStringWith("Could not create dialogue for message="))
class TestClientServer:
"""Client-Server end-to-end test."""
def setup_server(self):
"""Set up server connection."""
self.server_agent_address = "server_agent_address"
self.server_agent_identity = Identity(
"agent running server", address=self.server_agent_address
)
self.host = get_host()
self.port = get_unused_tcp_port()
self.connection_id = HTTPServerConnection.connection_id
self.protocol_id = PublicId.from_str("fetchai/http:0.4.0")
self.configuration = ConnectionConfig(
host=self.host,
port=self.port,
api_spec_path=None, # do not filter on API spec
connection_id=HTTPServerConnection.connection_id,
restricted_to_protocols=set([self.protocol_id]),
)
self.server = HTTPServerConnection(
configuration=self.configuration, identity=self.server_agent_identity,
)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.server.connect())
# skill side dialogues
self._server_dialogues = HttpDialogues(self.server_agent_address)
def setup_client(self):
"""Set up client connection."""
self.client_agent_address = "client_agent_address"
self.client_agent_identity = Identity(
"agent running client", address=self.client_agent_address
)
configuration = ConnectionConfig(
host="localost",
port="8888", # TODO: remove host/port for client?
connection_id=HTTPClientConnection.connection_id,
)
self.client = HTTPClientConnection(
configuration=configuration, identity=self.client_agent_identity
)
self.client.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.client.connect())
# skill side dialogues
self._client_dialogues = HttpDialogues(self.client_agent_address)
def setup(self):
"""Set up test case."""
self.setup_server()
self.setup_client()
def _make_request(
self, path: str, method: str = "get", headers: str = "", bodyy: bytes = b""
) -> Envelope:
"""Make request envelope."""
request_http_message = HttpMessage(
dialogue_reference=self._client_dialogues.new_self_initiated_dialogue_reference(),
target=0,
message_id=1,
performative=HttpMessage.Performative.REQUEST,
method=method,
url=f"http://{self.host}:{self.port}{path}",
headers="",
version="",
bodyy=b"",
)
request_http_message.counterparty = str(HTTPClientConnection.connection_id)
assert self._client_dialogues.update(request_http_message) is not None
request_envelope = Envelope(
to=request_http_message.counterparty,
sender=request_http_message.sender,
protocol_id=request_http_message.protocol_id,
message=request_http_message,
)
return request_envelope
def _make_response(
self, request_envelope: Envelope, status_code: int = 200, status_text: str = ""
) -> Envelope:
"""Make response envelope."""
incoming_message = cast(HttpMessage, request_envelope.message)
incoming_message.is_incoming = True
incoming_message.counterparty = str(HTTPServerConnection.connection_id)
dialogue = self._server_dialogues.update(incoming_message)
assert dialogue is not None
message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
target=incoming_message.message_id,
message_id=incoming_message.message_id + 1,
version=incoming_message.version,
headers=incoming_message.headers,
status_code=status_code,
status_text=status_text,
bodyy=incoming_message.bodyy,
)
message.counterparty = incoming_message.counterparty
assert dialogue.update(message) is not None
response_envelope = Envelope(
to=message.counterparty,
sender=message.sender,
protocol_id=message.protocol_id,
context=request_envelope.context,
message=message,
)
return response_envelope
@pytest.mark.asyncio
async def test_post_with_payload(self):
"""Test client and server with post request."""
initial_request = self._make_request("/test", "POST", bodyy=b"1234567890")
await self.client.send(initial_request)
request = await asyncio.wait_for(self.server.receive(), timeout=5)
# this is "inside" the server agent
initial_response = self._make_response(request)
await self.server.send(initial_response)
response = await asyncio.wait_for(self.client.receive(), timeout=5)
assert (
cast(HttpMessage, initial_request.message).bodyy
== cast(HttpMessage, response.message).bodyy
)
assert (
initial_request.message.dialogue_reference[0]
== response.message.dialogue_reference[0]
)
def teardown(self):
"""Tear down testcase."""
self.loop.run_until_complete(self.client.disconnect())
self.loop.run_until_complete(self.server.disconnect())
class WebhookChannel:
"""A wrapper for a Webhook."""
def __init__(
self,
agent_address: Address,
webhook_address: Address,
webhook_port: int,
webhook_url_path: str,
connection_id: PublicId,
logger: logging.Logger = _default_logger,
):
"""
Initialize a webhook channel.
:param agent_address: the address of the agent
:param webhook_address: webhook hostname / IP address
:param webhook_port: webhook port number
:param webhook_url_path: the url path to receive webhooks from
:param connection_id: the connection id
"""
self.agent_address = agent_address
self.webhook_address = webhook_address
self.webhook_port = webhook_port
self.webhook_url_path = webhook_url_path
self.webhook_site = None # type: Optional[web.TCPSite]
self.runner = None # type: Optional[web.AppRunner]
self.app = None # type: Optional[web.Application]
self.is_stopped = True
self.connection_id = connection_id
self.in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover
self.logger = logger
self.logger.info("Initialised a webhook channel")
self._dialogues = HttpDialogues(str(WebhookConnection.connection_id))
async def connect(self) -> None:
"""
Connect the webhook.
Connects the webhook via the webhook_address and webhook_port parameters
:return: None
"""
if self.is_stopped:
self.app = web.Application()
self.app.add_routes(
[web.post(self.webhook_url_path, self._receive_webhook)])
self.runner = web.AppRunner(self.app)
await self.runner.setup()
self.webhook_site = web.TCPSite(self.runner, self.webhook_address,
self.webhook_port)
await self.webhook_site.start()
self.is_stopped = False
async def disconnect(self) -> None:
"""
Disconnect.
Shut-off and cleanup the webhook site, the runner and the web app, then stop the channel.
:return: None
"""
assert (self.webhook_site is not None and self.runner is not None
and self.app
is not None), "Application not connected, call connect first!"
if not self.is_stopped:
await self.webhook_site.stop()
await self.runner.shutdown()
await self.runner.cleanup()
await self.app.shutdown()
await self.app.cleanup()
self.logger.info("Webhook app is shutdown.")
self.is_stopped = True
async def _receive_webhook(self, request: web.Request) -> web.Response:
"""
Receive a webhook request.
Get webhook request, turn it to envelop and send it to the agent to be picked up.
:param request: the webhook request
:return: Http response with a 200 code
"""
webhook_envelop = await self.to_envelope(request)
self.in_queue.put_nowait(webhook_envelop) # type: ignore
return web.Response(status=200)
async def send(self, envelope: Envelope) -> None:
"""
Send an envelope.
Sending envelopes via the webhook is not possible!
:param envelope: the envelope
"""
self.logger.warning(
"Dropping envelope={} as sending via the webhook is not possible!".
format(envelope))
async def to_envelope(self, request: web.Request) -> Envelope:
"""
Convert a webhook request object into an Envelope containing an HttpMessage `from the 'http' Protocol`.
:param request: the webhook request
:return: The envelop representing the webhook request
"""
payload_bytes = await request.read()
version = str(request.version[0]) + "." + str(request.version[1])
context = EnvelopeContext(uri=URI("aea/mail/base.py"))
http_message = HttpMessage(
performative=HttpMessage.Performative.REQUEST,
method=request.method,
url=str(request.url),
version=version,
headers=json.dumps(dict(request.headers)),
bodyy=payload_bytes if payload_bytes is not None else b"",
dialogue_reference=self._dialogues.
new_self_initiated_dialogue_reference(),
)
http_message.counterparty = self.agent_address
http_dialogue = self._dialogues.update(http_message)
assert http_dialogue is not None, "Could not create dialogue."
envelope = Envelope(
to=http_message.counterparty,
sender=http_message.sender,
protocol_id=http_message.protocol_id,
context=context,
message=http_message,
)
return envelope
class HTTPChannel(BaseAsyncChannel):
"""A wrapper for an RESTful API with an internal HTTPServer."""
RESPONSE_TIMEOUT = 300
def __init__(
self,
address: Address,
host: str,
port: int,
api_spec_path: Optional[str],
connection_id: PublicId,
restricted_to_protocols: Set[PublicId],
timeout_window: float = 5.0,
logger: logging.Logger = _default_logger,
):
"""
Initialize a channel and process the initial API specification from the file path (if given).
:param address: the address of the agent.
:param host: RESTful API hostname / IP address
:param port: RESTful API port number
:param api_spec_path: Directory API path and filename of the API spec YAML source file.
:param connection_id: public id of connection using this chanel.
:param restricted_to_protocols: set of restricted protocols
:param timeout_window: the timeout (in seconds) for a request to be handled.
"""
super().__init__(address=address, connection_id=connection_id)
self.host = host
self.port = port
self.server_address = "http://{}:{}".format(self.host, self.port)
self.restricted_to_protocols = restricted_to_protocols
self._api_spec = APISpec(api_spec_path, self.server_address, logger)
self.timeout_window = timeout_window
self.http_server: Optional[web.TCPSite] = None
self.pending_requests: Dict[RequestId, Future] = {}
self._dialogues = HttpDialogues(self.address)
self.logger = logger
@property
def api_spec(self) -> APISpec:
"""Get the api spec."""
return self._api_spec
async def connect(self, loop: AbstractEventLoop) -> None:
"""
Connect.
Upon HTTP Channel connection, kickstart the HTTP Server in its own thread.
:param loop: asyncio event loop
:return: None
"""
if self.is_stopped:
await super().connect(loop)
try:
await self._start_http_server()
self.logger.info(
"HTTP Server has connected to port: {}.".format(self.port))
except Exception: # pragma: nocover # pylint: disable=broad-except
self.is_stopped = True
self._in_queue = None
self.logger.exception(
"Failed to start server on {}:{}.".format(
self.host, self.port))
async def _http_handler(self, http_request: BaseRequest) -> Response:
"""
Verify the request then send the request to Agent as an envelope.
:param request: the request object
:return: a tuple of response code and response description
"""
request = await Request.create(http_request)
assert self._in_queue is not None, "Channel not connected!"
is_valid_request = self.api_spec.verify(request)
if not is_valid_request:
self.logger.warning(f"request is not valid: {request}")
return Response(status=NOT_FOUND, reason="Request Not Found")
try:
# turn request into envelope
envelope = request.to_envelope_and_set_id(
self.connection_id,
self.address,
dialogues=self._dialogues,
)
self.pending_requests[request.id] = Future()
# send the envelope to the agent's inbox (via self.in_queue)
await self._in_queue.put(envelope)
# wait for response envelope within given timeout window (self.timeout_window) to appear in dispatch_ready_envelopes
response_message = await asyncio.wait_for(
self.pending_requests[request.id],
timeout=self.RESPONSE_TIMEOUT,
)
return Response.from_message(response_message)
except asyncio.TimeoutError:
return Response(status=REQUEST_TIMEOUT, reason="Request Timeout")
except BaseException: # pragma: nocover # pylint: disable=broad-except
self.logger.exception("Error during handling incoming request")
return Response(status=SERVER_ERROR,
reason="Server Error",
text=format_exc())
finally:
if request.is_id_set:
self.pending_requests.pop(request.id, None)
async def _start_http_server(self) -> None:
"""Start http server."""
server = web.Server(self._http_handler)
runner = web.ServerRunner(server)
await runner.setup()
self.http_server = web.TCPSite(runner, self.host, self.port)
await self.http_server.start()
def send(self, envelope: Envelope) -> None:
"""
Send the envelope in_queue.
:param envelope: the envelope
:return: None
"""
assert self.http_server is not None, "Server not connected, call connect first!"
if envelope.protocol_id not in self.restricted_to_protocols:
self.logger.error(
"This envelope cannot be sent with the http connection: protocol_id={}"
.format(envelope.protocol_id))
raise ValueError("Cannot send message.")
http_message = cast(HttpMessage, envelope.message)
message = copy.copy(
http_message
) # TODO: fix; need to copy atm to avoid overwriting "is_incoming"
message.is_incoming = True # TODO: fix; should be done by framework
message.counterparty = envelope.sender # TODO: fix; should be done by framework
dialogue = self._dialogues.update(message)
if dialogue is None:
self.logger.warning(
"Could not create dialogue for message={}".format(message))
return
future = self.pending_requests.pop(dialogue.incomplete_dialogue_label,
None)
if not future:
self.logger.warning(
"Dropping message={} for incomplete_dialogue_label={} which has timed out."
.format(message, dialogue.incomplete_dialogue_label))
else:
future.set_result(message)
async def disconnect(self) -> None:
"""
Disconnect.
Shut-off the HTTP Server.
"""
assert self.http_server is not None, "Server not connected, call connect first!"
if not self.is_stopped:
await self.http_server.stop()
self.logger.info("HTTP Server has shutdown on port: {}.".format(
self.port))
self.is_stopped = True
self._in_queue = None
class TestHTTPServer:
"""Tests for HTTPServer connection."""
async def request(self, method: str, path: str, **kwargs) -> ClientResponse:
"""
Make a http request.
:param method: HTTP method: GET, POST etc
:param path: path to request on server. full url constructed automatically
:return: http response
"""
try:
url = f"http://{self.host}:{self.port}{path}"
async with aiohttp.ClientSession() as session:
async with session.request(method, url, **kwargs) as resp:
await resp.read()
return resp
except Exception:
print_exc()
raise
def setup(self):
"""Initialise the test case."""
self.identity = Identity("name", address="my_key")
self.agent_address = self.identity.address
self.host = get_host()
self.port = get_unused_tcp_port()
self.api_spec_path = os.path.join(
ROOT_DIR, "tests", "data", "petstore_sim.yaml"
)
self.connection_id = HTTPServerConnection.connection_id
self.protocol_id = PublicId.from_str("fetchai/http:0.4.0")
self.configuration = ConnectionConfig(
host=self.host,
port=self.port,
api_spec_path=self.api_spec_path,
connection_id=HTTPServerConnection.connection_id,
restricted_to_protocols=set([self.protocol_id]),
)
self.http_connection = HTTPServerConnection(
configuration=self.configuration, identity=self.identity,
)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.http_connection.connect())
self.connection_address = str(HTTPServerConnection.connection_id)
self._dialogues = HttpDialogues(self.connection_address)
self.original_timeout = self.http_connection.channel.RESPONSE_TIMEOUT
@pytest.mark.asyncio
async def test_http_connection_disconnect_channel(self):
"""Test the disconnect."""
await self.http_connection.channel.disconnect()
assert self.http_connection.channel.is_stopped
def _get_message_and_dialogue(
self, envelope: Envelope
) -> Tuple[HttpMessage, HttpDialogue]:
message = cast(HttpMessage, envelope.message)
message = copy.copy(
message
) # TODO: fix; need to copy atm to avoid overwriting "is_incoming"
message.is_incoming = True # TODO: fix; should be done by framework
message.counterparty = envelope.sender # TODO: fix; should be done by framework
dialogue = cast(HttpDialogue, self._dialogues.update(message))
assert dialogue is not None
return message, dialogue
@pytest.mark.asyncio
async def test_get_200(self):
"""Test send get request w/ 200 response."""
request_task = self.loop.create_task(self.request("get", "/pets"))
envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=20)
assert envelope
incoming_message, dialogue = self._get_message_and_dialogue(envelope)
message = HttpMessage(
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
performative=HttpMessage.Performative.RESPONSE,
version=incoming_message.version,
headers=incoming_message.headers,
message_id=incoming_message.message_id + 1,
target=incoming_message.message_id,
status_code=200,
status_text="Success",
bodyy=b"Response body",
)
message.counterparty = incoming_message.counterparty
assert dialogue.update(message)
response_envelope = Envelope(
to=envelope.sender,
sender=envelope.to,
protocol_id=envelope.protocol_id,
context=envelope.context,
message=message,
)
await self.http_connection.send(response_envelope)
response = await asyncio.wait_for(request_task, timeout=20,)
assert (
response.status == 200
and response.reason == "Success"
and await response.text() == "Response body"
)
@pytest.mark.asyncio
async def test_bad_performative_get_timeout_error(self):
"""Test send get request w/ 200 response."""
self.http_connection.channel.RESPONSE_TIMEOUT = 3
request_task = self.loop.create_task(self.request("get", "/pets"))
envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=10)
assert envelope
incoming_message, dialogue = self._get_message_and_dialogue(envelope)
message = HttpMessage(
performative=HttpMessage.Performative.REQUEST,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
target=incoming_message.message_id,
message_id=incoming_message.message_id + 1,
method="post",
url="/pets",
version=incoming_message.version,
headers=incoming_message.headers,
bodyy=b"Request body",
)
message.counterparty = incoming_message.counterparty
assert not dialogue.update(message)
response_envelope = Envelope(
to=envelope.sender,
sender=envelope.to,
protocol_id=envelope.protocol_id,
context=envelope.context,
message=message,
)
with patch.object(self.http_connection.logger, "warning") as mock_logger:
await self.http_connection.send(response_envelope)
mock_logger.assert_any_call(
f"Could not create dialogue for message={message}"
)
response = await asyncio.wait_for(request_task, timeout=10)
assert (
response.status == 408
and response.reason == "Request Timeout"
and await response.text() == ""
)
@pytest.mark.asyncio
async def test_late_message_get_timeout_error(self):
"""Test send get request w/ 200 response."""
self.http_connection.channel.RESPONSE_TIMEOUT = 1
request_task = self.loop.create_task(self.request("get", "/pets"))
envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=10)
assert envelope
incoming_message, dialogue = self._get_message_and_dialogue(envelope)
message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
target=incoming_message.message_id,
message_id=incoming_message.message_id + 1,
version=incoming_message.version,
headers=incoming_message.headers,
status_code=200,
status_text="Success",
bodyy=b"Response body",
)
message.counterparty = incoming_message.counterparty
assert dialogue.update(message)
response_envelope = Envelope(
to=envelope.sender,
sender=envelope.to,
protocol_id=envelope.protocol_id,
context=envelope.context,
message=message,
)
await asyncio.sleep(1.5)
with patch.object(self.http_connection.logger, "warning") as mock_logger:
await self.http_connection.send(response_envelope)
mock_logger.assert_any_call(
RegexComparator(
"Dropping message=.* for incomplete_dialogue_label=.* which has timed out."
)
)
response = await asyncio.wait_for(request_task, timeout=10)
assert (
response.status == 408
and response.reason == "Request Timeout"
and await response.text() == ""
)
@pytest.mark.asyncio
async def test_post_201(self):
"""Test send get request w/ 200 response."""
request_task = self.loop.create_task(self.request("post", "/pets",))
envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=20)
assert envelope
incoming_message, dialogue = self._get_message_and_dialogue(envelope)
message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
target=incoming_message.message_id,
message_id=incoming_message.message_id + 1,
version=incoming_message.version,
headers=incoming_message.headers,
status_code=201,
status_text="Created",
bodyy=b"Response body",
)
message.counterparty = incoming_message.counterparty
assert dialogue.update(message)
response_envelope = Envelope(
to=envelope.sender,
sender=envelope.to,
protocol_id=envelope.protocol_id,
context=envelope.context,
message=message,
)
await self.http_connection.send(response_envelope)
response = await asyncio.wait_for(request_task, timeout=20,)
assert (
response.status == 201
and response.reason == "Created"
and await response.text() == "Response body"
)
@pytest.mark.asyncio
async def test_get_404(self):
"""Test send post request w/ 404 response."""
response = await self.request("get", "/url-non-exists")
assert (
response.status == 404
and response.reason == "Request Not Found"
and await response.text() == ""
)
@pytest.mark.asyncio
async def test_post_404(self):
"""Test send post request w/ 404 response."""
response = await self.request("get", "/url-non-exists", data="some data")
assert (
response.status == 404
and response.reason == "Request Not Found"
and await response.text() == ""
)
@pytest.mark.asyncio
async def test_get_408(self):
"""Test send post request w/ 404 response."""
await self.http_connection.connect()
self.http_connection.channel.RESPONSE_TIMEOUT = 0.1
response = await self.request("get", "/pets")
assert (
response.status == 408
and response.reason == "Request Timeout"
and await response.text() == ""
)
@pytest.mark.asyncio
async def test_post_408(self):
"""Test send post request w/ 404 response."""
self.http_connection.channel.RESPONSE_TIMEOUT = 0.1
response = await self.request("post", "/pets", data="somedata")
assert (
response.status == 408
and response.reason == "Request Timeout"
and await response.text() == ""
)
@pytest.mark.asyncio
async def test_send_connection_drop(self):
"""Test unexpected response."""
message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
dialogue_reference=("", ""),
target=1,
message_id=2,
headers="",
version="",
status_code=200,
status_text="Success",
bodyy=b"",
)
message.counterparty = "to_key"
message.sender = "from_key"
envelope = Envelope(
to=message.counterparty,
sender=message.sender,
protocol_id=message.protocol_id,
message=message,
)
await self.http_connection.send(envelope)
@pytest.mark.asyncio
async def test_get_message_channel_not_connected(self):
"""Test error on channel get message if not connected."""
await self.http_connection.disconnect()
with pytest.raises(ValueError):
await self.http_connection.channel.get_message()
@pytest.mark.asyncio
async def test_fail_connect(self):
"""Test error on server connection."""
await self.http_connection.disconnect()
with patch.object(
self.http_connection.channel,
"_start_http_server",
side_effect=Exception("expected"),
):
await self.http_connection.connect()
assert not self.http_connection.is_connected
@pytest.mark.asyncio
async def test_server_error_on_send_response(self):
"""Test exception raised on response sending to the client."""
request_task = self.loop.create_task(self.request("post", "/pets",))
envelope = await asyncio.wait_for(self.http_connection.receive(), timeout=20)
assert envelope
incoming_message, dialogue = self._get_message_and_dialogue(envelope)
message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
target=incoming_message.message_id,
message_id=incoming_message.message_id + 1,
version=incoming_message.version,
headers=incoming_message.headers,
status_code=201,
status_text="Created",
bodyy=b"Response body",
)
message.counterparty = incoming_message.counterparty
assert dialogue.update(message)
response_envelope = Envelope(
to=envelope.sender,
sender=envelope.to,
protocol_id=envelope.protocol_id,
context=envelope.context,
message=message,
)
with patch.object(Response, "from_message", side_effect=Exception("expected")):
await self.http_connection.send(response_envelope)
response = await asyncio.wait_for(request_task, timeout=20,)
assert response and response.status == 500 and response.reason == "Server Error"
@pytest.mark.asyncio
async def test_send_envelope_restricted_to_protocols_fail(self):
"""Test fail on send if envelope protocol not supported."""
message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
dialogue_reference=("", ""),
target=1,
message_id=2,
version="1.0",
headers="",
status_code=200,
status_text="Success",
bodyy=b"Response body",
)
envelope = Envelope(
to="receiver",
sender="sender",
protocol_id=UNKNOWN_PROTOCOL_PUBLIC_ID,
message=message,
)
with patch.object(
self.http_connection.channel,
"restricted_to_protocols",
new=[HTTP_PROTOCOL_PUBLIC_ID],
):
with pytest.raises(ValueError):
await self.http_connection.send(envelope)
def teardown(self):
"""Teardown the test case."""
self.loop.run_until_complete(self.http_connection.disconnect())
self.http_connection.channel.RESPONSE_TIMEOUT = self.original_timeout
class HTTPClientAsyncChannel:
"""A wrapper for a HTTPClient."""
DEFAULT_TIMEOUT = 300 # default timeout in seconds
DEFAULT_EXCEPTION_CODE = (
600 # custom code to indicate there was exception during request
)
def __init__(
self,
agent_address: Address,
address: str,
port: int,
connection_id: PublicId,
excluded_protocols: Optional[Set[PublicId]] = None,
restricted_to_protocols: Optional[Set[PublicId]] = None,
):
"""
Initialize an http client channel.
:param agent_address: the address of the agent.
:param address: server hostname / IP address
:param port: server port number
:param excluded_protocols: this connection cannot handle messages adhering to any of the protocols in this set
:param restricted_to_protocols: this connection can only handle messages adhering to protocols in this set
"""
self.agent_address = agent_address
self.address = address
self.port = port
self.connection_id = connection_id
self.restricted_to_protocols = restricted_to_protocols
self._dialogues = HttpDialogues(str(
HTTPClientConnection.connection_id))
self._in_queue = None # type: Optional[asyncio.Queue] # pragma: no cover
self._loop = (
None
) # type: Optional[asyncio.AbstractEventLoop] # pragma: no cover
self.excluded_protocols = excluded_protocols
self.is_stopped = True
self._tasks: Set[Task] = set()
self.logger = logger
self.logger.info("Initialised the HTTP client channel")
async def connect(self, loop: AbstractEventLoop) -> None:
"""
Connect channel using loop.
:param loop: asyncio event loop to use
:return: None
"""
self._loop = loop
self._in_queue = asyncio.Queue()
self.is_stopped = False
def _get_message_and_dialogue(
self,
envelope: Envelope) -> Tuple[HttpMessage, Optional[HttpDialogue]]:
"""
Get a message copy and dialogue related to this message.
:param envelope: incoming envelope
:return: Tuple[MEssage, Optional[Dialogue]]
"""
orig_message = cast(HttpMessage, envelope.message)
message = copy.copy(
orig_message
) # TODO: fix; need to copy atm to avoid overwriting "is_incoming"
message.is_incoming = True # TODO: fix; should be done by framework
message.counterparty = (orig_message.sender
) # TODO: fix; should be done by framework
dialogue = cast(HttpDialogue, self._dialogues.update(message))
return message, dialogue
async def _http_request_task(self, request_envelope: Envelope) -> None:
"""
Perform http request and send back response.
:param request_http_message: HttpMessage with http request constructed.
:return: None
"""
if not self._loop: # pragma: nocover
raise ValueError("Channel is not connected")
request_http_message, dialogue = self._get_message_and_dialogue(
request_envelope)
if not dialogue:
self.logger.warning(
"Could not create dialogue for message={}".format(
request_http_message))
return
try:
resp = await asyncio.wait_for(
self._perform_http_request(request_http_message),
timeout=self.DEFAULT_TIMEOUT,
)
envelope = self.to_envelope(
self.connection_id,
request_http_message,
status_code=resp.status,
headers=resp.headers,
status_text=resp.reason,
bodyy=resp._body # pylint: disable=protected-access
if resp._body is not None # pylint: disable=protected-access
else b"",
dialogue=dialogue,
)
except Exception: # pragma: nocover # pylint: disable=broad-except
envelope = self.to_envelope(
self.connection_id,
request_http_message,
status_code=self.DEFAULT_EXCEPTION_CODE,
headers={},
status_text="HTTPConnection request error.",
bodyy=format_exc().encode("utf-8"),
dialogue=dialogue,
)
if self._in_queue is not None:
await self._in_queue.put(envelope)
async def _perform_http_request(
self, request_http_message: HttpMessage) -> ClientResponse:
"""
Perform http request and return response.
:param request_http_message: HttpMessage with http request constructed.
:return: aiohttp.ClientResponse
"""
try:
async with aiohttp.ClientSession() as session:
async with session.request(
method=request_http_message.method,
url=request_http_message.url,
headers=request_http_message.headers,
data=request_http_message.bodyy,
) as resp:
await resp.read()
return resp
except Exception: # pragma: nocover # pylint: disable=broad-except
self.logger.exception(
f"Exception raised during http call: {request_http_message.method} {request_http_message.url}"
)
raise
def send(self, request_envelope: Envelope) -> None:
"""
Send an envelope with http request data to request.
Convert an http envelope into an http request.
Send the http request
Wait for and receive its response
Translate the response into a response envelop.
Send the response envelope to the in-queue.
:param request_envelope: the envelope containing an http request
:return: None
"""
if self._loop is None or self.is_stopped:
raise ValueError("Can not send a message! Channel is not started!")
if request_envelope is None:
return
if request_envelope.protocol_id in (self.excluded_protocols or []):
self.logger.error(
"This envelope cannot be sent with the http client connection: protocol_id={}"
.format(request_envelope.protocol_id))
raise ValueError("Cannot send message.")
assert isinstance(request_envelope.message,
HttpMessage), "Message not of type HttpMessage"
request_http_message = cast(HttpMessage, request_envelope.message)
if (request_http_message.performative !=
HttpMessage.Performative.REQUEST): # pragma: nocover
self.logger.warning(
"The HTTPMessage performative must be a REQUEST. Envelop dropped."
)
return
task = self._loop.create_task(
self._http_request_task(request_envelope))
task.add_done_callback(self._task_done_callback)
self._tasks.add(task)
def _task_done_callback(self, task: Task) -> None:
"""
Handle http request task completed.
Removes tasks from _tasks.
:param task: Task completed.
:return: None
"""
self._tasks.remove(task)
self.logger.debug(f"Task completed: {task}")
async def get_message(self) -> Union["Envelope", None]:
"""
Get http response from in-queue.
:return: None or envelope with http response.
"""
if self._in_queue is None:
raise ValueError("Looks like channel is not connected!")
try:
return await self._in_queue.get()
except CancelledError: # pragma: nocover
return None
def to_envelope(
self,
connection_id: PublicId,
http_request_message: HttpMessage,
status_code: int,
headers: dict,
status_text: Optional[Any],
bodyy: bytes,
dialogue: HttpDialogue,
) -> Envelope:
"""
Convert an HTTP response object (from the 'requests' library) into an Envelope containing an HttpMessage (from the 'http' Protocol).
:param connection_id: the connection id
:param http_request_message: the message of the http request envelop
:param status_code: the http status code, int
:param headers: dict of http response headers
:param status_text: the http status_text, str
:param bodyy: bytes of http response content
:return: Envelope with http response data.
"""
context = EnvelopeContext(connection_id=connection_id)
http_message = HttpMessage(
performative=HttpMessage.Performative.RESPONSE,
status_code=status_code,
headers=json.dumps(dict(headers.items())),
status_text=status_text,
bodyy=bodyy,
version="",
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
target=http_request_message.message_id,
message_id=http_request_message.message_id + 1,
)
http_message.counterparty = http_request_message.counterparty
assert dialogue.update(http_message)
envelope = Envelope(
to=self.agent_address,
sender="HTTP Server",
protocol_id=PublicId.from_str("fetchai/http:0.4.0"),
context=context,
message=http_message,
)
return envelope
async def _cancel_tasks(self) -> None:
"""Cancel all requests tasks pending."""
for task in list(self._tasks):
if task.done(): # pragma: nocover
continue
task.cancel()
for task in list(self._tasks):
try:
await task
except KeyboardInterrupt: # pragma: nocover
raise
except BaseException: # pragma: nocover # pylint: disable=broad-except
pass # nosec
async def disconnect(self) -> None:
"""Disconnect."""
if not self.is_stopped:
self.logger.info("HTTP Client has shutdown on port: {}.".format(
self.port))
self.is_stopped = True
await self._cancel_tasks()
class TestWebhookConnection:
"""Tests the webhook connection's 'connect' functionality."""
def setup(self):
"""Initialise the class."""
self.host = get_host()
self.port = get_unused_tcp_port()
self.identity = Identity("", address="some string")
self.path = "/webhooks/topic/{topic}/"
self.loop = asyncio.get_event_loop()
configuration = ConnectionConfig(
webhook_address=self.host,
webhook_port=self.port,
webhook_url_path=self.path,
connection_id=WebhookConnection.connection_id,
)
self.webhook_connection = WebhookConnection(
configuration=configuration, identity=self.identity,
)
self.webhook_connection.loop = self.loop
self.dialogues = HttpDialogues(self.identity.address)
async def test_initialization(self):
"""Test the initialisation of the class."""
assert self.webhook_connection.address == self.identity.address
@pytest.mark.asyncio
async def test_connection(self):
"""Test the connect functionality of the webhook connection."""
await self.webhook_connection.connect()
assert self.webhook_connection.is_connected is True
@pytest.mark.asyncio
async def test_disconnect(self):
"""Test the disconnect functionality of the webhook connection."""
await self.webhook_connection.connect()
assert self.webhook_connection.is_connected is True
await self.webhook_connection.disconnect()
assert self.webhook_connection.is_connected is False
def teardown(self):
"""Close connection after testing."""
try:
self.loop.run_until_complete(self.webhook_connection.disconnect())
except Exception:
print_exc()
raise
@pytest.mark.asyncio
async def test_receive_post_ok(self):
"""Test the connect functionality of the webhook connection."""
await self.webhook_connection.connect()
assert self.webhook_connection.is_connected is True
payload = {"hello": "world"}
call_task = self.loop.create_task(self.call_webhook("test_topic", json=payload))
envelope = await asyncio.wait_for(self.webhook_connection.receive(), timeout=10)
assert envelope
orig_message = cast(HttpMessage, envelope.message)
message = copy.copy(orig_message)
message.counterparty = orig_message.sender
message.is_incoming = True
dialogue = self.dialogues.update(message)
assert dialogue is not None
assert message.method.upper() == "POST"
assert message.bodyy.decode("utf-8") == json.dumps(payload)
await call_task
@pytest.mark.asyncio
async def test_send(self):
"""Test the connect functionality of the webhook connection."""
await self.webhook_connection.connect()
assert self.webhook_connection.is_connected is True
http_message = HttpMessage(
dialogue_reference=("", ""),
target=0,
message_id=1,
performative=HttpMessage.Performative.REQUEST,
method="get",
url="/",
headers="",
bodyy="",
version="",
)
envelope = Envelope(
to="addr",
sender="my_id",
protocol_id=PublicId.from_str("fetchai/http:0.4.0"),
message=http_message,
)
with patch.object(self.webhook_connection.logger, "warning") as mock_logger:
await self.webhook_connection.send(envelope)
await asyncio.sleep(0.01)
mock_logger.assert_any_call(
RegexComparator(
"Dropping envelope=.* as sending via the webhook is not possible!"
)
)
async def call_webhook(self, topic: str, **kwargs) -> ClientResponse:
"""
Make a http request to a webhook.
:param topic: topic to use
:params **kwargs: data or json for payload
:return: http response
"""
path = self.path.format(topic=topic)
method = kwargs.get("method", "post")
url = f"http://{self.host}:{self.port}{path}"
try:
async with aiohttp.ClientSession() as session:
async with session.request(method, url, **kwargs) as resp:
await resp.read()
return resp
except Exception:
print_exc()
raise
class HttpPingPongHandler(Handler):
"""Dummy handler to handle messages."""
SUPPORTED_PROTOCOL = HttpMessage.protocol_id
def setup(self) -> None:
"""Noop setup."""
# pylint: disable=attribute-defined-outside-init, unused-argument
self.count: int = 0
self.rtt_total_time: float = 0.0
self.rtt_count: int = 0
self.latency_total_time: float = 0.0
self.latency_count: int = 0
def role(m: Message, addr: Address) -> Dialogue.Role:
return HttpDialogue.Role.CLIENT
self.dialogues = HttpDialogues(self.context.agent_address,
role_from_first_message=role)
def teardown(self) -> None:
"""Noop teardown."""
def handle(self, message: Message) -> None:
"""Handle incoming message."""
self.count += 1
message = cast(HttpMessage, message)
dialogue = self.dialogues.update(message)
if not dialogue:
raise Exception("something goes wrong")
rtt_ts, latency_ts = struct.unpack("dd", message.body) # type: ignore
if message.performative == HttpMessage.Performative.REQUEST:
self.latency_total_time += time.time() - latency_ts
self.latency_count += 1
self.make_response(cast(HttpDialogue, dialogue), message)
elif message.performative == HttpMessage.Performative.RESPONSE:
self.rtt_total_time += time.time() - rtt_ts
self.rtt_count += 1
# got response, make another requet to the same agent
self.make_request(message.sender)
def make_response(self, dialogue: HttpDialogue,
message: HttpMessage) -> None:
"""Construct and send a response for message received."""
response_message = dialogue.reply(
target_message=message,
performative=HttpMessage.Performative.RESPONSE,
version=message.version,
headers="",
status_code=200,
status_text="Success",
body=message.body,
)
self.context.outbox.put_message(response_message)
def make_request(self, recipient_addr: str) -> None:
"""Make initial http request."""
message, _ = self.dialogues.create(
recipient_addr,
performative=HttpMessage.Performative.REQUEST,
method="get",
url="some url",
headers="",
version="",
body=struct.pack("dd", time.time(), time.time()),
)
self.context.outbox.put_message(message)
class TestClientServer:
"""Client-Server end-to-end test."""
def setup_server(self):
"""Set up server connection."""
self.server_agent_address = "server_agent_address"
self.server_agent_identity = Identity(
"agent running server", address=self.server_agent_address)
self.host = get_host()
self.port = get_unused_tcp_port()
self.connection_id = HTTPServerConnection.connection_id
self.protocol_id = HttpMessage.protocol_id
self.configuration = ConnectionConfig(
host=self.host,
port=self.port,
api_spec_path=None, # do not filter on API spec
connection_id=HTTPServerConnection.connection_id,
restricted_to_protocols=set([self.protocol_id]),
)
self.server = HTTPServerConnection(
configuration=self.configuration,
identity=self.server_agent_identity,
)
self.loop = asyncio.get_event_loop()
self.loop.run_until_complete(self.server.connect())
# skill side dialogues
def role_from_first_message( # pylint: disable=unused-argument
message: Message,
receiver_address: Address) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
return HttpDialogue.Role.SERVER
self._server_dialogues = HttpDialogues(
self.server_agent_address,
role_from_first_message=role_from_first_message)
def setup_client(self):
"""Set up client connection."""
self.client_agent_address = "client_agent_address"
self.client_agent_identity = Identity(
"agent running client", address=self.client_agent_address)
configuration = ConnectionConfig(
host="localost",
port="8888", # TODO: remove host/port for client?
connection_id=HTTPClientConnection.connection_id,
)
self.client = HTTPClientConnection(configuration=configuration,
identity=self.client_agent_identity)
self.loop.run_until_complete(self.client.connect())
# skill side dialogues
def role_from_first_message( # pylint: disable=unused-argument
message: Message,
receiver_address: Address) -> BaseDialogue.Role:
"""Infer the role of the agent from an incoming/outgoing first message
:param message: an incoming/outgoing first message
:param receiver_address: the address of the receiving agent
:return: The role of the agent
"""
return HttpDialogue.Role.CLIENT
self._client_dialogues = HttpDialogues(
self.client_agent_address,
role_from_first_message=role_from_first_message)
def setup(self):
"""Set up test case."""
self.setup_server()
self.setup_client()
def _make_request(self,
path: str,
method: str = "get",
headers: str = "",
body: bytes = b"") -> Envelope:
"""Make request envelope."""
request_http_message, _ = self._client_dialogues.create(
counterparty=str(HTTPClientConnection.connection_id),
performative=HttpMessage.Performative.REQUEST,
method=method,
url=f"http://{self.host}:{self.port}{path}",
headers="",
version="",
body=b"",
)
request_envelope = Envelope(
to=request_http_message.to,
sender=request_http_message.sender,
protocol_id=request_http_message.protocol_id,
message=request_http_message,
)
return request_envelope
def _make_response(self,
request_envelope: Envelope,
status_code: int = 200,
status_text: str = "") -> Envelope:
"""Make response envelope."""
incoming_message = cast(HttpMessage, request_envelope.message)
dialogue = self._server_dialogues.update(incoming_message)
assert dialogue is not None
message = dialogue.reply(
target_message=incoming_message,
performative=HttpMessage.Performative.RESPONSE,
version=incoming_message.version,
headers=incoming_message.headers,
status_code=status_code,
status_text=status_text,
body=incoming_message.body,
)
response_envelope = Envelope(
to=message.to,
sender=message.sender,
protocol_id=message.protocol_id,
context=request_envelope.context,
message=message,
)
return response_envelope
@pytest.mark.asyncio
async def test_post_with_payload(self):
"""Test client and server with post request."""
initial_request = self._make_request("/test",
"POST",
body=b"1234567890")
await self.client.send(initial_request)
request = await asyncio.wait_for(self.server.receive(), timeout=5)
# this is "inside" the server agent
initial_response = self._make_response(request)
await self.server.send(initial_response)
response = await asyncio.wait_for(self.client.receive(), timeout=5)
assert (cast(HttpMessage, initial_request.message).body == cast(
HttpMessage, response.message).body)
assert (initial_request.message.dialogue_reference[0] ==
response.message.dialogue_reference[0])
@pytest.mark.asyncio
async def test_get_with_query(self):
"""Test client and server with url query."""
query = {"key": "value"}
path = "/test?{}".format(urllib.parse.urlencode(query))
initial_request = self._make_request(path, "GET")
await self.client.send(initial_request)
request = await asyncio.wait_for(self.server.receive(), timeout=5)
# this is "inside" the server agent
parsed_query = dict(
urllib.parse.parse_qsl(
urllib.parse.splitquery(
cast(HttpMessage, request.message).url)[1]))
assert parsed_query == query
initial_response = self._make_response(request)
await self.server.send(initial_response)
response = await asyncio.wait_for(self.client.receive(), timeout=5)
assert (initial_request.message.dialogue_reference[0] ==
response.message.dialogue_reference[0])
def teardown(self):
"""Tear down testcase."""
self.loop.run_until_complete(self.client.disconnect())
self.loop.run_until_complete(self.server.disconnect())