async def send_reset(self) -> GymDialogue:
"""Send a reset."""
msg = GymMessage(
performative=GymMessage.Performative.RESET,
dialogue_reference=self.dialogues.
new_self_initiated_dialogue_reference(),
)
msg.counterparty = self.gym_address
sending_dialogue = self.dialogues.update(msg)
assert sending_dialogue is not None
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
with patch.object(self.env, "reset") as mock:
await self.gym_con.send(envelope)
mock.assert_called()
response = await asyncio.wait_for(self.gym_con.receive(), timeout=3)
response_msg_orig = cast(GymMessage, response.message)
response_msg = copy.copy(response_msg_orig)
response_msg.is_incoming = True
response_msg.counterparty = response_msg_orig.sender
response_dialogue = self.dialogues.update(response_msg)
assert response_msg.performative == GymMessage.Performative.STATUS
assert response_msg.content == {"reset": "success"}
assert sending_dialogue == response_dialogue
return sending_dialogue
async def test_send_close_negative(self):
"""Test send close message with invalid reference and message id and target."""
incorrect_msg = GymMessage(
performative=GymMessage.Performative.CLOSE,
dialogue_reference=self.dialogues.
new_self_initiated_dialogue_reference(),
)
incorrect_msg.to = self.gym_address
incorrect_msg.sender = self.agent_address
# the incorrect message cannot be sent into a dialogue, so this is omitted.
envelope = Envelope(
to=incorrect_msg.to,
sender=incorrect_msg.sender,
protocol_specification_id=incorrect_msg.protocol_specification_id,
message=incorrect_msg,
)
await self.gym_con.connect()
with patch.object(self.gym_con.channel.logger,
"warning") as mock_logger:
await self.gym_con.send(envelope)
mock_logger.assert_any_call(
f"Could not create dialogue from message={incorrect_msg}")
def test_status_serialization():
"""Test the serialization for 'status' speech-act works."""
content_arg = {
"key_1": "value_1",
"key_2": "value_2",
}
msg = GymMessage(
message_id=1,
dialogue_reference=(str(0), ""),
target=0,
performative=GymMessage.Performative.STATUS,
content=content_arg,
)
msg.to = "receiver"
envelope = Envelope(
to=msg.to,
sender="sender",
message=msg,
)
envelope_bytes = envelope.encode()
actual_envelope = Envelope.decode(envelope_bytes)
expected_envelope = envelope
assert expected_envelope.to == actual_envelope.to
assert expected_envelope.sender == actual_envelope.sender
assert (expected_envelope.protocol_specification_id ==
actual_envelope.protocol_specification_id)
assert expected_envelope.message != actual_envelope.message
actual_msg = GymMessage.serializer.decode(actual_envelope.message)
actual_msg.to = actual_envelope.to
actual_msg.sender = actual_envelope.sender
expected_msg = msg
assert expected_msg == actual_msg
async def test_send_close(self):
"""Test send close message."""
sending_dialogue = await self.send_reset()
last_message = sending_dialogue.last_message
assert last_message is not None
msg = GymMessage(
performative=GymMessage.Performative.CLOSE,
dialogue_reference=sending_dialogue.dialogue_label.
dialogue_reference,
message_id=last_message.message_id + 1,
target=last_message.message_id,
)
msg.counterparty = self.gym_address
assert sending_dialogue.update(msg)
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
with patch.object(self.env, "close") as mock:
await self.gym_con.send(envelope)
mock.assert_called()
def decode(self, obj: bytes) -> Message:
"""
Decode the message.
:param obj: the bytes object
:return: the message
"""
json_msg = json.loads(obj.decode("utf-8"))
performative = GymMessage.Performative(json_msg["performative"])
new_body = copy.copy(json_msg)
new_body["performative"] = performative
if performative == GymMessage.Performative.ACT:
action_bytes = base64.b64decode(json_msg["action"])
action = pickle.loads(action_bytes) # nosec
new_body["action"] = action
new_body["step_id"] = json_msg["step_id"]
elif performative == GymMessage.Performative.PERCEPT:
# observation, reward and info are gym implementation specific, done is boolean
observation_bytes = base64.b64decode(json_msg["observation"])
observation = pickle.loads(observation_bytes) # nosec
new_body["observation"] = observation
reward_bytes = base64.b64decode(json_msg["reward"])
reward = pickle.loads(reward_bytes) # nosec
new_body["reward"] = reward
info_bytes = base64.b64decode(json_msg["info"])
info = pickle.loads(info_bytes) # nosec
new_body["info"] = info
new_body["step_id"] = json_msg["step_id"]
gym_message = GymMessage(performative=performative, body=new_body)
return gym_message
def test_act_serialization():
"""Test the serialization for 'act' speech-act works."""
msg = GymMessage(
message_id=1,
dialogue_reference=(str(0), ""),
target=0,
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("some_action"),
step_id=1,
)
msg.to = "receiver"
envelope = Envelope(
to=msg.to,
sender="sender",
message=msg,
)
envelope_bytes = envelope.encode()
actual_envelope = Envelope.decode(envelope_bytes)
expected_envelope = envelope
assert expected_envelope.to == actual_envelope.to
assert expected_envelope.sender == actual_envelope.sender
assert (expected_envelope.protocol_specification_id ==
actual_envelope.protocol_specification_id)
assert expected_envelope.message != actual_envelope.message
actual_msg = GymMessage.serializer.decode(actual_envelope.message)
actual_msg.to = actual_envelope.to
actual_msg.sender = actual_envelope.sender
expected_msg = msg
assert expected_msg == actual_msg
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
if not self._agent.multiplexer.is_connected:
self._connect()
gym_msg = GymMessage(
dialogue_reference=self.gym_dialogues.
new_self_initiated_dialogue_reference(),
performative=GymMessage.Performative.RESET,
)
gym_msg.counterparty = self.gym_address
gym_dialogue = cast(Optional[GymDialogue],
self.gym_dialogues.update(gym_msg))
assert gym_dialogue is not None
self._active_dialogue = gym_dialogue
self._agent.outbox.put_message(message=gym_msg)
# Wait (blocking!) for the response envelope from the environment
in_envelope = self._queue.get(block=True,
timeout=None) # type: GymMessage
self._decode_status(in_envelope)
def test_percept_serialization():
"""Test the serialization for 'percept' speech-act works."""
msg = GymMessage(
message_id=2,
dialogue_reference=(str(0), ""),
target=1,
performative=GymMessage.Performative.PERCEPT,
step_id=1,
observation=GymMessage.AnyObject("some_observation"),
reward=10.0,
done=False,
info=GymMessage.AnyObject("some_info"),
)
msg.to = "receiver"
envelope = Envelope(
to=msg.to,
sender="sender",
message=msg,
)
envelope_bytes = envelope.encode()
actual_envelope = Envelope.decode(envelope_bytes)
expected_envelope = envelope
assert expected_envelope.to == actual_envelope.to
assert expected_envelope.sender == actual_envelope.sender
assert (expected_envelope.protocol_specification_id ==
actual_envelope.protocol_specification_id)
assert expected_envelope.message != actual_envelope.message
actual_msg = GymMessage.serializer.decode(actual_envelope.message)
actual_msg.to = actual_envelope.to
actual_msg.sender = actual_envelope.sender
expected_msg = msg
assert expected_msg == actual_msg
def handle_gym_message(self, envelope: Envelope) -> None:
"""
Forward a message to gym.
:param envelope: the envelope
:return: None
"""
gym_message = GymSerializer().decode(envelope.message)
gym_message = cast(GymMessage, gym_message)
if gym_message.performative == GymMessage.Performative.ACT:
action = gym_message.action.any
step_id = gym_message.step_id
observation, reward, done, info = self.gym_env.step(action) # type: ignore
msg = GymMessage(
performative=GymMessage.Performative.PERCEPT,
observation=GymMessage.AnyObject(observation),
reward=reward,
done=done,
info=GymMessage.AnyObject(info),
step_id=step_id,
)
msg_bytes = GymSerializer().encode(msg)
envelope = Envelope(
to=envelope.sender,
sender=DEFAULT_GYM,
protocol_id=GymMessage.protocol_id,
message=msg_bytes,
)
self._send(envelope)
elif gym_message.performative == GymMessage.Performative.RESET:
self.gym_env.reset() # type: ignore
elif gym_message.performative == GymMessage.Performative.CLOSE:
self.gym_env.close() # type: ignore
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
self._step_count = 0
self._is_rl_agent_trained = False
gym_msg = GymMessage(
dialogue_reference=self.gym_dialogues.
new_self_initiated_dialogue_reference(),
performative=GymMessage.Performative.RESET,
)
gym_msg.counterparty = self.gym_address
gym_dialogue = cast(Optional[GymDialogue],
self.gym_dialogues.update(gym_msg))
assert gym_dialogue is not None
self._active_dialogue = gym_dialogue
self._skill_context.outbox.put_message(message=gym_msg)
# Wait (blocking!) for the response envelope from the environment
response_msg = self._queue.get(block=True,
timeout=None) # type: GymMessage
if response_msg.performative != GymMessage.Performative.STATUS:
raise ValueError(
"Unexpected performative. Expected={} got={}".format(
GymMessage.Performative.PERCEPT,
response_msg.performative))
def decode(obj: bytes) -> Message:
"""
Decode bytes into a 'Gym' message.
:param obj: the bytes object.
:return: the 'Gym' message.
"""
message_pb = ProtobufMessage()
gym_pb = gym_pb2.GymMessage()
message_pb.ParseFromString(obj)
message_id = message_pb.dialogue_message.message_id
dialogue_reference = (
message_pb.dialogue_message.dialogue_starter_reference,
message_pb.dialogue_message.dialogue_responder_reference,
)
target = message_pb.dialogue_message.target
gym_pb.ParseFromString(message_pb.dialogue_message.content)
performative = gym_pb.WhichOneof("performative")
performative_id = GymMessage.Performative(str(performative))
performative_content = dict() # type: Dict[str, Any]
if performative_id == GymMessage.Performative.ACT:
pb2_action = gym_pb.act.action
action = AnyObject.decode(pb2_action)
performative_content["action"] = action
step_id = gym_pb.act.step_id
performative_content["step_id"] = step_id
elif performative_id == GymMessage.Performative.PERCEPT:
step_id = gym_pb.percept.step_id
performative_content["step_id"] = step_id
pb2_observation = gym_pb.percept.observation
observation = AnyObject.decode(pb2_observation)
performative_content["observation"] = observation
reward = gym_pb.percept.reward
performative_content["reward"] = reward
done = gym_pb.percept.done
performative_content["done"] = done
pb2_info = gym_pb.percept.info
info = AnyObject.decode(pb2_info)
performative_content["info"] = info
elif performative_id == GymMessage.Performative.STATUS:
content = gym_pb.status.content
content_dict = dict(content)
performative_content["content"] = content_dict
elif performative_id == GymMessage.Performative.RESET:
pass
elif performative_id == GymMessage.Performative.CLOSE:
pass
else:
raise ValueError(
"Performative not valid: {}.".format(performative_id))
return GymMessage(message_id=message_id,
dialogue_reference=dialogue_reference,
target=target,
performative=performative,
**performative_content)
def close(self) -> None:
"""
Close the environment.
:return: None
"""
self._is_rl_agent_trained = True
gym_msg = GymMessage(performative=GymMessage.Performative.CLOSE)
gym_msg.counterparty = DEFAULT_GYM
self._skill_context.outbox.put_message(message=gym_msg)
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
self._step_count = 0
self._is_rl_agent_trained = False
gym_msg = GymMessage(performative=GymMessage.Performative.RESET)
gym_msg.counterparty = DEFAULT_GYM
self._skill_context.outbox.put_message(message=gym_msg)
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
if not self._agent.multiplexer.is_connected:
self._connect()
gym_msg = GymMessage(performative=GymMessage.Performative.RESET)
gym_msg.counterparty = DEFAULT_GYM
self._agent.outbox.put_message(message=gym_msg, sender=self._agent_address)
def close(self) -> None:
"""
Close the environment.
:return: None
"""
gym_msg = GymMessage(performative=GymMessage.Performative.CLOSE)
gym_msg.counterparty = DEFAULT_GYM
self._agent.outbox.put_message(message=gym_msg, sender=self._agent_address)
self._disconnect()
async def handle_gym_message(self, envelope: Envelope) -> None:
"""
Forward a message to gym.
:param envelope: the envelope
:return: None
"""
enforce(isinstance(envelope.message, GymMessage),
"Message not of type GymMessage")
gym_message, dialogue = self._get_message_and_dialogue(envelope)
if dialogue is None:
self.logger.warning(
"Could not create dialogue from message={}".format(
gym_message))
return
if gym_message.performative == GymMessage.Performative.ACT:
action = gym_message.action.any
step_id = gym_message.step_id
observation, reward, done, info = await self._run_in_executor(
self.gym_env.step, action)
msg = dialogue.reply(
performative=GymMessage.Performative.PERCEPT,
target_message=gym_message,
observation=GymMessage.AnyObject(observation),
reward=reward,
done=done,
info=GymMessage.AnyObject(info),
step_id=step_id,
)
elif gym_message.performative == GymMessage.Performative.RESET:
await self._run_in_executor(self.gym_env.reset)
msg = dialogue.reply(
performative=GymMessage.Performative.STATUS,
target_message=gym_message,
content={"reset": "success"},
)
elif gym_message.performative == GymMessage.Performative.CLOSE:
await self._run_in_executor(self.gym_env.close)
return
envelope = Envelope(
to=msg.to,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self._send(envelope)
async def test_send_act(self):
"""Test send act message."""
sending_dialogue = await self.send_reset()
last_message = sending_dialogue.last_message
assert last_message is not None
msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("any_action"),
step_id=1,
dialogue_reference=sending_dialogue.dialogue_label.
dialogue_reference,
message_id=last_message.message_id + 1,
target=last_message.message_id,
)
msg.counterparty = self.gym_address
assert sending_dialogue.update(msg)
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
observation = 1
reward = 1.0
done = True
info = "some info"
with patch.object(self.env,
"step",
return_value=(observation, reward, done,
info)) as mock:
await self.gym_con.send(envelope)
mock.assert_called()
response = await asyncio.wait_for(self.gym_con.receive(), timeout=3)
response_msg_orig = cast(GymMessage, response.message)
response_msg = copy.copy(response_msg_orig)
response_msg.is_incoming = True
response_msg.counterparty = response_msg_orig.sender
response_dialogue = self.dialogues.update(response_msg)
assert response_msg.performative == GymMessage.Performative.PERCEPT
assert response_msg.step_id == msg.step_id
assert response_msg.observation.any == observation
assert response_msg.reward == reward
assert response_msg.done == done
assert response_msg.info.any == info
assert sending_dialogue == response_dialogue
async def test_send_close(self):
"""Test send close message."""
msg = GymMessage(performative=GymMessage.Performative.CLOSE, )
msg.counterparty = "_to_key"
envelope = Envelope(
to="_to_key",
sender=self.my_address,
protocol_id=GymMessage.protocol_id,
message=msg,
)
await self.gym_con.connect()
await self.gym_con.send(envelope)
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(self.gym_con.receive(), timeout=0.5)
def _decode_percept(self, envelope: Envelope,
expected_step_id: int) -> Message:
"""
Receive the response from the gym environment in the form of an envelope and decode it.
The response is a PERCEPT message containing the usual 'observation', 'reward', 'done', 'info' parameters.
:param expected_step_id: the expected step id
:return: a message received as a response to the action performed in apply_action.
"""
if envelope is not None:
if envelope.protocol_id == PublicId.from_str("fetchai/gym:0.1.0"):
gym_msg = GymSerializer().decode(envelope.message)
gym_msg_performative = GymMessage.Performative(
gym_msg.get("performative"))
gym_msg_step_id = gym_msg.get("step_id")
if (gym_msg_performative == GymMessage.Performative.PERCEPT
and gym_msg_step_id == expected_step_id):
return gym_msg
else:
raise ValueError(
"Unexpected performative or no step_id: {}".format(
gym_msg_performative))
else:
raise ValueError("Unknown protocol_id: {}".format(
envelope.protocol_id))
else:
raise ValueError("Missing envelope.")
def test_gym_message_instantiation():
"""Test instantiation of the gym message."""
assert GymMessage(
performative=GymMessage.Performative.ACT, action="any_action", step_id=1
)
assert GymMessage(
performative=GymMessage.Performative.PERCEPT,
observation="any_observation",
reward=0.0,
info={"some_key": "some_value"},
done=True,
step_id=1,
)
assert GymMessage(performative=GymMessage.Performative.RESET)
assert GymMessage(performative=GymMessage.Performative.CLOSE)
assert str(GymMessage.Performative.CLOSE) == "close"
def _encode_and_send_action(self, action: Action, step_id: int) -> None:
"""
Encode the 'action' sent to the step function and send it.
:param action: the action that is the output of an RL algorithm.
:param step_id: the step id
:return: an envelope
"""
gym_msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject(action),
step_id=step_id,
)
gym_msg.counterparty = DEFAULT_GYM
# Send the message via the proxy agent and to the environment
self._skill_context.outbox.put_message(message=gym_msg)
async def test_send_connection_error(self):
"""Test send connection error."""
msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("any_action"),
step_id=1,
)
msg.counterparty = "_to_key"
envelope = Envelope(
to="_to_key",
sender="_from_key",
protocol_id=GymMessage.protocol_id,
message=msg,
)
with pytest.raises(ConnectionError):
await self.gym_con.send(envelope)
async def test_send_connection_error(self):
"""Test send connection error."""
msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("any_action"),
step_id=1,
)
msg_bytes = GymSerializer().encode(msg)
envelope = Envelope(
to="_to_key",
sender="_from_key",
protocol_id=GymMessage.protocol_id,
message=msg_bytes,
)
self.gym_con.connection_status.is_connected = False
with pytest.raises(ConnectionError):
await self.gym_con.send(envelope)
async def test_send_connection_error(self):
"""Test send connection error."""
msg = GymMessage(
performative=GymMessage.Performative.RESET,
dialogue_reference=self.dialogues.
new_self_initiated_dialogue_reference(),
)
msg.counterparty = self.gym_address
sending_dialogue = self.dialogues.update(msg)
assert sending_dialogue is not None
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
with pytest.raises(ConnectionError):
await self.gym_con.send(envelope)
def close(self) -> None:
"""
Close the environment.
:return: None
"""
self._is_rl_agent_trained = True
last_msg = self.active_gym_dialogue.last_message
assert last_msg is not None, "Cannot retrieve last message."
gym_msg = GymMessage(
dialogue_reference=self.active_gym_dialogue.dialogue_label.
dialogue_reference,
performative=GymMessage.Performative.CLOSE,
message_id=last_msg.message_id + 1,
target=last_msg.message_id,
)
gym_msg.counterparty = self.gym_address
assert self.active_gym_dialogue.update(gym_msg)
self._skill_context.outbox.put_message(message=gym_msg)
def test_incorrect_message(mocked_enforce):
"""Test that we raise an exception when the message is incorrect."""
with mock.patch.object(gym_message_logger, "error") as mock_logger:
GymMessage(
message_id=1,
dialogue_reference=(str(0), ""),
target=0,
performative=GymMessage.Performative.RESET,
)
mock_logger.assert_any_call("some error")
def _encode_action(self, action: Action, step_id: int) -> Envelope:
"""
Encode the 'action' sent to the step function as one or several envelopes.
:param action: the action that is the output of an RL algorithm.
:param step_id: the step id
:return: an envelope
"""
gym_msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject(action),
step_id=step_id,
)
gym_bytes = GymSerializer().encode(gym_msg)
envelope = Envelope(
to=DEFAULT_GYM,
sender=self._skill_context.agent_address,
protocol_id=GymMessage.protocol_id,
message=gym_bytes,
)
return envelope
async def handle_gym_message(self, envelope: Envelope) -> None:
"""
Forward a message to gym.
:param envelope: the envelope
:return: None
"""
assert isinstance(envelope.message,
GymMessage), "Message not of type GymMessage"
gym_message, dialogue = self._get_message_and_dialogue(envelope)
if dialogue is None:
self.logger.warning(
"Could not create dialogue from message={}".format(
gym_message))
return
if gym_message.performative == GymMessage.Performative.ACT:
action = gym_message.action.any
step_id = gym_message.step_id
observation, reward, done, info = await self._run_in_executor(
self.gym_env.step, action)
msg = GymMessage(
performative=GymMessage.Performative.PERCEPT,
observation=GymMessage.AnyObject(observation),
reward=reward,
done=done,
info=GymMessage.AnyObject(info),
step_id=step_id,
target=gym_message.message_id,
message_id=gym_message.message_id + 1,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
)
elif gym_message.performative == GymMessage.Performative.RESET:
await self._run_in_executor(self.gym_env.reset)
msg = GymMessage(
performative=GymMessage.Performative.STATUS,
content={"reset": "success"},
target=gym_message.message_id,
message_id=gym_message.message_id + 1,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
)
elif gym_message.performative == GymMessage.Performative.CLOSE:
await self._run_in_executor(self.gym_env.close)
return
msg.counterparty = gym_message.counterparty
assert dialogue.update(msg), "Error during dialogue update."
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self._send(envelope)
def test_gym_serialization():
"""Test that the serialization for the 'simple' protocol works for the ERROR message."""
msg = GymMessage(
performative=GymMessage.Performative.ACT, action="any_action", step_id=1
)
msg_bytes = GymSerializer().encode(msg)
actual_msg = GymSerializer().decode(msg_bytes)
expected_msg = msg
assert expected_msg == actual_msg
msg = GymMessage(
performative=GymMessage.Performative.PERCEPT,
observation="any_observation",
reward=0.0,
info={"some_key": "some_value"},
done=True,
step_id=1,
)
msg_bytes = GymSerializer().encode(msg)
actual_msg = GymSerializer().decode(msg_bytes)
expected_msg = msg
assert expected_msg == actual_msg
def test_encoding_unknown_performative():
"""Test that we raise an exception when the performative is unknown during encoding."""
msg = GymMessage(
message_id=1,
dialogue_reference=(str(0), ""),
target=0,
performative=GymMessage.Performative.RESET,
)
with pytest.raises(ValueError, match="Performative not valid:"):
with mock.patch.object(GymMessage.Performative,
"__eq__",
return_value=False):
GymMessage.serializer.encode(msg)
