def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
self._step_count = 0
self._is_rl_agent_trained = False
gym_msg = GymMessage(
dialogue_reference=self.gym_dialogues.
new_self_initiated_dialogue_reference(),
performative=GymMessage.Performative.RESET,
)
gym_msg.counterparty = self.gym_address
gym_dialogue = cast(Optional[GymDialogue],
self.gym_dialogues.update(gym_msg))
assert gym_dialogue is not None
self._active_dialogue = gym_dialogue
self._skill_context.outbox.put_message(message=gym_msg)
# Wait (blocking!) for the response envelope from the environment
response_msg = self._queue.get(block=True,
timeout=None) # type: GymMessage
if response_msg.performative != GymMessage.Performative.STATUS:
raise ValueError(
"Unexpected performative. Expected={} got={}".format(
GymMessage.Performative.PERCEPT,
response_msg.performative))
async def test_send_close(self):
"""Test send close message."""
sending_dialogue = await self.send_reset()
last_message = sending_dialogue.last_message
assert last_message is not None
msg = GymMessage(
performative=GymMessage.Performative.CLOSE,
dialogue_reference=sending_dialogue.dialogue_label.
dialogue_reference,
message_id=last_message.message_id + 1,
target=last_message.message_id,
)
msg.counterparty = self.gym_address
assert sending_dialogue.update(msg)
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
with patch.object(self.env, "close") as mock:
await self.gym_con.send(envelope)
mock.assert_called()
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
if not self._agent.multiplexer.is_connected:
self._connect()
gym_msg = GymMessage(
dialogue_reference=self.gym_dialogues.
new_self_initiated_dialogue_reference(),
performative=GymMessage.Performative.RESET,
)
gym_msg.counterparty = self.gym_address
gym_dialogue = cast(Optional[GymDialogue],
self.gym_dialogues.update(gym_msg))
assert gym_dialogue is not None
self._active_dialogue = gym_dialogue
self._agent.outbox.put_message(message=gym_msg)
# Wait (blocking!) for the response envelope from the environment
in_envelope = self._queue.get(block=True,
timeout=None) # type: GymMessage
self._decode_status(in_envelope)
async def send_reset(self) -> GymDialogue:
"""Send a reset."""
msg = GymMessage(
performative=GymMessage.Performative.RESET,
dialogue_reference=self.dialogues.
new_self_initiated_dialogue_reference(),
)
msg.counterparty = self.gym_address
sending_dialogue = self.dialogues.update(msg)
assert sending_dialogue is not None
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
with patch.object(self.env, "reset") as mock:
await self.gym_con.send(envelope)
mock.assert_called()
response = await asyncio.wait_for(self.gym_con.receive(), timeout=3)
response_msg_orig = cast(GymMessage, response.message)
response_msg = copy.copy(response_msg_orig)
response_msg.is_incoming = True
response_msg.counterparty = response_msg_orig.sender
response_dialogue = self.dialogues.update(response_msg)
assert response_msg.performative == GymMessage.Performative.STATUS
assert response_msg.content == {"reset": "success"}
assert sending_dialogue == response_dialogue
return sending_dialogue
def close(self) -> None:
"""
Close the environment.
:return: None
"""
self._is_rl_agent_trained = True
gym_msg = GymMessage(performative=GymMessage.Performative.CLOSE)
gym_msg.counterparty = DEFAULT_GYM
self._skill_context.outbox.put_message(message=gym_msg)
async def handle_gym_message(self, envelope: Envelope) -> None:
"""
Forward a message to gym.
:param envelope: the envelope
:return: None
"""
assert isinstance(envelope.message,
GymMessage), "Message not of type GymMessage"
gym_message, dialogue = self._get_message_and_dialogue(envelope)
if dialogue is None:
self.logger.warning(
"Could not create dialogue from message={}".format(
gym_message))
return
if gym_message.performative == GymMessage.Performative.ACT:
action = gym_message.action.any
step_id = gym_message.step_id
observation, reward, done, info = await self._run_in_executor(
self.gym_env.step, action)
msg = GymMessage(
performative=GymMessage.Performative.PERCEPT,
observation=GymMessage.AnyObject(observation),
reward=reward,
done=done,
info=GymMessage.AnyObject(info),
step_id=step_id,
target=gym_message.message_id,
message_id=gym_message.message_id + 1,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
)
elif gym_message.performative == GymMessage.Performative.RESET:
await self._run_in_executor(self.gym_env.reset)
msg = GymMessage(
performative=GymMessage.Performative.STATUS,
content={"reset": "success"},
target=gym_message.message_id,
message_id=gym_message.message_id + 1,
dialogue_reference=dialogue.dialogue_label.dialogue_reference,
)
elif gym_message.performative == GymMessage.Performative.CLOSE:
await self._run_in_executor(self.gym_env.close)
return
msg.counterparty = gym_message.counterparty
assert dialogue.update(msg), "Error during dialogue update."
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self._send(envelope)
def close(self) -> None:
"""
Close the environment.
:return: None
"""
gym_msg = GymMessage(performative=GymMessage.Performative.CLOSE)
gym_msg.counterparty = DEFAULT_GYM
self._agent.outbox.put_message(message=gym_msg, sender=self._agent_address)
self._disconnect()
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
self._step_count = 0
self._is_rl_agent_trained = False
gym_msg = GymMessage(performative=GymMessage.Performative.RESET)
gym_msg.counterparty = DEFAULT_GYM
self._skill_context.outbox.put_message(message=gym_msg)
def reset(self) -> None:
"""
Reset the environment.
:return: None
"""
if not self._agent.multiplexer.is_connected:
self._connect()
gym_msg = GymMessage(performative=GymMessage.Performative.RESET)
gym_msg.counterparty = DEFAULT_GYM
self._agent.outbox.put_message(message=gym_msg, sender=self._agent_address)
async def test_send_act(self):
"""Test send act message."""
sending_dialogue = await self.send_reset()
last_message = sending_dialogue.last_message
assert last_message is not None
msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("any_action"),
step_id=1,
dialogue_reference=sending_dialogue.dialogue_label.
dialogue_reference,
message_id=last_message.message_id + 1,
target=last_message.message_id,
)
msg.counterparty = self.gym_address
assert sending_dialogue.update(msg)
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
observation = 1
reward = 1.0
done = True
info = "some info"
with patch.object(self.env,
"step",
return_value=(observation, reward, done,
info)) as mock:
await self.gym_con.send(envelope)
mock.assert_called()
response = await asyncio.wait_for(self.gym_con.receive(), timeout=3)
response_msg_orig = cast(GymMessage, response.message)
response_msg = copy.copy(response_msg_orig)
response_msg.is_incoming = True
response_msg.counterparty = response_msg_orig.sender
response_dialogue = self.dialogues.update(response_msg)
assert response_msg.performative == GymMessage.Performative.PERCEPT
assert response_msg.step_id == msg.step_id
assert response_msg.observation.any == observation
assert response_msg.reward == reward
assert response_msg.done == done
assert response_msg.info.any == info
assert sending_dialogue == response_dialogue
def _encode_and_send_action(self, action: Action, step_id: int) -> None:
"""
Encode the 'action' sent to the step function and send it.
:param action: the action that is the output of an RL algorithm.
:param step_id: the step id
:return: an envelope
"""
gym_msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject(action),
step_id=step_id,
)
gym_msg.counterparty = DEFAULT_GYM
# Send the message via the proxy agent and to the environment
self._skill_context.outbox.put_message(message=gym_msg)
async def test_send_close(self):
"""Test send close message."""
msg = GymMessage(performative=GymMessage.Performative.CLOSE, )
msg.counterparty = "_to_key"
envelope = Envelope(
to="_to_key",
sender=self.my_address,
protocol_id=GymMessage.protocol_id,
message=msg,
)
await self.gym_con.connect()
await self.gym_con.send(envelope)
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(self.gym_con.receive(), timeout=0.5)
async def test_send_connection_error(self):
"""Test send connection error."""
msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("any_action"),
step_id=1,
)
msg.counterparty = "_to_key"
envelope = Envelope(
to="_to_key",
sender="_from_key",
protocol_id=GymMessage.protocol_id,
message=msg,
)
with pytest.raises(ConnectionError):
await self.gym_con.send(envelope)
def close(self) -> None:
"""
Close the environment.
:return: None
"""
self._is_rl_agent_trained = True
last_msg = self.active_gym_dialogue.last_message
assert last_msg is not None, "Cannot retrieve last message."
gym_msg = GymMessage(
dialogue_reference=self.active_gym_dialogue.dialogue_label.
dialogue_reference,
performative=GymMessage.Performative.CLOSE,
message_id=last_msg.message_id + 1,
target=last_msg.message_id,
)
gym_msg.counterparty = self.gym_address
assert self.active_gym_dialogue.update(gym_msg)
self._skill_context.outbox.put_message(message=gym_msg)
async def test_send_connection_error(self):
"""Test send connection error."""
msg = GymMessage(
performative=GymMessage.Performative.RESET,
dialogue_reference=self.dialogues.
new_self_initiated_dialogue_reference(),
)
msg.counterparty = self.gym_address
sending_dialogue = self.dialogues.update(msg)
assert sending_dialogue is not None
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
with pytest.raises(ConnectionError):
await self.gym_con.send(envelope)
def _encode_and_send_action(self, action: Action, step_id: int) -> None:
"""
Encode the 'action' sent to the step function and send it.
:param action: the action that is the output of an RL algorithm.
:param step_id: the step id
:return: an envelope
"""
last_msg = self.active_gym_dialogue.last_message
assert last_msg is not None, "Cannot retrieve last message."
gym_msg = GymMessage(
dialogue_reference=self.active_gym_dialogue.dialogue_label.
dialogue_reference,
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject(action),
step_id=step_id,
message_id=last_msg.message_id + 1,
target=last_msg.message_id,
)
gym_msg.counterparty = self.gym_address
assert self.active_gym_dialogue.update(gym_msg)
# Send the message via the proxy agent and to the environment
self._skill_context.outbox.put_message(message=gym_msg)
async def test_send_close_negative(self):
"""Test send close message with invalid reference and message id and target."""
msg = GymMessage(
performative=GymMessage.Performative.CLOSE,
dialogue_reference=self.dialogues.
new_self_initiated_dialogue_reference(),
)
msg.counterparty = self.gym_address
dialogue = self.dialogues.update(msg)
assert dialogue is None
msg.sender = self.agent_address
envelope = Envelope(
to=msg.counterparty,
sender=msg.sender,
protocol_id=msg.protocol_id,
message=msg,
)
await self.gym_con.connect()
with patch.object(self.gym_con.channel.logger,
"warning") as mock_logger:
await self.gym_con.send(envelope)
mock_logger.assert_any_call(
f"Could not create dialogue from message={msg}")
async def test_send_act(self):
"""Test send act message."""
msg = GymMessage(
performative=GymMessage.Performative.ACT,
action=GymMessage.AnyObject("any_action"),
step_id=1,
)
msg.counterparty = "_to_key"
envelope = Envelope(
to="_to_key",
sender=self.my_address,
protocol_id=GymMessage.protocol_id,
message=msg,
)
await self.gym_con.connect()
with patch.object(self.env,
"step",
return_value=(1, 1.0, True, "some info")) as mock:
await self.gym_con.send(envelope)
mock.assert_called()
assert await asyncio.wait_for(self.gym_con.receive(),
timeout=3) is not None