def setUp(self):
self.agent = Mock(["send_chat"])
self.memory = MCAgentMemory()
self.agent.memory = self.memory
self.dialogue_stack = DialogueStack(self.agent, self.memory)
self.dialogue_stack.append(
BotStackStatus(
agent=self.agent, memory=self.memory, dialogue_stack=self.dialogue_stack
)
)
class MethodsTests(unittest.TestCase):
def setUp(self):
self.memory = MCAgentMemory()
def test_peek_empty(self):
self.assertEqual(self.memory.task_stack_peek(), None)
def test_add_mob(self):
# add mob
chicken = {v: k for k, v in MOBS_BY_ID.items()}["chicken"]
mob_id, mob_type, pos, look = 42, chicken, Pos(3, 4, 5), Look(0.0, 0.0)
self.memory.set_mob_position(Mob(mob_id, mob_type, pos, look))
# get mob
self.assertIsNotNone(self.memory.get_entity_by_eid(mob_id))
# update mob
pos = Pos(6, 7, 8)
look = Look(120.0, 50.0)
self.memory.set_mob_position(Mob(mob_id, mob_type, pos, look))
# get mob
mob_node = self.memory.get_entity_by_eid(mob_id)
self.assertIsNotNone(mob_node)
self.assertEqual(mob_node.pos, (6, 7, 8), (120.0, 50.0))
def test_add_guardian_mob(self):
guardian = {v: k for k, v in MOBS_BY_ID.items()}["guardian"]
mob_id, mob_type, pos, look = 42, guardian, Pos(3, 4, 5), Look(0.0, 0.0)
self.memory.set_mob_position(Mob(mob_id, mob_type, pos, look))
class BotStackStatusTest(unittest.TestCase):
def setUp(self):
self.agent = Mock(["send_chat"])
self.memory = MCAgentMemory()
self.agent.memory = self.memory
self.dialogue_stack = DialogueStack(self.agent, self.memory)
self.dialogue_stack.append(
BotStackStatus(
agent=self.agent, memory=self.memory, dialogue_stack=self.dialogue_stack
)
)
def test_move(self):
self.memory.task_stack_push(tasks.Move(self.agent, {"target": (42, 42, 42)}))
self.memory.add_chat("test_agent", "test chat: where are you going?")
self.dialogue_stack.step()
self.agent.send_chat.assert_called()
def setUp(self):
self.memory = MCAgentMemory()
def init_memory(self):
self.memory = MCAgentMemory(
load_minecraft_specs=False) # don't load specs, it's slow
class FakeAgent(LocoMCAgent):
CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)]
def __init__(self, world, opts=None):
self.world = world
self.chat_count = 0
if not opts:
opts = Opt()
opts.nsp_model_dir = None
opts.nsp_data_dir = None
opts.nsp_embedding_path = None
opts.model_base_path = None
opts.QA_nsp_model_path = None
opts.ground_truth_file_path = ""
super(FakeAgent, self).__init__(opts)
self.no_default_behavior = True
self.last_task_memid = None
pos = (0, 63, 0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos, dtype="int")
self.logical_form = None
self._held_item: IDM = (0, 0)
self._look_vec = (1, 0) # (x, z) unit vec
self._changed_blocks: List[Block] = []
self._outgoing_chats: List[str] = []
def init_perception(self):
self.geoscorer = None
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self)
self.perception_modules[
"heuristic"] = heuristic_perception.PerceptionWrapper(self)
def init_physical_interfaces(self):
self.dig = Dig(self)
self.send_chat = SendChat(self)
self.set_held_item = SetHeldItem(self)
self.step_pos_x = StepPosX(self)
self.step_neg_x = StepNegX(self)
self.step_pos_z = StepPosZ(self)
self.step_neg_z = StepNegZ(self)
self.step_pos_y = StepPosY(self)
self.step_neg_y = StepNegY(self)
self.step_forward = StepForward(self)
self.turn_angle = TurnAngle(self)
self.turn_left = TurnLeft(self)
self.turn_right = TurnRight(self)
self.place_block = PlaceBlock(self)
def init_memory(self):
self.memory = MCAgentMemory(
load_minecraft_specs=False) # don't load specs, it's slow
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = GetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {
"logical_form": lf,
"chatstr": chatstr,
"speaker": speaker
}
def step(self):
if hasattr(self.world, "step"):
self.world.step()
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the CraftassistAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
pass
CraftAssistAgent.controller_step(self)
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(
self.memory.get_player_by_name(speaker_name).memid, chatstr)
obj = self.dialogue_manager.handle_logical_form(
speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
########################
## FAKE .PY METHODS ##
########################
def task_step(self):
CraftAssistAgent.task_step(self, sleep_time=0)
def point_at(*args):
pass
def perceive(self, force=False):
self.perception_modules["low_level"].perceive(force=force)
# self.perception_modules["heuristic"].perceive()
###################################
## FAKE C++ PERCEPTION METHODS ##
###################################
def get_blocks(self, xa, xb, ya, yb, za, zb):
return self.world.get_blocks(xa, xb, ya, yb, za, zb)
def get_local_blocks(self, r):
x, y, z = self.pos
return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r)
def get_incoming_chats(self):
c = self.chat_count
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player(self):
return Player(1, "fake_agent", Pos(*self.pos), Look(0, 0), Item(0, 0))
def get_mobs(self):
return self.world.get_mobs()
def get_other_players(self):
return self.world.players.copy()
def get_other_player_by_name(self):
raise NotImplementedError()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y,
player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
def get_changed_blocks(self) -> List[Block]:
# need a better solution here
r = self._changed_blocks.copy()
self._changed_blocks.clear()
return r
def safe_get_changed_blocks(self) -> List[Block]:
return self.get_changed_blocks()
######################################
## World setup
######################################
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block,
store in memory"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.perception_modules["low_level"].on_block_changed(abs_xyz, idm)
self.world.place_block((abs_xyz, idm))
def add_object(self, xyzbms: List[Block],
origin: XYZ = (0, 0, 0)) -> ObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an ObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
return self.memory.get_object_by_id(memid)
def init_memory(self):
T = FakeMCTime(self.world)
self.memory = MCAgentMemory(load_minecraft_specs=False, agent_time=T)
class FakeAgent(LocoMCAgent):
CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)]
default_frame = CraftAssistAgent.default_frame
coordinate_transforms = CraftAssistAgent.default_frame
def __init__(self, world, opts=None, do_heuristic_perception=False):
self.world = world
self.chat_count = 0
if not opts:
opts = Opt()
opts.nsp_model_dir = None
opts.nsp_data_dir = None
opts.nsp_embedding_path = None
opts.model_base_path = None
opts.QA_nsp_model_path = None
opts.ground_truth_data_dir = ""
opts.web_app = False
opts.no_ground_truth = True
super(FakeAgent, self).__init__(opts)
self.do_heuristic_perception = do_heuristic_perception
self.no_default_behavior = True
self.last_task_memid = None
pos = (0, 63, 0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos, dtype="int")
self.logical_form = None
self.world_interaction_occurred = False
self._held_item: IDM = (0, 0)
self._look_vec = (1, 0, 0)
self._changed_blocks: List[Block] = []
self._outgoing_chats: List[str] = []
CraftAssistAgent.add_self_memory_node(self)
def init_perception(self):
self.geoscorer = None
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(
self, perceive_freq=1)
self.perception_modules[
"heuristic"] = heuristic_perception.PerceptionWrapper(self)
def init_physical_interfaces(self):
self.dig = Dig(self)
self.send_chat = SendChat(self)
self.set_held_item = SetHeldItem(self)
self.step_pos_x = StepPosX(self)
self.step_neg_x = StepNegX(self)
self.step_pos_z = StepPosZ(self)
self.step_neg_z = StepNegZ(self)
self.step_pos_y = StepPosY(self)
self.step_neg_y = StepNegY(self)
self.step_forward = StepForward(self)
self.turn_angle = TurnAngle(self)
self.turn_left = TurnLeft(self)
self.turn_right = TurnRight(self)
self.set_look = SetLook(self)
self.place_block = PlaceBlock(self)
def init_memory(self):
T = FakeMCTime(self.world)
self.memory = MCAgentMemory(load_minecraft_specs=False, agent_time=T)
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = GetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {
"logical_form": lf,
"chatstr": chatstr,
"speaker": speaker
}
def step(self):
if hasattr(self.world, "step"):
if self.world_interaction_occurred or self.count % WORLD_STEP == 0:
self.world.step()
self.world_interaction_occurred = False
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the CraftassistAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
pass
CraftAssistAgent.controller_step(self)
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(
self.memory.get_player_by_name(speaker_name).memid, chatstr)
# force to get objects, speaker info
self.perceive(force=True)
obj = self.dialogue_manager.handle_logical_form(
speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
########################
## FAKE .PY METHODS ##
########################
def task_step(self):
CraftAssistAgent.task_step(self, sleep_time=0)
def point_at(*args):
pass
def perceive(self, force=False):
self.perception_modules["low_level"].perceive(force=force)
if self.do_heuristic_perception:
self.perception_modules["heuristic"].perceive()
###################################
## FAKE C++ PERCEPTION METHODS ##
###################################
def get_blocks(self, xa, xb, ya, yb, za, zb):
return self.world.get_blocks(xa, xb, ya, yb, za, zb)
def get_local_blocks(self, r):
x, y, z = self.pos
return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r)
def get_incoming_chats(self):
c = self.chat_count
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player(self):
return Player(1, "fake_agent", Pos(*self.pos), self.get_look(),
Item(*self._held_item))
def get_mobs(self):
return self.world.get_mobs()
def get_item_stacks(self):
return self.world.get_item_stacks()
def get_other_players(self):
return self.world.players.copy()
def get_other_player_by_name(self):
raise NotImplementedError()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_look(self):
pitch = -np.rad2deg(np.arcsin(self._look_vec[1]))
yaw = -np.rad2deg(np.arctan2(self._look_vec[0], self._look_vec[2]))
return Look(pitch, yaw)
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y,
player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
def get_changed_blocks(self) -> List[Block]:
# need a better solution here
r = self._changed_blocks.copy()
self._changed_blocks.clear()
return r
def safe_get_changed_blocks(self) -> List[Block]:
return self.get_changed_blocks()
######################################
## World setup
######################################
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block,
store in memory"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.perception_modules[
"low_level"].pending_agent_placed_blocks.add(abs_xyz)
# TODO add force option so we don't need to make it as if agent placed
self.perception_modules["low_level"].on_block_changed(abs_xyz, idm)
self.world.place_block((abs_xyz, idm))
def add_object(self,
xyzbms: List[Block],
origin: XYZ = (0, 0, 0),
relations={}) -> VoxelObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an VoxelObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
for pred, obj in relations.items():
self.memory.add_triple(subj=memid, pred_text=pred, obj_text=obj)
# sooooorrry FIXME? when we handle triples better in interpreter_helper
if "has_" in pred:
self.memory.tag(memid, obj)
return self.memory.get_object_by_id(memid)
######################################
## visualization
######################################
def draw_slice(self, h=None, r=5, c=None):
if not h:
h = self.pos[1]
if c:
c = [c[0], h, c[1]]
else:
c = [self.pos[0], h, self.pos[2]]
C = self.world.to_world_coords(c)
A = self.world.to_world_coords(self.pos)
shifted_agent_pos = [A[0] - C[0] + r, A[2] - C[2] + r]
npy = self.world.get_blocks(c[0] - r,
c[0] + r,
c[1],
c[1],
c[2] - r,
c[2] + r,
transpose=False)
npy = npy[:, 0, :, 0]
try:
npy[shifted_agent_pos[0], shifted_agent_pos[1]] = 1024
except:
pass
mobnums = {
"rabbit": -1,
"cow": -2,
"pig": -3,
"chicken": -4,
"sheep": -5
}
nummobs = {
-1: "rabbit",
-2: "cow",
-3: "pig",
-4: "chicken",
-5: "sheep"
}
for mob in self.world.mobs:
# todo only in the plane?
p = np.round(np.array(self.world.to_world_coords(mob.pos)))
p = p - C
try:
npy[p[0] + r, p[1] + r] = mobnums[mob.mobname]
except:
pass
mapslice = ""
height = npy.shape[0]
width = npy.shape[1]
def xs(x):
return x + int(self.pos[0]) - r
def zs(z):
return z + int(self.pos[2]) - r
mapslice = mapslice + " " * (width + 2) * 3 + "\n"
for i in reversed(range(height)):
mapslice = mapslice + str(xs(i)).center(3)
for j in range(width):
if npy[i, j] > 0:
if npy[i, j] == 1024:
mapslice = mapslice + " A "
else:
mapslice = mapslice + str(npy[i, j]).center(3)
elif npy[i, j] == 0:
mapslice = mapslice + " * "
else:
npy[i, j] = mapslice + " " + nummobs[npy[i, j]][0] + " "
mapslice = mapslice + "\n"
mapslice = mapslice + " "
for j in range(width):
mapslice = mapslice + " * "
mapslice = mapslice + "\n"
mapslice = mapslice + " "
for j in range(width):
mapslice = mapslice + str(zs(j)).center(3)
return mapslice