def setUp(self): self.agent = Mock(["send_chat"]) self.memory = MCAgentMemory() self.agent.memory = self.memory self.dialogue_stack = DialogueStack(self.agent, self.memory) self.dialogue_stack.append( BotStackStatus( agent=self.agent, memory=self.memory, dialogue_stack=self.dialogue_stack ) )
class MethodsTests(unittest.TestCase): def setUp(self): self.memory = MCAgentMemory() def test_peek_empty(self): self.assertEqual(self.memory.task_stack_peek(), None) def test_add_mob(self): # add mob chicken = {v: k for k, v in MOBS_BY_ID.items()}["chicken"] mob_id, mob_type, pos, look = 42, chicken, Pos(3, 4, 5), Look(0.0, 0.0) self.memory.set_mob_position(Mob(mob_id, mob_type, pos, look)) # get mob self.assertIsNotNone(self.memory.get_entity_by_eid(mob_id)) # update mob pos = Pos(6, 7, 8) look = Look(120.0, 50.0) self.memory.set_mob_position(Mob(mob_id, mob_type, pos, look)) # get mob mob_node = self.memory.get_entity_by_eid(mob_id) self.assertIsNotNone(mob_node) self.assertEqual(mob_node.pos, (6, 7, 8), (120.0, 50.0)) def test_add_guardian_mob(self): guardian = {v: k for k, v in MOBS_BY_ID.items()}["guardian"] mob_id, mob_type, pos, look = 42, guardian, Pos(3, 4, 5), Look(0.0, 0.0) self.memory.set_mob_position(Mob(mob_id, mob_type, pos, look))
class BotStackStatusTest(unittest.TestCase): def setUp(self): self.agent = Mock(["send_chat"]) self.memory = MCAgentMemory() self.agent.memory = self.memory self.dialogue_stack = DialogueStack(self.agent, self.memory) self.dialogue_stack.append( BotStackStatus( agent=self.agent, memory=self.memory, dialogue_stack=self.dialogue_stack ) ) def test_move(self): self.memory.task_stack_push(tasks.Move(self.agent, {"target": (42, 42, 42)})) self.memory.add_chat("test_agent", "test chat: where are you going?") self.dialogue_stack.step() self.agent.send_chat.assert_called()
def setUp(self): self.memory = MCAgentMemory()
def init_memory(self): self.memory = MCAgentMemory( load_minecraft_specs=False) # don't load specs, it's slow
class FakeAgent(LocoMCAgent): CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)] def __init__(self, world, opts=None): self.world = world self.chat_count = 0 if not opts: opts = Opt() opts.nsp_model_dir = None opts.nsp_data_dir = None opts.nsp_embedding_path = None opts.model_base_path = None opts.QA_nsp_model_path = None opts.ground_truth_file_path = "" super(FakeAgent, self).__init__(opts) self.no_default_behavior = True self.last_task_memid = None pos = (0, 63, 0) if hasattr(self.world, "agent_data"): pos = self.world.agent_data["pos"] self.pos = np.array(pos, dtype="int") self.logical_form = None self._held_item: IDM = (0, 0) self._look_vec = (1, 0) # (x, z) unit vec self._changed_blocks: List[Block] = [] self._outgoing_chats: List[str] = [] def init_perception(self): self.geoscorer = None self.perception_modules = {} self.perception_modules["low_level"] = LowLevelMCPerception(self) self.perception_modules[ "heuristic"] = heuristic_perception.PerceptionWrapper(self) def init_physical_interfaces(self): self.dig = Dig(self) self.send_chat = SendChat(self) self.set_held_item = SetHeldItem(self) self.step_pos_x = StepPosX(self) self.step_neg_x = StepNegX(self) self.step_pos_z = StepPosZ(self) self.step_neg_z = StepNegZ(self) self.step_pos_y = StepPosY(self) self.step_neg_y = StepNegY(self) self.step_forward = StepForward(self) self.turn_angle = TurnAngle(self) self.turn_left = TurnLeft(self) self.turn_right = TurnRight(self) self.place_block = PlaceBlock(self) def init_memory(self): self.memory = MCAgentMemory( load_minecraft_specs=False) # don't load specs, it's slow def init_controller(self): dialogue_object_classes = {} dialogue_object_classes["interpreter"] = Interpreter dialogue_object_classes["get_memory"] = GetMemoryHandler dialogue_object_classes["put_memory"] = PutMemoryHandler self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts) def set_logical_form(self, lf, chatstr, speaker): self.logical_form = { "logical_form": lf, "chatstr": chatstr, "speaker": speaker } def step(self): if hasattr(self.world, "step"): self.world.step() if hasattr(self, "recorder"): self.recorder.record_world() super().step() #### use the CraftassistAgent.controller_step() def controller_step(self): if self.logical_form is None: pass CraftAssistAgent.controller_step(self) else: # logical form given directly: # clear the chat buffer self.get_incoming_chats() # use the logical form as given... d = self.logical_form["logical_form"] chatstr = self.logical_form["chatstr"] speaker_name = self.logical_form["speaker"] self.memory.add_chat( self.memory.get_player_by_name(speaker_name).memid, chatstr) obj = self.dialogue_manager.handle_logical_form( speaker_name, d, chatstr) if obj is not None: self.dialogue_manager.dialogue_stack.append(obj) self.logical_form = None def setup_test(self): self.task_steps_count = 0 def clear_outgoing_chats(self): self._outgoing_chats.clear() def get_last_outgoing_chat(self): try: return self._outgoing_chats[-1] except IndexError: return None ######################## ## FAKE .PY METHODS ## ######################## def task_step(self): CraftAssistAgent.task_step(self, sleep_time=0) def point_at(*args): pass def perceive(self, force=False): self.perception_modules["low_level"].perceive(force=force) # self.perception_modules["heuristic"].perceive() ################################### ## FAKE C++ PERCEPTION METHODS ## ################################### def get_blocks(self, xa, xb, ya, yb, za, zb): return self.world.get_blocks(xa, xb, ya, yb, za, zb) def get_local_blocks(self, r): x, y, z = self.pos return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r) def get_incoming_chats(self): c = self.chat_count self.chat_count = len(self.world.chat_log) return self.world.chat_log[c:].copy() def get_player(self): return Player(1, "fake_agent", Pos(*self.pos), Look(0, 0), Item(0, 0)) def get_mobs(self): return self.world.get_mobs() def get_other_players(self): return self.world.players.copy() def get_other_player_by_name(self): raise NotImplementedError() def get_vision(self): raise NotImplementedError() def get_line_of_sight(self): raise NotImplementedError() def get_player_line_of_sight(self, player_struct): if hasattr(self.world, "get_line_of_sight"): pos = (player_struct.pos.x, player_struct.pos.y, player_struct.pos.z) pitch = player_struct.look.pitch yaw = player_struct.look.yaw xsect = self.world.get_line_of_sight(pos, yaw, pitch) if xsect is not None: return Pos(*xsect) else: raise NotImplementedError() def get_changed_blocks(self) -> List[Block]: # need a better solution here r = self._changed_blocks.copy() self._changed_blocks.clear() return r def safe_get_changed_blocks(self) -> List[Block]: return self.get_changed_blocks() ###################################### ## World setup ###################################### def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)): """Change the state of the world, block by block, store in memory""" for xyz, idm in xyzbms: abs_xyz = tuple(np.array(xyz) + origin) self.perception_modules["low_level"].on_block_changed(abs_xyz, idm) self.world.place_block((abs_xyz, idm)) def add_object(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)) -> ObjectNode: """Add an object to memory as if it was placed block by block Args: - xyzbms: a list of relative (xyz, idm) - origin: (x, y, z) of the corner Returns an ObjectNode """ self.set_blocks(xyzbms, origin) abs_xyz = tuple(np.array(xyzbms[0][0]) + origin) memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0] return self.memory.get_object_by_id(memid)
def init_memory(self): T = FakeMCTime(self.world) self.memory = MCAgentMemory(load_minecraft_specs=False, agent_time=T)
class FakeAgent(LocoMCAgent): CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)] default_frame = CraftAssistAgent.default_frame coordinate_transforms = CraftAssistAgent.default_frame def __init__(self, world, opts=None, do_heuristic_perception=False): self.world = world self.chat_count = 0 if not opts: opts = Opt() opts.nsp_model_dir = None opts.nsp_data_dir = None opts.nsp_embedding_path = None opts.model_base_path = None opts.QA_nsp_model_path = None opts.ground_truth_data_dir = "" opts.web_app = False opts.no_ground_truth = True super(FakeAgent, self).__init__(opts) self.do_heuristic_perception = do_heuristic_perception self.no_default_behavior = True self.last_task_memid = None pos = (0, 63, 0) if hasattr(self.world, "agent_data"): pos = self.world.agent_data["pos"] self.pos = np.array(pos, dtype="int") self.logical_form = None self.world_interaction_occurred = False self._held_item: IDM = (0, 0) self._look_vec = (1, 0, 0) self._changed_blocks: List[Block] = [] self._outgoing_chats: List[str] = [] CraftAssistAgent.add_self_memory_node(self) def init_perception(self): self.geoscorer = None self.perception_modules = {} self.perception_modules["low_level"] = LowLevelMCPerception( self, perceive_freq=1) self.perception_modules[ "heuristic"] = heuristic_perception.PerceptionWrapper(self) def init_physical_interfaces(self): self.dig = Dig(self) self.send_chat = SendChat(self) self.set_held_item = SetHeldItem(self) self.step_pos_x = StepPosX(self) self.step_neg_x = StepNegX(self) self.step_pos_z = StepPosZ(self) self.step_neg_z = StepNegZ(self) self.step_pos_y = StepPosY(self) self.step_neg_y = StepNegY(self) self.step_forward = StepForward(self) self.turn_angle = TurnAngle(self) self.turn_left = TurnLeft(self) self.turn_right = TurnRight(self) self.set_look = SetLook(self) self.place_block = PlaceBlock(self) def init_memory(self): T = FakeMCTime(self.world) self.memory = MCAgentMemory(load_minecraft_specs=False, agent_time=T) def init_controller(self): dialogue_object_classes = {} dialogue_object_classes["interpreter"] = Interpreter dialogue_object_classes["get_memory"] = GetMemoryHandler dialogue_object_classes["put_memory"] = PutMemoryHandler self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts) def set_logical_form(self, lf, chatstr, speaker): self.logical_form = { "logical_form": lf, "chatstr": chatstr, "speaker": speaker } def step(self): if hasattr(self.world, "step"): if self.world_interaction_occurred or self.count % WORLD_STEP == 0: self.world.step() self.world_interaction_occurred = False if hasattr(self, "recorder"): self.recorder.record_world() super().step() #### use the CraftassistAgent.controller_step() def controller_step(self): if self.logical_form is None: pass CraftAssistAgent.controller_step(self) else: # logical form given directly: # clear the chat buffer self.get_incoming_chats() # use the logical form as given... d = self.logical_form["logical_form"] chatstr = self.logical_form["chatstr"] speaker_name = self.logical_form["speaker"] self.memory.add_chat( self.memory.get_player_by_name(speaker_name).memid, chatstr) # force to get objects, speaker info self.perceive(force=True) obj = self.dialogue_manager.handle_logical_form( speaker_name, d, chatstr) if obj is not None: self.dialogue_manager.dialogue_stack.append(obj) self.logical_form = None def setup_test(self): self.task_steps_count = 0 def clear_outgoing_chats(self): self._outgoing_chats.clear() def get_last_outgoing_chat(self): try: return self._outgoing_chats[-1] except IndexError: return None ######################## ## FAKE .PY METHODS ## ######################## def task_step(self): CraftAssistAgent.task_step(self, sleep_time=0) def point_at(*args): pass def perceive(self, force=False): self.perception_modules["low_level"].perceive(force=force) if self.do_heuristic_perception: self.perception_modules["heuristic"].perceive() ################################### ## FAKE C++ PERCEPTION METHODS ## ################################### def get_blocks(self, xa, xb, ya, yb, za, zb): return self.world.get_blocks(xa, xb, ya, yb, za, zb) def get_local_blocks(self, r): x, y, z = self.pos return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r) def get_incoming_chats(self): c = self.chat_count self.chat_count = len(self.world.chat_log) return self.world.chat_log[c:].copy() def get_player(self): return Player(1, "fake_agent", Pos(*self.pos), self.get_look(), Item(*self._held_item)) def get_mobs(self): return self.world.get_mobs() def get_item_stacks(self): return self.world.get_item_stacks() def get_other_players(self): return self.world.players.copy() def get_other_player_by_name(self): raise NotImplementedError() def get_vision(self): raise NotImplementedError() def get_line_of_sight(self): raise NotImplementedError() def get_look(self): pitch = -np.rad2deg(np.arcsin(self._look_vec[1])) yaw = -np.rad2deg(np.arctan2(self._look_vec[0], self._look_vec[2])) return Look(pitch, yaw) def get_player_line_of_sight(self, player_struct): if hasattr(self.world, "get_line_of_sight"): pos = (player_struct.pos.x, player_struct.pos.y, player_struct.pos.z) pitch = player_struct.look.pitch yaw = player_struct.look.yaw xsect = self.world.get_line_of_sight(pos, yaw, pitch) if xsect is not None: return Pos(*xsect) else: raise NotImplementedError() def get_changed_blocks(self) -> List[Block]: # need a better solution here r = self._changed_blocks.copy() self._changed_blocks.clear() return r def safe_get_changed_blocks(self) -> List[Block]: return self.get_changed_blocks() ###################################### ## World setup ###################################### def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)): """Change the state of the world, block by block, store in memory""" for xyz, idm in xyzbms: abs_xyz = tuple(np.array(xyz) + origin) self.perception_modules[ "low_level"].pending_agent_placed_blocks.add(abs_xyz) # TODO add force option so we don't need to make it as if agent placed self.perception_modules["low_level"].on_block_changed(abs_xyz, idm) self.world.place_block((abs_xyz, idm)) def add_object(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0), relations={}) -> VoxelObjectNode: """Add an object to memory as if it was placed block by block Args: - xyzbms: a list of relative (xyz, idm) - origin: (x, y, z) of the corner Returns an VoxelObjectNode """ self.set_blocks(xyzbms, origin) abs_xyz = tuple(np.array(xyzbms[0][0]) + origin) memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0] for pred, obj in relations.items(): self.memory.add_triple(subj=memid, pred_text=pred, obj_text=obj) # sooooorrry FIXME? when we handle triples better in interpreter_helper if "has_" in pred: self.memory.tag(memid, obj) return self.memory.get_object_by_id(memid) ###################################### ## visualization ###################################### def draw_slice(self, h=None, r=5, c=None): if not h: h = self.pos[1] if c: c = [c[0], h, c[1]] else: c = [self.pos[0], h, self.pos[2]] C = self.world.to_world_coords(c) A = self.world.to_world_coords(self.pos) shifted_agent_pos = [A[0] - C[0] + r, A[2] - C[2] + r] npy = self.world.get_blocks(c[0] - r, c[0] + r, c[1], c[1], c[2] - r, c[2] + r, transpose=False) npy = npy[:, 0, :, 0] try: npy[shifted_agent_pos[0], shifted_agent_pos[1]] = 1024 except: pass mobnums = { "rabbit": -1, "cow": -2, "pig": -3, "chicken": -4, "sheep": -5 } nummobs = { -1: "rabbit", -2: "cow", -3: "pig", -4: "chicken", -5: "sheep" } for mob in self.world.mobs: # todo only in the plane? p = np.round(np.array(self.world.to_world_coords(mob.pos))) p = p - C try: npy[p[0] + r, p[1] + r] = mobnums[mob.mobname] except: pass mapslice = "" height = npy.shape[0] width = npy.shape[1] def xs(x): return x + int(self.pos[0]) - r def zs(z): return z + int(self.pos[2]) - r mapslice = mapslice + " " * (width + 2) * 3 + "\n" for i in reversed(range(height)): mapslice = mapslice + str(xs(i)).center(3) for j in range(width): if npy[i, j] > 0: if npy[i, j] == 1024: mapslice = mapslice + " A " else: mapslice = mapslice + str(npy[i, j]).center(3) elif npy[i, j] == 0: mapslice = mapslice + " * " else: npy[i, j] = mapslice + " " + nummobs[npy[i, j]][0] + " " mapslice = mapslice + "\n" mapslice = mapslice + " " for j in range(width): mapslice = mapslice + " * " mapslice = mapslice + "\n" mapslice = mapslice + " " for j in range(width): mapslice = mapslice + str(zs(j)).center(3) return mapslice