class FakeAgent(LocoMCAgent):
CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)]
def __init__(self, world, opts=None):
self.world = world
self.chat_count = 0
if not opts:
opts = Opt()
opts.nsp_model_dir = None
opts.nsp_data_dir = None
opts.nsp_embedding_path = None
opts.model_base_path = None
opts.QA_nsp_model_path = None
opts.ground_truth_file_path = ""
super(FakeAgent, self).__init__(opts)
self.no_default_behavior = True
self.last_task_memid = None
pos = (0, 63, 0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos, dtype="int")
self.logical_form = None
self._held_item: IDM = (0, 0)
self._look_vec = (1, 0) # (x, z) unit vec
self._changed_blocks: List[Block] = []
self._outgoing_chats: List[str] = []
def init_perception(self):
self.geoscorer = None
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self)
self.perception_modules[
"heuristic"] = heuristic_perception.PerceptionWrapper(self)
def init_physical_interfaces(self):
self.dig = Dig(self)
self.send_chat = SendChat(self)
self.set_held_item = SetHeldItem(self)
self.step_pos_x = StepPosX(self)
self.step_neg_x = StepNegX(self)
self.step_pos_z = StepPosZ(self)
self.step_neg_z = StepNegZ(self)
self.step_pos_y = StepPosY(self)
self.step_neg_y = StepNegY(self)
self.step_forward = StepForward(self)
self.turn_angle = TurnAngle(self)
self.turn_left = TurnLeft(self)
self.turn_right = TurnRight(self)
self.place_block = PlaceBlock(self)
def init_memory(self):
self.memory = MCAgentMemory(
load_minecraft_specs=False) # don't load specs, it's slow
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = GetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {
"logical_form": lf,
"chatstr": chatstr,
"speaker": speaker
}
def step(self):
if hasattr(self.world, "step"):
self.world.step()
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the CraftassistAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
pass
CraftAssistAgent.controller_step(self)
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(
self.memory.get_player_by_name(speaker_name).memid, chatstr)
obj = self.dialogue_manager.handle_logical_form(
speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
########################
## FAKE .PY METHODS ##
########################
def task_step(self):
CraftAssistAgent.task_step(self, sleep_time=0)
def point_at(*args):
pass
def perceive(self, force=False):
self.perception_modules["low_level"].perceive(force=force)
# self.perception_modules["heuristic"].perceive()
###################################
## FAKE C++ PERCEPTION METHODS ##
###################################
def get_blocks(self, xa, xb, ya, yb, za, zb):
return self.world.get_blocks(xa, xb, ya, yb, za, zb)
def get_local_blocks(self, r):
x, y, z = self.pos
return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r)
def get_incoming_chats(self):
c = self.chat_count
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player(self):
return Player(1, "fake_agent", Pos(*self.pos), Look(0, 0), Item(0, 0))
def get_mobs(self):
return self.world.get_mobs()
def get_other_players(self):
return self.world.players.copy()
def get_other_player_by_name(self):
raise NotImplementedError()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y,
player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
def get_changed_blocks(self) -> List[Block]:
# need a better solution here
r = self._changed_blocks.copy()
self._changed_blocks.clear()
return r
def safe_get_changed_blocks(self) -> List[Block]:
return self.get_changed_blocks()
######################################
## World setup
######################################
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block,
store in memory"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.perception_modules["low_level"].on_block_changed(abs_xyz, idm)
self.world.place_block((abs_xyz, idm))
def add_object(self, xyzbms: List[Block],
origin: XYZ = (0, 0, 0)) -> ObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an ObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
return self.memory.get_object_by_id(memid)
class FakeAgent(LocoMCAgent):
coordinate_transforms = rotation
def __init__(
self, world, opts=None, mv_speed=MV_SPEED, rot_speed=ROT_SPEED, head_speed=HEAD_SPEED
):
self.world = world
self.world.agent = self
self.chat_count = 0
if not opts:
opts = Opt()
opts.nsp_models_dir = ""
opts.nsp_data_dir = ""
opts.model_base_path = ""
opts.ground_truth_data_dir = ""
opts.no_ground_truth = True
super(FakeAgent, self).__init__(opts)
self.no_default_behavior = True
self.last_task_memid = None
pos = (0.0, 0.0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos)
self.base_yaw = 0.0
self.pan = 0.0
self.pitch = 0.0
self.mv_speed = mv_speed
self.rot_speed = rot_speed
self.head_speed = head_speed
self.logical_form = None
self._outgoing_chats = []
self.inventory = []
def init_perception(self):
self.perception_modules = {}
self.perception_modules["self"] = SelfPerception(self, perceive_freq=1)
self.perception_modules["vision"] = FakeDetectorPerception(self)
def init_physical_interfaces(self):
self.mover = FakeMover(self)
self.send_chat = self.mover.send_chat
def init_memory(self):
self.memory = LocoAgentMemory()
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = LocoInterpreter
# dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = LocoGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {"logical_form": lf, "chatstr": chatstr, "speaker": speaker}
def step(self):
if hasattr(self.world, "step"):
self.world.step()
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the LocobotAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
super().controller_step()
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(self.memory.get_player_by_name(speaker_name).memid, chatstr)
obj = self.dialogue_manager.handle_logical_form(speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
def get_info(self):
info = {}
# todo self.pos ---> self.camera_pos, self.base_pos
info["pos"] = self.pos
info["pitch"] = self.pitch
info["yaw"] = self.yaw
info["pan"] = self.pan
info["base_yaw"] = self.base_yaw
info["name"] = self.name
return info
########################
## FAKE .PY METHODS ##
########################
def point_at(*args):
pass
# TODO mirror this logic in real locobot
def get_incoming_chats(self):
c = self.chat_count
for raw_chatstr in self.world.chat_log[c:]:
match = re.search("^<([^>]+)> (.*)", raw_chatstr)
speaker_name = match.group(1)
if not self.memory.get_player_by_name(speaker_name):
# FIXME! name used as eid
PlayerNode.create(
self.memory,
to_player_struct((None, None, None), None, None, speaker_name, speaker_name),
)
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player_struct_by_name(self, speaker_name):
p = self.memory.get_player_by_name(speaker_name)
if p:
return p.get_struct()
else:
return None
def get_other_players(self):
return self.world.players.copy()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_look(self):
return Look(self.pitch, self.base_yaw + self.pan)
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y, player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
######################################
## World setup
######################################
# TODO put this in the world, decide if things can move through it, etc.
def add_object(self, xyz, tags=[], colour=None):
self.world.add_object(list(xyz), tags=tags, colour=colour)
# TODO do this better when less mocked
self.perception_modules["vision"].add_detected_object(xyz, properties=tags, colour=colour)
class FakeAgent(LocoMCAgent):
CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)]
default_frame = CraftAssistAgent.default_frame
coordinate_transforms = CraftAssistAgent.coordinate_transforms
def __init__(self, world, opts=None, do_heuristic_perception=False):
self.world = world
self.chat_count = 0
if not opts:
opts = MockOpt()
super(FakeAgent, self).__init__(opts)
self.do_heuristic_perception = do_heuristic_perception
self.no_default_behavior = True
self.last_task_memid = None
pos = (0, 63, 0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos, dtype="int")
self.logical_form = None
self.world_interaction_occurred = False
self._held_item: IDM = (0, 0)
self._look_vec = (1, 0, 0)
self._changed_blocks: List[Block] = []
self._outgoing_chats: List[str] = []
CraftAssistAgent.add_self_memory_node(self)
def init_perception(self):
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self, perceive_freq=1)
self.perception_modules["heuristic"] = PerceptionWrapper(self)
self.on_demand_perception = {}
self.on_demand_perception["geoscorer"] = None
self.on_demand_perception["check_inside"] = check_inside
def init_physical_interfaces(self):
self.dig = Dig(self)
self.send_chat = SendChat(self)
self.set_held_item = SetHeldItem(self)
self.step_pos_x = StepPosX(self)
self.step_neg_x = StepNegX(self)
self.step_pos_z = StepPosZ(self)
self.step_neg_z = StepNegZ(self)
self.step_pos_y = StepPosY(self)
self.step_neg_y = StepNegY(self)
self.step_forward = StepForward(self)
self.turn_angle = TurnAngle(self)
self.turn_left = TurnLeft(self)
self.turn_right = TurnRight(self)
self.set_look = SetLook(self)
self.place_block = PlaceBlock(self)
def init_memory(self):
T = FakeMCTime(self.world)
self.memory = MCAgentMemory(load_minecraft_specs=False, agent_time=T)
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["bot_capabilities"] = MCBotCapabilities
dialogue_object_classes["interpreter"] = MCInterpreter
dialogue_object_classes["get_memory"] = MCGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {"logical_form": lf, "chatstr": chatstr, "speaker": speaker}
def step(self):
if hasattr(self.world, "step"):
if self.world_interaction_occurred or self.count % WORLD_STEP == 0:
self.world.step()
self.world_interaction_occurred = False
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the CraftassistAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
pass
CraftAssistAgent.controller_step(self)
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(self.memory.get_player_by_name(speaker_name).memid, chatstr)
# force to get objects, speaker info
self.perceive(force=True)
obj = self.dialogue_manager.handle_logical_form(speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
########################
## FAKE .PY METHODS ##
########################
def task_step(self):
CraftAssistAgent.task_step(self, sleep_time=0)
def point_at(*args):
pass
def perceive(self, force=False):
self.perception_modules["low_level"].perceive(force=force)
if self.do_heuristic_perception:
self.perception_modules["heuristic"].perceive()
###################################
## FAKE C++ PERCEPTION METHODS ##
###################################
def get_blocks(self, xa, xb, ya, yb, za, zb):
return self.world.get_blocks(xa, xb, ya, yb, za, zb)
def get_local_blocks(self, r):
x, y, z = self.pos
return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r)
def get_incoming_chats(self):
c = self.chat_count
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player(self):
return Player(1, "fake_agent", Pos(*self.pos), self.get_look(), Item(*self._held_item))
def get_mobs(self):
return self.world.get_mobs()
def get_item_stacks(self):
return self.world.get_item_stacks()
def get_other_players(self):
return self.world.players.copy()
def get_other_player_by_name(self):
raise NotImplementedError()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_look(self):
pitch = -np.rad2deg(np.arcsin(self._look_vec[1]))
yaw = -np.rad2deg(np.arctan2(self._look_vec[0], self._look_vec[2]))
return Look(pitch, yaw)
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y, player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
def get_changed_blocks(self) -> List[Block]:
# need a better solution here
r = self._changed_blocks.copy()
self._changed_blocks.clear()
return r
def safe_get_changed_blocks(self) -> List[Block]:
return self.get_changed_blocks()
######################################
## World setup
######################################
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block,
store in memory"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.perception_modules["low_level"].pending_agent_placed_blocks.add(abs_xyz)
# TODO add force option so we don't need to make it as if agent placed
self.perception_modules["low_level"].on_block_changed(abs_xyz, idm)
self.world.place_block((abs_xyz, idm))
def add_object(
self, xyzbms: List[Block], origin: XYZ = (0, 0, 0), relations={}
) -> VoxelObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an VoxelObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
for pred, obj in relations.items():
self.memory.add_triple(subj=memid, pred_text=pred, obj_text=obj)
# sooooorrry FIXME? when we handle triples better in interpreter_helper
if "has_" in pred:
self.memory.tag(memid, obj)
return self.memory.get_object_by_id(memid)
# WARNING!! this does not step the world, but directly fast-forwards
# to count. Use only in world setup, once world is running!
def add_object_ff_time(
self, count, xyzbms: List[Block], origin: XYZ = (0, 0, 0), relations={}
) -> VoxelObjectNode:
assert count >= self.world.count
self.world.set_count(count)
return self.add_object(xyzbms, origin, relations=relations)
######################################
## visualization
######################################
def draw_slice(self, h=None, r=5, c=None):
if not h:
h = self.pos[1]
if c:
c = [c[0], h, c[1]]
else:
c = [self.pos[0], h, self.pos[2]]
C = self.world.to_world_coords(c)
A = self.world.to_world_coords(self.pos)
shifted_agent_pos = [A[0] - C[0] + r, A[2] - C[2] + r]
npy = self.world.get_blocks(
c[0] - r, c[0] + r, c[1], c[1], c[2] - r, c[2] + r, transpose=False
)
npy = npy[:, 0, :, 0]
try:
npy[shifted_agent_pos[0], shifted_agent_pos[1]] = 1024
except:
pass
mobnums = {"rabbit": -1, "cow": -2, "pig": -3, "chicken": -4, "sheep": -5}
nummobs = {-1: "rabbit", -2: "cow", -3: "pig", -4: "chicken", -5: "sheep"}
for mob in self.world.mobs:
# todo only in the plane?
p = np.round(np.array(self.world.to_world_coords(mob.pos)))
p = p - C
try:
npy[p[0] + r, p[1] + r] = mobnums[mob.mobname]
except:
pass
mapslice = ""
height = npy.shape[0]
width = npy.shape[1]
def xs(x):
return x + int(self.pos[0]) - r
def zs(z):
return z + int(self.pos[2]) - r
mapslice = mapslice + " " * (width + 2) * 3 + "\n"
for i in reversed(range(height)):
mapslice = mapslice + str(xs(i)).center(3)
for j in range(width):
if npy[i, j] > 0:
if npy[i, j] == 1024:
mapslice = mapslice + " A "
else:
mapslice = mapslice + str(npy[i, j]).center(3)
elif npy[i, j] == 0:
mapslice = mapslice + " * "
else:
npy[i, j] = mapslice + " " + nummobs[npy[i, j]][0] + " "
mapslice = mapslice + "\n"
mapslice = mapslice + " "
for j in range(width):
mapslice = mapslice + " * "
mapslice = mapslice + "\n"
mapslice = mapslice + " "
for j in range(width):
mapslice = mapslice + str(zs(j)).center(3)
return mapslice