def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["bot_capabilities"] = MCBotCapabilities
dialogue_object_classes["interpreter"] = MCInterpreter
dialogue_object_classes["get_memory"] = MCGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = LocoInterpreter
# dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = LocoGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
def init_controller(self):
"""Instantiates controllers - the components that convert a text chat to task(s)."""
dialogue_object_classes = {}
dialogue_object_classes["bot_capabilities"] = LocoBotCapabilities
dialogue_object_classes["interpreter"] = LocoInterpreter
dialogue_object_classes["get_memory"] = LocoGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
class FakeAgent(LocoMCAgent):
coordinate_transforms = rotation
def __init__(
self, world, opts=None, mv_speed=MV_SPEED, rot_speed=ROT_SPEED, head_speed=HEAD_SPEED
):
self.world = world
self.world.agent = self
self.chat_count = 0
if not opts:
opts = Opt()
opts.nsp_models_dir = ""
opts.nsp_data_dir = ""
opts.model_base_path = ""
opts.ground_truth_data_dir = ""
opts.no_ground_truth = True
super(FakeAgent, self).__init__(opts)
self.no_default_behavior = True
self.last_task_memid = None
pos = (0.0, 0.0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos)
self.base_yaw = 0.0
self.pan = 0.0
self.pitch = 0.0
self.mv_speed = mv_speed
self.rot_speed = rot_speed
self.head_speed = head_speed
self.logical_form = None
self._outgoing_chats = []
self.inventory = []
def init_perception(self):
self.perception_modules = {}
self.perception_modules["self"] = SelfPerception(self, perceive_freq=1)
self.perception_modules["vision"] = FakeDetectorPerception(self)
def init_physical_interfaces(self):
self.mover = FakeMover(self)
self.send_chat = self.mover.send_chat
def init_memory(self):
self.memory = LocoAgentMemory()
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = LocoInterpreter
# dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = LocoGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {"logical_form": lf, "chatstr": chatstr, "speaker": speaker}
def step(self):
if hasattr(self.world, "step"):
self.world.step()
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the LocobotAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
super().controller_step()
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(self.memory.get_player_by_name(speaker_name).memid, chatstr)
obj = self.dialogue_manager.handle_logical_form(speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
def get_info(self):
info = {}
# todo self.pos ---> self.camera_pos, self.base_pos
info["pos"] = self.pos
info["pitch"] = self.pitch
info["yaw"] = self.yaw
info["pan"] = self.pan
info["base_yaw"] = self.base_yaw
info["name"] = self.name
return info
########################
## FAKE .PY METHODS ##
########################
def point_at(*args):
pass
# TODO mirror this logic in real locobot
def get_incoming_chats(self):
c = self.chat_count
for raw_chatstr in self.world.chat_log[c:]:
match = re.search("^<([^>]+)> (.*)", raw_chatstr)
speaker_name = match.group(1)
if not self.memory.get_player_by_name(speaker_name):
# FIXME! name used as eid
PlayerNode.create(
self.memory,
to_player_struct((None, None, None), None, None, speaker_name, speaker_name),
)
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player_struct_by_name(self, speaker_name):
p = self.memory.get_player_by_name(speaker_name)
if p:
return p.get_struct()
else:
return None
def get_other_players(self):
return self.world.players.copy()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_look(self):
return Look(self.pitch, self.base_yaw + self.pan)
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y, player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
######################################
## World setup
######################################
# TODO put this in the world, decide if things can move through it, etc.
def add_object(self, xyz, tags=[], colour=None):
self.world.add_object(list(xyz), tags=tags, colour=colour)
# TODO do this better when less mocked
self.perception_modules["vision"].add_detected_object(xyz, properties=tags, colour=colour)
def init_controller(self):
dialogue_object_classes = {}
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
class FakeAgent(LocoMCAgent):
CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)]
def __init__(self, world, opts=None):
self.world = world
self.chat_count = 0
if not opts:
opts = Opt()
opts.nsp_model_dir = None
opts.nsp_data_dir = None
opts.nsp_embedding_path = None
opts.model_base_path = None
opts.QA_nsp_model_path = None
opts.ground_truth_file_path = ""
super(FakeAgent, self).__init__(opts)
self.no_default_behavior = True
self.last_task_memid = None
pos = (0, 63, 0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos, dtype="int")
self.logical_form = None
self._held_item: IDM = (0, 0)
self._look_vec = (1, 0) # (x, z) unit vec
self._changed_blocks: List[Block] = []
self._outgoing_chats: List[str] = []
def init_perception(self):
self.geoscorer = None
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self)
self.perception_modules[
"heuristic"] = heuristic_perception.PerceptionWrapper(self)
def init_physical_interfaces(self):
self.dig = Dig(self)
self.send_chat = SendChat(self)
self.set_held_item = SetHeldItem(self)
self.step_pos_x = StepPosX(self)
self.step_neg_x = StepNegX(self)
self.step_pos_z = StepPosZ(self)
self.step_neg_z = StepNegZ(self)
self.step_pos_y = StepPosY(self)
self.step_neg_y = StepNegY(self)
self.step_forward = StepForward(self)
self.turn_angle = TurnAngle(self)
self.turn_left = TurnLeft(self)
self.turn_right = TurnRight(self)
self.place_block = PlaceBlock(self)
def init_memory(self):
self.memory = MCAgentMemory(
load_minecraft_specs=False) # don't load specs, it's slow
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = GetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {
"logical_form": lf,
"chatstr": chatstr,
"speaker": speaker
}
def step(self):
if hasattr(self.world, "step"):
self.world.step()
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the CraftassistAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
pass
CraftAssistAgent.controller_step(self)
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(
self.memory.get_player_by_name(speaker_name).memid, chatstr)
obj = self.dialogue_manager.handle_logical_form(
speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
########################
## FAKE .PY METHODS ##
########################
def task_step(self):
CraftAssistAgent.task_step(self, sleep_time=0)
def point_at(*args):
pass
def perceive(self, force=False):
self.perception_modules["low_level"].perceive(force=force)
# self.perception_modules["heuristic"].perceive()
###################################
## FAKE C++ PERCEPTION METHODS ##
###################################
def get_blocks(self, xa, xb, ya, yb, za, zb):
return self.world.get_blocks(xa, xb, ya, yb, za, zb)
def get_local_blocks(self, r):
x, y, z = self.pos
return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r)
def get_incoming_chats(self):
c = self.chat_count
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player(self):
return Player(1, "fake_agent", Pos(*self.pos), Look(0, 0), Item(0, 0))
def get_mobs(self):
return self.world.get_mobs()
def get_other_players(self):
return self.world.players.copy()
def get_other_player_by_name(self):
raise NotImplementedError()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y,
player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
def get_changed_blocks(self) -> List[Block]:
# need a better solution here
r = self._changed_blocks.copy()
self._changed_blocks.clear()
return r
def safe_get_changed_blocks(self) -> List[Block]:
return self.get_changed_blocks()
######################################
## World setup
######################################
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block,
store in memory"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.perception_modules["low_level"].on_block_changed(abs_xyz, idm)
self.world.place_block((abs_xyz, idm))
def add_object(self, xyzbms: List[Block],
origin: XYZ = (0, 0, 0)) -> ObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an ObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
return self.memory.get_object_by_id(memid)
class CraftAssistAgent(LocoMCAgent):
def __init__(self, opts):
super(CraftAssistAgent, self).__init__(opts)
self.no_default_behavior = opts.no_default_behavior
self.point_targets = []
self.last_chat_time = 0
def init_memory(self):
self.memory = mc_memory.MCAgentMemory(
db_file=os.environ.get("DB_FILE", ":memory:"),
db_log_path="agent_memory.{}.log".format(self.name),
)
logging.info("Initialized agent memory")
def init_perception(self):
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self)
self.perception_modules[
"heuristic"] = heuristic_perception.PerceptionWrapper(self)
# set up the SubComponentClassifier model
if self.opts.semseg_model_path:
self.perception_modules["semseg"] = SubcomponentClassifierWrapper(
self, self.opts.semseg_model_path)
# set up the Geoscorer model
self.geoscorer = (Geoscorer(
merger_model_path=self.opts.geoscorer_model_path)
if self.opts.geoscorer_model_path else None)
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = GetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
def controller_step(self):
"""Process incoming chats and modify task stack"""
raw_incoming_chats = self.get_incoming_chats()
if raw_incoming_chats:
# force to get objects
self.perceive(force=True)
# logging.info("Incoming chats: {}".format(raw_incoming_chats))
incoming_chats = []
for raw_chat in raw_incoming_chats:
match = re.search("^<([^>]+)> (.*)", raw_chat)
if match is None:
logging.info("Ignoring chat: {}".format(raw_chat))
continue
speaker, chat = match.group(1), match.group(2)
speaker_hash = hash_user(speaker)
logging.info("Incoming chat: ['{}' -> {}]".format(
speaker_hash, chat))
if chat.startswith("/"):
continue
incoming_chats.append((speaker, chat))
self.memory.add_chat(
self.memory.get_player_by_name(speaker).memid, chat)
if len(incoming_chats) > 0:
# change this to memory.get_time() format?
self.last_chat_time = time.time()
# for now just process the first incoming chat
self.dialogue_manager.step(incoming_chats[0])
else:
# Maybe add default task
if not self.no_default_behavior:
self.maybe_run_slow_defaults()
self.dialogue_manager.step((None, ""))
def maybe_run_slow_defaults(self):
"""Pick a default task task to run
with a low probability"""
if self.memory.task_stack_peek() or len(
self.dialogue_manager.dialogue_stack) > 0:
return
# list of (prob, default function) pairs
visible_defaults = [
(0.001, default_behaviors.build_random_shape),
(0.005, default_behaviors.come_to_player),
]
# default behaviors of the agent not visible in the game
invisible_defaults = []
defaults = (visible_defaults + invisible_defaults if time.time() -
self.last_chat_time > DEFAULT_BEHAVIOUR_TIMEOUT else
invisible_defaults)
defaults = [(p, f) for (p, f) in defaults
if f not in self.memory.banned_default_behaviors]
def noop(*args):
pass
defaults.append(
(1 - sum(p
for p, _ in defaults), noop)) # noop with remaining prob
# weighted random choice of functions
p, fns = zip(*defaults)
fn = np.random.choice(fns, p=p)
if fn != noop:
logging.info("Default behavior: {}".format(fn))
fn(self)
def get_time(self):
# round to 100th of second, return as
# n hundreth of seconds since agent init
return self.memory.get_time()
def safe_get_changed_blocks(self):
blocks = self.cagent.get_changed_blocks()
safe_blocks = []
if len(self.point_targets) > 0:
for point_target in self.point_targets:
pt = point_target[0]
for b in blocks:
x, y, z = b[0]
xok = x < pt[0] or x > pt[3]
yok = y < pt[1] or y > pt[4]
zok = z < pt[2] or z > pt[5]
if xok and yok and zok:
safe_blocks.append(b)
else:
safe_blocks = blocks
return safe_blocks
def point_at(self, target, sleep=None):
"""Bot pointing.
Args:
target: list of x1 y1 z1 x2 y2 z2, where:
x1 <= x2,
y1 <= y2,
z1 <= z2.
"""
assert len(target) == 6
self.send_chat("/point {} {} {} {} {} {}".format(*target))
self.point_targets.append((target, time.time()))
# sleep before the bot can take any actions
# otherwise there might be bugs since the object is flashing
# deal with this in the task...
if sleep:
time.sleep(sleep)
def relative_head_pitch(self, angle):
# warning: pitch is flipped!
new_pitch = self.get_player().look.pitch - angle
self.set_look(self.get_player().look.yaw, new_pitch)
def send_chat(self, chat):
logging.info("Sending chat: {}".format(chat))
self.memory.add_chat(self.memory.self_memid, chat)
return self.cagent.send_chat(chat)
# TODO update client so we can just loop through these
# TODO rename things a bit- some perceptual things are here,
# but under current abstraction should be in init_perception
def init_physical_interfaces(self):
# For testing agent without cuberite server
if self.opts.port == -1:
return
logging.info("Attempting to connect to port {}".format(self.opts.port))
self.cagent = MCAgent("localhost", self.opts.port, self.name)
logging.info("Logged in to server")
self.dig = self.cagent.dig
# defined above...
# self.send_chat = self.cagent.send_chat
self.set_held_item = self.cagent.set_held_item
self.step_pos_x = self.cagent.step_pos_x
self.step_neg_x = self.cagent.step_neg_x
self.step_pos_z = self.cagent.step_pos_z
self.step_neg_z = self.cagent.step_neg_z
self.step_pos_y = self.cagent.step_pos_y
self.step_neg_y = self.cagent.step_neg_y
self.step_forward = self.cagent.step_forward
self.look_at = self.cagent.look_at
self.set_look = self.cagent.set_look
self.turn_angle = self.cagent.turn_angle
self.turn_left = self.cagent.turn_left
self.turn_right = self.cagent.turn_right
self.place_block = self.cagent.place_block
self.use_entity = self.cagent.use_entity
self.use_item = self.cagent.use_item
self.use_item_on_block = self.cagent.use_item_on_block
self.craft = self.cagent.craft
self.get_blocks = self.cagent.get_blocks
self.get_local_blocks = self.cagent.get_local_blocks
self.get_incoming_chats = self.cagent.get_incoming_chats
self.get_player = self.cagent.get_player
self.get_mobs = self.cagent.get_mobs
self.get_other_players = self.cagent.get_other_players
self.get_other_player_by_name = self.cagent.get_other_player_by_name
self.get_vision = self.cagent.get_vision
self.get_line_of_sight = self.cagent.get_line_of_sight
self.get_player_line_of_sight = self.cagent.get_player_line_of_sight
self.get_changed_blocks = self.cagent.get_changed_blocks
class FakeAgent(LocoMCAgent):
CCW_LOOK_VECS = [(1, 0), (0, 1), (-1, 0), (0, -1)]
default_frame = CraftAssistAgent.default_frame
coordinate_transforms = CraftAssistAgent.coordinate_transforms
def __init__(self, world, opts=None, do_heuristic_perception=False):
self.world = world
self.chat_count = 0
if not opts:
opts = MockOpt()
super(FakeAgent, self).__init__(opts)
self.do_heuristic_perception = do_heuristic_perception
self.no_default_behavior = True
self.last_task_memid = None
pos = (0, 63, 0)
if hasattr(self.world, "agent_data"):
pos = self.world.agent_data["pos"]
self.pos = np.array(pos, dtype="int")
self.logical_form = None
self.world_interaction_occurred = False
self._held_item: IDM = (0, 0)
self._look_vec = (1, 0, 0)
self._changed_blocks: List[Block] = []
self._outgoing_chats: List[str] = []
CraftAssistAgent.add_self_memory_node(self)
def init_perception(self):
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self, perceive_freq=1)
self.perception_modules["heuristic"] = PerceptionWrapper(self)
self.on_demand_perception = {}
self.on_demand_perception["geoscorer"] = None
self.on_demand_perception["check_inside"] = check_inside
def init_physical_interfaces(self):
self.dig = Dig(self)
self.send_chat = SendChat(self)
self.set_held_item = SetHeldItem(self)
self.step_pos_x = StepPosX(self)
self.step_neg_x = StepNegX(self)
self.step_pos_z = StepPosZ(self)
self.step_neg_z = StepNegZ(self)
self.step_pos_y = StepPosY(self)
self.step_neg_y = StepNegY(self)
self.step_forward = StepForward(self)
self.turn_angle = TurnAngle(self)
self.turn_left = TurnLeft(self)
self.turn_right = TurnRight(self)
self.set_look = SetLook(self)
self.place_block = PlaceBlock(self)
def init_memory(self):
T = FakeMCTime(self.world)
self.memory = MCAgentMemory(load_minecraft_specs=False, agent_time=T)
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["bot_capabilities"] = MCBotCapabilities
dialogue_object_classes["interpreter"] = MCInterpreter
dialogue_object_classes["get_memory"] = MCGetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self, dialogue_object_classes, self.opts)
def set_logical_form(self, lf, chatstr, speaker):
self.logical_form = {"logical_form": lf, "chatstr": chatstr, "speaker": speaker}
def step(self):
if hasattr(self.world, "step"):
if self.world_interaction_occurred or self.count % WORLD_STEP == 0:
self.world.step()
self.world_interaction_occurred = False
if hasattr(self, "recorder"):
self.recorder.record_world()
super().step()
#### use the CraftassistAgent.controller_step()
def controller_step(self):
if self.logical_form is None:
pass
CraftAssistAgent.controller_step(self)
else: # logical form given directly:
# clear the chat buffer
self.get_incoming_chats()
# use the logical form as given...
d = self.logical_form["logical_form"]
chatstr = self.logical_form["chatstr"]
speaker_name = self.logical_form["speaker"]
self.memory.add_chat(self.memory.get_player_by_name(speaker_name).memid, chatstr)
# force to get objects, speaker info
self.perceive(force=True)
obj = self.dialogue_manager.handle_logical_form(speaker_name, d, chatstr)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
self.logical_form = None
def setup_test(self):
self.task_steps_count = 0
def clear_outgoing_chats(self):
self._outgoing_chats.clear()
def get_last_outgoing_chat(self):
try:
return self._outgoing_chats[-1]
except IndexError:
return None
########################
## FAKE .PY METHODS ##
########################
def task_step(self):
CraftAssistAgent.task_step(self, sleep_time=0)
def point_at(*args):
pass
def perceive(self, force=False):
self.perception_modules["low_level"].perceive(force=force)
if self.do_heuristic_perception:
self.perception_modules["heuristic"].perceive()
###################################
## FAKE C++ PERCEPTION METHODS ##
###################################
def get_blocks(self, xa, xb, ya, yb, za, zb):
return self.world.get_blocks(xa, xb, ya, yb, za, zb)
def get_local_blocks(self, r):
x, y, z = self.pos
return self.get_blocks(x - r, x + r, y - r, y + r, z - r, z + r)
def get_incoming_chats(self):
c = self.chat_count
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def get_player(self):
return Player(1, "fake_agent", Pos(*self.pos), self.get_look(), Item(*self._held_item))
def get_mobs(self):
return self.world.get_mobs()
def get_item_stacks(self):
return self.world.get_item_stacks()
def get_other_players(self):
return self.world.players.copy()
def get_other_player_by_name(self):
raise NotImplementedError()
def get_vision(self):
raise NotImplementedError()
def get_line_of_sight(self):
raise NotImplementedError()
def get_look(self):
pitch = -np.rad2deg(np.arcsin(self._look_vec[1]))
yaw = -np.rad2deg(np.arctan2(self._look_vec[0], self._look_vec[2]))
return Look(pitch, yaw)
def get_player_line_of_sight(self, player_struct):
if hasattr(self.world, "get_line_of_sight"):
pos = (player_struct.pos.x, player_struct.pos.y, player_struct.pos.z)
pitch = player_struct.look.pitch
yaw = player_struct.look.yaw
xsect = self.world.get_line_of_sight(pos, yaw, pitch)
if xsect is not None:
return Pos(*xsect)
else:
raise NotImplementedError()
def get_changed_blocks(self) -> List[Block]:
# need a better solution here
r = self._changed_blocks.copy()
self._changed_blocks.clear()
return r
def safe_get_changed_blocks(self) -> List[Block]:
return self.get_changed_blocks()
######################################
## World setup
######################################
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block,
store in memory"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.perception_modules["low_level"].pending_agent_placed_blocks.add(abs_xyz)
# TODO add force option so we don't need to make it as if agent placed
self.perception_modules["low_level"].on_block_changed(abs_xyz, idm)
self.world.place_block((abs_xyz, idm))
def add_object(
self, xyzbms: List[Block], origin: XYZ = (0, 0, 0), relations={}
) -> VoxelObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an VoxelObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
for pred, obj in relations.items():
self.memory.add_triple(subj=memid, pred_text=pred, obj_text=obj)
# sooooorrry FIXME? when we handle triples better in interpreter_helper
if "has_" in pred:
self.memory.tag(memid, obj)
return self.memory.get_object_by_id(memid)
# WARNING!! this does not step the world, but directly fast-forwards
# to count. Use only in world setup, once world is running!
def add_object_ff_time(
self, count, xyzbms: List[Block], origin: XYZ = (0, 0, 0), relations={}
) -> VoxelObjectNode:
assert count >= self.world.count
self.world.set_count(count)
return self.add_object(xyzbms, origin, relations=relations)
######################################
## visualization
######################################
def draw_slice(self, h=None, r=5, c=None):
if not h:
h = self.pos[1]
if c:
c = [c[0], h, c[1]]
else:
c = [self.pos[0], h, self.pos[2]]
C = self.world.to_world_coords(c)
A = self.world.to_world_coords(self.pos)
shifted_agent_pos = [A[0] - C[0] + r, A[2] - C[2] + r]
npy = self.world.get_blocks(
c[0] - r, c[0] + r, c[1], c[1], c[2] - r, c[2] + r, transpose=False
)
npy = npy[:, 0, :, 0]
try:
npy[shifted_agent_pos[0], shifted_agent_pos[1]] = 1024
except:
pass
mobnums = {"rabbit": -1, "cow": -2, "pig": -3, "chicken": -4, "sheep": -5}
nummobs = {-1: "rabbit", -2: "cow", -3: "pig", -4: "chicken", -5: "sheep"}
for mob in self.world.mobs:
# todo only in the plane?
p = np.round(np.array(self.world.to_world_coords(mob.pos)))
p = p - C
try:
npy[p[0] + r, p[1] + r] = mobnums[mob.mobname]
except:
pass
mapslice = ""
height = npy.shape[0]
width = npy.shape[1]
def xs(x):
return x + int(self.pos[0]) - r
def zs(z):
return z + int(self.pos[2]) - r
mapslice = mapslice + " " * (width + 2) * 3 + "\n"
for i in reversed(range(height)):
mapslice = mapslice + str(xs(i)).center(3)
for j in range(width):
if npy[i, j] > 0:
if npy[i, j] == 1024:
mapslice = mapslice + " A "
else:
mapslice = mapslice + str(npy[i, j]).center(3)
elif npy[i, j] == 0:
mapslice = mapslice + " * "
else:
npy[i, j] = mapslice + " " + nummobs[npy[i, j]][0] + " "
mapslice = mapslice + "\n"
mapslice = mapslice + " "
for j in range(width):
mapslice = mapslice + " * "
mapslice = mapslice + "\n"
mapslice = mapslice + " "
for j in range(width):
mapslice = mapslice + str(zs(j)).center(3)
return mapslice
class CraftAssistAgent(LocoMCAgent):
default_frame = DEFAULT_FRAME
coordinate_transforms = rotation
def __init__(self, opts):
super(CraftAssistAgent, self).__init__(opts)
self.no_default_behavior = opts.no_default_behavior
self.point_targets = []
self.last_chat_time = 0
# areas must be perceived at each step
# List of tuple (XYZ, radius), each defines a cube
self.areas_to_perceive = []
self.add_self_memory_node()
self.init_inventory()
def init_inventory(self):
self.inventory = inventory.Inventory()
logging.info("Initialized agent inventory")
def init_memory(self):
self.memory = mc_memory.MCAgentMemory(
db_file=os.environ.get("DB_FILE", ":memory:"),
db_log_path="agent_memory.{}.log".format(self.name),
agent_time=MCTime(self.get_world_time),
)
file_log_handler = logging.FileHandler("agent.{}.log".format(
self.name))
file_log_handler.setFormatter(log_formatter)
logging.getLogger().addHandler(file_log_handler)
logging.info("Initialized agent memory")
def init_perception(self):
self.perception_modules = {}
self.perception_modules["low_level"] = LowLevelMCPerception(self)
self.perception_modules[
"heuristic"] = heuristic_perception.PerceptionWrapper(self)
# set up the SubComponentClassifier model
if self.opts.semseg_model_path:
self.perception_modules["semseg"] = SubcomponentClassifierWrapper(
self, self.opts.semseg_model_path, self.opts.semseg_vocab_path)
# set up the Geoscorer model
self.geoscorer = (Geoscorer(
merger_model_path=self.opts.geoscorer_model_path)
if self.opts.geoscorer_model_path else None)
def init_controller(self):
dialogue_object_classes = {}
dialogue_object_classes["interpreter"] = Interpreter
dialogue_object_classes["get_memory"] = GetMemoryHandler
dialogue_object_classes["put_memory"] = PutMemoryHandler
self.dialogue_manager = NSPDialogueManager(self,
dialogue_object_classes,
self.opts)
def perceive(self, force=False):
self.areas_to_perceive = cluster_areas(self.areas_to_perceive)
for v in self.perception_modules.values():
v.perceive(force=force)
self.areas_to_perceive = []
def controller_step(self):
"""Process incoming chats and modify task stack"""
raw_incoming_chats = self.get_incoming_chats()
incoming_chats = []
for raw_chat in raw_incoming_chats:
match = re.search("^<([^>]+)> (.*)", raw_chat)
if match is None:
logging.info("Ignoring chat: {}".format(raw_chat))
continue
speaker, chat = match.group(1), match.group(2)
speaker_hash = hash_user(speaker)
logging.info("Incoming chat: ['{}' -> {}]".format(
speaker_hash, chat))
if chat.startswith("/"):
continue
incoming_chats.append((speaker, chat))
self.memory.add_chat(
self.memory.get_player_by_name(speaker).memid, chat)
if incoming_chats:
# force to get objects, speaker info
self.perceive(force=True)
# logging.info("Incoming chats: {}".format(raw_incoming_chats))
# change this to memory.get_time() format?
self.last_chat_time = time.time()
# for now just process the first incoming chat
self.dialogue_manager.step(incoming_chats[0])
else:
# Maybe add default task
if not self.no_default_behavior:
self.maybe_run_slow_defaults()
self.dialogue_manager.step((None, ""))
def maybe_run_slow_defaults(self):
"""Pick a default task task to run
with a low probability"""
if self.memory.task_stack_peek() or len(
self.dialogue_manager.dialogue_stack) > 0:
return
# list of (prob, default function) pairs
visible_defaults = [
(0.001, default_behaviors.build_random_shape),
(0.005, default_behaviors.come_to_player),
]
# default behaviors of the agent not visible in the game
invisible_defaults = []
defaults = (visible_defaults + invisible_defaults if time.time() -
self.last_chat_time > DEFAULT_BEHAVIOUR_TIMEOUT else
invisible_defaults)
defaults = [(p, f) for (p, f) in defaults
if f not in self.memory.banned_default_behaviors]
def noop(*args):
pass
defaults.append(
(1 - sum(p
for p, _ in defaults), noop)) # noop with remaining prob
# weighted random choice of functions
p, fns = zip(*defaults)
fn = np.random.choice(fns, p=p)
if fn != noop:
logging.info("Default behavior: {}".format(fn))
fn(self)
def get_time(self):
# round to 100th of second, return as
# n hundreth of seconds since agent init
return self.memory.get_time()
def get_world_time(self):
# MC time is based on ticks, where 20 ticks happen every second.
# There are 24000 ticks in a day, making Minecraft days exactly 20 minutes long.
# The time of day in MC is based on the timestamp modulo 24000 (default).
# 0 is sunrise, 6000 is noon, 12000 is sunset, and 18000 is midnight.
return self.get_time_of_day()
def safe_get_changed_blocks(self):
blocks = self.cagent.get_changed_blocks()
safe_blocks = []
if len(self.point_targets) > 0:
for point_target in self.point_targets:
pt = point_target[0]
for b in blocks:
x, y, z = b[0]
xok = x < pt[0] or x > pt[3]
yok = y < pt[1] or y > pt[4]
zok = z < pt[2] or z > pt[5]
if xok and yok and zok:
safe_blocks.append(b)
else:
safe_blocks = blocks
return safe_blocks
def point_at(self, target, sleep=None):
"""Bot pointing.
Args:
target: list of x1 y1 z1 x2 y2 z2, where:
x1 <= x2,
y1 <= y2,
z1 <= z2.
"""
assert len(target) == 6
self.send_chat("/point {} {} {} {} {} {}".format(*target))
self.point_targets.append((target, time.time()))
# sleep before the bot can take any actions
# otherwise there might be bugs since the object is flashing
# deal with this in the task...
if sleep:
time.sleep(sleep)
def relative_head_pitch(self, angle):
# warning: pitch is flipped!
new_pitch = self.get_player().look.pitch - angle
self.set_look(self.get_player().look.yaw, new_pitch)
def send_chat(self, chat):
logging.info("Sending chat: {}".format(chat))
self.memory.add_chat(self.memory.self_memid, chat)
return self.cagent.send_chat(chat)
# TODO update client so we can just loop through these
# TODO rename things a bit- some perceptual things are here,
# but under current abstraction should be in init_perception
def init_physical_interfaces(self):
# For testing agent without cuberite server
if self.opts.port == -1:
return
logging.info("Attempting to connect to port {}".format(self.opts.port))
self.cagent = MCAgent("localhost", self.opts.port, self.name)
logging.info("Logged in to server")
self.dig = self.cagent.dig
self.drop_item_stack_in_hand = self.cagent.drop_item_stack_in_hand
self.drop_item_in_hand = self.cagent.drop_item_in_hand
self.drop_inventory_item_stack = self.cagent.drop_inventory_item_stack
self.set_inventory_slot = self.cagent.set_inventory_slot
self.get_player_inventory = self.cagent.get_player_inventory
self.get_inventory_item_count = self.cagent.get_inventory_item_count
self.get_inventory_items_counts = self.cagent.get_inventory_items_counts
# defined above...
# self.send_chat = self.cagent.send_chat
self.set_held_item = self.cagent.set_held_item
self.step_pos_x = self.cagent.step_pos_x
self.step_neg_x = self.cagent.step_neg_x
self.step_pos_z = self.cagent.step_pos_z
self.step_neg_z = self.cagent.step_neg_z
self.step_pos_y = self.cagent.step_pos_y
self.step_neg_y = self.cagent.step_neg_y
self.step_forward = self.cagent.step_forward
self.look_at = self.cagent.look_at
self.set_look = self.cagent.set_look
self.turn_angle = self.cagent.turn_angle
self.turn_left = self.cagent.turn_left
self.turn_right = self.cagent.turn_right
self.place_block = self.cagent.place_block
self.use_entity = self.cagent.use_entity
self.use_item = self.cagent.use_item
self.use_item_on_block = self.cagent.use_item_on_block
self.is_item_stack_on_ground = self.cagent.is_item_stack_on_ground
self.craft = self.cagent.craft
self.get_blocks = self.cagent.get_blocks
self.get_local_blocks = self.cagent.get_local_blocks
self.get_incoming_chats = self.cagent.get_incoming_chats
self.get_player = self.cagent.get_player
self.get_mobs = self.cagent.get_mobs
self.get_other_players = self.cagent.get_other_players
self.get_other_player_by_name = self.cagent.get_other_player_by_name
self.get_vision = self.cagent.get_vision
self.get_line_of_sight = self.cagent.get_line_of_sight
self.get_player_line_of_sight = self.cagent.get_player_line_of_sight
self.get_changed_blocks = self.cagent.get_changed_blocks
self.get_item_stacks = self.cagent.get_item_stacks
self.get_world_age = self.cagent.get_world_age
self.get_time_of_day = self.cagent.get_time_of_day
self.get_item_stack = self.cagent.get_item_stack
def add_self_memory_node(self):
# clean this up! FIXME!!!!! put in base_agent_memory?
# how/when to, memory is initialized before physical interfaces...
try:
p = self.get_player()
except: # this is for test/test_agent :(
return
self.memory._db_write(
"INSERT INTO ReferenceObjects(uuid, eid, name, ref_type, x, y, z, pitch, yaw) VALUES (?,?,?,?,?,?,?,?,?)",
self.memory.self_memid,
p.entityId,
p.name,
"player",
p.pos.x,
p.pos.y,
p.pos.z,
p.look.pitch,
p.look.yaw,
)
