class PutMemoryTestCase(BaseCraftassistTestCase):
def setUp(self):
super().setUp()
self.dialogue_manager = TtadModelDialogueManager(
self.agent,
TTAD_MODELS_DIR + "/ttad.pth",
TTAD_MODELS_DIR + "/ttad_ft_embeds.pth",
TTAD_MODELS_DIR + "/dialogue_grammar.json",
)
self.cube_right = self.add_object(shapes.cube(bid=(42, 0)), (9, 63, 4))
self.cube_left = self.add_object(shapes.cube(), (9, 63, 10))
self.set_looking_at(list(self.cube_right.blocks.keys())[0])
def test_come_here(self):
chat = "come here"
self.add_incoming_chat(chat)
self.dialogue_manager.step((self.speaker, chat))
self.flush()
self.assertLessEqual(
euclid_dist(self.agent.pos, self.get_speaker_pos()), 1)
def test_stop(self):
chat = "stop"
self.add_incoming_chat(chat)
self.dialogue_manager.step((self.speaker, chat))
self.flush()
def setUp(self):
super().setUp()
self.dialogue_manager = TtadModelDialogueManager(
self.agent, None, TTAD_MODEL_DIR, TTAD_BERT_DATA_DIR, None, None)
self.cube_right = self.add_object(shapes.cube(bid=(42, 0)), (9, 63, 4))
self.cube_left = self.add_object(shapes.cube(), (9, 63, 10))
self.set_looking_at(list(self.cube_right.blocks.keys())[0])
def setUp(self):
super().setUp()
self.dialogue_manager = TtadModelDialogueManager(
self.agent,
TTAD_MODELS_DIR + "/ttad.pth",
TTAD_MODELS_DIR + "/ttad_ft_embeds.pth",
TTAD_MODELS_DIR + "/dialogue_grammar.json",
)
self.cube_right = self.add_object(shapes.cube(bid=(42, 0)), (9, 63, 4))
self.cube_left = self.add_object(shapes.cube(), (9, 63, 10))
self.set_looking_at(list(self.cube_right.blocks.keys())[0])
def setUp(self):
self.memory = AgentMemory(
load_minecraft_specs=False) # don't load specs, it's slow
self.agent = FakeAgent(self.memory)
self.dialogue_manager = TtadModelDialogueManager(
self.agent, None, None, None, no_ground_truth_actions=True)
# More helpful error message to encourage test writers to use self.set_looking_at()
self.agent.get_player_line_of_sight = Mock(
side_effect=NotImplementedError(
"Cannot call into C++ function in this unit test. " +
"Call self.set_looking_at() to set the return value"))
# Add a speaker at position (5, 63, 5) looking in the +x direction
self.memory.update(self.agent)
self.speaker = list(self.memory.other_players.values())[0].name
class BaseCraftassistTestCase(unittest.TestCase):
def setUp(self):
self.memory = AgentMemory(
load_minecraft_specs=False) # don't load specs, it's slow
self.agent = FakeAgent(self.memory)
self.dialogue_manager = TtadModelDialogueManager(
self.agent, None, None, None, no_ground_truth_actions=True)
# More helpful error message to encourage test writers to use self.set_looking_at()
self.agent.get_player_line_of_sight = Mock(
side_effect=NotImplementedError(
"Cannot call into C++ function in this unit test. " +
"Call self.set_looking_at() to set the return value"))
# Add a speaker at position (5, 63, 5) looking in the +x direction
self.memory.update(self.agent)
self.speaker = list(self.memory.other_players.values())[0].name
def handle_action_dict(self,
d,
answer: str = None,
stop_on_chat=False,
max_steps=10000) -> Dict[XYZ, IDM]:
"""Handle an action dict and call self.flush()
If "answer" is specified and a question is asked by the agent, respond
with this string.
If "stop_on_chat" is specified, stop iterating if the agent says anything
"""
self.add_incoming_chat("TEST {}".format(d))
obj = self.dialogue_manager.handle_action_dict(self.speaker, d)
if obj is not None:
self.dialogue_manager.dialogue_stack.append(obj)
changes = self.flush(max_steps, stop_on_chat=stop_on_chat)
if len(self.dialogue_manager.dialogue_stack
) != 0 and answer is not None:
self.add_incoming_chat(answer)
changes.update(self.flush(max_steps, stop_on_chat=stop_on_chat))
return changes
def flush(self, max_steps=10000, stop_on_chat=False) -> Dict[XYZ, IDM]:
"""Update memory and step the dialogue and task stacks until they are empty
If "stop_on_chat" is specified, stop iterating if the agent says anything
Return the set of blocks that were changed.
"""
if stop_on_chat:
self.agent.clear_outgoing_chats()
world_before = self.agent._world.copy()
for _ in range(max_steps):
if (len(self.dialogue_manager.dialogue_stack) == 0
and not self.memory.task_stack_peek()):
break
self.memory.update(self.agent)
self.dialogue_manager.dialogue_stack.step()
self.agent.task_step()
if (isinstance(self.dialogue_manager.dialogue_stack.peek(),
AwaitResponse) and
not self.dialogue_manager.dialogue_stack.peek().finished
) or (stop_on_chat and self.agent.get_last_outgoing_chat()):
break
self.memory.update(self.agent)
# get changes
world_after = self.agent._world.copy()
changes = dict(set(world_after.items()) - set(world_before.items()))
changes.update({
k: (0, 0)
for k in set(world_before.keys()) - set(world_after.keys())
})
return changes
def set_looking_at(self, xyz: XYZ):
"""Set the return value for C++ call to get_player_line_of_sight"""
self.agent.get_player_line_of_sight = Mock(return_value=Pos(*xyz))
def set_blocks(self, xyzbms: List[Block], origin: XYZ = (0, 0, 0)):
"""Change the state of the world, block by block"""
for xyz, idm in xyzbms:
abs_xyz = tuple(np.array(xyz) + origin)
self.memory.on_block_changed(abs_xyz, idm)
self.agent._world[abs_xyz] = idm
def add_object(self, xyzbms: List[Block],
origin: XYZ = (0, 0, 0)) -> ObjectNode:
"""Add an object to memory as if it was placed block by block
Args:
- xyzbms: a list of relative (xyz, idm)
- origin: (x, y, z) of the corner
Returns an ObjectNode
"""
self.set_blocks(xyzbms, origin)
abs_xyz = tuple(np.array(xyzbms[0][0]) + origin)
memid = self.memory.get_block_object_ids_by_xyz(abs_xyz)[0]
return self.memory.get_object_by_id(memid)
def get_blocks(self, xyzs: Sequence[XYZ]) -> Dict[XYZ, IDM]:
"""Return the ground truth block state"""
d = {}
for (x, y, z) in xyzs:
B = self.agent.get_blocks(x, x, y, y, z, z)
d[(x, y, z)] = tuple(B[0, 0, 0, :])
return d
def add_incoming_chat(self, chat: str):
"""Add a chat to memory as if it was just spoken by SPEAKER"""
self.memory.add_chat(
self.memory.get_player_by_name(self.speaker).memid, chat)
def assert_schematics_equal(self, a, b):
"""Check equality between two list[(xyz, idm)] schematics
N.B. this compares the shapes and idms, but ignores absolute position offsets.
"""
a, _ = to_relative_pos(a)
b, _ = to_relative_pos(b)
self.assertEqual(set(a), set(b))
def last_outgoing_chat(self) -> str:
return self.agent.get_last_outgoing_chat()
def get_speaker_pos(self) -> XYZ:
return tuple(
pos_to_np(self.memory.get_player_struct_by_name(self.speaker).pos))
def setUp(self):
self.memory = AgentMemory(load_minecraft_specs=False) # don't load specs, it's slow
self.agent = FakeAgent(self.memory)
self.dialogue_manager = TtadModelDialogueManager(
self.agent, None, None, None, None, None, no_ground_truth_actions=True
)
# More helpful error message to encourage test writers to use self.set_looking_at()
self.agent.get_player_line_of_sight = Mock(
side_effect=NotImplementedError(
"Cannot call into C++ function in this unit test. "
+ "Call self.set_looking_at() to set the return value"
)
)
# Add a speaker at position (5, 63, 5) looking in the +x direction
self.memory.update(self.agent)
self.speaker = list(self.memory.other_players.values())[0].name
# Combinable actions to be used in test cases
self.possible_actions = {
"destroy_speaker_look": {
"action_type": "DESTROY",
"reference_object": {"location": {"location_type": "SPEAKER_LOOK"}},
},
"copy_speaker_look_to_agent_pos": {
"action_type": "BUILD",
"reference_object": {"location": {"location_type": "SPEAKER_LOOK"}},
"location": {"location_type": "AGENT_POS"},
},
"build_small_sphere": {
"action_type": "BUILD",
"schematic": {"has_name": "sphere", "has_size": "small"},
},
"build_1x1x1_cube": {
"action_type": "BUILD",
"schematic": {"has_name": "cube", "has_size": "1 x 1 x 1"},
},
"move_speaker_pos": {
"action_type": "MOVE",
"location": {"location_type": "SPEAKER_POS"},
},
"build_diamond": {"action_type": "BUILD", "schematic": {"has_name": "diamond"}},
"build_gold_cube": {
"action_type": "BUILD",
"schematic": {"has_block_type": "gold", "has_name": "cube"},
},
"fill_all_holes_speaker_look": {
"action_type": "FILL",
"location": {"location_type": "SPEAKER_LOOK"},
"repeat": {"repeat_key": "ALL"},
},
"go_to_tree": {
"action_type": "MOVE",
"location": {
"location_type": "REFERENCE_OBJECT",
"reference_object": {"has_name": "tree"},
},
},
"build_square_height_1": {
"action_type": "BUILD",
"schematic": {"has_name": "square", "has_height": "1"},
},
"stop": {"action_type": "STOP"},
"fill_speaker_look": {
"action_type": "FILL",
"location": {"location_type": "SPEAKER_LOOK"},
},
"fill_speaker_look_gold": {
"action_type": "FILL",
"has_block_type": "gold",
"location": {"location_type": "SPEAKER_LOOK"},
},
}
def __init__(
self,
host="localhost",
port=DEFAULT_PORT,
name=None,
ttad_prev_model_path=None,
ttad_model_dir=None,
ttad_bert_data_dir=None,
ttad_embeddings_path=None,
ttad_grammar_path=None,
semseg_model_path=None,
voxel_model_gpu_id=-1,
get_perception_interval=20,
draw_fn=None,
no_default_behavior=False,
geoscorer_model_path=None,
):
logging.info("CraftAssistAgent.__init__ started")
self.name = name or default_agent_name()
self.no_default_behavior = no_default_behavior
# files needed to set up ttad model
if ttad_prev_model_path is None:
ttad_prev_model_path = os.path.join(os.path.dirname(__file__),
"models/ttad/ttad.pth")
if ttad_model_dir is None:
ttad_model_dir = os.path.join(os.path.dirname(__file__),
"models/ttad_bert/model/")
if ttad_bert_data_dir is None:
ttad_bert_data_dir = os.path.join(
os.path.dirname(__file__), "models/ttad_bert/annotated_data/")
if ttad_embeddings_path is None:
ttad_embeddings_path = os.path.join(
os.path.dirname(__file__), "models/ttad/ttad_ft_embeds.pth")
if ttad_grammar_path is None:
ttad_grammar_path = os.path.join(
os.path.dirname(__file__), "models/ttad/dialogue_grammar.json")
# set up the SubComponentClassifier model
if semseg_model_path is not None:
self.subcomponent_classifier = SubComponentClassifier(
voxel_model_path=semseg_model_path)
else:
self.subcomponent_classifier = None
# set up the Geoscorer model
if geoscorer_model_path is not None:
self.geoscorer = Geoscorer(merger_model_path=geoscorer_model_path)
else:
self.geoscorer = None
self.memory = memory.AgentMemory(
db_file=os.environ.get("DB_FILE", ":memory:"),
db_log_path="agent_memory.{}.log".format(self.name),
)
logging.info("Initialized AgentMemory")
self.dialogue_manager = TtadModelDialogueManager(
self,
ttad_prev_model_path,
ttad_model_dir,
ttad_bert_data_dir,
ttad_embeddings_path,
ttad_grammar_path,
)
logging.info("Initialized DialogueManager")
# Log to file
fh = logging.FileHandler("agent.{}.log".format(self.name))
fh.setFormatter(log_formatter)
fh.setLevel(logging.DEBUG)
logging.getLogger().addHandler(fh)
# Login to server
logging.info("Attempting to connect to port {}".format(port))
super().__init__(host, port, self.name)
logging.info("Logged in to server")
# Wrap C++ agent methods
self._cpp_send_chat = self.send_chat
self.send_chat = self._send_chat
self.last_chat_time = 0
self.get_perception_interval = get_perception_interval
self.uncaught_error_count = 0
self.last_task_memid = None
self.point_targets = []
class CraftAssistAgent(Agent):
def __init__(
self,
host="localhost",
port=DEFAULT_PORT,
name=None,
ttad_prev_model_path=None,
ttad_model_dir=None,
ttad_bert_data_dir=None,
ttad_embeddings_path=None,
ttad_grammar_path=None,
semseg_model_path=None,
voxel_model_gpu_id=-1,
get_perception_interval=20,
draw_fn=None,
no_default_behavior=False,
geoscorer_model_path=None,
):
logging.info("CraftAssistAgent.__init__ started")
self.name = name or default_agent_name()
self.no_default_behavior = no_default_behavior
# files needed to set up ttad model
if ttad_prev_model_path is None:
ttad_prev_model_path = os.path.join(os.path.dirname(__file__),
"models/ttad/ttad.pth")
if ttad_model_dir is None:
ttad_model_dir = os.path.join(os.path.dirname(__file__),
"models/ttad_bert/model/")
if ttad_bert_data_dir is None:
ttad_bert_data_dir = os.path.join(
os.path.dirname(__file__), "models/ttad_bert/annotated_data/")
if ttad_embeddings_path is None:
ttad_embeddings_path = os.path.join(
os.path.dirname(__file__), "models/ttad/ttad_ft_embeds.pth")
if ttad_grammar_path is None:
ttad_grammar_path = os.path.join(
os.path.dirname(__file__), "models/ttad/dialogue_grammar.json")
# set up the SubComponentClassifier model
if semseg_model_path is not None:
self.subcomponent_classifier = SubComponentClassifier(
voxel_model_path=semseg_model_path)
else:
self.subcomponent_classifier = None
# set up the Geoscorer model
if geoscorer_model_path is not None:
self.geoscorer = Geoscorer(merger_model_path=geoscorer_model_path)
else:
self.geoscorer = None
self.memory = memory.AgentMemory(
db_file=os.environ.get("DB_FILE", ":memory:"),
db_log_path="agent_memory.{}.log".format(self.name),
)
logging.info("Initialized AgentMemory")
self.dialogue_manager = TtadModelDialogueManager(
self,
ttad_prev_model_path,
ttad_model_dir,
ttad_bert_data_dir,
ttad_embeddings_path,
ttad_grammar_path,
)
logging.info("Initialized DialogueManager")
# Log to file
fh = logging.FileHandler("agent.{}.log".format(self.name))
fh.setFormatter(log_formatter)
fh.setLevel(logging.DEBUG)
logging.getLogger().addHandler(fh)
# Login to server
logging.info("Attempting to connect to port {}".format(port))
super().__init__(host, port, self.name)
logging.info("Logged in to server")
# Wrap C++ agent methods
self._cpp_send_chat = self.send_chat
self.send_chat = self._send_chat
self.last_chat_time = 0
self.get_perception_interval = get_perception_interval
self.uncaught_error_count = 0
self.last_task_memid = None
self.point_targets = []
def start(self):
logging.info("CraftAssistAgent.start() called")
# start the subcomponent classification model
if self.subcomponent_classifier:
self.subcomponent_classifier.start()
for self.count in itertools.count(): # count forever
try:
if self.count == 0:
logging.info("First top-level step()")
self.step()
except Exception as e:
logging.exception(
"Default handler caught exception, db_log_idx={}".format(
self.memory.get_db_log_idx()))
self.send_chat(
"Oops! I got confused and wasn't able to complete my last task :("
)
sentry_sdk.capture_exception(e)
self.memory.task_stack_clear()
self.dialogue_manager.dialogue_stack.clear()
self.uncaught_error_count += 1
if self.uncaught_error_count >= 100:
sys.exit(1)
def step(self):
self.pos = to_block_pos(pos_to_np(self.get_player().pos))
# remove old point targets
self.point_targets = [
pt for pt in self.point_targets if time.time() - pt[1] < 6
]
# Update memory with current world state
# Removed get_perception call due to very slow updates on non-flatworlds
with TimingWarn(2):
self.memory.update(self)
# Process incoming chats
self.dialogue_step()
# Step topmost task on stack
self.task_step()
def task_step(self, sleep_time=0.25):
# Clean finished tasks
while (self.memory.task_stack_peek()
and self.memory.task_stack_peek().task.check_finished()):
self.memory.task_stack_pop()
# Maybe add default task
if not self.no_default_behavior:
self.maybe_run_slow_defaults()
# If nothing to do, wait a moment
if self.memory.task_stack_peek() is None:
time.sleep(sleep_time)
return
# If something to do, step the topmost task
task_mem = self.memory.task_stack_peek()
if task_mem.memid != self.last_task_memid:
logging.info("Starting task {}".format(task_mem.task))
self.last_task_memid = task_mem.memid
task_mem.task.step(self)
self.memory.task_stack_update_task(task_mem.memid, task_mem.task)
def get_time(self):
# round to 100th of second, return as
# n hundreth of seconds since agent init
return self.memory.get_time()
def get_perception(self, force=False):
"""
Get both block objects and component objects and put them
in memory
"""
if not force and (self.count % self.get_perception_interval != 0
or self.memory.task_stack_peek() is not None):
return
block_objs_for_vision = []
for obj in perception.all_nearby_objects(self.get_blocks, self.pos):
memory.BlockObjectNode.create(self.memory, obj)
# If any xyz of obj is has not been labeled
if any([(not self.memory.get_component_object_ids_by_xyz(xyz))
for xyz, _ in obj]):
block_objs_for_vision.append(obj)
# TODO formalize this, make a list of all perception calls to make, etc.
# note this directly adds the memories
perception.get_all_nearby_holes(self, self.pos, radius=15)
perception.get_nearby_airtouching_blocks(self, self.pos, radius=15)
if self.subcomponent_classifier is None:
return
for obj in block_objs_for_vision:
self.subcomponent_classifier.block_objs_q.put(obj)
# everytime we try to retrieve as many recognition results as possible
while not self.subcomponent_classifier.loc2labels_q.empty():
loc2labels, obj = self.subcomponent_classifier.loc2labels_q.get()
loc2ids = dict(obj)
label2blocks = {}
def contaminated(blocks):
"""
Check if blocks are still consistent with the current world
"""
mx, Mx, my, My, mz, Mz = shapes.get_bounds(blocks)
yzxb = self.get_blocks(mx, Mx, my, My, mz, Mz)
for b, _ in blocks:
x, y, z = b
if loc2ids[b][0] != yzxb[y - my, z - mz, x - mx, 0]:
return True
return False
for loc, labels in loc2labels.items():
b = (loc, loc2ids[loc])
for l in labels:
if l in label2blocks:
label2blocks[l].append(b)
else:
label2blocks[l] = [b]
for l, blocks in label2blocks.items():
## if the blocks are contaminated we just ignore
if not contaminated(blocks):
memory.ComponentObjectNode.create(self.memory, blocks, [l])
def maybe_run_slow_defaults(self):
"""Pick a default task task to run
with a low probability"""
if self.memory.task_stack_peek() or len(
self.dialogue_manager.dialogue_stack) > 0:
return
# list of (prob, default function) pairs
visible_defaults = [
(0.001, default_behaviors.build_random_shape),
(0.005, default_behaviors.come_to_player),
]
# default behaviors of the agent not visible in the game
invisible_defaults = []
defaults = (visible_defaults + invisible_defaults if time.time() -
self.last_chat_time > DEFAULT_BEHAVIOUR_TIMEOUT else
invisible_defaults)
defaults = [(p, f) for (p, f) in defaults
if f not in self.memory.banned_default_behaviors]
def noop(*args):
pass
defaults.append(
(1 - sum(p
for p, _ in defaults), noop)) # noop with remaining prob
# weighted random choice of functions
p, fns = zip(*defaults)
fn = np.random.choice(fns, p=p)
if fn != noop:
logging.info("Default behavior: {}".format(fn))
fn(self)
def dialogue_step(self):
"""Process incoming chats and modify task stack"""
raw_incoming_chats = self.get_incoming_chats()
if raw_incoming_chats:
# force to get objects
self.get_perception(force=True)
# logging.info("Incoming chats: {}".format(raw_incoming_chats))
incoming_chats = []
for raw_chat in raw_incoming_chats:
match = re.search("^<([^>]+)> (.*)", raw_chat)
if match is None:
logging.info("Ignoring chat: {}".format(raw_chat))
continue
speaker, chat = match.group(1), match.group(2)
speaker_hash = hash_user(speaker)
logging.info("Incoming chat: ['{}' -> {}]".format(
speaker_hash, chat))
if chat.startswith("/"):
continue
incoming_chats.append((speaker, chat))
self.memory.add_chat(
self.memory.get_player_by_name(speaker).memid, chat)
if len(incoming_chats) > 0:
# change this to memory.get_time() format?
self.last_chat_time = time.time()
# for now just process the first incoming chat
self.dialogue_manager.step(incoming_chats[0])
else:
self.dialogue_manager.step((None, ""))
# TODO reset all blocks in point area to what they
# were before the point action no matter what
# so e.g. player construction in pointing area during point
# is reverted
def safe_get_changed_blocks(self):
blocks = self.get_changed_blocks()
safe_blocks = []
if len(self.point_targets) > 0:
for point_target in self.point_targets:
pt = point_target[0]
for b in blocks:
x, y, z = b[0]
xok = x < pt[0] or x > pt[3]
yok = y < pt[1] or y > pt[4]
zok = z < pt[2] or z > pt[5]
if xok and yok and zok:
safe_blocks.append(b)
else:
safe_blocks = blocks
return safe_blocks
def point_at(self, target, sleep=None):
"""Bot pointing.
Args:
target: list of x1 y1 z1 x2 y2 z2, where:
x1 <= x2,
y1 <= y2,
z1 <= z2.
"""
assert len(target) == 6
self.send_chat("/point {} {} {} {} {} {}".format(*target))
self.point_targets.append((target, time.time()))
# sleep before the bot can take any actions
# otherwise there might be bugs since the object is flashing
# deal with this in the task...
if sleep:
time.sleep(sleep)
def relative_head_pitch(self, angle):
# warning: pitch is flipped!
new_pitch = self.get_player().look.pitch - angle
self.set_look(self.get_player().look.yaw, new_pitch)
def _send_chat(self, chat: str):
logging.info("Sending chat: {}".format(chat))
self.memory.add_chat(self.memory.self_memid, chat)
return self._cpp_send_chat(chat)