def add_self_memory_node(self):
"""Adds agent node into its own memory"""
# how/when to, memory is initialized before physical interfaces...
try:
p = self.get_player()
except: # this is for test/test_agent
return
PlayerNode.create(self.memory, p, memid=self.memory.self_memid)
def update_other_players(self, player_list: List):
# input is a list of player_structs from agent
for p in player_list:
mem = self.get_player_by_eid(p.entityId)
if mem is None:
memid = PlayerNode.create(self, p)
else:
memid = mem.memid
PlayerNode.update(self, p, memid)
def test_get_recent_entities(self):
self.memory = AgentMemory()
joe_memid = PlayerNode.create(
self.memory, Player(10, "joe", Pos(1, 0, 1), Look(0, 0)))
players = self.memory.get_recent_entities(memtype="Player")
assert len(players) == 1
jane_memid = PlayerNode.create(
self.memory, Player(11, "jane", Pos(-1, 0, 1), Look(0, 0)))
players = self.memory.get_recent_entities(memtype="Player")
assert len(players) == 2
def get_incoming_chats(self):
all_chats = []
speaker_name = "dashboard"
if self.dashboard_chat is not None:
if not self.memory.get_player_by_name(speaker_name):
PlayerNode.create(
self.memory,
to_player_struct((None, None, None), None, None, None, speaker_name),
)
all_chats.append(self.dashboard_chat)
self.dashboard_chat = None
return all_chats
def get_incoming_chats(self):
c = self.chat_count
for raw_chatstr in self.world.chat_log[c:]:
match = re.search("^<([^>]+)> (.*)", raw_chatstr)
speaker_name = match.group(1)
if not self.memory.get_player_by_name(speaker_name):
# FIXME! name used as eid
PlayerNode.create(
self.memory,
to_player_struct((None, None, None), None, None,
speaker_name, speaker_name),
)
self.chat_count = len(self.world.chat_log)
return self.world.chat_log[c:].copy()
def test_triggers(self):
self.memory = AgentMemory(agent_time=self.time)
joe_memid = PlayerNode.create(
self.memory, Player(10, "joe", Pos(1, 0, 1), Look(0, 0)))
joe_tag_memid = self.memory.tag(joe_memid, "joe")
jane_memid = PlayerNode.create(
self.memory, Player(11, "jane", Pos(-1, 0, 1), Look(0, 0)))
_, joe_mems = self.memory.basic_search(
"SELECT MEMORY FROM ReferenceObject WHERE name=joe")
_, jane_mems = self.memory.basic_search(
"SELECT MEMORY FROM ReferenceObject WHERE name=jane")
assert len(joe_mems) == 1
assert len(jane_mems) == 1
_, joe_mems_from_tag = self.memory.basic_search(
"SELECT MEMORY FROM ReferenceObject WHERE has_tag=joe")
_, jane_mems_from_tag = self.memory.basic_search(
"SELECT MEMORY FROM ReferenceObject WHERE has_tag=jane")
assert len(joe_mems_from_tag) == 1
assert len(jane_mems_from_tag) == 0
self.time.add_tick()
brother_of_memid = self.memory.add_triple(subj=joe_memid,
pred_text="brother_of",
obj=jane_memid)
sister_of_memid = self.memory.add_triple(subj=jane_memid,
pred_text="sister_of",
obj=joe_memid)
triples = self.memory.get_triples(subj=jane_memid,
pred_text="sister_of")
assert len(triples) == 1
self.time.add_tick()
self.memory.db_write("UPDATE ReferenceObjects SET x=? WHERE uuid=?", 2,
joe_memid)
cmd = "SELECT updated_time FROM Memories WHERE uuid=?"
joe_t = self.memory._db_read(cmd, joe_memid)[0][0]
jane_t = self.memory._db_read(cmd, jane_memid)[0][0]
assert joe_t == 2
assert jane_t == 0
self.memory.forget(joe_memid)
triples = self.memory.get_triples(subj=jane_memid,
pred_text="sister_of")
assert len(triples) == 0
def test_get_node_from_memid(self):
self.memory = AgentMemory()
joe_memid = PlayerNode.create(
self.memory, Player(10, "joe", Pos(1, 0, 1), Look(0, 0)))
assert self.memory.get_node_from_memid(joe_memid) == "Player"
loc_memid = LocationNode.create(self.memory, (0, 0, 0))
assert self.memory.get_node_from_memid(loc_memid) == "Location"
def get_player_by_id(self, memid) -> "PlayerNode":
"""Given memid, retrieve PlayerNode
Args:
memid (string): memory ID
"""
return PlayerNode(self, memid)
def test_place_field(self):
memory = AgentMemory()
PF = memory.place_field
joe_x = 1
joe_z = 2
joe_loc = (joe_x, 0, joe_z)
jane_loc = (-1, 0, 1)
joe_memid = PlayerNode.create(
memory, Player(10, "joe", Pos(*joe_loc), Look(0, 0)))
jane_memid = PlayerNode.create(
memory, Player(11, "jane", Pos(*jane_loc), Look(0, 0)))
wall_locs = [{"pos": (-i, 0, 4)} for i in range(5)]
changes = [{
"pos": joe_loc,
"memid": joe_memid
}, {
"pos": jane_loc,
"memid": jane_memid
}]
changes.extend(wall_locs)
PF.update_map(changes)
assert PF.maps[0]["map"].sum() == 7
jl = PF.memid2locs[joe_memid]
assert len(jl) == 1
recovered_pos = tuple(
int(i) for i in PF.map2real(*PF.idx2ijh(list(jl.keys())[0])))
assert recovered_pos == (joe_x, joe_z)
assert len(PF.memid2locs["NULL"]) == 5
changes = [{"pos": (-1, 0, 4), "is_delete": True}]
PF.update_map(changes)
assert len(PF.memid2locs["NULL"]) == 4
new_jane_x = -5
new_jane_z = 5
changes = [{
"pos": (new_jane_x, 0, new_jane_z),
"memid": jane_memid,
"is_move": True
}]
PF.update_map(changes)
jl = PF.memid2locs[jane_memid]
assert len(jl) == 1
recovered_pos = tuple(
int(i) for i in PF.map2real(*PF.idx2ijh(list(jl.keys())[0])))
assert recovered_pos == (new_jane_x, new_jane_z)
assert PF.maps[0]["map"].sum() == 6
def test_linear_extent_search(self):
self.memory = AgentMemory()
PlayerNode.create(
self.memory,
Player(0, "self", Pos(-1, 0, -1), Look(0, 0)),
memid=self.memory.self_memid,
)
joe_eid = 10
joe_memid = PlayerNode.create(
self.memory, Player(joe_eid, "joe", Pos(1, 0, 1), Look(0, 0)))
jane_memid = PlayerNode.create(
self.memory, Player(11, "jane", Pos(-10, 0, 3), Look(0, 0)))
jules_memid = PlayerNode.create(
self.memory, Player(12, "jules", Pos(-1, 0, 2), Look(0, 0)))
# FIXME shouldn't need to do this, interpreter should back off to something else
# if there is no AttentionNode to use in filter_by_sublocation
AttentionNode.create(self.memory, [1, 0, 1], joe_eid)
self.interpreter = Interpreter("joe", "NULL", self.memory)
self.interpreter.subinterpret["attribute"] = AttributeInterpreter()
self.interpreter.subinterpret[
"specify_locations"] = dummy_specify_locations
l = all_test_commands.LINEAR_EXTENTS["distance from joe"]
l["frame"] = "joe"
a = interpret_linear_extent(self.interpreter, "joe", l)
m = MemorySearcher()
comparator = {
"input_left": {
"value_extractor": {
"attribute": a
}
},
"input_right": {
"value_extractor": 5
},
"comparison_type": "GREATER_THAN",
}
query_dict = {
"output": "MEMORY",
"memory_type": "ReferenceObject",
"where_clause": {
"AND": [comparator]
},
}
memids, _ = m.search(self.memory, query=query_dict)
def test_get_entity_by_id(self):
self.memory = MCAgentMemory()
mob_memid = MobNode.create(self.memory,
Mob(10, 65, Pos(1, 2, 3), Look(0, 0)))
player_memid = PlayerNode.create(
self.memory, Player(20, "xyz", Pos(1, 1, 1), Look(1, 1)))
# Test get_entity_by_eid with entityId
assert self.memory.get_entity_by_eid(10).pos == (1.0, 2.0, 3.0)
assert self.memory.get_entity_by_eid(20).pos == (1.0, 1.0, 1.0)
def test_player_apis_memory(self):
self.memory = AgentMemory()
joe_memid = PlayerNode.create(
self.memory, Player(10, "joey", Pos(1, 0, 1), Look(0, 0)))
self.memory.tag(joe_memid, "basketball_player")
ann_memid = PlayerNode.create(
self.memory, Player(20, "ann", Pos(1, 0, 4), Look(0, 0)))
self.memory.tag(ann_memid, "swimmer")
# Test get_player_by_eid
assert self.memory.get_player_by_eid(10).name == "joey"
# Test get_player_by_name
assert self.memory.get_player_by_name("joey").eid == 10
# Test get_players_tagged
# Test get_player_by_id
assert self.memory.get_player_by_id(ann_memid).name == "ann"
def perceive(self):
bots = self.agent.world.get_bots()
for bot in bots:
# print(f"[Perception INFO]: Perceived bot [{bot.name}] in the world, update in memory]")
bot_node = self.agent.memory.get_player_by_eid(bot.entityId)
if bot_node is None:
memid = PlayerNode.create(self.agent.memory, bot)
bot_node = PlayerNode(self.agent.memory, memid)
self.agent.memory.tag(memid, "bot")
bot_node.update(self.agent.memory, bot, bot_node.memid)
print(
f"[Memory INFO]: update bot [{bot.name}] position: ({bot.pos.x}, {bot.pos.y}, {bot.pos.z})"
)
bot_memids = self.agent.memory.get_memids_by_tag("bot")
bots_in_world = [b.entityId for b in bots]
for memid in bot_memids:
bot_eid = self.agent.memory.get_mem_by_id(memid).eid
if bot_eid not in bots_in_world:
self.agent.memory.forget(memid)
print(f"[Memory INFO]: delete bot [{bot_eid}] from memory")
def get_player_by_eid(self, eid) -> Optional["PlayerNode"]:
"""Given eid, retrieve PlayerNode
Args:
eid (int): Entity ID
"""
r = self._db_read_one("SELECT uuid FROM ReferenceObjects WHERE eid=?",
eid)
if r:
return PlayerNode(self, r[0])
else:
return None
def test_untag(self):
self.memory = AgentMemory()
player_memid = PlayerNode.create(
self.memory, Player(10, "rachel", Pos(1, 0, 1), Look(0, 0)))
self.memory.tag(subj_memid=player_memid, tag_text="girl")
self.memory.tag(subj_memid=player_memid, tag_text="plays_football")
assert len(self.memory.get_triples(subj=player_memid,
obj_text="girl")) == 1
self.memory.untag(subj_memid=player_memid, tag_text="girl")
assert len(self.memory.get_triples(subj=player_memid,
obj_text="girl")) == 0
def get_player_by_name(self, name) -> Optional["PlayerNode"]:
"""Given player name, retrieve PlayerNode
Args:
name (string): Player name
"""
r = self._db_read_one(
'SELECT uuid FROM ReferenceObjects WHERE ref_type="player" AND name=?',
name)
# r = self._db_read_one("SELECT uuid FROM Players WHERE name=?", name)
if r:
return PlayerNode(self, r[0])
else:
return None
def setUp(self):
super().setUp()
memory = self.agent.memory
PlayerNode.create(memory,
Player(10, "agent10", Pos(1, 63, 1), Look(0, 0)))
PlayerNode.create(memory,
Player(11, "agent11", Pos(-2, 63, 1), Look(0, 0)))
PlayerNode.create(memory,
Player(12, "agent12", Pos(-4, 63, -3), Look(0, 0)))
self.set_looking_at([1, 63, 1])
def test_memids_and_tags(self):
self.memory = AgentMemory()
player_memid = PlayerNode.create(
self.memory, Player(10, "rache", Pos(1, 0, 1), Look(0, 0)))
self.memory.tag(subj_memid=player_memid, tag_text="girl")
self.memory.tag(subj_memid=player_memid, tag_text="plays_football")
# test get_memids_by_tag
self.memory.get_memids_by_tag(tag="girl")
assert len(self.memory.get_memids_by_tag(tag="girl")) == 1
assert self.memory.get_memids_by_tag(tag="girl")[0] == player_memid
# test_get_tags_by_memid
assert "girl" in self.memory.get_tags_by_memid(player_memid)
# test get_triples
assert len(self.memory.get_triples(subj=player_memid,
obj_text="girl")) == 1
def update(self,
perception_output: namedtuple = None,
areas_to_perceive: List = []):
"""
Updates the world with updates from agent's perception module.
Args:
perception_output: namedtuple with attributes-
mob: All mobs in perception range.
agent_pickable_items: Dict containing - items in agent's perception that can be picked up
and all items that can be picked up by the agent.
agent_attributes: Agent's player attributes including
other_player_list: List of other in-game players
changed_block_attributes: marked attributes (interesting, player_placed, agent_placed)
of changed blocks
in_perceive_area : blockobjects, holes and blocks in the area agent will be running perception in
near_agent: block objects, holes and blocks near the agent
labeled_blocks: labels and resulting locations from semantic segmentation model
:return:
updated_areas_to_perceive: list of (xyz, idm) representing the area agent should perceive
"""
if not perception_output:
return areas_to_perceive
output = {}
updated_areas_to_perceive = areas_to_perceive
"""Perform update the memory with input from low_level perception module"""
# 1. Handle all mobs in agent's perception range
if perception_output.mobs:
map_changes = []
for mob in perception_output.mobs:
mob_memid = self.set_mob_position(mob)
mp = (mob.pos.x, mob.pos.y, mob.pos.z)
map_changes.append({
"pos": mp,
"is_obstacle": False,
"memid": mob_memid,
"is_move": True
})
self.place_field.update_map(map_changes)
# 2. Handle all items that the agent can pick up in-game
if perception_output.agent_pickable_items:
# FIXME PUT IN MEMORY PROPERLY
# 2.1 Items that are in perception range
if perception_output.agent_pickable_items["in_perception_items"]:
for pickable_items in perception_output.agent_pickable_items[
"in_perception_items"]:
self.set_item_stack_position(pickable_items)
# 2.2 Update previous pickable_item_stack based on perception
if perception_output.agent_pickable_items["all_items"]:
# Note: item stacks are not stored properly in memory right now @Yuxuan to fix this.
old_item_stacks = self.get_all_item_stacks()
if old_item_stacks:
for old_item_stack in old_item_stacks:
memid = old_item_stack[0]
eid = old_item_stack[1]
# NIT3: return untag set and tag set
if eid not in perception_output.agent_pickable_items[
"all_items"]:
self.untag(memid, "_on_ground")
else:
self.tag(memid, "_on_ground")
# 3. Update agent's current position and attributes in memory
if perception_output.agent_attributes:
agent_player = perception_output.agent_attributes
memid = self.get_player_by_eid(agent_player.entityId).memid
cmd = (
"UPDATE ReferenceObjects SET eid=?, name=?, x=?, y=?, z=?, pitch=?, yaw=? WHERE "
)
cmd = cmd + "uuid=?"
self.db_write(
cmd,
agent_player.entityId,
agent_player.name,
agent_player.pos.x,
agent_player.pos.y,
agent_player.pos.z,
agent_player.look.pitch,
agent_player.look.yaw,
memid,
)
ap = (agent_player.pos.x, agent_player.pos.y, agent_player.pos.z)
self.place_field.update_map([{
"pos": ap,
"is_obstacle": True,
"memid": memid,
"is_move": True
}])
# 4. Update other in-game players in agent's memory
if perception_output.other_player_list:
player_list = perception_output.other_player_list
for player, location in player_list:
mem = self.get_player_by_eid(player.entityId)
if mem is None:
memid = PlayerNode.create(self, player)
else:
memid = mem.memid
cmd = "UPDATE ReferenceObjects SET eid=?, name=?, x=?, y=?, z=?, pitch=?, yaw=? WHERE "
cmd = cmd + "uuid=?"
self.db_write(
cmd,
player.entityId,
player.name,
player.pos.x,
player.pos.y,
player.pos.z,
player.look.pitch,
player.look.yaw,
memid,
)
pp = (player.pos.x, player.pos.y, player.pos.z)
self.place_field.update_map([{
"pos": pp,
"is_obstacle": True,
"memid": memid,
"is_move": True
}])
memids = self._db_read_one(
'SELECT uuid FROM ReferenceObjects WHERE ref_type="attention" AND type_name=?',
player.entityId,
)
if memids:
self.db_write(
"UPDATE ReferenceObjects SET x=?, y=?, z=? WHERE uuid=?",
location[0],
location[1],
location[2],
memids[0],
)
else:
AttentionNode.create(self,
location,
attender=player.entityId)
# 5. Update the state of the world when a block is changed.
if perception_output.changed_block_attributes:
for (xyz, idm) in perception_output.changed_block_attributes:
# 5.1 Update old instance segmentation if needed
self.maybe_remove_inst_seg(xyz)
# 5.2 Update agent's memory with blocks that have been destroyed.
updated_areas_to_perceive = self.maybe_remove_block_from_memory(
xyz, idm, areas_to_perceive)
# 5.3 Update blocks in memory when any change in the environment is caused either by agent or player
(
interesting,
player_placed,
agent_placed,
) = perception_output.changed_block_attributes[(xyz, idm)]
self.maybe_add_block_to_memory(interesting, player_placed,
agent_placed, xyz, idm)
"""Now perform update the memory with input from heuristic perception module"""
# 1. Process everything in area to attend for perception
if perception_output.in_perceive_area:
# 1.1 Add colors of all block objects
if perception_output.in_perceive_area["block_object_attributes"]:
for block_object_attr in perception_output.in_perceive_area[
"block_object_attributes"]:
block_object, color_tags = block_object_attr
memid = BlockObjectNode.create(self, block_object)
for color_tag in list(set(color_tags)):
self.add_triple(subj=memid,
pred_text="has_colour",
obj_text=color_tag)
# 1.2 Update all holes with their block type in memory
if perception_output.in_perceive_area["holes"]:
self.add_holes_to_mem(
perception_output.in_perceive_area["holes"])
# 1.3 Update tags of air-touching blocks
if "airtouching_blocks" in perception_output.in_perceive_area:
for c, tags in perception_output.in_perceive_area[
"airtouching_blocks"]:
InstSegNode.create(self, c, tags=tags)
# 2. Process everything near agent's current position
if perception_output.near_agent:
# 2.1 Add colors of all block objects
if perception_output.near_agent["block_object_attributes"]:
for block_object_attr in perception_output.near_agent[
"block_object_attributes"]:
block_object, color_tags = block_object_attr
memid = BlockObjectNode.create(self, block_object)
for color_tag in list(set(color_tags)):
self.add_triple(subj=memid,
pred_text="has_colour",
obj_text=color_tag)
# 2.2 Update all holes with their block type in memory
if perception_output.near_agent["holes"]:
self.add_holes_to_mem(perception_output.near_agent["holes"])
# 2.3 Update tags of air-touching blocks
if "airtouching_blocks" in perception_output.near_agent:
for c, tags in perception_output.near_agent[
"airtouching_blocks"]:
InstSegNode.create(self, c, tags=tags)
"""Update the memory with labeled blocks from SubComponent classifier"""
if perception_output.labeled_blocks:
for label, locations in perception_output.labeled_blocks.items():
InstSegNode.create(self, locations, [label])
"""Update the memory with holes"""
if perception_output.holes:
hole_memories = self.add_holes_to_mem(perception_output.holes)
output["holes"] = hole_memories
output["areas_to_perceive"] = updated_areas_to_perceive
return output
def test_sql_form(self):
self.memory = AgentMemory()
# FIXME? should this be in memory init?
# FIXME!! in agents use SelfNode instead of PlayerNode
self_memid = SelfNode.create(self.memory,
Player(1, "robot", Pos(0, 0, 0),
Look(0, 0)),
memid=self.memory.self_memid)
rachel_memid = PlayerNode.create(
self.memory, Player(10, "rachel", Pos(1, 0, 1), Look(0, 0)))
self.memory.tag(subj_memid=rachel_memid, tag_text="girl")
self.memory.tag(subj_memid=rachel_memid, tag_text="plays_football")
robert_memid = PlayerNode.create(
self.memory, Player(11, "robert", Pos(4, 0, 5), Look(0, 0)))
self.memory.tag(subj_memid=robert_memid, tag_text="boy")
self.memory.tag(subj_memid=robert_memid, tag_text="plays_football")
sam_memid = PlayerNode.create(
self.memory, Player(12, "sam", Pos(-2, 0, 5), Look(0, 0)))
self.memory.tag(subj_memid=sam_memid, tag_text="girl")
self.memory.tag(subj_memid=sam_memid, tag_text="plays_volleyball")
# test NOT
m = MemorySearcher()
query = "SELECT MEMORY FROM ReferenceObject WHERE (NOT has_tag=girl)"
memids, _ = m.search(self.memory, query=query)
assert robert_memid in memids
assert sam_memid not in memids
assert rachel_memid not in memids
# test OR
m = MemorySearcher()
query = "SELECT MEMORY FROM ReferenceObject WHERE ((has_tag=plays_volleyball) OR (NOT has_tag=girl))"
memids, _ = m.search(self.memory, query=query)
assert robert_memid in memids
assert sam_memid in memids
assert rachel_memid not in memids
# test that text form and dict form return same records
query_dict = {
"output": "MEMORY",
"memory_type": "ReferenceObject",
"where_clause": {
"OR": [
{
"pred_text": "has_tag",
"obj_text": "plays_volleyball"
},
{
"NOT": [{
"pred_text": "has_tag",
"obj_text": "girl"
}]
},
]
},
}
memids_d, _ = m.search(self.memory, query=query_dict)
assert set(memids_d) == set(memids)
# test table property with tag
m = MemorySearcher()
query = "SELECT MEMORY FROM ReferenceObject WHERE ((has_tag=plays_volleyball) AND (x<0))"
memids, _ = m.search(self.memory, query=query)
assert robert_memid not in memids
assert sam_memid in memids
assert rachel_memid not in memids
query = "SELECT (x, y) FROM ReferenceObject WHERE ((has_tag=plays_volleyball) AND (x<0))"
memids, vals = m.search(self.memory, query=query)
assert abs(vals[0][0] + 2.0) < 0.01
assert abs(vals[0][1]) < 0.01
triple_memid = TripleNode.create(self.memory,
subj=sam_memid,
pred_text="mother_of",
obj=robert_memid)
query = "SELECT MEMORY FROM ReferenceObject WHERE <<#{}, mother_of, ?>>".format(
sam_memid)
memids, _ = m.search(self.memory, query=query)
assert robert_memid in memids
assert len(memids) == 1
# test FROM works
query = "SELECT MEMORY FROM Triple WHERE create_time > -100"
memids, _ = m.search(self.memory, query=query)
assert all(
[type(self.memory.get_mem_by_id(m)) is TripleNode for m in memids])
assert triple_memid in memids
assert robert_memid not in memids
def test_get_mem_by_id(self):
self.memory = AgentMemory()
kavya_memid = PlayerNode.create(
self.memory, Player(10, "kavya", Pos(1, 0, 1), Look(0, 0)))
assert self.memory.get_mem_by_id(kavya_memid).NODE_TYPE == "Player"
