def perceive(self, force=False):
"""
run the classifiers, put tags in memory
Args:
force (boolean): set to True to run all perceptual heuristics right now,
as opposed to waiting for perceive_freq steps (default: False)
"""
if self.perceive_freq == 0 and not force:
return
if self.perceive_freq > 0 and self.agent.count % self.perceive_freq != 0 and not force:
return
if self.subcomponent_classifier is None:
return
# TODO don't all_nearby_objects again, search in memory instead
to_label = []
# add all blocks in marked areas
for pos, radius in self.agent.areas_to_perceive:
for obj in all_nearby_objects(self.agent.get_blocks, pos, radius):
to_label.append(obj)
# add all blocks near the agent
for obj in all_nearby_objects(self.agent.get_blocks, self.agent.pos):
to_label.append(obj)
for obj in to_label:
self.subcomponent_classifier.block_objs_q.put(obj)
# everytime we try to retrieve as many recognition results as possible
while not self.subcomponent_classifier.loc2labels_q.empty():
loc2labels, obj = self.subcomponent_classifier.loc2labels_q.get()
loc2ids = dict(obj)
label2blocks = {}
def contaminated(blocks):
"""
Check if blocks are still consistent with the current world
"""
mx, Mx, my, My, mz, Mz = get_bounds(blocks)
yzxb = self.agent.get_blocks(mx, Mx, my, My, mz, Mz)
for b, _ in blocks:
x, y, z = b
if loc2ids[b][0] != yzxb[y - my, z - mz, x - mx, 0]:
return True
return False
for loc, labels in loc2labels.items():
b = (loc, loc2ids[loc])
for l in labels:
if l in label2blocks:
label2blocks[l].append(b)
else:
label2blocks[l] = [b]
for l, blocks in label2blocks.items():
## if the blocks are contaminated we just ignore
if not contaminated(blocks):
locs = [loc for loc, idm in blocks]
InstSegNode.create(self.memory, locs, [l])
def get_nearby_airtouching_blocks(agent, location, radius=15):
"""Get all blocks in 'radius' of 'location'
that are touching air on either side.
Returns:
A list of blocktypes
"""
gh = ground_height(agent, location, 0)[0, 0]
x, y, z = location
ymin = int(max(y - radius, gh))
yzxb = agent.get_blocks(x - radius, x + radius, ymin, y + radius,
z - radius, z + radius)
xyzb = yzxb.transpose([2, 0, 1, 3]).copy()
components = connected_components(xyzb, unique_idm=True)
blocktypes = []
for c in components:
tags = None
for loc in c:
idm = tuple(xyzb[loc[0], loc[1], loc[2], :])
for coord in range(3):
for d in [-1, 1]:
off = [0, 0, 0]
off[coord] = d
l = (loc[0] + off[0], loc[1] + off[1], loc[2] + off[2])
if l[coord] >= 0 and l[coord] < xyzb.shape[coord]:
if xyzb[l[0], l[1], l[2], 0] == 0:
try:
blocktypes.append(idm)
type_name = BLOCK_DATA["bid_to_name"][idm]
tags = [type_name]
colours = COLOUR_DATA["name_to_colors"].get(
type_name, [])
colours.extend(
[c for c in COLOUR_LIST if c in type_name])
if colours:
tags.extend(colours)
tags.extend([{
"has_colour": c
} for c in colours])
tags.extend(
BLOCK_PROPERTY_DATA["name_to_properties"].
get(type_name, []))
except:
logging.debug(
"I see a weird block, ignoring: ({}, {})".
format(idm[0], idm[1]))
if tags:
shifted_c = [(l[0] + x - radius, l[1] + ymin, l[2] + z - radius)
for l in c]
if len(shifted_c) > 0:
InstSegNode.create(agent.memory, shifted_c, tags=tags)
return blocktypes
def get_object_by_id(self, memid: str, table="BlockObjects") -> "VoxelObjectNode":
if table == "BlockObjects":
return BlockObjectNode(self, memid)
elif table == "InstSeg":
return InstSegNode(self, memid)
else:
raise ValueError("Bad table={}".format(table))
def get_object_by_id(self,
memid: str,
table="BlockObjects") -> "VoxelObjectNode":
"""
Returns:
The memory node for the given memid
"""
if table == "BlockObjects":
return BlockObjectNode(self, memid)
elif table == "InstSeg":
return InstSegNode(self, memid)
else:
raise ValueError("Bad table={}".format(table))
def get_all_nearby_holes(agent, location, radius=15, store_inst_seg=True):
"""Returns:
a list of holes. Each hole is tuple(list[xyz], idm)"""
sx, sy, sz = location
max_height = sy + 5
map_size = radius * 2 + 1
height_map = [[sz] * map_size for i in range(map_size)]
hid_map = [[-1] * map_size for i in range(map_size)]
idm_map = [[(0, 0)] * map_size for i in range(map_size)]
visited = set([])
global current_connected_comp, current_idm
current_connected_comp = []
current_idm = (2, 0)
# helper functions
def get_block_info(x, z): # fudge factor 5
height = max_height
while True:
B = agent.get_blocks(x, x, height, height, z, z)
if ((B[0, 0, 0, 0] != 0) and (x != sx or z != sz or height != sy)
and (x != agent.pos[0] or z != agent.pos[2]
or height != agent.pos[1]) and
(B[0, 0, 0, 0] !=
383)): # if it's not a mobile block (agent, speaker, mobs)
return height, tuple(B[0, 0, 0])
height -= 1
gx = [0, 0, -1, 1]
gz = [1, -1, 0, 0]
def dfs(x, y, z):
""" Traverse current connected component and return minimum
height of all surrounding blocks """
build_height = 100000
if (x, y, z) in visited:
return build_height
global current_connected_comp, current_idm
current_connected_comp.append(
(x - radius + sx, y, z - radius + sz)) # absolute positions
visited.add((x, y, z))
for d in range(4):
nx = x + gx[d]
nz = z + gz[d]
if nx >= 0 and nz >= 0 and nx < map_size and nz < map_size:
if height_map[x][z] == height_map[nx][nz]:
build_height = min(build_height, dfs(nx, y, nz))
else:
build_height = min(build_height, height_map[nx][nz])
current_idm = idm_map[nx][nz]
else:
# bad ... hole is not within defined radius
return -100000
return build_height
# find all holes
blocks_queue = []
for i in range(map_size):
for j in range(map_size):
height_map[i][j], idm_map[i][j] = get_block_info(
i - radius + sx, j - radius + sz)
heapq.heappush(blocks_queue, (height_map[i][j] + 1,
(i, height_map[i][j] + 1, j)))
holes = []
while len(blocks_queue) > 0:
hxyz = heapq.heappop(blocks_queue)
h, (x, y, z) = hxyz # NB: relative positions
if (x, y, z) in visited or y > max_height:
continue
assert h == height_map[x][z] + 1, " h=%d heightmap=%d, x,z=%d,%d" % (
h,
height_map[x][z],
x,
z,
) # sanity check
current_connected_comp = []
current_idm = (2, 0)
build_height = dfs(x, y, z)
if build_height >= h:
holes.append((current_connected_comp.copy(), current_idm))
cur_hid = len(holes) - 1
for n, xyz in enumerate(current_connected_comp):
x, y, z = xyz
rx, ry, rz = x - sx + radius, y + 1, z - sz + radius
heapq.heappush(blocks_queue, (ry, (rx, ry, rz)))
height_map[rx][rz] += 1
if hid_map[rx][rz] != -1:
holes[cur_hid][0].extend(holes[hid_map[rx][rz]][0])
holes[hid_map[rx][rz]] = ([], (0, 0))
hid_map[rx][rz] = cur_hid
# A bug in the algorithm above produces holes that include non-air blocks.
# Just patch the problem here, since this function will eventually be
# performed by an ML model
for i, (xyzs, idm) in enumerate(holes):
blocks = fill_idmeta(agent, xyzs)
xyzs = [xyz for xyz, (d, _) in blocks
if d == 0] # remove non-air blocks
holes[i] = (xyzs, idm)
# remove 0-length holes
holes = [h for h in holes if len(h[0]) > 0]
if store_inst_seg:
for hole in holes:
InstSegNode.create(agent.memory,
hole[0],
tags=["hole", "pit", "mine"])
return holes