def perceive(self, force=False):
# FIXME (low pri) remove these in code, get from sql
self.agent.pos = to_block_pos(pos_to_np(self.agent.get_player().pos))
if self.agent.count % self.perceive_freq == 0 or force:
for mob in self.agent.get_mobs():
if euclid_dist(self.agent.pos, pos_to_np(
mob.pos)) < self.memory.perception_range:
self.memory.set_mob_position(mob)
item_stack_set = set()
for item_stack in self.agent.get_item_stacks():
item_stack_set.add(item_stack.entityId)
if (euclid_dist(self.agent.pos, pos_to_np(item_stack.pos)) <
self.memory.perception_range):
self.memory.set_item_stack_position(item_stack)
old_item_stacks = self.memory.get_all_item_stacks()
if old_item_stacks:
for old_item_stack in old_item_stacks:
memid = old_item_stack[0]
eid = old_item_stack[1]
if eid not in item_stack_set:
self.memory.untag(memid, "_on_ground")
else:
self.memory.tag(memid, "_on_ground")
# note: no "force"; these run on every perceive call. assumed to be fast
self.update_self_memory()
self.update_other_players(self.agent.get_other_players())
# use safe_get_changed_blocks to deal with pointing
for (xyz, idm) in self.agent.safe_get_changed_blocks():
self.on_block_changed(xyz, idm)
def compute_location_heuristic(player_look, player_pos, mems, steps, reldir):
loc = mems[0].get_pos()
if reldir is not None:
steps = steps or DEFAULT_NUM_STEPS
if reldir == "BETWEEN":
loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2
loc = (loc[0], loc[1], loc[2])
elif reldir == "INSIDE":
for i in range(len(mems)):
mem = mems[i]
locs = heuristic_perception.find_inside(mem)
if len(locs) > 0:
break
if len(locs) == 0:
raise ErrorWithResponse("I don't know how to go inside there")
else:
loc = locs[0]
elif reldir == "NEAR":
pass
elif reldir == "AROUND":
pass
else: # LEFT, RIGHT, etc...
reldir_vec = rotation.DIRECTIONS[reldir]
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec,
player_look.yaw,
0,
inverted=True)
loc = steps * np.array(dir_vec) + to_block_center(loc)
elif steps is not None:
loc = to_block_center(loc) + [0, 0, steps]
return to_block_pos(loc)
def __init__(self, agent, task_data):
super(Move, self).__init__()
if self.finished:
return
self.target = to_block_pos(np.array(task_data["target"]))
self.approx = task_data.get("approx", 1)
self.path = None
self.replace = set()
self.last_stepped_time = agent.memory.get_time()
def handle_fill(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
r = d.get("reference_object")
self.finished = True
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self, speaker, candidates, r, limit=repeat, loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill."
)
return None, None
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
# FIXME use filters properly...
triples = d.get("triples", [])
block_types = [
t.get("obj_text") for t in triples if t.get("pred_text", "") == "has_block_type"
]
try:
fill_idm = get_block_type(block_types[0])
except:
fill_idm = hole_idm
task_data = {"action_dict": d, "schematic": poss, "block_idm": fill_idm}
self.append_new_task(self.task_objects["fill"], task_data)
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
self.finished = True
return None, None
def __call__(
self,
interpreter,
speaker,
mems,
steps,
reldir,
repeat_num=1,
repeat_dir=None,
objects=[],
padding=(1, 1, 1),
enable_geoscorer=False,
):
agent = interpreter.agent
repeat_num = max(repeat_num, len(objects))
player_look = agent.perception_modules[
"low_level"].get_player_struct_by_name(speaker).look
player_pos = pos_to_np(agent.get_player().pos)
speaker_pos = pos_to_np(
agent.perception_modules["low_level"].get_player_struct_by_name(
speaker).pos)
origin = compute_location_heuristic(player_look, player_pos, mems,
steps, reldir)
agent_geoscorer = agent.on_demand_perception["geoscorer"]
if enable_geoscorer and agent_geoscorer is not None and agent_geoscorer.use(
steps, reldir):
r = agent_geoscorer.radius
# Use heuristic to get starting point only for for between and inside
if reldir not in ("BETWEEN", "INSIDE"):
origin = to_block_pos(mems[0].get_pos())
minc = (origin[0] - r, origin[1] - r, origin[2] - r)
maxc = (minc[0] + 2 * r - 1, minc[1] + 2 * r - 1,
minc[2] + 2 * r - 1)
context = agent.get_blocks(minc[0], maxc[0], minc[1], maxc[1],
minc[2], maxc[2])
origin, offsets = agent_geoscorer.produce_object_positions(
objects, context, minc, reldir, speaker_pos)
else:
if repeat_num > 1:
schematic = None if len(objects) == 0 else objects[0][0]
offsets = get_repeat_arrangement(player_look, repeat_num,
repeat_dir, mems, schematic,
padding)
else:
offsets = [(0, 0, 0)]
origin = post_process_loc(origin, interpreter)
offsets = [post_process_loc(o, interpreter) for o in offsets]
return origin, offsets
def handle_fill(self, speaker, d) -> Tuple[Optional[str], Any]:
r = d.get("reference_object")
self.finished = True
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = interpret_reference_location(self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = compute_locations(self, speaker, mems, steps, reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self, speaker, candidates, r, limit=repeat, loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill."
)
return None, None
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
fill_idm = get_block_type(d["has_block_type"]) if "has_block_type" in d else hole_idm
task_data = {"action_dict": d, "schematic": poss, "block_idm": fill_idm}
self.append_new_task(tasks.Fill, task_data)
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
self.finished = True
return None, None
def get_repeat_arrangement(player_look,
repeat_num,
repeat_dir,
ref_mems,
schematic=None,
padding=(1, 1, 1)):
shapeparams = {}
# default repeat dir is LEFT
if not repeat_dir:
repeat_dir = "LEFT"
# eventually fix this to allow number based on shape
shapeparams["N"] = repeat_num
if repeat_dir == "AROUND":
# TODO vertical "around"
shapeparams["orient"] = "xy"
shapeparams["extra_space"] = max(padding)
central_object = ref_mems[0]
bounds = central_object.get_bounds()
b = max(bounds[1] - bounds[0], bounds[3] - bounds[2],
bounds[5] - bounds[4])
shapeparams["encircled_object_radius"] = b
offsets = shapes.arrange("circle", schematic, shapeparams)
else:
reldir_vec = rotation.DIRECTIONS[repeat_dir]
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec,
player_look.yaw,
0,
inverted=True)
max_ind = np.argmax(dir_vec)
shapeparams["extra_space"] = padding[max_ind]
shapeparams["orient"] = dir_vec
offsets = shapes.arrange("line", schematic, shapeparams)
offsets = [tuple(to_block_pos(o)) for o in offsets]
return offsets
def find_inside(entity):
"""Return a point inside the entity if it can find one.
TODO: heuristic quick check to find that there aren't any,
and maybe make this not d^3"""
# is this a negative object? if yes, just return its mean:
if hasattr(entity, "blocks"):
if all(b == (0, 0) for b in entity.blocks.values()):
m = np.mean(list(entity.blocks.keys()), axis=0)
return [to_block_pos(m)]
l = get_locs_from_entity(entity)
if l is None:
return []
m = np.round(np.mean(l, axis=0))
maxes = np.max(l, axis=0)
mins = np.min(l, axis=0)
inside = []
for x in range(mins[0], maxes[0] + 1):
for y in range(mins[1], maxes[1] + 1):
for z in range(mins[2], maxes[2] + 1):
if check_inside([(x, y, z), entity]):
inside.append((x, y, z))
return sorted(inside, key=lambda x: euclid_dist(x, m))
def post_process_loc(loc, interpreter):
return to_block_pos(loc)
def target_to_memory(self, target):
return to_block_pos(np.array(target))
def handle_fill(self, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
tasks = []
r = d.get("reference_object")
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(
self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self,
speaker,
candidates,
r,
limit=repeat,
loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill.")
return None, None, None
tasks = []
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
schematic, tags = interpret_fill_schematic(self, speaker,
d.get("schematic", {}),
poss, hole_idm)
origin = np.min([xyz for (xyz, bid) in schematic], axis=0)
task_data = {
"blocks_list": schematic,
"force": True,
"origin": origin,
"verbose": False,
"embed": True,
"fill_message": True,
"schematic_tags": tags,
}
tasks.append(self.task_objects["build"](self.agent, task_data))
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
return maybe_task_list_to_control_block(tasks, self.agent), None, None