def get_holes(interpreter,
speaker,
location,
limit=1,
all_proximity=10) -> List[Tuple[XYZ, Hole]]:
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(
interpreter.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
if len(candidates) > 0:
# NB(demiguo): by default, we fill the hole the player is looking at
player_struct = interpreter.agent.perception_modules[
"low_level"].get_player_struct_by_name(speaker)
centroid_hole = object_looked_at(interpreter.agent,
candidates,
player_struct,
limit=limit)
if centroid_hole is None or len(centroid_hole) == 0:
# NB(demiguo): if there's no hole in front of the player, we will fill the nearest hole
speaker_pos = (interpreter.agent.perception_modules["low_level"].
get_player_struct_by_name(speaker).pos)
speaker_pos = to_block_pos(pos_to_np(speaker_pos))
if limit == "ALL":
return list(
filter(
lambda c: euclid_dist(c[0], speaker_pos) <=
all_proximity, candidates))
else:
candidates.sort(key=lambda c: euclid_dist(c[0], speaker_pos))
return candidates[:limit]
else:
return centroid_hole
else:
return []
def handle_fill(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
r = d.get("reference_object")
self.finished = True
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self, speaker, candidates, r, limit=repeat, loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill."
)
return None, None
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
# FIXME use filters properly...
triples = d.get("triples", [])
block_types = [
t.get("obj_text") for t in triples if t.get("pred_text", "") == "has_block_type"
]
try:
fill_idm = get_block_type(block_types[0])
except:
fill_idm = hole_idm
task_data = {"action_dict": d, "schematic": poss, "block_idm": fill_idm}
self.append_new_task(self.task_objects["fill"], task_data)
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
self.finished = True
return None, None
def handle_fill(self, speaker, d) -> Tuple[Optional[str], Any]:
r = d.get("reference_object")
self.finished = True
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = interpret_reference_location(self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = compute_locations(self, speaker, mems, steps, reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self, speaker, candidates, r, limit=repeat, loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill."
)
return None, None
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
fill_idm = get_block_type(d["has_block_type"]) if "has_block_type" in d else hole_idm
task_data = {"action_dict": d, "schematic": poss, "block_idm": fill_idm}
self.append_new_task(tasks.Fill, task_data)
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
self.finished = True
return None, None
def handle_fill(self, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
tasks = []
r = d.get("reference_object")
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(
self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self,
speaker,
candidates,
r,
limit=repeat,
loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill.")
return None, None, None
tasks = []
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
schematic, tags = interpret_fill_schematic(self, speaker,
d.get("schematic", {}),
poss, hole_idm)
origin = np.min([xyz for (xyz, bid) in schematic], axis=0)
task_data = {
"blocks_list": schematic,
"force": True,
"origin": origin,
"verbose": False,
"embed": True,
"fill_message": True,
"schematic_tags": tags,
}
tasks.append(self.task_objects["build"](self.agent, task_data))
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
return maybe_task_list_to_control_block(tasks, self.agent), None, None