def interpret_named_schematic(
interpreter, speaker,
d) -> Tuple[List[Block], Optional[str], List[Tuple[str, str]]]:
"""Return a tuple of 3 values:
- the schematic blocks, list[(xyz, idm)]
- a SchematicNode memid, or None
- a list of (pred, val) tags
"""
if "has_name" not in d:
raise ErrorWithResponse("I don't know what you want me to build.")
name = d["has_name"]
stemmed_name = stemmer.stemWord(name)
shapename = SPECIAL_SHAPES_CANONICALIZE.get(
name) or SPECIAL_SHAPES_CANONICALIZE.get(stemmed_name)
if shapename:
blocks, tags = interpret_shape_schematic(speaker,
d,
shapename=shapename)
return blocks, None, tags
schematic = interpreter.memory.get_schematic_by_name(name)
if schematic is None:
schematic = interpreter.memory.get_schematic_by_name(stemmed_name)
if schematic is None:
raise ErrorWithResponse("I don't know what you want me to build.")
tags = [(p, v)
for (_, p,
v) in interpreter.memory.get_triples(subj=schematic.memid)]
return list(schematic.blocks.items()), schematic.memid, tags
def maybe_get_location_memory(interpreter, speaker, d):
location_type = d.get("location_type", "SPEAKER_LOOK")
if location_type == "REFERENCE_OBJECT" or d.get(
"reference_object") is not None:
if d.get("relative_direction") == "BETWEEN":
if d.get("reference_object_1"):
mem1 = interpret_reference_object(interpreter,
speaker,
d["reference_object_1"],
loose_speakerlook=True)[0]
mem2 = interpret_reference_object(interpreter,
speaker,
d["reference_object_2"],
loose_speakerlook=True)[0]
mems = [mem1, mem2]
else:
mems = interpret_reference_object(interpreter,
speaker,
d["reference_object"],
limit=2,
loose_speakerlook=True)
if len(mems) < 2:
mem1 = None
else:
mem1, mem2 = mems
if not mem1:
# TODO specify the ref object in the error message
raise ErrorWithResponse(
"I don't know what you're referring to")
loc = (np.add(mem1.get_pos(), mem2.get_pos())) / 2
loc = (loc[0], loc[1], loc[2])
else:
mems = interpret_reference_object(interpreter, speaker,
d["reference_object"])
if len(mems) == 0:
tags = set(tags_from_dict(d["reference_object"]))
cands = interpreter.memory.get_recent_entities("Mobs")
mems = [
c for c in cands
if any(set.intersection(set(c.get_tags()), tags))
]
if len(mems) == 0:
cands = interpreter.memory.get_recent_entities(
"BlockObjects")
mems = [
c for c in cands
if any(set.intersection(set(c.get_tags()), tags))
]
if len(mems) == 0:
raise ErrorWithResponse(
"I don't know what you're referring to")
assert len(mems) == 1, mems
interpreter.memory.update_recent_entities(mems)
mem = mems[0]
loc = mem.get_pos()
mems = [mem]
return loc, mems
return None, None
def __init__(
self,
agent,
ref_object=None,
relative_direction="CLOCKWISE", # this is the memory of the object
):
self.agent = agent
self.tick = 0
blocks = [(bpos, bid) for bpos, bid in ref_object.blocks.items()]
bounds = shapes.get_bounds(blocks)
center = np.mean([b[0] for b in blocks], axis=0)
d = max(bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4])
if relative_direction == "CLOCKWISE":
offsets = shapes.arrange(
"circle", schematic=None, shapeparams={"encircled_object_radius": d}
)
elif relative_direction == "ANTICLOCKWISE":
offsets = shapes.arrange(
"circle", schematic=None, shapeparams={"encircled_object_radius": d}
)
offsets = offsets[::-1]
else:
raise NotImplementedError("TODO other kinds of paths")
self.path = [np.round(center + o) for o in offsets]
self.path.append(self.path[0])
# check each offset to find a nearby reachable point, see if a path
# is possible now, and error otherwise
for i in range(len(self.path) - 1):
path = search.astar(agent, self.path[i + 1], approx=2, pos=self.path[i])
if path is None:
raise ErrorWithResponse("I cannot find an appropriate path.")
def interpret_facing(interpreter, speaker, d):
current_pitch = interpreter.agent.get_player().look.pitch
current_yaw = interpreter.agent.get_player().look.yaw
if d.get("yaw_pitch"):
span = d["yaw_pitch"]
# for now assumed in (yaw, pitch) or yaw, pitch or yaw pitch formats
yp = span.replace("(", "").replace(")", "").split()
return {"head_yaw_pitch": (int(yp[0]), int(yp[1]))}
elif d.get("yaw"):
# for now assumed span is yaw as word or number
w = d["yaw"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (word_to_num(w), current_pitch)}
elif d.get("pitch"):
# for now assumed span is pitch as word or number
w = d["pitch"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (current_yaw, word_to_num(w))}
elif d.get("relative_yaw"):
# TODO in the task use turn angle
if d["relative_yaw"].get("angle"):
return {"relative_yaw": int(d["relative_yaw"]["angle"])}
else:
pass
elif d.get("relative_pitch"):
if d["relative_pitch"].get("angle"):
# TODO in the task make this relative!
return {"relative_pitch": int(d["relative_pitch"]["angle"])}
else:
pass
elif d.get("location"):
loc, _ = interpret_location(interpreter, speaker, d["location"])
return {"head_xyz": loc}
else:
raise ErrorWithResponse("I am not sure where you want me to turn")
def interpret_point_target(interpreter, speaker, d):
if d.get("location") is None:
# TODO other facings
raise ErrorWithResponse("I am not sure where you want me to point")
loc, mem = interpret_location(interpreter, speaker, d["location"])
if mem is not None:
return mem.get_point_at_target()
else:
return (loc[0], loc[1] + 1, loc[2], loc[0], loc[1] + 1, loc[2])
def interpret_named_schematic(
interpreter, speaker,
d) -> Tuple[List[Block], Optional[str], List[Tuple[str, str]]]:
"""Return a tuple of 3 values:
- the schematic blocks, list[(xyz, idm)]
- a SchematicNode memid, or None
- a list of (pred, val) tags
"""
if "has_name" not in d:
raise ErrorWithResponse("I don't know what you want me to build.")
name = d["has_name"]
stemmed_name = name
shapename = SPECIAL_SHAPES_CANONICALIZE.get(
name) or SPECIAL_SHAPES_CANONICALIZE.get(stemmed_name)
if shapename:
shape_blocks, tags = interpret_shape_schematic(interpreter,
speaker,
d,
shapename=shapename)
return shape_blocks, None, tags
schematic = interpreter.memory.get_schematic_by_name(name)
if schematic is None:
schematic = interpreter.memory.get_schematic_by_name(stemmed_name)
if schematic is None:
raise ErrorWithResponse("I don't know what you want me to build.")
tags = [(p, v)
for (_, p,
v) in interpreter.memory.get_triples(subj=schematic.memid)]
blocks = schematic.blocks
# TODO generalize to more general block properties
# Longer term: remove and put a call to the modify model here
if d.get("has_colour"):
old_idm = most_common_idm(blocks.values())
c = block_data.COLOR_BID_MAP.get(d["has_colour"])
if c is not None:
new_idm = random.choice(c)
for l in blocks:
if blocks[l] == old_idm:
blocks[l] = new_idm
return list(blocks.items()), schematic.memid, tags
def __init__(
self,
agent,
ref_object=None,
relative_direction="CLOCKWISE", # this is the memory of the object
):
self.agent = agent
self.tick = 0
if ref_object is None or ref_object == "AGENT_POS":
x, y, z = agent.pos
bounds = (x, x, y, y, z, z)
center = (x, y, z)
else:
bounds = ref_object.get_bounds()
center = ref_object.get_pos()
d = max(bounds[1] - bounds[0], bounds[3] - bounds[2],
bounds[5] - bounds[4])
if relative_direction == "CLOCKWISE":
offsets = shapes.arrange(
"circle",
schematic=None,
shapeparams={"encircled_object_radius": d})
elif relative_direction == "ANTICLOCKWISE":
offsets = shapes.arrange(
"circle",
schematic=None,
shapeparams={"encircled_object_radius": d})
offsets = offsets[::-1]
else:
raise NotImplementedError("TODO other kinds of paths")
self.path = [np.round(np.add(center, o)) for o in offsets]
self.path.append(self.path[0])
# check each offset to find a nearby reachable point, see if a path
# is possible now, and error otherwise
for i in range(len(self.path) - 1):
path = search.astar(agent,
self.path[i + 1],
approx=2,
pos=self.path[i])
if path is None:
raise ErrorWithResponse("I cannot find an appropriate path.")
def interpret_reference_object(
interpreter,
speaker,
d,
ignore_mobs=False,
limit=1,
loose_speakerlook=False) -> List[ReferenceObjectNode]:
if d.get("coref_resolve", "NULL") != "NULL":
mem = d["coref_resolve"]
if isinstance(mem, ReferenceObjectNode):
return [mem]
else:
logging.error("bad coref_resolve -> {}".format(mem))
if len(interpreter.progeny_data) == 0:
tags = [
stemmer.stemWord(tag.lstrip("the ")) for key, tag in d.items()
if key.startswith("has_") and isinstance(tag, str)
]
candidates = (get_reference_objects(interpreter, *tags)
if not ignore_mobs else get_objects(interpreter, *tags))
if len(candidates) > 0:
location_d = d.get("location", {"location_type": "SPEAKER_LOOK"})
if limit == 1:
# override with input value
limit = get_repeat_num(d)
r = filter_by_sublocation(interpreter,
speaker,
candidates,
location_d,
limit=limit,
loose=loose_speakerlook)
return [mem for _, mem in r]
else:
# no candidates found; ask Clarification
# TODO: move ttad call to dialogue manager and remove this logic
interpreter.action_dict_frozen = True
player = interpreter.memory.get_player_struct_by_name(speaker)
confirm_candidates = get_objects(interpreter) # no tags
objects = object_looked_at(interpreter.agent,
confirm_candidates,
player,
limit=1)
if len(objects) == 0:
raise ErrorWithResponse(
"I don't know what you're referring to")
_, mem = objects[0]
blocks = list(mem.blocks.keys())
interpreter.provisional["object_mem"] = mem
interpreter.provisional["object"] = blocks
interpreter.provisional["d"] = d
interpreter.dialogue_stack.append_new(ConfirmReferenceObject,
blocks)
raise NextDialogueStep()
else:
# clarification answered
r = interpreter.progeny_data[-1].get("response")
if r == "yes":
# TODO: learn from the tag! put it in memory!
return [interpreter.provisional.get("object_mem")] * limit
else:
# TODO: error handling here ?
return []
def filter_by_sublocation(
interpreter,
speaker,
candidates: List[Tuple[XYZ, T]],
location: Dict,
limit=1,
all_proximity=10,
loose=False,
) -> List[Tuple[XYZ, T]]:
"""Select from a list of candidate (xyz, object) tuples given a sublocation
If limit == 'ALL', return all matching candidates
Returns a list of (xyz, mem) tuples
"""
# handle SPEAKER_LOOK separately due to slightly different semantics
# (proximity to ray instead of point)
if location.get("location_type") == "SPEAKER_LOOK":
player = interpreter.memory.get_player_struct_by_name(speaker)
return object_looked_at(interpreter.agent,
candidates,
player,
limit=limit,
loose=loose)
reldir = location.get("relative_direction")
if reldir:
if reldir == "INSIDE":
if location.get("reference_object"):
# this is ugly, should probably return from interpret_location...
ref_mems = interpret_reference_object(
interpreter, speaker, location["reference_object"])
for l, candidate_mem in candidates:
if perception.check_inside([candidate_mem, ref_mems[0]]):
return [(l, candidate_mem)]
raise ErrorWithResponse("I can't find something inside that")
elif reldir == "AWAY":
raise ErrorWithResponse("I don't know which object you mean")
elif reldir == "NEAR":
pass # fall back to no reference direction
else:
# reference object location, i.e. the "X" in "left of X"
ref_loc = interpret_location(interpreter,
speaker,
location,
ignore_reldir=True)
# relative direction, i.e. the "LEFT" in "left of X"
reldir_vec = rotation.DIRECTIONS[reldir]
# transform each object into the speaker look coordinate system,
# and project onto the reldir vector
look = interpreter.memory.get_player_struct_by_name(speaker).look
proj = [
rotation.transform(np.array(l) - ref_loc, look.yaw, 0)
@ reldir_vec for (l, _) in candidates
]
# filter by relative dir, e.g. "left of Y"
proj_cands = [(p, c) for (p, c) in zip(proj, candidates) if p > 0]
# "the X left of Y" = the right-most X that is left of Y
if limit == "ALL":
limit = len(proj_cands)
return [c for (_, c) in sorted(proj_cands, key=lambda p: p[0])
][:limit]
else:
# no reference direction: choose the closest
if limit == "ALL":
return list(
filter(lambda c: euclid_dist(c[0], ref_loc) <= all_proximity,
candidates))
else:
candidates.sort(key=lambda c: euclid_dist(c[0], ref_loc))
return candidates[:limit]
return [] # this fixes flake but seems awful?
def interpret_location(interpreter, speaker, d, ignore_reldir=False) -> XYZ:
"""Location dict -> coordinates
Side effect: adds mems to agent_memory.recent_entities
if a reference object is interpreted;
and loc to memory
"""
location_type = d.get("location_type", "SPEAKER_LOOK")
if location_type == "REFERENCE_OBJECT":
mems = interpret_reference_object(interpreter, speaker,
d["reference_object"])
if len(mems) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
assert len(mems) == 1, mems
interpreter.memory.update_recent_entities(mems)
mem = mems[0]
loc = mem.get_pos()
elif location_type == "SPEAKER_LOOK":
player = interpreter.memory.get_player_struct_by_name(speaker)
loc = capped_line_of_sight(interpreter.agent, player)
elif location_type == "SPEAKER_POS":
loc = pos_to_np(
interpreter.memory.get_player_struct_by_name(speaker).pos)
elif location_type == "AGENT_POS":
loc = pos_to_np(interpreter.agent.get_player().pos)
elif location_type == "COORDINATES":
loc = cast(
XYZ,
tuple(
int(float(w))
for w in re.findall("[-0-9.]+", d["coordinates"])))
if len(loc) != 3:
logging.error("Bad coordinates: {}".format(d["coordinates"]))
raise ErrorWithResponse(
"I don't understand what location you're referring to")
else:
raise ValueError(
"Can't handle Location type: {}".format(location_type))
# handle relative direction
reldir = d.get("relative_direction")
if reldir is not None and not ignore_reldir:
if reldir == "INSIDE":
if location_type == "REFERENCE_OBJECT":
locs = perception.find_inside(mem)
if len(locs) == 0:
raise ErrorWithResponse(
"I don't know how to go inside there")
else:
loc = locs[0]
elif reldir == "AWAY":
apos = pos_to_np(interpreter.agent.get_player().pos)
dir_vec = (apos - loc) / np.linalg.norm(apos - loc)
num_steps = word_to_num(d.get("steps", "5"))
loc = num_steps * np.array(dir_vec) + to_block_center(loc)
elif reldir == "NEAR":
pass
else: # LEFT, RIGHT, etc...
reldir_vec = rotation.DIRECTIONS[reldir]
look = interpreter.memory.get_player_struct_by_name(speaker).look
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec,
look.yaw,
0,
inverted=True)
num_steps = word_to_num(d.get("steps", "5"))
loc = num_steps * np.array(dir_vec) + to_block_center(loc)
# if steps without relative direction
elif "steps" in d:
num_steps = word_to_num(d.get("steps", "5"))
loc = to_block_center(loc) + [0, 0, num_steps]
return to_block_pos(loc)