def special_reference_search_data(interpreter,
speaker,
S,
entity_id=None,
agent_memory=None):
"""make a search query for a MemorySearcher to return the special ReferenceObject"""
# TODO/FIXME! add things to workspace memory
agent_memory = agent_memory or interpreter.memory
if type(S) is dict:
coord_span = S["coordinates_span"]
loc = cast(
XYZ,
tuple(int(float(w)) for w in re.findall("[-0-9.]+", coord_span)))
if len(loc) != 3:
logging.error("Bad coordinates: {}".format(coord_span))
raise ErrorWithResponse(
"I don't understand what location you're referring to")
memid = agent_memory.add_location(
(int(loc[0]), int(loc[1]), int(loc[2])))
mem = agent_memory.get_location_by_id(memid)
q = "SELECT MEMORY FROM ReferenceObject WHERE uuid={}".format(memid)
else:
if S == "AGENT" or S == "SELF" or S == "SPEAKER":
q = "SELECT MEMORY FROM Player WHERE eid={}".format(entity_id)
elif S == "SPEAKER_LOOK":
q = "SELECT MEMORY FROM Attention WHERE type_name={}".format(
entity_id)
else:
raise ErrorWithResponse("unknown special reference: {}".format(S))
return q
def handle_destroy(self, agent, speaker,
d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'destroy' command by either pushing a dialogue object
or pushing a Destroy task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
default_ref_d = {"filters": {"location": SPEAKERLOOK}}
ref_d = d.get("reference_object", default_ref_d)
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_destructible"])
if len(objs) == 0:
raise ErrorWithResponse(
"I don't understand what you want me to destroy.")
# don't kill mobs
if all(isinstance(obj, MobNode) for obj in objs):
raise ErrorWithResponse("I don't kill animals, sorry!")
if all(isinstance(obj, PlayerNode) for obj in objs):
raise ErrorWithResponse("I don't kill players, sorry!")
objs = [obj for obj in objs if not isinstance(obj, MobNode)]
tasks = []
for obj in objs:
if hasattr(obj, "blocks"):
schematic = list(obj.blocks.items())
task_data = {"schematic": schematic, "action_dict": d}
tasks.append(self.task_objects["destroy"](agent, task_data))
logging.info("Added {} Destroy tasks to stack".format(len(tasks)))
return maybe_task_list_to_control_block(tasks, agent), None, None
def handle_get(self, agent, speaker, d) -> Tuple[Optional[str], Any]:
default_ref_d = {"filters": {"location": AGENTPOS}}
ref_d = d.get("reference_object", default_ref_d)
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_physical_object"]
)
if len(objs) == 0:
raise ErrorWithResponse("I don't know what you want me to get.")
if all(isinstance(obj, PlayerNode) for obj in objs):
raise ErrorWithResponse("I can't get a person, sorry!")
objs = [obj for obj in objs if not isinstance(obj, PlayerNode)]
if d.get("receiver") is None:
receiver_d = None
else:
receiver_d = d.get("receiver").get("reference_object")
receiver = None
if receiver_d:
receiver = self.subinterpret["reference_objects"](self, speaker, receiver_d)
if len(receiver) == 0:
raise ErrorWithResponse("I don't know where you want me to take it")
receiver = receiver[0].memid
tasks = []
for obj in objs:
task_data = {"get_target": obj.memid, "give_target": receiver, "action_dict": d}
tasks.append(self.task_objects["get"](agent, task_data))
# logging.info("Added {} Get tasks to stack".format(len(tasks)))
return maybe_task_list_to_control_block(tasks, agent), None, None
def handle_drop(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'drop' command by either pushing a dialogue object
or pushing a Drop task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
ref_d = d.get("reference_object", None)
if not ref_d:
raise ErrorWithResponse(
"I don't understand what you want me to drop.")
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_in_inventory"])
if len(objs) == 0:
raise ErrorWithResponse(
"I don't understand what you want me to drop.")
obj = [obj for obj in objs if isinstance(obj, ItemStackNode)][0]
item_stack = agent.get_item_stack(obj.eid)
idm = (item_stack.item.id, item_stack.item.meta)
task_data = {"eid": obj.eid, "idm": idm, "obj_memid": obj.memid}
return self.task_objects["drop"](agent, task_data), None, None
def interpret_named_schematic(
interpreter, speaker, d, block_data_info, color_bid_map,
special_shape_function
) -> Tuple[List[Block], Optional[str], List[Tuple[str, str]]]:
"""Return a tuple of 3 values:
- the schematic blocks, list[(xyz, idm)]
- a SchematicNode memid, or None
- a list of (pred, val) tags
"""
# FIXME! this is not compositional, and is not using full FILTERS handlers
filters_d = d.get("filters", {})
flattened_clauses = get_flattened_clauses(filters_d)
name = get_properties_from_clauses(flattened_clauses,
["has_name"]).get("has_name", "")
if not name:
raise ErrorWithResponse("I don't know what you want me to build.")
stemmed_name = name.strip("s") # why aren't we using stemmer anymore?
shapename = SPECIAL_SHAPES_CANONICALIZE.get(
name) or SPECIAL_SHAPES_CANONICALIZE.get(stemmed_name)
if shapename:
shape_blocks, tags = interpret_shape_schematic(
interpreter,
speaker,
d,
block_data_info,
color_bid_map,
special_shape_function,
shapename=shapename,
)
return shape_blocks, None, tags
schematic = interpreter.memory.get_schematic_by_name(name)
if schematic is None:
schematic = interpreter.memory.get_schematic_by_name(stemmed_name)
if schematic is None:
raise ErrorWithResponse("I don't know what you want me to build.")
triples = [(p, v)
for (_, p,
v) in interpreter.memory.get_triples(subj=schematic.memid)]
blocks = schematic.blocks
# TODO generalize to more general block properties
# Longer term: remove and put a call to the modify model here
colour = get_properties_from_clauses(flattened_clauses,
["has_colour"]).get("has_colour", "")
if colour:
old_idm = most_common_idm(blocks.values())
c = color_bid_map.get(colour)
if c is not None:
new_idm = random.choice(c)
for l in blocks:
if blocks[l] == old_idm:
blocks[l] = new_idm
return list(blocks.items()), schematic.memid, triples
def interpret_mob_schematic(interpreter, speaker, filters_d):
spawn_clause = {"pred_text": "has_tag", "obj_text": "_spawn"}
where = filters_d.get("where_clause", {"AND": [spawn_clause]})
if where.get("AND"):
where["AND"].append(spawn_clause)
else:
new_where = {"AND": [spawn_clause, deepcopy(where)]}
where = new_where
# HACK for nsp/data weirdness: for now don't allow
# 'same': 'DISALLOWED' in Selector so could not e.g.
# "spawn three different kinds of mobs". we don't have examples
# like that yet anyway ...
if filters_d.get("selector", {}):
if filters_d["selector"].get("same", "ALLOWED") == "DISALLOWED":
filters_d["selector"]["same"] = "ALLOWED"
# FIXME! we don't need to recopy this here, do more composably
W = interpret_where_backoff(interpreter,
speaker,
where,
memory_type="Schematic")
F = maybe_apply_selector(interpreter, speaker, filters_d, W)
schematic_memids, _ = F()
object_idms = [
list(SchematicNode(interpreter.memory, m).blocks.values())[0]
for m in schematic_memids
]
if not object_idms:
raise ErrorWithResponse("I don't know how to spawn that")
return object_idms
def compute_location_heuristic(player_mem, mems, steps, reldir,
get_locs_from_entity):
loc = mems[0].get_pos()
if reldir is not None:
steps = steps or DEFAULT_NUM_STEPS
if reldir == "BETWEEN":
loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2
loc = (loc[0], loc[1], loc[2])
elif reldir == "INSIDE":
for i in range(len(mems)):
mem = mems[i]
# FIXME
locs = heuristic_perception.find_inside(
mem, get_locs_from_entity)
if len(locs) > 0:
break
if len(locs) == 0:
raise ErrorWithResponse("I don't know how to go inside there")
else:
loc = locs[0]
elif reldir == "NEAR":
pass
elif reldir == "AROUND":
pass
else: # LEFT, RIGHT, etc...
reldir_vec = rotation.DIRECTIONS[reldir]
# this should be an inverse transform so we set inverted=True
yaw, _ = player_mem.get_yaw_pitch()
dir_vec = rotation.transform(reldir_vec, yaw, 0, inverted=True)
loc = steps * np.array(dir_vec) + to_block_center(loc)
elif steps is not None:
loc = to_block_center(loc) + [0, 0, steps]
return to_block_pos(loc)
def __call__(self, interpreter, speaker, d):
self_mem = interpreter.memory.get_mem_by_id(
interpreter.memory.self_memid)
current_yaw, current_pitch = self_mem.get_yaw_pitch()
if d.get("yaw_pitch"):
span = d["yaw_pitch"]
# for now assumed in (yaw, pitch) or yaw, pitch or yaw pitch formats
yp = span.replace("(", "").replace(")", "").split()
return {"head_yaw_pitch": (int(yp[0]), int(yp[1]))}
elif d.get("yaw"):
# for now assumed span is yaw as word or number
w = d["yaw"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (word_to_num(w), current_pitch)}
elif d.get("pitch"):
# for now assumed span is pitch as word or number
w = d["pitch"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (current_yaw, word_to_num(w))}
elif d.get("relative_yaw"):
# TODO in the task use turn angle
if "left" in d["relative_yaw"] or "right" in d["relative_yaw"]:
left = "left" in d["relative_yaw"] or "leave" in d[
"relative_yaw"] # lemmatizer :)
degrees = number_from_span(d["relative_yaw"]) or 90
if degrees > 0 and left:
return {"relative_yaw": -degrees}
else:
return {"relative_yaw": degrees}
else:
try:
degrees = int(number_from_span(d["relative_yaw"]))
return {"relative_yaw": degrees}
except:
pass
elif d.get("relative_pitch"):
if "down" in d["relative_pitch"] or "up" in d["relative_pitch"]:
down = "down" in d["relative_pitch"]
degrees = number_from_span(d["relative_pitch"]) or 90
if degrees > 0 and down:
return {"relative_pitch": -degrees}
else:
return {"relative_pitch": degrees}
else:
# TODO in the task make this relative!
try:
deg = int(number_from_span(d["relative_pitch"]))
return {"relative_pitch": deg}
except:
pass
elif d.get("location"):
mems = interpreter.subinterpret["reference_locations"](
interpreter, speaker, d["location"])
steps, reldir = interpret_relative_direction(
interpreter, d["location"])
loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
return {"head_xyz": loc}
else:
raise ErrorWithResponse("I am not sure where you want me to turn")
def get_special_reference_object(interpreter,
speaker,
S,
agent_memory=None,
eid=None):
"""subinterpret a special reference object.
args:
interpreter: the root interpreter
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
S: the special reference object logical form from
"""
# TODO/FIXME! add things to workspace memory
agent_memory = agent_memory or interpreter.memory
if not eid:
eid = get_eid_from_special(agent_memory, S, speaker=speaker)
q = special_reference_search_data(None,
speaker,
S,
entity_id=eid,
agent_memory=agent_memory)
_, mems = agent_memory.basic_search(q)
if not mems:
# need a better interface for this, don't need to run full perception
# just to force speakerlook in memory
# TODO force if look is stale, not just if it doesn't exist
# this branch shouldn't occur
# interpreter.agent.perceive(force=True)
raise ErrorWithResponse(
"I think you are pointing at something but I don't know what it is"
)
return mems[0]
def safe_call(f, *args, **kwargs):
try:
return f(*args, **kwargs)
except Pyro4.errors.ConnectionClosedError as e:
msg = "{} - {}".format(f._RemoteMethod__name, e)
raise ErrorWithResponse(msg)
except Exception as e:
print("Pyro traceback:")
print("".join(Pyro4.util.getPyroTraceback()))
raise e
def __call__(self, interpreter, speaker, d) -> POINT_AT_TARGET:
if d.get("location") is None:
# TODO other facings
raise ErrorWithResponse("I am not sure where you want me to point")
# TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure
mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, d["location"])
steps, reldir = interpret_relative_direction(interpreter, d)
loc, _ = interpreter.subinterpret["specify_locations"](
interpreter, speaker, mems, steps, reldir
)
return self.point_to_region(loc)
def handle_task_refobj_string(self, task, refobj_attr):
if refobj_attr == "name":
for pred, val in task.task.target:
if pred == "has_name":
return "I am going to the " + prepend_a_an(val), None
elif refobj_attr == "location":
target = tuple(task.task.target)
return "I am going to {}".format(target), None
else:
raise ErrorWithResponse(
"trying get attribute {} from action".format(refobj_attr))
def __call__(self, interpreter, speaker, d) -> Optional[Condition]:
"""subinterpreter for Conditions
args:
interpreter: root interpreter.
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
d: logical form from semantic parser
"""
ct = d.get("condition_type")
if ct:
if self.condition_types.get(ct):
# condition_type NEVER doesn't have a "condition" sibling
if ct == "NEVER":
return self.condition_types[ct](interpreter, speaker, d)
if not d.get("condition"):
raise ErrorWithResponse(
"I thought there was a condition but I don't understand it"
)
return self.condition_types[ct](interpreter, speaker,
d["condition"])
else:
raise ErrorWithResponse("I don't understand that condition")
else:
return None
def handle_task_refobj_string(self, task, refobj_attr):
if refobj_attr == "name":
assert isinstance(task.task, Build), task.task
for pred, val in task.task.schematic_tags:
if pred == "has_name":
return "I am building " + prepend_a_an(val), None
return "I am building something that is {}".format(val), None
elif refobj_attr == "location":
assert task.action_name == "Move", task.action_name
target = tuple(task.task.target)
return "I am going to {}".format(target), None
else:
raise ErrorWithResponse(
"trying get attribute {} from action".format(refobj_attr))
return None, None
def __call__(self, interpreter, speaker, filters_d, get_all=False):
"""
This is a subinterpreter to handle FILTERS dictionaries
Args:
interpreter: root interpreter.
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
filters_d: FILTERS logical form from semantic parser
get_all (bool): if True, output attributes are set with get_all=True
Outputs a (chain) of MemoryFilter objects
"""
val_map = get_val_map(interpreter, speaker, filters_d, get_all=get_all)
# NB (kavyasrinet) output can be string and have value "memory" too here
# is this a specific memory?
# ... then return
specific_mem_filter = maybe_handle_specific_mem(
interpreter, speaker, filters_d, val_map)
if specific_mem_filter is not None:
return specific_mem_filter
memtype = filters_d.get("memory_type", "REFERENCE_OBJECT")
# FIXME/TODO: these share lots of code, refactor
if memtype == "REFERENCE_OBJECT":
# just using this to check if SELF is a possibility, TODO finer control
tags, _ = backoff_where(filters_d.get("where_clause"), {})
F = interpret_where_backoff(
interpreter,
speaker,
filters_d.get("where_clause", {}),
memory_type="ReferenceObject",
ignore_self=not ("SELF" in tags),
)
elif memtype == "TASKS":
F = interpret_task_filter(interpreter, speaker, filters_d)
else:
memtype_key = memtype.lower() + "_filters"
try:
F = interpreter.subinterpret[memtype_key](interpreter, speaker,
filters_d)
except:
raise ErrorWithResponse(
"failed at interpreting filters of type {}".format(
memtype))
F = maybe_apply_selector(interpreter, speaker, filters_d, F)
return maybe_append_left(F, to_append=val_map)
def handle_build(self, agent, speaker,
d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'build' command by either pushing a dialogue object
or pushing a Build task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
# Get the segment to build
if "reference_object" in d:
# handle copy
##########FIXME remove this when DSL updated!!!
md = deepcopy(d)
objs = self.subinterpret["reference_objects"](
self,
speaker,
md["reference_object"],
extra_tags=["VOXEL_OBJECT"],
loose_speakerlook=True,
)
if len(objs) == 0:
raise ErrorWithResponse(
"I don't understand what you want me to build")
tagss = [[(p, v)
for (_, p, v) in self.memory.get_triples(subj=obj.memid)]
for obj in objs]
interprets = [[list(obj.blocks.items()), obj.memid, tags]
for (obj, tags) in zip(objs, tagss)]
else: # a schematic
interprets = interpret_schematic(
self,
speaker,
d.get("schematic", {}),
self.block_data,
self.color_bid_map,
self.special_shape_functions,
)
tasks_data = self.get_block_task_data(speaker, interprets, d)
tasks = []
for td in tasks_data:
t = self.task_objects["build"](agent, td)
tasks.append(t)
logging.info("Adding {} Build tasks to stack".format(len(tasks)))
return maybe_task_list_to_control_block(tasks, agent), None, None
def interpret_time(self, interpreter, speaker, d):
event = None
if d.get("special_time_event"):
return TimeCondition(interpreter.memory, d["special_time_event"])
else:
if not d.get("comparator"):
raise ErrorWithResponse(
"I don't know how to interpret this time condition")
dc = d["comparator"]
dc["input_left"] = {"value_extractor": "NULL"}
comparator = interpret_comparator(interpreter, speaker, dc)
if d.get("event"):
event = self(interpreter, speaker, d["event"])
return TimeCondition(interpreter.memory, comparator, event=event)
def handle_triple(self, agent) -> Tuple[Optional[str], Any]:
"""Writes a triple of type : (subject, predicate_text, object_text)
to the memory and returns a confirmation.
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
ref_obj_d = {"filters": self.logical_form["filters"]}
r = self.subinterpret["reference_objects"](
self, self.speaker, ref_obj_d, extra_tags=["_physical_object"]
)
if len(r) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
mem = r[0]
name = "it"
triples = self.memory.get_triples(subj=mem.memid, pred_text="has_tag")
if len(triples) > 0:
name = triples[0][2].strip("_")
schematic_memid = (
self.memory.convert_block_object_to_schematic(mem.memid).memid
if isinstance(mem, VoxelObjectNode) and len(mem.blocks) > 0
else None
)
for t in self.logical_form["upsert"]["memory_data"].get("triples", []):
if t.get("pred_text") and t.get("obj_text"):
logging.debug("Tagging {} {} {}".format(mem.memid, t["pred_text"], t["obj_text"]))
self.memory.add_triple(
subj=mem.memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
if schematic_memid:
self.memory.add_triple(
subj=schematic_memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
point_at_target = mem.get_point_at_target()
# FIXME agent : This is the only place in file using the agent from the .step()
task = self.task_objects["point"](agent, {"target": point_at_target})
# FIXME? higher pri, make sure this runs now...?
TaskNode(self.memory, task.memid)
r = "OK I'm tagging this %r as %r %r " % (name, t["pred_text"], t["obj_text"])
Say(agent, task_data={"response_options": r})
return
def do_answer(self, agent, mems: Sequence[Any],
vals: Sequence[Any]) -> Tuple[Optional[str], Any]:
"""This function uses the action dictionary and memory state to return an answer.
Args:
mems: Sequence of memories
vals: Sequence of values
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
self.finished = True # noqa
output_type = self.output_type
try:
if type(output_type) is str and output_type.lower() == "count":
# FIXME will multiple count if getting tags
if not any(vals):
Say(agent, task_data={"response_options": "none"})
Say(agent, task_data={"response_options": str(vals[0])})
elif type(output_type) is dict and output_type.get("attribute"):
attrib = output_type["attribute"]
if type(attrib) is str and attrib.lower() == "location":
# add a Point task if attribute is a location
if self.subinterpret.get(
"point_target") and self.task_objects.get("point"):
target = self.subinterpret[
"point_target"].point_to_region(vals[0])
# FIXME agent : This is the only place in file using the agent from the .step()
t = self.task_objects["point"](agent, {
"target": target
})
# FIXME? higher pri, make sure this runs now...?
TaskNode(self.memory, t.memid)
Say(agent, task_data={"response_options": str(vals)})
elif type(output_type) is str and output_type.lower() == "memory":
self.handle_exists(mems)
else:
raise ValueError("Bad answer_type={}".format(output_type))
except IndexError: # index error indicates no answer available
logging.error("No answer available from do_answer")
raise ErrorWithResponse("I don't understand what you're asking")
except Exception as e:
logging.exception(e)
def handle_thicken(interpreter, speaker, modify_dict, obj):
old_blocks = list(obj.blocks.items())
bounds = droidlet.base_util.get_bounds()
mx, my, mz = (bounds[0], bounds[2], bounds[4])
origin = [mx, my, mz]
if modify_dict.get("modify_type") == "THICKER":
num_blocks = modify_dict.get("num_blocks", 1)
new_blocks = thicker(old_blocks, delta=num_blocks)
else:
raise ErrorWithResponse("I don't know how thin out blocks yet")
destroy_task_data = {"schematic": old_blocks}
# FIXME deal with tags!!!
build_task_data = {
"blocks_list": new_blocks,
"origin": origin,
# "schematic_tags": tags,
}
return destroy_task_data, build_task_data
def __call__(self, interpreter, speaker, d):
if d.get("location") is None:
# TODO other facings
raise ErrorWithResponse("I am not sure where you want me to point")
# TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure
# what the +1s are in the return value
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
d["location"])
steps, reldir = interpret_relative_direction(interpreter, d)
# if directly point at a reference object, call built-in fn to get pointed target
if steps is None and reldir is None:
loc = mems[0].get_point_at_target()
else:
loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
return self.point_to_region(loc)
def handle_set(self, agent) -> Tuple[Optional[str], Any]:
"""creates a set of memories
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
ref_obj_d = {"filters": self.logical_form["filters"]}
ref_objs = self.subinterpret["reference_objects"](
self, self.speaker, ref_obj_d, extra_tags=["_physical_object"]
)
if len(ref_objs) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
triples_d = self.logical_form["upsert"]["memory_data"].get("triples")
if len(triples_d) == 1 and triples_d[0]["pred_text"] == "has_name":
# the set has a name; check to see if one with that name exists,
# if so add to it, else create one with that name
name = triples_d[0]["obj_text"]
set_memids, _ = self.memory.basic_search(
"SELECT MEMORY FROM Set WHERE (has_name={} OR name={})".format(name, name)
)
if not set_memids:
# make a new set, and name it
set_memid = SetNode.create(self.memory)
self.memory.add_triple(subj=set_memid, pred_text="has_name", obj_text=name)
else:
# FIXME, which one
set_memid = set_memids[0]
else:
# an anonymous set, assuming its new, and defined to hold the triple(s)
set_memid = SetNode.create(self.memory)
for t in triples_d:
self.memory.add_triple(
subj=set_memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
for r in ref_objs:
self.memory.add_triple(subj=r.memid, pred_text="member_of", obj=set_memid)
# FIXME point to the objects put in the set, otherwise explain this better
Say(agent, task_data={"response_options": "OK made those objects into a set "})
return
def __init__(
self,
agent,
ref_object=None,
relative_direction="CLOCKWISE", # this is the memory of the object
):
self.agent = agent
self.tick = 0
if not ref_object:
x, y, z = agent.pos
bounds = (x, x, y, y, z, z)
center = (x, y, z)
else:
bounds = ref_object.get_bounds()
center = ref_object.get_pos()
d = max(bounds[1] - bounds[0], bounds[3] - bounds[2],
bounds[5] - bounds[4])
if relative_direction == "CLOCKWISE" or relative_direction == "AROUND":
offsets = droidlet.shared_data_struct.craftassist_shared_utils.arrange(
"circle",
schematic=None,
shapeparams={"encircled_object_radius": d})
elif relative_direction == "ANTICLOCKWISE":
offsets = droidlet.shared_data_struct.craftassist_shared_utils.arrange(
"circle",
schematic=None,
shapeparams={"encircled_object_radius": d})
offsets = offsets[::-1]
else:
raise NotImplementedError("TODO other kinds of paths")
self.path = [np.round(np.add(center, o)) for o in offsets]
self.path.append(self.path[0])
# check each offset to find a nearby reachable point, see if a path
# is possible now, and error otherwise
for i in range(len(self.path) - 1):
path = astar(agent, self.path[i + 1], approx=2, pos=self.path[i])
if path is None:
raise ErrorWithResponse("I cannot find an appropriate path.")
def handle_undo(self, agent, speaker, d) -> Tuple[Optional[str], Any]:
Undo = self.task_objects["undo"]
task_name = d.get("undo_action")
if task_name:
task_name = task_name.split("_")[0].strip()
old_task = self.memory.get_last_finished_root_task(task_name)
if old_task is None:
raise ErrorWithResponse("Nothing to be undone ...")
undo_tasks = [Undo(agent, {"memid": old_task.memid})]
for u in undo_tasks:
agent.memory.get_mem_by_id(u.memid).get_update_status(
{"paused": 1})
undo_command = old_task.get_chat().chat_text
logging.debug("Pushing ConfirmTask tasks={}".format(undo_tasks))
confirm_data = {
"task_memids": [u.memid for u in undo_tasks],
"question":
'Do you want me to undo the command: "{}" ?'.format(undo_command),
}
ConfirmTask(agent, confirm_data)
self.finished = True
return None, None
def new_tasks():
# TODO if we do this better will be able to handle "stay between the x"
default_loc = getattr(self, "default_loc", SPEAKERLOOK)
location_d = d.get("location", default_loc)
# FIXME, this is hacky. need more careful way of storing this in task
if loop_mem:
mems = [loop_mem]
else:
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
# FIXME this should go in the ref_location subinterpret:
steps, reldir = interpret_relative_direction(self, location_d)
pos, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
# TODO: can this actually happen?
if pos is None:
raise ErrorWithResponse(
"I don't understand where you want me to move.")
pos = self.post_process_loc(pos, self)
task_data = {"target": pos, "action_dict": d}
task = Move(agent, task_data)
return task
def new_tasks():
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE"
or rd == "ANTICLOCKWISE"):
ref_obj = None
location_reference_object = location_d.get(
"reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object)
if len(objmems) == 0:
raise ErrorWithResponse(
"I don't understand where you want me to go.")
ref_obj = objmems[0]
refmove = dance.RefObjMovement(
agent,
ref_object=ref_obj,
relative_direction=location_d["relative_direction"],
)
t = self.task_objects["dance"](agent, {
"movement": refmove
})
return t
dance_type = d.get("dance_type", {})
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker,
dance_type["point"])
t = self.task_objects["point"](agent, {"target": target})
# MC bot does not control body turn separate from head
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn") or dance_type.get("body_turn")
f = self.subinterpret["facing"](self, speaker, lt)
t = self.task_objects["dancemove"](agent, f)
else:
if location_d is None:
dance_location = None
else:
mems = self.subinterpret["reference_locations"](self,
speaker,
location_d)
steps, reldir = interpret_relative_direction(
self, location_d)
dance_location, _ = self.subinterpret["specify_locations"](
self, speaker, mems, steps, reldir)
filters_d = dance_type.get("filters", {})
filters_d["memory_type"] = "DANCES"
F = self.subinterpret["filters"](self, speaker,
dance_type.get("filters", {}))
dance_memids, _ = F()
# TODO correct selector in filters
if dance_memids:
dance_memid = random.choice(dance_memids)
dance_mem = self.memory.get_mem_by_id(dance_memid)
dance_obj = dance.Movement(agent=agent,
move_fn=dance_mem.dance_fn,
dance_location=dance_location)
t = self.task_objects["dance"](agent, {
"movement": dance_obj
})
else:
# dance out of scope
raise ErrorWithResponse(
"I don't know how to do that movement yet.")
return t
def handle_fill(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
tasks = []
r = d.get("reference_object")
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
"""
FIXME: We need to fix this and perhaps put this in reasoning. Agent should run perception
and put into memory. Interpreter shouldn't be perceiving, but should be able to
ask the agent to do it when needed.
"""
# Get nearby holes
perception_holes = self.get_all_holes_fn(
agent, location, self.block_data,
agent.low_level_data["fill_idmeta"])
perception_output = CraftAssistPerceptionData(holes=perception_holes)
output = self.memory.update(perception_output=perception_output)
holes = output.get("holes", [])
# Choose the best ones to fill
holes = filter_by_sublocation(self, speaker, holes, r, loose=True)
if holes is None:
# FIXME: in stage III, replace agent with the lowlevel interface to sending chats
raise ErrorWithResponse(
"I don't understand what holes you want me to fill.")
tasks = []
for hole in holes:
poss = list(hole.blocks.keys())
try:
fill_memid = agent.memory.get_triples(
subj=hole.memid, pred_text="has_fill_type")[0][2]
fill_block_mem = self.memory.get_mem_by_id(fill_memid)
fill_idm = (fill_block_mem.b, fill_block_mem.m)
except:
# FIXME use a constant name
fill_idm = (3, 0)
schematic, tags = interpret_fill_schematic(
self,
speaker,
d.get("schematic", {}),
poss,
fill_idm,
self.block_data,
self.color_bid_map,
)
origin = np.min([xyz for (xyz, bid) in schematic], axis=0)
task_data = {
"blocks_list": schematic,
"force": True,
"origin": origin,
"verbose": False,
"embed": True,
"fill_message": True,
"schematic_tags": tags,
}
tasks.append(self.task_objects["build"](agent, task_data))
if len(holes) > 1:
Say(agent,
task_data={"response_options": "Ok. I'll fill up the holes."})
else:
Say(agent,
task_data={"response_options": "Ok. I'll fill that hole up."})
return maybe_task_list_to_control_block(tasks, agent), None, None
def handle_modify(self, agent, speaker,
d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and handles a 'modify' command by either replying back or pushing
appropriate tasks to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
default_ref_d = {"filters": {"location": SPEAKERLOOK}}
ref_d = d.get("reference_object", default_ref_d)
# only modify blockobjects...
objs = self.subinterpret["reference_objects"](
self,
speaker,
ref_d,
extra_tags=["_physical_object", "VOXEL_OBJECT"])
if len(objs) == 0:
raise ErrorWithResponse(
"I don't understand what you want me to modify.")
m_d = d.get("modify_dict")
if not m_d:
raise ErrorWithResponse(
"I think you want me to modify an object but am not sure what to do"
)
tasks = []
for obj in objs:
if m_d["modify_type"] == "THINNER" or m_d[
"modify_type"] == "THICKER":
destroy_task_data, build_task_data = handle_thicken(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "REPLACE":
destroy_task_data, build_task_data = handle_replace(
self,
speaker,
m_d,
obj,
block_data=self.block_data,
color_bid_map=self.color_bid_map,
)
elif m_d["modify_type"] == "SCALE":
destroy_task_data, build_task_data = handle_scale(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "RIGIDMOTION":
destroy_task_data, build_task_data = handle_rigidmotion(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "FILL" or m_d["modify_type"] == "HOLLOW":
destroy_task_data, build_task_data = handle_fill(
self,
speaker,
m_d,
obj,
block_data=self.block_data,
color_bid_map=self.color_bid_map,
)
else:
raise ErrorWithResponse(
"I think you want me to modify an object but am not sure what to do (parse error)"
)
if build_task_data:
tasks.append(self.task_objects["build"](agent,
build_task_data))
if destroy_task_data:
tasks.append(self.task_objects["build"](agent,
destroy_task_data))
return maybe_task_list_to_control_block(tasks, agent), None, None
def filter_by_sublocation(interpreter,
speaker,
candidates: List[T],
d: Dict,
all_proximity=10,
loose=False) -> List[T]:
"""Select from a list of candidate reference_object mems given a sublocation
also handles random sampling
Returns a list of mems
"""
filters_d = d.get("filters")
assert filters_d is not None, "no filters: {}".format(d)
default_loc = getattr(interpreter, "default_loc", SPEAKERLOOK)
location = filters_d.get("selector", {}).get("location", default_loc)
reldir = location.get("relative_direction")
distance_sorted = False
location_filtered_candidates = []
if reldir:
if reldir == "INSIDE":
# FIXME formalize this better, make extensible
if location.get("reference_object"):
# should probably return from interpret_reference_location...
ref_mems = interpret_reference_object(
interpreter, speaker, location["reference_object"])
# FIXME !!! this should be more clearly delineated
# between perception and memory
I = getattr(interpreter.memory, "check_inside", None)
if I:
for candidate_mem in candidates:
if I([candidate_mem, ref_mems[0]]):
location_filtered_candidates.append(candidate_mem)
else:
raise ErrorWithResponse("I don't know how to check inside")
if not location_filtered_candidates:
raise ErrorWithResponse("I can't find something inside that")
elif reldir == "AWAY":
raise ErrorWithResponse("I don't know which object you mean")
elif reldir == "NEAR":
pass # fall back to no reference direction
elif reldir == "BETWEEN":
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
location)
steps, reldir = interpret_relative_direction(interpreter, d)
ref_loc, _ = interpreter.subinterpret["specify_locations"](
interpreter, speaker, mems, steps, reldir)
distance_sorted = True
location_filtered_candidates = candidates
location_filtered_candidates.sort(
key=lambda c: euclid_dist(c.get_pos(), ref_loc))
else:
# reference object location, i.e. the "X" in "left of X"
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
location)
if not mems:
raise ErrorWithResponse("I don't know which object you mean")
# FIXME!!! handle frame better, might want agent's frame instead
# FIXME use the subinterpreter, don't directly call the attribute
eid = interpreter.memory.get_player_by_name(speaker).eid
self_mem = interpreter.memory.get_mem_by_id(
interpreter.memory.self_memid)
L = LinearExtentAttribute(interpreter.memory, {
"frame": eid,
"relative_direction": reldir
},
mem=self_mem)
c_proj = L(candidates)
m_proj = L(mems)
# FIXME don't just take the first...
m_proj = m_proj[0]
# filter by relative dir, e.g. "left of Y"
location_filtered_candidates = [
c for (p, c) in zip(c_proj, candidates) if p > m_proj
]
# "the X left of Y" = the right-most X that is left of Y
location_filtered_candidates.sort(key=lambda p: p.get_pos())
distance_sorted = True
else:
# no reference direction: sort by closest
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
location)
steps, reldir = interpret_relative_direction(interpreter, d)
ref_loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
location_filtered_candidates = [
c for c in candidates
if euclid_dist(c.get_pos(), ref_loc) <= all_proximity
]
location_filtered_candidates.sort(
key=lambda c: euclid_dist(c.get_pos(), ref_loc))
distance_sorted = True
mems = location_filtered_candidates
if location_filtered_candidates: # could be [], if so will return []
default_selector_d = {"return_quantity": "ALL"}
# default_selector_d = {"location": {"location_type": "SPEAKER_LOOK"}}
selector_d = filters_d.get("selector", default_selector_d)
S = interpret_selector(interpreter, speaker, selector_d)
if S:
memids, _ = S(
[c.memid for c in location_filtered_candidates],
[None] * len(location_filtered_candidates),
)
mems = [interpreter.memory.get_mem_by_id(m) for m in memids]
else:
pass
# FIXME, warn/error here; mems is still the candidates
return mems
def interpret_reference_object(
interpreter,
speaker,
d,
extra_tags=[],
loose_speakerlook=False,
allow_clarification=True,
all_proximity=100,
) -> List[ReferenceObjectNode]:
"""this tries to find a ref obj memory matching the criteria from the
ref_obj_dict
args:
interpreter: root interpreter.
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
d: logical form from semantic parser
extra_tags (list of strings): tags added by parent to narrow the search
allow_clarification (bool): should a Clarification object be put on the DialogueStack
"""
filters_d = d.get("filters")
special = d.get("special_reference")
# filters_d can be empty...
assert (
filters_d is not None
) or special, "no filters or special_reference sub-dicts {}".format(d)
if special:
mem = get_special_reference_object(interpreter, speaker, special)
return [mem]
if filters_d.get("contains_coreference", "NULL") != "NULL":
mem = filters_d["contains_coreference"]
if isinstance(mem, ReferenceObjectNode):
update_attended_and_link_lf(interpreter, [mem])
return [mem]
elif mem == "resolved":
pass
else:
logging.error("bad coref_resolve -> {}".format(mem))
clarification_query = (
"SELECT MEMORY FROM Task WHERE reference_object_confirmation=#={}".
format(interpreter.memid))
_, clarification_task_mems = interpreter.memory.basic_search(
clarification_query)
# does a clarification task referencing this interpreter exist?
if not clarification_task_mems:
mems = maybe_get_text_span_mems(interpreter, speaker, d)
if mems:
update_attended_and_link_lf(interpreter, mems)
# No filter by sublocation etc if a mem matches the text_span exactly...
return mems
if any(extra_tags):
extra_clauses = []
for tag in extra_tags:
extra_clauses.append({"pred_text": "has_tag", "obj_text": tag})
if not filters_d.get("where_clause"):
filters_d["where_clause"] = {"AND": []}
if filters_d["where_clause"].get(
"OR") or filters_d["where_clause"].get("NOT"):
subclause = deepcopy(filters_d["where_clause"])
filters_d["where_clause"] = {"AND": [subclause]}
filters_d["where_clause"]["AND"].extend(extra_clauses)
# TODO Add ignore_player maybe?
# FIXME! see above. currently removing selector to get candidates, and filtering after
# instead of letting filter interpreters handle.
filters_no_select = deepcopy(filters_d)
filters_no_select.pop("selector", None)
# filters_no_select.pop("location", None)
candidate_mems = apply_memory_filters(interpreter, speaker,
filters_no_select)
if len(candidate_mems) > 0:
mems = filter_by_sublocation(
interpreter,
speaker,
candidate_mems,
d,
loose=loose_speakerlook,
all_proximity=all_proximity,
)
update_attended_and_link_lf(interpreter, mems)
return mems
elif allow_clarification:
# no candidates found; ask Clarification
confirm_candidates = apply_memory_filters(interpreter, speaker,
filters_d)
objects = object_looked_at(interpreter.memory,
confirm_candidates,
speaker=speaker)
if len(objects) == 0:
raise ErrorWithResponse(
"I don't know what you're referring to")
_, mem = objects[0]
interpreter.memory.add_triple(subj=interpreter.memid,
pred_text="provisional_refobj_memid",
obj=mem.memid)
task_egg = {
"class": ConfirmReferenceObject,
"task_data": {
"reference_object": mem
}
}
cmemid = TaskNode.create(interpreter.memory, task_egg)
interpreter.memory.add_triple(
subj=cmemid,
pred_text="reference_object_confirmation",
obj=self.memid)
raise NextDialogueStep()
else:
raise ErrorWithResponse("I don't know what you're referring to")
else:
# there is a clarification task. is it active?
task_mem = clarification_task_mems[
0] # FIXME, error if there are many?
if task_mem.prio > -2:
raise NextDialogueStep()
# clarification task finished.
query = "SELECT dialogue_task_output FROM Task WHERE uuid={}".format(
task_mem.memid)
_, r = interpreter.memory.basic_search(query)
if r and r[0] == "yes":
# TODO: learn from the tag! put it in memory!
query = "SELECT MEMORY FROM ReferenceObject WHERE << {}, reference_object_confirmation, ?>>".format(
self.memid)
_, ref_obj_mems = interpreter.memory.basic_search(query)
update_attended_and_link_lf(interpreter, ref_obj_mems)
return ref_obj_mems
else:
raise ErrorWithResponse("I don't know what you're referring to")
