def handle_destroy(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'destroy' command by either pushing a dialogue object
or pushing a Destroy task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
default_ref_d = {"filters": {"location": SPEAKERLOOK}}
ref_d = d.get("reference_object", default_ref_d)
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_destructible"]
)
if len(objs) == 0:
raise ErrorWithResponse("I don't understand what you want me to destroy.")
# don't kill mobs
if all(isinstance(obj, MobNode) for obj in objs):
raise ErrorWithResponse("I don't kill animals, sorry!")
if all(isinstance(obj, PlayerNode) for obj in objs):
raise ErrorWithResponse("I don't kill players, sorry!")
objs = [obj for obj in objs if not isinstance(obj, MobNode)]
num_destroy_tasks = 0
for obj in objs:
if hasattr(obj, "blocks"):
schematic = list(obj.blocks.items())
task_data = {"schematic": schematic, "action_dict": d}
self.append_new_task(self.task_objects["destroy"], task_data)
num_destroy_tasks += 1
logging.info("Added {} Destroy tasks to stack".format(num_destroy_tasks))
self.finished = True
return None, None
def handle_drop(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'drop' command by either pushing a dialogue object
or pushing a Drop task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
ref_d = d.get("reference_object", None)
if not ref_d:
raise ErrorWithResponse("I don't understand what you want me to drop.")
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_in_inventory"]
)
if len(objs) == 0:
raise ErrorWithResponse("I don't understand what you want me to drop.")
obj = [obj for obj in objs if isinstance(obj, ItemStackNode)][0]
item_stack = self.agent.get_item_stack(obj.eid)
idm = (item_stack.item.id, item_stack.item.meta)
task_data = {"eid": obj.eid, "idm": idm, "memid": obj.memid}
self.append_new_task(self.task_objects["drop"], task_data)
self.finished = True
return None, None
def handle_spawn(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and handles a 'spawn' command by either replying back or
pushing a Spawn task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
# FIXME! use filters appropriately, don't search by hand
spawn_triples = d.get("reference_object", {}).get("filters", {}).get("triples", [])
if not spawn_triples:
raise ErrorWithResponse("I don't understand what you want me to spawn.")
names = [t.get("obj_text") for t in spawn_triples if t.get("pred_text", "") == "has_name"]
if not any(names):
raise ErrorWithResponse("I don't understand what you want me to spawn.")
# if multiple possible has_name triples, just pick the first:
object_name = names[0]
schematic = self.memory.get_mob_schematic_by_name(object_name)
if not schematic:
raise ErrorWithResponse("I don't know how to spawn: %r." % (object_name))
object_idm = list(schematic.blocks.values())[0]
location_d = d.get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
pos, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir)
repeat_times = get_repeat_num(d)
for i in range(repeat_times):
task_data = {"object_idm": object_idm, "pos": pos, "action_dict": d}
self.append_new_task(self.task_objects["spawn"], task_data)
self.finished = True
return None, None
def handle_get(self, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'get' command by either pushing a dialogue object
or pushing a Get task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
ref_d = d.get("reference_object", None)
if not ref_d:
raise ErrorWithResponse(
"I don't understand what you want me to get.")
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_on_ground"])
if len(objs) == 0:
raise ErrorWithResponse(
"I don't understand what you want me to get.")
obj = [obj for obj in objs if isinstance(obj, ItemStackNode)][0]
item_stack = self.agent.get_item_stack(obj.eid)
idm = (item_stack.item.id, item_stack.item.meta)
task_data = {
"idm": idm,
"pos": obj.pos,
"eid": obj.eid,
"memid": obj.memid
}
return self.task_objects["get"](self.agent, task_data), None, None
def handle_modify(self, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and handles a 'modify' command by either replying back or pushing
appropriate tasks to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
default_ref_d = {"filters": {"location": SPEAKERLOOK}}
ref_d = d.get("reference_object", default_ref_d)
# only modify blockobjects...
objs = self.subinterpret["reference_objects"](
self,
speaker,
ref_d,
extra_tags=["_physical_object", "VOXEL_OBJECT"])
if len(objs) == 0:
raise ErrorWithResponse(
"I don't understand what you want me to modify.")
m_d = d.get("modify_dict")
if not m_d:
raise ErrorWithResponse(
"I think you want me to modify an object but am not sure what to do"
)
tasks = []
for obj in objs:
if m_d["modify_type"] == "THINNER" or m_d[
"modify_type"] == "THICKER":
destroy_task_data, build_task_data = handle_thicken(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "REPLACE":
destroy_task_data, build_task_data = handle_replace(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "SCALE":
destroy_task_data, build_task_data = handle_scale(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "RIGIDMOTION":
destroy_task_data, build_task_data = handle_rigidmotion(
self, speaker, m_d, obj)
elif m_d["modify_type"] == "FILL" or m_d["modify_type"] == "HOLLOW":
destroy_task_data, build_task_data = handle_fill(
self, speaker, m_d, obj)
else:
raise ErrorWithResponse(
"I think you want me to modify an object but am not sure what to do (parse error)"
)
if build_task_data:
tasks.append(self.task_objects["build"](self.agent,
build_task_data))
if destroy_task_data:
tasks.append(self.task_objects["build"](self.agent,
destroy_task_data))
return maybe_task_list_to_control_block(tasks, self.agent), None, None
def interpret_named_schematic(
interpreter, speaker, d
) -> Tuple[List[Block], Optional[str], List[Tuple[str, str]]]:
"""Return a tuple of 3 values:
- the schematic blocks, list[(xyz, idm)]
- a SchematicNode memid, or None
- a list of (pred, val) tags
warning: if multiple possibilities are given for the same tag, current
heursitic just picks one. e.g. if the lf is
"triples" : [{"pred_text": "has_colour", "obj_text": "red"},
{"pred_text": "has_colour", "obj_text": "blue"}]
will currently just pick red. Same for other properties encoded in triples
"""
# FIXME! this is not compositional, and is not using full FILTERS handlers
filters_d = d.get("filters", {})
triples = filters_d.get("triples", [])
names = get_properties_from_triples(triples, "has_name")
if not any(names):
raise ErrorWithResponse("I don't know what you want me to build.")
name = names[0]
stemmed_name = name.strip("s") # why aren't we using stemmer anymore?
shapename = SPECIAL_SHAPES_CANONICALIZE.get(name) or SPECIAL_SHAPES_CANONICALIZE.get(
stemmed_name
)
if shapename:
shape_blocks, tags = interpret_shape_schematic(
interpreter, speaker, d, shapename=shapename
)
return shape_blocks, None, tags
schematic = interpreter.memory.get_schematic_by_name(name)
if schematic is None:
schematic = interpreter.memory.get_schematic_by_name(stemmed_name)
if schematic is None:
raise ErrorWithResponse("I don't know what you want me to build.")
tags = [(p, v) for (_, p, v) in interpreter.memory.get_triples(subj=schematic.memid)]
blocks = schematic.blocks
# TODO generalize to more general block properties
# Longer term: remove and put a call to the modify model here
colours = get_properties_from_triples(triples, "has_colour")
if any(colours):
colour = colours[0]
old_idm = most_common_idm(blocks.values())
c = block_data.COLOR_BID_MAP.get(colour)
if c is not None:
new_idm = random.choice(c)
for l in blocks:
if blocks[l] == old_idm:
blocks[l] = new_idm
return list(blocks.items()), schematic.memid, tags
def get_special_reference_object(interpreter,
speaker,
S,
agent_memory=None,
eid=None):
""" subinterpret a special reference object.
args:
interpreter: the root interpreter
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
S: the special reference object logical form from
"""
# TODO/FIXME! add things to workspace memory
agent_memory = agent_memory or interpreter.agent.memory
if not eid:
eid = get_eid_from_special(agent_memory, S, speaker=speaker)
sd = special_reference_search_data(None,
speaker,
S,
entity_id=eid,
agent_memory=agent_memory)
mems = agent_memory.basic_search(sd)
if not mems:
# need a better interface for this, don't need to run full perception
# just to force speakerlook in memory
# TODO force if look is stale, not just if it doesn't exist
# this branch shouldn't occur
# interpreter.agent.perceive(force=True)
raise ErrorWithResponse(
"I think you are pointing at something but I don't know what it is"
)
return mems[0]
def __call__(self, interpreter, speaker, d):
# get these from memory, not player struct!!!!! FIXME!!!
current_pitch = interpreter.agent.get_player().look.pitch
current_yaw = interpreter.agent.get_player().look.yaw
if d.get("yaw_pitch"):
span = d["yaw_pitch"]
# for now assumed in (yaw, pitch) or yaw, pitch or yaw pitch formats
yp = span.replace("(", "").replace(")", "").split()
return {"head_yaw_pitch": (int(yp[0]), int(yp[1]))}
elif d.get("yaw"):
# for now assumed span is yaw as word or number
w = d["yaw"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (word_to_num(w), current_pitch)}
elif d.get("pitch"):
# for now assumed span is pitch as word or number
w = d["pitch"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (current_yaw, word_to_num(w))}
elif d.get("relative_yaw"):
# TODO in the task use turn angle
if "left" in d["relative_yaw"] or "right" in d["relative_yaw"]:
left = "left" in span or "leave" in span # lemmatizer :)
degrees = number_from_span(span) or 90
if degrees > 0 and left:
return {"relative_yaw": -degrees}
else:
return {"relative_yaw": degrees}
else:
try:
deg = int(d["relative_yaw"])
return {"relative_yaw": deg}
except:
pass
elif d.get("relative_pitch"):
if "down" in d["relative_pitch"] or "up" in d["relative_pitch"]:
down = "down" in d["relative_pitch"]
degrees = number_from_span(d["relative_pitch"]) or 90
if degrees > 0 and down:
return {"relative_pitch": -degrees}
else:
return {"relative_pitch": degrees}
else:
# TODO in the task make this relative!
try:
deg = int(d["relative_pitch"]["angle"])
return {"relative_pitch": deg}
except:
pass
elif d.get("location"):
mems = interpreter.subinterpret["reference_locations"](
interpreter, speaker, d["location"])
steps, reldir = interpret_relative_direction(
interpreter, d["location"])
loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
return {"head_xyz": loc}
else:
raise ErrorWithResponse("I am not sure where you want me to turn")
def handle_task_refobj_string(self, task, refobj_attr):
if refobj_attr == "name":
for pred, val in task.task.target:
if pred == "has_name":
return "I am going to the " + prepend_a_an(val), None
elif refobj_attr == "location":
target = tuple(task.task.target)
return "I am going to {}".format(target), None
else:
raise ErrorWithResponse(
"trying get attribute {} from action".format(refobj_attr))
def __call__(self, interpreter, speaker, d) -> POINT_AT_TARGET:
if d.get("location") is None:
# TODO other facings
raise ErrorWithResponse("I am not sure where you want me to point")
# TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
d["location"])
steps, reldir = interpret_relative_direction(interpreter.agent, d)
loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker, mems,
steps, reldir)
return self.point_to_region(loc)
def handle_task_refobj_string(self, task, refobj_attr):
if refobj_attr == "name":
assert isinstance(task.task, Build), task.task
for pred, val in task.task.schematic_tags:
if pred == "has_name":
return "I am building " + prepend_a_an(val), None
return "I am building something that is {}".format(val), None
elif refobj_attr == "location":
assert task.action_name == "Move", task.action_name
target = tuple(task.task.target)
return "I am going to {}".format(target), None
else:
raise ErrorWithResponse("trying get attribute {} from action".format(refobj_attr))
return None, None
def handle_get(self, speaker, d) -> Tuple[Optional[str], Any]:
default_ref_d = {"filters": {"location": AGENTPOS}}
ref_d = d.get("reference_object", default_ref_d)
objs = self.subinterpret["reference_objects"](
self, speaker, ref_d, extra_tags=["_physical_object"])
if len(objs) == 0:
raise ErrorWithResponse("I don't know what you want me to get.")
if all(isinstance(obj, PlayerNode) for obj in objs):
raise ErrorWithResponse("I can't get a person, sorry!")
objs = [obj for obj in objs if not isinstance(obj, PlayerNode)]
if d.get("receiver") is None:
receiver_d = None
else:
receiver_d = d.get("receiver").get("reference_object")
receiver = None
if receiver_d:
receiver = self.subinterpret["reference_objects"](self, speaker,
receiver_d)
if len(receiver) == 0:
raise ErrorWithResponse(
"I don't know where you want me to take it")
receiver = receiver[0].memid
num_get_tasks = 0
for obj in objs:
task_data = {
"get_target": obj.memid,
"give_target": receiver,
"action_dict": d
}
self.append_new_task(self.task_objects["get"], task_data)
num_get_tasks += 1
# logging.info("Added {} Get tasks to stack".format(num_get_tasks))
self.finished = True
return None, None
def step(self) -> Tuple[Optional[str], Any]:
assert self.action_dict["dialogue_type"] == "HUMAN_GIVE_COMMAND"
try:
actions = []
if "action" in self.action_dict:
actions.append(self.action_dict["action"])
elif "action_sequence" in self.action_dict:
actions = self.action_dict["action_sequence"]
if len(actions) == 0:
# The action dict is in an unexpected state
raise ErrorWithResponse(
"I thought you wanted me to do something, but now I don't know what"
)
tasks_to_push = []
for action_def in actions:
action_type = action_def["action_type"]
r = self.action_handlers[action_type](self.speaker, action_def)
if len(r) == 3:
task, response, dialogue_data = r
else:
# FIXME don't use this branch, uniformize the signatures
task = None
response, dialogue_data = r
if task:
tasks_to_push.append(task)
task_mem = None
if tasks_to_push:
T = maybe_task_list_to_control_block(tasks_to_push, self.agent)
task_mem = TaskNode(self.agent.memory, tasks_to_push[0].memid)
if task_mem:
chat = self.agent.memory.get_most_recent_incoming_chat()
TripleNode.create(
self.agent.memory,
subj=chat.memid,
pred_text="chat_effect_",
obj=task_mem.memid,
)
self.finished = True
return response, dialogue_data
except NextDialogueStep:
return None, None
except ErrorWithResponse as err:
self.finished = True
return err.chat, None
def new_tasks():
# TODO if we do this better will be able to handle "stay between the x"
default_loc = getattr(self, "default_loc", SPEAKERLOOK)
location_d = d.get("location", default_loc)
if self.loop_data and hasattr(self.loop_data, "get_pos"):
mems = [self.loop_data]
else:
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
# FIXME this should go in the ref_location subinterpret:
steps, reldir = interpret_relative_direction(self, location_d)
pos, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir)
# TODO: can this actually happen?
if pos is None:
raise ErrorWithResponse("I don't understand where you want me to move.")
pos = self.post_process_loc(pos, self)
task_data = {"target": pos, "action_dict": d}
task = Move(self.agent, task_data)
return [task]
def handle_triple(self) -> Tuple[Optional[str], Any]:
"""Writes a triple of type : (subject, predicate_text, object_text)
to the memory and returns a confirmation.
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
ref_obj_d = {"filters": self.action_dict["filters"]}
r = self.subinterpret["reference_objects"](
self, self.speaker_name, ref_obj_d, extra_tags=["_physical_object"]
)
if len(r) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
mem = r[0]
name = "it"
triples = self.memory.get_triples(subj=mem.memid, pred_text="has_tag")
if len(triples) > 0:
name = triples[0][2].strip("_")
schematic_memid = (
self.memory.convert_block_object_to_schematic(mem.memid).memid
if isinstance(mem, VoxelObjectNode)
else None
)
for t in self.action_dict["upsert"]["memory_data"].get("triples", []):
if t.get("pred_text") and t.get("obj_text"):
logging.info("Tagging {} {} {}".format(mem.memid, t["pred_text"], t["obj_text"]))
self.memory.add_triple(
subj=mem.memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
if schematic_memid:
self.memory.add_triple(
subj=schematic_memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
point_at_target = mem.get_point_at_target()
self.agent.send_chat(
"OK I'm tagging this %r as %r %r " % (name, t["pred_text"], t["obj_text"])
)
self.agent.point_at(list(point_at_target))
return "Done!", None
def handle_thicken(interpreter, speaker, modify_dict, obj):
old_blocks = list(obj.blocks.items())
bounds = obj.get_bounds()
mx, my, mz = (bounds[0], bounds[2], bounds[4])
origin = [mx, my, mz]
if modify_dict.get("modify_type") == "THICKER":
num_blocks = modify_dict.get("num_blocks", 1)
new_blocks = thicker(old_blocks, delta=num_blocks)
else:
raise ErrorWithResponse("I don't know how thin out blocks yet")
destroy_task_data = {"schematic": old_blocks}
# FIXME deal with tags!!!
build_task_data = {
"blocks_list": new_blocks,
"origin": origin,
# "schematic_tags": tags,
}
return destroy_task_data, build_task_data
def __call__(self, interpreter, speaker, filters_d, get_all=False):
"""
This is a subinterpreter to handle FILTERS dictionaries
Args:
interpreter: root interpreter.
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
filters_d: FILTERS logical form from semantic parser
get_all (bool): if True, output attributes are set with get_all=True
Outputs a (chain) of MemoryFilter objects
"""
val_map = get_val_map(interpreter, speaker, filters_d, get_all=get_all)
# NB (kavyasrinet) output can be string and have value "memory" too here
# is this a specific memory?
# ... then return
specific_mem = maybe_handle_specific_mem(interpreter, speaker,
filters_d, val_map)
if specific_mem is not None:
return specific_mem
memtype = filters_d.get("memory_type", "REFERENCE_OBJECT")
# FIXME/TODO: these share lots of code, refactor
if memtype == "REFERENCE_OBJECT":
F = interpret_ref_obj_filter(interpreter, speaker, filters_d)
elif memtype == "TASKS":
F = interpret_task_filter(interpreter, speaker, filters_d)
else:
memtype_key = memtype.lower() + "_filters"
try:
F = interpreter.subinterpret[memtype_key](interpreter, speaker,
filters_d)
except:
raise ErrorWithResponse(
"failed at interpreting filters of type {}".format(
memtype))
F = maybe_apply_selector(interpreter, speaker, filters_d, F)
return maybe_append_left(F, to_append=val_map)
def do_answer(self, mems: Sequence[Any],
vals: Sequence[Any]) -> Tuple[Optional[str], Any]:
"""This function uses the action dictionary and memory state to return an answer.
Args:
mems: Sequence of memories
vals: Sequence of values
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
self.finished = True # noqa
output_type = self.action_dict_orig["filters"].get("output")
try:
if type(output_type) is str and output_type.lower() == "count":
# FIXME will multiple count if getting tags
if not any(vals):
return "none", None
return str(vals[0]), None
elif type(output_type) is dict and output_type.get("attribute"):
attrib = output_type["attribute"]
if type(attrib) is str and attrib.lower() == "location":
# add a Point task if attribute is a location
target = self.subinterpret["point_target"].point_to_region(
vals[0])
t = self.task_objects["point"](self.agent, {
"target": target
})
self.append_new_task(t)
return str(vals[0]), None
elif type(output_type) is str and output_type.lower() == "memory":
return self.handle_exists(mems)
else:
raise ValueError("Bad answer_type={}".format(output_type))
except IndexError: # index error indicates no answer available
logging.error("No answer available from do_answer")
raise ErrorWithResponse("I don't understand what you're asking")
except Exception as e:
logging.exception(e)
def handle_undo(self, speaker, d) -> Tuple[Optional[str], Any]:
Undo = self.task_objects["undo"]
task_name = d.get("undo_action")
if task_name:
task_name = task_name.split("_")[0].strip()
old_task = self.memory.get_last_finished_root_task(task_name)
if old_task is None:
raise ErrorWithResponse("Nothing to be undone ...")
undo_tasks = [Undo(self.agent, {"memid": old_task.memid})]
# undo_tasks = [
# tasks.Undo(self.agent, {"memid": task.memid})
# for task in old_task.all_descendent_tasks(include_root=True)
# ]
undo_command = old_task.get_chat().chat_text
logging.debug("Pushing ConfirmTask tasks={}".format(undo_tasks))
self.dialogue_stack.append_new(
ConfirmTask,
'Do you want me to undo the command: "{}" ?'.format(undo_command),
undo_tasks,
)
self.finished = True
return None, None
def step(self) -> Tuple[Optional[str], Any]:
assert self.action_dict["dialogue_type"] == "HUMAN_GIVE_COMMAND"
try:
actions = []
if "action" in self.action_dict:
actions.append(self.action_dict["action"])
elif "action_sequence" in self.action_dict:
actions = self.action_dict["action_sequence"]
actions.reverse()
if len(actions) == 0:
# The action dict is in an unexpected state
raise ErrorWithResponse(
"I thought you wanted me to do something, but now I don't know what"
)
for action_def in actions:
action_type = action_def["action_type"]
response = self.action_handlers[action_type](self.speaker, action_def)
return response
except NextDialogueStep:
return None, None
except ErrorWithResponse as err:
self.finished = True
return err.chat, None
def special_reference_search_data(interpreter,
speaker,
S,
entity_id=None,
agent_memory=None):
""" make a search dictionary for a BasicMemorySearcher to return the special ReferenceObject"""
# TODO/FIXME! add things to workspace memory
agent_memory = agent_memory or interpreter.agent.memory
if type(S) is dict:
coord_span = S["coordinates_span"]
loc = cast(
XYZ,
tuple(int(float(w)) for w in re.findall("[-0-9.]+", coord_span)))
if len(loc) != 3:
logging.error("Bad coordinates: {}".format(coord_span))
raise ErrorWithResponse(
"I don't understand what location you're referring to")
memid = agent_memory.add_location(
(int(loc[0]), int(loc[1]), int(loc[2])))
mem = agent_memory.get_location_by_id(memid)
f = {"special": {"DUMMY": mem}}
else:
f = {"special": {S: entity_id}}
return f
def new_tasks():
repeat = get_repeat_num(d)
tasks_to_do = []
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE"
or rd == "ANTICLOCKWISE"):
ref_obj = None
location_reference_object = location_d.get(
"reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object)
if len(objmems) == 0:
raise ErrorWithResponse(
"I don't understand where you want me to go.")
ref_obj = objmems[0]
for i in range(repeat):
refmove = dance.RefObjMovement(
self.agent,
ref_object=ref_obj,
relative_direction=location_d[
"relative_direction"],
)
t = self.task_objects["dance"](self.agent, {
"movement": refmove
})
tasks_to_do.append(t)
return list(reversed(tasks_to_do))
dance_type = d.get("dance_type", {"dance_type_name": "dance"})
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker,
dance_type["point"])
for i in range(repeat):
t = self.task_objects["point"](self.agent, {
"target": target
})
tasks_to_do.append(t)
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn")
if lt:
f = self.subinterpret["facing"](self,
speaker,
lt,
head_or_body="head")
T = self.task_objects["look"]
else:
bt = dance_type.get("body_turn")
f = self.subinterpret["facing"](self,
speaker,
bt,
head_or_body="body")
T = self.task_objects["turn"]
for i in range(repeat):
tasks_to_do.append(T(self.agent, f))
elif dance_type["dance_type_name"] == "wave":
new_task = self.task_objects["dance"](self.agent, {
"movement_type": "wave"
})
tasks_to_do.append(new_task)
else:
# FIXME ! merge dances, refactor. search by name in sql
raise ErrorWithResponse(
"I don't know how to do that dance yet!")
return list(reversed(tasks_to_do))
def handle_scale(interpreter, speaker, modify_dict, obj):
old_blocks = list(obj.blocks.items())
bounds = obj.get_bounds()
mx, my, mz = (bounds[0], bounds[2], bounds[4])
csf = modify_dict.get("categorical_scale_factor")
origin = [mx, my, mz]
if not csf:
if modify_dict.get("numerical_scale_factor"):
raise ErrorWithResponse("I don't know how to handle numerical_scale_factor yet")
else:
raise ErrorWithResponse(
"I think I am supposed to scale something but I don't know which dimensions to scale"
)
destroy_task_data = {"schematic": old_blocks}
if csf == "WIDER":
if bounds[1] - bounds[0] > bounds[5] - bounds[4]:
lam = (2.0, 1.0, 1.0)
else:
lam = (1.0, 1.0, 2.0)
new_blocks = maybe_convert_to_list(scale(old_blocks, lam))
destroy_task_data = None
elif csf == "NARROWER":
if bounds[1] - bounds[0] > bounds[5] - bounds[4]:
lam = (0.5, 1.0, 1.0)
else:
lam = (1.0, 1.0, 0.5)
new_blocks = maybe_convert_to_list(shrink_sample(old_blocks, lam))
elif csf == "TALLER":
lam = (1.0, 2.0, 1.0)
new_blocks = maybe_convert_to_list(scale(old_blocks, lam))
destroy_task_data = None
elif csf == "SHORTER":
lam = (1.0, 0.5, 1.0)
new_blocks = maybe_convert_to_list(shrink_sample(old_blocks, lam))
elif csf == "SKINNIER":
lam = (0.5, 1.0, 0.5)
new_blocks = maybe_convert_to_list(shrink_sample(old_blocks, lam))
elif csf == "FATTER":
lam = (2.0, 1.0, 2.0)
new_blocks = maybe_convert_to_list(scale(old_blocks, lam))
destroy_task_data = None
elif csf == "BIGGER":
lam = (2.0, 2.0, 2.0)
destroy_task_data = None
new_blocks = maybe_convert_to_list(scale(old_blocks, lam))
elif csf == "SMALLER":
lam = (0.5, 0.5, 0.5)
new_blocks = maybe_convert_to_list(shrink_sample(old_blocks, lam))
M_new = np.max([l for l, idm in new_blocks], axis=0)
m_new = np.min([l for l, idm in new_blocks], axis=0)
new_extent = (M_new[0] - m_new[0], M_new[1] - m_new[1], M_new[2] - m_new[2])
old_extent = (bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4])
origin = (
mx - (new_extent[0] - old_extent[0]) // 2,
my,
mz - (new_extent[2] - old_extent[2]) // 2,
)
# FIXME deal with tags!!!
build_task_data = {
"blocks_list": new_blocks,
"origin": origin,
# "schematic_tags": tags,
}
return destroy_task_data, build_task_data
def new_tasks():
repeat = get_repeat_num(d)
tasks_to_do = []
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE"
or rd == "ANTICLOCKWISE"):
ref_obj = None
location_reference_object = location_d.get(
"reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object)
if len(objmems) == 0:
raise ErrorWithResponse(
"I don't understand where you want me to go.")
ref_obj = objmems[0]
for i in range(repeat):
refmove = dance.RefObjMovement(
self.agent,
ref_object=ref_obj,
relative_direction=location_d[
"relative_direction"],
)
t = self.task_objects["dance"](self.agent, {
"movement": refmove
})
tasks_to_do.append(t)
return maybe_task_list_to_control_block(
tasks_to_do, self.agent)
dance_type = d.get("dance_type", {})
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker,
dance_type["point"])
for i in range(repeat):
t = self.task_objects["point"](self.agent, {
"target": target
})
tasks_to_do.append(t)
# MC bot does not control body turn separate from head
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn") or dance_type.get("body_turn")
f = self.subinterpret["facing"](self, speaker, lt)
for i in range(repeat):
t = self.task_objects["dancemove"](self.agent, f)
tasks_to_do.append(t)
else:
if location_d is None:
dance_location = None
else:
mems = self.subinterpret["reference_locations"](self,
speaker,
location_d)
steps, reldir = interpret_relative_direction(
self, location_d)
dance_location, _ = self.subinterpret["specify_locations"](
self, speaker, mems, steps, reldir)
filters_d = dance_type.get("filters", {})
filters_d["memory_type"] = "DANCES"
F = self.subinterpret["filters"](self, speaker,
dance_type.get("filters", {}))
dance_memids, _ = F()
# TODO correct selector in filters
dance_memid = random.choice(dance_memids)
dance_mem = self.memory.get_mem_by_id(dance_memid)
for i in range(repeat):
dance_obj = dance.Movement(agent=self.agent,
move_fn=dance_mem.dance_fn,
dance_location=dance_location)
t = self.task_objects["dance"](self.agent, {
"movement": dance_obj
})
tasks_to_do.append(t)
return maybe_task_list_to_control_block(tasks_to_do, self.agent)
def filter_by_sublocation(
interpreter,
speaker,
candidates: List[Tuple[XYZ, T]],
d: Dict,
limit=1,
all_proximity=10,
loose=False,
) -> List[Tuple[XYZ, T]]:
"""Select from a list of candidate (xyz, object) tuples given a sublocation
If limit == 'ALL', return all matching candidates
Returns a list of (xyz, mem) tuples
"""
F = d.get("filters")
assert F is not None, "no filters: {}".format(d)
default_loc = getattr(interpreter, "default_loc", SPEAKERLOOK)
location = F.get("location", default_loc)
# if limit == 1:
# limit = get_repeat_num(d)
reldir = location.get("relative_direction")
if reldir:
if reldir == "INSIDE":
# FIXME formalize this better, make extensible
if location.get("reference_object"):
# should probably return from interpret_reference_location...
ref_mems = interpret_reference_object(
interpreter, speaker, location["reference_object"])
for l, candidate_mem in candidates:
I = interpreter.agent.on_demand_perception.get[
"check_inside"]
if I:
if I([candidate_mem, ref_mems[0]]):
return [(l, candidate_mem)]
else:
raise ErrorWithResponse(
"I don't know how to check inside")
raise ErrorWithResponse("I can't find something inside that")
elif reldir == "AWAY":
raise ErrorWithResponse("I don't know which object you mean")
elif reldir == "NEAR":
pass # fall back to no reference direction
elif reldir == "BETWEEN":
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
location)
steps, reldir = interpret_relative_direction(interpreter, d)
ref_loc, _ = interpreter.subinterpret["specify_locations"](
interpreter, speaker, mems, steps, reldir)
candidates.sort(key=lambda c: euclid_dist(c[0], ref_loc))
return candidates[:limit]
else:
# FIXME need some tests here
# reference object location, i.e. the "X" in "left of X"
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
location)
# FIXME!!! handle frame better, might want agent's frame instead
eid = interpreter.agent.memory.get_player_by_name(speaker).eid
self_mem = interpreter.agent.memory.get_mem_by_id(
interpreter.agent.memory.self_memid)
L = LinearExtentAttribute(interpreter.agent, {
"frame": eid,
"relative_direction": reldir
},
mem=self_mem)
proj = L([c[1] for c in candidates])
# filter by relative dir, e.g. "left of Y"
proj_cands = [(p, c) for (p, c) in zip(proj, candidates) if p > 0]
# "the X left of Y" = the right-most X that is left of Y
if limit == "ALL":
limit = len(proj_cands)
return [c for (_, c) in sorted(proj_cands, key=lambda p: p[0])
][:limit]
else:
# no reference direction: choose the closest
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
location)
steps, reldir = interpret_relative_direction(interpreter, d)
ref_loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
if limit == "ALL":
return list(
filter(lambda c: euclid_dist(c[0], ref_loc) <= all_proximity,
candidates))
else:
candidates.sort(key=lambda c: euclid_dist(c[0], ref_loc))
return candidates[:limit]
return [] # this fixes flake but seems awful?
def safe_call(f, *args):
try:
return f(*args)
except Pyro4.errors.ConnectionClosedError as e:
msg = "{} - {}".format(f._RemoteMethod__name, e)
raise ErrorWithResponse(msg)
def new_tasks():
repeat = get_repeat_num(d)
tasks_to_do = []
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (
rd == "AROUND" or rd == "CLOCKWISE" or rd == "ANTICLOCKWISE"
):
ref_obj = None
location_reference_object = location_d.get("reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object
)
if len(objmems) == 0:
raise ErrorWithResponse("I don't understand where you want me to go.")
ref_obj = objmems[0]
for i in range(repeat):
refmove = dance.RefObjMovement(
self.agent,
ref_object=ref_obj,
relative_direction=location_d["relative_direction"],
)
t = self.task_objects["dance"](self.agent, {"movement": refmove})
tasks_to_do.append(t)
return list(reversed(tasks_to_do))
dance_type = d.get("dance_type", {"dance_type_name": "dance"})
# FIXME holdover from old dict format
if type(dance_type) is str:
dance_type = dance_type = {"dance_type_name": "dance"}
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker, dance_type["point"])
for i in range(repeat):
t = self.task_objects["point"](self.agent, {"target": target})
tasks_to_do.append(t)
# MC bot does not control body turn separate from head
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn") or dance_type.get("body_turn")
f = self.subinterpret["facing"](self, speaker, lt)
for i in range(repeat):
t = self.task_objects["dancemove"](self.agent, f)
tasks_to_do.append(t)
else:
if location_d is None:
dance_location = None
else:
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
dance_location, _ = self.subinterpret["specify_locations"](
self, speaker, mems, steps, reldir
)
# TODO use name!
if dance_type.get("dance_type_span") is not None:
dance_name = dance_type["dance_type_span"]
if dance_name == "dance":
dance_name = "ornamental_dance"
dance_memids = self.memory._db_read(
"SELECT DISTINCT(Dances.uuid) FROM Dances INNER JOIN Triples on Dances.uuid=Triples.subj WHERE Triples.obj_text=?",
dance_name,
)
else:
dance_memids = self.memory._db_read(
"SELECT DISTINCT(Dances.uuid) FROM Dances INNER JOIN Triples on Dances.uuid=Triples.subj WHERE Triples.obj_text=?",
"ornamental_dance",
)
dance_memid = random.choice(dance_memids)[0]
dance_fn = self.memory.dances[dance_memid]
for i in range(repeat):
dance_obj = dance.Movement(
agent=self.agent, move_fn=dance_fn, dance_location=dance_location
)
t = self.task_objects["dance"](self.agent, {"movement": dance_obj})
tasks_to_do.append(t)
return list(reversed(tasks_to_do))
def __call__(self, interpreter, speaker, d, head_or_body="head"):
current_pitch = interpreter.agent.pitch
if head_or_body == "head":
current_yaw = interpreter.agent.pan
else:
current_yaw = interpreter.agent.base_yaw
if d.get("yaw_pitch"):
# make everything relative:
span = d["yaw_pitch"]
# for now assumed in (yaw, pitch) or yaw, pitch or yaw pitch formats
yp = span.replace("(", "").replace(")", "").split()
# negated in look_at in locobot_mover
rel_yaw = current_yaw - float(yp[0])
rel_pitch = current_pitch - float(yp[1])
return {"yaw": rel_yaw, "pitch": rel_pitch}
elif d.get("yaw"):
# make everything relative:
# for now assumed span is yaw as word or number
w = float(word_to_num(d["yaw"].strip(" degrees").strip(" degree")))
return {"yaw": current_yaw - w}
elif d.get("pitch"):
# make everything relative:
# for now assumed span is pitch as word or number
w = float(
word_to_num(d["pitch"].strip(" degrees").strip(" degree")))
return {"yaw": current_pitch - w}
elif d.get("relative_yaw"):
if d["relative_yaw"].get("angle"):
return {"yaw": float(d["relative_yaw"]["angle"])}
elif d["relative_yaw"].get("yaw_span"):
span = d["relative_yaw"].get("yaw_span")
left = "left" in span or "leave" in span # lemmatizer :)
degrees = number_from_span(span) or 90
if degrees > 0 and left:
print(degrees)
return {"yaw": degrees}
else:
print(-degrees)
return {"yaw": -degrees}
else:
pass
elif d.get("relative_pitch"):
if d["relative_pitch"].get("angle"):
# TODO in the task make this relative!
return {"pitch": int(d["relative_pitch"]["angle"])}
elif d["relative_pitch"].get("pitch_span"):
span = d["relative_pitch"].get("pitch_span")
down = "down" in span
degrees = number_from_span(span) or 90
if degrees > 0 and down:
return {"pitch": -degrees}
else:
return {"pitch": degrees}
else:
pass
elif d.get("location"):
loc_mems = interpreter.subinterpret["reference_locations"](
interpreter, speaker, d["location"])
loc = loc_mems[0].get_pos()
return {"target": loc}
else:
raise ErrorWithResponse("I am not sure where you want me to turn")
def handle_build(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'build' command by either pushing a dialogue object
or pushing a Build task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
# Get the segment to build
if "reference_object" in d:
# handle copy
repeat = get_repeat_num(d)
objs = self.subinterpret["reference_objects"](
self,
speaker,
d["reference_object"],
limit=repeat,
extra_tags=["_voxel_object"],
loose_speakerlook=True,
)
if len(objs) == 0:
raise ErrorWithResponse("I don't understand what you want me to build")
tagss = [
[(p, v) for (_, p, v) in self.memory.get_triples(subj=obj.memid)] for obj in objs
]
interprets = [
[list(obj.blocks.items()), obj.memid, tags] for (obj, tags) in zip(objs, tagss)
]
else: # a schematic
if d.get("repeat") is not None:
repeat_dict = d
else:
repeat_dict = None
interprets = interpret_schematic(
self, speaker, d.get("schematic", {}), repeat_dict=repeat_dict
)
# Get the locations to build
location_d = d.get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
origin, offsets = self.subinterpret["specify_locations"](
self,
speaker,
mems,
steps,
reldir,
repeat_dir=get_repeat_dir(location_d),
objects=interprets,
enable_geoscorer=True,
)
interprets_with_offsets = [
(blocks, mem, tags, off) for (blocks, mem, tags), off in zip(interprets, offsets)
]
tasks_todo = []
for schematic, schematic_memid, tags, offset in interprets_with_offsets:
og = np.array(origin) + offset
task_data = {
"blocks_list": schematic,
"origin": og,
"schematic_memid": schematic_memid,
"schematic_tags": tags,
"action_dict": d,
}
tasks_todo.append(task_data)
for task_data in reversed(tasks_todo):
self.append_new_task(self.task_objects["build"], task_data)
logging.info("Added {} Build tasks to stack".format(len(tasks_todo)))
self.finished = True
return None, None
def interpret_reference_object(
interpreter,
speaker,
d,
extra_tags=[],
limit=1,
loose_speakerlook=False,
allow_clarification=True,
) -> List[ReferenceObjectNode]:
"""this tries to find a ref obj memory matching the criteria from the
ref_obj_dict
args:
interpreter: root interpreter.
speaker (str): The name of the player/human/agent who uttered
the chat resulting in this interpreter
d: logical form from semantic parser
extra_tags (list of strings): tags added by parent to narrow the search
limit (natural number): maximum number of reference objects to return
allow_clarification (bool): should a Clarification object be put on the DialogueStack
"""
F = d.get("filters")
special = d.get("special_reference")
# F can be empty...
assert (
F is not None
) or special, "no filters or special_reference sub-dicts {}".format(d)
if special:
mem = get_special_reference_object(interpreter, speaker, special)
return [mem]
if F.get("contains_coreference", "NULL") != "NULL":
mem = F["contains_coreference"]
if isinstance(mem, ReferenceObjectNode):
return [mem]
elif mem == "resolved":
pass
else:
logging.error("bad coref_resolve -> {}".format(mem))
if len(interpreter.progeny_data) == 0:
if any(extra_tags):
if not F.get("triples"):
F["triples"] = []
for tag in extra_tags:
F["triples"].append({"pred_text": "has_tag", "obj_text": tag})
# TODO Add ignore_player maybe?
candidate_mems = apply_memory_filters(interpreter, speaker, F)
if len(candidate_mems) > 0:
# FIXME?
candidates = [(c.get_pos(), c) for c in candidate_mems]
r = filter_by_sublocation(interpreter,
speaker,
candidates,
d,
limit=limit,
loose=loose_speakerlook)
return [mem for _, mem in r]
elif allow_clarification:
# no candidates found; ask Clarification
# TODO: move ttad call to dialogue manager and remove this logic
interpreter.action_dict_frozen = True
confirm_candidates = apply_memory_filters(interpreter, speaker, F)
objects = object_looked_at(interpreter.agent,
confirm_candidates,
speaker=speaker)
if len(objects) == 0:
raise ErrorWithResponse(
"I don't know what you're referring to")
_, mem = objects[0]
interpreter.provisional["object_mem"] = mem
interpreter.provisional["F"] = F
interpreter.dialogue_stack.append_new(ConfirmReferenceObject, mem)
raise NextDialogueStep()
else:
raise ErrorWithResponse("I don't know what you're referring to")
else:
# clarification answered
r = interpreter.progeny_data[-1].get("response")
if r == "yes":
# TODO: learn from the tag! put it in memory!
return [interpreter.provisional.get("object_mem")] * limit
else:
raise ErrorWithResponse("I don't know what you're referring to")