def step(self, agent) -> Tuple[Optional[str], Any]:
"""Take immediate actions based on action dictionary and
mark the dialogueObject as finished.
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
try:
# FIXME this will fail at clarifications
self.finished = True
memory_type = self.logical_form["upsert"]["memory_data"]["memory_type"]
if memory_type == "REWARD":
self.handle_reward(agent)
elif memory_type == "SET":
self.handle_set(agent)
elif memory_type == "TRIPLE":
self.handle_triple(agent)
else:
logging.debug(
"unknown memory type {} encountered in PutMemory handler".format(
self.logical_form
)
)
except ErrorWithResponse as err:
self.finished = True
Say(agent, task_data={"response_options": err.chat})
return
def do_answer(self, agent, mems: Sequence[Any],
vals: Sequence[Any]) -> Tuple[Optional[str], Any]:
"""This function uses the action dictionary and memory state to return an answer.
Args:
mems: Sequence of memories
vals: Sequence of values
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
self.finished = True # noqa
output_type = self.output_type
try:
if type(output_type) is str and output_type.lower() == "count":
# FIXME will multiple count if getting tags
if not any(vals):
Say(agent, task_data={"response_options": "none"})
Say(agent, task_data={"response_options": str(vals[0])})
elif type(output_type) is dict and output_type.get("attribute"):
attrib = output_type["attribute"]
if type(attrib) is str and attrib.lower() == "location":
# add a Point task if attribute is a location
if self.subinterpret.get(
"point_target") and self.task_objects.get("point"):
target = self.subinterpret[
"point_target"].point_to_region(vals[0])
# FIXME agent : This is the only place in file using the agent from the .step()
t = self.task_objects["point"](agent, {
"target": target
})
# FIXME? higher pri, make sure this runs now...?
TaskNode(self.memory, t.memid)
Say(agent, task_data={"response_options": str(vals)})
elif type(output_type) is str and output_type.lower() == "memory":
self.handle_exists(mems)
else:
raise ValueError("Bad answer_type={}".format(output_type))
except IndexError: # index error indicates no answer available
logging.error("No answer available from do_answer")
raise ErrorWithResponse("I don't understand what you're asking")
except Exception as e:
logging.exception(e)
def handle_action(self, agent) -> Tuple[Optional[str], Any]:
"""This function handles questions about the attributes and status of
the current action.
"""
# no clarifications etc? FIXME:
self.finished = True # noqa
f = self.logical_form["filters"]
F = self.subinterpret["filters"](self, self.speaker, f, get_all=True)
mems, vals = F()
Say(agent, task_data={"response_options": str(vals)})
def handle_exists(self,
mems: Sequence[MemoryNode]) -> Tuple[Optional[str], Any]:
"""Check if a memory exists.
Args:
mems: Sequence of memories
"""
# we check the clarification because if it exists, there was a confirmation,
# and the interpret reference object failed to find the object
# so it does not have the proper tag. this is an unused opportunity to learn...
# also note if the answer is going to be no, bot will always ask. maybe should fix this.
clarification_query = (
"SELECT MEMORY FROM Task WHERE reference_object_confirmation=#={}".
format(self.memid))
_, clarification_task_mems = self.memory.basic_search(
clarification_query)
if len(mems) > 0 and len(clarification_task_mems) == 0:
Say(agent, task_data={"response_options": "yes"})
else:
Say(agent, task_data={"response_options": "no"})
def handle_triple(self, agent) -> Tuple[Optional[str], Any]:
"""Writes a triple of type : (subject, predicate_text, object_text)
to the memory and returns a confirmation.
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
ref_obj_d = {"filters": self.logical_form["filters"]}
r = self.subinterpret["reference_objects"](
self, self.speaker, ref_obj_d, extra_tags=["_physical_object"]
)
if len(r) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
mem = r[0]
name = "it"
triples = self.memory.get_triples(subj=mem.memid, pred_text="has_tag")
if len(triples) > 0:
name = triples[0][2].strip("_")
schematic_memid = (
self.memory.convert_block_object_to_schematic(mem.memid).memid
if isinstance(mem, VoxelObjectNode) and len(mem.blocks) > 0
else None
)
for t in self.logical_form["upsert"]["memory_data"].get("triples", []):
if t.get("pred_text") and t.get("obj_text"):
logging.debug("Tagging {} {} {}".format(mem.memid, t["pred_text"], t["obj_text"]))
self.memory.add_triple(
subj=mem.memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
if schematic_memid:
self.memory.add_triple(
subj=schematic_memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
point_at_target = mem.get_point_at_target()
# FIXME agent : This is the only place in file using the agent from the .step()
task = self.task_objects["point"](agent, {"target": point_at_target})
# FIXME? higher pri, make sure this runs now...?
TaskNode(self.memory, task.memid)
r = "OK I'm tagging this %r as %r %r " % (name, t["pred_text"], t["obj_text"])
Say(agent, task_data={"response_options": r})
return
def build_random_shape(agent,
shape_util_dict,
rand_range=(10, 0, 10),
no_chat=False):
"""Pick a random shape from shapes.py and build that"""
target_loc = agent.pos
for i in range(3):
target_loc[i] += np.random.randint(-rand_range[i], rand_range[i] + 1)
shape = random.choice(shape_util_dict["shape_names"])
opts = shape_util_dict["shape_option_fn_map"][shape]()
opts["bid"] = shape_util_dict["bid"]
schematic = shape_util_dict["shape_fns"][shape](**opts)
relations = [
{
"pred": "has_name",
"obj": shape.lower()
},
{
"pred": "has_tag",
"obj": shape.lower()
},
]
task_data = {
"blocks_list": schematic,
"origin": target_loc,
"verbose": False,
"schematic_tags": relations,
"default_behavior":
"build_random_shape", # must == function name. Hacky and I hate it.
}
logging.debug("Default behavior: building {}".format(shape))
TaskNode(agent.memory, tasks.Build(agent, task_data).memid)
if not no_chat:
shape_name = prepend_a_an(shape.lower())
# FIXME agent , also don't push directly to stack, ask the manager?
Say(
agent,
task_data={
"response_options":
"I am building {} while you decide what you want me to do!".
format(shape_name)
},
)
return schematic
def handle_set(self, agent) -> Tuple[Optional[str], Any]:
"""creates a set of memories
Returns:
output_chat: An optional string for when the agent wants to send a chat
step_data: Any other data that this step would like to send to the task
"""
ref_obj_d = {"filters": self.logical_form["filters"]}
ref_objs = self.subinterpret["reference_objects"](
self, self.speaker, ref_obj_d, extra_tags=["_physical_object"]
)
if len(ref_objs) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
triples_d = self.logical_form["upsert"]["memory_data"].get("triples")
if len(triples_d) == 1 and triples_d[0]["pred_text"] == "has_name":
# the set has a name; check to see if one with that name exists,
# if so add to it, else create one with that name
name = triples_d[0]["obj_text"]
set_memids, _ = self.memory.basic_search(
"SELECT MEMORY FROM Set WHERE (has_name={} OR name={})".format(name, name)
)
if not set_memids:
# make a new set, and name it
set_memid = SetNode.create(self.memory)
self.memory.add_triple(subj=set_memid, pred_text="has_name", obj_text=name)
else:
# FIXME, which one
set_memid = set_memids[0]
else:
# an anonymous set, assuming its new, and defined to hold the triple(s)
set_memid = SetNode.create(self.memory)
for t in triples_d:
self.memory.add_triple(
subj=set_memid, pred_text=t["pred_text"], obj_text=t["obj_text"]
)
for r in ref_objs:
self.memory.add_triple(subj=r.memid, pred_text="member_of", obj=set_memid)
# FIXME point to the objects put in the set, otherwise explain this better
Say(agent, task_data={"response_options": "OK made those objects into a set "})
return
def handle_reward(self, agent) -> Tuple[Optional[str], Any]:
"""Creates a new node of memory type : RewardNode and
returns a confirmation.
"""
self.finished = True
try:
reward_value = self.logical_form["upsert"]["memory_data"]["reward_value"]
assert reward_value in ("POSITIVE", "NEGATIVE")
except:
# no subinterpreters called, etc; just fail silently if somehow this came from user...
logging.debug(
"unknown reward type {} encountered in PutMemory handler".format(self.logical_form)
)
return
RewardNode.create(self.memory, reward_value)
if reward_value == "POSITIVE":
r = "Thank you!"
else:
r = "I'll try to do better in the future."
Say(agent, task_data={"response_options": r})
def handle_fill(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
tasks = []
r = d.get("reference_object")
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
"""
FIXME: We need to fix this and perhaps put this in reasoning. Agent should run perception
and put into memory. Interpreter shouldn't be perceiving, but should be able to
ask the agent to do it when needed.
"""
# Get nearby holes
perception_holes = self.get_all_holes_fn(
agent, location, self.block_data,
agent.low_level_data["fill_idmeta"])
perception_output = CraftAssistPerceptionData(holes=perception_holes)
output = self.memory.update(perception_output=perception_output)
holes = output.get("holes", [])
# Choose the best ones to fill
holes = filter_by_sublocation(self, speaker, holes, r, loose=True)
if holes is None:
# FIXME: in stage III, replace agent with the lowlevel interface to sending chats
raise ErrorWithResponse(
"I don't understand what holes you want me to fill.")
tasks = []
for hole in holes:
poss = list(hole.blocks.keys())
try:
fill_memid = agent.memory.get_triples(
subj=hole.memid, pred_text="has_fill_type")[0][2]
fill_block_mem = self.memory.get_mem_by_id(fill_memid)
fill_idm = (fill_block_mem.b, fill_block_mem.m)
except:
# FIXME use a constant name
fill_idm = (3, 0)
schematic, tags = interpret_fill_schematic(
self,
speaker,
d.get("schematic", {}),
poss,
fill_idm,
self.block_data,
self.color_bid_map,
)
origin = np.min([xyz for (xyz, bid) in schematic], axis=0)
task_data = {
"blocks_list": schematic,
"force": True,
"origin": origin,
"verbose": False,
"embed": True,
"fill_message": True,
"schematic_tags": tags,
}
tasks.append(self.task_objects["build"](agent, task_data))
if len(holes) > 1:
Say(agent,
task_data={"response_options": "Ok. I'll fill up the holes."})
else:
Say(agent,
task_data={"response_options": "Ok. I'll fill that hole up."})
return maybe_task_list_to_control_block(tasks, agent), None, None
def step(self, agent) -> Tuple[Optional[str], Any]:
start_time = datetime.datetime.now()
assert self.logical_form["dialogue_type"] == "HUMAN_GIVE_COMMAND"
try:
actions = []
if "action" in self.logical_form:
actions.append(self.logical_form["action"])
elif "action_sequence" in self.logical_form:
actions = self.logical_form["action_sequence"]
if len(actions) == 0:
# The action dict is in an unexpected state
raise ErrorWithResponse(
"I thought you wanted me to do something, but now I don't know what"
)
tasks_to_push = []
for action_def in actions:
action_type = action_def["action_type"]
# FIXME!: THIS IS A HACK to be removed by dec2021:
# special case to push loops into the handlers if
# there is only one action in the sequence but
# there is a control structure around the sequence:
if len(actions) == 1 and self.logical_form.get(
"remove_condition") is not None:
action_def = deepcopy(action_def)
action_def["remove_condition"] = deepcopy(
self.logical_form.get("remove_condition"))
r = self.action_handlers[action_type](agent, self.speaker,
action_def)
if len(r) == 3:
task, response, dialogue_data = r
else:
# FIXME don't use this branch, uniformize the signatures
task = None
response, dialogue_data = r
if task:
tasks_to_push.append(task)
task_mem = None
if tasks_to_push:
T = maybe_task_list_to_control_block(tasks_to_push, agent)
# task_mem = TaskNode(self.memory, tasks_to_push[0].memid)
task_mem = TaskNode(self.memory, T.memid)
if task_mem:
chat = self.memory.get_most_recent_incoming_chat()
TripleNode.create(self.memory,
subj=chat.memid,
pred_text="chat_effect_",
obj=task_mem.memid)
self.finished = True
end_time = datetime.datetime.now()
hook_data = {
"name": "interpreter",
"start_time": start_time,
"end_time": end_time,
"elapsed_time": (end_time - start_time).total_seconds(),
"agent_time": self.memory.get_time(),
"tasks_to_push": tasks_to_push,
"task_mem": task_mem,
}
dispatch.send("interpreter", data=hook_data)
except NextDialogueStep:
return
except ErrorWithResponse as err:
Say(agent, task_data={"response_options": err.chat})
self.finished = True
return