def handle_spawn(self, agent, speaker,
d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and handles a 'spawn' command by either replying back or
pushing a Spawn task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
# FIXME! use filters appropriately, don't search by hand
filters_d = d.get("reference_object", {}).get("filters", {})
object_idms = interpret_mob_schematic(self, speaker, filters_d)
location_d = d.get("location", SPEAKERLOOK)
locmems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
pos, _ = self.subinterpret["specify_locations"](self, speaker, locmems,
steps, reldir)
tasks = []
for i in range(len(object_idms)):
task_data = {
"object_idm": object_idms[i],
"pos": pos,
"action_dict": d
}
tasks.append(self.task_objects["spawn"](agent, task_data))
return maybe_task_list_to_control_block(tasks, agent), None, None
def __call__(self, interpreter, speaker, d):
self_mem = interpreter.memory.get_mem_by_id(
interpreter.memory.self_memid)
current_yaw, current_pitch = self_mem.get_yaw_pitch()
if d.get("yaw_pitch"):
span = d["yaw_pitch"]
# for now assumed in (yaw, pitch) or yaw, pitch or yaw pitch formats
yp = span.replace("(", "").replace(")", "").split()
return {"head_yaw_pitch": (int(yp[0]), int(yp[1]))}
elif d.get("yaw"):
# for now assumed span is yaw as word or number
w = d["yaw"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (word_to_num(w), current_pitch)}
elif d.get("pitch"):
# for now assumed span is pitch as word or number
w = d["pitch"].strip(" degrees").strip(" degree")
return {"head_yaw_pitch": (current_yaw, word_to_num(w))}
elif d.get("relative_yaw"):
# TODO in the task use turn angle
if "left" in d["relative_yaw"] or "right" in d["relative_yaw"]:
left = "left" in d["relative_yaw"] or "leave" in d[
"relative_yaw"] # lemmatizer :)
degrees = number_from_span(d["relative_yaw"]) or 90
if degrees > 0 and left:
return {"relative_yaw": -degrees}
else:
return {"relative_yaw": degrees}
else:
try:
degrees = int(number_from_span(d["relative_yaw"]))
return {"relative_yaw": degrees}
except:
pass
elif d.get("relative_pitch"):
if "down" in d["relative_pitch"] or "up" in d["relative_pitch"]:
down = "down" in d["relative_pitch"]
degrees = number_from_span(d["relative_pitch"]) or 90
if degrees > 0 and down:
return {"relative_pitch": -degrees}
else:
return {"relative_pitch": degrees}
else:
# TODO in the task make this relative!
try:
deg = int(number_from_span(d["relative_pitch"]))
return {"relative_pitch": deg}
except:
pass
elif d.get("location"):
mems = interpreter.subinterpret["reference_locations"](
interpreter, speaker, d["location"])
steps, reldir = interpret_relative_direction(
interpreter, d["location"])
loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
return {"head_xyz": loc}
else:
raise ErrorWithResponse("I am not sure where you want me to turn")
def __call__(self, interpreter, speaker, d) -> POINT_AT_TARGET:
if d.get("location") is None:
# TODO other facings
raise ErrorWithResponse("I am not sure where you want me to point")
# TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure
mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, d["location"])
steps, reldir = interpret_relative_direction(interpreter, d)
loc, _ = interpreter.subinterpret["specify_locations"](
interpreter, speaker, mems, steps, reldir
)
return self.point_to_region(loc)
def __call__(self, interpreter, speaker, d):
if d.get("location") is None:
# TODO other facings
raise ErrorWithResponse("I am not sure where you want me to point")
# TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure
# what the +1s are in the return value
mems = interpreter.subinterpret["reference_locations"](interpreter,
speaker,
d["location"])
steps, reldir = interpret_relative_direction(interpreter, d)
# if directly point at a reference object, call built-in fn to get pointed target
if steps is None and reldir is None:
loc = mems[0].get_point_at_target()
else:
loc, _ = interpreter.subinterpret["specify_locations"](interpreter,
speaker,
mems, steps,
reldir)
return self.point_to_region(loc)
def get_block_task_data(self, speaker, schematic_info, d):
"""
takes schematic_info returned e.g. from interpret_schematic
and the location logical form
and returns task data for a list of build or dig tasks
"""
# Get the locations to build
location_d = d.get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
origin, offsets = self.subinterpret["specify_locations"](
self,
speaker,
mems,
steps,
reldir,
repeat_dir=get_repeat_dir(location_d),
objects=schematic_info,
)
schematic_info_with_offsets = [
(blocks, mem, tags, off)
for (blocks, mem, tags), off in zip(schematic_info, offsets)
]
tasks_data = []
for schematic, schematic_memid, tags, offset in schematic_info_with_offsets:
og = np.array(origin) + offset
task_data = {
"blocks_list": schematic,
"origin": og,
"schematic_memid": schematic_memid,
"schematic_tags": tags,
"action_dict": d,
}
tasks_data.append(task_data)
return tasks_data
def new_tasks():
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE"
or rd == "ANTICLOCKWISE"):
ref_obj = None
location_reference_object = location_d.get(
"reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object)
if len(objmems) == 0:
raise ErrorWithResponse(
"I don't understand where you want me to go.")
ref_obj = objmems[0]
refmove = dance.RefObjMovement(
agent,
ref_object=ref_obj,
relative_direction=location_d["relative_direction"],
)
t = self.task_objects["dance"](agent, {
"movement": refmove
})
return t
dance_type = d.get("dance_type", {})
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker,
dance_type["point"])
t = self.task_objects["point"](agent, {"target": target})
# MC bot does not control body turn separate from head
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn") or dance_type.get("body_turn")
f = self.subinterpret["facing"](self, speaker, lt)
t = self.task_objects["dancemove"](agent, f)
else:
if location_d is None:
dance_location = None
else:
mems = self.subinterpret["reference_locations"](self,
speaker,
location_d)
steps, reldir = interpret_relative_direction(
self, location_d)
dance_location, _ = self.subinterpret["specify_locations"](
self, speaker, mems, steps, reldir)
filters_d = dance_type.get("filters", {})
filters_d["memory_type"] = "DANCES"
F = self.subinterpret["filters"](self, speaker,
dance_type.get("filters", {}))
dance_memids, _ = F()
# TODO correct selector in filters
if dance_memids:
dance_memid = random.choice(dance_memids)
dance_mem = self.memory.get_mem_by_id(dance_memid)
dance_obj = dance.Movement(agent=agent,
move_fn=dance_mem.dance_fn,
dance_location=dance_location)
t = self.task_objects["dance"](agent, {
"movement": dance_obj
})
else:
# dance out of scope
raise ErrorWithResponse(
"I don't know how to do that movement yet.")
return t
def handle_fill(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
tasks = []
r = d.get("reference_object")
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
"""
FIXME: We need to fix this and perhaps put this in reasoning. Agent should run perception
and put into memory. Interpreter shouldn't be perceiving, but should be able to
ask the agent to do it when needed.
"""
# Get nearby holes
perception_holes = self.get_all_holes_fn(
agent, location, self.block_data,
agent.low_level_data["fill_idmeta"])
perception_output = CraftAssistPerceptionData(holes=perception_holes)
output = self.memory.update(perception_output=perception_output)
holes = output.get("holes", [])
# Choose the best ones to fill
holes = filter_by_sublocation(self, speaker, holes, r, loose=True)
if holes is None:
# FIXME: in stage III, replace agent with the lowlevel interface to sending chats
raise ErrorWithResponse(
"I don't understand what holes you want me to fill.")
tasks = []
for hole in holes:
poss = list(hole.blocks.keys())
try:
fill_memid = agent.memory.get_triples(
subj=hole.memid, pred_text="has_fill_type")[0][2]
fill_block_mem = self.memory.get_mem_by_id(fill_memid)
fill_idm = (fill_block_mem.b, fill_block_mem.m)
except:
# FIXME use a constant name
fill_idm = (3, 0)
schematic, tags = interpret_fill_schematic(
self,
speaker,
d.get("schematic", {}),
poss,
fill_idm,
self.block_data,
self.color_bid_map,
)
origin = np.min([xyz for (xyz, bid) in schematic], axis=0)
task_data = {
"blocks_list": schematic,
"force": True,
"origin": origin,
"verbose": False,
"embed": True,
"fill_message": True,
"schematic_tags": tags,
}
tasks.append(self.task_objects["build"](agent, task_data))
if len(holes) > 1:
Say(agent,
task_data={"response_options": "Ok. I'll fill up the holes."})
else:
Say(agent,
task_data={"response_options": "Ok. I'll fill that hole up."})
return maybe_task_list_to_control_block(tasks, agent), None, None