def interpret_schematic(
interpreter,
speaker,
d,
repeat_dict=None
) -> List[Tuple[List[Block], Optional[str], List[Tuple[str, str]]]]:
"""Return a list of 3-tuples, each with values:
- the schematic blocks, list[(xyz, idm)]
- a SchematicNode memid, or None
- a list of (pred, val) tags
"""
# hack, fixme in grammar/standardize. sometimes the repeat is a sibling of action
if repeat_dict is not None:
repeat = cast(int, get_repeat_num(repeat_dict))
else:
repeat = cast(int, get_repeat_num(d))
assert type(repeat) == int, "bad repeat={}".format(repeat)
triples = d.get("triples", [{
"pred_text": "has_shape",
"obj_text": "cube"
}])
shapes = get_properties_from_triples(triples, "has_shape")
if any(shapes):
blocks, tags = interpret_shape_schematic(interpreter, speaker, d)
return [(blocks, None, tags)] * repeat
else:
return [interpret_named_schematic(interpreter, speaker, d)] * repeat
def handle_spawn(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and handles a 'spawn' command by either replying back or
pushing a Spawn task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
# FIXME! use filters appropriately, don't search by hand
spawn_triples = d.get("reference_object", {}).get("filters", {}).get("triples", [])
if not spawn_triples:
raise ErrorWithResponse("I don't understand what you want me to spawn.")
names = [t.get("obj_text") for t in spawn_triples if t.get("pred_text", "") == "has_name"]
if not any(names):
raise ErrorWithResponse("I don't understand what you want me to spawn.")
# if multiple possible has_name triples, just pick the first:
object_name = names[0]
schematic = self.memory.get_mob_schematic_by_name(object_name)
if not schematic:
raise ErrorWithResponse("I don't know how to spawn: %r." % (object_name))
object_idm = list(schematic.blocks.values())[0]
location_d = d.get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
pos, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir)
repeat_times = get_repeat_num(d)
for i in range(repeat_times):
task_data = {"object_idm": object_idm, "pos": pos, "action_dict": d}
self.append_new_task(self.task_objects["spawn"], task_data)
self.finished = True
return None, None
def handle_fill(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
r = d.get("reference_object")
self.finished = True
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self, speaker, candidates, r, limit=repeat, loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill."
)
return None, None
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
# FIXME use filters properly...
triples = d.get("triples", [])
block_types = [
t.get("obj_text") for t in triples if t.get("pred_text", "") == "has_block_type"
]
try:
fill_idm = get_block_type(block_types[0])
except:
fill_idm = hole_idm
task_data = {"action_dict": d, "schematic": poss, "block_idm": fill_idm}
self.append_new_task(self.task_objects["fill"], task_data)
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
self.finished = True
return None, None
def new_tasks():
attrs = {}
schematic_d = d.get("schematic", {"has_size": 2})
# set the attributes of the hole to be dug.
for dim, default in [("depth", 1), ("length", 1), ("width", 1)]:
key = "has_{}".format(dim)
if key in schematic_d:
attrs[dim] = word_to_num(schematic_d[key])
elif "has_size" in schematic_d:
attrs[dim] = interpret_size(self, schematic_d["has_size"])
else:
attrs[dim] = default
# minecraft world is [z, x, y]
padding = (attrs["depth"] + 4, attrs["length"] + 4, attrs["width"] + 4)
print("attrs", attrs)
print("padding", padding)
location_d = d.get("location", SPEAKERLOOK)
repeat_num = get_repeat_num(d)
repeat_dir = get_repeat_dir(d)
print("loc d in dig", location_d, "repeat", repeat_num, repeat_dir)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
origin, offsets = self.subinterpret["specify_locations"](
self,
speaker,
mems,
steps,
reldir,
repeat_num=repeat_num,
repeat_dir=repeat_dir,
padding=padding,
)
print("origin from dig", origin, "offsets", offsets)
# add dig tasks in a loop
tasks_todo = []
for offset in offsets:
og = np.array(origin) + offset
t = self.task_objects["dig"](self.agent, {"origin": og, "action_dict": d, **attrs})
print("append task:", t, og, d, attrs)
tasks_todo.append(t)
return list(reversed(tasks_todo))
def new_tasks():
repeat = get_repeat_num(d)
tasks_to_do = []
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (
rd == "AROUND" or rd == "CLOCKWISE" or rd == "ANTICLOCKWISE"
):
ref_obj = None
location_reference_object = location_d.get("reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object
)
if len(objmems) == 0:
raise ErrorWithResponse("I don't understand where you want me to go.")
ref_obj = objmems[0]
for i in range(repeat):
refmove = dance.RefObjMovement(
self.agent,
ref_object=ref_obj,
relative_direction=location_d["relative_direction"],
)
t = self.task_objects["dance"](self.agent, {"movement": refmove})
tasks_to_do.append(t)
return list(reversed(tasks_to_do))
dance_type = d.get("dance_type", {"dance_type_name": "dance"})
# FIXME holdover from old dict format
if type(dance_type) is str:
dance_type = dance_type = {"dance_type_name": "dance"}
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker, dance_type["point"])
for i in range(repeat):
t = self.task_objects["point"](self.agent, {"target": target})
tasks_to_do.append(t)
# MC bot does not control body turn separate from head
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn") or dance_type.get("body_turn")
f = self.subinterpret["facing"](self, speaker, lt)
for i in range(repeat):
t = self.task_objects["dancemove"](self.agent, f)
tasks_to_do.append(t)
else:
if location_d is None:
dance_location = None
else:
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
dance_location, _ = self.subinterpret["specify_locations"](
self, speaker, mems, steps, reldir
)
# TODO use name!
if dance_type.get("dance_type_span") is not None:
dance_name = dance_type["dance_type_span"]
if dance_name == "dance":
dance_name = "ornamental_dance"
dance_memids = self.memory._db_read(
"SELECT DISTINCT(Dances.uuid) FROM Dances INNER JOIN Triples on Dances.uuid=Triples.subj WHERE Triples.obj_text=?",
dance_name,
)
else:
dance_memids = self.memory._db_read(
"SELECT DISTINCT(Dances.uuid) FROM Dances INNER JOIN Triples on Dances.uuid=Triples.subj WHERE Triples.obj_text=?",
"ornamental_dance",
)
dance_memid = random.choice(dance_memids)[0]
dance_fn = self.memory.dances[dance_memid]
for i in range(repeat):
dance_obj = dance.Movement(
agent=self.agent, move_fn=dance_fn, dance_location=dance_location
)
t = self.task_objects["dance"](self.agent, {"movement": dance_obj})
tasks_to_do.append(t)
return list(reversed(tasks_to_do))
def handle_build(self, speaker, d) -> Tuple[Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'build' command by either pushing a dialogue object
or pushing a Build task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
# Get the segment to build
if "reference_object" in d:
# handle copy
repeat = get_repeat_num(d)
objs = self.subinterpret["reference_objects"](
self,
speaker,
d["reference_object"],
limit=repeat,
extra_tags=["_voxel_object"],
loose_speakerlook=True,
)
if len(objs) == 0:
raise ErrorWithResponse("I don't understand what you want me to build")
tagss = [
[(p, v) for (_, p, v) in self.memory.get_triples(subj=obj.memid)] for obj in objs
]
interprets = [
[list(obj.blocks.items()), obj.memid, tags] for (obj, tags) in zip(objs, tagss)
]
else: # a schematic
if d.get("repeat") is not None:
repeat_dict = d
else:
repeat_dict = None
interprets = interpret_schematic(
self, speaker, d.get("schematic", {}), repeat_dict=repeat_dict
)
# Get the locations to build
location_d = d.get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker, location_d)
steps, reldir = interpret_relative_direction(self, location_d)
origin, offsets = self.subinterpret["specify_locations"](
self,
speaker,
mems,
steps,
reldir,
repeat_dir=get_repeat_dir(location_d),
objects=interprets,
enable_geoscorer=True,
)
interprets_with_offsets = [
(blocks, mem, tags, off) for (blocks, mem, tags), off in zip(interprets, offsets)
]
tasks_todo = []
for schematic, schematic_memid, tags, offset in interprets_with_offsets:
og = np.array(origin) + offset
task_data = {
"blocks_list": schematic,
"origin": og,
"schematic_memid": schematic_memid,
"schematic_tags": tags,
"action_dict": d,
}
tasks_todo.append(task_data)
for task_data in reversed(tasks_todo):
self.append_new_task(self.task_objects["build"], task_data)
logging.info("Added {} Build tasks to stack".format(len(tasks_todo)))
self.finished = True
return None, None
def new_tasks():
repeat = get_repeat_num(d)
tasks_to_do = []
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE"
or rd == "ANTICLOCKWISE"):
ref_obj = None
location_reference_object = location_d.get(
"reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object)
if len(objmems) == 0:
raise ErrorWithResponse(
"I don't understand where you want me to go.")
ref_obj = objmems[0]
for i in range(repeat):
refmove = dance.RefObjMovement(
self.agent,
ref_object=ref_obj,
relative_direction=location_d[
"relative_direction"],
)
t = self.task_objects["dance"](self.agent, {
"movement": refmove
})
tasks_to_do.append(t)
return list(reversed(tasks_to_do))
dance_type = d.get("dance_type", {"dance_type_name": "dance"})
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker,
dance_type["point"])
for i in range(repeat):
t = self.task_objects["point"](self.agent, {
"target": target
})
tasks_to_do.append(t)
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn")
if lt:
f = self.subinterpret["facing"](self,
speaker,
lt,
head_or_body="head")
T = self.task_objects["look"]
else:
bt = dance_type.get("body_turn")
f = self.subinterpret["facing"](self,
speaker,
bt,
head_or_body="body")
T = self.task_objects["turn"]
for i in range(repeat):
tasks_to_do.append(T(self.agent, f))
elif dance_type["dance_type_name"] == "wave":
new_task = self.task_objects["dance"](self.agent, {
"movement_type": "wave"
})
tasks_to_do.append(new_task)
else:
# FIXME ! merge dances, refactor. search by name in sql
raise ErrorWithResponse(
"I don't know how to do that dance yet!")
return list(reversed(tasks_to_do))
def new_tasks():
repeat = get_repeat_num(d)
tasks_to_do = []
# only go around the x has "around"; FIXME allow other kinds of dances
location_d = d.get("location")
if location_d is not None:
rd = location_d.get("relative_direction")
if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE"
or rd == "ANTICLOCKWISE"):
ref_obj = None
location_reference_object = location_d.get(
"reference_object")
if location_reference_object:
objmems = self.subinterpret["reference_objects"](
self, speaker, location_reference_object)
if len(objmems) == 0:
raise ErrorWithResponse(
"I don't understand where you want me to go.")
ref_obj = objmems[0]
for i in range(repeat):
refmove = dance.RefObjMovement(
self.agent,
ref_object=ref_obj,
relative_direction=location_d[
"relative_direction"],
)
t = self.task_objects["dance"](self.agent, {
"movement": refmove
})
tasks_to_do.append(t)
return maybe_task_list_to_control_block(
tasks_to_do, self.agent)
dance_type = d.get("dance_type", {})
if dance_type.get("point"):
target = self.subinterpret["point_target"](self, speaker,
dance_type["point"])
for i in range(repeat):
t = self.task_objects["point"](self.agent, {
"target": target
})
tasks_to_do.append(t)
# MC bot does not control body turn separate from head
elif dance_type.get("look_turn") or dance_type.get("body_turn"):
lt = dance_type.get("look_turn") or dance_type.get("body_turn")
f = self.subinterpret["facing"](self, speaker, lt)
for i in range(repeat):
t = self.task_objects["dancemove"](self.agent, f)
tasks_to_do.append(t)
else:
if location_d is None:
dance_location = None
else:
mems = self.subinterpret["reference_locations"](self,
speaker,
location_d)
steps, reldir = interpret_relative_direction(
self, location_d)
dance_location, _ = self.subinterpret["specify_locations"](
self, speaker, mems, steps, reldir)
filters_d = dance_type.get("filters", {})
filters_d["memory_type"] = "DANCES"
F = self.subinterpret["filters"](self, speaker,
dance_type.get("filters", {}))
dance_memids, _ = F()
# TODO correct selector in filters
dance_memid = random.choice(dance_memids)
dance_mem = self.memory.get_mem_by_id(dance_memid)
for i in range(repeat):
dance_obj = dance.Movement(agent=self.agent,
move_fn=dance_mem.dance_fn,
dance_location=dance_location)
t = self.task_objects["dance"](self.agent, {
"movement": dance_obj
})
tasks_to_do.append(t)
return maybe_task_list_to_control_block(tasks_to_do, self.agent)
def handle_fill(self, speaker, d) -> Tuple[Any, Optional[str], Any]:
"""This function reads the dictionary, resolves the missing details using memory
and perception and handles a 'fill' command by either pushing a dialogue object
or pushing a Fill task to the task stack.
Args:
speaker: speaker_id or name.
d: the complete action dictionary
"""
tasks = []
r = d.get("reference_object")
if not r.get("filters"):
r["filters"] = {"location", SPEAKERLOOK}
# Get the reference location
location_d = r["filters"].get("location", SPEAKERLOOK)
mems = self.subinterpret["reference_locations"](self, speaker,
location_d)
steps, reldir = interpret_relative_direction(self, location_d)
location, _ = self.subinterpret["specify_locations"](self, speaker,
mems, steps,
reldir)
# Get nearby holes
holes: List[Hole] = heuristic_perception.get_all_nearby_holes(
self.agent, location)
candidates: List[Tuple[XYZ, Hole]] = [
(to_block_pos(np.mean(hole[0], axis=0)), hole) for hole in holes
]
# Choose the best ones to fill
repeat = get_repeat_num(d)
holes = filter_by_sublocation(self,
speaker,
candidates,
r,
limit=repeat,
loose=True)
if holes is None:
self.dialogue_stack.append_new(
Say, "I don't understand what holes you want me to fill.")
return None, None, None
tasks = []
for hole in holes:
_, hole_info = hole
poss, hole_idm = hole_info
schematic, tags = interpret_fill_schematic(self, speaker,
d.get("schematic", {}),
poss, hole_idm)
origin = np.min([xyz for (xyz, bid) in schematic], axis=0)
task_data = {
"blocks_list": schematic,
"force": True,
"origin": origin,
"verbose": False,
"embed": True,
"fill_message": True,
"schematic_tags": tags,
}
tasks.append(self.task_objects["build"](self.agent, task_data))
if len(holes) > 1:
self.dialogue_stack.append_new(Say, "Ok. I'll fill up the holes.")
else:
self.dialogue_stack.append_new(Say, "Ok. I'll fill that hole up.")
return maybe_task_list_to_control_block(tasks, self.agent), None, None