def special_reference_search_data(interpreter, speaker, S, entity_id=None, agent_memory=None): """make a search query for a MemorySearcher to return the special ReferenceObject""" # TODO/FIXME! add things to workspace memory agent_memory = agent_memory or interpreter.memory if type(S) is dict: coord_span = S["coordinates_span"] loc = cast( XYZ, tuple(int(float(w)) for w in re.findall("[-0-9.]+", coord_span))) if len(loc) != 3: logging.error("Bad coordinates: {}".format(coord_span)) raise ErrorWithResponse( "I don't understand what location you're referring to") memid = agent_memory.add_location( (int(loc[0]), int(loc[1]), int(loc[2]))) mem = agent_memory.get_location_by_id(memid) q = "SELECT MEMORY FROM ReferenceObject WHERE uuid={}".format(memid) else: if S == "AGENT" or S == "SELF" or S == "SPEAKER": q = "SELECT MEMORY FROM Player WHERE eid={}".format(entity_id) elif S == "SPEAKER_LOOK": q = "SELECT MEMORY FROM Attention WHERE type_name={}".format( entity_id) else: raise ErrorWithResponse("unknown special reference: {}".format(S)) return q
def handle_destroy(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]: """This function reads the dictionary, resolves the missing details using memory and perception and handles a 'destroy' command by either pushing a dialogue object or pushing a Destroy task to the task stack. Args: speaker: speaker_id or name. d: the complete action dictionary """ default_ref_d = {"filters": {"location": SPEAKERLOOK}} ref_d = d.get("reference_object", default_ref_d) objs = self.subinterpret["reference_objects"]( self, speaker, ref_d, extra_tags=["_destructible"]) if len(objs) == 0: raise ErrorWithResponse( "I don't understand what you want me to destroy.") # don't kill mobs if all(isinstance(obj, MobNode) for obj in objs): raise ErrorWithResponse("I don't kill animals, sorry!") if all(isinstance(obj, PlayerNode) for obj in objs): raise ErrorWithResponse("I don't kill players, sorry!") objs = [obj for obj in objs if not isinstance(obj, MobNode)] tasks = [] for obj in objs: if hasattr(obj, "blocks"): schematic = list(obj.blocks.items()) task_data = {"schematic": schematic, "action_dict": d} tasks.append(self.task_objects["destroy"](agent, task_data)) logging.info("Added {} Destroy tasks to stack".format(len(tasks))) return maybe_task_list_to_control_block(tasks, agent), None, None
def handle_get(self, agent, speaker, d) -> Tuple[Optional[str], Any]: default_ref_d = {"filters": {"location": AGENTPOS}} ref_d = d.get("reference_object", default_ref_d) objs = self.subinterpret["reference_objects"]( self, speaker, ref_d, extra_tags=["_physical_object"] ) if len(objs) == 0: raise ErrorWithResponse("I don't know what you want me to get.") if all(isinstance(obj, PlayerNode) for obj in objs): raise ErrorWithResponse("I can't get a person, sorry!") objs = [obj for obj in objs if not isinstance(obj, PlayerNode)] if d.get("receiver") is None: receiver_d = None else: receiver_d = d.get("receiver").get("reference_object") receiver = None if receiver_d: receiver = self.subinterpret["reference_objects"](self, speaker, receiver_d) if len(receiver) == 0: raise ErrorWithResponse("I don't know where you want me to take it") receiver = receiver[0].memid tasks = [] for obj in objs: task_data = {"get_target": obj.memid, "give_target": receiver, "action_dict": d} tasks.append(self.task_objects["get"](agent, task_data)) # logging.info("Added {} Get tasks to stack".format(len(tasks))) return maybe_task_list_to_control_block(tasks, agent), None, None
def handle_drop(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]: """This function reads the dictionary, resolves the missing details using memory and perception and handles a 'drop' command by either pushing a dialogue object or pushing a Drop task to the task stack. Args: speaker: speaker_id or name. d: the complete action dictionary """ ref_d = d.get("reference_object", None) if not ref_d: raise ErrorWithResponse( "I don't understand what you want me to drop.") objs = self.subinterpret["reference_objects"]( self, speaker, ref_d, extra_tags=["_in_inventory"]) if len(objs) == 0: raise ErrorWithResponse( "I don't understand what you want me to drop.") obj = [obj for obj in objs if isinstance(obj, ItemStackNode)][0] item_stack = agent.get_item_stack(obj.eid) idm = (item_stack.item.id, item_stack.item.meta) task_data = {"eid": obj.eid, "idm": idm, "obj_memid": obj.memid} return self.task_objects["drop"](agent, task_data), None, None
def interpret_named_schematic( interpreter, speaker, d, block_data_info, color_bid_map, special_shape_function ) -> Tuple[List[Block], Optional[str], List[Tuple[str, str]]]: """Return a tuple of 3 values: - the schematic blocks, list[(xyz, idm)] - a SchematicNode memid, or None - a list of (pred, val) tags """ # FIXME! this is not compositional, and is not using full FILTERS handlers filters_d = d.get("filters", {}) flattened_clauses = get_flattened_clauses(filters_d) name = get_properties_from_clauses(flattened_clauses, ["has_name"]).get("has_name", "") if not name: raise ErrorWithResponse("I don't know what you want me to build.") stemmed_name = name.strip("s") # why aren't we using stemmer anymore? shapename = SPECIAL_SHAPES_CANONICALIZE.get( name) or SPECIAL_SHAPES_CANONICALIZE.get(stemmed_name) if shapename: shape_blocks, tags = interpret_shape_schematic( interpreter, speaker, d, block_data_info, color_bid_map, special_shape_function, shapename=shapename, ) return shape_blocks, None, tags schematic = interpreter.memory.get_schematic_by_name(name) if schematic is None: schematic = interpreter.memory.get_schematic_by_name(stemmed_name) if schematic is None: raise ErrorWithResponse("I don't know what you want me to build.") triples = [(p, v) for (_, p, v) in interpreter.memory.get_triples(subj=schematic.memid)] blocks = schematic.blocks # TODO generalize to more general block properties # Longer term: remove and put a call to the modify model here colour = get_properties_from_clauses(flattened_clauses, ["has_colour"]).get("has_colour", "") if colour: old_idm = most_common_idm(blocks.values()) c = color_bid_map.get(colour) if c is not None: new_idm = random.choice(c) for l in blocks: if blocks[l] == old_idm: blocks[l] = new_idm return list(blocks.items()), schematic.memid, triples
def interpret_mob_schematic(interpreter, speaker, filters_d): spawn_clause = {"pred_text": "has_tag", "obj_text": "_spawn"} where = filters_d.get("where_clause", {"AND": [spawn_clause]}) if where.get("AND"): where["AND"].append(spawn_clause) else: new_where = {"AND": [spawn_clause, deepcopy(where)]} where = new_where # HACK for nsp/data weirdness: for now don't allow # 'same': 'DISALLOWED' in Selector so could not e.g. # "spawn three different kinds of mobs". we don't have examples # like that yet anyway ... if filters_d.get("selector", {}): if filters_d["selector"].get("same", "ALLOWED") == "DISALLOWED": filters_d["selector"]["same"] = "ALLOWED" # FIXME! we don't need to recopy this here, do more composably W = interpret_where_backoff(interpreter, speaker, where, memory_type="Schematic") F = maybe_apply_selector(interpreter, speaker, filters_d, W) schematic_memids, _ = F() object_idms = [ list(SchematicNode(interpreter.memory, m).blocks.values())[0] for m in schematic_memids ] if not object_idms: raise ErrorWithResponse("I don't know how to spawn that") return object_idms
def compute_location_heuristic(player_mem, mems, steps, reldir, get_locs_from_entity): loc = mems[0].get_pos() if reldir is not None: steps = steps or DEFAULT_NUM_STEPS if reldir == "BETWEEN": loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2 loc = (loc[0], loc[1], loc[2]) elif reldir == "INSIDE": for i in range(len(mems)): mem = mems[i] # FIXME locs = heuristic_perception.find_inside( mem, get_locs_from_entity) if len(locs) > 0: break if len(locs) == 0: raise ErrorWithResponse("I don't know how to go inside there") else: loc = locs[0] elif reldir == "NEAR": pass elif reldir == "AROUND": pass else: # LEFT, RIGHT, etc... reldir_vec = rotation.DIRECTIONS[reldir] # this should be an inverse transform so we set inverted=True yaw, _ = player_mem.get_yaw_pitch() dir_vec = rotation.transform(reldir_vec, yaw, 0, inverted=True) loc = steps * np.array(dir_vec) + to_block_center(loc) elif steps is not None: loc = to_block_center(loc) + [0, 0, steps] return to_block_pos(loc)
def __call__(self, interpreter, speaker, d): self_mem = interpreter.memory.get_mem_by_id( interpreter.memory.self_memid) current_yaw, current_pitch = self_mem.get_yaw_pitch() if d.get("yaw_pitch"): span = d["yaw_pitch"] # for now assumed in (yaw, pitch) or yaw, pitch or yaw pitch formats yp = span.replace("(", "").replace(")", "").split() return {"head_yaw_pitch": (int(yp[0]), int(yp[1]))} elif d.get("yaw"): # for now assumed span is yaw as word or number w = d["yaw"].strip(" degrees").strip(" degree") return {"head_yaw_pitch": (word_to_num(w), current_pitch)} elif d.get("pitch"): # for now assumed span is pitch as word or number w = d["pitch"].strip(" degrees").strip(" degree") return {"head_yaw_pitch": (current_yaw, word_to_num(w))} elif d.get("relative_yaw"): # TODO in the task use turn angle if "left" in d["relative_yaw"] or "right" in d["relative_yaw"]: left = "left" in d["relative_yaw"] or "leave" in d[ "relative_yaw"] # lemmatizer :) degrees = number_from_span(d["relative_yaw"]) or 90 if degrees > 0 and left: return {"relative_yaw": -degrees} else: return {"relative_yaw": degrees} else: try: degrees = int(number_from_span(d["relative_yaw"])) return {"relative_yaw": degrees} except: pass elif d.get("relative_pitch"): if "down" in d["relative_pitch"] or "up" in d["relative_pitch"]: down = "down" in d["relative_pitch"] degrees = number_from_span(d["relative_pitch"]) or 90 if degrees > 0 and down: return {"relative_pitch": -degrees} else: return {"relative_pitch": degrees} else: # TODO in the task make this relative! try: deg = int(number_from_span(d["relative_pitch"])) return {"relative_pitch": deg} except: pass elif d.get("location"): mems = interpreter.subinterpret["reference_locations"]( interpreter, speaker, d["location"]) steps, reldir = interpret_relative_direction( interpreter, d["location"]) loc, _ = interpreter.subinterpret["specify_locations"](interpreter, speaker, mems, steps, reldir) return {"head_xyz": loc} else: raise ErrorWithResponse("I am not sure where you want me to turn")
def get_special_reference_object(interpreter, speaker, S, agent_memory=None, eid=None): """subinterpret a special reference object. args: interpreter: the root interpreter speaker (str): The name of the player/human/agent who uttered the chat resulting in this interpreter S: the special reference object logical form from """ # TODO/FIXME! add things to workspace memory agent_memory = agent_memory or interpreter.memory if not eid: eid = get_eid_from_special(agent_memory, S, speaker=speaker) q = special_reference_search_data(None, speaker, S, entity_id=eid, agent_memory=agent_memory) _, mems = agent_memory.basic_search(q) if not mems: # need a better interface for this, don't need to run full perception # just to force speakerlook in memory # TODO force if look is stale, not just if it doesn't exist # this branch shouldn't occur # interpreter.agent.perceive(force=True) raise ErrorWithResponse( "I think you are pointing at something but I don't know what it is" ) return mems[0]
def safe_call(f, *args, **kwargs): try: return f(*args, **kwargs) except Pyro4.errors.ConnectionClosedError as e: msg = "{} - {}".format(f._RemoteMethod__name, e) raise ErrorWithResponse(msg) except Exception as e: print("Pyro traceback:") print("".join(Pyro4.util.getPyroTraceback())) raise e
def __call__(self, interpreter, speaker, d) -> POINT_AT_TARGET: if d.get("location") is None: # TODO other facings raise ErrorWithResponse("I am not sure where you want me to point") # TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, d["location"]) steps, reldir = interpret_relative_direction(interpreter, d) loc, _ = interpreter.subinterpret["specify_locations"]( interpreter, speaker, mems, steps, reldir ) return self.point_to_region(loc)
def handle_task_refobj_string(self, task, refobj_attr): if refobj_attr == "name": for pred, val in task.task.target: if pred == "has_name": return "I am going to the " + prepend_a_an(val), None elif refobj_attr == "location": target = tuple(task.task.target) return "I am going to {}".format(target), None else: raise ErrorWithResponse( "trying get attribute {} from action".format(refobj_attr))
def __call__(self, interpreter, speaker, d) -> Optional[Condition]: """subinterpreter for Conditions args: interpreter: root interpreter. speaker (str): The name of the player/human/agent who uttered the chat resulting in this interpreter d: logical form from semantic parser """ ct = d.get("condition_type") if ct: if self.condition_types.get(ct): # condition_type NEVER doesn't have a "condition" sibling if ct == "NEVER": return self.condition_types[ct](interpreter, speaker, d) if not d.get("condition"): raise ErrorWithResponse( "I thought there was a condition but I don't understand it" ) return self.condition_types[ct](interpreter, speaker, d["condition"]) else: raise ErrorWithResponse("I don't understand that condition") else: return None
def handle_task_refobj_string(self, task, refobj_attr): if refobj_attr == "name": assert isinstance(task.task, Build), task.task for pred, val in task.task.schematic_tags: if pred == "has_name": return "I am building " + prepend_a_an(val), None return "I am building something that is {}".format(val), None elif refobj_attr == "location": assert task.action_name == "Move", task.action_name target = tuple(task.task.target) return "I am going to {}".format(target), None else: raise ErrorWithResponse( "trying get attribute {} from action".format(refobj_attr)) return None, None
def __call__(self, interpreter, speaker, filters_d, get_all=False): """ This is a subinterpreter to handle FILTERS dictionaries Args: interpreter: root interpreter. speaker (str): The name of the player/human/agent who uttered the chat resulting in this interpreter filters_d: FILTERS logical form from semantic parser get_all (bool): if True, output attributes are set with get_all=True Outputs a (chain) of MemoryFilter objects """ val_map = get_val_map(interpreter, speaker, filters_d, get_all=get_all) # NB (kavyasrinet) output can be string and have value "memory" too here # is this a specific memory? # ... then return specific_mem_filter = maybe_handle_specific_mem( interpreter, speaker, filters_d, val_map) if specific_mem_filter is not None: return specific_mem_filter memtype = filters_d.get("memory_type", "REFERENCE_OBJECT") # FIXME/TODO: these share lots of code, refactor if memtype == "REFERENCE_OBJECT": # just using this to check if SELF is a possibility, TODO finer control tags, _ = backoff_where(filters_d.get("where_clause"), {}) F = interpret_where_backoff( interpreter, speaker, filters_d.get("where_clause", {}), memory_type="ReferenceObject", ignore_self=not ("SELF" in tags), ) elif memtype == "TASKS": F = interpret_task_filter(interpreter, speaker, filters_d) else: memtype_key = memtype.lower() + "_filters" try: F = interpreter.subinterpret[memtype_key](interpreter, speaker, filters_d) except: raise ErrorWithResponse( "failed at interpreting filters of type {}".format( memtype)) F = maybe_apply_selector(interpreter, speaker, filters_d, F) return maybe_append_left(F, to_append=val_map)
def handle_build(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]: """This function reads the dictionary, resolves the missing details using memory and perception and handles a 'build' command by either pushing a dialogue object or pushing a Build task to the task stack. Args: speaker: speaker_id or name. d: the complete action dictionary """ # Get the segment to build if "reference_object" in d: # handle copy ##########FIXME remove this when DSL updated!!! md = deepcopy(d) objs = self.subinterpret["reference_objects"]( self, speaker, md["reference_object"], extra_tags=["VOXEL_OBJECT"], loose_speakerlook=True, ) if len(objs) == 0: raise ErrorWithResponse( "I don't understand what you want me to build") tagss = [[(p, v) for (_, p, v) in self.memory.get_triples(subj=obj.memid)] for obj in objs] interprets = [[list(obj.blocks.items()), obj.memid, tags] for (obj, tags) in zip(objs, tagss)] else: # a schematic interprets = interpret_schematic( self, speaker, d.get("schematic", {}), self.block_data, self.color_bid_map, self.special_shape_functions, ) tasks_data = self.get_block_task_data(speaker, interprets, d) tasks = [] for td in tasks_data: t = self.task_objects["build"](agent, td) tasks.append(t) logging.info("Adding {} Build tasks to stack".format(len(tasks))) return maybe_task_list_to_control_block(tasks, agent), None, None
def interpret_time(self, interpreter, speaker, d): event = None if d.get("special_time_event"): return TimeCondition(interpreter.memory, d["special_time_event"]) else: if not d.get("comparator"): raise ErrorWithResponse( "I don't know how to interpret this time condition") dc = d["comparator"] dc["input_left"] = {"value_extractor": "NULL"} comparator = interpret_comparator(interpreter, speaker, dc) if d.get("event"): event = self(interpreter, speaker, d["event"]) return TimeCondition(interpreter.memory, comparator, event=event)
def handle_triple(self, agent) -> Tuple[Optional[str], Any]: """Writes a triple of type : (subject, predicate_text, object_text) to the memory and returns a confirmation. Returns: output_chat: An optional string for when the agent wants to send a chat step_data: Any other data that this step would like to send to the task """ ref_obj_d = {"filters": self.logical_form["filters"]} r = self.subinterpret["reference_objects"]( self, self.speaker, ref_obj_d, extra_tags=["_physical_object"] ) if len(r) == 0: raise ErrorWithResponse("I don't know what you're referring to") mem = r[0] name = "it" triples = self.memory.get_triples(subj=mem.memid, pred_text="has_tag") if len(triples) > 0: name = triples[0][2].strip("_") schematic_memid = ( self.memory.convert_block_object_to_schematic(mem.memid).memid if isinstance(mem, VoxelObjectNode) and len(mem.blocks) > 0 else None ) for t in self.logical_form["upsert"]["memory_data"].get("triples", []): if t.get("pred_text") and t.get("obj_text"): logging.debug("Tagging {} {} {}".format(mem.memid, t["pred_text"], t["obj_text"])) self.memory.add_triple( subj=mem.memid, pred_text=t["pred_text"], obj_text=t["obj_text"] ) if schematic_memid: self.memory.add_triple( subj=schematic_memid, pred_text=t["pred_text"], obj_text=t["obj_text"] ) point_at_target = mem.get_point_at_target() # FIXME agent : This is the only place in file using the agent from the .step() task = self.task_objects["point"](agent, {"target": point_at_target}) # FIXME? higher pri, make sure this runs now...? TaskNode(self.memory, task.memid) r = "OK I'm tagging this %r as %r %r " % (name, t["pred_text"], t["obj_text"]) Say(agent, task_data={"response_options": r}) return
def do_answer(self, agent, mems: Sequence[Any], vals: Sequence[Any]) -> Tuple[Optional[str], Any]: """This function uses the action dictionary and memory state to return an answer. Args: mems: Sequence of memories vals: Sequence of values Returns: output_chat: An optional string for when the agent wants to send a chat step_data: Any other data that this step would like to send to the task """ self.finished = True # noqa output_type = self.output_type try: if type(output_type) is str and output_type.lower() == "count": # FIXME will multiple count if getting tags if not any(vals): Say(agent, task_data={"response_options": "none"}) Say(agent, task_data={"response_options": str(vals[0])}) elif type(output_type) is dict and output_type.get("attribute"): attrib = output_type["attribute"] if type(attrib) is str and attrib.lower() == "location": # add a Point task if attribute is a location if self.subinterpret.get( "point_target") and self.task_objects.get("point"): target = self.subinterpret[ "point_target"].point_to_region(vals[0]) # FIXME agent : This is the only place in file using the agent from the .step() t = self.task_objects["point"](agent, { "target": target }) # FIXME? higher pri, make sure this runs now...? TaskNode(self.memory, t.memid) Say(agent, task_data={"response_options": str(vals)}) elif type(output_type) is str and output_type.lower() == "memory": self.handle_exists(mems) else: raise ValueError("Bad answer_type={}".format(output_type)) except IndexError: # index error indicates no answer available logging.error("No answer available from do_answer") raise ErrorWithResponse("I don't understand what you're asking") except Exception as e: logging.exception(e)
def handle_thicken(interpreter, speaker, modify_dict, obj): old_blocks = list(obj.blocks.items()) bounds = droidlet.base_util.get_bounds() mx, my, mz = (bounds[0], bounds[2], bounds[4]) origin = [mx, my, mz] if modify_dict.get("modify_type") == "THICKER": num_blocks = modify_dict.get("num_blocks", 1) new_blocks = thicker(old_blocks, delta=num_blocks) else: raise ErrorWithResponse("I don't know how thin out blocks yet") destroy_task_data = {"schematic": old_blocks} # FIXME deal with tags!!! build_task_data = { "blocks_list": new_blocks, "origin": origin, # "schematic_tags": tags, } return destroy_task_data, build_task_data
def __call__(self, interpreter, speaker, d): if d.get("location") is None: # TODO other facings raise ErrorWithResponse("I am not sure where you want me to point") # TODO: We might want to specifically check for BETWEEN/INSIDE, I'm not sure # what the +1s are in the return value mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, d["location"]) steps, reldir = interpret_relative_direction(interpreter, d) # if directly point at a reference object, call built-in fn to get pointed target if steps is None and reldir is None: loc = mems[0].get_point_at_target() else: loc, _ = interpreter.subinterpret["specify_locations"](interpreter, speaker, mems, steps, reldir) return self.point_to_region(loc)
def handle_set(self, agent) -> Tuple[Optional[str], Any]: """creates a set of memories Returns: output_chat: An optional string for when the agent wants to send a chat step_data: Any other data that this step would like to send to the task """ ref_obj_d = {"filters": self.logical_form["filters"]} ref_objs = self.subinterpret["reference_objects"]( self, self.speaker, ref_obj_d, extra_tags=["_physical_object"] ) if len(ref_objs) == 0: raise ErrorWithResponse("I don't know what you're referring to") triples_d = self.logical_form["upsert"]["memory_data"].get("triples") if len(triples_d) == 1 and triples_d[0]["pred_text"] == "has_name": # the set has a name; check to see if one with that name exists, # if so add to it, else create one with that name name = triples_d[0]["obj_text"] set_memids, _ = self.memory.basic_search( "SELECT MEMORY FROM Set WHERE (has_name={} OR name={})".format(name, name) ) if not set_memids: # make a new set, and name it set_memid = SetNode.create(self.memory) self.memory.add_triple(subj=set_memid, pred_text="has_name", obj_text=name) else: # FIXME, which one set_memid = set_memids[0] else: # an anonymous set, assuming its new, and defined to hold the triple(s) set_memid = SetNode.create(self.memory) for t in triples_d: self.memory.add_triple( subj=set_memid, pred_text=t["pred_text"], obj_text=t["obj_text"] ) for r in ref_objs: self.memory.add_triple(subj=r.memid, pred_text="member_of", obj=set_memid) # FIXME point to the objects put in the set, otherwise explain this better Say(agent, task_data={"response_options": "OK made those objects into a set "}) return
def __init__( self, agent, ref_object=None, relative_direction="CLOCKWISE", # this is the memory of the object ): self.agent = agent self.tick = 0 if not ref_object: x, y, z = agent.pos bounds = (x, x, y, y, z, z) center = (x, y, z) else: bounds = ref_object.get_bounds() center = ref_object.get_pos() d = max(bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4]) if relative_direction == "CLOCKWISE" or relative_direction == "AROUND": offsets = droidlet.shared_data_struct.craftassist_shared_utils.arrange( "circle", schematic=None, shapeparams={"encircled_object_radius": d}) elif relative_direction == "ANTICLOCKWISE": offsets = droidlet.shared_data_struct.craftassist_shared_utils.arrange( "circle", schematic=None, shapeparams={"encircled_object_radius": d}) offsets = offsets[::-1] else: raise NotImplementedError("TODO other kinds of paths") self.path = [np.round(np.add(center, o)) for o in offsets] self.path.append(self.path[0]) # check each offset to find a nearby reachable point, see if a path # is possible now, and error otherwise for i in range(len(self.path) - 1): path = astar(agent, self.path[i + 1], approx=2, pos=self.path[i]) if path is None: raise ErrorWithResponse("I cannot find an appropriate path.")
def handle_undo(self, agent, speaker, d) -> Tuple[Optional[str], Any]: Undo = self.task_objects["undo"] task_name = d.get("undo_action") if task_name: task_name = task_name.split("_")[0].strip() old_task = self.memory.get_last_finished_root_task(task_name) if old_task is None: raise ErrorWithResponse("Nothing to be undone ...") undo_tasks = [Undo(agent, {"memid": old_task.memid})] for u in undo_tasks: agent.memory.get_mem_by_id(u.memid).get_update_status( {"paused": 1}) undo_command = old_task.get_chat().chat_text logging.debug("Pushing ConfirmTask tasks={}".format(undo_tasks)) confirm_data = { "task_memids": [u.memid for u in undo_tasks], "question": 'Do you want me to undo the command: "{}" ?'.format(undo_command), } ConfirmTask(agent, confirm_data) self.finished = True return None, None
def new_tasks(): # TODO if we do this better will be able to handle "stay between the x" default_loc = getattr(self, "default_loc", SPEAKERLOOK) location_d = d.get("location", default_loc) # FIXME, this is hacky. need more careful way of storing this in task if loop_mem: mems = [loop_mem] else: mems = self.subinterpret["reference_locations"](self, speaker, location_d) # FIXME this should go in the ref_location subinterpret: steps, reldir = interpret_relative_direction(self, location_d) pos, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir) # TODO: can this actually happen? if pos is None: raise ErrorWithResponse( "I don't understand where you want me to move.") pos = self.post_process_loc(pos, self) task_data = {"target": pos, "action_dict": d} task = Move(agent, task_data) return task
def new_tasks(): # only go around the x has "around"; FIXME allow other kinds of dances location_d = d.get("location") if location_d is not None: rd = location_d.get("relative_direction") if rd is not None and (rd == "AROUND" or rd == "CLOCKWISE" or rd == "ANTICLOCKWISE"): ref_obj = None location_reference_object = location_d.get( "reference_object") if location_reference_object: objmems = self.subinterpret["reference_objects"]( self, speaker, location_reference_object) if len(objmems) == 0: raise ErrorWithResponse( "I don't understand where you want me to go.") ref_obj = objmems[0] refmove = dance.RefObjMovement( agent, ref_object=ref_obj, relative_direction=location_d["relative_direction"], ) t = self.task_objects["dance"](agent, { "movement": refmove }) return t dance_type = d.get("dance_type", {}) if dance_type.get("point"): target = self.subinterpret["point_target"](self, speaker, dance_type["point"]) t = self.task_objects["point"](agent, {"target": target}) # MC bot does not control body turn separate from head elif dance_type.get("look_turn") or dance_type.get("body_turn"): lt = dance_type.get("look_turn") or dance_type.get("body_turn") f = self.subinterpret["facing"](self, speaker, lt) t = self.task_objects["dancemove"](agent, f) else: if location_d is None: dance_location = None else: mems = self.subinterpret["reference_locations"](self, speaker, location_d) steps, reldir = interpret_relative_direction( self, location_d) dance_location, _ = self.subinterpret["specify_locations"]( self, speaker, mems, steps, reldir) filters_d = dance_type.get("filters", {}) filters_d["memory_type"] = "DANCES" F = self.subinterpret["filters"](self, speaker, dance_type.get("filters", {})) dance_memids, _ = F() # TODO correct selector in filters if dance_memids: dance_memid = random.choice(dance_memids) dance_mem = self.memory.get_mem_by_id(dance_memid) dance_obj = dance.Movement(agent=agent, move_fn=dance_mem.dance_fn, dance_location=dance_location) t = self.task_objects["dance"](agent, { "movement": dance_obj }) else: # dance out of scope raise ErrorWithResponse( "I don't know how to do that movement yet.") return t
def handle_fill(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]: """This function reads the dictionary, resolves the missing details using memory and perception and handles a 'fill' command by either pushing a dialogue object or pushing a Fill task to the task stack. Args: speaker: speaker_id or name. d: the complete action dictionary """ tasks = [] r = d.get("reference_object") if not r.get("filters"): r["filters"] = {"location", SPEAKERLOOK} # Get the reference location location_d = r["filters"].get("location", SPEAKERLOOK) mems = self.subinterpret["reference_locations"](self, speaker, location_d) steps, reldir = interpret_relative_direction(self, location_d) location, _ = self.subinterpret["specify_locations"](self, speaker, mems, steps, reldir) """ FIXME: We need to fix this and perhaps put this in reasoning. Agent should run perception and put into memory. Interpreter shouldn't be perceiving, but should be able to ask the agent to do it when needed. """ # Get nearby holes perception_holes = self.get_all_holes_fn( agent, location, self.block_data, agent.low_level_data["fill_idmeta"]) perception_output = CraftAssistPerceptionData(holes=perception_holes) output = self.memory.update(perception_output=perception_output) holes = output.get("holes", []) # Choose the best ones to fill holes = filter_by_sublocation(self, speaker, holes, r, loose=True) if holes is None: # FIXME: in stage III, replace agent with the lowlevel interface to sending chats raise ErrorWithResponse( "I don't understand what holes you want me to fill.") tasks = [] for hole in holes: poss = list(hole.blocks.keys()) try: fill_memid = agent.memory.get_triples( subj=hole.memid, pred_text="has_fill_type")[0][2] fill_block_mem = self.memory.get_mem_by_id(fill_memid) fill_idm = (fill_block_mem.b, fill_block_mem.m) except: # FIXME use a constant name fill_idm = (3, 0) schematic, tags = interpret_fill_schematic( self, speaker, d.get("schematic", {}), poss, fill_idm, self.block_data, self.color_bid_map, ) origin = np.min([xyz for (xyz, bid) in schematic], axis=0) task_data = { "blocks_list": schematic, "force": True, "origin": origin, "verbose": False, "embed": True, "fill_message": True, "schematic_tags": tags, } tasks.append(self.task_objects["build"](agent, task_data)) if len(holes) > 1: Say(agent, task_data={"response_options": "Ok. I'll fill up the holes."}) else: Say(agent, task_data={"response_options": "Ok. I'll fill that hole up."}) return maybe_task_list_to_control_block(tasks, agent), None, None
def handle_modify(self, agent, speaker, d) -> Tuple[Any, Optional[str], Any]: """This function reads the dictionary, resolves the missing details using memory and handles a 'modify' command by either replying back or pushing appropriate tasks to the task stack. Args: speaker: speaker_id or name. d: the complete action dictionary """ default_ref_d = {"filters": {"location": SPEAKERLOOK}} ref_d = d.get("reference_object", default_ref_d) # only modify blockobjects... objs = self.subinterpret["reference_objects"]( self, speaker, ref_d, extra_tags=["_physical_object", "VOXEL_OBJECT"]) if len(objs) == 0: raise ErrorWithResponse( "I don't understand what you want me to modify.") m_d = d.get("modify_dict") if not m_d: raise ErrorWithResponse( "I think you want me to modify an object but am not sure what to do" ) tasks = [] for obj in objs: if m_d["modify_type"] == "THINNER" or m_d[ "modify_type"] == "THICKER": destroy_task_data, build_task_data = handle_thicken( self, speaker, m_d, obj) elif m_d["modify_type"] == "REPLACE": destroy_task_data, build_task_data = handle_replace( self, speaker, m_d, obj, block_data=self.block_data, color_bid_map=self.color_bid_map, ) elif m_d["modify_type"] == "SCALE": destroy_task_data, build_task_data = handle_scale( self, speaker, m_d, obj) elif m_d["modify_type"] == "RIGIDMOTION": destroy_task_data, build_task_data = handle_rigidmotion( self, speaker, m_d, obj) elif m_d["modify_type"] == "FILL" or m_d["modify_type"] == "HOLLOW": destroy_task_data, build_task_data = handle_fill( self, speaker, m_d, obj, block_data=self.block_data, color_bid_map=self.color_bid_map, ) else: raise ErrorWithResponse( "I think you want me to modify an object but am not sure what to do (parse error)" ) if build_task_data: tasks.append(self.task_objects["build"](agent, build_task_data)) if destroy_task_data: tasks.append(self.task_objects["build"](agent, destroy_task_data)) return maybe_task_list_to_control_block(tasks, agent), None, None
def filter_by_sublocation(interpreter, speaker, candidates: List[T], d: Dict, all_proximity=10, loose=False) -> List[T]: """Select from a list of candidate reference_object mems given a sublocation also handles random sampling Returns a list of mems """ filters_d = d.get("filters") assert filters_d is not None, "no filters: {}".format(d) default_loc = getattr(interpreter, "default_loc", SPEAKERLOOK) location = filters_d.get("selector", {}).get("location", default_loc) reldir = location.get("relative_direction") distance_sorted = False location_filtered_candidates = [] if reldir: if reldir == "INSIDE": # FIXME formalize this better, make extensible if location.get("reference_object"): # should probably return from interpret_reference_location... ref_mems = interpret_reference_object( interpreter, speaker, location["reference_object"]) # FIXME !!! this should be more clearly delineated # between perception and memory I = getattr(interpreter.memory, "check_inside", None) if I: for candidate_mem in candidates: if I([candidate_mem, ref_mems[0]]): location_filtered_candidates.append(candidate_mem) else: raise ErrorWithResponse("I don't know how to check inside") if not location_filtered_candidates: raise ErrorWithResponse("I can't find something inside that") elif reldir == "AWAY": raise ErrorWithResponse("I don't know which object you mean") elif reldir == "NEAR": pass # fall back to no reference direction elif reldir == "BETWEEN": mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, location) steps, reldir = interpret_relative_direction(interpreter, d) ref_loc, _ = interpreter.subinterpret["specify_locations"]( interpreter, speaker, mems, steps, reldir) distance_sorted = True location_filtered_candidates = candidates location_filtered_candidates.sort( key=lambda c: euclid_dist(c.get_pos(), ref_loc)) else: # reference object location, i.e. the "X" in "left of X" mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, location) if not mems: raise ErrorWithResponse("I don't know which object you mean") # FIXME!!! handle frame better, might want agent's frame instead # FIXME use the subinterpreter, don't directly call the attribute eid = interpreter.memory.get_player_by_name(speaker).eid self_mem = interpreter.memory.get_mem_by_id( interpreter.memory.self_memid) L = LinearExtentAttribute(interpreter.memory, { "frame": eid, "relative_direction": reldir }, mem=self_mem) c_proj = L(candidates) m_proj = L(mems) # FIXME don't just take the first... m_proj = m_proj[0] # filter by relative dir, e.g. "left of Y" location_filtered_candidates = [ c for (p, c) in zip(c_proj, candidates) if p > m_proj ] # "the X left of Y" = the right-most X that is left of Y location_filtered_candidates.sort(key=lambda p: p.get_pos()) distance_sorted = True else: # no reference direction: sort by closest mems = interpreter.subinterpret["reference_locations"](interpreter, speaker, location) steps, reldir = interpret_relative_direction(interpreter, d) ref_loc, _ = interpreter.subinterpret["specify_locations"](interpreter, speaker, mems, steps, reldir) location_filtered_candidates = [ c for c in candidates if euclid_dist(c.get_pos(), ref_loc) <= all_proximity ] location_filtered_candidates.sort( key=lambda c: euclid_dist(c.get_pos(), ref_loc)) distance_sorted = True mems = location_filtered_candidates if location_filtered_candidates: # could be [], if so will return [] default_selector_d = {"return_quantity": "ALL"} # default_selector_d = {"location": {"location_type": "SPEAKER_LOOK"}} selector_d = filters_d.get("selector", default_selector_d) S = interpret_selector(interpreter, speaker, selector_d) if S: memids, _ = S( [c.memid for c in location_filtered_candidates], [None] * len(location_filtered_candidates), ) mems = [interpreter.memory.get_mem_by_id(m) for m in memids] else: pass # FIXME, warn/error here; mems is still the candidates return mems
def interpret_reference_object( interpreter, speaker, d, extra_tags=[], loose_speakerlook=False, allow_clarification=True, all_proximity=100, ) -> List[ReferenceObjectNode]: """this tries to find a ref obj memory matching the criteria from the ref_obj_dict args: interpreter: root interpreter. speaker (str): The name of the player/human/agent who uttered the chat resulting in this interpreter d: logical form from semantic parser extra_tags (list of strings): tags added by parent to narrow the search allow_clarification (bool): should a Clarification object be put on the DialogueStack """ filters_d = d.get("filters") special = d.get("special_reference") # filters_d can be empty... assert ( filters_d is not None ) or special, "no filters or special_reference sub-dicts {}".format(d) if special: mem = get_special_reference_object(interpreter, speaker, special) return [mem] if filters_d.get("contains_coreference", "NULL") != "NULL": mem = filters_d["contains_coreference"] if isinstance(mem, ReferenceObjectNode): update_attended_and_link_lf(interpreter, [mem]) return [mem] elif mem == "resolved": pass else: logging.error("bad coref_resolve -> {}".format(mem)) clarification_query = ( "SELECT MEMORY FROM Task WHERE reference_object_confirmation=#={}". format(interpreter.memid)) _, clarification_task_mems = interpreter.memory.basic_search( clarification_query) # does a clarification task referencing this interpreter exist? if not clarification_task_mems: mems = maybe_get_text_span_mems(interpreter, speaker, d) if mems: update_attended_and_link_lf(interpreter, mems) # No filter by sublocation etc if a mem matches the text_span exactly... return mems if any(extra_tags): extra_clauses = [] for tag in extra_tags: extra_clauses.append({"pred_text": "has_tag", "obj_text": tag}) if not filters_d.get("where_clause"): filters_d["where_clause"] = {"AND": []} if filters_d["where_clause"].get( "OR") or filters_d["where_clause"].get("NOT"): subclause = deepcopy(filters_d["where_clause"]) filters_d["where_clause"] = {"AND": [subclause]} filters_d["where_clause"]["AND"].extend(extra_clauses) # TODO Add ignore_player maybe? # FIXME! see above. currently removing selector to get candidates, and filtering after # instead of letting filter interpreters handle. filters_no_select = deepcopy(filters_d) filters_no_select.pop("selector", None) # filters_no_select.pop("location", None) candidate_mems = apply_memory_filters(interpreter, speaker, filters_no_select) if len(candidate_mems) > 0: mems = filter_by_sublocation( interpreter, speaker, candidate_mems, d, loose=loose_speakerlook, all_proximity=all_proximity, ) update_attended_and_link_lf(interpreter, mems) return mems elif allow_clarification: # no candidates found; ask Clarification confirm_candidates = apply_memory_filters(interpreter, speaker, filters_d) objects = object_looked_at(interpreter.memory, confirm_candidates, speaker=speaker) if len(objects) == 0: raise ErrorWithResponse( "I don't know what you're referring to") _, mem = objects[0] interpreter.memory.add_triple(subj=interpreter.memid, pred_text="provisional_refobj_memid", obj=mem.memid) task_egg = { "class": ConfirmReferenceObject, "task_data": { "reference_object": mem } } cmemid = TaskNode.create(interpreter.memory, task_egg) interpreter.memory.add_triple( subj=cmemid, pred_text="reference_object_confirmation", obj=self.memid) raise NextDialogueStep() else: raise ErrorWithResponse("I don't know what you're referring to") else: # there is a clarification task. is it active? task_mem = clarification_task_mems[ 0] # FIXME, error if there are many? if task_mem.prio > -2: raise NextDialogueStep() # clarification task finished. query = "SELECT dialogue_task_output FROM Task WHERE uuid={}".format( task_mem.memid) _, r = interpreter.memory.basic_search(query) if r and r[0] == "yes": # TODO: learn from the tag! put it in memory! query = "SELECT MEMORY FROM ReferenceObject WHERE << {}, reference_object_confirmation, ?>>".format( self.memid) _, ref_obj_mems = interpreter.memory.basic_search(query) update_attended_and_link_lf(interpreter, ref_obj_mems) return ref_obj_mems else: raise ErrorWithResponse("I don't know what you're referring to")