def compute_location_heuristic(player_look, player_pos, mems, steps, reldir): loc = mems[0].get_pos() if reldir is not None: steps = steps or DEFAULT_NUM_STEPS if reldir == "BETWEEN": loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2 loc = (loc[0], loc[1], loc[2]) elif reldir == "INSIDE": for i in range(len(mems)): mem = mems[i] locs = heuristic_perception.find_inside(mem) if len(locs) > 0: break if len(locs) == 0: raise ErrorWithResponse("I don't know how to go inside there") else: loc = locs[0] elif reldir == "NEAR": pass elif reldir == "AROUND": pass else: # LEFT, RIGHT, etc... reldir_vec = rotation.DIRECTIONS[reldir] # this should be an inverse transform so we set inverted=True dir_vec = rotation.transform(reldir_vec, player_look.yaw, 0, inverted=True) loc = steps * np.array(dir_vec) + to_block_center(loc) elif steps is not None: loc = to_block_center(loc) + [0, 0, steps] return to_block_pos(loc)
def object_looked_at( agent, candidates: Sequence[Tuple[XYZ, T]], player_struct, limit=1, max_distance=30, loose=False, ) -> List[Tuple[XYZ, T]]: """Return the object that `player` is looking at Args: - agent: agent object, for API access - candidates: list of (centroid, object) tuples - player_struct: player struct whose POV to use for calculation - limit: 'ALL' or int; max candidates to return - loose: if True, don't filter candaidates behind agent Returns: a list of (xyz, mem) tuples, max length `limit` """ if len(candidates) == 0: return [] pos = pos_to_np(player_struct.pos) yaw, pitch = player_struct.look.yaw, player_struct.look.pitch # append to each candidate its relative position to player, rotated to # player-centric coordinates candidates_ = [(p, obj, rotation.transform(p - pos, yaw, pitch)) for (p, obj) in candidates] FRONT = rotation.DIRECTIONS["FRONT"] LEFT = rotation.DIRECTIONS["LEFT"] UP = rotation.DIRECTIONS["UP"] # reject objects behind player or not in cone of sight (but always include # an object if it's directly looked at) xsect = tuple(capped_line_of_sight(agent, player_struct, 25)) if not loose: candidates_ = [(p, o, r) for (p, o, r) in candidates_ if xsect in getattr(o, "blocks", {}) or r @ FRONT > ( (r @ LEFT)**2 + (r @ UP)**2)**0.5] # if looking directly at an object, sort by proximity to look intersection if euclid_dist(pos, xsect) <= 25: candidates_.sort(key=lambda c: euclid_dist(c[0], xsect)) else: # otherwise, sort by closest to look vector candidates_.sort( key=lambda c: ((c[2] @ LEFT)**2 + (c[2] @ UP)**2)**0.5) # linit returns of things too far away candidates_ = [ c for c in candidates_ if euclid_dist(pos, c[0]) < max_distance ] # limit number of returns if limit == "ALL": limit = len(candidates_) return [(p, o) for (p, o, r) in candidates_[:limit]]
def assert_move(self, reldir, steps, changes): old_pos = changes[0]["agent"]["pos"] new_pos = changes[1]["agent"]["pos"] start_base_yaw = changes[0]["agent"]["base_yaw"] reldir_vec = rotation.DIRECTIONS[reldir] dir_vec = rotation.transform(reldir_vec, start_base_yaw, 0, inverted=True) dir_vec = np.array([dir_vec[0], dir_vec[2]], dtype="float32") tocheck_pos = np.around(old_pos + steps * dir_vec, 2) self.assertEqual(new_pos[0], tocheck_pos[0]) self.assertEqual(new_pos[1], tocheck_pos[1])
def handle_replace(interpreter, speaker, modify_dict, obj): old_blocks = list(obj.blocks.items()) bounds = obj.get_bounds() mx, my, mz = (bounds[0], bounds[2], bounds[4]) origin = (mx, my, mz) new_block_type = get_block_type(modify_dict["new_block"]) destroy_task_data = None if modify_dict.get("old_block"): # TODO FILTERS, also in build # TODO "make the red blocks green" etc- currently get_block type does not return a list of possibilities old_block_type = get_block_type(modify_dict["old_block"]) new_blocks = replace_by_blocktype( old_blocks, new_idm=new_block_type, current_idm=old_block_type ) else: geom_d = modify_dict.get("replace_geometry") geometry = {} schematic = maybe_convert_to_npy(old_blocks) geometry["offset"] = np.array(schematic.shape[:3]) / 2 reldir = geom_d.get("relative_direction", "TOP") if reldir == "TOP": reldir = "UP" elif reldir == "BOTTOM": reldir = "DOWN" reldir_vec = rotation.DIRECTIONS[reldir] look = ( interpreter.agent.perception_modules["low_level"] .get_player_struct_by_name(speaker) .look ) dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True) geometry["v"] = dir_vec projections = [] for l, idm in old_blocks: projections.append((np.array(l) - geometry["offset"]) @ reldir_vec) a = geom_d.get("amount", "HALF") if a == "QUARTER": geometry["threshold"] = (np.max(projections) - np.min(projections)) / 4 else: geometry["threshold"] = 0.0 new_blocks = replace_by_halfspace(old_blocks, new_idm=new_block_type, geometry=geometry) # FIXME deal with tags!!! build_task_data = { "blocks_list": maybe_convert_to_list(new_blocks), "origin": origin, # "schematic_tags": tags, } return destroy_task_data, build_task_data
def compute_location_heuristic(interpreter, speaker, d, mems): # handle relative direction reldir = d.get("relative_direction") loc = mems[0].get_pos() if reldir is not None: if reldir == "BETWEEN": loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2 loc = (loc[0], loc[1], loc[2]) elif reldir == "INSIDE": ref_obj_dict = d.get("reference_object", SPEAKERLOOK["reference_object"]) special = ref_obj_dict.get("special_reference") if not special: for i in range(len(mems)): mem = mems[i] locs = heuristic_perception.find_inside(mem) if len(locs) > 0: break if len(locs) == 0: raise ErrorWithResponse("I don't know how to go inside there") else: interpreter.memory.update_recent_entities([mem]) loc = locs[0] else: raise ErrorWithResponse("I don't know how to go inside there") elif reldir == "AWAY": apos = pos_to_np(interpreter.agent.get_player().pos) dir_vec = (apos - loc) / np.linalg.norm(apos - loc) num_steps = word_to_num(d.get("steps", "5")) loc = num_steps * np.array(dir_vec) + to_block_center(loc) elif reldir == "NEAR": pass else: # LEFT, RIGHT, etc... reldir_vec = rotation.DIRECTIONS[reldir] look = ( interpreter.agent.perception_modules["low_level"] .get_player_struct_by_name(speaker) .look ) # this should be an inverse transform so we set inverted=True dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True) num_steps = word_to_num(d.get("steps", "5")) loc = num_steps * np.array(dir_vec) + to_block_center(loc) # if steps without relative direction elif "steps" in d: num_steps = word_to_num(d.get("steps", "5")) loc = to_block_center(loc) + [0, 0, num_steps] return post_process_loc(loc, interpreter)
def get_repeat_arrangement(d, interpreter, speaker, schematic, repeat_num=-1, extra_space=1) -> List[XYZ]: shapeparams = {} # eventually fix this to allow number based on shape shapeparams["N"] = repeat_num shapeparams["extra_space"] = extra_space if "repeat" in d: direction_name = d.get("repeat", {}).get("repeat_dir", "FRONT") elif "schematic" in d: direction_name = d["schematic"].get("repeat", {}).get("repeat_dir", "FRONT") if direction_name != "AROUND": reldir_vec = rotation.DIRECTIONS[direction_name] look = (interpreter.agent.perception_modules["low_level"]. get_player_struct_by_name(speaker).look) # this should be an inverse transform so we set inverted=True dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True) shapeparams["orient"] = dir_vec offsets = shapes.arrange("line", schematic, shapeparams) else: # TODO vertical "around" shapeparams["orient"] = "xy" shapeparams["encircled_object_radius"] = 1 if d.get("location") is not None: central_object = interpret_reference_object( interpreter, speaker, d["location"]["reference_object"], limit=1) # FIXME: .blocks is unsafe, assumes BlockObject only object could be Mob. Ignoring for now. central_object_blocks = central_object[0].blocks # type: ignore # .blocks returns a dict of (x, y, z) : (block_id, meta), convert to list # to get bounds central_object_list = [ tuple([k, v]) for k, v in central_object_blocks.items() ] bounds = shapes.get_bounds(central_object_list) b = max(bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4]) shapeparams["encircled_object_radius"] = b offsets = shapes.arrange("circle", schematic, shapeparams) offsets = [tuple(to_block_pos(o)) for o in offsets] return offsets
def get_repeat_arrangement(player_look, repeat_num, repeat_dir, ref_mems, schematic=None, padding=(1, 1, 1)): shapeparams = {} # default repeat dir is LEFT if not repeat_dir: repeat_dir = "LEFT" # eventually fix this to allow number based on shape shapeparams["N"] = repeat_num if repeat_dir == "AROUND": # TODO vertical "around" shapeparams["orient"] = "xy" shapeparams["extra_space"] = max(padding) central_object = ref_mems[0] bounds = central_object.get_bounds() b = max(bounds[1] - bounds[0], bounds[3] - bounds[2], bounds[5] - bounds[4]) shapeparams["encircled_object_radius"] = b offsets = shapes.arrange("circle", schematic, shapeparams) else: reldir_vec = rotation.DIRECTIONS[repeat_dir] # this should be an inverse transform so we set inverted=True dir_vec = rotation.transform(reldir_vec, player_look.yaw, 0, inverted=True) max_ind = np.argmax(dir_vec) shapeparams["extra_space"] = padding[max_ind] shapeparams["orient"] = dir_vec offsets = shapes.arrange("line", schematic, shapeparams) offsets = [tuple(to_block_pos(o)) for o in offsets] return offsets
def filter_by_sublocation( interpreter, speaker, candidates: List[Tuple[XYZ, T]], location: Dict, limit=1, all_proximity=10, loose=False, ) -> List[Tuple[XYZ, T]]: """Select from a list of candidate (xyz, object) tuples given a sublocation If limit == 'ALL', return all matching candidates Returns a list of (xyz, mem) tuples """ # handle SPEAKER_LOOK separately due to slightly different semantics # (proximity to ray instead of point) if location.get("location_type") == "SPEAKER_LOOK": player = interpreter.memory.get_player_struct_by_name(speaker) return object_looked_at(interpreter.agent, candidates, player, limit=limit, loose=loose) reldir = location.get("relative_direction") if reldir: if reldir == "INSIDE": if location.get("reference_object"): # this is ugly, should probably return from interpret_location... ref_mems = interpret_reference_object( interpreter, speaker, location["reference_object"]) for l, candidate_mem in candidates: if perception.check_inside([candidate_mem, ref_mems[0]]): return [(l, candidate_mem)] raise ErrorWithResponse("I can't find something inside that") elif reldir == "AWAY": raise ErrorWithResponse("I don't know which object you mean") elif reldir == "NEAR": pass # fall back to no reference direction else: # reference object location, i.e. the "X" in "left of X" ref_loc = interpret_location(interpreter, speaker, location, ignore_reldir=True) # relative direction, i.e. the "LEFT" in "left of X" reldir_vec = rotation.DIRECTIONS[reldir] # transform each object into the speaker look coordinate system, # and project onto the reldir vector look = interpreter.memory.get_player_struct_by_name(speaker).look proj = [ rotation.transform(np.array(l) - ref_loc, look.yaw, 0) @ reldir_vec for (l, _) in candidates ] # filter by relative dir, e.g. "left of Y" proj_cands = [(p, c) for (p, c) in zip(proj, candidates) if p > 0] # "the X left of Y" = the right-most X that is left of Y if limit == "ALL": limit = len(proj_cands) return [c for (_, c) in sorted(proj_cands, key=lambda p: p[0]) ][:limit] else: # no reference direction: choose the closest if limit == "ALL": return list( filter(lambda c: euclid_dist(c[0], ref_loc) <= all_proximity, candidates)) else: candidates.sort(key=lambda c: euclid_dist(c[0], ref_loc)) return candidates[:limit] return [] # this fixes flake but seems awful?
def interpret_location(interpreter, speaker, d, ignore_reldir=False) -> XYZ: """Location dict -> coordinates Side effect: adds mems to agent_memory.recent_entities if a reference object is interpreted; and loc to memory """ location_type = d.get("location_type", "SPEAKER_LOOK") if location_type == "REFERENCE_OBJECT": mems = interpret_reference_object(interpreter, speaker, d["reference_object"]) if len(mems) == 0: raise ErrorWithResponse("I don't know what you're referring to") assert len(mems) == 1, mems interpreter.memory.update_recent_entities(mems) mem = mems[0] loc = mem.get_pos() elif location_type == "SPEAKER_LOOK": player = interpreter.memory.get_player_struct_by_name(speaker) loc = capped_line_of_sight(interpreter.agent, player) elif location_type == "SPEAKER_POS": loc = pos_to_np( interpreter.memory.get_player_struct_by_name(speaker).pos) elif location_type == "AGENT_POS": loc = pos_to_np(interpreter.agent.get_player().pos) elif location_type == "COORDINATES": loc = cast( XYZ, tuple( int(float(w)) for w in re.findall("[-0-9.]+", d["coordinates"]))) if len(loc) != 3: logging.error("Bad coordinates: {}".format(d["coordinates"])) raise ErrorWithResponse( "I don't understand what location you're referring to") else: raise ValueError( "Can't handle Location type: {}".format(location_type)) # handle relative direction reldir = d.get("relative_direction") if reldir is not None and not ignore_reldir: if reldir == "INSIDE": if location_type == "REFERENCE_OBJECT": locs = perception.find_inside(mem) if len(locs) == 0: raise ErrorWithResponse( "I don't know how to go inside there") else: loc = locs[0] elif reldir == "AWAY": apos = pos_to_np(interpreter.agent.get_player().pos) dir_vec = (apos - loc) / np.linalg.norm(apos - loc) num_steps = word_to_num(d.get("steps", "5")) loc = num_steps * np.array(dir_vec) + to_block_center(loc) elif reldir == "NEAR": pass else: # LEFT, RIGHT, etc... reldir_vec = rotation.DIRECTIONS[reldir] look = interpreter.memory.get_player_struct_by_name(speaker).look # this should be an inverse transform so we set inverted=True dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True) num_steps = word_to_num(d.get("steps", "5")) loc = num_steps * np.array(dir_vec) + to_block_center(loc) # if steps without relative direction elif "steps" in d: num_steps = word_to_num(d.get("steps", "5")) loc = to_block_center(loc) + [0, 0, num_steps] return to_block_pos(loc)