def compute_location_heuristic(player_look, player_pos, mems, steps, reldir):
loc = mems[0].get_pos()
if reldir is not None:
steps = steps or DEFAULT_NUM_STEPS
if reldir == "BETWEEN":
loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2
loc = (loc[0], loc[1], loc[2])
elif reldir == "INSIDE":
for i in range(len(mems)):
mem = mems[i]
locs = heuristic_perception.find_inside(mem)
if len(locs) > 0:
break
if len(locs) == 0:
raise ErrorWithResponse("I don't know how to go inside there")
else:
loc = locs[0]
elif reldir == "NEAR":
pass
elif reldir == "AROUND":
pass
else: # LEFT, RIGHT, etc...
reldir_vec = rotation.DIRECTIONS[reldir]
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec,
player_look.yaw,
0,
inverted=True)
loc = steps * np.array(dir_vec) + to_block_center(loc)
elif steps is not None:
loc = to_block_center(loc) + [0, 0, steps]
return to_block_pos(loc)
def object_looked_at(
agent,
candidates: Sequence[Tuple[XYZ, T]],
player_struct,
limit=1,
max_distance=30,
loose=False,
) -> List[Tuple[XYZ, T]]:
"""Return the object that `player` is looking at
Args:
- agent: agent object, for API access
- candidates: list of (centroid, object) tuples
- player_struct: player struct whose POV to use for calculation
- limit: 'ALL' or int; max candidates to return
- loose: if True, don't filter candaidates behind agent
Returns: a list of (xyz, mem) tuples, max length `limit`
"""
if len(candidates) == 0:
return []
pos = pos_to_np(player_struct.pos)
yaw, pitch = player_struct.look.yaw, player_struct.look.pitch
# append to each candidate its relative position to player, rotated to
# player-centric coordinates
candidates_ = [(p, obj, rotation.transform(p - pos, yaw, pitch))
for (p, obj) in candidates]
FRONT = rotation.DIRECTIONS["FRONT"]
LEFT = rotation.DIRECTIONS["LEFT"]
UP = rotation.DIRECTIONS["UP"]
# reject objects behind player or not in cone of sight (but always include
# an object if it's directly looked at)
xsect = tuple(capped_line_of_sight(agent, player_struct, 25))
if not loose:
candidates_ = [(p, o, r) for (p, o, r) in candidates_
if xsect in getattr(o, "blocks", {}) or r @ FRONT > (
(r @ LEFT)**2 + (r @ UP)**2)**0.5]
# if looking directly at an object, sort by proximity to look intersection
if euclid_dist(pos, xsect) <= 25:
candidates_.sort(key=lambda c: euclid_dist(c[0], xsect))
else:
# otherwise, sort by closest to look vector
candidates_.sort(
key=lambda c: ((c[2] @ LEFT)**2 + (c[2] @ UP)**2)**0.5)
# linit returns of things too far away
candidates_ = [
c for c in candidates_ if euclid_dist(pos, c[0]) < max_distance
]
# limit number of returns
if limit == "ALL":
limit = len(candidates_)
return [(p, o) for (p, o, r) in candidates_[:limit]]
def assert_move(self, reldir, steps, changes):
old_pos = changes[0]["agent"]["pos"]
new_pos = changes[1]["agent"]["pos"]
start_base_yaw = changes[0]["agent"]["base_yaw"]
reldir_vec = rotation.DIRECTIONS[reldir]
dir_vec = rotation.transform(reldir_vec,
start_base_yaw,
0,
inverted=True)
dir_vec = np.array([dir_vec[0], dir_vec[2]], dtype="float32")
tocheck_pos = np.around(old_pos + steps * dir_vec, 2)
self.assertEqual(new_pos[0], tocheck_pos[0])
self.assertEqual(new_pos[1], tocheck_pos[1])
def handle_replace(interpreter, speaker, modify_dict, obj):
old_blocks = list(obj.blocks.items())
bounds = obj.get_bounds()
mx, my, mz = (bounds[0], bounds[2], bounds[4])
origin = (mx, my, mz)
new_block_type = get_block_type(modify_dict["new_block"])
destroy_task_data = None
if modify_dict.get("old_block"):
# TODO FILTERS, also in build
# TODO "make the red blocks green" etc- currently get_block type does not return a list of possibilities
old_block_type = get_block_type(modify_dict["old_block"])
new_blocks = replace_by_blocktype(
old_blocks, new_idm=new_block_type, current_idm=old_block_type
)
else:
geom_d = modify_dict.get("replace_geometry")
geometry = {}
schematic = maybe_convert_to_npy(old_blocks)
geometry["offset"] = np.array(schematic.shape[:3]) / 2
reldir = geom_d.get("relative_direction", "TOP")
if reldir == "TOP":
reldir = "UP"
elif reldir == "BOTTOM":
reldir = "DOWN"
reldir_vec = rotation.DIRECTIONS[reldir]
look = (
interpreter.agent.perception_modules["low_level"]
.get_player_struct_by_name(speaker)
.look
)
dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True)
geometry["v"] = dir_vec
projections = []
for l, idm in old_blocks:
projections.append((np.array(l) - geometry["offset"]) @ reldir_vec)
a = geom_d.get("amount", "HALF")
if a == "QUARTER":
geometry["threshold"] = (np.max(projections) - np.min(projections)) / 4
else:
geometry["threshold"] = 0.0
new_blocks = replace_by_halfspace(old_blocks, new_idm=new_block_type, geometry=geometry)
# FIXME deal with tags!!!
build_task_data = {
"blocks_list": maybe_convert_to_list(new_blocks),
"origin": origin,
# "schematic_tags": tags,
}
return destroy_task_data, build_task_data
def compute_location_heuristic(interpreter, speaker, d, mems):
# handle relative direction
reldir = d.get("relative_direction")
loc = mems[0].get_pos()
if reldir is not None:
if reldir == "BETWEEN":
loc = (np.add(mems[0].get_pos(), mems[1].get_pos())) / 2
loc = (loc[0], loc[1], loc[2])
elif reldir == "INSIDE":
ref_obj_dict = d.get("reference_object", SPEAKERLOOK["reference_object"])
special = ref_obj_dict.get("special_reference")
if not special:
for i in range(len(mems)):
mem = mems[i]
locs = heuristic_perception.find_inside(mem)
if len(locs) > 0:
break
if len(locs) == 0:
raise ErrorWithResponse("I don't know how to go inside there")
else:
interpreter.memory.update_recent_entities([mem])
loc = locs[0]
else:
raise ErrorWithResponse("I don't know how to go inside there")
elif reldir == "AWAY":
apos = pos_to_np(interpreter.agent.get_player().pos)
dir_vec = (apos - loc) / np.linalg.norm(apos - loc)
num_steps = word_to_num(d.get("steps", "5"))
loc = num_steps * np.array(dir_vec) + to_block_center(loc)
elif reldir == "NEAR":
pass
else: # LEFT, RIGHT, etc...
reldir_vec = rotation.DIRECTIONS[reldir]
look = (
interpreter.agent.perception_modules["low_level"]
.get_player_struct_by_name(speaker)
.look
)
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True)
num_steps = word_to_num(d.get("steps", "5"))
loc = num_steps * np.array(dir_vec) + to_block_center(loc)
# if steps without relative direction
elif "steps" in d:
num_steps = word_to_num(d.get("steps", "5"))
loc = to_block_center(loc) + [0, 0, num_steps]
return post_process_loc(loc, interpreter)
def get_repeat_arrangement(d,
interpreter,
speaker,
schematic,
repeat_num=-1,
extra_space=1) -> List[XYZ]:
shapeparams = {}
# eventually fix this to allow number based on shape
shapeparams["N"] = repeat_num
shapeparams["extra_space"] = extra_space
if "repeat" in d:
direction_name = d.get("repeat", {}).get("repeat_dir", "FRONT")
elif "schematic" in d:
direction_name = d["schematic"].get("repeat",
{}).get("repeat_dir", "FRONT")
if direction_name != "AROUND":
reldir_vec = rotation.DIRECTIONS[direction_name]
look = (interpreter.agent.perception_modules["low_level"].
get_player_struct_by_name(speaker).look)
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec, look.yaw, 0, inverted=True)
shapeparams["orient"] = dir_vec
offsets = shapes.arrange("line", schematic, shapeparams)
else:
# TODO vertical "around"
shapeparams["orient"] = "xy"
shapeparams["encircled_object_radius"] = 1
if d.get("location") is not None:
central_object = interpret_reference_object(
interpreter,
speaker,
d["location"]["reference_object"],
limit=1)
# FIXME: .blocks is unsafe, assumes BlockObject only object could be Mob. Ignoring for now.
central_object_blocks = central_object[0].blocks # type: ignore
# .blocks returns a dict of (x, y, z) : (block_id, meta), convert to list
# to get bounds
central_object_list = [
tuple([k, v]) for k, v in central_object_blocks.items()
]
bounds = shapes.get_bounds(central_object_list)
b = max(bounds[1] - bounds[0], bounds[3] - bounds[2],
bounds[5] - bounds[4])
shapeparams["encircled_object_radius"] = b
offsets = shapes.arrange("circle", schematic, shapeparams)
offsets = [tuple(to_block_pos(o)) for o in offsets]
return offsets
def get_repeat_arrangement(player_look,
repeat_num,
repeat_dir,
ref_mems,
schematic=None,
padding=(1, 1, 1)):
shapeparams = {}
# default repeat dir is LEFT
if not repeat_dir:
repeat_dir = "LEFT"
# eventually fix this to allow number based on shape
shapeparams["N"] = repeat_num
if repeat_dir == "AROUND":
# TODO vertical "around"
shapeparams["orient"] = "xy"
shapeparams["extra_space"] = max(padding)
central_object = ref_mems[0]
bounds = central_object.get_bounds()
b = max(bounds[1] - bounds[0], bounds[3] - bounds[2],
bounds[5] - bounds[4])
shapeparams["encircled_object_radius"] = b
offsets = shapes.arrange("circle", schematic, shapeparams)
else:
reldir_vec = rotation.DIRECTIONS[repeat_dir]
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec,
player_look.yaw,
0,
inverted=True)
max_ind = np.argmax(dir_vec)
shapeparams["extra_space"] = padding[max_ind]
shapeparams["orient"] = dir_vec
offsets = shapes.arrange("line", schematic, shapeparams)
offsets = [tuple(to_block_pos(o)) for o in offsets]
return offsets
def filter_by_sublocation(
interpreter,
speaker,
candidates: List[Tuple[XYZ, T]],
location: Dict,
limit=1,
all_proximity=10,
loose=False,
) -> List[Tuple[XYZ, T]]:
"""Select from a list of candidate (xyz, object) tuples given a sublocation
If limit == 'ALL', return all matching candidates
Returns a list of (xyz, mem) tuples
"""
# handle SPEAKER_LOOK separately due to slightly different semantics
# (proximity to ray instead of point)
if location.get("location_type") == "SPEAKER_LOOK":
player = interpreter.memory.get_player_struct_by_name(speaker)
return object_looked_at(interpreter.agent,
candidates,
player,
limit=limit,
loose=loose)
reldir = location.get("relative_direction")
if reldir:
if reldir == "INSIDE":
if location.get("reference_object"):
# this is ugly, should probably return from interpret_location...
ref_mems = interpret_reference_object(
interpreter, speaker, location["reference_object"])
for l, candidate_mem in candidates:
if perception.check_inside([candidate_mem, ref_mems[0]]):
return [(l, candidate_mem)]
raise ErrorWithResponse("I can't find something inside that")
elif reldir == "AWAY":
raise ErrorWithResponse("I don't know which object you mean")
elif reldir == "NEAR":
pass # fall back to no reference direction
else:
# reference object location, i.e. the "X" in "left of X"
ref_loc = interpret_location(interpreter,
speaker,
location,
ignore_reldir=True)
# relative direction, i.e. the "LEFT" in "left of X"
reldir_vec = rotation.DIRECTIONS[reldir]
# transform each object into the speaker look coordinate system,
# and project onto the reldir vector
look = interpreter.memory.get_player_struct_by_name(speaker).look
proj = [
rotation.transform(np.array(l) - ref_loc, look.yaw, 0)
@ reldir_vec for (l, _) in candidates
]
# filter by relative dir, e.g. "left of Y"
proj_cands = [(p, c) for (p, c) in zip(proj, candidates) if p > 0]
# "the X left of Y" = the right-most X that is left of Y
if limit == "ALL":
limit = len(proj_cands)
return [c for (_, c) in sorted(proj_cands, key=lambda p: p[0])
][:limit]
else:
# no reference direction: choose the closest
if limit == "ALL":
return list(
filter(lambda c: euclid_dist(c[0], ref_loc) <= all_proximity,
candidates))
else:
candidates.sort(key=lambda c: euclid_dist(c[0], ref_loc))
return candidates[:limit]
return [] # this fixes flake but seems awful?
def interpret_location(interpreter, speaker, d, ignore_reldir=False) -> XYZ:
"""Location dict -> coordinates
Side effect: adds mems to agent_memory.recent_entities
if a reference object is interpreted;
and loc to memory
"""
location_type = d.get("location_type", "SPEAKER_LOOK")
if location_type == "REFERENCE_OBJECT":
mems = interpret_reference_object(interpreter, speaker,
d["reference_object"])
if len(mems) == 0:
raise ErrorWithResponse("I don't know what you're referring to")
assert len(mems) == 1, mems
interpreter.memory.update_recent_entities(mems)
mem = mems[0]
loc = mem.get_pos()
elif location_type == "SPEAKER_LOOK":
player = interpreter.memory.get_player_struct_by_name(speaker)
loc = capped_line_of_sight(interpreter.agent, player)
elif location_type == "SPEAKER_POS":
loc = pos_to_np(
interpreter.memory.get_player_struct_by_name(speaker).pos)
elif location_type == "AGENT_POS":
loc = pos_to_np(interpreter.agent.get_player().pos)
elif location_type == "COORDINATES":
loc = cast(
XYZ,
tuple(
int(float(w))
for w in re.findall("[-0-9.]+", d["coordinates"])))
if len(loc) != 3:
logging.error("Bad coordinates: {}".format(d["coordinates"]))
raise ErrorWithResponse(
"I don't understand what location you're referring to")
else:
raise ValueError(
"Can't handle Location type: {}".format(location_type))
# handle relative direction
reldir = d.get("relative_direction")
if reldir is not None and not ignore_reldir:
if reldir == "INSIDE":
if location_type == "REFERENCE_OBJECT":
locs = perception.find_inside(mem)
if len(locs) == 0:
raise ErrorWithResponse(
"I don't know how to go inside there")
else:
loc = locs[0]
elif reldir == "AWAY":
apos = pos_to_np(interpreter.agent.get_player().pos)
dir_vec = (apos - loc) / np.linalg.norm(apos - loc)
num_steps = word_to_num(d.get("steps", "5"))
loc = num_steps * np.array(dir_vec) + to_block_center(loc)
elif reldir == "NEAR":
pass
else: # LEFT, RIGHT, etc...
reldir_vec = rotation.DIRECTIONS[reldir]
look = interpreter.memory.get_player_struct_by_name(speaker).look
# this should be an inverse transform so we set inverted=True
dir_vec = rotation.transform(reldir_vec,
look.yaw,
0,
inverted=True)
num_steps = word_to_num(d.get("steps", "5"))
loc = num_steps * np.array(dir_vec) + to_block_center(loc)
# if steps without relative direction
elif "steps" in d:
num_steps = word_to_num(d.get("steps", "5"))
loc = to_block_center(loc) + [0, 0, num_steps]
return to_block_pos(loc)
