def dump_scene(scene_name, base_dir, renderObjectImage=False, renderDepthImage=False, renderClassImage=False):
controller = ai2thor.controller.Controller()
controller.start(player_screen_height=448, player_screen_width=448)
controller.reset(scene_name)
event = controller.step(dict(action='Initialize', fieldOfView=90, gridSize=0.25, renderDepthImage=renderDepthImage, renderObjectImage=renderObjectImage, renderClassImage=renderClassImage))
dump_scene_controller(base_dir, controller)
controller.stop()
def run(file_name=None):
# file_name = file_path.split('/')[-1].split('.')[0]
controller = ai2thor.controller.Controller()
controller.start()
controller.reset("FloorPlan203")
y_coord = 1.25
event = controller.step(
dict(action='Initialize',
gridSize=0.5,
cameraY=y_coord,
visibilityDistance=1.0))
all_visible_objects = list(
np.unique([obj['objectType'] for obj in event.metadata['objects']]))
rotation = 0.0
while True: # making a loop
try: # used try so that if user pressed other than the given key error will not be shown
key = click.getchar()
if key == 'a': # Rotate Left
rotation -= 22.5
if rotation < 0:
rotation = rotation + 360
event = controller.step(
dict(action='Rotate', rotation=rotation))
elif key == 'd':
rotation += 22.5
if rotation > 360:
rotation = rotation - 360
event = controller.step(
dict(action='Rotate', rotation=rotation))
elif key == 'w':
event = controller.step(dict(action='MoveAhead'))
elif key == 's':
event = controller.step(dict(action='MoveBack'))
elif key == 'z':
event = controller.step(dict(action='LookDown'))
elif key == 'x':
event = controller.step(dict(action='LookUp'))
elif key == 'q':
controller.stop()
break
elif key == 'r':
scene = input("Scene id: ")
controller.reset('FloorPlan{}'.format(scene))
event = controller.step(
dict(action='Initialize', gridSize=0.5, cameraY=y_coord))
else:
print("Key not supported! Try a, d, w, s, q, r.")
print((event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z'],
event.metadata['agent']['rotation']))
# print([(obj['objectType'], obj['distance']) for obj in event.metadata['objects'] if obj['visible']])
except:
print("Key not supported! Try a, d, w, s, q, r.")
def worker(steps_per_proc, sync_event, queue, gpu_id=0, actions=actions):
os.environ['DISPLAY'] = f":{gpu_id}"
controller = ai2thor.controller.Controller()
controller.start()
controller.step(dict(action='Initialize', gridSize=0.25))
print("Worker with pid:", os.getpid(), "is intialized")
np.random.seed(os.getpid())
#inform main process that intialization is successful
queue.put(1)
sync_event.wait()
count = len(actions)
for _ in range(steps_per_proc):
a = np.random.randint(count)
controller.step(dict(action=actions[a]))
queue.put(1)
print("Worker with pid:", os.getpid(), " funished job")
sync_event.clear()
sync_event.wait()
controller.stop()
def dump_scene(
scene_name,
base_dir,
renderInstanceSegmentation=False,
renderDepthImage=False,
renderSemanticSegmentation=False,
):
controller = ai2thor.controller.Controller()
controller.start(height=448, width=448)
controller.reset(scene_name)
event = controller.step(
dict(
action="Initialize",
fieldOfView=90,
gridSize=0.25,
renderDepthImage=renderDepthImage,
renderInstanceSegmentation=renderInstanceSegmentation,
renderSemanticSegmentation=renderSemanticSegmentation,
))
dump_scene_controller(base_dir, controller)
controller.stop()
def check_size(scene):
f = h5py.File("dumped/{}.hdf5".format(scene), "w")
locations = []
visible_objects = []
controller = ai2thor.controller.Controller()
controller.start()
controller.reset(scene)
controller.random_initialize(unique_object_types=True)
event = controller.step(dict(action='Initialize', gridSize=0.5))
y_coord = event.metadata['agent']['position']['y']
locations.append((event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z']))
# Using BFS to discover all reachable positions in current environment.
visited = set()
visited.add((event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z']))
while len(locations) > 0:
loc = locations.pop(0)
for act in ALL_POSSIBLE_ACTIONS:
controller.step(
dict(action='Teleport', x=loc[0], y=y_coord, z=loc[1]))
event = controller.step(dict(action=act))
if event.metadata['lastActionSuccess']:
new_loc = (event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z'])
if new_loc not in visited:
visited.add(new_loc)
locations.append(new_loc)
all_locs = list(visited)
controller.stop()
return len(all_locs)
def dump_feature(scene, cat2idx):
f = h5py.File("dumped/{}.hdf5".format(scene), "a")
states = f['locations'][()]
laser = f['lasers'][()]
dump_features = []
controller = ai2thor.controller.Controller()
controller.start()
controller.reset(scene)
event = controller.step(
dict(action='Initialize', gridSize=0.5, visibilityDistance=1000.0))
for i, state in enumerate(states):
event = controller.step(
dict(action='TeleportFull',
x=state[0],
y=1.25,
z=state[1],
rotation=state[2],
horizon=30))
visible = [obj for obj in event.metadata['objects'] if obj['visible']]
df = np.zeros(len(cat2idx) + 4)
df[-4:] = laser[i].tolist()
for obj in visible:
try:
obj_id = cat2idx[obj['objectType']]
df[obj_id] = obj['distance']
except:
print(obj['objectType'])
dump_features.append(df)
controller.stop()
f.create_dataset("dump_features",
data=np.asarray(dump_features, np.float32))
f.close()
def start_controller(args, ep1, obj_list, target_parents):
controller = ai2thor.controller.Controller()
controller.start(player_screen_height=500, player_screen_width=500)
controller.reset(ep1[0])
event = controller.step(
dict(action='Initialize',
gridSize=0.25,
fieldOfView=90,
renderObjectImage=True))
event = controller.step(
dict(action='TeleportFull',
x=ep1[2][0],
y=ep1[2][1],
z=ep1[2][2],
rotation=ep1[2][3],
horizon=ep1[2][4]))
frames = []
rotation_list = ['RotateRight', 'RotateLeft']
angle = 0
for i in range(len(ep1) - 5):
print(ep1[i + 4])
time.sleep(0.5)
img = img_bbx(args, event, ep1[i + 4], obj_list, ep1, target_parents)
pos = event.metadata['agent']['position']
rot = event.metadata['agent']['rotation']
if ep1[i + 4] == 'RotateLeft':
event = controller.step(
dict(action='TeleportFull',
x=pos['x'],
y=pos['y'],
z=pos['z'],
rotation=rot['y'] - 45,
horizon=angle))
elif ep1[i + 4] == 'RotateRight':
event = controller.step(
dict(action='TeleportFull',
x=pos['x'],
y=pos['y'],
z=pos['z'],
rotation=rot['y'] + 45,
horizon=angle))
elif ep1[i + 4] == 'LookDown':
event = controller.step(dict(action=ep1[i + 4]))
angle += 30
angle = np.clip(angle, -60, 60)
elif ep1[i + 4] == 'LookUp':
event = controller.step(dict(action=ep1[i + 4]))
angle -= 30
angle = np.clip(angle, -60, 60)
else:
event = controller.step(
dict(action='TeleportFull',
x=pos['x'],
y=pos['y'],
z=pos['z'],
rotation=rot['y'],
horizon=angle))
event = controller.step(dict(action=ep1[i + 4]))
frames.append(img)
time.sleep(1.5)
img = img_bbx(args, event, ep1[-1], obj_list, ep1, target_parents)
frames.append(img)
time.sleep(3)
controller.stop()
cv2.destroyAllWindows()
return frames
def teardown_module(module):
controller.stop()
def inference_worker(worker_ind: int, in_queue: mp.Queue,
out_queue: mp.Queue, agent_class: Any,
agent_kwargs: Dict[str,
Any], controller_kwargs: Dict[str,
Any],
max_steps: int, test: bool):
agent = agent_class(**agent_kwargs)
controller = ai2thor.controller.Controller(**controller_kwargs)
while True:
try:
e = in_queue.get(timeout=1)
except queue.Empty:
break
logger.info(
"Task Start id:{id} scene:{scene} target_object:{object_type} initial_position:{initial_position} rotation:{initial_orientation}"
.format(**e))
controller.initialization_parameters[
"robothorChallengeEpisodeId"] = e["id"]
print(e["scene"])
controller.reset(e["scene"])
teleport_action = {
"action": "TeleportFull",
**e["initial_position"], "rotation": {
"x": 0,
"y": e["initial_orientation"],
"z": 0
},
"horizon": e["initial_horizon"],
"standing": True
}
controller.step(action=teleport_action)
total_steps = 0
agent.reset()
episode_metrics = {
"trajectory": [{
**e["initial_position"], "rotation":
float(e["initial_orientation"]),
"horizon":
e["initial_horizon"]
}],
"actions_taken": []
}
stopped = False
while total_steps < max_steps and stopped is False:
total_steps += 1
event = controller.last_event
event.metadata.clear()
action = agent.act({
"object_goal": e["object_type"],
"depth": event.depth_frame,
"rgb": event.frame
})
if action not in ALLOWED_ACTIONS:
raise ValueError(
"Invalid action: {action}".format(action=action))
logger.info("Agent action: {action}".format(action=action))
event = controller.step(action=action)
episode_metrics["trajectory"].append({
**event.metadata["agent"]["position"], "rotation":
event.metadata["agent"]["rotation"]["y"],
"horizon":
event.metadata["agent"]["cameraHorizon"]
})
episode_metrics["actions_taken"].append({
"action":
action,
"success":
event.metadata["lastActionSuccess"]
})
stopped = action == "Stop"
if not test:
target_obj = get_object_by_type(event.metadata["objects"],
e["object_type"])
assert target_obj is not None
target_visible = target_obj["visible"]
episode_metrics["success"] = stopped and target_visible
if not test:
episode_result = {
"path": episode_metrics["trajectory"],
"shortest_path": e["shortest_path"],
"success": episode_metrics["success"]
}
else:
episode_result = None
out_queue.put((e["id"], episode_metrics, episode_result))
controller.stop()
print(f"Worker {worker_ind} Finished.")
renderClassImage=True,
renderObjectImage=True))
def stop(self):
self.controller.stop()
def apply(self, action):
if action in self.moves:
event = self.controller.step(dict(action=self.moves[action]))
elif action in self.rotates:
self.rotation += self.rotates[action]
event = self.controller.step(
dict(action='Rotate', rotation=self.rotation))
elif action in self.looks:
self.horizon += self.looks[action]
event = self.controller.step(
dict(action='Look', horizon=self.horizon))
if __name__ == '__main__':
controller = Controller()
import time
controller.start()
while True:
controller.apply('quaytrai')
time.sleep(0.1)
controller.stop()
def create_scene_meta(args,
scene,
width=300,
height=300,
objects_filter=None,
filter_file=None):
"""
Creates a dataset for the robothor challenge in `intermediate_directory`
named `robothor-dataset.json`
"""
angle = args.rotate_by
grid_size = args.grid_size
visibility_distance = args.visibility_distance
targets = [target.replace(" ", "") for target in args.targets]
desired_points = args.desired_points
rotations = list(range(0, 360, args.rotate_by))
# Restrict points visibility_multiplier_filter * visibility_distance away from the target object
visibility_multiplier_filter = 2
print("Visibility distance: {}".format(visibility_distance))
controller = ai2thor.controller.Controller(
width=width,
height=height,
scene=scene,
# Unity params
gridSize=grid_size,
rotateStepDegrees=angle,
agentMode='bot',
visibilityDistance=visibility_distance,
)
failed_points = []
if objects_filter is not None:
obj_filter = set([o for o in objects_filter.split(",")])
targets = [o for o in targets if o.replace(" ", "") in obj_filter]
# event = controller.step(
# dict(
# action='GetScenesInBuild',
# )
# )
# scenes_in_build = event.metadata['actionReturn']
objects_types_in_scene = set()
# dataset_flat = [])
scenes = args.scenes
# scenes = sorted(
# [scene for scene in scenes_in_build if 'physics' not in scene],
# key=key_sort_func
# )
# if scene_filter is not None:
# scene_filter_set = set([o for o in scene_filter.split(",")])
# scenes = [s for s in scenes if s in scene_filter_set]
print("Sorted scenes: {}".format(scenes))
scene_episodes = [[] for i in range(args.max_diff_level)]
for objectType in targets:
points = get_points(controller, objectType, scene,
objects_types_in_scene, failed_points, grid_size,
args.rotate_by, desired_points)
if points is not None:
split_by_difficulty(points,
scene_episodes,
args.distance_upgrade_step,
max_diff_level=args.max_diff_level)
scene_out_file = "{}_curriculum_{}_{}_{}.json".format(
scene, args.num_ep_per_stage, args.distance_upgrade_step,
args.penalty_decay)
if not os.path.exists(os.path.join(args.out_dir, scene)):
os.makedirs(os.path.join(args.out_dir, scene))
with open(os.path.join(args.out_dir, scene, scene_out_file), 'w') as f:
scene_curriculum_meta = {
"num_ep_per_stage": args.num_ep_per_stage,
"distance_upgrade_step": args.distance_upgrade_step,
"penalty_decay": args.penalty_decay,
"episodes": {
scene: scene_episodes
}
}
json.dump(scene_curriculum_meta, f, indent=4)
# collect episodes for this scene
# episodes[scene] = scene_episodes
#
# curriculum_meta['episodes'] = episodes
#
# out_file_name = "curriculum_{}_{}_{}.json".format(
# args.num_ep_per_stage,
# args.distance_upgrade_step,
# args.penalty_decay
# )
# with open(os.path.join(args.out_dir, out_file_name), 'w') as f:
# json.dump(curriculum_meta, f, indent=4)
num_ep = sum([len(ep_of_diff) for ep_of_diff in scene_episodes])
print("Finished scene {} ,total episodes: {}".format(scene, num_ep))
print("Object types in scene {}: {}".format(scene, objects_types_in_scene))
controller.stop()
return failed_points
def dump(scene, angle, resolution=(300, 300)):
'''
Dump needed data to hdf5 file to speed up training. Dumped file can be loaded using: f = h5py.File(filename, 'r'), where:
- f['locations'][()]: numpy array of all states in format (x, z, rotation, looking angle)
- f['observations'][()]: numpy array of RGB images of corresponding states in f['locations']
- f['graph'][()]: numpy array representing transition graph between states. e.g: f[0] = array([ 16., 272., 1., -1.], dtype=float32)
means from 1st locations, the agent will reach 16th state by taking action 0 (move forward), 272th state by taking action 1 (move backward),
reach 1th state by taking action 2 (rotate right) and cannot take action 3 (rotate left) indicated by -1 value.
- f['visible_objects'][()]: visible objects at corresponding states in f['locations']
- f['shortest'][()]: numpy array with shape of (num_states, num_states) indicating the shortest path length between
every pair of states.
'''
f = h5py.File("dumped/{}.hdf5".format(scene), "w")
observations = []
dump_features = []
locations = []
visible_objects = []
controller = ai2thor.controller.Controller()
controller.start()
controller.reset(scene)
event = controller.step(
dict(action='Initialize',
gridSize=0.5,
cameraY=1.0,
visibilityDistance=1.0))
y_coord = event.metadata['agent']['position']['y']
locations.append((event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z']))
# Using BFS to discover all reachable positions in current environment.
visited = set()
visited.add((event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z']))
while len(locations) > 0:
loc = locations.pop(0)
for act in ALL_POSSIBLE_ACTIONS:
controller.step(
dict(action='Teleport', x=loc[0], y=y_coord, z=loc[1]))
event = controller.step(dict(action=act))
if event.metadata['lastActionSuccess']:
new_loc = (event.metadata['agent']['position']['x'],
event.metadata['agent']['position']['z'])
if new_loc not in visited:
visited.add(new_loc)
locations.append(new_loc)
all_locs = list(visited)
print("{} locations".format(len(all_locs)))
states = []
rotations = np.linspace(0, 360, int(360 // angle) + 1)[:-1].tolist()
movement = {
0: [1, 0],
90.0: [0, 1],
180.0: [-1, 0],
270.0: [0, -1],
22.5: [1, 0],
67.5: [0, 1],
45.0: [1, 1],
112.5: [0, 1],
135.0: [-1, 1],
157.5: [-1, 0],
202.5: [-1, 0],
225.0: [-1, -1],
247.5: [0, -1],
292.5: [0, -1],
315: [1, -1],
337.5: [1, 0]
}
# Adding rotations and looking angles
for loc in all_locs:
for rot in rotations:
states.append((loc[0], loc[1], rot))
# for horot in [-30, 0, 30, 60]:
# states.append((loc[0], loc[1], rot, horot))
# ------------------------------------------------------------------------------
## Calculate shortest path length array
sta2idx = dict(zip(states, range(len(states))))
loc2idx = dict(zip(all_locs, range(len(all_locs))))
shortest_loc = np.zeros((len(all_locs), len(all_locs)))
for i in range(len(all_locs)):
dists = cal_min_dist(i, all_locs, loc2idx)
for j, d in enumerate(dists):
if j != i:
shortest_loc[i, j] = d
shortest_loc[j, i] = d
shortest_state = np.zeros((len(states), len(states)))
for i in range(len(states)):
for j in range(len(states)):
if i != j:
from_loc = loc2idx[states[i][0], states[i][1]]
to_loc = loc2idx[states[j][0], states[j][1]]
shortest_state[i, j] = shortest_loc[from_loc, to_loc]
shortest_state[j, i] = shortest_state[i, j]
# ------------------------------------------------------------------------------
# Building transition graph
graph = np.zeros(shape=(len(states), 4), dtype=int)
for state in states:
loc = (state[0], state[1])
rot = state[2]
to_states = []
move = movement[rot]
to_states.append((loc[0] + move[1] * 0.5, loc[1] + move[0] * 0.5,
rot)) # move ahead
to_states.append(
(loc[0] - move[1] * 0.5, loc[1] - move[0] * 0.5, rot)) # move back
to_states.append(
(loc[0], loc[1],
rot + angle if rot + angle < 360 else 0)) # turn right
to_states.append(
(loc[0], loc[1],
rot - angle if rot - angle >= 0 else 360 - angle)) # turn left
# to_states.append((loc[0], loc[1], rot + 30)) # look down
# to_states.append((loc[0], loc[1], rot, horot - 30)) # look up
state_idx = sta2idx[state]
for i, new_state in enumerate(to_states):
if new_state in sta2idx:
graph[state_idx][i] = sta2idx[new_state]
else:
graph[state_idx][i] = -1
# ------------------------------------------------------------------------------
laser = {}
## Calculate laser
for loc in all_locs:
pos = (loc[0], loc[1], 0)
north = 0
while graph[sta2idx[pos]][0] != -1:
north += 1
pos = states[graph[sta2idx[pos]][0]]
assert pos[2] == 0
pos = (loc[0], loc[1], 0)
south = 0
while graph[sta2idx[pos]][1] != -1:
south += 1
pos = states[graph[sta2idx[pos]][1]]
assert pos[2] == 0
pos = (loc[0], loc[1], 90)
right = 0
while graph[sta2idx[pos]][0] != -1:
right += 1
pos = states[graph[sta2idx[pos]][0]]
assert pos[2] == 90
pos = (loc[0], loc[1], 90)
left = 0
while graph[sta2idx[pos]][1] != -1:
left += 1
pos = states[graph[sta2idx[pos]][1]]
assert pos[2] == 90
for r in rotations:
if r > 315.0 or r < 45.0:
laser[(loc[0], loc[1], r)] = [north, south, right, left]
elif r > 45.0 and r < 135.0:
laser[(loc[0], loc[1], r)] = [right, left, south, north]
elif r > 135.0 and r < 225.0:
laser[(loc[0], loc[1], r)] = [south, north, left, right]
elif r > 225.0 and r < 315.0:
laser[(loc[0], loc[1], r)] = [left, right, north, south]
if 45.0 in rotations:
for loc in all_locs:
pos = (loc[0], loc[1], 45.0)
north = 0
while graph[sta2idx[pos]][0] != -1:
north += 1
pos = states[graph[sta2idx[pos]][0]]
assert pos[2] == 45.0
pos = (loc[0], loc[1], 45.0)
south = 0
while graph[sta2idx[pos]][1] != -1:
south += 1
pos = states[graph[sta2idx[pos]][1]]
assert pos[2] == 45.0
pos = (loc[0], loc[1], 135.0)
right = 0
while graph[sta2idx[pos]][0] != -1:
right += 1
pos = states[graph[sta2idx[pos]][0]]
assert pos[2] == 135.0
pos = (loc[0], loc[1], 135.0)
left = 0
while graph[sta2idx[pos]][1] != -1:
left += 1
pos = states[graph[sta2idx[pos]][1]]
assert pos[2] == 135.0
laser[(loc[0], loc[1], 45.0)] = [north, south, right, left]
laser[(loc[0], loc[1], 225.0)] = [south, north, left, right]
laser[(loc[0], loc[1], 135.0)] = [right, left, south, north]
laser[(loc[0], loc[1], 315.0)] = [left, right, north, south]
lasers = []
for state in states:
lasers.append(laser[state])
# ------------------------------------------------------------------------------
# Adding observations
for state in states:
vis_objects = set()
event = controller.step(
dict(action='TeleportFull',
x=state[0],
y=1.25,
z=state[1],
rotation=state[2],
horizon=30))
resized_frame = cv2.resize(event.frame, (resolution[0], resolution[1]))
observations.append(resized_frame)
visible = [obj for obj in event.metadata['objects'] if obj['visible']]
for obj in visible:
vis_objects.add(obj['objectType'])
if len(vis_objects) > 0:
visible_objects.append(",".join(list(vis_objects)))
else:
visible_objects.append("")
# ------------------------------------------------------------------------------
print("{} states".format(len(states)))
all_visible_objects = list(
set(",".join([o for o in visible_objects if o != '']).split(',')))
all_visible_objects.sort()
for c in ['Lamp', 'PaperTowelRoll', 'Glassbottle']:
if c in all_visible_objects:
all_visible_objects.remove(c)
controller.stop()
f.create_dataset("locations", data=np.asarray(states, np.float32))
f.create_dataset("observations", data=np.asarray(observations, np.uint8))
f.create_dataset("graph", data=graph)
f.create_dataset("visible_objects",
data=np.array(visible_objects, dtype=object),
dtype=h5py.special_dtype(vlen=str))
f.create_dataset("all_visible_objects",
data=np.array(all_visible_objects, dtype=object),
dtype=h5py.special_dtype(vlen=str))
f.create_dataset("shortest", data=shortest_state)
f.create_dataset("lasers", data=np.asarray(lasers, np.float32))
f.close()
return y_coord
