def dump_scene(scene_name, base_dir, renderObjectImage=False, renderDepthImage=False, renderClassImage=False): controller = ai2thor.controller.Controller() controller.start(player_screen_height=448, player_screen_width=448) controller.reset(scene_name) event = controller.step(dict(action='Initialize', fieldOfView=90, gridSize=0.25, renderDepthImage=renderDepthImage, renderObjectImage=renderObjectImage, renderClassImage=renderClassImage)) dump_scene_controller(base_dir, controller) controller.stop()
def run(file_name=None): # file_name = file_path.split('/')[-1].split('.')[0] controller = ai2thor.controller.Controller() controller.start() controller.reset("FloorPlan203") y_coord = 1.25 event = controller.step( dict(action='Initialize', gridSize=0.5, cameraY=y_coord, visibilityDistance=1.0)) all_visible_objects = list( np.unique([obj['objectType'] for obj in event.metadata['objects']])) rotation = 0.0 while True: # making a loop try: # used try so that if user pressed other than the given key error will not be shown key = click.getchar() if key == 'a': # Rotate Left rotation -= 22.5 if rotation < 0: rotation = rotation + 360 event = controller.step( dict(action='Rotate', rotation=rotation)) elif key == 'd': rotation += 22.5 if rotation > 360: rotation = rotation - 360 event = controller.step( dict(action='Rotate', rotation=rotation)) elif key == 'w': event = controller.step(dict(action='MoveAhead')) elif key == 's': event = controller.step(dict(action='MoveBack')) elif key == 'z': event = controller.step(dict(action='LookDown')) elif key == 'x': event = controller.step(dict(action='LookUp')) elif key == 'q': controller.stop() break elif key == 'r': scene = input("Scene id: ") controller.reset('FloorPlan{}'.format(scene)) event = controller.step( dict(action='Initialize', gridSize=0.5, cameraY=y_coord)) else: print("Key not supported! Try a, d, w, s, q, r.") print((event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z'], event.metadata['agent']['rotation'])) # print([(obj['objectType'], obj['distance']) for obj in event.metadata['objects'] if obj['visible']]) except: print("Key not supported! Try a, d, w, s, q, r.")
def worker(steps_per_proc, sync_event, queue, gpu_id=0, actions=actions): os.environ['DISPLAY'] = f":{gpu_id}" controller = ai2thor.controller.Controller() controller.start() controller.step(dict(action='Initialize', gridSize=0.25)) print("Worker with pid:", os.getpid(), "is intialized") np.random.seed(os.getpid()) #inform main process that intialization is successful queue.put(1) sync_event.wait() count = len(actions) for _ in range(steps_per_proc): a = np.random.randint(count) controller.step(dict(action=actions[a])) queue.put(1) print("Worker with pid:", os.getpid(), " funished job") sync_event.clear() sync_event.wait() controller.stop()
def dump_scene( scene_name, base_dir, renderInstanceSegmentation=False, renderDepthImage=False, renderSemanticSegmentation=False, ): controller = ai2thor.controller.Controller() controller.start(height=448, width=448) controller.reset(scene_name) event = controller.step( dict( action="Initialize", fieldOfView=90, gridSize=0.25, renderDepthImage=renderDepthImage, renderInstanceSegmentation=renderInstanceSegmentation, renderSemanticSegmentation=renderSemanticSegmentation, )) dump_scene_controller(base_dir, controller) controller.stop()
def check_size(scene): f = h5py.File("dumped/{}.hdf5".format(scene), "w") locations = [] visible_objects = [] controller = ai2thor.controller.Controller() controller.start() controller.reset(scene) controller.random_initialize(unique_object_types=True) event = controller.step(dict(action='Initialize', gridSize=0.5)) y_coord = event.metadata['agent']['position']['y'] locations.append((event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z'])) # Using BFS to discover all reachable positions in current environment. visited = set() visited.add((event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z'])) while len(locations) > 0: loc = locations.pop(0) for act in ALL_POSSIBLE_ACTIONS: controller.step( dict(action='Teleport', x=loc[0], y=y_coord, z=loc[1])) event = controller.step(dict(action=act)) if event.metadata['lastActionSuccess']: new_loc = (event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z']) if new_loc not in visited: visited.add(new_loc) locations.append(new_loc) all_locs = list(visited) controller.stop() return len(all_locs)
def dump_feature(scene, cat2idx): f = h5py.File("dumped/{}.hdf5".format(scene), "a") states = f['locations'][()] laser = f['lasers'][()] dump_features = [] controller = ai2thor.controller.Controller() controller.start() controller.reset(scene) event = controller.step( dict(action='Initialize', gridSize=0.5, visibilityDistance=1000.0)) for i, state in enumerate(states): event = controller.step( dict(action='TeleportFull', x=state[0], y=1.25, z=state[1], rotation=state[2], horizon=30)) visible = [obj for obj in event.metadata['objects'] if obj['visible']] df = np.zeros(len(cat2idx) + 4) df[-4:] = laser[i].tolist() for obj in visible: try: obj_id = cat2idx[obj['objectType']] df[obj_id] = obj['distance'] except: print(obj['objectType']) dump_features.append(df) controller.stop() f.create_dataset("dump_features", data=np.asarray(dump_features, np.float32)) f.close()
def start_controller(args, ep1, obj_list, target_parents): controller = ai2thor.controller.Controller() controller.start(player_screen_height=500, player_screen_width=500) controller.reset(ep1[0]) event = controller.step( dict(action='Initialize', gridSize=0.25, fieldOfView=90, renderObjectImage=True)) event = controller.step( dict(action='TeleportFull', x=ep1[2][0], y=ep1[2][1], z=ep1[2][2], rotation=ep1[2][3], horizon=ep1[2][4])) frames = [] rotation_list = ['RotateRight', 'RotateLeft'] angle = 0 for i in range(len(ep1) - 5): print(ep1[i + 4]) time.sleep(0.5) img = img_bbx(args, event, ep1[i + 4], obj_list, ep1, target_parents) pos = event.metadata['agent']['position'] rot = event.metadata['agent']['rotation'] if ep1[i + 4] == 'RotateLeft': event = controller.step( dict(action='TeleportFull', x=pos['x'], y=pos['y'], z=pos['z'], rotation=rot['y'] - 45, horizon=angle)) elif ep1[i + 4] == 'RotateRight': event = controller.step( dict(action='TeleportFull', x=pos['x'], y=pos['y'], z=pos['z'], rotation=rot['y'] + 45, horizon=angle)) elif ep1[i + 4] == 'LookDown': event = controller.step(dict(action=ep1[i + 4])) angle += 30 angle = np.clip(angle, -60, 60) elif ep1[i + 4] == 'LookUp': event = controller.step(dict(action=ep1[i + 4])) angle -= 30 angle = np.clip(angle, -60, 60) else: event = controller.step( dict(action='TeleportFull', x=pos['x'], y=pos['y'], z=pos['z'], rotation=rot['y'], horizon=angle)) event = controller.step(dict(action=ep1[i + 4])) frames.append(img) time.sleep(1.5) img = img_bbx(args, event, ep1[-1], obj_list, ep1, target_parents) frames.append(img) time.sleep(3) controller.stop() cv2.destroyAllWindows() return frames
def teardown_module(module): controller.stop()
def inference_worker(worker_ind: int, in_queue: mp.Queue, out_queue: mp.Queue, agent_class: Any, agent_kwargs: Dict[str, Any], controller_kwargs: Dict[str, Any], max_steps: int, test: bool): agent = agent_class(**agent_kwargs) controller = ai2thor.controller.Controller(**controller_kwargs) while True: try: e = in_queue.get(timeout=1) except queue.Empty: break logger.info( "Task Start id:{id} scene:{scene} target_object:{object_type} initial_position:{initial_position} rotation:{initial_orientation}" .format(**e)) controller.initialization_parameters[ "robothorChallengeEpisodeId"] = e["id"] print(e["scene"]) controller.reset(e["scene"]) teleport_action = { "action": "TeleportFull", **e["initial_position"], "rotation": { "x": 0, "y": e["initial_orientation"], "z": 0 }, "horizon": e["initial_horizon"], "standing": True } controller.step(action=teleport_action) total_steps = 0 agent.reset() episode_metrics = { "trajectory": [{ **e["initial_position"], "rotation": float(e["initial_orientation"]), "horizon": e["initial_horizon"] }], "actions_taken": [] } stopped = False while total_steps < max_steps and stopped is False: total_steps += 1 event = controller.last_event event.metadata.clear() action = agent.act({ "object_goal": e["object_type"], "depth": event.depth_frame, "rgb": event.frame }) if action not in ALLOWED_ACTIONS: raise ValueError( "Invalid action: {action}".format(action=action)) logger.info("Agent action: {action}".format(action=action)) event = controller.step(action=action) episode_metrics["trajectory"].append({ **event.metadata["agent"]["position"], "rotation": event.metadata["agent"]["rotation"]["y"], "horizon": event.metadata["agent"]["cameraHorizon"] }) episode_metrics["actions_taken"].append({ "action": action, "success": event.metadata["lastActionSuccess"] }) stopped = action == "Stop" if not test: target_obj = get_object_by_type(event.metadata["objects"], e["object_type"]) assert target_obj is not None target_visible = target_obj["visible"] episode_metrics["success"] = stopped and target_visible if not test: episode_result = { "path": episode_metrics["trajectory"], "shortest_path": e["shortest_path"], "success": episode_metrics["success"] } else: episode_result = None out_queue.put((e["id"], episode_metrics, episode_result)) controller.stop() print(f"Worker {worker_ind} Finished.")
renderClassImage=True, renderObjectImage=True)) def stop(self): self.controller.stop() def apply(self, action): if action in self.moves: event = self.controller.step(dict(action=self.moves[action])) elif action in self.rotates: self.rotation += self.rotates[action] event = self.controller.step( dict(action='Rotate', rotation=self.rotation)) elif action in self.looks: self.horizon += self.looks[action] event = self.controller.step( dict(action='Look', horizon=self.horizon)) if __name__ == '__main__': controller = Controller() import time controller.start() while True: controller.apply('quaytrai') time.sleep(0.1) controller.stop()
def create_scene_meta(args, scene, width=300, height=300, objects_filter=None, filter_file=None): """ Creates a dataset for the robothor challenge in `intermediate_directory` named `robothor-dataset.json` """ angle = args.rotate_by grid_size = args.grid_size visibility_distance = args.visibility_distance targets = [target.replace(" ", "") for target in args.targets] desired_points = args.desired_points rotations = list(range(0, 360, args.rotate_by)) # Restrict points visibility_multiplier_filter * visibility_distance away from the target object visibility_multiplier_filter = 2 print("Visibility distance: {}".format(visibility_distance)) controller = ai2thor.controller.Controller( width=width, height=height, scene=scene, # Unity params gridSize=grid_size, rotateStepDegrees=angle, agentMode='bot', visibilityDistance=visibility_distance, ) failed_points = [] if objects_filter is not None: obj_filter = set([o for o in objects_filter.split(",")]) targets = [o for o in targets if o.replace(" ", "") in obj_filter] # event = controller.step( # dict( # action='GetScenesInBuild', # ) # ) # scenes_in_build = event.metadata['actionReturn'] objects_types_in_scene = set() # dataset_flat = []) scenes = args.scenes # scenes = sorted( # [scene for scene in scenes_in_build if 'physics' not in scene], # key=key_sort_func # ) # if scene_filter is not None: # scene_filter_set = set([o for o in scene_filter.split(",")]) # scenes = [s for s in scenes if s in scene_filter_set] print("Sorted scenes: {}".format(scenes)) scene_episodes = [[] for i in range(args.max_diff_level)] for objectType in targets: points = get_points(controller, objectType, scene, objects_types_in_scene, failed_points, grid_size, args.rotate_by, desired_points) if points is not None: split_by_difficulty(points, scene_episodes, args.distance_upgrade_step, max_diff_level=args.max_diff_level) scene_out_file = "{}_curriculum_{}_{}_{}.json".format( scene, args.num_ep_per_stage, args.distance_upgrade_step, args.penalty_decay) if not os.path.exists(os.path.join(args.out_dir, scene)): os.makedirs(os.path.join(args.out_dir, scene)) with open(os.path.join(args.out_dir, scene, scene_out_file), 'w') as f: scene_curriculum_meta = { "num_ep_per_stage": args.num_ep_per_stage, "distance_upgrade_step": args.distance_upgrade_step, "penalty_decay": args.penalty_decay, "episodes": { scene: scene_episodes } } json.dump(scene_curriculum_meta, f, indent=4) # collect episodes for this scene # episodes[scene] = scene_episodes # # curriculum_meta['episodes'] = episodes # # out_file_name = "curriculum_{}_{}_{}.json".format( # args.num_ep_per_stage, # args.distance_upgrade_step, # args.penalty_decay # ) # with open(os.path.join(args.out_dir, out_file_name), 'w') as f: # json.dump(curriculum_meta, f, indent=4) num_ep = sum([len(ep_of_diff) for ep_of_diff in scene_episodes]) print("Finished scene {} ,total episodes: {}".format(scene, num_ep)) print("Object types in scene {}: {}".format(scene, objects_types_in_scene)) controller.stop() return failed_points
def dump(scene, angle, resolution=(300, 300)): ''' Dump needed data to hdf5 file to speed up training. Dumped file can be loaded using: f = h5py.File(filename, 'r'), where: - f['locations'][()]: numpy array of all states in format (x, z, rotation, looking angle) - f['observations'][()]: numpy array of RGB images of corresponding states in f['locations'] - f['graph'][()]: numpy array representing transition graph between states. e.g: f[0] = array([ 16., 272., 1., -1.], dtype=float32) means from 1st locations, the agent will reach 16th state by taking action 0 (move forward), 272th state by taking action 1 (move backward), reach 1th state by taking action 2 (rotate right) and cannot take action 3 (rotate left) indicated by -1 value. - f['visible_objects'][()]: visible objects at corresponding states in f['locations'] - f['shortest'][()]: numpy array with shape of (num_states, num_states) indicating the shortest path length between every pair of states. ''' f = h5py.File("dumped/{}.hdf5".format(scene), "w") observations = [] dump_features = [] locations = [] visible_objects = [] controller = ai2thor.controller.Controller() controller.start() controller.reset(scene) event = controller.step( dict(action='Initialize', gridSize=0.5, cameraY=1.0, visibilityDistance=1.0)) y_coord = event.metadata['agent']['position']['y'] locations.append((event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z'])) # Using BFS to discover all reachable positions in current environment. visited = set() visited.add((event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z'])) while len(locations) > 0: loc = locations.pop(0) for act in ALL_POSSIBLE_ACTIONS: controller.step( dict(action='Teleport', x=loc[0], y=y_coord, z=loc[1])) event = controller.step(dict(action=act)) if event.metadata['lastActionSuccess']: new_loc = (event.metadata['agent']['position']['x'], event.metadata['agent']['position']['z']) if new_loc not in visited: visited.add(new_loc) locations.append(new_loc) all_locs = list(visited) print("{} locations".format(len(all_locs))) states = [] rotations = np.linspace(0, 360, int(360 // angle) + 1)[:-1].tolist() movement = { 0: [1, 0], 90.0: [0, 1], 180.0: [-1, 0], 270.0: [0, -1], 22.5: [1, 0], 67.5: [0, 1], 45.0: [1, 1], 112.5: [0, 1], 135.0: [-1, 1], 157.5: [-1, 0], 202.5: [-1, 0], 225.0: [-1, -1], 247.5: [0, -1], 292.5: [0, -1], 315: [1, -1], 337.5: [1, 0] } # Adding rotations and looking angles for loc in all_locs: for rot in rotations: states.append((loc[0], loc[1], rot)) # for horot in [-30, 0, 30, 60]: # states.append((loc[0], loc[1], rot, horot)) # ------------------------------------------------------------------------------ ## Calculate shortest path length array sta2idx = dict(zip(states, range(len(states)))) loc2idx = dict(zip(all_locs, range(len(all_locs)))) shortest_loc = np.zeros((len(all_locs), len(all_locs))) for i in range(len(all_locs)): dists = cal_min_dist(i, all_locs, loc2idx) for j, d in enumerate(dists): if j != i: shortest_loc[i, j] = d shortest_loc[j, i] = d shortest_state = np.zeros((len(states), len(states))) for i in range(len(states)): for j in range(len(states)): if i != j: from_loc = loc2idx[states[i][0], states[i][1]] to_loc = loc2idx[states[j][0], states[j][1]] shortest_state[i, j] = shortest_loc[from_loc, to_loc] shortest_state[j, i] = shortest_state[i, j] # ------------------------------------------------------------------------------ # Building transition graph graph = np.zeros(shape=(len(states), 4), dtype=int) for state in states: loc = (state[0], state[1]) rot = state[2] to_states = [] move = movement[rot] to_states.append((loc[0] + move[1] * 0.5, loc[1] + move[0] * 0.5, rot)) # move ahead to_states.append( (loc[0] - move[1] * 0.5, loc[1] - move[0] * 0.5, rot)) # move back to_states.append( (loc[0], loc[1], rot + angle if rot + angle < 360 else 0)) # turn right to_states.append( (loc[0], loc[1], rot - angle if rot - angle >= 0 else 360 - angle)) # turn left # to_states.append((loc[0], loc[1], rot + 30)) # look down # to_states.append((loc[0], loc[1], rot, horot - 30)) # look up state_idx = sta2idx[state] for i, new_state in enumerate(to_states): if new_state in sta2idx: graph[state_idx][i] = sta2idx[new_state] else: graph[state_idx][i] = -1 # ------------------------------------------------------------------------------ laser = {} ## Calculate laser for loc in all_locs: pos = (loc[0], loc[1], 0) north = 0 while graph[sta2idx[pos]][0] != -1: north += 1 pos = states[graph[sta2idx[pos]][0]] assert pos[2] == 0 pos = (loc[0], loc[1], 0) south = 0 while graph[sta2idx[pos]][1] != -1: south += 1 pos = states[graph[sta2idx[pos]][1]] assert pos[2] == 0 pos = (loc[0], loc[1], 90) right = 0 while graph[sta2idx[pos]][0] != -1: right += 1 pos = states[graph[sta2idx[pos]][0]] assert pos[2] == 90 pos = (loc[0], loc[1], 90) left = 0 while graph[sta2idx[pos]][1] != -1: left += 1 pos = states[graph[sta2idx[pos]][1]] assert pos[2] == 90 for r in rotations: if r > 315.0 or r < 45.0: laser[(loc[0], loc[1], r)] = [north, south, right, left] elif r > 45.0 and r < 135.0: laser[(loc[0], loc[1], r)] = [right, left, south, north] elif r > 135.0 and r < 225.0: laser[(loc[0], loc[1], r)] = [south, north, left, right] elif r > 225.0 and r < 315.0: laser[(loc[0], loc[1], r)] = [left, right, north, south] if 45.0 in rotations: for loc in all_locs: pos = (loc[0], loc[1], 45.0) north = 0 while graph[sta2idx[pos]][0] != -1: north += 1 pos = states[graph[sta2idx[pos]][0]] assert pos[2] == 45.0 pos = (loc[0], loc[1], 45.0) south = 0 while graph[sta2idx[pos]][1] != -1: south += 1 pos = states[graph[sta2idx[pos]][1]] assert pos[2] == 45.0 pos = (loc[0], loc[1], 135.0) right = 0 while graph[sta2idx[pos]][0] != -1: right += 1 pos = states[graph[sta2idx[pos]][0]] assert pos[2] == 135.0 pos = (loc[0], loc[1], 135.0) left = 0 while graph[sta2idx[pos]][1] != -1: left += 1 pos = states[graph[sta2idx[pos]][1]] assert pos[2] == 135.0 laser[(loc[0], loc[1], 45.0)] = [north, south, right, left] laser[(loc[0], loc[1], 225.0)] = [south, north, left, right] laser[(loc[0], loc[1], 135.0)] = [right, left, south, north] laser[(loc[0], loc[1], 315.0)] = [left, right, north, south] lasers = [] for state in states: lasers.append(laser[state]) # ------------------------------------------------------------------------------ # Adding observations for state in states: vis_objects = set() event = controller.step( dict(action='TeleportFull', x=state[0], y=1.25, z=state[1], rotation=state[2], horizon=30)) resized_frame = cv2.resize(event.frame, (resolution[0], resolution[1])) observations.append(resized_frame) visible = [obj for obj in event.metadata['objects'] if obj['visible']] for obj in visible: vis_objects.add(obj['objectType']) if len(vis_objects) > 0: visible_objects.append(",".join(list(vis_objects))) else: visible_objects.append("") # ------------------------------------------------------------------------------ print("{} states".format(len(states))) all_visible_objects = list( set(",".join([o for o in visible_objects if o != '']).split(','))) all_visible_objects.sort() for c in ['Lamp', 'PaperTowelRoll', 'Glassbottle']: if c in all_visible_objects: all_visible_objects.remove(c) controller.stop() f.create_dataset("locations", data=np.asarray(states, np.float32)) f.create_dataset("observations", data=np.asarray(observations, np.uint8)) f.create_dataset("graph", data=graph) f.create_dataset("visible_objects", data=np.array(visible_objects, dtype=object), dtype=h5py.special_dtype(vlen=str)) f.create_dataset("all_visible_objects", data=np.array(all_visible_objects, dtype=object), dtype=h5py.special_dtype(vlen=str)) f.create_dataset("shortest", data=shortest_state) f.create_dataset("lasers", data=np.asarray(lasers, np.float32)) f.close() return y_coord