def __init__(self, all_metadata_available: bool = True, **kwargs): self.config = dict( rotateStepDegrees=30.0, visibilityDistance=1.0, gridSize=0.25, agentType="stochastic", continuousMode=True, snapToGrid=False, agentMode="locobot", width=640, height=480, agentCount=1, server_class=FifoServer, ) if "agentCount" in kwargs: assert kwargs["agentCount"] > 0 recursive_update(self.config, {**kwargs, "agentMode": "locobot"}) self.controller = Controller(**self.config, ) self.all_metadata_available = all_metadata_available self.scene_to_reachable_positions: Optional[Dict[str, Any]] = None self.distance_cache: Optional[DynamicDistanceCache] = None if self.all_metadata_available: self.scene_to_reachable_positions = { self.scene_name: copy.deepcopy(self.currently_reachable_points) } assert len(self.scene_to_reachable_positions[self.scene_name]) > 10 self.distance_cache = DynamicDistanceCache(rounding=1) self.agent_count = self.config["agentCount"]
def restart(self): ### reset the unity to avoid some latency issue ... if isinstance(self.controller, type(None)): self.controller = Controller(local_executable_path=PATH, scene="FloorPlan201_physics", x_display=self.x_display, agentMode='drone', fieldOfView=60, port=self.port) else: self.controller.stop() self.controller = Controller(local_executable_path=PATH, scene="FloorPlan201_physics", x_display=self.x_display, agentMode='drone', fieldOfView=60, port=self.port) _ = self.controller.reset(self.trajectory['scene'][0]) _ = self.controller.step( dict(action='ChangeAutoResetTimeScale', timeScale=self.cfg.thor.time_scale)) _ = self.controller.step( dict(action='ChangeFixedDeltaTime', fixedDeltaTime=self.cfg.thor.delta_time)) print('Thread:' + str(self.pid) + ' [' + str(self.iters + 1) + ']: restart finish')
def __init__(self, **kwargs): self.config = dict( rotateStepDegrees=30.0, visibilityDistance=1.0, gridSize=0.25, agentType="stochastic", continuousMode=True, snapToGrid=False, agentMode="bot", width=640, height=480, ) recursive_update(self.config, {**kwargs, "agentMode": "bot"}) self.controller = Controller(**self.config) self.known_good_locations: Dict[str, Any] = { self.scene_name: copy.deepcopy(self.currently_reachable_points) } assert len(self.known_good_locations[self.scene_name]) > 10 # onames = [o['objectId'] for o in self.last_event.metadata['objects']] # removed = [] # for oname in onames: # if 'Painting' in oname: # self.controller.step("RemoveFromScene", objectId=oname) # removed.append(oname) # get_logger().info("Removed {} Paintings from {}".format(len(removed), self.scene_name)) # get_logger().warning("init to scene {} in pos {}".format(self.scene_name, self.agent_state())) # npoints = len(self.currently_reachable_points) # assert npoints > 100, "only {} reachable points after init".format(npoints) self.grids: Dict[str, Tuple[Dict[str, np.array], int, int, int, int]] = {} self.initialize_grid()
def __init__(self, fov=60.0, camera_Y=0.675, grid_size=0.25, visibility_distance=1.5, player_screen_width=300, player_screen_height=300, full_scrn=False, depth_image=False, class_image=False, top_view_cam=False, object_image=False, third_party_cam=False, scene="FloorPlan220", object_name="Television"): self.scene = scene self.grid_size = grid_size self.depth_image = depth_image self.class_image = class_image self.object_image = object_image self.visibility_distance = visibility_distance self.camera_Y = camera_Y self.fov = fov self.object_name = object_name self.player_screen_width = player_screen_width self.player_screen_height = player_screen_height self.top_view_cam = top_view_cam self.third_party_cam = third_party_cam self.full_scrn = full_scrn self.ctrl = Controller()
def run_episodes(self): self.ep_idx = 0 # self.objects = [] for episode in range(self.num_episodes): print("STARTING EPISODE ", episode) mapname = self.mapnames[episode] print("MAPNAME=", mapname) self.controller = Controller( scene=mapname, gridSize=0.25, width=self.W, height=self.H, fieldOfView=self.fov, renderObjectImage=True, renderDepthImage=True, ) self.basepath = self.homepath + f"/{mapname}_{episode}" print("BASEPATH: ", self.basepath) # self.basepath = f"/hdd/ayushj/habitat_data/{mapname}_{episode}" if not os.path.exists(self.basepath): os.mkdir(self.basepath) self.run() self.controller.stop() time.sleep(1) self.ep_idx += 1
def __init__(self, **kwargs): self.config = dict( rotateStepDegrees=30.0, visibilityDistance=1.0, gridSize=0.25, agentType="stochastic", continuousMode=True, snapToGrid=False, agentMode="bot", width=640, height=480, agentCount=1, ) if "agentCount" in kwargs: assert kwargs["agentCount"] > 0 recursive_update(self.config, {**kwargs, "agentMode": "bot"}) self.controller = Controller( **self.config, server_class=ai2thor.fifo_server.FifoServer) self.known_good_locations: Dict[str, Any] = { self.scene_name: copy.deepcopy(self.currently_reachable_points) } self.distance_cache = DynamicDistanceCache(rounding=1) assert len(self.known_good_locations[self.scene_name]) > 10 self.agent_count = self.config["agentCount"]
def run_val(self, mapnames, BATCH_SIZE): batch = {"rewards": [], "obs": [], "actions": []} episode_rewards = 0.0 episode_steps = [] iter_idx = 0 while True: mapname = np.random.choice(mapnames) # self.basepath = self.homepath + f"/{mapname}_{episode}" # print("BASEPATH: ", self.basepath) # # self.basepath = f"/hdd/ayushj/habitat_data/{mapname}_{episode}" # if not os.path.exists(self.basepath): # os.mkdir(self.basepath) self.controller = Controller( scene=mapname, gridSize=0.25, width=self.W, height=self.H, fieldOfView=self.fov, renderObjectImage=True, renderDepthImage=True, ) episode_rewards, obs, actions = self.run() print("Total reward for val batch # ", iter_idx, " :", episode_rewards) self.controller.stop() time.sleep(1) if episode_rewards is None: print("NO EPISODE REWARDS.. SKIPPING BATCH INSTANCE") continue batch["rewards"].append(episode_rewards) batch["obs"].append(obs) batch["actions"].append(actions) train, labels, mean_rewards, boudary = self.elite_batch( batch, self.percentile) with torch.no_grad(): scores = self.cemnet(train, softmax=False) loss_value = self.loss(scores, labels) iter_idx += 1 if len(batch["rewards"]) == BATCH_SIZE: break return loss_value, mean_rewards
def teleport_to_base_location(controller: Controller): assert ( controller.last_event.metadata["sceneName"].replace("_physics", "") == "FloorPlan28" ) controller.step("TeleportFull", **BASE_FP28_LOCATION) assert controller.last_event.metadata["lastActionSuccess"]
def run_load(dataset_path, memory, conn): init_logging() controller = Controller( x_display='0.%d' % global_config.actor_gpu, visibilityDistance=actor_config.visibilityDistance, renderDepthImage=global_config.depth) dataset = ActiveDataset(dataset_path, memory, controller, conn) dataset.process() controller.stop()
def make_controller(self): if self.controller is None: self.controller = Controller() self.controller.step({"action": "ChangeQuality", "quality": "Very High"}) self.controller.step( { "action": "ChangeResolution", "x": self.viz_rows_cols[1], "y": self.viz_rows_cols[0], } )
def batch_iteration(self, mapnames, NN, BATCH_SIZE): batch = {"rewards": [], "obs": [], "actions": []} episode_rewards = 0.0 iter_idx = 0 while True: mapname = np.random.choice(mapnames) # self.basepath = self.homepath + f"/{mapname}_{episode}" # print("BASEPATH: ", self.basepath) # # self.basepath = f"/hdd/ayushj/habitat_data/{mapname}_{episode}" # if not os.path.exists(self.basepath): # os.mkdir(self.basepath) self.controller = Controller( scene=mapname, gridSize=0.25, width=self.W, height=self.H, fieldOfView=self.fov, renderObjectImage=True, renderDepthImage=True, ) episode_rewards, obs, actions = self.run() print("Total reward for train batch # ", iter_idx, " :", episode_rewards) self.controller.stop() time.sleep(1) if episode_rewards is None: print("NO EPISODE REWARDS.. SKIPPING BATCH INSTANCE") continue batch["rewards"].append(episode_rewards) batch["obs"].append(obs) batch["actions"].append(actions) iter_idx += 1 print(len(batch["rewards"])) if len(batch["rewards"]) == BATCH_SIZE: yield batch iter_idx = 0 batch = {"rewards": [], "obs": [], "actions": []} if len(batch["rewards"]) > BATCH_SIZE: st()
def get_agent_map_data(c: Controller): c.step(dict(action='ToggleMapView')) cam_position = c.last_event.metadata["cameraPosition"] cam_orth_size = c.last_event.metadata["cameraOrthSize"] pos_converted = converter(c.last_event.frame.shape, get_position(cam_position), cam_orth_size) to_return = { "frame": c.last_event.frame, "cam_position": cam_position, "cam_orth_size": cam_orth_size, "pos_converted": pos_converted } c.step(dict(action='ToggleMapView')) return to_return
def get_agent_map_data(c: Controller): c.step({"action": "ToggleMapView"}) cam_position = c.last_event.metadata["cameraPosition"] cam_orth_size = c.last_event.metadata["cameraOrthSize"] pos_translator = ThorPositionTo2DFrameTranslator( c.last_event.frame.shape, position_to_tuple(cam_position), cam_orth_size) to_return = { "frame": c.last_event.frame, "cam_position": cam_position, "cam_orth_size": cam_orth_size, "pos_translator": pos_translator, } c.step({"action": "ToggleMapView"}) return to_return
def open_objs(object_names_to_open: List[str], controller: Controller) -> Dict[str, Optional[float]]: """Opens up the chosen pickupable objects if they're openable.""" out: Dict[str, Optional[float]] = defaultdict(lambda: None) for obj_name in object_names_to_open: obj = get_object_by_name(obj_name, controller) last_openness = obj["openness"] new_openness = last_openness while abs(last_openness - new_openness) <= OPENNESS_THRESHOLD: new_openness = random.random() event = controller.step( "OpenObject", objectId=obj["objectId"], openness=new_openness, forceAction=True, ) obj_after_open = get_object_by_name(obj_name, controller) if abs(obj_after_open["openness"] - new_openness) > 0.001: raise RuntimeError( f"In scene {event.metadata['sceneName']}, {obj['name']} was supposed to open to {new_openness}" f" from {last_openness} but instead reached {obj_after_open['openness']}. Last action success was:" f" {event.metadata['lastActionSuccess']}") out[obj["name"]] = obj_after_open["openness"] return out
def create_controller(): return Controller( x_display=self.x_display, player_screen_width=self._start_player_screen_width, player_screen_height=self._start_player_screen_height, local_executable_path=self._local_thor_build, quality=self._quality, )
def check_object_opens(obj: Dict[str, Any], controller: Controller): controller.step( "OpenObject", objectId=obj["objectId"], openness=1.0, forceAction=True, ) obj_opened_fully = controller.last_event.metadata["lastActionSuccess"] controller.step( "CloseObject", objectId=obj["objectId"], forceAction=True, ) obj_closed_fully = controller.last_event.metadata["lastActionSuccess"] return obj_opened_fully and obj_closed_fully
def get_all_objtype_in_room(room_type: str): ''' Get all Object Types in a certern room type: kitchen, living_room, bedroom, bathroom ''' if room_type == "kitchen": room_start_index = 0 elif room_type == "living_room": room_start_index = 200 elif room_type == "bedroom": room_start_index = 300 else: room_start_index = 400 all_obj_type = [] controller = Controller(scene="FloorPlan1", renderInstanceSegmentation=True, width=1080, height=1080) for i in range(1, 31): controller.reset(scene="FloorPlan" + str(room_start_index + i)) event = controller.step("Done") all_obj = get_all_objtype_in_event(event) for objtype in all_obj: if objtype not in all_obj_type: all_obj_type.append(objtype) controller.stop() return all_obj_type
def open_objs(objects_to_open: List[dict], controller: Controller) -> Dict[int, Union[float, None]]: """Opens up the chosen pickupable objects if they're openable.""" out: Dict[int, Union[float, None]] = defaultdict(lambda: None) for obj in objects_to_open: last_openness = obj["openness"] new_openness = last_openness while abs(last_openness - new_openness) <= 0.2: new_openness = random.random() controller.step( "OpenObject", objectId=obj["objectId"], openness=new_openness, forceAction=True, ) out[obj["name"]] = new_openness return out
def build_controller(**args): default_args = dict(scene="FloorPlan28", local_build=True) default_args.update(args) # during a ci-build we will get a warning that we are using a commit_id for the # build instead of 'local' with warnings.catch_warnings(): warnings.simplefilter("ignore") c = Controller(**default_args) return c
def __init__(self, grid_size=0.25, fov=90.0, local_executable_path=None, randomize_objects=False, seed=1): random.seed(seed) self.controller = Controller() if local_executable_path: self.controller.local_executable_path = local_executable_path self.grid_size = grid_size self._reachable_points = {} self.fov = fov self.offline_data_dir = './datasets/floorplans' self.y = None self.randomize_objects = randomize_objects self.stored_scene_name = None
def __init__(self, all_metadata_available: bool = True, **kwargs): self.config = dict( rotateStepDegrees=30.0, visibilityDistance=1.0, gridSize=0.25, continuousMode=True, snapToGrid=False, agentMode="locobot", width=640, height=480, agentCount=1, server_class=FifoServer, ) if "agentCount" in kwargs: assert kwargs["agentCount"] > 0 kwargs["agentMode"] = kwargs.get("agentMode", "locobot") if kwargs["agentMode"] not in ["bot", "locobot"]: warnings.warn(f"The RoboTHOR environment has not been tested using" f" an agent of mode '{kwargs['agentMode']}'.") recursive_update(self.config, kwargs) self.controller = Controller(**self.config, ) self.all_metadata_available = all_metadata_available self.scene_to_reachable_positions: Optional[Dict[str, Any]] = None self.distance_cache: Optional[DynamicDistanceCache] = None if self.all_metadata_available: self.scene_to_reachable_positions = { self.scene_name: copy.deepcopy(self.currently_reachable_points) } assert len(self.scene_to_reachable_positions[self.scene_name]) > 10 self.distance_cache = DynamicDistanceCache(rounding=1) self.agent_count = self.config["agentCount"] self._extra_teleport_kwargs: Dict[str, Any] = { } # Used for backwards compatability with the teleport action
def remove_objects_until_all_have_identical_meshes(controller: Controller): obj_type_to_obj_list = defaultdict(lambda: []) for obj in controller.last_event.metadata["objects"]: obj_type_to_obj_list[obj["objectType"]].append(obj) for obj_type in OBJECT_TYPES_THAT_CAN_HAVE_IDENTICAL_MESHES: objs_of_type = list( sorted(obj_type_to_obj_list[obj_type], key=lambda x: x["name"])) random.shuffle(objs_of_type) objs_to_remove = objs_of_type[:-1] for obj_to_remove in objs_to_remove: obj_to_remove_name = obj_to_remove["name"] obj_id_to_remove = next( obj["objectId"] for obj in controller.last_event.metadata["objects"] if obj["name"] == obj_to_remove_name) controller.step("RemoveFromScene", objectId=obj_id_to_remove) if not controller.last_event.metadata["lastActionSuccess"]: return False return True
def start( self, scene_name: Optional[str], move_mag: float = 0.25, **kwargs, ) -> None: """Starts the ai2thor controller if it was previously stopped. After starting, `reset` will be called with the scene name and move magnitude. # Parameters scene_name : The scene to load. move_mag : The amount of distance the agent moves in a single `MoveAhead` step. kwargs : additional kwargs, passed to reset. """ if self._started: raise RuntimeError( "Trying to start the environment but it is already started.") self.controller = Controller( x_display=self.x_display, width=self._start_player_screen_width, height=self._start_player_screen_height, local_executable_path=self._local_thor_build, quality=self._quality, server_class=ai2thor.fifo_server.FifoServer, ) if ( self._start_player_screen_height, self._start_player_screen_width, ) != self.current_frame.shape[:2]: self.controller.step({ "action": "ChangeResolution", "x": self._start_player_screen_width, "y": self._start_player_screen_height, }) self._started = True self.reset(scene_name=scene_name, move_mag=move_mag, **kwargs)
def make_top_down_views(self) -> Dict[str, np.ndarray]: top_downs = {} for roomname in self.iterate_scenes(self.scenes): fname = self.cached_image_path(roomname) if not os.path.exists(fname): if self.controller is None: self.controller = Controller() self.controller.step({ "action": "ChangeQuality", "quality": "Very High" }) self.controller.step({ "action": "ChangeResolution", "x": self.viz_rows_cols[1], "y": self.viz_rows_cols[0], }) self.dump_top_down_view(roomname, fname) top_downs[roomname] = cv2.imread(fname) return top_downs
def __init__(self, fov=60.0, action_n=6, camera_Y=0.675, grid_size=0.20, visibility_distance=1.5, player_screen_width=300, player_screen_height=300, full_scrn=False, depth_image=False, class_image=False, top_view_cam=False, object_image=False, third_party_cam=False, random_init=False, random_goals=False, scene="FloorPlan220", object_name="Television"): # self.scene = scene self.grid_size = grid_size self.depth_image = depth_image self.class_image = class_image self.object_image = object_image self.visibility_distance = visibility_distance self.camera_Y = camera_Y self.fov = fov self.scene = scene self.object_name = object_name self.player_screen_width = player_screen_width self.player_screen_height = player_screen_height self.top_view_cam = top_view_cam self.third_party_cam = third_party_cam self.full_scrn = full_scrn self.random_init = random_init self.orientations = [0.0, 90.0, 180.0, 270.0, 360.0] self.action_n = action_n self.random_goal = random_goals self.ctrl = Controller() # headless=True
def __init__(self, offline_root='', scene='FloorPlan_Train1_1', verbose=False): self.verbose = verbose self.online_controller = Controller(width=640, height=480, rotateStepDegrees=30, applyActionNoise=False, gridSize=0.125, snapToGrid=False, agentMode='bot') # self.online_controller = ai2thor.controller.Controller( # start_unity=True, # width=640, # height=480, # agentMode='bot', # gridSize=0.125, # rotateStepDegrees=30, # applyActionNoise=False, # snapToGrid=False) self.scene = scene if offline_root != '': self.offline_controller = OfflineControllerWithSmallRotation( grid_size=0.125, fov=90, offline_data_dir=offline_root, visualize=False, rotate_by=30, state_decimal=3) else: self.offline_controller = None if scene != '': self.online_controller.reset(scene) if self.offline_controller is not None: self.offline_controller.reset(scene)
def __init__(self, pipe, gpu, configs, ground_truth=0, run=True): self.pipe = pipe self.gc, self.ac = configs self.ground_truth = ground_truth self.directions = [ dict(x=a, y=b, z=c) for a in [-1, 1] for b in [1, 2] for c in [-1, 1] ] self.controller = Controller( x_display='0.%d' % gpu, visibilityDistance=self.ac.visibilityDistance, renderDepthImage=self.gc.depth or ground_truth > 0, renderClassImage=ground_truth > 0, renderObjectImage=ground_truth > 0) self.grid = [(x, y) for x in range(self.gc.grid_size) for y in range(self.gc.grid_size)] self.depth_correction = self._make_depth_correction( self.gc.resolution, self.gc.resolution, 90) self.kernel_size = (self.ac.check_change_kernel.shape[0] - 1) // 2 if run: self.run()
def __init__(self,env_name,timelimit,stack_frame=1,color=True,image_size=128,render_all=True): assert env_name in ENV_LIST self.env_name = env_name self.timelimit = timelimit if render_all == True: self.controller = Controller(scene=self.env_name, agentMode='bot',gridSize=0.25,rotateStepDegrees=10,renderDepthImage=True,renderObjectImage=True,renderClassImage=True) else: self.controller = Controller(scene=self.env_name, agentMode='bot',gridSize=0.25,rotateStepDegrees=10) self.time = 0 self.score = 0 self.color = color self.stack_frame = stack_frame self.image_size =image_size self.render_all = render_all self.action_space = Discrete(5) if color==True: self.observation_space = Box(-1,1,(image_size,image_size,3*stack_frame,)) self.channel = 3 else: self.observation_space = Box(-1,1,(image_size,image_size,1*stack_frame,)) self.channel = 1 self.state = np.zeros(self.observation_space.shape,dtype=np.float32)
def get_translator(self) -> Dict[str, Any]: roomname = list(ThorViz.iterate_scenes(self.scenes))[0] json_file = self.cached_map_data_path(roomname) if not os.path.exists(json_file): if self.controller is None: self.controller = Controller() self.controller.reset(roomname) map_data = self.get_agent_map_data() get_logger().info("Dumping {}".format(json_file)) with open(json_file, "w") as f: json.dump(map_data, f, indent=4, sort_keys=True) else: with open(json_file, "r") as f: map_data = json.load(f) pos_translator = ThorPositionTo2DFrameTranslator( self.viz_rows_cols, self.position_to_tuple(map_data["cam_position"]), map_data["cam_orth_size"], ) map_data["pos_translator"] = pos_translator get_logger().debug("Using map_data {}".format(map_data)) return map_data
def main(args): c = Controller() c.start() sence = args.sence c.reset("FloorPlan" + str(sence)) t = get_agent_map_data(c) new_frame = add_agent_view_triangle( position_to_tuple(c.last_event.metadata["agent"]["position"]), c.last_event.metadata["agent"]["rotation"]["y"], t["frame"], t["pos_translator"], ) plt.imshow(new_frame) plt.axis('off') plt.savefig('topview.png') plt.show() im = Image.open('topview.png') im = trim(im) im.save('topview.png')