def __init__(self, all_metadata_available: bool = True, **kwargs):
self.config = dict(
rotateStepDegrees=30.0,
visibilityDistance=1.0,
gridSize=0.25,
agentType="stochastic",
continuousMode=True,
snapToGrid=False,
agentMode="locobot",
width=640,
height=480,
agentCount=1,
server_class=FifoServer,
)
if "agentCount" in kwargs:
assert kwargs["agentCount"] > 0
recursive_update(self.config, {**kwargs, "agentMode": "locobot"})
self.controller = Controller(**self.config, )
self.all_metadata_available = all_metadata_available
self.scene_to_reachable_positions: Optional[Dict[str, Any]] = None
self.distance_cache: Optional[DynamicDistanceCache] = None
if self.all_metadata_available:
self.scene_to_reachable_positions = {
self.scene_name: copy.deepcopy(self.currently_reachable_points)
}
assert len(self.scene_to_reachable_positions[self.scene_name]) > 10
self.distance_cache = DynamicDistanceCache(rounding=1)
self.agent_count = self.config["agentCount"]
def restart(self):
### reset the unity to avoid some latency issue ...
if isinstance(self.controller, type(None)):
self.controller = Controller(local_executable_path=PATH,
scene="FloorPlan201_physics",
x_display=self.x_display,
agentMode='drone',
fieldOfView=60,
port=self.port)
else:
self.controller.stop()
self.controller = Controller(local_executable_path=PATH,
scene="FloorPlan201_physics",
x_display=self.x_display,
agentMode='drone',
fieldOfView=60,
port=self.port)
_ = self.controller.reset(self.trajectory['scene'][0])
_ = self.controller.step(
dict(action='ChangeAutoResetTimeScale',
timeScale=self.cfg.thor.time_scale))
_ = self.controller.step(
dict(action='ChangeFixedDeltaTime',
fixedDeltaTime=self.cfg.thor.delta_time))
print('Thread:' + str(self.pid) + ' [' + str(self.iters + 1) +
']: restart finish')
def __init__(self, **kwargs):
self.config = dict(
rotateStepDegrees=30.0,
visibilityDistance=1.0,
gridSize=0.25,
agentType="stochastic",
continuousMode=True,
snapToGrid=False,
agentMode="bot",
width=640,
height=480,
)
recursive_update(self.config, {**kwargs, "agentMode": "bot"})
self.controller = Controller(**self.config)
self.known_good_locations: Dict[str, Any] = {
self.scene_name: copy.deepcopy(self.currently_reachable_points)
}
assert len(self.known_good_locations[self.scene_name]) > 10
# onames = [o['objectId'] for o in self.last_event.metadata['objects']]
# removed = []
# for oname in onames:
# if 'Painting' in oname:
# self.controller.step("RemoveFromScene", objectId=oname)
# removed.append(oname)
# get_logger().info("Removed {} Paintings from {}".format(len(removed), self.scene_name))
# get_logger().warning("init to scene {} in pos {}".format(self.scene_name, self.agent_state()))
# npoints = len(self.currently_reachable_points)
# assert npoints > 100, "only {} reachable points after init".format(npoints)
self.grids: Dict[str, Tuple[Dict[str, np.array], int, int, int,
int]] = {}
self.initialize_grid()
def __init__(self,
fov=60.0,
camera_Y=0.675,
grid_size=0.25,
visibility_distance=1.5,
player_screen_width=300,
player_screen_height=300,
full_scrn=False,
depth_image=False,
class_image=False,
top_view_cam=False,
object_image=False,
third_party_cam=False,
scene="FloorPlan220",
object_name="Television"):
self.scene = scene
self.grid_size = grid_size
self.depth_image = depth_image
self.class_image = class_image
self.object_image = object_image
self.visibility_distance = visibility_distance
self.camera_Y = camera_Y
self.fov = fov
self.object_name = object_name
self.player_screen_width = player_screen_width
self.player_screen_height = player_screen_height
self.top_view_cam = top_view_cam
self.third_party_cam = third_party_cam
self.full_scrn = full_scrn
self.ctrl = Controller()
def run_episodes(self):
self.ep_idx = 0
# self.objects = []
for episode in range(self.num_episodes):
print("STARTING EPISODE ", episode)
mapname = self.mapnames[episode]
print("MAPNAME=", mapname)
self.controller = Controller(
scene=mapname,
gridSize=0.25,
width=self.W,
height=self.H,
fieldOfView=self.fov,
renderObjectImage=True,
renderDepthImage=True,
)
self.basepath = self.homepath + f"/{mapname}_{episode}"
print("BASEPATH: ", self.basepath)
# self.basepath = f"/hdd/ayushj/habitat_data/{mapname}_{episode}"
if not os.path.exists(self.basepath):
os.mkdir(self.basepath)
self.run()
self.controller.stop()
time.sleep(1)
self.ep_idx += 1
def __init__(self, **kwargs):
self.config = dict(
rotateStepDegrees=30.0,
visibilityDistance=1.0,
gridSize=0.25,
agentType="stochastic",
continuousMode=True,
snapToGrid=False,
agentMode="bot",
width=640,
height=480,
agentCount=1,
)
if "agentCount" in kwargs:
assert kwargs["agentCount"] > 0
recursive_update(self.config, {**kwargs, "agentMode": "bot"})
self.controller = Controller(
**self.config, server_class=ai2thor.fifo_server.FifoServer)
self.known_good_locations: Dict[str, Any] = {
self.scene_name: copy.deepcopy(self.currently_reachable_points)
}
self.distance_cache = DynamicDistanceCache(rounding=1)
assert len(self.known_good_locations[self.scene_name]) > 10
self.agent_count = self.config["agentCount"]
def run_val(self, mapnames, BATCH_SIZE):
batch = {"rewards": [], "obs": [], "actions": []}
episode_rewards = 0.0
episode_steps = []
iter_idx = 0
while True:
mapname = np.random.choice(mapnames)
# self.basepath = self.homepath + f"/{mapname}_{episode}"
# print("BASEPATH: ", self.basepath)
# # self.basepath = f"/hdd/ayushj/habitat_data/{mapname}_{episode}"
# if not os.path.exists(self.basepath):
# os.mkdir(self.basepath)
self.controller = Controller(
scene=mapname,
gridSize=0.25,
width=self.W,
height=self.H,
fieldOfView=self.fov,
renderObjectImage=True,
renderDepthImage=True,
)
episode_rewards, obs, actions = self.run()
print("Total reward for val batch # ", iter_idx, " :",
episode_rewards)
self.controller.stop()
time.sleep(1)
if episode_rewards is None:
print("NO EPISODE REWARDS.. SKIPPING BATCH INSTANCE")
continue
batch["rewards"].append(episode_rewards)
batch["obs"].append(obs)
batch["actions"].append(actions)
train, labels, mean_rewards, boudary = self.elite_batch(
batch, self.percentile)
with torch.no_grad():
scores = self.cemnet(train, softmax=False)
loss_value = self.loss(scores, labels)
iter_idx += 1
if len(batch["rewards"]) == BATCH_SIZE:
break
return loss_value, mean_rewards
def teleport_to_base_location(controller: Controller):
assert (
controller.last_event.metadata["sceneName"].replace("_physics", "")
== "FloorPlan28"
)
controller.step("TeleportFull", **BASE_FP28_LOCATION)
assert controller.last_event.metadata["lastActionSuccess"]
def run_load(dataset_path, memory, conn):
init_logging()
controller = Controller(
x_display='0.%d' % global_config.actor_gpu,
visibilityDistance=actor_config.visibilityDistance,
renderDepthImage=global_config.depth)
dataset = ActiveDataset(dataset_path, memory, controller, conn)
dataset.process()
controller.stop()
def make_controller(self):
if self.controller is None:
self.controller = Controller()
self.controller.step({"action": "ChangeQuality", "quality": "Very High"})
self.controller.step(
{
"action": "ChangeResolution",
"x": self.viz_rows_cols[1],
"y": self.viz_rows_cols[0],
}
)
def batch_iteration(self, mapnames, NN, BATCH_SIZE):
batch = {"rewards": [], "obs": [], "actions": []}
episode_rewards = 0.0
iter_idx = 0
while True:
mapname = np.random.choice(mapnames)
# self.basepath = self.homepath + f"/{mapname}_{episode}"
# print("BASEPATH: ", self.basepath)
# # self.basepath = f"/hdd/ayushj/habitat_data/{mapname}_{episode}"
# if not os.path.exists(self.basepath):
# os.mkdir(self.basepath)
self.controller = Controller(
scene=mapname,
gridSize=0.25,
width=self.W,
height=self.H,
fieldOfView=self.fov,
renderObjectImage=True,
renderDepthImage=True,
)
episode_rewards, obs, actions = self.run()
print("Total reward for train batch # ", iter_idx, " :",
episode_rewards)
self.controller.stop()
time.sleep(1)
if episode_rewards is None:
print("NO EPISODE REWARDS.. SKIPPING BATCH INSTANCE")
continue
batch["rewards"].append(episode_rewards)
batch["obs"].append(obs)
batch["actions"].append(actions)
iter_idx += 1
print(len(batch["rewards"]))
if len(batch["rewards"]) == BATCH_SIZE:
yield batch
iter_idx = 0
batch = {"rewards": [], "obs": [], "actions": []}
if len(batch["rewards"]) > BATCH_SIZE:
st()
def get_agent_map_data(c: Controller):
c.step(dict(action='ToggleMapView'))
cam_position = c.last_event.metadata["cameraPosition"]
cam_orth_size = c.last_event.metadata["cameraOrthSize"]
pos_converted = converter(c.last_event.frame.shape,
get_position(cam_position), cam_orth_size)
to_return = {
"frame": c.last_event.frame,
"cam_position": cam_position,
"cam_orth_size": cam_orth_size,
"pos_converted": pos_converted
}
c.step(dict(action='ToggleMapView'))
return to_return
def get_agent_map_data(c: Controller):
c.step({"action": "ToggleMapView"})
cam_position = c.last_event.metadata["cameraPosition"]
cam_orth_size = c.last_event.metadata["cameraOrthSize"]
pos_translator = ThorPositionTo2DFrameTranslator(
c.last_event.frame.shape, position_to_tuple(cam_position),
cam_orth_size)
to_return = {
"frame": c.last_event.frame,
"cam_position": cam_position,
"cam_orth_size": cam_orth_size,
"pos_translator": pos_translator,
}
c.step({"action": "ToggleMapView"})
return to_return
def open_objs(object_names_to_open: List[str],
controller: Controller) -> Dict[str, Optional[float]]:
"""Opens up the chosen pickupable objects if they're openable."""
out: Dict[str, Optional[float]] = defaultdict(lambda: None)
for obj_name in object_names_to_open:
obj = get_object_by_name(obj_name, controller)
last_openness = obj["openness"]
new_openness = last_openness
while abs(last_openness - new_openness) <= OPENNESS_THRESHOLD:
new_openness = random.random()
event = controller.step(
"OpenObject",
objectId=obj["objectId"],
openness=new_openness,
forceAction=True,
)
obj_after_open = get_object_by_name(obj_name, controller)
if abs(obj_after_open["openness"] - new_openness) > 0.001:
raise RuntimeError(
f"In scene {event.metadata['sceneName']}, {obj['name']} was supposed to open to {new_openness}"
f" from {last_openness} but instead reached {obj_after_open['openness']}. Last action success was:"
f" {event.metadata['lastActionSuccess']}")
out[obj["name"]] = obj_after_open["openness"]
return out
def create_controller():
return Controller(
x_display=self.x_display,
player_screen_width=self._start_player_screen_width,
player_screen_height=self._start_player_screen_height,
local_executable_path=self._local_thor_build,
quality=self._quality,
)
def check_object_opens(obj: Dict[str, Any], controller: Controller):
controller.step(
"OpenObject",
objectId=obj["objectId"],
openness=1.0,
forceAction=True,
)
obj_opened_fully = controller.last_event.metadata["lastActionSuccess"]
controller.step(
"CloseObject",
objectId=obj["objectId"],
forceAction=True,
)
obj_closed_fully = controller.last_event.metadata["lastActionSuccess"]
return obj_opened_fully and obj_closed_fully
def get_all_objtype_in_room(room_type: str):
'''
Get all Object Types in a certern room type: kitchen, living_room, bedroom, bathroom
'''
if room_type == "kitchen":
room_start_index = 0
elif room_type == "living_room":
room_start_index = 200
elif room_type == "bedroom":
room_start_index = 300
else:
room_start_index = 400
all_obj_type = []
controller = Controller(scene="FloorPlan1",
renderInstanceSegmentation=True,
width=1080,
height=1080)
for i in range(1, 31):
controller.reset(scene="FloorPlan" + str(room_start_index + i))
event = controller.step("Done")
all_obj = get_all_objtype_in_event(event)
for objtype in all_obj:
if objtype not in all_obj_type:
all_obj_type.append(objtype)
controller.stop()
return all_obj_type
def open_objs(objects_to_open: List[dict],
controller: Controller) -> Dict[int, Union[float, None]]:
"""Opens up the chosen pickupable objects if they're openable."""
out: Dict[int, Union[float, None]] = defaultdict(lambda: None)
for obj in objects_to_open:
last_openness = obj["openness"]
new_openness = last_openness
while abs(last_openness - new_openness) <= 0.2:
new_openness = random.random()
controller.step(
"OpenObject",
objectId=obj["objectId"],
openness=new_openness,
forceAction=True,
)
out[obj["name"]] = new_openness
return out
def build_controller(**args):
default_args = dict(scene="FloorPlan28", local_build=True)
default_args.update(args)
# during a ci-build we will get a warning that we are using a commit_id for the
# build instead of 'local'
with warnings.catch_warnings():
warnings.simplefilter("ignore")
c = Controller(**default_args)
return c
def __init__(self,
grid_size=0.25,
fov=90.0,
local_executable_path=None,
randomize_objects=False,
seed=1):
random.seed(seed)
self.controller = Controller()
if local_executable_path:
self.controller.local_executable_path = local_executable_path
self.grid_size = grid_size
self._reachable_points = {}
self.fov = fov
self.offline_data_dir = './datasets/floorplans'
self.y = None
self.randomize_objects = randomize_objects
self.stored_scene_name = None
def __init__(self, all_metadata_available: bool = True, **kwargs):
self.config = dict(
rotateStepDegrees=30.0,
visibilityDistance=1.0,
gridSize=0.25,
continuousMode=True,
snapToGrid=False,
agentMode="locobot",
width=640,
height=480,
agentCount=1,
server_class=FifoServer,
)
if "agentCount" in kwargs:
assert kwargs["agentCount"] > 0
kwargs["agentMode"] = kwargs.get("agentMode", "locobot")
if kwargs["agentMode"] not in ["bot", "locobot"]:
warnings.warn(f"The RoboTHOR environment has not been tested using"
f" an agent of mode '{kwargs['agentMode']}'.")
recursive_update(self.config, kwargs)
self.controller = Controller(**self.config, )
self.all_metadata_available = all_metadata_available
self.scene_to_reachable_positions: Optional[Dict[str, Any]] = None
self.distance_cache: Optional[DynamicDistanceCache] = None
if self.all_metadata_available:
self.scene_to_reachable_positions = {
self.scene_name: copy.deepcopy(self.currently_reachable_points)
}
assert len(self.scene_to_reachable_positions[self.scene_name]) > 10
self.distance_cache = DynamicDistanceCache(rounding=1)
self.agent_count = self.config["agentCount"]
self._extra_teleport_kwargs: Dict[str, Any] = {
} # Used for backwards compatability with the teleport action
def remove_objects_until_all_have_identical_meshes(controller: Controller):
obj_type_to_obj_list = defaultdict(lambda: [])
for obj in controller.last_event.metadata["objects"]:
obj_type_to_obj_list[obj["objectType"]].append(obj)
for obj_type in OBJECT_TYPES_THAT_CAN_HAVE_IDENTICAL_MESHES:
objs_of_type = list(
sorted(obj_type_to_obj_list[obj_type], key=lambda x: x["name"]))
random.shuffle(objs_of_type)
objs_to_remove = objs_of_type[:-1]
for obj_to_remove in objs_to_remove:
obj_to_remove_name = obj_to_remove["name"]
obj_id_to_remove = next(
obj["objectId"]
for obj in controller.last_event.metadata["objects"]
if obj["name"] == obj_to_remove_name)
controller.step("RemoveFromScene", objectId=obj_id_to_remove)
if not controller.last_event.metadata["lastActionSuccess"]:
return False
return True
def start(
self,
scene_name: Optional[str],
move_mag: float = 0.25,
**kwargs,
) -> None:
"""Starts the ai2thor controller if it was previously stopped.
After starting, `reset` will be called with the scene name and move magnitude.
# Parameters
scene_name : The scene to load.
move_mag : The amount of distance the agent moves in a single `MoveAhead` step.
kwargs : additional kwargs, passed to reset.
"""
if self._started:
raise RuntimeError(
"Trying to start the environment but it is already started.")
self.controller = Controller(
x_display=self.x_display,
width=self._start_player_screen_width,
height=self._start_player_screen_height,
local_executable_path=self._local_thor_build,
quality=self._quality,
server_class=ai2thor.fifo_server.FifoServer,
)
if (
self._start_player_screen_height,
self._start_player_screen_width,
) != self.current_frame.shape[:2]:
self.controller.step({
"action": "ChangeResolution",
"x": self._start_player_screen_width,
"y": self._start_player_screen_height,
})
self._started = True
self.reset(scene_name=scene_name, move_mag=move_mag, **kwargs)
def make_top_down_views(self) -> Dict[str, np.ndarray]:
top_downs = {}
for roomname in self.iterate_scenes(self.scenes):
fname = self.cached_image_path(roomname)
if not os.path.exists(fname):
if self.controller is None:
self.controller = Controller()
self.controller.step({
"action": "ChangeQuality",
"quality": "Very High"
})
self.controller.step({
"action": "ChangeResolution",
"x": self.viz_rows_cols[1],
"y": self.viz_rows_cols[0],
})
self.dump_top_down_view(roomname, fname)
top_downs[roomname] = cv2.imread(fname)
return top_downs
def __init__(self,
fov=60.0,
action_n=6,
camera_Y=0.675,
grid_size=0.20,
visibility_distance=1.5,
player_screen_width=300,
player_screen_height=300,
full_scrn=False,
depth_image=False,
class_image=False,
top_view_cam=False,
object_image=False,
third_party_cam=False,
random_init=False,
random_goals=False,
scene="FloorPlan220",
object_name="Television"):
# self.scene = scene
self.grid_size = grid_size
self.depth_image = depth_image
self.class_image = class_image
self.object_image = object_image
self.visibility_distance = visibility_distance
self.camera_Y = camera_Y
self.fov = fov
self.scene = scene
self.object_name = object_name
self.player_screen_width = player_screen_width
self.player_screen_height = player_screen_height
self.top_view_cam = top_view_cam
self.third_party_cam = third_party_cam
self.full_scrn = full_scrn
self.random_init = random_init
self.orientations = [0.0, 90.0, 180.0, 270.0, 360.0]
self.action_n = action_n
self.random_goal = random_goals
self.ctrl = Controller() # headless=True
def __init__(self,
offline_root='',
scene='FloorPlan_Train1_1',
verbose=False):
self.verbose = verbose
self.online_controller = Controller(width=640,
height=480,
rotateStepDegrees=30,
applyActionNoise=False,
gridSize=0.125,
snapToGrid=False,
agentMode='bot')
# self.online_controller = ai2thor.controller.Controller(
# start_unity=True,
# width=640,
# height=480,
# agentMode='bot',
# gridSize=0.125,
# rotateStepDegrees=30,
# applyActionNoise=False,
# snapToGrid=False)
self.scene = scene
if offline_root != '':
self.offline_controller = OfflineControllerWithSmallRotation(
grid_size=0.125,
fov=90,
offline_data_dir=offline_root,
visualize=False,
rotate_by=30,
state_decimal=3)
else:
self.offline_controller = None
if scene != '':
self.online_controller.reset(scene)
if self.offline_controller is not None:
self.offline_controller.reset(scene)
def __init__(self, pipe, gpu, configs, ground_truth=0, run=True):
self.pipe = pipe
self.gc, self.ac = configs
self.ground_truth = ground_truth
self.directions = [
dict(x=a, y=b, z=c) for a in [-1, 1] for b in [1, 2]
for c in [-1, 1]
]
self.controller = Controller(
x_display='0.%d' % gpu,
visibilityDistance=self.ac.visibilityDistance,
renderDepthImage=self.gc.depth or ground_truth > 0,
renderClassImage=ground_truth > 0,
renderObjectImage=ground_truth > 0)
self.grid = [(x, y) for x in range(self.gc.grid_size)
for y in range(self.gc.grid_size)]
self.depth_correction = self._make_depth_correction(
self.gc.resolution, self.gc.resolution, 90)
self.kernel_size = (self.ac.check_change_kernel.shape[0] - 1) // 2
if run:
self.run()
def __init__(self,env_name,timelimit,stack_frame=1,color=True,image_size=128,render_all=True):
assert env_name in ENV_LIST
self.env_name = env_name
self.timelimit = timelimit
if render_all == True:
self.controller = Controller(scene=self.env_name, agentMode='bot',gridSize=0.25,rotateStepDegrees=10,renderDepthImage=True,renderObjectImage=True,renderClassImage=True)
else:
self.controller = Controller(scene=self.env_name, agentMode='bot',gridSize=0.25,rotateStepDegrees=10)
self.time = 0
self.score = 0
self.color = color
self.stack_frame = stack_frame
self.image_size =image_size
self.render_all = render_all
self.action_space = Discrete(5)
if color==True:
self.observation_space = Box(-1,1,(image_size,image_size,3*stack_frame,))
self.channel = 3
else:
self.observation_space = Box(-1,1,(image_size,image_size,1*stack_frame,))
self.channel = 1
self.state = np.zeros(self.observation_space.shape,dtype=np.float32)
def get_translator(self) -> Dict[str, Any]:
roomname = list(ThorViz.iterate_scenes(self.scenes))[0]
json_file = self.cached_map_data_path(roomname)
if not os.path.exists(json_file):
if self.controller is None:
self.controller = Controller()
self.controller.reset(roomname)
map_data = self.get_agent_map_data()
get_logger().info("Dumping {}".format(json_file))
with open(json_file, "w") as f:
json.dump(map_data, f, indent=4, sort_keys=True)
else:
with open(json_file, "r") as f:
map_data = json.load(f)
pos_translator = ThorPositionTo2DFrameTranslator(
self.viz_rows_cols,
self.position_to_tuple(map_data["cam_position"]),
map_data["cam_orth_size"],
)
map_data["pos_translator"] = pos_translator
get_logger().debug("Using map_data {}".format(map_data))
return map_data
def main(args):
c = Controller()
c.start()
sence = args.sence
c.reset("FloorPlan" + str(sence))
t = get_agent_map_data(c)
new_frame = add_agent_view_triangle(
position_to_tuple(c.last_event.metadata["agent"]["position"]),
c.last_event.metadata["agent"]["rotation"]["y"],
t["frame"],
t["pos_translator"],
)
plt.imshow(new_frame)
plt.axis('off')
plt.savefig('topview.png')
plt.show()
im = Image.open('topview.png')
im = trim(im)
im.save('topview.png')
