def run_sim(rank, params, shared_model, shared_optimizer, count, lock): if not os.path.exists('./' + params.weight_dir): os.mkdir('./' + params.weight_dir) if not os.path.exists('./log'): os.mkdir('./log') logging.basicConfig(filename='./log/' + params.log_file + '.log', level=logging.INFO) ptitle('Training Agent: {}'.format(rank)) gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)] api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id) cfg = load_config('config.json') torch.manual_seed(random.randint(0, 1000) + rank) if gpu_id >= 0: torch.cuda.manual_seed(random.randint(0, 1000) + rank) model = A3C_LSTM_GA() with torch.cuda.device(gpu_id): model = model.cuda() Agent = run_agent(model, gpu_id) house_id = params.house_id if house_id == -1: house_id = rank if house_id > 50: house_id = house_id % 50 env = Environment(api, get_house_id(house_id, params.difficulty), cfg) task = RoomNavTask(env, hardness=params.hardness, segment_input=params.semantic_mode, max_steps=params.max_steps, discrete_action=True) n_train = 0 best_rate = 0.0 save_model_index = 0 while True: n_train += 1 training(task, gpu_id, shared_model, Agent, shared_optimizer, params, lock, count) if n_train % 1000 == 0: with lock: n_update = count.value with torch.cuda.device(gpu_id): Agent.model.load_state_dict(shared_model.state_dict()) start_time = time.time() best_rate, save_model_index = testing(lock, n_update, gpu_id, Agent, task, best_rate, params, save_model_index, start_time, logging, house_id)
def create_env(house="02f594bb5d8c5871bde0d8c8db20125b"): api = objrender.RenderAPI(w=600, h=450, device=0) cfg = create_default_config('/home/rythei/guide/SUNCG/house') if house is None: env = Environment(api, TEST_HOUSE, cfg) else: env = Environment(api, house, cfg) env.reset() return env
def _initialize(self): h, w = self.screen_size api = objrender.RenderAPI(w=w, h=h, device=0) env = Environment(api, self.scene, self.configuration) env.reset() env = RoomNavTask(env, discrete_action=True, depth_signal=False, segment_input=False, reward_type=None, hardness=self.hardness) self._env = env
def worker(device, id_queue, suncg_dir, width, height, result_dir, invalid_houses): assert osp.isfile(cfg['modelCategoryFile']) and osp.isfile( cfg['colorFile']) #Try allocating device api = objrender.RenderAPI(width, height, device=device) # api.printContextInfo() NN_cache_dir = osp.join(suncg_dir, 'object_vox', 'object_vox_data', '__cache__') os.makedirs(NN_cache_dir, exist_ok=True) model_dir = osp.join(suncg_dir, 'object_vox', 'object_vox_data') obj_cache = NNCache(NN_cache_dir, model_dir) while True: # start = time.time() #Check if we have work to do try: house_id = id_queue.get_nowait() except queue.Empty: return #Get Cameras cam_path = osp.join(suncg_dir, 'camera', house_id, 'room_camera.txt') with open(cam_path) as f: csvreader = csv.reader(f, delimiter=' ', quoting=csv.QUOTE_NONNUMERIC) cameras = [c for c in csvreader] #Create result dir id_result_dir = osp.join(result_dir, house_id) os.makedirs(id_result_dir, exist_ok=True) #If already renderered, skip work if image_folder_is_complete(len(cameras), id_result_dir): logger.info('Skipping images for {}'.format(house_id)) else: invalid_houses.invalidate(house_id) render_scene_images(cameras, api, suncg_dir, house_id, id_result_dir) if voxel_folder_is_complete(len(cameras), id_result_dir): logger.info('Skipping voxels for {}'.format(house_id)) else: invalid_houses.invalidate(house_id) generate_scene_voxel_grids(cameras, suncg_dir, house_id, id_result_dir, obj_cache) invalid_houses.validate(house_id) #Free memory del api
def reset(self, house_id=None, x=None, y=None, yaw=None): if not self.train_mode: obs_map = self.env.house.obsMap.T self.obs_pos = obs_map == 1 self.traj = [] self.traj_actions = [] self.grid_traj = [] if house_id is None: house_id = self.np_random.choice(self.houses_id, 1)[0] self.hid = house_id if self.env is not None: del self.api del self.env self.api = objrender.RenderAPI(self.render_width, self.render_height, device=RENDERING_GPU) self.env = Environment(self.api, house_id, self.config, GridDet=self.configs['GridDet'], RenderDoor=self.render_door, StartIndoor=self.start_indoor) self.tracker = [] self.num_rotate = 0 self.right_rotate = 0 self.L_min = self.env.house.L_lo self.L_max = self.env.house.L_hi self.grid_size = self.env.house.grid_det grid_num = np.array( [self.env.house.n_row[0] + 1, self.env.house.n_row[1] + 1]) self.grids_mat = np.zeros(tuple(grid_num), dtype=np.uint8) self.max_grid_size = np.max(grid_num) self.max_seen_area = float(np.prod(grid_num)) self.env.reset(x=x, y=y, yaw=yaw) self.start_pos, self.grid_start_pos = self.get_camera_grid_pos() if not self.train_mode: self.traj.append(self.start_pos.tolist()) self.grid_traj.append(self.grid_start_pos.tolist()) rgb, depth, true_depth, extrinsics = self.get_obs() self.seen_area = self.get_seen_area(rgb, true_depth, extrinsics, self.grids_mat) self.ep_len = 0 self.ep_reward = 0 self.collision_times = 0 ret_obs = np.concatenate((rgb, depth), axis=-1) return ret_obs
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data_file', default='../path_data/cleaned_human_demo_data.json', type=str) parser.add_argument('--suncg_house_dir', default='../../suncg_data/house', type=str) parser.add_argument('--mode', default='rgb', type=str, choices=['rgb', 'semantic', 'instance', 'depth']) parser.add_argument('--demo_id', default=0, type=int) parser.add_argument('--width', type=int, default=800) parser.add_argument('--height', type=int, default=600) parser.add_argument('--device', type=int, default=0) args = parser.parse_args() with open(args.data_file, 'r') as f: data = json.load(f) demo = data[args.demo_id] cfg = create_default_config('.') api = objrender.RenderAPI(args.width, args.height, device=args.device) api.printContextInfo() obj_path = pathlib.Path(args.suncg_house_dir).joinpath( demo['house_id'], 'house.obj') print('You select house id: {0:s}'.format(demo['house_id'])) api.loadScene(str(obj_path.resolve()), cfg['modelCategoryFile'], cfg['colorFile']) cam = api.getCamera() api_mode = getattr(RenderMode, args.mode.upper()) api.setMode(api_mode) for t in count(): mat = np.array(api.render()) if api_mode == RenderMode.DEPTH: infmask = mat[:, :, 1] mat = mat[:, :, 0] * (infmask == 0) else: mat = mat[:, :, ::-1] if api_mode == RenderMode.INSTANCE: center_rgb = mat[args.height // 2, args.width // 2, ::-1] center_instance = api.getNameFromInstanceColor( center_rgb[0], center_rgb[1], center_rgb[2]) print("Instance ID in the center: ", center_instance) exit_flag = key_control(mat, cam) if exit_flag: break
def GymHouseEnv2(scene='05cac5f7fdd5f8138234164e76a97383', screen_size=(84, 84), goals=['living_room'], configuration=None): h, w = screen_size api = objrender.RenderAPI(w=w, h=h, device=0) env = Environment(api, scene, create_configuration(configuration)) env.reset() env = RoomNavTask2(env, discrete_action=True, depth_signal=False, segment_input=False, reward_type=None) env.observation_space.dtype = np.uint8 return GymHouseWrapper(env, room_types=goals, screen_size=screen_size)
def atari_env(env_id, args, type): api = objrender.RenderAPI(w=OBSERVATION_SPACE_SHAPE[2], h=OBSERVATION_SPACE_SHAPE[1], device=args.gpu_ids[0]) cfg = load_config('config_namazu.json') if type == 'train': pre_env = Environment(api, HOUSE_train[env_id], cfg, seed=args.seed) # TODO: temporarily use HOUSE # pre_env = Environment(api, HOUSE[env_id], cfg) elif type == 'test': pre_env = Environment(api, HOUSE_test[env_id], cfg, seed=args.seed) # TODO: temporarily use HOUSE # pre_env = Environment(api, HOUSE[env_id], cfg) env = RoomNavTask(pre_env, hardness=args.hardness, depth_signal=False, discrete_action=True, max_steps=args.max_episode_length) return env
def worker(idx, house_id, device): colormapFile = "../metadata/colormap_coarse.csv" api = objrender.RenderAPI(w=args.width, h=args.height, device=device) env = Environment(api, house_id, cfg) N = 15000 start = time.time() cnt = 0 env.reset() for t in range(N): cnt += 1 env.move_forward(random.random() * 3, random.random() * 3) mat = env.render() if (cnt % 50 == 0): env.reset() end = time.time() print("Worker {}, speed {:.3f} fps".format(idx, N / (end - start)))
def gen_cache_files(ids, skip_file): configs = get_configs() config = load_config(configs['path'], prefix=configs['par_path']) render_height = configs['render_height'] render_width = configs['render_width'] with open(skip_file, 'r') as f: skip_houses = json.load(f) for idx in tqdm(ids): if idx in skip_houses: continue print(idx) api = objrender.RenderAPI(render_width, render_height, device=0) try: env = Environment(api, idx, config, GridDet=configs['GridDet']) except AssertionError: skip_houses.append(idx) return skip_houses
def test_render(self): api = objrender.RenderAPI(w=SIDE, h=SIDE, device=0) cfg = load_config('config.json') houseID, house = find_first_good_house(cfg) env = Environment(api, house, cfg) location = house.getRandomLocation(ROOM_TYPE) env.reset(*location) # Check RGB env.set_render_mode(RenderMode.RGB) rgb = env.render_cube_map() self.assertEqual(rgb.shape, (SIDE, SIDE * 6, 3)) # Check SEMANTIC env.set_render_mode(RenderMode.RGB) semantic = env.render_cube_map() self.assertEqual(semantic.shape, (SIDE, SIDE * 6, 3)) # Check INSTANCE env.set_render_mode(RenderMode.INSTANCE) instance = env.render_cube_map() self.assertEqual(instance.shape, (SIDE, SIDE * 6, 3)) # Check DEPTH env.set_render_mode(RenderMode.DEPTH) depth = env.render_cube_map() self.assertEqual(depth.dtype, np.uint8) self.assertEqual(depth.shape, (SIDE, SIDE * 6, 2)) depth_value = depth[0, 0, 0] * 20.0 / 255.0 # Check INVDEPTH env.set_render_mode(RenderMode.INVDEPTH) invdepth = env.render_cube_map() self.assertEqual(invdepth.dtype, np.uint8) self.assertEqual(invdepth.shape, (SIDE, SIDE * 6, 3)) # Convert to 16 bit and then to depth id16 = 256 * invdepth[:, :, 0] + invdepth[:, :, 1] depth2 = depth_of_inverse_depth(id16) self.assertEqual(depth2.dtype, np.float) self.assertEqual(depth2.shape, (SIDE, SIDE * 6)) # Check that depth value is within 5% of depth from RenderMode.DEPTH self.assertAlmostEqual( depth2[0, 0], depth_value, delta=depth_value * 0.05)
def worker(idx, device, num_iter): api = objrender.RenderAPI(args.width, args.height, device=device) api.printContextInfo() mappingFile = cfg['modelCategoryFile'] colormapFile = cfg['colorFile'] assert os.path.isfile(mappingFile) and os.path.isfile(colormapFile) api.loadScene(args.obj, mappingFile, colormapFile) cam = api.getCamera() start = time.time() for t in range(num_iter): if t % 2 == 1: api.setMode(RenderMode.RGB) else: api.setMode(RenderMode.SEMANTIC) mat = np.array(api.render(), copy=False) end = time.time() print("Worker {}, speed {:.3f} fps".format(idx, num_iter / (end - start)))
def run_test(rank, params, loaded_model, lock, seen_succ, seen_length, unseen_succ, unseen_length): logging.basicConfig(filename='./log/' + params.log_file + '.log', level=logging.INFO) ptitle('Test Agent: {}'.format(rank)) gpu_id = params.gpu_ids_test[rank % len(params.gpu_ids_test)] api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id) cfg = load_config('config.json') torch.manual_seed(params.seed + rank) if gpu_id >= 0: with torch.cuda.device(gpu_id): torch.cuda.manual_seed(params.seed + rank) model = A3C_LSTM_GA() with torch.cuda.device(gpu_id): model = model.cuda() load_model = torch.load( loaded_model, map_location=lambda storage, loc: storage.cuda(gpu_id)) model.load_state_dict(load_model) model.eval() Agent = run_agent(model, gpu_id) n_test = 0 start_time = time.time() while True: house_id = rank + (n_test * params.n_process) if house_id >= 70: break else: if house_id < 20: seen = True house = get_house_id(house_id) else: seen = False house = get_eval_house_id(house_id - (n_test * params.n_process)) env = Environment(api, house, cfg) task = RoomNavTask(env, hardness=params.hardness, segment_input=params.semantic_mode, max_steps=params.max_steps, discrete_action=True) #reward_type='indicator' eval = [] for i in range(params.n_test): next_observation = task.reset() target = task.info['target_room'] target = get_instruction_idx(target) with torch.cuda.device(gpu_id): target = Variable(torch.LongTensor(target)).cuda() Agent.cx = Variable(torch.zeros(1, 256).cuda()) Agent.hx = Variable(torch.zeros(1, 256).cuda()) Agent.target = target step, total_rew, good = 0, 0, 0 done = False while not done: observation = next_observation act = Agent.action_test(observation, target) next_observation, rew, done, info = task.step(actions[act[0]]) total_rew += rew if rew == 10: # success good = 1 step += 1 if done: break eval.append((step, total_rew, good)) if len(eval) > 0: succ = [e for e in eval if e[2] > 0] succ_rate = (len(succ) / len(eval)) * 100 avg_reward = sum([e[1] for e in eval]) / len(eval) avg_length = sum([e[0] for e in eval]) / len(eval) if seen: msg_seen = "Seen" msg_house = house_id else: msg_seen = "Unseen" msg_house = house_id - 20 msg = " ".join([ "++++++++++ Task Stats +++++++++++\n", "Time {}\n".format( time.strftime("%dd %Hh %Mm %Ss", time.gmtime(time.time() - start_time))), "Episode Played: {:d}\n".format(len(eval)), "{:s} House id: {:d}\n".format(msg_seen, msg_house), "Avg Reward = {:5.3f}\n".format(avg_reward), "Avg Length = {:.3f}\n".format(avg_length), "Success rate {:3.2f}%".format(succ_rate) ]) print(msg) logging.info(msg) with lock: if seen: seen_succ.value += len(succ) seen_length.value += sum([e[0] for e in eval]) else: unseen_succ.value += len(succ) unseen_length.value += sum([e[0] for e in eval]) n_test += 1
def test(rank, params, shared_model, count, lock, best_acc, evaluation=True): if not os.path.exists('./' + params.weight_dir): os.mkdir('./' + params.weight_dir) if not os.path.exists('./log'): os.mkdir('./log') logging.basicConfig(filename='./log/' + params.log_file + '.log', level=logging.INFO) ptitle('Test Agent: {}'.format(rank)) gpu_id = params.gpu_ids_test[rank % len(params.gpu_ids_test)] api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id) cfg = load_config('config.json') best_rate = 0.0 save_model_index = 0 n_update = 0 torch.manual_seed(params.seed + rank) if gpu_id >= 0: torch.cuda.manual_seed(params.seed + rank) model = A3C_LSTM_GA() with torch.cuda.device(gpu_id): model = model.cuda() Agent = run_agent(model, gpu_id) house_id = params.house_id if house_id == -1: house_id = rank if house_id >= 20: house_id = house_id % 20 #time.sleep(rank*30) env = Environment(api, get_house_id(house_id), cfg) task = RoomNavTask(env, hardness=params.hardness, segment_input=params.semantic_mode, max_steps=params.max_steps, discrete_action=True) #reward_type='indicator' start_time = time.time() if evaluation is True: max_episode = params.max_episode n_try = params.n_eval else: max_episode = 1 # for loaded model test n_try = params.n_test for episode in range(max_episode): eval = [] if evaluation is True: with lock: n_update = count.value with torch.cuda.device(gpu_id): Agent.model.load_state_dict(shared_model.state_dict()) else: with torch.cuda.device(gpu_id): Agent.model.load_state_dict(shared_model) Agent.model.eval() for i in range(n_try): next_observation = task.reset() target = task.info['target_room'] target = get_instruction_idx(target) with torch.cuda.device(gpu_id): target = Variable(torch.LongTensor(target)).cuda() Agent.cx = Variable(torch.zeros(1, 256).cuda()) Agent.hx = Variable(torch.zeros(1, 256).cuda()) Agent.target = target step, total_rew, good = 0, 0, 0 done = False while not done: observation = next_observation act = Agent.action_test(observation, target) next_observation, rew, done, info = task.step(actions[act[0]]) total_rew += rew if rew == 10: # success good = 1 step += 1 if done: break eval.append((step, total_rew, good)) if len(eval) > 0: succ = [e for e in eval if e[2] > 0] succ_rate = (len(succ) / len(eval)) * 100 if evaluation is True: # evaluation mode with lock: #if best_acc.value >= best_rate: # best_rate = best_acc.value if succ_rate >= best_rate: best_rate = succ_rate with torch.cuda.device(gpu_id): torch.save( Agent.model.state_dict(), params.weight_dir + 'model' + str(n_update) + '.ckpt') save_model_index += 1 #if best_rate > best_acc.value: # best_acc.value = best_rate avg_reward = sum([e[1] for e in eval]) / len(eval) avg_length = sum([e[0] for e in eval]) / len(eval) msg = " ".join([ "++++++++++ Task Stats +++++++++++\n", "Time {}\n".format( time.strftime("%dd %Hh %Mm %Ss", time.gmtime(time.time() - start_time))), "Episode Played: {:d}\n".format(len(eval)), "N_Update = {:d}\n".format(n_update), "House id: {:d}\n".format(house_id), "Avg Reward = {:5.3f}\n".format(avg_reward), "Avg Length = {:.3f}\n".format(avg_length), "Best rate {:3.2f}, Success rate {:3.2f}%".format( best_rate, succ_rate) ]) print(msg) logging.info(msg)
from House3D import objrender, create_default_config from House3D.objrender import RenderMode if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('obj') parser.add_argument('--width', type=int, default=800) parser.add_argument('--height', type=int, default=600) parser.add_argument('--device', type=int, default=0) parser.add_argument('--interactive', action='store_true', help='run interactive rendering (does not work under ssh)') args = parser.parse_args() cfg = create_default_config('.') api = objrender.RenderAPI(args.width, args.height, device=args.device) api.printContextInfo() api.loadScene(args.obj, cfg['modelCategoryFile'], cfg['colorFile']) cam = api.getCamera() modes = [RenderMode.RGB, RenderMode.SEMANTIC, RenderMode.INSTANCE, RenderMode.DEPTH] for t in count(): mode = modes[t % len(modes)] api.setMode(mode) mat = np.array(api.render()) if mode == RenderMode.DEPTH: infmask = mat[:, :, 1] mat = mat[:, :, 0] * (infmask == 0) else: mat = mat[:, :, ::-1] # cv expects bgr
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data_file', default='../path_data/cleaned_human_demo_data.json', type=str) parser.add_argument('--demo_id', default=0, type=int) parser.add_argument('--width', type=int, default=600) parser.add_argument('--height', type=int, default=450) parser.add_argument('--filter_height', type=bool, default=True) args = parser.parse_args() with open(args.data_file, 'r') as f: data = json.load(f) demo = data[args.demo_id] print('Total time steps:', len(demo['loc'])) locs = np.array(demo['loc'][1:]) ques = demo['question'] answer = demo['answer'] text = 'Q: {0:s} A: {1:s}'.format(ques, answer) text_height = 60 cfg = load_config('config.json') api = objrender.RenderAPI(w=args.width, h=args.height, device=0) env = Environment(api, demo['house_id'], cfg) L_min = env.house.L_min_coor L_max = env.house.L_max_coor L_min = np.array([[env.house.L_lo[0], L_min[1], env.house.L_lo[1]]]) L_max = np.array([[env.house.L_hi[0], L_max[1], env.house.L_hi[1]]]) grid_size = env.house.grid_det n_row = env.house.n_row grid_num = np.array([ n_row[0] + 1, int((L_max[0][1] - L_min[0][1]) / (grid_size + 1e-8)) + 1, n_row[1] + 1 ]) print('Grid size:', grid_size) print('Number of grid in [x, y, z]:', grid_num) all_grids = np.zeros(tuple(grid_num), dtype=bool) grid_colors = np.zeros(tuple(grid_num) + (3, ), dtype=np.uint8) loc_map = env.gen_locmap() obs_map = env.house.obsMap.T obs_pos = obs_map == 1 for t in tqdm(range(len(locs))): env.reset(x=locs[t][0], y=locs[t][2], yaw=locs[t][3]) depth = env.render(RenderMode.DEPTH) rgb = env.render(RenderMode.RGB) semantic = env.render(RenderMode.SEMANTIC) infmask = depth[:, :, 1] depth = depth[:, :, 0] * (infmask == 0) true_depth = depth.astype(np.float32) / 255.0 * 20.0 extrinsics = env.cam.getExtrinsicsNumpy() points, points_colors = gen_point_cloud(true_depth, rgb, extrinsics) grid_locs = np.floor((points - L_min) / grid_size).astype(int) all_grids[grid_locs[:, 0], grid_locs[:, 1], grid_locs[:, 2]] = True grid_colors[grid_locs[:, 0], grid_locs[:, 1], grid_locs[:, 2]] = points_colors depth = np.stack([depth] * 3, axis=2) loc_map[grid_locs[:, 2], grid_locs[:, 0], :] = np.array([250, 120, 120]) loc_map[obs_pos] = 0 rad = env.house.robotRad / env.house.grid_det x, y = env.cam.pos.x, env.cam.pos.z x, y = env.house.to_grid(x, y) loc_map_cp = loc_map.copy() cv2.circle(loc_map_cp, (x, y), int(rad), (0, 0, 255), thickness=-1) loc_map_resized = cv2.resize(loc_map_cp, env.resolution) concat1 = np.concatenate((rgb, semantic), axis=1) concat2 = np.concatenate((depth, loc_map_resized), axis=1) ret = np.concatenate((concat1, concat2), axis=0) ret = ret[:, :, ::-1] pad_text = np.ones((text_height, ret.shape[1], ret.shape[2]), dtype=np.uint8) mat_w_text = np.concatenate((np.copy(ret), pad_text), axis=0) cv2.putText(mat_w_text, text, (10, mat_w_text.shape[0] - 10), FONT, 1, (255, 255, 255), 2, cv2.LINE_AA) cv2.imshow("human_demo", mat_w_text) cv2.waitKey(20) valid_grids = np.argwhere(all_grids) valid_grid_center = valid_grids * grid_size + L_min valid_grid_color = grid_colors[valid_grids[:, 0], valid_grids[:, 1], valid_grids[:, 2]] if args.filter_height: height_lim = [-0.1, 2.3] valid_idx = np.logical_and(valid_grid_center[:, 1] >= height_lim[0], valid_grid_center[:, 1] <= height_lim[1]) valid_grid_center = valid_grid_center[valid_idx] valid_grid_color = valid_grid_color[valid_idx] valid_grids = valid_grids[valid_idx] loc_map = env.gen_locmap() obs_map = env.house.obsMap.T loc_map[valid_grids[:, 2], valid_grids[:, 0], :] = np.array([250, 120, 120]) loc_map[obs_map == 1] = 0 loc_map = cv2.resize(loc_map, env.resolution) cv2.imwrite('seen_map.png', loc_map) if USE_OPEN3D: pcd = PointCloud() pcd.points = Vector3dVector(valid_grid_center) pcd.colors = Vector3dVector(valid_grid_color / 255.0) coord = create_mesh_coordinate_frame(3, [40, 0, 35]) draw_geometries([pcd, coord])
def remove_files(paths): for path in paths: try: os.remove(path) except Exception as e: print(path, e) if __name__ == '__main__': # Get house ids to generate data for house_ids = open(args.house_list, 'r').readlines()[args.hid_lo:args.hid_hi] # Initialize renderer API and config api = objrender.RenderAPI(w=args.render_width, h=args.render_height, device=0) cfg = load_config(args.render_config_path) for h in house_ids: house_id = h[:-1] check_path = os.path.join(args.question_dir, house_id + '.json') print(check_path) if os.path.exists(check_path): print('Already have questions for this house --- SKIPPING') continue # Create house .obj and .mtl files in SUNCG scn2scn_path = os.path.join(args.suncgtoolbox_dir, 'gaps/bin/x86_64/scn2scn') house_file_dir = os.path.join(args.suncg_dir, 'house', house_id)
if __name__ == '__main__': from environments.gym_house.env import create_configuration from configuration import configuration import deep_rl from deep_rl.common.console_util import print_progress deep_rl.configure(**configuration) with open(os.path.join(os.path.dirname(__file__),'jobs', 'houses'), 'r') as f: houses = [a.strip() for a in f.readlines()] samples_per_room = 20 screen_size = (512, 512) cfg = create_configuration(deep_rl.configuration.get('house3d').as_dict()) room_target_object = load_target_object_data(cfg['roomTargetFile']) api = objrender.RenderAPI(w=screen_size[1], h=screen_size[0], device=0) for i, houseID in enumerate(houses): print('Processing house %s (%s/%s)' % (houseID, i + 1, len(houses))) house = create_house(houseID, cfg) env = Environment(api, house, cfg) types_to_rooms = get_valid_room_dict(house) for room_type, rooms in types_to_rooms.items(): print('Processing house %s (%s/%s) - %s' % (houseID, i + 1, len(houses), room_type)) locations = get_valid_locations(house, rooms) if len(locations) == 0: print('ERROR: no locations for room %s' % room_type) continue for j in range(samples_per_room):
# All rights reserved. # # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. import tqdm import cv2 import numpy as np import os import pdb from House3D import objrender, Environment, load_config if __name__ == '__main__': api = objrender.RenderAPI(w=250, h=250, device=0) cfg = load_config('config.json') houseID = np.random.choice(os.listdir(cfg['prefix'])) env = Environment(api, houseID, cfg) cam = api.getCamera() for t in tqdm.trange(1000): env.reset() mat = env.renderCubeMap() cv2.imshow("aaa", mat) key = cv2.waitKey(0) while key in [ord('h'), ord('l'), ord('s')]: if key == ord('h'): cam.yaw -= 5
def create_api(): return objrender.RenderAPI(w=250, h=250, device=0)
def gendp(): img = env.render(RenderMode.INVDEPTH) img_16 = img.astype(np.uint16) inverse_depth_16 = img_16[:, :, 0] * 256 + img_16[:, :, 1] PIXEL_MAX = np.iinfo(np.uint16).max NEAR = 0.3 # has to match minDepth parameter depth_float = NEAR * PIXEL_MAX / inverse_depth_16.astype(np.float) #depth_float_3 = depth_float[depth_float > 3.0] = 3.0 depth_float_3 = np.minimum(depth_float, 3.0, depth_float) #depth_float_3 = int() return depth_float_3 #返回的是小于3m的浮点值 if __name__ == '__main__': api = objrender.RenderAPI(w=600, h=450, device=0) #随机地从obsMap中采样 cfg = load_config('config.json') env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg) con = [] loc = [] for i in range(env.house.obsMap.shape[0]): for j in range(env.house.obsMap.shape[1]): if env.house.obsMap[i][j] == 0: con.append([i, j]) for i in range(100): gx, gy = random.choice(con) x, y = env.house.to_coor(gx, gy, True)
# Copyright 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. import cv2 from House3D import objrender, Environment, load_config if __name__ == '__main__': api = objrender.RenderAPI(w=600, h=450, device=0) cfg = load_config('config.json') env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg) env.reset() # put the agent into the house # fourcc = cv2.VideoWriter_fourcc(*'X264') # writer = cv2.VideoWriter('out.avi', fourcc, 30, (1200, 900)) while True: mat = env.debug_render() # writer.write(mat) cv2.imshow("aaa", mat) key = cv2.waitKey(0) if not env.keyboard_control(key): break # writer.release()
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('obj') parser.add_argument('--width', type=int, default=800) parser.add_argument('--height', type=int, default=600) parser.add_argument('--device', type=int, default=0) parser.add_argument( '--interactive', action='store_true', help='run interactive rendering (does not work under ssh)') args = parser.parse_args() cfg = create_default_config('.') api = objrender.RenderAPI(args.width, args.height, device=args.device) #声明一个api,调用了预先编译的库 api.printContextInfo() api.loadScene(args.obj, cfg['modelCategoryFile'], cfg['colorFile']) cam = api.getCamera() modes = [ RenderMode.RGB, RenderMode.SEMANTIC, RenderMode.INSTANCE, RenderMode.DEPTH ] for t in tqdm.trange(1000000): #mode = modes[t % len(modes)] mode = RenderMode.RGB api.setMode(mode) mat = np.array(api.render()) print(mat.shape)
def run_sim(rank, params, shared_model, shared_optimizer, count, lock): ptitle('Training Agent: {}'.format(rank)) gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)] api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id) cfg = load_config('config.json') if shared_optimizer is None: optimizer = optim.Adam(shared_model.parameters(), lr=params.lr, amsgrad=params.amsgrad, weight_decay=params.weight_decay) #optimizer.share_memory() else: optimizer = shared_optimizer torch.manual_seed(params.seed + rank) if gpu_id >= 0: torch.cuda.manual_seed(params.seed + rank) model = A3C_LSTM_GA() with torch.cuda.device(gpu_id): model = model.cuda() Agent = run_agent(model, gpu_id) house_id = params.house_id if house_id == -1: house_id = rank if house_id >= 20: house_id = house_id % 20 env = Environment(api, get_house_id(house_id), cfg) task = RoomNavTask(env, hardness=params.hardness, segment_input=params.semantic_mode, max_steps=params.max_steps, discrete_action=True) for episode in range(params.max_episode): next_observation = task.reset() target = task.info['target_room'] target = get_instruction_idx(target) with torch.cuda.device(gpu_id): target = Variable(torch.LongTensor(target)).cuda() Agent.model.load_state_dict(shared_model.state_dict()) Agent.cx = Variable(torch.zeros(1, 256).cuda()) Agent.hx = Variable(torch.zeros(1, 256).cuda()) Agent.target = target total_reward, num_steps, good = 0, 0, 0 Agent.done = False done = False Agent.eps_len = 0 while not done: num_steps += 1 observation = next_observation act, entropy, value, log_prob = Agent.action_train( observation, target) next_observation, reward, done, info = task.step(actions[act[0]]) rew = np.clip(reward, -1.0, 1.0) Agent.put_reward(rew, entropy, value, log_prob) if num_steps % params.num_steps == 0 or done: if done: Agent.done = done with lock: count.value += 1 Agent.training(next_observation, shared_model, optimizer, params) if done: break
def main(): parser = argparse.ArgumentParser() parser.add_argument('--data_file', default='../path_data/eqa_humans.json', type=str) parser.add_argument('--demo_id', default=0, type=int) parser.add_argument('--width', type=int, default=600) parser.add_argument('--height', type=int, default=450) parser.add_argument('--no_display', action='store_true', help='disable display on screen') parser.add_argument('--play_mode', default='auto', type=str, choices=['key', 'auto'], help='play the human demonstration in ' 'keyboard-control mode or auto-play mode') args = parser.parse_args() with open(args.data_file, 'r') as f: data = json.load(f) demo = data[args.demo_id] print('Total time steps:', len(demo['loc'])) locs = np.array(demo['loc'][1:]) ques = demo['question'] answer = demo['answer'] text = 'Q: {0:s} A: {1:s}'.format(ques, answer) text_height = 60 auto_freq = 15 # 15 hz cfg = load_config('config.json') api = objrender.RenderAPI(w=args.width, h=args.height, device=0) env = Environment(api, demo['house_id'], cfg, GridDet=0.05) save_folder = 'demo_{0:d}'.format(args.demo_id) os.makedirs(save_folder, exist_ok=True) if not args.no_display: print_keyboard_help(args.play_mode) t = 0 while True: # print(t) if t < len(locs): env.reset(x=locs[t][0], y=locs[t][2], yaw=locs[t][3]) mat = get_mat(env, text_height, text) cv2.imshow("human_demo", mat) if args.play_mode == 'key': key = cv2.waitKey(0) if key == 27 or key == ord('q'): # esc break elif key == ord('n'): if t < len(locs) - 1: t += 1 elif key == ord('p'): if t > 0: t -= 1 elif key == ord('r'): t = 0 else: print("Unknown key: {}".format(key)) else: key = cv2.waitKey(int(1000 / auto_freq)) if key == 27 or key == ord('q'): # esc break elif key == -1: if t < len(locs) - 1: t += 1 elif key == ord('r'): t = 0 auto_freq = 15 elif key == ord('i'): if auto_freq < 30: auto_freq += 4 elif key == ord('d'): if auto_freq > 3: auto_freq -= 4 else: print("Unknown key: {}".format(key))
from House3D import Environment, objrender, load_config from House3D.roomnav import RoomNavTask from collections import deque import tqdm import numpy as np EPISODE=1000 api = objrender.RenderAPI( w=400, h=300, device=0) cfg = load_config('config.json') houses = ['00065ecbdd7300d35ef4328ffe871505', 'cf57359cd8603c3d9149445fb4040d90', '31966fdc9f9c87862989fae8ae906295', '7995c2a93311717a3a9c48d789563590', '8b8c1994f3286bfc444a7527ffacde86', '32e53679b33adfcc5a5660b8c758cc96', '492c5839f8a534a673c92912aedc7b63', 'e3ae3f7b32cf99b29d3c8681ec3be321', '1dba3a1039c6ec1a3c141a1cb0ad0757', '5f3f959c7b3e6f091898caa8e828f110'] env = Environment(api, np.random.choice(houses, 1)[0], cfg) task = RoomNavTask(env, hardness=0.6, discrete_action=True) succ = deque(maxlen=500) for i in range(EPISODE): step, total_rew, good = 0, 0, 0 task.reset() while True:
def run_sim(rank, params, state_Queue, action_done, actions, reward_Queue, lock): ptitle('Training Agent: {}'.format(rank)) gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)] api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id) cfg = load_config('config.json') house_id = params.house_id if house_id == -1: house_id = rank if house_id > 50: house_id = house_id % 50 env = Environment(api, get_house_id(house_id, params.difficulty), cfg) task = RoomNavTask(env, hardness=params.hardness, segment_input=params.semantic_mode, max_steps=params.max_steps, discrete_action=True) while True: next_observation = task.reset() target = task.info['target_room'] target = get_instruction_idx(target) # with torch.cuda.device(gpu_id): # target = Variable(torch.LongTensor(target)).cuda() total_reward, num_steps, good = 0, 0, 0 done = False test = False while not done: num_steps += 1 observation = next_observation state = rank, [observation, target] state_Queue.put(state) state_Queue.join() # action_done.get() # action done action = actions[rank] if action == 99: test = True break # call for test next_observation, reward, done, info = task.step(action) reward = np.clip(reward, -1.0, 10.0) if reward != -1.0 and reward != 10.0: # make sparse reward reward = 0.0 total_reward += reward rew = [rank, done, reward] # print("send - rank: {:d}, reward: {:3.2f}".format(rank, reward)) reward_Queue.put(rew) reward_Queue.join() if done: break