def run_sim(rank, params, shared_model, shared_optimizer, count, lock):
if not os.path.exists('./' + params.weight_dir):
os.mkdir('./' + params.weight_dir)
if not os.path.exists('./log'):
os.mkdir('./log')
logging.basicConfig(filename='./log/' + params.log_file + '.log',
level=logging.INFO)
ptitle('Training Agent: {}'.format(rank))
gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
torch.manual_seed(random.randint(0, 1000) + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(random.randint(0, 1000) + rank)
model = A3C_LSTM_GA()
with torch.cuda.device(gpu_id):
model = model.cuda()
Agent = run_agent(model, gpu_id)
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id > 50:
house_id = house_id % 50
env = Environment(api, get_house_id(house_id, params.difficulty), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True)
n_train = 0
best_rate = 0.0
save_model_index = 0
while True:
n_train += 1
training(task, gpu_id, shared_model, Agent, shared_optimizer, params,
lock, count)
if n_train % 1000 == 0:
with lock:
n_update = count.value
with torch.cuda.device(gpu_id):
Agent.model.load_state_dict(shared_model.state_dict())
start_time = time.time()
best_rate, save_model_index = testing(lock, n_update, gpu_id,
Agent, task, best_rate,
params, save_model_index,
start_time, logging,
house_id)
def create_env(house="02f594bb5d8c5871bde0d8c8db20125b"):
api = objrender.RenderAPI(w=600, h=450, device=0)
cfg = create_default_config('/home/rythei/guide/SUNCG/house')
if house is None:
env = Environment(api, TEST_HOUSE, cfg)
else:
env = Environment(api, house, cfg)
env.reset()
return env
def _initialize(self):
h, w = self.screen_size
api = objrender.RenderAPI(w=w, h=h, device=0)
env = Environment(api, self.scene, self.configuration)
env.reset()
env = RoomNavTask(env,
discrete_action=True,
depth_signal=False,
segment_input=False,
reward_type=None,
hardness=self.hardness)
self._env = env
def worker(device, id_queue, suncg_dir, width, height, result_dir,
invalid_houses):
assert osp.isfile(cfg['modelCategoryFile']) and osp.isfile(
cfg['colorFile'])
#Try allocating device
api = objrender.RenderAPI(width, height, device=device)
# api.printContextInfo()
NN_cache_dir = osp.join(suncg_dir, 'object_vox', 'object_vox_data',
'__cache__')
os.makedirs(NN_cache_dir, exist_ok=True)
model_dir = osp.join(suncg_dir, 'object_vox', 'object_vox_data')
obj_cache = NNCache(NN_cache_dir, model_dir)
while True:
# start = time.time()
#Check if we have work to do
try:
house_id = id_queue.get_nowait()
except queue.Empty:
return
#Get Cameras
cam_path = osp.join(suncg_dir, 'camera', house_id, 'room_camera.txt')
with open(cam_path) as f:
csvreader = csv.reader(f,
delimiter=' ',
quoting=csv.QUOTE_NONNUMERIC)
cameras = [c for c in csvreader]
#Create result dir
id_result_dir = osp.join(result_dir, house_id)
os.makedirs(id_result_dir, exist_ok=True)
#If already renderered, skip work
if image_folder_is_complete(len(cameras), id_result_dir):
logger.info('Skipping images for {}'.format(house_id))
else:
invalid_houses.invalidate(house_id)
render_scene_images(cameras, api, suncg_dir, house_id,
id_result_dir)
if voxel_folder_is_complete(len(cameras), id_result_dir):
logger.info('Skipping voxels for {}'.format(house_id))
else:
invalid_houses.invalidate(house_id)
generate_scene_voxel_grids(cameras, suncg_dir, house_id,
id_result_dir, obj_cache)
invalid_houses.validate(house_id)
#Free memory
del api
def reset(self, house_id=None, x=None, y=None, yaw=None):
if not self.train_mode:
obs_map = self.env.house.obsMap.T
self.obs_pos = obs_map == 1
self.traj = []
self.traj_actions = []
self.grid_traj = []
if house_id is None:
house_id = self.np_random.choice(self.houses_id, 1)[0]
self.hid = house_id
if self.env is not None:
del self.api
del self.env
self.api = objrender.RenderAPI(self.render_width,
self.render_height,
device=RENDERING_GPU)
self.env = Environment(self.api,
house_id,
self.config,
GridDet=self.configs['GridDet'],
RenderDoor=self.render_door,
StartIndoor=self.start_indoor)
self.tracker = []
self.num_rotate = 0
self.right_rotate = 0
self.L_min = self.env.house.L_lo
self.L_max = self.env.house.L_hi
self.grid_size = self.env.house.grid_det
grid_num = np.array(
[self.env.house.n_row[0] + 1, self.env.house.n_row[1] + 1])
self.grids_mat = np.zeros(tuple(grid_num), dtype=np.uint8)
self.max_grid_size = np.max(grid_num)
self.max_seen_area = float(np.prod(grid_num))
self.env.reset(x=x, y=y, yaw=yaw)
self.start_pos, self.grid_start_pos = self.get_camera_grid_pos()
if not self.train_mode:
self.traj.append(self.start_pos.tolist())
self.grid_traj.append(self.grid_start_pos.tolist())
rgb, depth, true_depth, extrinsics = self.get_obs()
self.seen_area = self.get_seen_area(rgb, true_depth, extrinsics,
self.grids_mat)
self.ep_len = 0
self.ep_reward = 0
self.collision_times = 0
ret_obs = np.concatenate((rgb, depth), axis=-1)
return ret_obs
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--data_file',
default='../path_data/cleaned_human_demo_data.json',
type=str)
parser.add_argument('--suncg_house_dir',
default='../../suncg_data/house',
type=str)
parser.add_argument('--mode',
default='rgb',
type=str,
choices=['rgb', 'semantic', 'instance', 'depth'])
parser.add_argument('--demo_id', default=0, type=int)
parser.add_argument('--width', type=int, default=800)
parser.add_argument('--height', type=int, default=600)
parser.add_argument('--device', type=int, default=0)
args = parser.parse_args()
with open(args.data_file, 'r') as f:
data = json.load(f)
demo = data[args.demo_id]
cfg = create_default_config('.')
api = objrender.RenderAPI(args.width, args.height, device=args.device)
api.printContextInfo()
obj_path = pathlib.Path(args.suncg_house_dir).joinpath(
demo['house_id'], 'house.obj')
print('You select house id: {0:s}'.format(demo['house_id']))
api.loadScene(str(obj_path.resolve()), cfg['modelCategoryFile'],
cfg['colorFile'])
cam = api.getCamera()
api_mode = getattr(RenderMode, args.mode.upper())
api.setMode(api_mode)
for t in count():
mat = np.array(api.render())
if api_mode == RenderMode.DEPTH:
infmask = mat[:, :, 1]
mat = mat[:, :, 0] * (infmask == 0)
else:
mat = mat[:, :, ::-1]
if api_mode == RenderMode.INSTANCE:
center_rgb = mat[args.height // 2, args.width // 2, ::-1]
center_instance = api.getNameFromInstanceColor(
center_rgb[0], center_rgb[1], center_rgb[2])
print("Instance ID in the center: ", center_instance)
exit_flag = key_control(mat, cam)
if exit_flag:
break
def GymHouseEnv2(scene='05cac5f7fdd5f8138234164e76a97383',
screen_size=(84, 84),
goals=['living_room'],
configuration=None):
h, w = screen_size
api = objrender.RenderAPI(w=w, h=h, device=0)
env = Environment(api, scene, create_configuration(configuration))
env.reset()
env = RoomNavTask2(env,
discrete_action=True,
depth_signal=False,
segment_input=False,
reward_type=None)
env.observation_space.dtype = np.uint8
return GymHouseWrapper(env, room_types=goals, screen_size=screen_size)
def atari_env(env_id, args, type):
api = objrender.RenderAPI(w=OBSERVATION_SPACE_SHAPE[2], h=OBSERVATION_SPACE_SHAPE[1], device=args.gpu_ids[0])
cfg = load_config('config_namazu.json')
if type == 'train':
pre_env = Environment(api, HOUSE_train[env_id], cfg, seed=args.seed)
# TODO: temporarily use HOUSE
# pre_env = Environment(api, HOUSE[env_id], cfg)
elif type == 'test':
pre_env = Environment(api, HOUSE_test[env_id], cfg, seed=args.seed)
# TODO: temporarily use HOUSE
# pre_env = Environment(api, HOUSE[env_id], cfg)
env = RoomNavTask(pre_env, hardness=args.hardness, depth_signal=False, discrete_action=True,
max_steps=args.max_episode_length)
return env
def worker(idx, house_id, device):
colormapFile = "../metadata/colormap_coarse.csv"
api = objrender.RenderAPI(w=args.width, h=args.height, device=device)
env = Environment(api, house_id, cfg)
N = 15000
start = time.time()
cnt = 0
env.reset()
for t in range(N):
cnt += 1
env.move_forward(random.random() * 3, random.random() * 3)
mat = env.render()
if (cnt % 50 == 0):
env.reset()
end = time.time()
print("Worker {}, speed {:.3f} fps".format(idx, N / (end - start)))
def gen_cache_files(ids, skip_file):
configs = get_configs()
config = load_config(configs['path'], prefix=configs['par_path'])
render_height = configs['render_height']
render_width = configs['render_width']
with open(skip_file, 'r') as f:
skip_houses = json.load(f)
for idx in tqdm(ids):
if idx in skip_houses:
continue
print(idx)
api = objrender.RenderAPI(render_width, render_height, device=0)
try:
env = Environment(api, idx, config, GridDet=configs['GridDet'])
except AssertionError:
skip_houses.append(idx)
return skip_houses
def test_render(self):
api = objrender.RenderAPI(w=SIDE, h=SIDE, device=0)
cfg = load_config('config.json')
houseID, house = find_first_good_house(cfg)
env = Environment(api, house, cfg)
location = house.getRandomLocation(ROOM_TYPE)
env.reset(*location)
# Check RGB
env.set_render_mode(RenderMode.RGB)
rgb = env.render_cube_map()
self.assertEqual(rgb.shape, (SIDE, SIDE * 6, 3))
# Check SEMANTIC
env.set_render_mode(RenderMode.RGB)
semantic = env.render_cube_map()
self.assertEqual(semantic.shape, (SIDE, SIDE * 6, 3))
# Check INSTANCE
env.set_render_mode(RenderMode.INSTANCE)
instance = env.render_cube_map()
self.assertEqual(instance.shape, (SIDE, SIDE * 6, 3))
# Check DEPTH
env.set_render_mode(RenderMode.DEPTH)
depth = env.render_cube_map()
self.assertEqual(depth.dtype, np.uint8)
self.assertEqual(depth.shape, (SIDE, SIDE * 6, 2))
depth_value = depth[0, 0, 0] * 20.0 / 255.0
# Check INVDEPTH
env.set_render_mode(RenderMode.INVDEPTH)
invdepth = env.render_cube_map()
self.assertEqual(invdepth.dtype, np.uint8)
self.assertEqual(invdepth.shape, (SIDE, SIDE * 6, 3))
# Convert to 16 bit and then to depth
id16 = 256 * invdepth[:, :, 0] + invdepth[:, :, 1]
depth2 = depth_of_inverse_depth(id16)
self.assertEqual(depth2.dtype, np.float)
self.assertEqual(depth2.shape, (SIDE, SIDE * 6))
# Check that depth value is within 5% of depth from RenderMode.DEPTH
self.assertAlmostEqual(
depth2[0, 0], depth_value, delta=depth_value * 0.05)
def worker(idx, device, num_iter):
api = objrender.RenderAPI(args.width, args.height, device=device)
api.printContextInfo()
mappingFile = cfg['modelCategoryFile']
colormapFile = cfg['colorFile']
assert os.path.isfile(mappingFile) and os.path.isfile(colormapFile)
api.loadScene(args.obj, mappingFile, colormapFile)
cam = api.getCamera()
start = time.time()
for t in range(num_iter):
if t % 2 == 1:
api.setMode(RenderMode.RGB)
else:
api.setMode(RenderMode.SEMANTIC)
mat = np.array(api.render(), copy=False)
end = time.time()
print("Worker {}, speed {:.3f} fps".format(idx, num_iter / (end - start)))
def run_test(rank, params, loaded_model, lock, seen_succ, seen_length,
unseen_succ, unseen_length):
logging.basicConfig(filename='./log/' + params.log_file + '.log',
level=logging.INFO)
ptitle('Test Agent: {}'.format(rank))
gpu_id = params.gpu_ids_test[rank % len(params.gpu_ids_test)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
torch.manual_seed(params.seed + rank)
if gpu_id >= 0:
with torch.cuda.device(gpu_id):
torch.cuda.manual_seed(params.seed + rank)
model = A3C_LSTM_GA()
with torch.cuda.device(gpu_id):
model = model.cuda()
load_model = torch.load(
loaded_model,
map_location=lambda storage, loc: storage.cuda(gpu_id))
model.load_state_dict(load_model)
model.eval()
Agent = run_agent(model, gpu_id)
n_test = 0
start_time = time.time()
while True:
house_id = rank + (n_test * params.n_process)
if house_id >= 70:
break
else:
if house_id < 20:
seen = True
house = get_house_id(house_id)
else:
seen = False
house = get_eval_house_id(house_id -
(n_test * params.n_process))
env = Environment(api, house, cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True) #reward_type='indicator'
eval = []
for i in range(params.n_test):
next_observation = task.reset()
target = task.info['target_room']
target = get_instruction_idx(target)
with torch.cuda.device(gpu_id):
target = Variable(torch.LongTensor(target)).cuda()
Agent.cx = Variable(torch.zeros(1, 256).cuda())
Agent.hx = Variable(torch.zeros(1, 256).cuda())
Agent.target = target
step, total_rew, good = 0, 0, 0
done = False
while not done:
observation = next_observation
act = Agent.action_test(observation, target)
next_observation, rew, done, info = task.step(actions[act[0]])
total_rew += rew
if rew == 10: # success
good = 1
step += 1
if done:
break
eval.append((step, total_rew, good))
if len(eval) > 0:
succ = [e for e in eval if e[2] > 0]
succ_rate = (len(succ) / len(eval)) * 100
avg_reward = sum([e[1] for e in eval]) / len(eval)
avg_length = sum([e[0] for e in eval]) / len(eval)
if seen:
msg_seen = "Seen"
msg_house = house_id
else:
msg_seen = "Unseen"
msg_house = house_id - 20
msg = " ".join([
"++++++++++ Task Stats +++++++++++\n", "Time {}\n".format(
time.strftime("%dd %Hh %Mm %Ss",
time.gmtime(time.time() - start_time))),
"Episode Played: {:d}\n".format(len(eval)),
"{:s} House id: {:d}\n".format(msg_seen, msg_house),
"Avg Reward = {:5.3f}\n".format(avg_reward),
"Avg Length = {:.3f}\n".format(avg_length),
"Success rate {:3.2f}%".format(succ_rate)
])
print(msg)
logging.info(msg)
with lock:
if seen:
seen_succ.value += len(succ)
seen_length.value += sum([e[0] for e in eval])
else:
unseen_succ.value += len(succ)
unseen_length.value += sum([e[0] for e in eval])
n_test += 1
def test(rank, params, shared_model, count, lock, best_acc, evaluation=True):
if not os.path.exists('./' + params.weight_dir):
os.mkdir('./' + params.weight_dir)
if not os.path.exists('./log'):
os.mkdir('./log')
logging.basicConfig(filename='./log/' + params.log_file + '.log',
level=logging.INFO)
ptitle('Test Agent: {}'.format(rank))
gpu_id = params.gpu_ids_test[rank % len(params.gpu_ids_test)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
best_rate = 0.0
save_model_index = 0
n_update = 0
torch.manual_seed(params.seed + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(params.seed + rank)
model = A3C_LSTM_GA()
with torch.cuda.device(gpu_id):
model = model.cuda()
Agent = run_agent(model, gpu_id)
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id >= 20:
house_id = house_id % 20
#time.sleep(rank*30)
env = Environment(api, get_house_id(house_id), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True) #reward_type='indicator'
start_time = time.time()
if evaluation is True:
max_episode = params.max_episode
n_try = params.n_eval
else:
max_episode = 1 # for loaded model test
n_try = params.n_test
for episode in range(max_episode):
eval = []
if evaluation is True:
with lock:
n_update = count.value
with torch.cuda.device(gpu_id):
Agent.model.load_state_dict(shared_model.state_dict())
else:
with torch.cuda.device(gpu_id):
Agent.model.load_state_dict(shared_model)
Agent.model.eval()
for i in range(n_try):
next_observation = task.reset()
target = task.info['target_room']
target = get_instruction_idx(target)
with torch.cuda.device(gpu_id):
target = Variable(torch.LongTensor(target)).cuda()
Agent.cx = Variable(torch.zeros(1, 256).cuda())
Agent.hx = Variable(torch.zeros(1, 256).cuda())
Agent.target = target
step, total_rew, good = 0, 0, 0
done = False
while not done:
observation = next_observation
act = Agent.action_test(observation, target)
next_observation, rew, done, info = task.step(actions[act[0]])
total_rew += rew
if rew == 10: # success
good = 1
step += 1
if done:
break
eval.append((step, total_rew, good))
if len(eval) > 0:
succ = [e for e in eval if e[2] > 0]
succ_rate = (len(succ) / len(eval)) * 100
if evaluation is True: # evaluation mode
with lock:
#if best_acc.value >= best_rate:
# best_rate = best_acc.value
if succ_rate >= best_rate:
best_rate = succ_rate
with torch.cuda.device(gpu_id):
torch.save(
Agent.model.state_dict(), params.weight_dir +
'model' + str(n_update) + '.ckpt')
save_model_index += 1
#if best_rate > best_acc.value:
# best_acc.value = best_rate
avg_reward = sum([e[1] for e in eval]) / len(eval)
avg_length = sum([e[0] for e in eval]) / len(eval)
msg = " ".join([
"++++++++++ Task Stats +++++++++++\n", "Time {}\n".format(
time.strftime("%dd %Hh %Mm %Ss",
time.gmtime(time.time() - start_time))),
"Episode Played: {:d}\n".format(len(eval)),
"N_Update = {:d}\n".format(n_update),
"House id: {:d}\n".format(house_id),
"Avg Reward = {:5.3f}\n".format(avg_reward),
"Avg Length = {:.3f}\n".format(avg_length),
"Best rate {:3.2f}, Success rate {:3.2f}%".format(
best_rate, succ_rate)
])
print(msg)
logging.info(msg)
from House3D import objrender, create_default_config
from House3D.objrender import RenderMode
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('obj')
parser.add_argument('--width', type=int, default=800)
parser.add_argument('--height', type=int, default=600)
parser.add_argument('--device', type=int, default=0)
parser.add_argument('--interactive', action='store_true',
help='run interactive rendering (does not work under ssh)')
args = parser.parse_args()
cfg = create_default_config('.')
api = objrender.RenderAPI(args.width, args.height, device=args.device)
api.printContextInfo()
api.loadScene(args.obj, cfg['modelCategoryFile'], cfg['colorFile'])
cam = api.getCamera()
modes = [RenderMode.RGB, RenderMode.SEMANTIC, RenderMode.INSTANCE, RenderMode.DEPTH]
for t in count():
mode = modes[t % len(modes)]
api.setMode(mode)
mat = np.array(api.render())
if mode == RenderMode.DEPTH:
infmask = mat[:, :, 1]
mat = mat[:, :, 0] * (infmask == 0)
else:
mat = mat[:, :, ::-1] # cv expects bgr
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--data_file',
default='../path_data/cleaned_human_demo_data.json',
type=str)
parser.add_argument('--demo_id', default=0, type=int)
parser.add_argument('--width', type=int, default=600)
parser.add_argument('--height', type=int, default=450)
parser.add_argument('--filter_height', type=bool, default=True)
args = parser.parse_args()
with open(args.data_file, 'r') as f:
data = json.load(f)
demo = data[args.demo_id]
print('Total time steps:', len(demo['loc']))
locs = np.array(demo['loc'][1:])
ques = demo['question']
answer = demo['answer']
text = 'Q: {0:s} A: {1:s}'.format(ques, answer)
text_height = 60
cfg = load_config('config.json')
api = objrender.RenderAPI(w=args.width, h=args.height, device=0)
env = Environment(api, demo['house_id'], cfg)
L_min = env.house.L_min_coor
L_max = env.house.L_max_coor
L_min = np.array([[env.house.L_lo[0], L_min[1], env.house.L_lo[1]]])
L_max = np.array([[env.house.L_hi[0], L_max[1], env.house.L_hi[1]]])
grid_size = env.house.grid_det
n_row = env.house.n_row
grid_num = np.array([
n_row[0] + 1,
int((L_max[0][1] - L_min[0][1]) / (grid_size + 1e-8)) + 1, n_row[1] + 1
])
print('Grid size:', grid_size)
print('Number of grid in [x, y, z]:', grid_num)
all_grids = np.zeros(tuple(grid_num), dtype=bool)
grid_colors = np.zeros(tuple(grid_num) + (3, ), dtype=np.uint8)
loc_map = env.gen_locmap()
obs_map = env.house.obsMap.T
obs_pos = obs_map == 1
for t in tqdm(range(len(locs))):
env.reset(x=locs[t][0], y=locs[t][2], yaw=locs[t][3])
depth = env.render(RenderMode.DEPTH)
rgb = env.render(RenderMode.RGB)
semantic = env.render(RenderMode.SEMANTIC)
infmask = depth[:, :, 1]
depth = depth[:, :, 0] * (infmask == 0)
true_depth = depth.astype(np.float32) / 255.0 * 20.0
extrinsics = env.cam.getExtrinsicsNumpy()
points, points_colors = gen_point_cloud(true_depth, rgb, extrinsics)
grid_locs = np.floor((points - L_min) / grid_size).astype(int)
all_grids[grid_locs[:, 0], grid_locs[:, 1], grid_locs[:, 2]] = True
grid_colors[grid_locs[:, 0], grid_locs[:, 1],
grid_locs[:, 2]] = points_colors
depth = np.stack([depth] * 3, axis=2)
loc_map[grid_locs[:, 2], grid_locs[:,
0], :] = np.array([250, 120, 120])
loc_map[obs_pos] = 0
rad = env.house.robotRad / env.house.grid_det
x, y = env.cam.pos.x, env.cam.pos.z
x, y = env.house.to_grid(x, y)
loc_map_cp = loc_map.copy()
cv2.circle(loc_map_cp, (x, y), int(rad), (0, 0, 255), thickness=-1)
loc_map_resized = cv2.resize(loc_map_cp, env.resolution)
concat1 = np.concatenate((rgb, semantic), axis=1)
concat2 = np.concatenate((depth, loc_map_resized), axis=1)
ret = np.concatenate((concat1, concat2), axis=0)
ret = ret[:, :, ::-1]
pad_text = np.ones((text_height, ret.shape[1], ret.shape[2]),
dtype=np.uint8)
mat_w_text = np.concatenate((np.copy(ret), pad_text), axis=0)
cv2.putText(mat_w_text, text, (10, mat_w_text.shape[0] - 10), FONT, 1,
(255, 255, 255), 2, cv2.LINE_AA)
cv2.imshow("human_demo", mat_w_text)
cv2.waitKey(20)
valid_grids = np.argwhere(all_grids)
valid_grid_center = valid_grids * grid_size + L_min
valid_grid_color = grid_colors[valid_grids[:, 0], valid_grids[:, 1],
valid_grids[:, 2]]
if args.filter_height:
height_lim = [-0.1, 2.3]
valid_idx = np.logical_and(valid_grid_center[:, 1] >= height_lim[0],
valid_grid_center[:, 1] <= height_lim[1])
valid_grid_center = valid_grid_center[valid_idx]
valid_grid_color = valid_grid_color[valid_idx]
valid_grids = valid_grids[valid_idx]
loc_map = env.gen_locmap()
obs_map = env.house.obsMap.T
loc_map[valid_grids[:, 2], valid_grids[:,
0], :] = np.array([250, 120, 120])
loc_map[obs_map == 1] = 0
loc_map = cv2.resize(loc_map, env.resolution)
cv2.imwrite('seen_map.png', loc_map)
if USE_OPEN3D:
pcd = PointCloud()
pcd.points = Vector3dVector(valid_grid_center)
pcd.colors = Vector3dVector(valid_grid_color / 255.0)
coord = create_mesh_coordinate_frame(3, [40, 0, 35])
draw_geometries([pcd, coord])
def remove_files(paths):
for path in paths:
try:
os.remove(path)
except Exception as e:
print(path, e)
if __name__ == '__main__':
# Get house ids to generate data for
house_ids = open(args.house_list, 'r').readlines()[args.hid_lo:args.hid_hi]
# Initialize renderer API and config
api = objrender.RenderAPI(w=args.render_width,
h=args.render_height,
device=0)
cfg = load_config(args.render_config_path)
for h in house_ids:
house_id = h[:-1]
check_path = os.path.join(args.question_dir, house_id + '.json')
print(check_path)
if os.path.exists(check_path):
print('Already have questions for this house --- SKIPPING')
continue
# Create house .obj and .mtl files in SUNCG
scn2scn_path = os.path.join(args.suncgtoolbox_dir,
'gaps/bin/x86_64/scn2scn')
house_file_dir = os.path.join(args.suncg_dir, 'house', house_id)
if __name__ == '__main__':
from environments.gym_house.env import create_configuration
from configuration import configuration
import deep_rl
from deep_rl.common.console_util import print_progress
deep_rl.configure(**configuration)
with open(os.path.join(os.path.dirname(__file__),'jobs', 'houses'), 'r') as f:
houses = [a.strip() for a in f.readlines()]
samples_per_room = 20
screen_size = (512, 512)
cfg = create_configuration(deep_rl.configuration.get('house3d').as_dict())
room_target_object = load_target_object_data(cfg['roomTargetFile'])
api = objrender.RenderAPI(w=screen_size[1], h=screen_size[0], device=0)
for i, houseID in enumerate(houses):
print('Processing house %s (%s/%s)' % (houseID, i + 1, len(houses)))
house = create_house(houseID, cfg)
env = Environment(api, house, cfg)
types_to_rooms = get_valid_room_dict(house)
for room_type, rooms in types_to_rooms.items():
print('Processing house %s (%s/%s) - %s' % (houseID, i + 1, len(houses), room_type))
locations = get_valid_locations(house, rooms)
if len(locations) == 0:
print('ERROR: no locations for room %s' % room_type)
continue
for j in range(samples_per_room):
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import tqdm
import cv2
import numpy as np
import os
import pdb
from House3D import objrender, Environment, load_config
if __name__ == '__main__':
api = objrender.RenderAPI(w=250, h=250, device=0)
cfg = load_config('config.json')
houseID = np.random.choice(os.listdir(cfg['prefix']))
env = Environment(api, houseID, cfg)
cam = api.getCamera()
for t in tqdm.trange(1000):
env.reset()
mat = env.renderCubeMap()
cv2.imshow("aaa", mat)
key = cv2.waitKey(0)
while key in [ord('h'), ord('l'), ord('s')]:
if key == ord('h'):
cam.yaw -= 5
def create_api():
return objrender.RenderAPI(w=250, h=250, device=0)
def gendp():
img = env.render(RenderMode.INVDEPTH)
img_16 = img.astype(np.uint16)
inverse_depth_16 = img_16[:, :, 0] * 256 + img_16[:, :, 1]
PIXEL_MAX = np.iinfo(np.uint16).max
NEAR = 0.3 # has to match minDepth parameter
depth_float = NEAR * PIXEL_MAX / inverse_depth_16.astype(np.float)
#depth_float_3 = depth_float[depth_float > 3.0] = 3.0
depth_float_3 = np.minimum(depth_float, 3.0, depth_float)
#depth_float_3 = int()
return depth_float_3 #返回的是小于3m的浮点值
if __name__ == '__main__':
api = objrender.RenderAPI(w=600, h=450, device=0) #随机地从obsMap中采样
cfg = load_config('config.json')
env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg)
con = []
loc = []
for i in range(env.house.obsMap.shape[0]):
for j in range(env.house.obsMap.shape[1]):
if env.house.obsMap[i][j] == 0:
con.append([i, j])
for i in range(100):
gx, gy = random.choice(con)
x, y = env.house.to_coor(gx, gy, True)
# Copyright 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import cv2
from House3D import objrender, Environment, load_config
if __name__ == '__main__':
api = objrender.RenderAPI(w=600, h=450, device=0)
cfg = load_config('config.json')
env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg)
env.reset() # put the agent into the house
# fourcc = cv2.VideoWriter_fourcc(*'X264')
# writer = cv2.VideoWriter('out.avi', fourcc, 30, (1200, 900))
while True:
mat = env.debug_render()
# writer.write(mat)
cv2.imshow("aaa", mat)
key = cv2.waitKey(0)
if not env.keyboard_control(key):
break
# writer.release()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('obj')
parser.add_argument('--width', type=int, default=800)
parser.add_argument('--height', type=int, default=600)
parser.add_argument('--device', type=int, default=0)
parser.add_argument(
'--interactive',
action='store_true',
help='run interactive rendering (does not work under ssh)')
args = parser.parse_args()
cfg = create_default_config('.')
api = objrender.RenderAPI(args.width, args.height,
device=args.device) #声明一个api,调用了预先编译的库
api.printContextInfo()
api.loadScene(args.obj, cfg['modelCategoryFile'], cfg['colorFile'])
cam = api.getCamera()
modes = [
RenderMode.RGB, RenderMode.SEMANTIC, RenderMode.INSTANCE,
RenderMode.DEPTH
]
for t in tqdm.trange(1000000):
#mode = modes[t % len(modes)]
mode = RenderMode.RGB
api.setMode(mode)
mat = np.array(api.render())
print(mat.shape)
def run_sim(rank, params, shared_model, shared_optimizer, count, lock):
ptitle('Training Agent: {}'.format(rank))
gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
if shared_optimizer is None:
optimizer = optim.Adam(shared_model.parameters(),
lr=params.lr,
amsgrad=params.amsgrad,
weight_decay=params.weight_decay)
#optimizer.share_memory()
else:
optimizer = shared_optimizer
torch.manual_seed(params.seed + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(params.seed + rank)
model = A3C_LSTM_GA()
with torch.cuda.device(gpu_id):
model = model.cuda()
Agent = run_agent(model, gpu_id)
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id >= 20:
house_id = house_id % 20
env = Environment(api, get_house_id(house_id), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True)
for episode in range(params.max_episode):
next_observation = task.reset()
target = task.info['target_room']
target = get_instruction_idx(target)
with torch.cuda.device(gpu_id):
target = Variable(torch.LongTensor(target)).cuda()
Agent.model.load_state_dict(shared_model.state_dict())
Agent.cx = Variable(torch.zeros(1, 256).cuda())
Agent.hx = Variable(torch.zeros(1, 256).cuda())
Agent.target = target
total_reward, num_steps, good = 0, 0, 0
Agent.done = False
done = False
Agent.eps_len = 0
while not done:
num_steps += 1
observation = next_observation
act, entropy, value, log_prob = Agent.action_train(
observation, target)
next_observation, reward, done, info = task.step(actions[act[0]])
rew = np.clip(reward, -1.0, 1.0)
Agent.put_reward(rew, entropy, value, log_prob)
if num_steps % params.num_steps == 0 or done:
if done:
Agent.done = done
with lock:
count.value += 1
Agent.training(next_observation, shared_model, optimizer,
params)
if done:
break
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--data_file',
default='../path_data/eqa_humans.json',
type=str)
parser.add_argument('--demo_id', default=0, type=int)
parser.add_argument('--width', type=int, default=600)
parser.add_argument('--height', type=int, default=450)
parser.add_argument('--no_display',
action='store_true',
help='disable display on screen')
parser.add_argument('--play_mode',
default='auto',
type=str,
choices=['key', 'auto'],
help='play the human demonstration in '
'keyboard-control mode or auto-play mode')
args = parser.parse_args()
with open(args.data_file, 'r') as f:
data = json.load(f)
demo = data[args.demo_id]
print('Total time steps:', len(demo['loc']))
locs = np.array(demo['loc'][1:])
ques = demo['question']
answer = demo['answer']
text = 'Q: {0:s} A: {1:s}'.format(ques, answer)
text_height = 60
auto_freq = 15 # 15 hz
cfg = load_config('config.json')
api = objrender.RenderAPI(w=args.width, h=args.height, device=0)
env = Environment(api, demo['house_id'], cfg, GridDet=0.05)
save_folder = 'demo_{0:d}'.format(args.demo_id)
os.makedirs(save_folder, exist_ok=True)
if not args.no_display:
print_keyboard_help(args.play_mode)
t = 0
while True:
# print(t)
if t < len(locs):
env.reset(x=locs[t][0], y=locs[t][2], yaw=locs[t][3])
mat = get_mat(env, text_height, text)
cv2.imshow("human_demo", mat)
if args.play_mode == 'key':
key = cv2.waitKey(0)
if key == 27 or key == ord('q'): # esc
break
elif key == ord('n'):
if t < len(locs) - 1:
t += 1
elif key == ord('p'):
if t > 0:
t -= 1
elif key == ord('r'):
t = 0
else:
print("Unknown key: {}".format(key))
else:
key = cv2.waitKey(int(1000 / auto_freq))
if key == 27 or key == ord('q'): # esc
break
elif key == -1:
if t < len(locs) - 1:
t += 1
elif key == ord('r'):
t = 0
auto_freq = 15
elif key == ord('i'):
if auto_freq < 30:
auto_freq += 4
elif key == ord('d'):
if auto_freq > 3:
auto_freq -= 4
else:
print("Unknown key: {}".format(key))
from House3D import Environment, objrender, load_config
from House3D.roomnav import RoomNavTask
from collections import deque
import tqdm
import numpy as np
EPISODE=1000
api = objrender.RenderAPI(
w=400, h=300, device=0)
cfg = load_config('config.json')
houses = ['00065ecbdd7300d35ef4328ffe871505',
'cf57359cd8603c3d9149445fb4040d90', '31966fdc9f9c87862989fae8ae906295',
'7995c2a93311717a3a9c48d789563590', '8b8c1994f3286bfc444a7527ffacde86',
'32e53679b33adfcc5a5660b8c758cc96',
'492c5839f8a534a673c92912aedc7b63',
'e3ae3f7b32cf99b29d3c8681ec3be321',
'1dba3a1039c6ec1a3c141a1cb0ad0757',
'5f3f959c7b3e6f091898caa8e828f110']
env = Environment(api, np.random.choice(houses, 1)[0], cfg)
task = RoomNavTask(env, hardness=0.6, discrete_action=True)
succ = deque(maxlen=500)
for i in range(EPISODE):
step, total_rew, good = 0, 0, 0
task.reset()
while True:
def run_sim(rank, params, state_Queue, action_done, actions, reward_Queue,
lock):
ptitle('Training Agent: {}'.format(rank))
gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id > 50:
house_id = house_id % 50
env = Environment(api, get_house_id(house_id, params.difficulty), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True)
while True:
next_observation = task.reset()
target = task.info['target_room']
target = get_instruction_idx(target)
# with torch.cuda.device(gpu_id):
# target = Variable(torch.LongTensor(target)).cuda()
total_reward, num_steps, good = 0, 0, 0
done = False
test = False
while not done:
num_steps += 1
observation = next_observation
state = rank, [observation, target]
state_Queue.put(state)
state_Queue.join()
# action_done.get() # action done
action = actions[rank]
if action == 99:
test = True
break # call for test
next_observation, reward, done, info = task.step(action)
reward = np.clip(reward, -1.0, 10.0)
if reward != -1.0 and reward != 10.0: # make sparse reward
reward = 0.0
total_reward += reward
rew = [rank, done, reward]
# print("send - rank: {:d}, reward: {:3.2f}".format(rank, reward))
reward_Queue.put(rew)
reward_Queue.join()
if done:
break
