def create_env(self,
config_path,
house_id,
render_width=args.render_width,
render_height=args.render_height):
api = objrender.RenderAPIThread(w=render_width,
h=render_height,
device=0)
cfg = load_config(config_path)
self.env = Environment(api, house_id, cfg)
def _initialize(self):
h, w = self.screen_size
api = objrender.RenderAPI(w=w, h=h, device=0)
env = Environment(api, self.scene, self.configuration)
env.reset()
env = RoomNavTask(env,
discrete_action=True,
depth_signal=False,
segment_input=False,
reward_type=None,
hardness=self.hardness)
self._env = env
def reset(self, house_id=None, x=None, y=None, yaw=None):
if not self.train_mode:
obs_map = self.env.house.obsMap.T
self.obs_pos = obs_map == 1
self.traj = []
self.traj_actions = []
self.grid_traj = []
if house_id is None:
house_id = self.np_random.choice(self.houses_id, 1)[0]
self.hid = house_id
if self.env is not None:
del self.api
del self.env
self.api = objrender.RenderAPI(self.render_width,
self.render_height,
device=RENDERING_GPU)
self.env = Environment(self.api,
house_id,
self.config,
GridDet=self.configs['GridDet'],
RenderDoor=self.render_door,
StartIndoor=self.start_indoor)
self.tracker = []
self.num_rotate = 0
self.right_rotate = 0
self.L_min = self.env.house.L_lo
self.L_max = self.env.house.L_hi
self.grid_size = self.env.house.grid_det
grid_num = np.array(
[self.env.house.n_row[0] + 1, self.env.house.n_row[1] + 1])
self.grids_mat = np.zeros(tuple(grid_num), dtype=np.uint8)
self.max_grid_size = np.max(grid_num)
self.max_seen_area = float(np.prod(grid_num))
self.env.reset(x=x, y=y, yaw=yaw)
self.start_pos, self.grid_start_pos = self.get_camera_grid_pos()
if not self.train_mode:
self.traj.append(self.start_pos.tolist())
self.grid_traj.append(self.grid_start_pos.tolist())
rgb, depth, true_depth, extrinsics = self.get_obs()
self.seen_area = self.get_seen_area(rgb, true_depth, extrinsics,
self.grids_mat)
self.ep_len = 0
self.ep_reward = 0
self.collision_times = 0
ret_obs = np.concatenate((rgb, depth), axis=-1)
return ret_obs
def GymHouseEnv2(scene='05cac5f7fdd5f8138234164e76a97383',
screen_size=(84, 84),
goals=['living_room'],
configuration=None):
h, w = screen_size
api = objrender.RenderAPI(w=w, h=h, device=0)
env = Environment(api, scene, create_configuration(configuration))
env.reset()
env = RoomNavTask2(env,
discrete_action=True,
depth_signal=False,
segment_input=False,
reward_type=None)
env.observation_space.dtype = np.uint8
return GymHouseWrapper(env, room_types=goals, screen_size=screen_size)
def atari_env(env_id, args, type):
api = objrender.RenderAPI(w=OBSERVATION_SPACE_SHAPE[2], h=OBSERVATION_SPACE_SHAPE[1], device=args.gpu_ids[0])
cfg = load_config('config_namazu.json')
if type == 'train':
pre_env = Environment(api, HOUSE_train[env_id], cfg, seed=args.seed)
# TODO: temporarily use HOUSE
# pre_env = Environment(api, HOUSE[env_id], cfg)
elif type == 'test':
pre_env = Environment(api, HOUSE_test[env_id], cfg, seed=args.seed)
# TODO: temporarily use HOUSE
# pre_env = Environment(api, HOUSE[env_id], cfg)
env = RoomNavTask(pre_env, hardness=args.hardness, depth_signal=False, discrete_action=True,
max_steps=args.max_episode_length)
return env
def _load_env(self, house):
# For testing (ipynb) only, we wanna load just one house.
start = time.time()
self.all_houses = [local_create_house(house, self.cfg, self.map_resolution)]
env = Environment(self.api_threads[0], self.all_houses[0], self.cfg)
self.env_loaded[house] = House3DUtils(env, target_obj_conn_map_dir=self.target_obj_conn_map_dir)
print('[%.02f] Loaded 1 house3d envs' % (time.time() - start))
def run_sim(rank, params, shared_model, shared_optimizer, count, lock):
if not os.path.exists('./' + params.weight_dir):
os.mkdir('./' + params.weight_dir)
if not os.path.exists('./log'):
os.mkdir('./log')
logging.basicConfig(filename='./log/' + params.log_file + '.log',
level=logging.INFO)
ptitle('Training Agent: {}'.format(rank))
gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
torch.manual_seed(random.randint(0, 1000) + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(random.randint(0, 1000) + rank)
model = A3C_LSTM_GA()
with torch.cuda.device(gpu_id):
model = model.cuda()
Agent = run_agent(model, gpu_id)
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id > 50:
house_id = house_id % 50
env = Environment(api, get_house_id(house_id, params.difficulty), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True)
n_train = 0
best_rate = 0.0
save_model_index = 0
while True:
n_train += 1
training(task, gpu_id, shared_model, Agent, shared_optimizer, params,
lock, count)
if n_train % 1000 == 0:
with lock:
n_update = count.value
with torch.cuda.device(gpu_id):
Agent.model.load_state_dict(shared_model.state_dict())
start_time = time.time()
best_rate, save_model_index = testing(lock, n_update, gpu_id,
Agent, task, best_rate,
params, save_model_index,
start_time, logging,
house_id)
def worker(idx, house_id, device):
colormapFile = "../metadata/colormap_coarse.csv"
api = objrender.RenderAPI(w=args.width, h=args.height, device=device)
env = Environment(api, house_id, cfg)
N = 15000
start = time.time()
cnt = 0
env.reset()
for t in range(N):
cnt += 1
env.move_forward(random.random() * 3, random.random() * 3)
mat = env.render()
if (cnt % 50 == 0):
env.reset()
end = time.time()
print("Worker {}, speed {:.3f} fps".format(idx, N / (end - start)))
def cache_path_images(hid_to_qns, args):
# make last several key frames of navigation
api_thread = objrender.RenderAPIThread(w=args.render_width, h=args.render_height)
# make frames
problematic_qids = []
for hid, qns in tqdm(hid_to_qns.items()):
print('Processing:', hid)
# initialize environment
env = Environment(api_thread, hid, cfg, ColideRes=500)
# rgb, semantic, cube_rgb (Licheng: assume pre-trained cnn is good at cnn -> depth)
for qn in qns:
# path json
path_name = '.'.join([qn['house']] + [box['id'] for box in qn['bbox']])+'.json'
path_file = osp.join(args.shortest_path_dir, path_name)
paths = json.load(open(path_file)) # list of [{positions, actions, best_iou, ordered}]
# path h5
output_h5 = osp.join(args.output_dir, '.'.join([qn['house']] + [box['id'] for box in qn['bbox']])+'.h5')
# if osp.exists(output_h5):
# continue
# f = h5py.File(output_h5, 'w')
f = h5py.File(output_h5, 'r+')
if 'cube_rgb0' in list(f.keys()): # close if already written.
print('cube_rgb already exists in %s' % output_h5)
f.close()
continue
# render
num_paths = min(args.num_paths, len(paths))
for pix in range(num_paths):
path = paths[pix] # on default we use the first sampled path
positions, actions, key_ixs, ordered = get_positions(path)
# ego_rgb = np.zeros((len(positions), args.render_height, args.render_width, 3)) # (n, h, w, 3)
# ego_depth = np.zeros((len(positions), args.render_height, args.render_width, 2)) # (n, h, w, 2)
# ego_sem = np.zeros((len(positions), args.render_height, args.render_width, 3)) # (n, h, w, 3)
cube_rgb = np.zeros((len(positions), 4, args.render_height, args.render_width, 3)) # (n, 4, h, w, 3)
# cube_depth = np.zeros((len(positions), 4, args.render_height, args.render_width, 2)) # (n, 4, h, w, 2)
for i, pos in enumerate(positions):
# ego_rgb[i] = render(env, pos, mode='rgb')
# ego_depth[i] = render(env, pos, mode='depth')
# ego_sem[i] = render(env, pos, mode='semantic')
cube_rgb[i] = render_cube_map(env, pos, mode='rgb')
# cube_depth[i] = render_cube_map(env, pos, mode='depth')
# save
# f.create_dataset('ego_rgb%s'%pix, dtype=np.uint8, data=ego_rgb)
# f.create_dataset('ego_depth%s'%pix, dtype=np.uint8, data=ego_depth)
# f.create_dataset('ego_sem%s'%pix, dtype=np.uint8, data=ego_sem)
f.create_dataset('cube_rgb%s'%pix, dtype=np.uint8, data=cube_rgb)
# f.create_dataset('cube_depth%s'%pix, dtype=np.uint8, data=cube_depth)
# f.create_dataset('key_ixs%s'%pix, dtype=np.uint8, data=np.array(key_ixs))
# f.create_dataset('ordered%s'%pix, dtype=np.uint8, data=np.array([ordered]))
# f.create_dataset('positions%s'%pix, dtype=np.float32, data=np.array(positions))
# f.create_dataset('actions%s'%pix, dtype=np.uint8, data=np.array(actions))
# f.create_dataset('num_paths', dtype=np.uint8, data=np.array([num_paths]))
f.close()
print('%s cached.' % output_h5)
def _load_envs(self, start_idx=-1, in_order=False):
if start_idx == -1: # next env
start_idx = self.env_set.index(self.pruned_env_set[-1]) + 1
# pick envs
self.pruned_env_set = self._pick_envs_to_load(self.split,
self.max_threads_per_gpu,
start_idx, in_order)
if len(self.pruned_env_set) == 0:
return
# Load api threads
start = time.time()
if len(self.api_threads) == 0:
for i in range(self.max_threads_per_gpu):
self.api_threads.append(
objrender.RenderAPIThread(w=self.width,
h=self.height,
device=self.gpu_id))
print('[%.2f] Loaded %d api threads' %
(time.time() - start, len(self.api_threads)))
# Load houses
start = time.time()
from multiprocessing import Pool
_args = ([h, self.cfg, self.map_resolution]
for h in self.pruned_env_set)
with Pool(len(self.pruned_env_set)) as pool:
self.all_houses = pool.starmap(local_create_house, _args)
print('[%.02f] Loaded %d houses' %
(time.time() - start, len(self.all_houses)))
# Load envs
start = time.time()
self.env_loaded = {}
for i in range(len(self.all_houses)):
print('[%02d/%d][split:%s][gpu:%d][house:%s]' %
(i + 1, len(self.all_houses), self.split, self.gpu_id,
self.all_houses[i].house['id']))
env = Environment(self.api_threads[i], self.all_houses[i],
self.cfg)
self.env_loaded[self.all_houses[i].house['id']] = \
House3DUtils(env, target_obj_conn_map_dir=self.target_obj_conn_map_dir)
print('[%.02f] Loaded %d house3d envs' %
(time.time() - start, len(self.env_loaded)))
for i in range(len(self.all_houses)):
self.visited_envs.add(self.all_houses[i].house['id'])
# Mark available data indices
self.available_idx = [
i for i, v in enumerate(self.env_list) if v in self.env_loaded
]
print('Available inds: %d' % len(self.available_idx))
def worker():
api_thread = objrender.RenderAPIThread(w=224, h=224)
while True:
i, house_id, qns = q.get()
print('Processing house[%s] %s/%s' %
(house_id, i + 1, len(house_to_qns)))
# api_thread = objrender.RenderAPIThread(w=224, h=224)
env = Environment(api_thread,
house_id,
cfg,
ColideRes=args.colide_resolution)
build_graph = True
if osp.exists(
osp.join(args.graph_dir, env.house.house['id'] + '.pkl')):
build_graph = False
h3d = House3DUtils(
env,
build_graph=build_graph,
graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir)
# make connMap for each qn
for qn in qns:
try:
if 'object' in qn['type']:
# object_color_compare_xroom(inroom), object_size_compare_xroom(inroom), object_dist_compare_inroom
for bbox in qn['bbox']:
assert bbox['type'] == 'object'
obj = h3d.objects[bbox['id']]
h3d.set_target_object(
obj) # connMap computed and saved
# we also store its room connMap
room = h3d.rooms[bbox['room_id']]
h3d.set_target_room(
room) # connMap computed and saved
elif qn['type'] in ['room_size_compare']:
for bbox in qn['bbox']:
assert 'room' in qn['type']
room = h3d.rooms[bbox['id']]
h3d.set_target_room(
room) # connMap computed and saved
except:
print('Error found for qn[%s]' % qn['id'])
invalid.append(qn['id'])
q.task_done()
def create_env(house="02f594bb5d8c5871bde0d8c8db20125b"):
api = objrender.RenderAPI(w=600, h=450, device=0)
cfg = create_default_config('/home/rythei/guide/SUNCG/house')
if house is None:
env = Environment(api, TEST_HOUSE, cfg)
else:
env = Environment(api, house, cfg)
env.reset()
return env
def gen_cache_files(ids, skip_file):
configs = get_configs()
config = load_config(configs['path'], prefix=configs['par_path'])
render_height = configs['render_height']
render_width = configs['render_width']
with open(skip_file, 'r') as f:
skip_houses = json.load(f)
for idx in tqdm(ids):
if idx in skip_houses:
continue
print(idx)
api = objrender.RenderAPI(render_width, render_height, device=0)
try:
env = Environment(api, idx, config, GridDet=configs['GridDet'])
except AssertionError:
skip_houses.append(idx)
return skip_houses
def house_renderer(cfg, house_queue, progress_queue):
while True:
houseID, house = house_queue.get()
api = objrender.RenderAPIThread(w=args.width, h=args.height, device=0)
env = Environment(api, house, cfg)
cam = api.getCamera()
loc_idx = 0
valid_rooms = get_valid_rooms(house)
for room in valid_rooms:
for i in range(SAMPLES_PER_ROOM):
reset_random(env, house, room)
render_current_location(env, houseID, room['id'], loc_idx)
loc_idx += 1
house_queue.task_done()
progress_queue.put(1)
print('Rendered house {}'.format(houseID))
def worker():
api_thread = objrender.RenderAPIThread(w=224, h=224)
while True:
i, house_id, qns = q.get()
print('Processing house[%s] %s/%s' %
(house_id, i + 1, len(house_to_qns)))
# api_thread = objrender.RenderAPIThread(w=224, h=224)
env = Environment(api_thread,
house_id,
cfg,
ColideRes=args.colide_resolution)
build_graph = True
if osp.exists(
osp.join(args.graph_dir, env.house.house['id'] + '.pkl')):
build_graph = False
h3d = House3DUtils(
env,
build_graph=build_graph,
graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir)
# objects in the house of interest
object_ids = []
for qn in qns:
if 'object' in qn['type']:
for bbox in qn['bbox']:
assert bbox['type'] == 'object'
object_ids.append(bbox['id'])
object_ids = list(set(object_ids))
# sample good-view points for each object
for obj_id in tqdm(object_ids):
save_file = osp.join(args.output_dir,
house_id + '_' + obj_id + '.json')
if osp.exists(save_file):
continue
# run and save
points, ious = get_best_view_points(h3d, obj_id, args)
json.dump({
'points': points,
'ious': ious
}, open(save_file, 'w'))
q.task_done()
def house_renderer(cfg, house_queue, progress_queue):
while True:
houseID, house = house_queue.get()
api = objrender.RenderAPIThread(w=512, h=512, device=0)
env = Environment(api, house, cfg)
loc_idx = 0
valid_rooms = get_valid_rooms(house)
for room, room_type in valid_rooms:
for _i in range(SAMPLES_PER_ROOM):
if not reset_random(env, house, room):
print('Unable to sample location for house {}'.format(
houseID))
break
render_current_location(env, houseID, room['id'], loc_idx, room_type)
loc_idx += 1
house_queue.task_done()
progress_queue.put(1)
print('Rendered house {}'.format(houseID))
import deep_rl
from deep_rl.common.console_util import print_progress
deep_rl.configure(**configuration)
with open(os.path.join(os.path.dirname(__file__),'jobs', 'houses'), 'r') as f:
houses = [a.strip() for a in f.readlines()]
samples_per_room = 20
screen_size = (512, 512)
cfg = create_configuration(deep_rl.configuration.get('house3d').as_dict())
room_target_object = load_target_object_data(cfg['roomTargetFile'])
api = objrender.RenderAPI(w=screen_size[1], h=screen_size[0], device=0)
for i, houseID in enumerate(houses):
print('Processing house %s (%s/%s)' % (houseID, i + 1, len(houses)))
house = create_house(houseID, cfg)
env = Environment(api, house, cfg)
types_to_rooms = get_valid_room_dict(house)
for room_type, rooms in types_to_rooms.items():
print('Processing house %s (%s/%s) - %s' % (houseID, i + 1, len(houses), room_type))
locations = get_valid_locations(house, rooms)
if len(locations) == 0:
print('ERROR: no locations for room %s' % room_type)
continue
for j in range(samples_per_room):
#print_progress(0, samples_per_room)
location = sample_true_object(room_target_object, env, house, locations, room_type)
if location is not None:
env.reset(*location[1])
# Copyright 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import tqdm
import cv2
from House3D import objrender, Environment, load_config
if __name__ == '__main__':
api = objrender.RenderAPI(w=600, h=450, device=0)
cfg = load_config('config.json')
env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg)
# fourcc = cv2.VideoWriter_fourcc(*'X264')
# writer = cv2.VideoWriter('out.avi', fourcc, 30, (1200, 900))
for t in tqdm.trange(1000):
if t % 1000 == 0:
env.reset()
mat = env.debug_render()
# writer.write(mat)
cv2.imshow("aaa", mat)
key = cv2.waitKey(0)
if not env.keyboard_control(key):
break
# writer.release()
for j in range(high):
if ([i,j] not in [cen,[cen[0]-1,cen[1]-1],[cen[0]-1,cen[1]],[cen[0]-1,cen[1]+1],[cen[0],cen[1]-1],[cen[0],cen[1]+1],[cen[0]+1,cen[1]-1],[cen[0]+1,cen[1]],[cen[0]+1,cen[1]+1]]):
if (loctd1[i][j] == 1.0):
pass
else:
line = createLineIterator(np.array(cen), np.array([i,j]), loctd)
if (np.max(line[:,2]) == 1.0):
loctd1[i][j] = 1.0
return loctd1
if __name__ == '__main__':
api = objrender.RenderAPI(w=600, h=450, device=0) #随机地从obsMap中采样
cfg = load_config('config.json')
env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg)
con = []
loc = []
for i in range(env.house.obsMap.shape[0]):
for j in range(env.house.obsMap.shape[1]):
if env.house.obsMap[i][j] == 0:
con.append([i,j])
for i in range(30):
gx, gy = random.choice(con)
x, y = env.house.to_coor(gx, gy, True)
loc.append([x,y])
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-questions_json',
default='/private/home/akadian/eqa-data/suncg-data/eqa_v1.json')
parser.add_argument(
'-graph_dir',
default='/private/home/akadian/eqa-data/suncg-data/a-star')
parser.add_argument(
'-target_obj_conn_map_dir',
default=
'/private/home/akadian/eqa-data/suncg-data/a-star/target_obj_conn_map_dir'
)
parser.add_argument('-shortest_path_dir')
parser.add_argument(
'-invalids_dir',
default='/private/home/akadian/eqa-data/suncg-data/invalids/')
parser.add_argument('-env_id', default=None)
parser.add_argument('-debug', action='store_true')
parser.add_argument('-map_resolution', default=1000, type=int)
parser.add_argument('-seed', type=int, required=True)
parser.add_argument('-check_validity', action="store_true")
parser.add_argument('-log_path', default=None)
parser.add_argument('-source_candidate_fraction', type=float, default=0.05)
args = parser.parse_args()
if args.log_path is None:
args.log_path = 'seed_{}_resolution_{}.{}.log'.format(
args.seed, args.map_resolution,
str(datetime.now()).replace(' ', '_'))
logging.basicConfig(filename=args.log_path,
level=logging.INFO,
format='%(asctime)-15s %(message)s')
random.seed(args.seed)
np.random.seed(args.seed)
if not os.path.exists(args.shortest_path_dir):
os.makedirs(args.shortest_path_dir)
args.gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(',')
args.gpus = [int(x) for x in args.gpus]
# create specific directories corresponding to the resolution
args.graph_dir = os.path.join(args.graph_dir, str(args.map_resolution))
args.target_obj_conn_map_dir = os.path.join(args.target_obj_conn_map_dir,
str(args.map_resolution))
# load house3d renderer
cfg = load_config('../../House3D/tests/config.json')
api_thread = objrender.RenderAPIThread(w=224, h=224, device=args.gpus[0])
# load envs list from questions json
data = json.load(open(args.questions_json, 'r'))
qns = data['questions']
if args.env_id is None:
envs = sorted(list(set(qns.keys())))
else:
envs = [args.env_id]
random.shuffle(envs)
invalid = []
count_path_found = 0
count_valid = 0
count_path_not_found = 0
count_no_source_cands = 0
shortest_path_lengths = []
for h in tqdm(envs):
# `scn2scn` from suncg-toolbox segfaults for this env :/
if h == '436d655f24d385512e1e782b5ba88c6b':
continue
for q in qns[h]:
logging.info("count_path_found: {}".format(count_path_found))
logging.info("count_valid: {}".format(count_valid))
logging.info(
"count_path_not_found: {}".format(count_path_not_found))
logging.info(
"count_no_source_cands: {}".format(count_no_source_cands))
if len(shortest_path_lengths) > 0:
logging.info(
"shortest path length mean: {}, median: {}, min: {}, max: {}"
.format(np.mean(shortest_path_lengths),
np.median(shortest_path_lengths),
np.min(shortest_path_lengths),
np.max(shortest_path_lengths)))
logging.info("env, question pair: {}_{}".format(h, q['id']))
logging.info("{} {} {}".format(h, q['question'], q['answer']))
env = Environment(api_thread,
h,
cfg,
ColideRes=args.map_resolution)
h3d = House3DUtils(
env,
graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir,
build_graph=False)
if os.path.exists(
os.path.join(args.shortest_path_dir,
"{}_{}.pkl".format(h, q['id']))):
logging.info("Shortest path exists")
continue
# set target object
bbox_obj = [
x for x in q['bbox']
if x['type'] == 'object' and x['target'] is True
][0]
obj_id = []
for x in h3d.objects:
if all([h3d.objects[x]['bbox']['min'][i] == bbox_obj['box']['min'][i] for i in range(3)]) and \
all([h3d.objects[x]['bbox']['max'][i] == bbox_obj['box']['max'][i] for i in range(3)]):
obj_id.append(x)
if h3d.objects[x]['fine_class'] != bbox_obj['name']:
logging.info('Name not matched {} {}'.format(
h3d.objects[x]['fine_class'], bbox_obj['name']))
assert len(obj_id) == 1
bbox_room = [
x for x in q['bbox']
if x['type'] == 'room' and x['target'] is False
][0]
target_room = False
for room in h3d.env.house.all_rooms:
if all([room['bbox']['min'][i] == bbox_room['box']['min'][i] for i in range(3)]) and \
all([room['bbox']['max'][i] == bbox_room['box']['max'][i] for i in range(3)]):
target_room = room
break
target_obj = obj_id[0]
h3d.set_target_object(h3d.objects[target_obj], target_room)
# sample a close enough target point
target_point_cands = np.argwhere((env.house.connMap >= 0)
& (env.house.connMap <= 5))
target_point_idx = np.random.choice(target_point_cands.shape[0])
target_yaw, best_coverage = h3d._get_best_yaw_obj_from_pos(
target_obj, [
target_point_cands[target_point_idx][0],
target_point_cands[target_point_idx][1]
],
height=1.0)
target_point = (target_point_cands[target_point_idx][0],
target_point_cands[target_point_idx][1],
target_yaw)
# graph creation used for selecting a source point
t1 = time()
if os.path.exists(
os.path.join(
h3d.graph_dir, h3d.env.house.house['id'] + '_' +
target_obj + '.pkl')):
print('loading graph')
h3d.load_graph(
os.path.join(
h3d.graph_dir,
h3d.env.house.house['id'] + '_' + target_obj + '.pkl'))
else:
print('building graph')
h3d.build_graph(save_path=os.path.join(
h3d.graph_dir, h3d.env.house.house['id'] + '_' +
target_obj + '.pkl'))
connmap_values = env.house.connMap.flatten()
connmap_values.sort()
# threshold for --source_candidate_fraction number of points
thresh = connmap_values[int(
(1.0 - args.source_candidate_fraction) *
connmap_values.shape[0])]
source_point_cands = np.argwhere((env.house.connMap != -1)
& (env.house.connMap >= thresh))
if thresh < 50:
# sanity check to prevent scenario when agent is spawned close to target location
logging.info("No source candidates")
invalid.append(h)
count_no_source_cands += 1
continue
t2 = time()
logging.info("Time spent for graph creation {:.6f}s".format(t2 -
t1))
for it in range(10):
logging.info("Try: {}".format(it))
try:
source_point_idx = np.random.choice(
source_point_cands.shape[0])
source_point = (source_point_cands[source_point_idx][0],
source_point_cands[source_point_idx][1],
np.random.choice(h3d.angles))
# A* for shortest path
t3 = time()
target_x, target_y, target_yaw = target_point
source_continous = h3d.env.house.to_coor(source_point[0],
source_point[1],
shft=True)
target_continous = h3d.env.house.to_coor(target_x,
target_y,
shft=True)
points_queue = []
distances_source = dict()
prev_pos = dict()
distances_source[source_point] = 0
prev_pos[source_point] = (-1.0, -1.0, -1.0, -1.0, -1.0)
# schema for point in points_queue:
# (x-grid-location, y-grid-location, yaw, x-continous-coordinate, y-continous-coordinate)
source_point = (source_point[0], source_point[1],
source_point[2], source_continous[0],
source_continous[1])
heappush(points_queue, (heuristic_estimate(
source_continous, target_continous), source_point))
while True:
if len(points_queue) == 0:
count_path_not_found += 1
logging.info("A* not able to find path to target")
raise ValueError(
"Path not found to target {} {}".format(
source_point[:3], target_point))
f_dist, point = heappop(points_queue)
add_neighbors(h3d, points_queue, point,
distances_source, prev_pos,
target_continous)
if point[0] == target_x and point[
1] == target_y and point[2] == target_yaw:
# store path
shortest_path_nodes = []
while True:
shortest_path_nodes.append(point)
point = prev_pos[point[:3]]
if point[0] == -1:
break
shortest_path_nodes.reverse()
break
t4 = time()
logging.info(
"Time spent for coupled graph generation and A*: {:.6f}s"
.format(t4 - t3))
# bookkeeping
act_q, pos_q, coord_q, actual_q = [], [], [], []
episode_images = []
movemap = None
for i in range(len(shortest_path_nodes) - 1):
u = shortest_path_nodes[i]
v = shortest_path_nodes[i + 1]
pos_q.append(
(float(u[3]), 1.0, float(u[4]), float(u[2])))
coord_q.append(h3d.env.house.to_grid(u[3], u[4]))
curr_x, curr_y, curr_yaw = u[3], u[4], u[2]
next_x, next_y, next_yaw = v[3], v[4], v[2]
if curr_yaw != next_yaw:
if next_yaw == 171 and curr_yaw == -180:
act_q.append(1)
elif next_yaw == -180 and curr_yaw == 171:
act_q.append(2)
elif next_yaw < curr_yaw:
act_q.append(1)
else:
act_q.append(2)
else:
act_q.append(0)
pos_q.append((shortest_path_nodes[-1][3], 1.0,
shortest_path_nodes[-1][4],
shortest_path_nodes[-1][2]))
act_q.append(3)
if args.check_validity:
h3d.env.reset(x=pos_q[0][0],
y=pos_q[0][2],
yaw=pos_q[0][3])
h3d_yaw = pos_q[0][
3] # dummy yaw limited to [-180, 180)
actual_q.append(
(float(h3d.env.cam.pos.x), 1.0,
float(h3d.env.cam.pos.z), float(h3d.env.cam.yaw)))
for i, action in enumerate(act_q[:-1]):
pre_pos = [
h3d.env.cam.pos.x, h3d.env.cam.pos.z,
h3d.env.cam.yaw
]
img, _, episode_done = h3d.step(action)
episode_images.append(img)
post_pos = [
h3d.env.cam.pos.x, h3d.env.cam.pos.z,
h3d.env.cam.yaw
]
actual_q.append((float(h3d.env.cam.pos.x), 1.0,
float(h3d.env.cam.pos.z),
float(h3d.env.cam.yaw)))
if all([
np.abs(pre_pos[x] - post_pos[x]) < 1e-9
for x in range(3)
]):
raise ValueError("Invalid action")
angle_delta = post_pos[2] - pre_pos[2]
h3d_yaw = (h3d_yaw + 180 + angle_delta) % 360 - 180
assert np.abs(h3d.env.cam.pos.x -
pos_q[i + 1][0]) < 1e-3
assert np.abs(h3d.env.cam.pos.z -
pos_q[i + 1][2]) < 1e-3
assert h3d_yaw == pos_q[i + 1][3]
count_valid += 1
movemap = h3d.env.house._showMoveMap(visualize=False)
logging.info("Valid")
result = {
"actions": act_q,
"actual_q": actual_q,
"answer": q['answer'],
"coordinates": coord_q,
"images": episode_images,
"movemap": movemap,
"positions": pos_q,
"question": q['question'],
}
with open(
os.path.join(args.shortest_path_dir,
"{}_{}.pkl".format(h, q['id'])),
"wb") as f:
pickle.dump(result, f)
logging.info("Saved {}_{}.pkl".format(h, q['id']))
logging.info("Length of shortest path: {}".format(
len(shortest_path_nodes)))
shortest_path_lengths.append(len(shortest_path_nodes))
count_path_found += 1
break
except KeyboardInterrupt:
raise
except:
invalid.append("env, question pair: {}_{}".format(
h, q['id']))
traceback.print_exc()
class House3DRGBD:
def __init__(self,
train_mode=True,
area_reward_scale=1,
collision_penalty=0.1,
step_penalty=0.0005,
max_depth=3.0,
render_door=False,
start_indoor=False,
ignore_collision=False,
ob_dilation_kernel=5,
large_map_size=80):
self.seed()
self.configs = get_configs()
self.configs['large_map_size'] = large_map_size
self.env = None
self.train_mode = train_mode
self.render_door = render_door
self.ignore_collision = ignore_collision
self.start_indoor = start_indoor
self.render_height = self.configs['render_height']
self.render_width = self.configs['render_width']
self.img_height = self.configs['output_height']
self.img_width = self.configs['output_width']
self.ob_dilation_kernel = ob_dilation_kernel
self.config = load_config(self.configs['path'],
prefix=self.configs['par_path'])
self.move_sensitivity = self.configs['move_sensitivity']
self.rot_sensitivity = self.configs['rot_sensitivity']
self.train_houses = self.configs['train_houses']
self.test_houses = self.configs['test_houses']
if train_mode:
self.houses_id = self.train_houses
# print("Number of traning houses:", len(self.houses_id))
else:
self.houses_id = None # self.test_houses
self.depth_threshold = (0, max_depth)
self.area_reward_scale = area_reward_scale
self.collision_penalty = collision_penalty
self.step_penalty = step_penalty
self.observation_space = [self.img_width, self.img_height, 3]
self.action_space = [6]
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def reset(self, house_id=None, x=None, y=None, yaw=None):
if house_id is None:
house_id = self.np_random.choice(self.houses_id, 1)[0]
self.hid = house_id
if self.env is not None:
del self.api
del self.env
self.api = objrender.RenderAPI(self.render_width,
self.render_height,
device=RENDERING_GPU)
self.env = Environment(self.api,
house_id,
self.config,
GridDet=self.configs['GridDet'],
RenderDoor=self.render_door,
StartIndoor=self.start_indoor)
if not self.train_mode:
self.loc_map = self.env.gen_locmap()
obs_map = self.env.house.obsMap.T
self.obs_pos = obs_map == 1
self.traj = []
self.traj_actions = []
self.grid_traj = []
self.L_min = self.env.house.L_lo
self.L_max = self.env.house.L_hi
self.grid_size = self.env.house.grid_det
grid_num = np.array(
[self.env.house.n_row[0] + 1, self.env.house.n_row[1] + 1])
self.grids_mat = np.zeros(tuple(grid_num), dtype=np.uint8)
self.max_grid_size = np.max(grid_num)
self.max_seen_area = float(np.prod(grid_num))
self.env.reset(x=x, y=y, yaw=yaw)
self.start_pos, self.grid_start_pos = self.get_camera_grid_pos()
if not self.train_mode:
print('start pose: ', self.start_pos)
self.traj.append(self.start_pos.tolist())
self.grid_traj.append(self.grid_start_pos.tolist())
rgb, depth, extrinsics = self.get_obs()
large_loc_map, small_loc_map = self.get_loc_map()
self.seen_area = self.get_seen_area(rgb, depth, extrinsics,
self.grids_mat)
self.ep_len = 0
self.ep_reward = 0
self.collision_times = 0
ob = (self.resize_img(rgb), large_loc_map, small_loc_map)
return ob
def step(self, action):
if self.ignore_collision:
collision_flag = self.motion_primitive_no_check(action)
else:
collision_flag = self.motion_primitive(action)
rgb, depth, extrinsics = self.get_obs()
large_loc_map, small_loc_map = self.get_loc_map()
if not self.train_mode:
current_pos, grid_current_pos = self.get_camera_grid_pos()
self.traj_actions.append(int(action))
self.traj.append(current_pos.tolist())
self.grid_traj.append(grid_current_pos.tolist())
reward, seen_area, raw_reward = self.cal_reward(
rgb, depth, extrinsics, collision_flag)
self.ep_len += 1
self.ep_reward += reward
if collision_flag:
self.collision_times += 1
info = {
'reward_so_far': self.ep_reward,
'steps_so_far': self.ep_len,
'seen_area': seen_area,
'collisions': self.collision_times,
'start_pose': self.start_pos,
'house_id': self.hid,
'collision_flag': collision_flag
}
info = {**info, **raw_reward}
done = False
ob = (self.resize_img(rgb), large_loc_map, small_loc_map)
return ob, reward, done, info
def render(self):
loc_map, small_map = self.get_loc_map()
if not self.train_mode:
rad = self.env.house.robotRad / self.env.house.grid_det
x = int(loc_map.shape[0] / 2)
y = int(loc_map.shape[1] / 2)
cv2.circle(loc_map, (x, y), 1, (255, 0, 255), thickness=-1)
x = int(small_map.shape[0] / 2)
y = int(small_map.shape[1] / 2)
cv2.circle(small_map, (x, y),
int(rad) * 2, (255, 0, 255),
thickness=-1)
loc_map = cv2.resize(loc_map, (self.render_width, self.render_height),
interpolation=cv2.INTER_CUBIC)
if not self.train_mode:
x = int(loc_map.shape[0] / 2)
y = int(loc_map.shape[1] / 2)
cv2.circle(loc_map, (x, y), 2, (255, 0, 255), thickness=-1)
self.env.set_render_mode('rgb')
rgb = self.env.render()
img = np.concatenate((rgb, loc_map), axis=1)
img = img[:, :, ::-1]
img = cv2.resize(img, (img.shape[1] * 3, img.shape[0] * 3),
interpolation=cv2.INTER_CUBIC)
cv2.imshow("nav", img)
cv2.waitKey(40)
def resize_img(self, img):
return cv2.resize(img, (self.img_width, self.img_height),
interpolation=cv2.INTER_AREA)
def motion_primitive(self, action):
# 0: Forward
# 1: Turn Left
# 2: Turn Right
# 3: Strafe Left
# 4: Strafe Right
# 5: Backward
collision_flag = False
if action == 0:
if not self.train_mode:
print('Action: Forward')
if not self.env.move_forward(self.move_sensitivity):
if not self.train_mode:
print('Cannot move forward, collision!!!')
collision_flag = True
elif action == 1:
if not self.train_mode:
print('Action: Turn Left')
self.env.rotate(-self.rot_sensitivity)
elif action == 2:
if not self.train_mode:
print('Action: Turn Right')
self.env.rotate(self.rot_sensitivity)
elif action == 3:
if not self.train_mode:
print('Action: Strafe Left')
if not self.env.move_forward(dist_fwd=0,
dist_hor=-self.move_sensitivity):
if not self.train_mode:
print('Cannot strafe left, collision!!!')
collision_flag = True
elif action == 4:
if not self.train_mode:
print('Action: Strafe Right')
if not self.env.move_forward(dist_fwd=0,
dist_hor=self.move_sensitivity):
if not self.train_mode:
print('Cannot strafe right, collision!!!')
collision_flag = True
elif action == 5:
if not self.train_mode:
print('Action: Backward')
if not self.env.move_forward(-self.move_sensitivity):
if not self.train_mode:
print('Cannot move backward, collision!!!')
collision_flag = True
else:
raise ValueError('unknown action type: [{0:d}]'.format(action))
return collision_flag
def move_forward(self, dist_fwd, dist_hor=0):
"""
Move with `fwd` distance to the front and `hor` distance to the right.
Both distance are float numbers.
Ignore collision !!!
"""
pos = self.env.cam.pos
pos = pos + self.env.cam.front * dist_fwd
pos = pos + self.env.cam.right * dist_hor
self.env.cam.pos.x = pos.x
self.env.cam.pos.z = pos.z
def motion_primitive_no_check(self, action):
# motion primitive without collision checking
# 0: Forward
# 1: Turn Left
# 2: Turn Right
# 3: Strafe Left
# 4: Strafe Right
# 5: Backward
collision_flag = False
if action == 0:
self.move_forward(self.move_sensitivity)
elif action == 1:
self.env.rotate(-self.rot_sensitivity)
elif action == 2:
self.env.rotate(self.rot_sensitivity)
elif action == 3:
self.move_forward(dist_fwd=0, dist_hor=-self.move_sensitivity)
elif action == 4:
self.move_forward(dist_fwd=0, dist_hor=self.move_sensitivity)
elif action == 5:
self.move_forward(-self.move_sensitivity)
else:
raise ValueError('unknown action type: [{0:d}]'.format(action))
return collision_flag
def get_obs(self):
self.env.set_render_mode('rgb')
rgb = self.env.render()
self.env.set_render_mode('depth')
depth = self.env.render()
infmask = depth[:, :, 1]
depth = depth[:, :, 0] * (infmask == 0)
true_depth = depth.astype(np.float32) / 255.0 * 20.0
extrinsics = self.env.cam.getExtrinsicsNumpy()
return rgb, true_depth, extrinsics
def get_seen_area(self, rgb, depth, extrinsics, out_mat, inv_E=True):
points, points_colors = gen_point_cloud(
depth,
rgb,
extrinsics,
depth_threshold=self.depth_threshold,
inv_E=inv_E)
grid_locs = np.floor(
(points[:, [0, 2]] - self.L_min) / self.grid_size).astype(int)
grids_mat = np.zeros(
(self.grids_mat.shape[0], self.grids_mat.shape[1]), dtype=np.uint8)
high_filter_idx = points[:, 1] < HEIGHT_THRESHOLD[1]
low_filter_idx = points[:, 1] > HEIGHT_THRESHOLD[0]
obstacle_idx = np.logical_and(high_filter_idx, low_filter_idx)
self.safe_assign(grids_mat, grid_locs[high_filter_idx, 0],
grid_locs[high_filter_idx, 1], 2)
kernel = np.ones((3, 3), np.uint8)
grids_mat = cv2.morphologyEx(grids_mat, cv2.MORPH_CLOSE, kernel)
obs_mat = np.zeros((self.grids_mat.shape[0], self.grids_mat.shape[1]),
dtype=np.uint8)
self.safe_assign(obs_mat, grid_locs[obstacle_idx, 0],
grid_locs[obstacle_idx, 1], 1)
kernel = np.ones((self.ob_dilation_kernel, self.ob_dilation_kernel),
np.uint8)
obs_mat = cv2.morphologyEx(obs_mat, cv2.MORPH_CLOSE, kernel)
obs_idx = np.where(obs_mat == 1)
self.safe_assign(grids_mat, obs_idx[0], obs_idx[1], 1)
out_mat[np.where(grids_mat == 2)] = 2
out_mat[np.where(grids_mat == 1)] = 1
seen_area = np.sum(out_mat > 0)
return seen_area
def cal_reward(self, rgb, depth, extrinsics, collision_flag):
filled_grid_num = self.get_seen_area(rgb,
depth,
extrinsics,
self.grids_mat,
inv_E=True)
area_reward = (filled_grid_num - self.seen_area)
reward = area_reward * self.area_reward_scale
if collision_flag:
reward -= self.collision_penalty
reward -= self.step_penalty
self.seen_area = filled_grid_num
raw_reward = {'area': area_reward, 'collision_flag': collision_flag}
return reward, filled_grid_num, raw_reward
def get_loc_map(self):
top_down_map = self.grids_mat.T.copy()
half_size = max(top_down_map.shape[0], top_down_map.shape[1],
self.configs['large_map_range']) * 3
ego_map = np.ones(
(half_size * 2, half_size * 2, 3), dtype=np.uint8) * 255
loc_map = np.zeros((top_down_map.shape[0], top_down_map.shape[1], 3),
dtype=np.uint8)
loc_map[top_down_map == 0] = np.array([255, 255, 255])
loc_map[top_down_map == 1] = np.array([0, 0, 255])
loc_map[top_down_map == 2] = np.array([0, 255, 0])
current_pos, grid_current_pos = self.get_camera_grid_pos()
x_start = half_size - grid_current_pos[1]
y_start = half_size - grid_current_pos[0]
x_end = x_start + top_down_map.shape[0]
y_end = y_start + top_down_map.shape[1]
assert x_start >= 0 and y_start >= 0 and \
x_end <= ego_map.shape[0] and y_end <= ego_map.shape[1]
ego_map[x_start:x_end, y_start:y_end] = loc_map
center = (half_size, half_size)
rot_angle = self.constrain_to_pm_pi(90 + current_pos[2])
M = cv2.getRotationMatrix2D(center, rot_angle, 1.0)
ego_map = cv2.warpAffine(ego_map,
M, (ego_map.shape[1], ego_map.shape[0]),
flags=cv2.INTER_AREA,
borderMode=cv2.BORDER_CONSTANT,
borderValue=(255, 255, 255))
start = half_size - self.configs['small_map_range']
end = half_size + self.configs['small_map_range']
small_ego_map = ego_map[start:end, start:end]
start = half_size - self.configs['large_map_range']
end = half_size + self.configs['large_map_range']
assert start >= 0
assert end <= ego_map.shape[0]
large_ego_map = ego_map[start:end, start:end]
return cv2.resize(
large_ego_map,
(self.configs['large_map_size'], self.configs['large_map_size']),
interpolation=cv2.INTER_AREA), small_ego_map
def safe_assign(self, im_map, x_idx, y_idx, value):
try:
im_map[x_idx, y_idx] = value
except IndexError:
valid_idx1 = np.logical_and(x_idx >= 0, x_idx < im_map.shape[0])
valid_idx2 = np.logical_and(y_idx >= 0, y_idx < im_map.shape[1])
valid_idx = np.logical_and(valid_idx1, valid_idx2)
im_map[x_idx[valid_idx], y_idx[valid_idx]] = value
def constrain_to_pm_pi(self, theta):
# make sure theta is within [-180, 180)
return (theta + 180) % 360 - 180
def get_camera_grid_pos(self):
current_pos = np.array([
self.env.cam.pos.x, self.env.cam.pos.z,
self.constrain_to_pm_pi(self.env.cam.yaw)
])
grid_pos = np.array(
self.env.house.to_grid(current_pos[0], current_pos[1]))
return current_pos, grid_pos
def truncated_norm(self, mu, sigma, lower_limit, upper_limit, size):
if sigma == 0:
return mu
lower_limit = (lower_limit - mu) / sigma
upper_limit = (upper_limit - mu) / sigma
r = truncnorm(lower_limit, upper_limit, loc=mu, scale=sigma)
return r.rvs(size)
def _load_envs(self, start_idx=-1, in_order=False):
#self._clear_memory()
if start_idx == -1:
start_idx = self.env_set.index(self.pruned_env_set[-1]) + 1
# Pick envs
self.pruned_env_set = self._pick_envs_to_load(
split=self.split,
max_envs=self.max_threads_per_gpu,
start_idx=start_idx,
in_order=in_order)
if len(self.pruned_env_set) == 0:
return
# Load api threads
start = time.time()
if len(self.api_threads) == 0:
for i in range(self.max_threads_per_gpu):
self.api_threads.append(
objrender.RenderAPIThread(w=224, h=224,
device=self.gpu_id))
self.cfg = load_config('../../House3D/tests/config.json')
print('[%.02f] Loaded %d api threads' %
(time.time() - start, len(self.api_threads)))
start = time.time()
# Load houses
from multiprocessing import Pool
_args = ([h, self.cfg, self.map_resolution]
for h in self.pruned_env_set)
with Pool(len(self.pruned_env_set)) as pool:
self.all_houses = pool.starmap(local_create_house, _args)
print('[%.02f] Loaded %d houses' %
(time.time() - start, len(self.all_houses)))
start = time.time()
# Load envs
self.env_loaded = {}
for i in range(len(self.all_houses)):
print('[%02d/%d][split:%s][gpu:%d][house:%s]' %
(i + 1, len(self.all_houses), self.split, self.gpu_id,
self.all_houses[i].house['id']))
environment = Environment(self.api_threads[i], self.all_houses[i],
self.cfg)
self.env_loaded[self.all_houses[i].house['id']] = House3DUtils(
environment,
target_obj_conn_map_dir=self.target_obj_conn_map_dir,
build_graph=False)
# [TODO] Unused till now
self.env_ptr = -1
print('[%.02f] Loaded %d house3d envs' %
(time.time() - start, len(self.env_loaded)))
# Mark available data indices
self.available_idx = [
i for i, v in enumerate(self.env_list) if v in self.env_loaded
]
# [TODO] only keeping legit sequences
# needed for things to play well with old data
temp_available_idx = self.available_idx.copy()
for i in range(len(temp_available_idx)):
if self.action_lengths[temp_available_idx[i]] < 5:
self.available_idx.remove(temp_available_idx[i])
print('Available inds: %d' % len(self.available_idx))
# Flag to check if loaded envs have been cycled through or not
# [TODO] Unused till now
self.all_envs_loaded = False
img_16 = img.astype(np.uint16)
inverse_depth_16 = img_16[:, :, 0] * 256 + img_16[:, :, 1]
PIXEL_MAX = np.iinfo(np.uint16).max
NEAR = 0.3 # has to match minDepth parameter
depth_float = NEAR * PIXEL_MAX / inverse_depth_16.astype(np.float)
#depth_float_3 = depth_float[depth_float > 3.0] = 3.0
depth_float_3 = np.minimum(depth_float, 3.0, depth_float)
#depth_float_3 = int()
return depth_float_3 #返回的是小于3m的浮点值
if __name__ == '__main__':
api = objrender.RenderAPI(w=600, h=450, device=0) #随机地从obsMap中采样
cfg = load_config('config.json')
env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg)
con = []
loc = []
for i in range(env.house.obsMap.shape[0]):
for j in range(env.house.obsMap.shape[1]):
if env.house.obsMap[i][j] == 0:
con.append([i, j])
for i in range(100):
gx, gy = random.choice(con)
x, y = env.house.to_coor(gx, gy, True)
loc.append([x, y])
print([gx, gy, x, y])
# Copyright 2017-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import cv2
from House3D import objrender, Environment, load_config
if __name__ == '__main__':
api = objrender.RenderAPI(w=600, h=450, device=0)
cfg = load_config('config.json')
env = Environment(api, '00065ecbdd7300d35ef4328ffe871505', cfg)
env.reset() # put the agent into the house
# fourcc = cv2.VideoWriter_fourcc(*'X264')
# writer = cv2.VideoWriter('out.avi', fourcc, 30, (1200, 900))
while True:
mat = env.debug_render()
# writer.write(mat)
cv2.imshow("aaa", mat)
key = cv2.waitKey(0)
if not env.keyboard_control(key):
break
# writer.release()
def run_sim(rank, params, state_Queue, action_done, actions, reward_Queue,
lock):
ptitle('Training Agent: {}'.format(rank))
gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id > 50:
house_id = house_id % 50
env = Environment(api, get_house_id(house_id, params.difficulty), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True)
while True:
next_observation = task.reset()
target = task.info['target_room']
target = get_instruction_idx(target)
# with torch.cuda.device(gpu_id):
# target = Variable(torch.LongTensor(target)).cuda()
total_reward, num_steps, good = 0, 0, 0
done = False
test = False
while not done:
num_steps += 1
observation = next_observation
state = rank, [observation, target]
state_Queue.put(state)
state_Queue.join()
# action_done.get() # action done
action = actions[rank]
if action == 99:
test = True
break # call for test
next_observation, reward, done, info = task.step(action)
reward = np.clip(reward, -1.0, 10.0)
if reward != -1.0 and reward != 10.0: # make sparse reward
reward = 0.0
total_reward += reward
rew = [rank, done, reward]
# print("send - rank: {:d}, reward: {:3.2f}".format(rank, reward))
reward_Queue.put(rew)
reward_Queue.join()
if done:
break
class TrajectoryGenerator():
def __init__(self,
traj_dir,
house_id=args.house_id,
traj_id=None,
load_graph=True):
self.house_id = house_id
self.traj_dir = traj_dir
if house_id != None:
self.create_env(args.config_path, house_id, args.render_width,
args.render_height)
# Contains methods for calculataing distances, room location, etc
self.hp = HouseParse(dataDir=args.data_dir)
# Loads house graph and generates shortest paths
self.utils = House3DUtils(
self.env,
rotation_sensitivity=45,
target_obj_conn_map_dir=False,
# Changed load_graph method to use pickle directly and Graph(g) initialisation;
# self.graph.load(path) left the graph empty!
build_graph=load_graph,
graph_dir=args.graph_dir)
self.house = {}
self.current_room = None
self.env_coors = None
self.traj_id = None
if traj_id != None:
self.update_trajectory(traj_id)
"""
Initialize environment for a given house.
"""
def create_env(self,
config_path,
house_id,
render_width=args.render_width,
render_height=args.render_height):
api = objrender.RenderAPIThread(w=render_width,
h=render_height,
device=0)
cfg = load_config(config_path)
self.env = Environment(api, house_id, cfg)
"""
Load given trajectory from file.
"""
def update_trajectory(self, traj_id):
self.traj_id = traj_id
load_path = os.path.join(self.traj_dir, self.house_id + '.npy')
self.env_coors = np.load(load_path)[traj_id]
# Add look-arounds when entering rooms
print(
'Preprocessing trajectory for room views (90 degrees left and right)'
)
self.house = {}
self.add_180s_to_trajectory()
"""
Update the agent's position.
"""
def update_env(self, new_pos):
self.env.cam.pos.x = new_pos[0]
self.env.cam.pos.y = 1.2
self.env.cam.pos.z = new_pos[2]
self.env.cam.yaw = new_pos[3]
self.env.cam.updateDirection()
"""
Preprocesses trajectory by adding frames where the agent looks around when entering a room.
"""
def add_180s_to_trajectory(self):
# Index house rooms first
self.build_rooms_and_objects_description()
new_coors = []
for i in range(len(self.env_coors)):
self.update_env(self.env_coors[i])
new_coors.append(self.env_coors[i])
# Entered a new room, look around
if self.update_current_room(self.env_coors[i]):
init_yaw = new_coors[-1][3]
new_coor = new_coors[-1]
# Look around (left, right) up to 90 degrees in increments of 30
yaw_adds = [1, 1, 1, -1, -1, -1, -1, -1, -1, 1, 1, 1]
for yaw_add in yaw_adds:
new_coor = (new_coor[0], new_coor[1], new_coor[2],
new_coor[3] + 30 * yaw_add)
new_coors.append(new_coor)
self.env_coors = new_coors
"""
Render the trajectory step by step.
"""
def render_and_save_trajectory(self, frame_dir):
self.env.set_render_mode(RenderMode.RGB)
s = self.env_coors[0]
d = self.env_coors[-1]
assert os.path.exists(
frame_dir), 'Can\'t save frames, non-existent directory!'
filename = '%s/%s_%04d.mp4' % (frame_dir, self.house_id, self.traj_id)
print('Generating', filename)
video = cv2.VideoWriter(filename, cv2.VideoWriter_fourcc(*'mp4v'),
args.fps,
(args.render_width, args.render_height))
for i in range(len(self.env_coors)):
self.update_env(self.env_coors[i])
img = np.array(self.env.render(), copy=False)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
# Write video frame
video.write(img)
cv2.destroyAllWindows()
video.release()
"""
Render the agent's current view in a given mode.
"""
def render_frame(self, mode=RenderMode.RGB):
self.env.set_render_mode(mode)
img = self.env.render()
if mode in [RenderMode.SEMANTIC, RenderMode.RGB]:
img = np.array(img, copy=False, dtype=np.int32)
return img
elif mode == RenderMode.DEPTH:
img = img[:, :, 0]
img = np.array(img, copy=False, dtype=np.float32)
return img
else:
return None
"""
Returns a set of all the simple room types in the given list, with the following changes:
- guest_room -> bedroom
- toilet -> bathroom
"""
@staticmethod
def get_room_types(types):
room_types = types
if 'toilet' in room_types:
room_types.remove('toilet')
if not 'bathroom' in room_types:
room_types.append('bathroom')
if 'guest_room' in room_types:
room_types.remove('guest_room')
if not 'bedroom' in room_types:
room_types.append('bedroom')
return list(sorted(room_types))
"""
Trajectory/question generation requires frame-by-frame semantic processing and establishing
object/room attributes. As we have access to the ground truth information, we can just lookup
the properties in the dict that this method returns.
"""
def build_rooms_and_objects_description(self):
obj_id_to_color = json.load(open(args.obj_color_path))
room_unique_id = 0
for room in self.utils.rooms:
# Add room type to dict
room_types = TrajectoryGenerator.get_room_types(room['type'])
room_type = '|'.join(room_types)
if not room_type in self.house:
self.house[room_type] = {
'room_list': [],
'count': 0,
'been_here_count': 0
}
self.house[room_type]['count'] += 1
# Prepare property container for room
room_unique_id += 1
room_desc = {
'been_here': False,
'room_type': room_type,
'bbox': room['bbox'],
'objects': {},
'room_id': room_type + str(room_unique_id)
}
objects_in_room = self.get_object_objects_in_room(room)
for obj in objects_in_room:
if not obj['coarse_class'] in constants.query_objects:
continue
# Add object type to dict
obj_type = obj['coarse_class']
if not obj_type in room_desc['objects']:
room_desc['objects'][obj_type] = {
'obj_list': [],
'count': 0,
'seen_count': 0
}
room_desc['objects'][obj_type]['count'] += 1
# Prepare property container for object
color = None
node = '.0_' + obj['id'][2:]
if self.house_id + node in obj_id_to_color:
color = obj_id_to_color[self.house_id + node]
obj_desc = {
'node': node,
'bbox': obj['bbox'],
'color': color,
'seen': False,
'room_location': room_type,
'obj_type': obj_type,
'room_id': room_type + str(room_unique_id)
}
room_desc['objects'][obj_type]['obj_list'].append(obj_desc)
self.house[room_type]['room_list'].append(room_desc)
"""
Index all objects in the given room.
"""
def get_object_objects_in_room(self, room):
return [
self.utils.objects['0_' + str(node)] for node in room['nodes']
if '0_' + str(node) in self.utils.objects and self.utils.objects[
'0_' + str(node)]['coarse_class'] in constants.query_objects
]
"""
Generate a shortest path between random locations in room1 and room2.
"""
def generate_random_path(self, room1, room2):
# Disabled assert in getRandomLocation for ALLOWED_TARGET_ROOM_TYPES
c1 = self.env.house.getRandomLocation(
room1, return_grid_loc=True, mixedTp=True) + (0, )
c2 = self.env.house.getRandomLocation(
room2, return_grid_loc=True, mixedTp=True) + (0, )
print('Generating random path from %s to %s' % (room1, room2), c1, c2)
start = time.time()
path = self.utils.compute_shortest_path(c1, c2)
print(time.time() - start, 'seconds to generate path of length',
len(path.nodes))
return path
"""
Turn grid path into a trajectory inside the house.
"""
def get_graph_to_env_coors(self, path):
env_coors = []
for node in path.nodes:
to_coor = self.env.house.to_coor(node[0], node[1])
env_coor = (to_coor[0], self.env.house.robotHei, to_coor[1],
node[2])
env_coors.append(env_coor)
return env_coors
"""
Returns whether the agent entered a new room.
"""
def update_current_room(self, agent_pos):
agent_new_pos_obj = TrajectoryGenerator.get_agent_pos_obj(agent_pos)
if self.current_room == None or \
not (self.hp.isContained(self.current_room, agent_new_pos_obj, axis=0) and \
self.hp.isContained(self.current_room, agent_new_pos_obj, axis=2)):
# New room
for room_type in self.house:
for room in self.house[room_type]['room_list']:
if self.hp.isContained(room, agent_new_pos_obj, axis=0) and \
self.hp.isContained(room, agent_new_pos_obj, axis=2):
self.current_room = room
# print("New room entered:", self.current_room['room_id'])
self.current_room['been_here'] = True
self.house[self.current_room['room_type']][
'been_here_count'] += 1
return True
return False
"""
Given a door node, finds at most two rooms which are on either side of the door and adds this
information to the door object.
"""
def find_adjacent_rooms_for_door(self, door_node):
door_obj = self.doors[door_node]
# One adjacent room is the room which the door belongs to
door_obj['adjacent_rooms'] = [door_obj['room_id']]
# If the door doesn't belong to the room we're currently in, append the latter to the list
# as the other adjacent room
if door_obj['room_id'] != self.current_room['room_id']:
door_obj['adjacent_rooms'].append(self.current_room['room_id'])
return
# Looking for the second adjacent room
for room_type in self.house:
for room_obj in self.house[room_type]['room_list']:
if room_obj['room_id'] == door_obj['room_id']:
continue
if self.hp.isContained(room_obj, door_obj, axis=0) or\
self.hp.isContained(room_obj, door_obj, axis=2):
# Found the other adjacent room
door_obj['adjacent_rooms'].append(room_obj['room_id'])
return
"""
Given a list of objects in the current view (type->count, approx_depths) and the ground truth
information, mark doors that were seen. Unlike match_seen_to_ground_truth(), this method tries
to find seen doors in the entire house. (Uses approximation when computing distances, not 100%
accurate.)
"""
def match_seen_to_doors(self, objs_in_frame, agent_pos):
if 'door' not in objs_in_frame:
return []
agent_pos_obj = TrajectoryGenerator.get_agent_pos_obj(agent_pos)
doors_seen = []
count = objs_in_frame['door']['count']
depths = objs_in_frame['door']['depths']
for i in range(count):
curr_depth = depths[i]
for room_type in self.house:
for room in self.house[room_type]['room_list']:
if not 'door' in room['objects']:
continue
door_objs_in_room = room['objects']['door']['obj_list']
for door_obj in door_objs_in_room:
# Distance from agent to bbox centre of door
coord_bbox = list((np.array(door_obj['bbox']['min']) +\
np.array(door_obj['bbox']['max'])) / 2)
bbox_centre = {
'bbox': {
'min': coord_bbox,
'max': coord_bbox
}
}
true_dist1 = self.hp.getClosestDistance(
agent_pos_obj, bbox_centre)
# Default distance computation
true_dist2 = self.hp.getClosestDistance(
agent_pos_obj, door_obj)
# Check if the door object corresponds to the door seen
if isclose(curr_depth, true_dist1, rtol=0.25) or\
isclose(curr_depth, true_dist2, rtol=0.25):
doors_seen.append(door_obj)
break
return doors_seen
"""
Given a list of objects in the current view (type->count, approx_depths) and the ground truth
information, mark objects that were seen. (Uses approximation when computing distances, not 100%
accurate.)
"""
def match_seen_to_ground_truth(self, objs_in_frame, agent_pos):
if self.current_room == None:
return []
agent_pos_obj = TrajectoryGenerator.get_agent_pos_obj(agent_pos)
obj_nodes_seen = []
for obj_type in objs_in_frame:
count = objs_in_frame[obj_type]['count']
depths = objs_in_frame[obj_type]['depths']
for i in range(count):
if not obj_type in self.current_room['objects']:
continue
curr_depth = depths[i]
for j in range(
self.current_room['objects'][obj_type]['count']):
# An object is usually visible across multiple frames - only mark it once
if not self.current_room['objects'][obj_type]['obj_list'][
j]['seen']:
# Get distance between bboxes of agent and object
coord1 = list((np.array(
self.current_room['objects'][obj_type]['obj_list'][j]['bbox']['min']) +\
np.array(
self.current_room['objects'][obj_type]['obj_list'][j]['bbox']['max']))
/ 2)
bbox_centre = {'bbox': {'min': coord1, 'max': coord1}}
true_dist1 = self.hp.getClosestDistance(
agent_pos_obj, bbox_centre)
true_dist2 = self.hp.getClosestDistance(
agent_pos_obj, self.current_room['objects']
[obj_type]['obj_list'][j])
# Check if the object in view is in the room
if isclose(curr_depth, true_dist1, rtol=0.25) or\
isclose(curr_depth, true_dist2, rtol=0.25):
obj_nodes_seen.append(self.current_room['objects']
[obj_type]['obj_list'][j])
self.current_room['objects'][obj_type]['obj_list'][
j]['seen'] = True
self.current_room['objects'][obj_type][
'seen_count'] += 1
break
return obj_nodes_seen
"""
Generate a trajectory and gather seen rooms and objects. Optionally returns objects in all video
frames corresponding to the trajectory.
"""
def generate_trajectory_and_seen_items(self,
frame_dir=None,
compute_seen_doors=False,
return_objects_in_frames=False):
self.build_rooms_and_objects_description()
if frame_dir:
self.render_and_save_trajectory(frame_dir)
rgb_to_obj = TrajectoryGenerator.get_semantic_to_object_mapping(
args.csv_path)
self.current_room = None
if return_objects_in_frames:
objects_in_frames = []
# Parse frames
start = time.time()
for c in range(len(self.env_coors)):
# Update agent position in the environment
self.update_env(self.env_coors[c])
semantic_img = self.render_frame(mode=RenderMode.SEMANTIC)
depth_img = self.render_frame(mode=RenderMode.DEPTH)
# Mark current room as visited
self.update_current_room(self.env_coors[c])
# Get objects types and approximate depths from current frame
objs_in_frame = TrajectoryGenerator.get_objects_in_frame(
semantic_img, rgb_to_obj, depth_img)
if return_objects_in_frames:
objects_in_frames.append(list(objs_in_frame.keys()))
# Mark objects in ground truth room that correspond to current view
seen_nodes = self.match_seen_to_ground_truth(
objs_in_frame, self.env_coors[c])
if compute_seen_doors:
# Store objects corresponding to all seen doors (in the entire house)
seen_doors = self.match_seen_to_doors(objs_in_frame,
self.env_coors[c])
for door in seen_doors:
if door['node'] in self.doors:
continue
# See which rooms are on both sides of the new door
self.doors[door['node']] = door
self.find_adjacent_rooms_for_door(door['node'])
print(time.time() - start, 'seconds to process',
str(len(self.env_coors)), 'frames.')
if return_objects_in_frames:
return objects_in_frames
"""
Returns a list of the valid room types in the house.
"""
def get_all_valid_room_types(self):
return [
x for x in self.env.house.all_roomTypes
if TrajectoryGenerator.valid_room_type(x)
]
"""
Get pairs of nearby objects for a visited room with marked objects.
"""
def get_nearby_object_pairs(self, room_desc):
assert room_desc['been_here'], 'This room has not been visited!'
obj_container = []
for obj_type in room_desc['objects']:
cnt_type = 0
for obj_entry in room_desc['objects'][obj_type]['obj_list']:
# Make sure the object was seen on the trajectory
if not obj_entry['seen']:
continue
cnt_type += 1
obj = ItemInfo(name=obj_type + str(cnt_type), meta=obj_entry)
obj_container.append(obj)
if len(obj_container) > 0:
return self.hp.getNearbyPairs(obj_container,
hthreshold=args.h_threshold,
vthreshold=args.v_threshold)
return {'on': [], 'next_to': []}
"""
Returns a dict with keys ['on', 'next_to'] and values as list of tuples (obj1, obj2, dist)
showing spatial relationships between objects.
"""
def get_all_nearby_object_pairs(self):
all_pairs = {'on': [], 'next_to': []}
for room_type in self.house:
for room_obj in self.house[room_type]['room_list']:
# Only look at visited rooms
if room_obj['been_here']:
pairs = self.get_nearby_object_pairs(room_obj)
for rel in ['on', 'next_to']:
all_pairs[rel] += pairs[rel]
return all_pairs
"""
Get the list of all objects (either on the trajectory or in the entire house).
"""
def get_all_objects(self,
include_unseen_objects=False,
include_objects_in_all_rooms=False):
obj_list = []
for room_type in self.house:
for room in self.house[room_type]['room_list']:
if not room['been_here'] and not include_objects_in_all_rooms:
continue
for obj_type in room['objects']:
for obj in room['objects'][obj_type]['obj_list']:
if obj['seen'] or include_unseen_objects:
obj_list.append(obj)
return obj_list
"""
Get the list of all rooms (either on the trajectory or in the entire house). Does not include
object list for rooms.
"""
def get_all_rooms(self, include_unseen_rooms=False):
room_list = []
for room_type in self.house:
for room in self.house[room_type]['room_list']:
if room['been_here'] or include_unseen_rooms:
# Don't include object list
room_list.append({
'been_here': True,
'room_type': room_type,
'bbox': room['bbox'],
'room_id': room['room_id']
})
return room_list
"""
Return agent's current position inside an object with 'bbox' attribute. Useful for calling
HouseParse methods.
"""
@staticmethod
def get_agent_pos_obj(agent_pos):
agent_new_pos_obj = {
'bbox': {
'min': agent_pos[0:3],
'max': agent_pos[0:3],
},
}
return agent_new_pos_obj
"""
Given a depth map and an image with numbered disjoint regions corresponding to a single object
type, return an __approximate__ depth for each one of them.
"""
@staticmethod
def get_approx_depths_for_object_type(depth_img, labeled_objs_img,
num_objs):
approx_depths = []
for i in range(num_objs):
first_idx = next(idx
for idx, val in np.ndenumerate(labeled_objs_img)
if val == i + 1)
approx_depths.append(depth_img[first_idx] / 255.0 * 20.0)
return approx_depths
"""
Extract objects from a frame.
object_type -> (num_objects, object_depths)
"""
@staticmethod
def get_objects_in_frame(semantic_img, rgb_to_obj, depth_img):
label_img = TrajectoryGenerator.rgb_to_int_image(semantic_img)
labels = np.unique(label_img)
objs_in_frame = {}
# Process information about each unique type of object in the current frame
only_curr_obj_img = np.zeros(shape=label_img.shape, dtype=np.int32)
for i in range(len(labels)):
# "Paint" on a background only the objects of the current type
only_curr_obj_img[:, :] = 0
only_curr_obj_img[np.where(label_img == labels[i])] = 1
# Find number of occurrences of object in frame (might be misleading, occlusions or
# several objects overlapping e.g. chairs)
s = [[1, 1, 1], [1, 1, 1], [1, 1, 1]]
num_objs = label(only_curr_obj_img, output=only_curr_obj_img)
# Find semantic color of object and convert it to object type
first_idx = next(idx
for idx, val in np.ndenumerate(only_curr_obj_img)
if val == 1)
rgb = (semantic_img[first_idx[0], first_idx[1],
0], semantic_img[first_idx[0], first_idx[1],
1], semantic_img[first_idx[0],
first_idx[1],
2])
obj_name = rgb_to_obj[rgb]
# Check if we want to ask questions about the object type
if not obj_name in constants.query_objects:
continue
objs_in_frame[obj_name] = {}
# Get number of objects of this type in the frame
objs_in_frame[obj_name]['count'] = num_objs
# Get approximate depths
depths = TrajectoryGenerator.get_approx_depths_for_object_type(
depth_img, only_curr_obj_img, num_objs)
objs_in_frame[obj_name]['depths'] = depths
return objs_in_frame
"""
Open CSV category file to map semantic frames to sets of objects.
"""
@staticmethod
def get_semantic_to_object_mapping(path):
f = open(path, newline="")
reader = csv.DictReader(f)
all_objects_dict = {}
for line in reader:
all_objects_dict[(int(line['r']), int(line['g']),
int(line['b']))] = line['name']
return all_objects_dict
"""
Map RGB values in an image to integers: (r,g,b) -> (256^2 * r + 256 * g + b).
"""
@staticmethod
def rgb_to_int_image(img):
out = np.zeros(shape=(img.shape[0], img.shape[1]), dtype=np.int32)
out = (img[:, :, 0] << 16) | (img[:, :, 1] << 8) | (img[:, :, 2])
return out
"""
Blacklist some room types.
"""
@staticmethod
def valid_room_type(room_type):
return len(room_type) > 0 and\
all([not tp.lower() in constants.exclude_rooms for tp in room_type])
def run_sim(rank, params, shared_model, shared_optimizer, count, lock):
ptitle('Training Agent: {}'.format(rank))
gpu_id = params.gpu_ids_train[rank % len(params.gpu_ids_train)]
api = objrender.RenderAPI(w=params.width, h=params.height, device=gpu_id)
cfg = load_config('config.json')
if shared_optimizer is None:
optimizer = optim.Adam(shared_model.parameters(),
lr=params.lr,
amsgrad=params.amsgrad,
weight_decay=params.weight_decay)
#optimizer.share_memory()
else:
optimizer = shared_optimizer
torch.manual_seed(params.seed + rank)
if gpu_id >= 0:
torch.cuda.manual_seed(params.seed + rank)
model = A3C_LSTM_GA()
with torch.cuda.device(gpu_id):
model = model.cuda()
Agent = run_agent(model, gpu_id)
house_id = params.house_id
if house_id == -1:
house_id = rank
if house_id >= 20:
house_id = house_id % 20
env = Environment(api, get_house_id(house_id), cfg)
task = RoomNavTask(env,
hardness=params.hardness,
segment_input=params.semantic_mode,
max_steps=params.max_steps,
discrete_action=True)
for episode in range(params.max_episode):
next_observation = task.reset()
target = task.info['target_room']
target = get_instruction_idx(target)
with torch.cuda.device(gpu_id):
target = Variable(torch.LongTensor(target)).cuda()
Agent.model.load_state_dict(shared_model.state_dict())
Agent.cx = Variable(torch.zeros(1, 256).cuda())
Agent.hx = Variable(torch.zeros(1, 256).cuda())
Agent.target = target
total_reward, num_steps, good = 0, 0, 0
Agent.done = False
done = False
Agent.eps_len = 0
while not done:
num_steps += 1
observation = next_observation
act, entropy, value, log_prob = Agent.action_train(
observation, target)
next_observation, reward, done, info = task.step(actions[act[0]])
rew = np.clip(reward, -1.0, 1.0)
Agent.put_reward(rew, entropy, value, log_prob)
if num_steps % params.num_steps == 0 or done:
if done:
Agent.done = done
with lock:
count.value += 1
Agent.training(next_observation, shared_model, optimizer,
params)
if done:
break
def worker():
api_thread = objrender.RenderAPIThread(w=224, h=224)
while True:
i, house_id, qns = q.get()
print('Processing house[%s] %s/%s' % (house_id, i+1, len(house_to_qns)))
env = Environment(api_thread, house_id, cfg, ColideRes=args.colide_resolution)
h3d = House3DUtils(env, build_graph=False, graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir)
# compute shortest path for each qn
for qn in qns:
tic = time.time()
samples = []
for _ in range(args.num_samples):
sample = {'shortest_paths': [], 'positions': []}
if qn['type'] in ['object_color_compare_inroom', 'object_color_compare_xroom', 'object_size_compare_inroom', 'object_size_compare_xroom']:
ntnp = '2obj4p'
# 2 objects
assert len(qn['bbox']) == 2
obj1_id, obj2_id = qn['bbox'][0]['id'], qn['bbox'][1]['id']
obj1_name, obj2_name = qn['bbox'][0]['name'], qn['bbox'][1]['name']
# 4 points
try:
source_point, obj1_point, obj2_point, end_point, obj1_iou, obj2_iou = get_4points_for_2objects(h3d, obj1_id, obj2_id, args)
except:
print('4 points for 2 objects not found.')
continue
# compute shortest paths
try:
ordered = True
positions, actions = sample_paths(h3d, [source_point, obj1_point, obj2_point, end_point])
samples.append({'positions': positions, 'actions': actions, 'best_iou': {obj1_name: obj1_iou, obj2_name: obj2_iou}, 'ordered': ordered})
except:
print('shortest path not found for question[%s](%s).' % (qn['id'], qn['type']))
continue
elif qn['type'] == 'object_dist_compare_inroom':
ntnp = '3obj5p'
# 3 objects
assert len(qn['bbox']) == 3
obj1_id, obj2_id, obj3_id = qn['bbox'][0]['id'], qn['bbox'][1]['id'], qn['bbox'][2]['id']
obj1_name, obj2_name, obj3_name = qn['bbox'][0]['name'], qn['bbox'][1]['name'], qn['bbox'][2]['name']
# 5 points
try:
source_point, obj1_point, obj2_point, obj3_point, end_point, obj1_iou, obj2_iou, obj3_iou = \
get_5points_for_3objects(h3d, obj1_id, obj2_id, obj3_id, args)
except:
print('5 points for 3 objects not found.')
continue
# compute shortest paths
try:
ordered = True
positions, actions = sample_paths(h3d, [source_point, obj1_point, obj2_point, obj3_point, end_point])
samples.append({'positions': positions, 'actions': actions, 'best_iou': {obj1_name: obj1_iou, obj2_name: obj2_iou, obj3_name: obj3_iou}, 'ordered': ordered})
except:
print('shortest path not found for question[%s](%s).' % (qn['id'], qn['type']))
continue
elif qn['type'] == 'room_size_compare':
ntnp = '2rm4p'
# 2 rooms
assert len(qn['bbox']) == 2
room1_id, room2_id = qn['bbox'][0]['id'], qn['bbox'][1]['id']
# 4 points
try:
source_point, room1_point, room2_point, end_point = get_4points_for_2rooms(h3d, room1_id, room2_id, args)
except:
print('4 points for 2 rooms not found.')
continue
# compute shortest paths
try:
ordered = True
positions, actions = sample_paths(h3d, [source_point, room1_point, room2_point, end_point])
samples.append({'positions': positions, 'actions': actions, 'ordered': ordered})
except:
print('shortest path not found for question[%s][%s].' % (qn['id'], qn['type']))
continue
# save
if len(samples) == 0:
invalid.append(qn['id']) # do not use += here!
else:
print('%s [%s] samples for question[%s] in %.2fs.' % (len(samples), ntnp, qn['id'], time.time()-tic))
fname = [str(qn['house'])] + [str(box['id'])for box in qn['bbox']]
fname = '.'.join(fname) + '.json'
with open(osp.join(args.shortest_path_dir, fname), 'w') as f:
json.dump(samples, f)
# finished this job
q.task_done()
EPISODE=1000
api = objrender.RenderAPI(
w=400, h=300, device=0)
cfg = load_config('config.json')
houses = ['00065ecbdd7300d35ef4328ffe871505',
'cf57359cd8603c3d9149445fb4040d90', '31966fdc9f9c87862989fae8ae906295',
'7995c2a93311717a3a9c48d789563590', '8b8c1994f3286bfc444a7527ffacde86',
'32e53679b33adfcc5a5660b8c758cc96',
'492c5839f8a534a673c92912aedc7b63',
'e3ae3f7b32cf99b29d3c8681ec3be321',
'1dba3a1039c6ec1a3c141a1cb0ad0757',
'5f3f959c7b3e6f091898caa8e828f110']
env = Environment(api, np.random.choice(houses, 1)[0], cfg)
task = RoomNavTask(env, hardness=0.6, discrete_action=True)
succ = deque(maxlen=500)
for i in range(EPISODE):
step, total_rew, good = 0, 0, 0
task.reset()
while True:
act = task._action_space.sample()
obs, rew, done, info = task.step(act)
total_rew += rew
step += 1
houses = [
'00065ecbdd7300d35ef4328ffe871505', 'cf57359cd8603c3d9149445fb4040d90',
'31966fdc9f9c87862989fae8ae906295', 'ff32675f2527275171555259b4a1b3c3',
'7995c2a93311717a3a9c48d789563590', '8b8c1994f3286bfc444a7527ffacde86',
'775941abe94306edc1b5820e3a992d75', '32e53679b33adfcc5a5660b8c758cc96',
'4383029c98c14177640267bd34ad2f3c', '0884337c703e7c25949d3a237101f060',
'492c5839f8a534a673c92912aedc7b63', 'a7e248efcdb6040c92ac0cdc3b2351a6',
'2364b7dcc432c6d6dcc59dba617b5f4b', 'e3ae3f7b32cf99b29d3c8681ec3be321',
'f10ce4008da194626f38f937fb9c1a03', 'e6f24af5f87558d31db17b86fe269cf2',
'1dba3a1039c6ec1a3c141a1cb0ad0757', 'b814705bc93d428507a516b866efda28',
'26e33980e4b4345587d6278460746ec4', '5f3f959c7b3e6f091898caa8e828f110',
'b5bd72478fce2a2dbd1beb1baca48abd', '9be4c7bee6c0ba81936ab0e757ab3d61'
]
#env = MultiHouseEnv(api, houses[:3], cfg) # use 3 houses
env = Environment(api, houses[0], cfg)
task = RoomNavTask(env, hardness=0.6, discrete_action=True)
action_dict = dict(k=7,
l=9,
j=10,
o=3,
u=4,
f=5,
a=6,
i=8,
d=11,
s=12,
r=-1,
h=-2,
q=-3)
