def __init__(self,
traj_dir,
house_id=args.house_id,
traj_id=None,
load_graph=True):
self.house_id = house_id
self.traj_dir = traj_dir
if house_id != None:
self.create_env(args.config_path, house_id, args.render_width,
args.render_height)
# Contains methods for calculataing distances, room location, etc
self.hp = HouseParse(dataDir=args.data_dir)
# Loads house graph and generates shortest paths
self.utils = House3DUtils(
self.env,
rotation_sensitivity=45,
target_obj_conn_map_dir=False,
# Changed load_graph method to use pickle directly and Graph(g) initialisation;
# self.graph.load(path) left the graph empty!
build_graph=load_graph,
graph_dir=args.graph_dir)
self.house = {}
self.current_room = None
self.env_coors = None
self.traj_id = None
if traj_id != None:
self.update_trajectory(traj_id)
def _load_env(self, house):
# For testing (ipynb) only, we wanna load just one house.
start = time.time()
self.all_houses = [local_create_house(house, self.cfg, self.map_resolution)]
env = Environment(self.api_threads[0], self.all_houses[0], self.cfg)
self.env_loaded[house] = House3DUtils(env, target_obj_conn_map_dir=self.target_obj_conn_map_dir)
print('[%.02f] Loaded 1 house3d envs' % (time.time() - start))
def _load_envs(self, start_idx=-1, in_order=False):
if start_idx == -1: # next env
start_idx = self.env_set.index(self.pruned_env_set[-1]) + 1
# pick envs
self.pruned_env_set = self._pick_envs_to_load(self.split,
self.max_threads_per_gpu,
start_idx, in_order)
if len(self.pruned_env_set) == 0:
return
# Load api threads
start = time.time()
if len(self.api_threads) == 0:
for i in range(self.max_threads_per_gpu):
self.api_threads.append(
objrender.RenderAPIThread(w=self.width,
h=self.height,
device=self.gpu_id))
print('[%.2f] Loaded %d api threads' %
(time.time() - start, len(self.api_threads)))
# Load houses
start = time.time()
from multiprocessing import Pool
_args = ([h, self.cfg, self.map_resolution]
for h in self.pruned_env_set)
with Pool(len(self.pruned_env_set)) as pool:
self.all_houses = pool.starmap(local_create_house, _args)
print('[%.02f] Loaded %d houses' %
(time.time() - start, len(self.all_houses)))
# Load envs
start = time.time()
self.env_loaded = {}
for i in range(len(self.all_houses)):
print('[%02d/%d][split:%s][gpu:%d][house:%s]' %
(i + 1, len(self.all_houses), self.split, self.gpu_id,
self.all_houses[i].house['id']))
env = Environment(self.api_threads[i], self.all_houses[i],
self.cfg)
self.env_loaded[self.all_houses[i].house['id']] = \
House3DUtils(env, target_obj_conn_map_dir=self.target_obj_conn_map_dir)
print('[%.02f] Loaded %d house3d envs' %
(time.time() - start, len(self.env_loaded)))
for i in range(len(self.all_houses)):
self.visited_envs.add(self.all_houses[i].house['id'])
# Mark available data indices
self.available_idx = [
i for i, v in enumerate(self.env_list) if v in self.env_loaded
]
print('Available inds: %d' % len(self.available_idx))
def worker():
api_thread = objrender.RenderAPIThread(w=224, h=224)
while True:
i, house_id, qns = q.get()
print('Processing house[%s] %s/%s' %
(house_id, i + 1, len(house_to_qns)))
# api_thread = objrender.RenderAPIThread(w=224, h=224)
env = Environment(api_thread,
house_id,
cfg,
ColideRes=args.colide_resolution)
build_graph = True
if osp.exists(
osp.join(args.graph_dir, env.house.house['id'] + '.pkl')):
build_graph = False
h3d = House3DUtils(
env,
build_graph=build_graph,
graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir)
# make connMap for each qn
for qn in qns:
try:
if 'object' in qn['type']:
# object_color_compare_xroom(inroom), object_size_compare_xroom(inroom), object_dist_compare_inroom
for bbox in qn['bbox']:
assert bbox['type'] == 'object'
obj = h3d.objects[bbox['id']]
h3d.set_target_object(
obj) # connMap computed and saved
# we also store its room connMap
room = h3d.rooms[bbox['room_id']]
h3d.set_target_room(
room) # connMap computed and saved
elif qn['type'] in ['room_size_compare']:
for bbox in qn['bbox']:
assert 'room' in qn['type']
room = h3d.rooms[bbox['id']]
h3d.set_target_room(
room) # connMap computed and saved
except:
print('Error found for qn[%s]' % qn['id'])
invalid.append(qn['id'])
q.task_done()
def worker():
api_thread = objrender.RenderAPIThread(w=224, h=224)
while True:
i, house_id, qns = q.get()
print('Processing house[%s] %s/%s' %
(house_id, i + 1, len(house_to_qns)))
# api_thread = objrender.RenderAPIThread(w=224, h=224)
env = Environment(api_thread,
house_id,
cfg,
ColideRes=args.colide_resolution)
build_graph = True
if osp.exists(
osp.join(args.graph_dir, env.house.house['id'] + '.pkl')):
build_graph = False
h3d = House3DUtils(
env,
build_graph=build_graph,
graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir)
# objects in the house of interest
object_ids = []
for qn in qns:
if 'object' in qn['type']:
for bbox in qn['bbox']:
assert bbox['type'] == 'object'
object_ids.append(bbox['id'])
object_ids = list(set(object_ids))
# sample good-view points for each object
for obj_id in tqdm(object_ids):
save_file = osp.join(args.output_dir,
house_id + '_' + obj_id + '.json')
if osp.exists(save_file):
continue
# run and save
points, ious = get_best_view_points(h3d, obj_id, args)
json.dump({
'points': points,
'ious': ious
}, open(save_file, 'w'))
q.task_done()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-questions_json',
default='/private/home/akadian/eqa-data/suncg-data/eqa_v1.json')
parser.add_argument(
'-graph_dir',
default='/private/home/akadian/eqa-data/suncg-data/a-star')
parser.add_argument(
'-target_obj_conn_map_dir',
default=
'/private/home/akadian/eqa-data/suncg-data/a-star/target_obj_conn_map_dir'
)
parser.add_argument('-shortest_path_dir')
parser.add_argument(
'-invalids_dir',
default='/private/home/akadian/eqa-data/suncg-data/invalids/')
parser.add_argument('-env_id', default=None)
parser.add_argument('-debug', action='store_true')
parser.add_argument('-map_resolution', default=1000, type=int)
parser.add_argument('-seed', type=int, required=True)
parser.add_argument('-check_validity', action="store_true")
parser.add_argument('-log_path', default=None)
parser.add_argument('-source_candidate_fraction', type=float, default=0.05)
args = parser.parse_args()
if args.log_path is None:
args.log_path = 'seed_{}_resolution_{}.{}.log'.format(
args.seed, args.map_resolution,
str(datetime.now()).replace(' ', '_'))
logging.basicConfig(filename=args.log_path,
level=logging.INFO,
format='%(asctime)-15s %(message)s')
random.seed(args.seed)
np.random.seed(args.seed)
if not os.path.exists(args.shortest_path_dir):
os.makedirs(args.shortest_path_dir)
args.gpus = os.environ['CUDA_VISIBLE_DEVICES'].split(',')
args.gpus = [int(x) for x in args.gpus]
# create specific directories corresponding to the resolution
args.graph_dir = os.path.join(args.graph_dir, str(args.map_resolution))
args.target_obj_conn_map_dir = os.path.join(args.target_obj_conn_map_dir,
str(args.map_resolution))
# load house3d renderer
cfg = load_config('../../House3D/tests/config.json')
api_thread = objrender.RenderAPIThread(w=224, h=224, device=args.gpus[0])
# load envs list from questions json
data = json.load(open(args.questions_json, 'r'))
qns = data['questions']
if args.env_id is None:
envs = sorted(list(set(qns.keys())))
else:
envs = [args.env_id]
random.shuffle(envs)
invalid = []
count_path_found = 0
count_valid = 0
count_path_not_found = 0
count_no_source_cands = 0
shortest_path_lengths = []
for h in tqdm(envs):
# `scn2scn` from suncg-toolbox segfaults for this env :/
if h == '436d655f24d385512e1e782b5ba88c6b':
continue
for q in qns[h]:
logging.info("count_path_found: {}".format(count_path_found))
logging.info("count_valid: {}".format(count_valid))
logging.info(
"count_path_not_found: {}".format(count_path_not_found))
logging.info(
"count_no_source_cands: {}".format(count_no_source_cands))
if len(shortest_path_lengths) > 0:
logging.info(
"shortest path length mean: {}, median: {}, min: {}, max: {}"
.format(np.mean(shortest_path_lengths),
np.median(shortest_path_lengths),
np.min(shortest_path_lengths),
np.max(shortest_path_lengths)))
logging.info("env, question pair: {}_{}".format(h, q['id']))
logging.info("{} {} {}".format(h, q['question'], q['answer']))
env = Environment(api_thread,
h,
cfg,
ColideRes=args.map_resolution)
h3d = House3DUtils(
env,
graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir,
build_graph=False)
if os.path.exists(
os.path.join(args.shortest_path_dir,
"{}_{}.pkl".format(h, q['id']))):
logging.info("Shortest path exists")
continue
# set target object
bbox_obj = [
x for x in q['bbox']
if x['type'] == 'object' and x['target'] is True
][0]
obj_id = []
for x in h3d.objects:
if all([h3d.objects[x]['bbox']['min'][i] == bbox_obj['box']['min'][i] for i in range(3)]) and \
all([h3d.objects[x]['bbox']['max'][i] == bbox_obj['box']['max'][i] for i in range(3)]):
obj_id.append(x)
if h3d.objects[x]['fine_class'] != bbox_obj['name']:
logging.info('Name not matched {} {}'.format(
h3d.objects[x]['fine_class'], bbox_obj['name']))
assert len(obj_id) == 1
bbox_room = [
x for x in q['bbox']
if x['type'] == 'room' and x['target'] is False
][0]
target_room = False
for room in h3d.env.house.all_rooms:
if all([room['bbox']['min'][i] == bbox_room['box']['min'][i] for i in range(3)]) and \
all([room['bbox']['max'][i] == bbox_room['box']['max'][i] for i in range(3)]):
target_room = room
break
target_obj = obj_id[0]
h3d.set_target_object(h3d.objects[target_obj], target_room)
# sample a close enough target point
target_point_cands = np.argwhere((env.house.connMap >= 0)
& (env.house.connMap <= 5))
target_point_idx = np.random.choice(target_point_cands.shape[0])
target_yaw, best_coverage = h3d._get_best_yaw_obj_from_pos(
target_obj, [
target_point_cands[target_point_idx][0],
target_point_cands[target_point_idx][1]
],
height=1.0)
target_point = (target_point_cands[target_point_idx][0],
target_point_cands[target_point_idx][1],
target_yaw)
# graph creation used for selecting a source point
t1 = time()
if os.path.exists(
os.path.join(
h3d.graph_dir, h3d.env.house.house['id'] + '_' +
target_obj + '.pkl')):
print('loading graph')
h3d.load_graph(
os.path.join(
h3d.graph_dir,
h3d.env.house.house['id'] + '_' + target_obj + '.pkl'))
else:
print('building graph')
h3d.build_graph(save_path=os.path.join(
h3d.graph_dir, h3d.env.house.house['id'] + '_' +
target_obj + '.pkl'))
connmap_values = env.house.connMap.flatten()
connmap_values.sort()
# threshold for --source_candidate_fraction number of points
thresh = connmap_values[int(
(1.0 - args.source_candidate_fraction) *
connmap_values.shape[0])]
source_point_cands = np.argwhere((env.house.connMap != -1)
& (env.house.connMap >= thresh))
if thresh < 50:
# sanity check to prevent scenario when agent is spawned close to target location
logging.info("No source candidates")
invalid.append(h)
count_no_source_cands += 1
continue
t2 = time()
logging.info("Time spent for graph creation {:.6f}s".format(t2 -
t1))
for it in range(10):
logging.info("Try: {}".format(it))
try:
source_point_idx = np.random.choice(
source_point_cands.shape[0])
source_point = (source_point_cands[source_point_idx][0],
source_point_cands[source_point_idx][1],
np.random.choice(h3d.angles))
# A* for shortest path
t3 = time()
target_x, target_y, target_yaw = target_point
source_continous = h3d.env.house.to_coor(source_point[0],
source_point[1],
shft=True)
target_continous = h3d.env.house.to_coor(target_x,
target_y,
shft=True)
points_queue = []
distances_source = dict()
prev_pos = dict()
distances_source[source_point] = 0
prev_pos[source_point] = (-1.0, -1.0, -1.0, -1.0, -1.0)
# schema for point in points_queue:
# (x-grid-location, y-grid-location, yaw, x-continous-coordinate, y-continous-coordinate)
source_point = (source_point[0], source_point[1],
source_point[2], source_continous[0],
source_continous[1])
heappush(points_queue, (heuristic_estimate(
source_continous, target_continous), source_point))
while True:
if len(points_queue) == 0:
count_path_not_found += 1
logging.info("A* not able to find path to target")
raise ValueError(
"Path not found to target {} {}".format(
source_point[:3], target_point))
f_dist, point = heappop(points_queue)
add_neighbors(h3d, points_queue, point,
distances_source, prev_pos,
target_continous)
if point[0] == target_x and point[
1] == target_y and point[2] == target_yaw:
# store path
shortest_path_nodes = []
while True:
shortest_path_nodes.append(point)
point = prev_pos[point[:3]]
if point[0] == -1:
break
shortest_path_nodes.reverse()
break
t4 = time()
logging.info(
"Time spent for coupled graph generation and A*: {:.6f}s"
.format(t4 - t3))
# bookkeeping
act_q, pos_q, coord_q, actual_q = [], [], [], []
episode_images = []
movemap = None
for i in range(len(shortest_path_nodes) - 1):
u = shortest_path_nodes[i]
v = shortest_path_nodes[i + 1]
pos_q.append(
(float(u[3]), 1.0, float(u[4]), float(u[2])))
coord_q.append(h3d.env.house.to_grid(u[3], u[4]))
curr_x, curr_y, curr_yaw = u[3], u[4], u[2]
next_x, next_y, next_yaw = v[3], v[4], v[2]
if curr_yaw != next_yaw:
if next_yaw == 171 and curr_yaw == -180:
act_q.append(1)
elif next_yaw == -180 and curr_yaw == 171:
act_q.append(2)
elif next_yaw < curr_yaw:
act_q.append(1)
else:
act_q.append(2)
else:
act_q.append(0)
pos_q.append((shortest_path_nodes[-1][3], 1.0,
shortest_path_nodes[-1][4],
shortest_path_nodes[-1][2]))
act_q.append(3)
if args.check_validity:
h3d.env.reset(x=pos_q[0][0],
y=pos_q[0][2],
yaw=pos_q[0][3])
h3d_yaw = pos_q[0][
3] # dummy yaw limited to [-180, 180)
actual_q.append(
(float(h3d.env.cam.pos.x), 1.0,
float(h3d.env.cam.pos.z), float(h3d.env.cam.yaw)))
for i, action in enumerate(act_q[:-1]):
pre_pos = [
h3d.env.cam.pos.x, h3d.env.cam.pos.z,
h3d.env.cam.yaw
]
img, _, episode_done = h3d.step(action)
episode_images.append(img)
post_pos = [
h3d.env.cam.pos.x, h3d.env.cam.pos.z,
h3d.env.cam.yaw
]
actual_q.append((float(h3d.env.cam.pos.x), 1.0,
float(h3d.env.cam.pos.z),
float(h3d.env.cam.yaw)))
if all([
np.abs(pre_pos[x] - post_pos[x]) < 1e-9
for x in range(3)
]):
raise ValueError("Invalid action")
angle_delta = post_pos[2] - pre_pos[2]
h3d_yaw = (h3d_yaw + 180 + angle_delta) % 360 - 180
assert np.abs(h3d.env.cam.pos.x -
pos_q[i + 1][0]) < 1e-3
assert np.abs(h3d.env.cam.pos.z -
pos_q[i + 1][2]) < 1e-3
assert h3d_yaw == pos_q[i + 1][3]
count_valid += 1
movemap = h3d.env.house._showMoveMap(visualize=False)
logging.info("Valid")
result = {
"actions": act_q,
"actual_q": actual_q,
"answer": q['answer'],
"coordinates": coord_q,
"images": episode_images,
"movemap": movemap,
"positions": pos_q,
"question": q['question'],
}
with open(
os.path.join(args.shortest_path_dir,
"{}_{}.pkl".format(h, q['id'])),
"wb") as f:
pickle.dump(result, f)
logging.info("Saved {}_{}.pkl".format(h, q['id']))
logging.info("Length of shortest path: {}".format(
len(shortest_path_nodes)))
shortest_path_lengths.append(len(shortest_path_nodes))
count_path_found += 1
break
except KeyboardInterrupt:
raise
except:
invalid.append("env, question pair: {}_{}".format(
h, q['id']))
traceback.print_exc()
def _load_envs(self, start_idx=-1, in_order=False):
#self._clear_memory()
if start_idx == -1:
start_idx = self.env_set.index(self.pruned_env_set[-1]) + 1
# Pick envs
self.pruned_env_set = self._pick_envs_to_load(
split=self.split,
max_envs=self.max_threads_per_gpu,
start_idx=start_idx,
in_order=in_order)
if len(self.pruned_env_set) == 0:
return
# Load api threads
start = time.time()
if len(self.api_threads) == 0:
for i in range(self.max_threads_per_gpu):
self.api_threads.append(
objrender.RenderAPIThread(w=224, h=224,
device=self.gpu_id))
self.cfg = load_config('../../House3D/tests/config.json')
print('[%.02f] Loaded %d api threads' %
(time.time() - start, len(self.api_threads)))
start = time.time()
# Load houses
from multiprocessing import Pool
_args = ([h, self.cfg, self.map_resolution]
for h in self.pruned_env_set)
with Pool(len(self.pruned_env_set)) as pool:
self.all_houses = pool.starmap(local_create_house, _args)
print('[%.02f] Loaded %d houses' %
(time.time() - start, len(self.all_houses)))
start = time.time()
# Load envs
self.env_loaded = {}
for i in range(len(self.all_houses)):
print('[%02d/%d][split:%s][gpu:%d][house:%s]' %
(i + 1, len(self.all_houses), self.split, self.gpu_id,
self.all_houses[i].house['id']))
environment = Environment(self.api_threads[i], self.all_houses[i],
self.cfg)
self.env_loaded[self.all_houses[i].house['id']] = House3DUtils(
environment,
target_obj_conn_map_dir=self.target_obj_conn_map_dir,
build_graph=False)
# [TODO] Unused till now
self.env_ptr = -1
print('[%.02f] Loaded %d house3d envs' %
(time.time() - start, len(self.env_loaded)))
# Mark available data indices
self.available_idx = [
i for i, v in enumerate(self.env_list) if v in self.env_loaded
]
# [TODO] only keeping legit sequences
# needed for things to play well with old data
temp_available_idx = self.available_idx.copy()
for i in range(len(temp_available_idx)):
if self.action_lengths[temp_available_idx[i]] < 5:
self.available_idx.remove(temp_available_idx[i])
print('Available inds: %d' % len(self.available_idx))
# Flag to check if loaded envs have been cycled through or not
# [TODO] Unused till now
self.all_envs_loaded = False
def worker():
api_thread = objrender.RenderAPIThread(w=224, h=224)
while True:
i, house_id, qns = q.get()
print('Processing house[%s] %s/%s' % (house_id, i+1, len(house_to_qns)))
env = Environment(api_thread, house_id, cfg, ColideRes=args.colide_resolution)
h3d = House3DUtils(env, build_graph=False, graph_dir=args.graph_dir,
target_obj_conn_map_dir=args.target_obj_conn_map_dir)
# compute shortest path for each qn
for qn in qns:
tic = time.time()
samples = []
for _ in range(args.num_samples):
sample = {'shortest_paths': [], 'positions': []}
if qn['type'] in ['object_color_compare_inroom', 'object_color_compare_xroom', 'object_size_compare_inroom', 'object_size_compare_xroom']:
ntnp = '2obj4p'
# 2 objects
assert len(qn['bbox']) == 2
obj1_id, obj2_id = qn['bbox'][0]['id'], qn['bbox'][1]['id']
obj1_name, obj2_name = qn['bbox'][0]['name'], qn['bbox'][1]['name']
# 4 points
try:
source_point, obj1_point, obj2_point, end_point, obj1_iou, obj2_iou = get_4points_for_2objects(h3d, obj1_id, obj2_id, args)
except:
print('4 points for 2 objects not found.')
continue
# compute shortest paths
try:
ordered = True
positions, actions = sample_paths(h3d, [source_point, obj1_point, obj2_point, end_point])
samples.append({'positions': positions, 'actions': actions, 'best_iou': {obj1_name: obj1_iou, obj2_name: obj2_iou}, 'ordered': ordered})
except:
print('shortest path not found for question[%s](%s).' % (qn['id'], qn['type']))
continue
elif qn['type'] == 'object_dist_compare_inroom':
ntnp = '3obj5p'
# 3 objects
assert len(qn['bbox']) == 3
obj1_id, obj2_id, obj3_id = qn['bbox'][0]['id'], qn['bbox'][1]['id'], qn['bbox'][2]['id']
obj1_name, obj2_name, obj3_name = qn['bbox'][0]['name'], qn['bbox'][1]['name'], qn['bbox'][2]['name']
# 5 points
try:
source_point, obj1_point, obj2_point, obj3_point, end_point, obj1_iou, obj2_iou, obj3_iou = \
get_5points_for_3objects(h3d, obj1_id, obj2_id, obj3_id, args)
except:
print('5 points for 3 objects not found.')
continue
# compute shortest paths
try:
ordered = True
positions, actions = sample_paths(h3d, [source_point, obj1_point, obj2_point, obj3_point, end_point])
samples.append({'positions': positions, 'actions': actions, 'best_iou': {obj1_name: obj1_iou, obj2_name: obj2_iou, obj3_name: obj3_iou}, 'ordered': ordered})
except:
print('shortest path not found for question[%s](%s).' % (qn['id'], qn['type']))
continue
elif qn['type'] == 'room_size_compare':
ntnp = '2rm4p'
# 2 rooms
assert len(qn['bbox']) == 2
room1_id, room2_id = qn['bbox'][0]['id'], qn['bbox'][1]['id']
# 4 points
try:
source_point, room1_point, room2_point, end_point = get_4points_for_2rooms(h3d, room1_id, room2_id, args)
except:
print('4 points for 2 rooms not found.')
continue
# compute shortest paths
try:
ordered = True
positions, actions = sample_paths(h3d, [source_point, room1_point, room2_point, end_point])
samples.append({'positions': positions, 'actions': actions, 'ordered': ordered})
except:
print('shortest path not found for question[%s][%s].' % (qn['id'], qn['type']))
continue
# save
if len(samples) == 0:
invalid.append(qn['id']) # do not use += here!
else:
print('%s [%s] samples for question[%s] in %.2fs.' % (len(samples), ntnp, qn['id'], time.time()-tic))
fname = [str(qn['house'])] + [str(box['id'])for box in qn['bbox']]
fname = '.'.join(fname) + '.json'
with open(osp.join(args.shortest_path_dir, fname), 'w') as f:
json.dump(samples, f)
# finished this job
q.task_done()