def generate_fovs(image_path, node_path, fov_prefix, fov_size=400):
'''Generates FoV images for given list of objects.
'''
cam = camera(output_image_shape=(fov_size, fov_size))
cam.load_img(image_path)
nodes = np.load(node_path, allow_pickle=True)[()]
for jj, n in enumerate(nodes.keys()):
idx, xlng, ylat = nodes[n]['id'], nodes[n]['xlng'], nodes[n]['ylat']
cam.look(xlng, ylat)
fov_img = cam.get_image()
fov_file = fov_prefix + '{}.jpg'.format(idx)
# cv2.imwrite(fov_file, fov_img)
pil_img = Image.fromarray(fov_img)
pil_img.save(fov_file)
def generate_gt_moves(image_path,
move_list,
move_ids,
fov_root='',
fov_size=400):
cam = camera(output_image_shape=(fov_size, fov_size))
cam.load_img(image_path)
for jj, (move_id, fovs) in enumerate(zip(move_ids, move_list)):
for kk, fov in enumerate(fovs):
xlng, ylat = fov[0], fov[1]
cam.look(xlng, ylat)
fov_img = cam.get_image()
fov_prefix = os.path.join(fov_root, '{}.gt_move.'.format(move_id))
fov_file = fov_prefix + 'move{}.jpg'.format(kk)
# cv2.imwrite(fov_file, fov_img)
pil_img = Image.fromarray(fov_img)
pil_img.save(fov_file)
def generate_balanced_fovs(image_path,
nodes,
fov_prefix,
full_w=4552,
full_h=2276,
fov_size=400,
n_fovs=50):
cam = camera(output_image_shape=(fov_size, fov_size))
cam.load_img(image_path)
size = 20
canvas = np.zeros((full_h, full_w, 3), dtype='uint8')
THRESHOLD = 100
fov_nodes = []
oclr = (255, 0, 0)
fclr = (0, 0, 255)
font = cv2.FONT_HERSHEY_SIMPLEX
for jj, n in enumerate(nodes.keys()):
idx, olat, olng, ox, oy = nodes[n]['id'], nodes[n]['lat'], nodes[n][
'lng'], nodes[n]['x'], nodes[n]['y']
if idx == 0:
continue
cam.look(olat, olng)
canvas[oy - size:oy + size, ox - size:ox + size, 0] = 255.
canvas[oy - size:oy + size, ox - size:ox + size, 1:] = 0
cv2.putText(canvas, str(idx), (ox + size, oy + size), font, 3, oclr, 5)
for x_coeff in np.arange(-1.5, 2, 1):
for y_coeff in np.arange(-1.5, 2, 1):
if not x_coeff and not y_coeff:
continue
x_diff = fov_size * x_coeff + random.randint(0, size)
y_diff = fov_size * y_coeff + random.randint(0, size)
if ox + x_diff > full_w:
new_x = (ox + x_diff) - full_w
elif ox + x_diff < 0:
new_x = 2 * full_w - (ox + x_diff)
else:
new_x = ox + x_diff
if oy + y_diff > full_h:
new_y = (oy + y_diff) - full_h
elif oy + y_diff < 0:
new_y = 2 * full_h - (oy + y_diff)
else:
new_y = oy + y_diff
new_x = int(new_x)
new_y = int(new_y)
# flag = True
# for fov in fov_nodes:
# px, py = fov[0], fov[1]
# d = ((px - new_x)**2 + (py - new_y)**2)**0.5
# if d < THRESHOLD:
# flag = False
# break
# if flag:
fov_nodes.append([new_x, new_y])
canvas[new_y - size:new_y + size,
new_x - size:new_x + size, :2] = 0
canvas[new_y - size:new_y + size, new_x - size:new_x + size,
2] = 255
# cv2.putText(canvas, 'x{}|y{}'.format(x_coeff, y_coeff),
# (new_x+size, new_y+size), font, 3, fclr, 5)
fov_data = np.array(fov_nodes)
kmeans = KMeans(n_clusters=n_fovs, random_state=0).fit(fov_data)
fov_nodes = kmeans.cluster_centers_.tolist()
node_dict = {}
for jj, fov in enumerate(fov_nodes):
x, y = int(fov[0]), int(fov[1])
canvas[y - size:y + size, x - size:x + size, 0] = 0
canvas[y - size:y + size, x - size:x + size, 1] = 255
canvas[y - size:y + size, x - size:x + size, 2] = 0
lat = (x * 1.0 / full_w) * 360 - 180
lng = (full_h - y * 1.0) * (180. / full_h) - 90
cam.look(lat, lng)
#pixel_map = cam.get_map()
# test_lat = pixel_map[200][200][0].item()
# test_lng = pixel_map[200][200][1].item()
fov_img = cam.get_image()
fov_file = fov_prefix + '{}.jpg'.format(jj)
pil_img = Image.fromarray(fov_img)
pil_img.save(fov_file)
n = {
'id': jj,
'lat': lat,
'lng': lng,
'obj_label': 'fov',
'obj_id': 0,
'x': x,
'y': y,
'boxes': [],
'neighbors': []
}
node_dict[jj] = n
canvas = cv2.resize(canvas, (800, 400))
return node_dict, canvas
def generate_graph(
image_path,
predictor,
vg_classes,
full_w=2048, # full_w=4552,
full_h=1024, # full_h=2276,
left_w=32, # left_w=128,
topk=100):
font = cv2.FONT_HERSHEY_SIMPLEX
THRESHOLD = 50
size = 20
all_objects = []
all_nodes = []
all_scores = []
slat, slng = rad2degree(np.random.uniform(0, 6),
np.random.uniform(1, 1.5),
adjust=True)
sx = int(full_w * ((slat + 180) / 360.0))
sy = int(full_h - full_h * ((slng + 90) / 180.0))
all_objects.append(
(slat, slng, -1, sx, sy, [sx - 1, sy - 1, sx + 1, sy + 1]))
all_nodes.append([sx, sy])
all_scores = [1]
for fov_size in [400, 1200]:
cam = camera(output_image_shape=(fov_size, fov_size))
cam.load_img(image_path)
for lng in range(-45, 90, 45):
for lat in range(-165, 180, 30):
cam.look(lat, lng)
pixel_map = cam.get_map()
img = cam.get_image()
detectron_outputs = predictor(img)
boxes, object_types, det_scores = get_det2_features(
detectron_outputs["instances"].to("cpu"))
for b, o, s in zip(boxes, object_types, det_scores):
center_x = int((b[0] + b[2]) / 2)
center_y = int((b[1] + b[3]) / 2)
o_lat = pixel_map[center_y][center_x][0].item()
o_lng = pixel_map[center_y][center_x][1].item()
flag = True
for r in all_objects:
d = ((r[0] - o_lng)**2 + (r[1] - o_lat)**2)**0.5
if d < THRESHOLD and r[2] == o:
flag = False
if flag:
x = int(full_w * ((o_lat + 180) / 360.0))
y = int(full_h - full_h * ((o_lng + 90) / 180.0))
all_objects.append((o_lat, o_lng, o, x, y, b.tolist()))
all_nodes.append([x, y])
all_scores.append(s)
scores, nodes, objects = zip(
*sorted(zip(all_scores, all_nodes, all_objects), reverse=True))
scores = list(scores[:topk])
nodes = list(nodes[:topk])
objects = list(objects[:topk])
print('# of detected objects {} | {} used objects'.format(
len(all_nodes), len(nodes)))
tri, order2nid, n_nodes = get_triangulation(nodes, left_w, full_w)
canvas = np.zeros((full_h, full_w, 3), dtype='uint8')
clr = (255, 0, 0)
node_dict = dict()
for kk, o in enumerate(objects):
o_type, ox, oy = o[2], o[3], o[4]
canvas[oy - size:oy + size, ox - size:ox + size, 0] = 255.
canvas[oy - size:oy + size, ox - size:ox + size, 1:] = 0
o_label = '<START>'
if o_type > 0:
o_label = vg_classes[o_type]
cv2.putText(canvas, o_label, (ox + size, oy + size), font, 3, clr, 5)
n = {
'id': kk,
'lat': o[0],
'lng': o[1],
'obj_label': o_label,
'obj_id': o_type,
'x': o[3],
'y': o[4],
'boxes': o[5],
'neighbors': []
}
node_dict[kk] = n
debug_color = (0, 0, 256)
color = (125, 125, 125)
for s in tri.simplices:
x0 = nodes[s[0]][0]
color1 = color2 = color3 = color
if x0 > full_w:
x0 -= full_w
color1 = debug_color
color2 = debug_color
x1 = nodes[s[1]][0]
if x1 > full_w:
x1 -= full_w
color1 = debug_color
color3 = debug_color
x2 = nodes[s[2]][0]
if x2 > full_w:
x2 -= full_w
color2 = debug_color
color3 = debug_color
n0 = order2nid[s[0]]
n1 = order2nid[s[1]]
n2 = order2nid[s[2]]
node_dict[n0]['neighbors'] += [n1]
node_dict[n1]['neighbors'] += [n0]
node_dict[n0]['neighbors'] += [n2]
node_dict[n2]['neighbors'] += [n0]
node_dict[n1]['neighbors'] += [n2]
node_dict[n2]['neighbors'] += [n1]
cv2.line(canvas, (x0, nodes[s[0]][1]), (x1, nodes[s[1]][1]), color1, 3,
8)
cv2.line(canvas, (x0, nodes[s[0]][1]), (x2, nodes[s[2]][1]), color2, 3,
8)
cv2.line(canvas, (x2, nodes[s[2]][1]), (x1, nodes[s[1]][1]), color3, 3,
8)
canvas = cv2.resize(canvas, (800, 400))
return node_dict, canvas