def draw_frame_grids(self, scr, grids):
grids_meta = [(0, "street boundaries"),
(3, "crashables (except cars)"), (7, "street markings"),
(4, "current lane"), (1, "cars"), (2, "cars in mirrors")]
titles = [title for idx, title in grids_meta]
grids = to_numpy(grids[0])
grids = [grids[idx] for idx, title in grids_meta]
#self.grid_to_graph(scr, grids[0])
bgcolor = [0, 0, 0]
image = np.zeros((720, 1280, 3), dtype=np.uint8) + bgcolor
scr_main = ia.imresize_single_image(
scr, (int(720 * 0.58), int(1280 * 0.58)))
#util.draw_image(image, y=720-scr_main.shape[0], x=1080-scr_main.shape[1], other_img=scr_main, copy=False)
util.draw_image(image,
y=int((image.shape[0] - scr_main.shape[0]) / 2),
x=1280 - scr_main.shape[1] - 2,
other_img=scr_main,
copy=False)
image = util.draw_text(
image,
x=1280 - (scr_main.shape[1] // 2) - 125,
y=image.shape[0] - int(
(image.shape[0] - scr_main.shape[0]) / 2) + 10,
text="Framerate matches the one that the model sees (10fps).",
size=10,
color=[128, 128, 128])
grid_rel_size = 0.19
scr_small = ia.imresize_single_image(
scr, (int(720 * grid_rel_size), int(1280 * grid_rel_size)))
grid_hms = []
for grid, title in zip(grids, titles):
grid = (grid * 255).astype(np.uint8)[:, :, np.newaxis]
grid = ia.imresize_single_image(
grid, (int(720 * grid_rel_size), int(1280 * grid_rel_size)),
interpolation="nearest")
grid_hm = util.draw_heatmap_overlay(scr_small, grid / 255)
grid_hm = np.pad(grid_hm, ((2, 0), (2, 2), (0, 0)),
mode="constant",
constant_values=np.average(bgcolor))
#grid_hm = np.pad(grid_hm, ((0, 20), (0, 0), (0, 0)), mode="constant", constant_values=0)
#grid_hm[-20:, 2:-2, :] = [128, 128, 255]
#grid_hm = util.draw_text(grid_hm, x=4, y=grid_hm.shape[0]-16, text=title, size=10, color=[255, 255, 255])
grid_hm = np.pad(grid_hm, ((40, 0), (0, 0), (0, 0)),
mode="constant",
constant_values=0)
grid_hm = util.draw_text(grid_hm,
x=4,
y=20,
text=title,
size=12,
color=[255, 255, 255])
grid_hms.append(grid_hm)
grid_hms = ia.draw_grid(grid_hms, cols=2)
util.draw_image(image, y=70, x=0, other_img=grid_hms, copy=False)
return image
def _generate_heatmap(self):
return util.draw_heatmap_overlay(self.current_state.screenshot_rs, self.grid, alpha=self.heatmap_alpha)
def generate_debug_image(images, images_prev, \
outputs_ae_gt, outputs_grids_gt, outputs_atts_gt, \
outputs_multiactions_gt, outputs_flow_gt, outputs_canny_gt, \
outputs_flipped_gt, \
outputs_ae_pred, outputs_grids_pred, outputs_atts_pred, \
outputs_multiactions_pred, outputs_flow_pred, outputs_canny_pred, \
outputs_flipped_pred, \
grids_annotated, atts_annotated):
image = to_numpy(images)[0]
grids_annotated = grids_annotated[0]
atts_annotated = atts_annotated[0]
ae_gt = to_numpy(outputs_ae_gt)[0]
grids_gt = to_numpy(outputs_grids_gt)[0]
atts_gt = to_numpy(outputs_atts_gt)[0]
multiactions_gt = to_numpy(outputs_multiactions_gt)[0]
flow_gt = to_numpy(outputs_flow_gt)[0]
canny_gt = to_numpy(outputs_canny_gt)[0]
flipped_gt = to_numpy(outputs_flipped_gt)[0]
ae_pred = to_numpy(outputs_ae_pred)[0]
grids_pred = to_numpy(outputs_grids_pred)[0]
atts_pred = to_numpy(outputs_atts_pred)[0]
multiactions_pred = to_numpy(outputs_multiactions_pred)[0]
flow_pred = to_numpy(outputs_flow_pred)[0]
canny_pred = to_numpy(outputs_canny_pred)[0]
flipped_pred = to_numpy(outputs_flipped_pred)[0]
image = (np.squeeze(image).transpose(1, 2, 0) * 255).astype(np.uint8)
ae_pred = (np.squeeze(ae_pred).transpose(1, 2, 0) * 255).astype(np.uint8)
grids_gt = (np.squeeze(grids_gt).transpose(1, 2, 0) * 255).astype(np.uint8)
grids_pred = (np.squeeze(grids_pred).transpose(1, 2, 0) * 255).astype(
np.uint8)
atts_gt = np.squeeze(atts_gt)
atts_pred = np.squeeze(atts_pred)
multiactions_gt = np.squeeze(multiactions_gt)
multiactions_pred = np.squeeze(multiactions_pred)
flow_gt = (flow_gt.transpose(1, 2, 0) * 255).astype(np.uint8)
flow_pred = (flow_pred.transpose(1, 2, 0) * 255).astype(np.uint8)
canny_gt = (canny_gt.transpose(1, 2, 0) * 255).astype(np.uint8)
canny_pred = (canny_pred.transpose(1, 2, 0) * 255).astype(np.uint8)
#print(image.shape, grid_gt.shape, grid_pred.shape)
h, w = int(image.shape[0] * 0.5), int(image.shape[1] * 0.5)
#h, w = image.shape[0:2]
image_rs = ia.imresize_single_image(image, (h, w), interpolation="cubic")
grids_vis = []
for i in xrange(grids_gt.shape[2]):
grid_gt_rs = ia.imresize_single_image(grids_gt[..., i][:, :,
np.newaxis],
(h, w),
interpolation="cubic")
grid_pred_rs = ia.imresize_single_image(grids_pred[..., i][:, :,
np.newaxis],
(h, w),
interpolation="cubic")
grid_gt_hm = util.draw_heatmap_overlay(image_rs,
np.squeeze(grid_gt_rs) / 255)
grid_pred_hm = util.draw_heatmap_overlay(
image_rs,
np.squeeze(grid_pred_rs) / 255)
if grids_annotated[i] == 0:
grid_gt_hm[::4, ::4, :] = [255, 0, 0]
grids_vis.append(np.hstack((grid_gt_hm, grid_pred_hm)))
"""
lst = [image[0:3]] \
+ [image[3:6]] \
+ [ia.imresize_single_image(ae_pred[:, :, 0:3], (image.shape[0], image.shape[1]), interpolation="cubic")] \
+ [ia.imresize_single_image(ae_pred[:, :, 3:6], (image.shape[0], image.shape[1]), interpolation="cubic")] \
+ [ia.imresize_single_image(np.tile(flow_pred[:, :, 0][:, :, np.newaxis], (1, 1, 3)), (image.shape[0], image.shape[1]), interpolation="cubic")] \
+ [ia.imresize_single_image(np.tile(flow_pred[:, :, 1][:, :, np.newaxis], (1, 1, 3)), (image.shape[0], image.shape[1]), interpolation="cubic")] \
+ grids_vis
print([s.shape for s in lst])
"""
def downscale(im):
return ia.imresize_single_image(
im, (image.shape[0] // 2, image.shape[1] // 2),
interpolation="cubic")
def to_rgb(im):
if im.ndim == 2:
im = im[:, :, np.newaxis]
return np.tile(im, (1, 1, 3))
#print(canny_gt.shape, canny_gt[...,0].shape, to_rgb(canny_gt[...,0]).shape, downscale(to_rgb(canny_gt[...,0])).shape)
#print(canny_pred.shape, canny_gt[...,0].shape, to_rgb(canny_pred[...,0]).shape, downscale(to_rgb(canny_pred[...,0])).shape)
current_image = np.vstack(
#[image[:, :, 0:3]]
[image[:, :, 0:3]] + grids_vis + [
np.hstack([
downscale(ae_pred[:, :, 0:3]),
downscale(to_rgb(ae_pred[:, :, 3]))
])
] + [
np.hstack([
downscale(to_rgb(ae_pred[:, :, 4])),
# downscale(to_rgb(ae_pred[:, :, 5]))
np.zeros_like(downscale(to_rgb(ae_pred[:, :, 4])))
])
] + [
np.hstack([
downscale(to_rgb(flow_gt[..., 0])),
downscale(to_rgb(flow_pred[..., 0]))
])
] + [
np.hstack([
downscale(to_rgb(canny_gt[..., 0])),
downscale(to_rgb(canny_pred[..., 0]))
])
])
y_grids_start = image.shape[0]
grid_height = grids_vis[0].shape[0]
for i, name in enumerate(train.GRIDS_ORDER):
current_image = util.draw_text(current_image,
x=2,
y=y_grids_start +
(i + 1) * grid_height - 12,
text=name,
size=8,
color=[0, 255, 0])
current_image = np.pad(current_image, ((0, 280), (0, 280), (0, 0)),
mode="constant",
constant_values=0)
texts = []
att_idx = 0
for i, att_group in enumerate(ATTRIBUTE_GROUPS):
texts.append(att_group.name_shown)
for j, att in enumerate(att_group.attributes):
if atts_annotated[0] == 0:
texts.append(" %s | ? | %.2f" % (att.name, atts_pred[att_idx]))
else:
texts.append(" %s | %.2f | %.2f" %
(att.name, atts_gt[att_idx], atts_pred[att_idx]))
att_idx += 1
current_image = util.draw_text(current_image,
x=current_image.shape[1] - 256 + 1,
y=1,
text="\n".join(texts),
size=8,
color=[0, 255, 0])
ma_texts = ["multiactions (prev avg, next avg, curr, next)"]
counter = 0
while counter < multiactions_gt.shape[0]:
ma_texts_sub = ([], [])
for j in xrange(9):
ma_texts_sub[0].append("%.2f" % (multiactions_gt[counter], ))
ma_texts_sub[1].append("%.2f" % (multiactions_pred[counter], ))
counter += 1
ma_texts.append(" ".join(ma_texts_sub[0]))
ma_texts.append(" ".join(ma_texts_sub[1]))
ma_texts.append("")
current_image = util.draw_text(current_image,
x=current_image.shape[1] - 256 + 1,
y=650,
text="\n".join(ma_texts),
size=8,
color=[0, 255, 0])
flipped_texts = [
"flipped", " ".join(
["%.2f" % (flipped_gt[i], ) for i in xrange(flipped_gt.shape[0])]),
" ".join([
"%.2f" % (flipped_pred[i], ) for i in xrange(flipped_pred.shape[0])
])
]
current_image = util.draw_text(current_image,
x=current_image.shape[1] - 256 + 1,
y=810,
text="\n".join(flipped_texts),
size=8,
color=[0, 255, 0])
return current_image
def generate_overview_image(current_state, last_state, \
action_up_down_bpe, action_left_right_bpe, \
memory, memory_val, \
ticks, last_train_tick, \
plans, plan_to_rewards_direct, plan_to_reward_indirect, \
plan_to_reward, plans_ranking, current_plan, best_plan_ae_decodings,
idr_v, idr_adv,
grids, args):
h, w = current_state.screenshot_rs.shape[0:2]
scr = np.copy(current_state.screenshot_rs)
scr = ia.imresize_single_image(scr, (h // 2, w // 2))
if best_plan_ae_decodings is not None:
ae_decodings = (to_numpy(best_plan_ae_decodings) * 255).astype(
np.uint8).transpose((0, 2, 3, 1))
ae_decodings = [
ia.imresize_single_image(ae_decodings[i, ...], (h // 4, w // 4))
for i in xrange(ae_decodings.shape[0])
]
ae_decodings = ia.draw_grid(ae_decodings, cols=5)
#ae_decodings = np.vstack([
# np.hstack(ae_decodings[0:5]),
# np.hstack(ae_decodings[5:10])
#])
else:
ae_decodings = np.zeros((1, 1, 3), dtype=np.uint8)
if grids is not None:
scr_rs = ia.imresize_single_image(scr, (h // 4, w // 4))
grids = (to_numpy(grids)[0] * 255).astype(np.uint8)
grids = [
ia.imresize_single_image(grids[i, ...][:, :, np.newaxis],
(h // 4, w // 4))
for i in xrange(grids.shape[0])
]
grids = [
util.draw_heatmap_overlay(
scr_rs,
np.squeeze(grid / 255).astype(np.float32)) for grid in grids
]
grids = ia.draw_grid(grids, cols=4)
else:
grids = np.zeros((1, 1, 3), dtype=np.uint8)
plans_text = []
if idr_v is not None and idr_adv is not None:
idr_v = to_numpy(idr_v[0])
idr_adv = to_numpy(idr_adv[0])
plans_text.append("V(s): %+07.2f" % (idr_v[0], ))
adv_texts = []
curr = []
for i, ma in enumerate(actionslib.ALL_MULTIACTIONS):
curr.append("A(%s%s): %+07.2f" %
(ma[0] if ma[0] != "~WS" else "_",
ma[1] if ma[1] != "~AD" else "_", idr_adv[i]))
if (i + 1) % 3 == 0 or (i + 1) == len(actionslib.ALL_MULTIACTIONS):
adv_texts.append(" ".join(curr))
curr = []
plans_text.extend(adv_texts)
if current_plan is not None:
plans_text.append("")
plans_text.append("Current Plan:")
actions_ud_text = []
actions_lr_text = []
for multiaction in current_plan:
actions_ud_text.append(
"%s" % (multiaction[0] if multiaction[0] != "~WS" else "_", ))
actions_lr_text.append(
"%s" % (multiaction[1] if multiaction[1] != "~AD" else "_", ))
plans_text.extend(
[" ".join(actions_ud_text), " ".join(actions_lr_text)])
plans_text.append("")
plans_text.append("Best Plans:")
if plan_to_rewards_direct is not None:
for plan_idx in plans_ranking[::-1][0:5]:
plan = plans[plan_idx]
rewards_direct = plan_to_rewards_direct[plan_idx]
reward_indirect = plan_to_reward_indirect[plan_idx]
reward = plan_to_reward[plan_idx]
actions_ud_text = []
actions_lr_text = []
rewards_text = []
for multiaction in plan:
actions_ud_text.append(
"%s" %
(multiaction[0] if multiaction[0] != "~WS" else "_", ))
actions_lr_text.append(
"%s" %
(multiaction[1] if multiaction[1] != "~AD" else "_", ))
for rewards_t in rewards_direct:
rewards_text.append("%+04.1f" % (rewards_t, ))
rewards_text.append("| %+07.2f (V(s')=%+07.2f)" %
(reward, reward_indirect))
plans_text.extend([
"", " ".join(actions_ud_text), " ".join(actions_lr_text),
" ".join(rewards_text)
])
plans_text = "\n".join(plans_text)
stats_texts = [
"u/d bpe: %s" % (action_up_down_bpe.rjust(5)),
" l/r bpe: %s" % (action_left_right_bpe.rjust(5)),
"u/d ape: %s %s" %
(current_state.action_up_down.rjust(5),
"[C]" if action_up_down_bpe != current_state.action_up_down else ""),
" l/r ape: %s %s" %
(current_state.action_left_right.rjust(5), "[C]"
if action_left_right_bpe != current_state.action_left_right else ""),
"speed: %03d" % (current_state.speed, )
if current_state.speed is not None else "speed: None",
"is_reverse: yes" if current_state.is_reverse else "is_reverse: no",
"is_damage_shown: yes" if current_state.is_damage_shown else
"is_damage_shown: no", "is_offence_shown: yes"
if current_state.is_offence_shown else "is_offence_shown: no",
"steering wheel: %05.2f (%05.2f)" %
(current_state.steering_wheel_cnn,
current_state.steering_wheel_raw_cnn),
"reward for last state: %05.2f" % (last_state.reward, )
if last_state is not None else "reward for last state: None",
"p_explore: %.2f%s" %
(current_state.p_explore if args.p_explore is None else args.p_explore,
"" if args.p_explore is None else " (constant)"),
"memory size (train/val): %06d / %06d" %
(memory.size, memory_val.size),
"ticks: %06d" % (ticks, ),
"last train: %06d" % (last_train_tick, )
]
stats_text = "\n".join(stats_texts)
all_texts = plans_text + "\n\n\n" + stats_text
result = np.zeros((720, 590, 3), dtype=np.uint8)
util.draw_image(result, x=0, y=0, other_img=scr, copy=False)
util.draw_image(result,
x=0,
y=scr.shape[0] + 10,
other_img=ae_decodings,
copy=False)
util.draw_image(result,
x=0,
y=scr.shape[0] + 10 + ae_decodings.shape[0] + 10,
other_img=grids,
copy=False)
result = util.draw_text(result,
x=0,
y=scr.shape[0] + 10 + ae_decodings.shape[0] + 10 +
grids.shape[0] + 10,
size=8,
text=all_texts,
color=[255, 255, 255])
return result