def _image_summaries(self, data, embed, image_pred):
summary_size = 6 # nr images to be shown
summary_length = 5 # nr step observed before dreaming
if self._c.obs_type in ['image', 'lidar']:
truth = data[self._c.obs_type][:summary_size] + 0.5
recon = image_pred.mode()[:summary_size]
init, _ = self._dynamics.observe(embed[:summary_size, :summary_length],
data['action'][:summary_size, :summary_length])
init = {k: v[:, -1] for k, v in init.items()}
prior = self._dynamics.imagine(data['action'][:summary_size, summary_length:], init)
openl = self._decode(self._dynamics.get_feat(prior)).mode()
model = tf.concat([recon[:, :summary_length] + 0.5, openl + 0.5], 1)
if self._c.obs_type == "lidar":
truth = tools.lidar_to_image(truth)
model = tools.lidar_to_image(model)
error = model - truth
else:
error = (model - truth + 1) / 2
openl = tf.concat([truth, model, error], 2)
elif self._c.obs_type == 'lidar_occupancy':
truth = data[self._c.obs_type][:summary_size]
recon = image_pred.mode()[:summary_size]
recon = tf.cast(recon, tf.float32) # concatenation requires same type
init, _ = self._dynamics.observe(embed[:summary_size, :summary_length],
data['action'][:summary_size, :summary_length])
init = {k: v[:, -1] for k, v in init.items()}
prior = self._dynamics.imagine(data['action'][:summary_size, summary_length:], init)
openl = self._decode(self._dynamics.get_feat(prior)).mode()
openl = tf.cast(openl, tf.float32)
model = tf.concat([recon[:, :summary_length], openl],
1) # note: recon/openl is already 0 or 1, no need scaling
error = (model - truth + 1) / 2
openl = tf.concat([truth, model, error], 2)
tools.graph_summary(self._writer, tools.video_summary,
'agent/train/autoencoder', openl, self._step, int(100 / self._c.action_repeat))
def _reward_summaries(self, data, reward_pred):
summary_size = 6 # nr images to be shown
truth = tools.reward_to_image(data['reward'][:summary_size])
model = tools.reward_to_image(reward_pred.mode()[:summary_size])
error = model - truth
video_image = tf.concat([truth, model, error], 1) # note: no T dimension, then stack over dim 1
video_image = tf.expand_dims(video_image, axis=1) # since no gif, expand dim=1 (T), B,H,W,C -> B,T,H,W,C
tools.graph_summary(self._writer, tools.video_summary,
'agent/train/reward', video_image, self._step, int(100 / self._c.action_repeat))
def _image_summaries(self, data, embed, image_pred):
truth = data["image"][:6] + 0.5
recon = image_pred.mode()[:6]
init, _ = self._dynamics.observe(embed[:6, :5], data["action"][:6, :5])
init = {k: v[:, -1] for k, v in init.items()}
prior = self._dynamics.imagine(data["action"][:6, 5:], init)
openl = self._decode(self._dynamics.get_feat(prior)).mode()
model = tf.concat([recon[:, :5] + 0.5, openl + 0.5], 1)
error = (model - truth + 1) / 2
openl = tf.concat([truth, model, error], 2)
tools.graph_summary(self._writer, tools.video_summary, "agent/openl", openl)
def _image_summaries(self, data, embed, image_pred):
# print("data['obs']:", data["obs"].shape) # (50, 10, 64, 64, 1)
truth = data["obp1s"][:6] + 0.5
recon = image_pred.mode()[:6]
init, _ = self.dynamics.observe(
embed[:6, :5],
data["actions"][:6, :5]) # scaning st-1 to get post of st
init = {k: v[:, -1] for k, v in init.items()}
prior = self.dynamics.imagine(data["actions"][:6, 5:], init) # scaning
openl = self.decoder(self.dynamics.get_feat(prior)).mode()
model = tf.concat([recon[:, :5] + 0.5, openl + 0.5], 1)
error = (model - truth + 1) / 2
openl = tf.concat([truth, model, error], 2)
tools.graph_summary(self._writer, tools.video_summary, "agent/openl",
openl)
