def predict(model, model_path, dataset, gpu, in_episodes, out_episodes, out,
start_from, images_num):
if gpu >= 0:
info("Using GPU %d" % gpu)
chainer.cuda.get_device_from_id(gpu).use()
model.to_gpu()
else:
info("Using CPU")
chainer.serializers.load_npz(model_path, model)
os.makedirs(out, exist_ok=True)
chainer.config.train = False
info("Forwarding")
xp = model.xp
for n in range(start_from, images_num):
data = dataset[n]
in_data, next_data = data[:in_episodes], data[in_episodes:in_episodes +
out_episodes]
in_data, next_data = in_data[np.newaxis, :], next_data[np.newaxis, :]
in_data = chainer.Variable(in_data)
if gpu >= 0:
with chainer.cuda.get_device_from_id(gpu):
in_data.to_gpu()
out_data = model(in_data)
reconst, pred = out_data[0], out_data[1]
fig = plt.figure()
ax = fig.add_subplot(2, 2, 1)
ax = vis_episode(in_data, ax=ax, title="Input")
ax = fig.add_subplot(2, 2, 2)
ax = vis_episode(next_data, ax=ax, title="Predicted Ground Truth")
ax = fig.add_subplot(2, 2, 3)
ax = vis_episode(reconst, ax=ax, title="Reconstruction")
ax = fig.add_subplot(2, 2, 4)
ax = vis_episode(pred, ax=ax, title="Prediction")
out_path = os.path.join(out, "result_%03d.png" % n)
plt.savefig(out_path, bbox_inches="tight", dpi=160)
info("saved to %s" % out_path)
plt.close(fig)
def show_dataset(index, split, vertical, out):
dataset = chainervr.datasets.UCF101Dataset(split=split)
in_data = dataset[index]
ax = vis_episode(in_data, vertical=vertical)
plt.title("index %s" % index)
plt.savefig(out, bbox_inches="tight", dpi=300)
print("Saved to %s" % out)
def detect(model_path, features_path, input_num, top_n, gpu, out, split,
in_episodes, out_episodes, hidden_channels):
"""Extract hidden features for each data in dataset"""
import chainer
import chainervr
import os
import numpy as np
import pandas as pd
from sklearn.metrics.pairwise import cosine_similarity
import matplotlib.pyplot as plt
assert in_episodes == out_episodes
model = chainervr.models.DeepEpisodicMemory(
hidden_channels=hidden_channels, num_episodes=in_episodes)
model.reset_state()
if gpu >= 0:
P.info("Using GPU %d" % gpu)
chainer.cuda.get_device_from_id(gpu).use()
model.to_gpu()
else:
P.info("Using CPU")
chainer.serializers.load_npz(model_path, model)
P.info("Loading dataset")
dataset = chainervr.datasets.MovingMnistDataset(split=split,
channels_num=3)
os.makedirs(out, exist_ok=True)
chainer.config.train = False
P.info("Loading features")
features = np.load(features_path)["features"]
P.info("Forwarding: %d" % input_num)
in_data = dataset[input_num][:in_episodes][np.newaxis, :]
in_data = chainer.Variable(in_data)
if gpu >= 0:
with chainer.cuda.get_device_from_id(gpu):
in_data.to_gpu()
# forwarding
_, _, hidden = model(in_data)
if gpu >= 0:
hidden.to_cpu()
feature = hidden.array.reshape(-1)
# compute similarities
similarities = cosine_similarity(
features, np.broadcast_to(feature, (len(dataset), feature.shape[0])))
similarities = np.diag(similarities)
ranking = (-similarities).argsort()
rows = top_n + 1
fig = plt.figure(figsize=(3, rows))
ax = fig.add_subplot(rows, 1, 1)
ax = vis_episode(in_data, ax=ax, title="Input")
for n in range(top_n):
index = ranking[n]
similarity = similarities[index]
comp_data = dataset[index][:in_episodes][np.newaxis, :]
ax = fig.add_subplot(rows, 1, n + 2)
ax = vis_episode(comp_data,
ax=ax,
title="Top %d (index: %d, similarity: %f)" %
(n + 1, index, similarity))
os.makedirs(out, exist_ok=True)
out_path = os.path.join(out, "rank.png")
plt.savefig(out_path, bbox_inches="tight", dpi=300)
P.info("saved to %s" % out_path)
plt.close(fig)
data = utils.load_episode(videoname,
num_episodes=self.num_episodes)
if self.with_class:
return data, cls
else:
return data
if __name__ == '__main__':
import os
if not os.getenv("DISPLAY", None):
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from chainervr.visualizations import vis_episode
dataset = UCF101Dataset()
fig = plt.figure()
n = 10
for i in range(n):
idx = (len(dataset) // n) * i
episode, cls = dataset[idx]
cls_str = utils.ucf101_class_names[cls]
ax = fig.add_subplot(n, 1, i+1)
vis_episode(episode, ax=ax, title=cls_str, fontsize=4)
plt.savefig("image.jpg", bbox_inches="tight", dpi=300)
plt.close()
def predict_summary(model,
model_dir,
dataset,
gpu,
in_episodes,
out_episodes,
out,
image_num,
skip_num=None):
if gpu >= 0:
info("Using GPU %d" % gpu)
chainer.cuda.get_device_from_id(gpu).use()
model.to_gpu()
else:
info("Using CPU")
glob_ex = os.path.join(model_dir, "model_iter_")
model_paths = glob.glob(glob_ex + "*")
model_paths.sort(key=lambda s: int(s[len(glob_ex):]))
if skip_num is not None:
model_paths = model_paths[::skip_num]
info("Found %d models" % len(model_paths))
data = dataset[image_num]
in_data, next_data = data[:in_episodes], data[in_episodes:in_episodes +
out_episodes]
in_data, next_data = in_data[np.newaxis, :], next_data[np.newaxis, :]
in_data = chainer.Variable(in_data)
if gpu >= 0:
with chainer.cuda.get_device_from_id(gpu):
in_data.to_gpu()
rows = len(model_paths) + 1
fig = plt.figure(figsize=(3, rows))
i = 1
ax = fig.add_subplot(rows, 2, i)
ax = vis_episode(in_data, ax=ax, title="Input")
i += 1
ax = fig.add_subplot(rows, 2, i)
ax = vis_episode(in_data, ax=ax, title="Predicted Ground Truth")
i += 1
for model_path in model_paths:
model_name = os.path.basename(model_path)
chainer.serializers.load_npz(model_path, model)
chainer.config.train = False
out_data = model(in_data)
reconst, pred = out_data[0], out_data[1]
ax = fig.add_subplot(rows, 2, i)
ax = vis_episode(reconst, ax=ax, title="%s Reconst" % model_name)
i += 1
ax = fig.add_subplot(rows, 2, i)
ax = vis_episode(pred, ax=ax, title="%s Pred" % model_name)
i += 1
info("Generated image using %s" % model_path)
os.makedirs(out, exist_ok=True)
out_path = os.path.join(out, "summary.png")
plt.savefig(out_path, bbox_inches="tight", dpi=300)
info("saved to %s" % out_path)
plt.close(fig)
channels_num = 3
self.channels_num = channels_num
def __len__(self):
return self.dataset.shape[0]
def get_example(self, i):
data = self.dataset[i] # FHW
data = np.broadcast_to(data,
(self.channels_num, ) + data.shape) # CFHW
# CFHW -> FCHW
return data.transpose((1, 0, 2, 3))
if __name__ == '__main__':
import os
if not os.getenv("DISPLAY", None):
import matplotlib
matplotlib.use('Agg')
from chainervr.visualizations import vis_episode
import matplotlib.pyplot as plt
dataset = MovingMnistDataset()
for i, frames in enumerate(dataset):
vis_episode(frames)
plt.title("Dataset %d/%d" % (i, len(dataset)))
fname = "image_%05d.jpg" % i
plt.savefig(fname, dpi=300, bbox_inches="tight")
print("Saved to %s" % fname)
plt.close(plt.figure())