def plot_predictions(model, data_loader, key, plt=None):
if plt is None:
plt = matplotlib.pyplot
model.eval()
n_examples = 10
pred = utils.apply_on_dataset(
model=model,
dataset=data_loader.dataset,
output_keys_regexp=key,
max_num_examples=n_examples,
description='plot_predictions:{}'.format(key))[key]
probs = torch.softmax(pred, dim=1)
probs = utils.to_numpy(probs)
data = [data_loader.dataset[i][0] for i in range(n_examples)]
labels = [data_loader.dataset[i][1] for i in range(n_examples)]
samples = torch.stack(data, dim=0)
samples = revert_normalization(samples, data_loader.dataset)
samples = utils.to_numpy(samples)
fig, ax = plt.subplots(nrows=n_examples,
ncols=2,
figsize=(2 * 2, 2 * n_examples))
for i in range(n_examples):
ax[i][0].imshow(get_image(samples[i]), vmin=0, vmax=1)
ax[i][0].set_axis_off()
ax[i][0].set_title('labels as {}'.format(labels[i]))
ax[i][1].bar(range(model.num_classes), probs[i])
ax[i][1].set_xticks(range(model.num_classes))
return fig, plt
def latent_space_tsne(model, data_loader, plt=None):
"""A scatter plot of latent factors on some 2-d subspace, with points colored according to test labels."""
model.eval()
if plt is None:
plt = matplotlib.pyplot
tab = [
'tab:blue', 'tab:orange', 'tab:green', 'tab:red', 'tab:purple',
'tab:brown', 'tab:pink', 'tab:gray', 'tab:olive', 'tab:cyan'
]
z = []
labels = []
for batch_data, batch_labels in data_loader:
z_batch = model(inputs=[batch_data])['z']
z.append(utils.to_numpy(z_batch))
labels.append(utils.to_numpy(batch_labels))
z = np.concatenate(z, axis=0)
labels = np.concatenate(labels, axis=0)
fig, ax = plt.subplots(1)
legend = []
# if labels are not vectors, take the first coordinate
if len(labels.shape) > 1:
assert len(labels.shape) == 2
labels = labels[:, 0]
# replace labels with integers
possible_labels = list(np.unique(labels))
labels = [possible_labels.index(label) for label in labels]
# run TSNE for embedding z into 2 dimensional space
z = TSNE(n_components=2).fit_transform(z)
# select up to 10000 points
cnt = min(10000, len(labels))
indices = np.random.choice(len(labels), cnt)
z = z[indices]
labels = [labels[idx] for idx in indices]
for i in np.unique(labels):
indices = (labels == i)
ax.scatter(z[indices, 0],
z[indices, 1],
marker='.',
color=tab[i],
alpha=0.5,
edgecolor='',
label=i)
legend.append(str(i))
fig.legend(legend)
L = np.percentile(z, q=5, axis=0)
R = np.percentile(z, q=95, axis=0)
ax.set_xlim(L[0], R[0])
ax.set_ylim(L[1], R[1])
ax.set_title('Latent space TSNE plot')
return fig, plt
def ce_gradient_pair_scatter(
model, data_loader, d1=0, d2=1, max_num_examples=2000, plt=None
):
if plt is None:
plt = matplotlib.pyplot
model.eval()
pred = utils.apply_on_dataset(
model=model,
dataset=data_loader.dataset,
output_keys_regexp="pred",
max_num_examples=max_num_examples,
description="grad-pair-scatter:pred",
)["pred"]
n_examples = min(len(data_loader.dataset), max_num_examples)
labels = []
for idx in range(n_examples):
labels.append(data_loader.dataset[idx][1])
labels = torch.tensor(labels, dtype=torch.long)
labels = F.one_hot(labels, num_classes=model.num_classes).float()
labels = utils.to_cpu(labels)
grad_wrt_logits = torch.softmax(pred, dim=-1) - labels
grad_wrt_logits = utils.to_numpy(grad_wrt_logits)
fig, ax = plt.subplots(1, figsize=(5, 5))
plt.scatter(grad_wrt_logits[:, d1], grad_wrt_logits[:, d2])
ax.set_xlabel(str(d1))
ax.set_ylabel(str(d2))
# L = np.percentile(grad_wrt_logits, q=5, axis=0)
# R = np.percentile(grad_wrt_logits, q=95, axis=0)
# ax.set_xlim(L[d1], R[d1])
# ax.set_ylim(L[d2], R[d2])
ax.set_title("Two coordinates of grad wrt to logits")
return fig, plt
def ce_gradient_norm_histogram(
model, data_loader, tensorboard, epoch, name, max_num_examples=5000
):
model.eval()
pred = utils.apply_on_dataset(
model=model,
dataset=data_loader.dataset,
output_keys_regexp="pred",
description="grad-histogram:pred",
max_num_examples=max_num_examples,
)["pred"]
n_examples = min(len(data_loader.dataset), max_num_examples)
labels = []
for idx in range(n_examples):
labels.append(data_loader.dataset[idx][1])
labels = torch.tensor(labels, dtype=torch.long)
labels = F.one_hot(labels, num_classes=model.num_classes).float()
labels = utils.to_cpu(labels)
grad_wrt_logits = torch.softmax(pred, dim=-1) - labels
grad_norms = torch.sum(grad_wrt_logits ** 2, dim=-1)
grad_norms = utils.to_numpy(grad_norms)
try:
tensorboard.add_histogram(tag=name, values=grad_norms, global_step=epoch)
except ValueError as e:
print("Tensorboard histogram error: {}".format(e))
def pred_gradient_pair_scatter(model,
data_loader,
d1=0,
d2=1,
max_num_examples=2000,
plt=None):
if plt is None:
plt = matplotlib.pyplot
model.eval()
grad_pred = utils.apply_on_dataset(
model=model,
dataset=data_loader.dataset,
output_keys_regexp='grad_pred',
max_num_examples=max_num_examples,
description='grad-pair-scatter:grad_pred')['grad_pred']
grad_pred = utils.to_numpy(grad_pred)
fig, ax = plt.subplots(1, figsize=(5, 5))
plt.scatter(grad_pred[:, d1], grad_pred[:, d2])
ax.set_xlabel(str(d1))
ax.set_ylabel(str(d2))
# L = np.percentile(grad_pred, q=5, axis=0)
# R = np.percentile(grad_pred, q=95, axis=0)
# ax.set_xlim(L[d1], R[d1])
# ax.set_ylim(L[d2], R[d2])
ax.set_title('Two coordinates of grad wrt to logits')
return fig, plt
def reconstruction_plot(model, train_data, val_data, n_samples=5, plt=None):
"""Plots reconstruction examples for training & validation sets."""
model.eval()
if plt is None:
plt = pyplot
train_samples = [train_data[i][0] for i in range(n_samples)]
val_samples = [val_data[i][0] for i in range(n_samples)]
samples = torch.stack(train_samples + val_samples, dim=0)
x_rec = model(inputs=[samples])["x_rec"]
x_rec = x_rec.reshape(samples.shape)
samples = revert_normalization(samples, train_data)
samples = utils.to_numpy(samples)
x_rec = utils.to_numpy(x_rec)
fig, ax = plt.subplots(nrows=2 * n_samples, ncols=2, figsize=(2, 2 * n_samples))
for i in range(2 * n_samples):
ax[i][0].imshow(get_image(samples[i]), vmin=0, vmax=1)
ax[i][0].set_axis_off()
ax[i][1].imshow(get_image(x_rec[i]), vmin=0, vmax=1)
ax[i][1].set_axis_off()
return fig, plt
def plot_confusion_matrix(Q, plt=None):
if plt is None:
plt = matplotlib.pyplot
num_classes = Q.shape[0]
fig, ax = plt.subplots(1, figsize=(5, 5))
im = ax.imshow(utils.to_numpy(Q))
fig.colorbar(im)
ax.set_xticks(range(num_classes))
ax.set_yticks(range(num_classes))
ax.set_xlabel("observed")
ax.set_ylabel("true")
return fig, plt
def pred_gradient_norm_histogram(
model, data_loader, tensorboard, epoch, name, max_num_examples=5000
):
model.eval()
grad_pred = utils.apply_on_dataset(
model=model,
dataset=data_loader.dataset,
output_keys_regexp="grad_pred",
description="grad-histogram:grad_pred",
max_num_examples=max_num_examples,
)["grad_pred"]
grad_norms = torch.sum(grad_pred ** 2, dim=-1)
grad_norms = utils.to_numpy(grad_norms)
try:
tensorboard.add_histogram(tag=name, values=grad_norms, global_step=epoch)
except ValueError as e:
print("Tensorboard histogram error: {}".format(e))
def estimate_transition(load_from, data_loader, device="cpu", batch_size=256):
""" Estimates the label noise matrix. The code is adapted form the original implementation.
Source: https://github.com/giorgiop/loss-correction/.
"""
assert load_from is not None
model = utils.load(load_from, device=device)
pred = utils.apply_on_dataset(
model=model,
dataset=data_loader.dataset,
batch_size=batch_size,
cpu=True,
description="Estimating transition matrix",
output_keys_regexp="pred",
)["pred"]
pred = torch.softmax(pred, dim=1)
pred = utils.to_numpy(pred)
c = model.num_classes
T = np.zeros((c, c))
filter_outlier = True
# find a 'perfect example' for each class
for i in range(c):
if not filter_outlier:
idx_best = np.argmax(pred[:, i])
else:
thresh = np.percentile(pred[:, i], 97, interpolation="higher")
robust_eta = pred[:, i]
robust_eta[robust_eta >= thresh] = 0.0
idx_best = np.argmax(robust_eta)
for j in range(c):
T[i, j] = pred[idx_best, j]
# row normalize
row_sums = T.sum(axis=1, keepdims=True)
T /= row_sums
T = torch.tensor(T, dtype=torch.float).to(device)
print(T)
return T
def manifold_plot(model,
example_shape,
low=-1.0,
high=+1.0,
n_points=20,
d1=0,
d2=1,
plt=None):
"""Plots reconstruction for varying dimensions d1 and d2, while the remaining dimensions are kept fixed."""
model.eval()
if plt is None:
plt = pyplot
image = np.zeros((example_shape[0], n_points * example_shape[1],
n_points * example_shape[2]),
dtype=np.float32)
z = np.random.uniform(low=low, high=high, size=(model.hidden_shape[-1], ))
z1_grid = np.linspace(low, high, n_points)
z2_grid = np.linspace(low, high, n_points)
for i, z1 in enumerate(z1_grid):
for j, z2 in enumerate(z2_grid):
cur_z = np.copy(z)
z[d1] = z1
z[d2] = z2
cur_z = cur_z.reshape((1, -1))
x = utils.decode(model, cur_z).reshape(example_shape)
x = utils.to_numpy(x)
image[:, example_shape[1] * i:example_shape[1] * (i + 1),
example_shape[2] * j:example_shape[2] * (j + 1)] = x
fig, ax = plt.subplots(1, figsize=(10, 10))
if image.shape[0] == 1:
ax.imshow(image[0], vmin=0, vmax=1, cmap='gray')
else:
image = image.transpose((1, 2, 0))
image = (255 * image).astype(np.uint8)
ax.imshow(image)
ax.axis('off')
ax.set_ylabel('z_{}'.format(d1))
ax.set_xlabel('z_{}'.format(d2))
return fig, plt
def on_iteration_end(self, info, batch_labels, partition, **kwargs):
self._current_iteration[partition] += 1
pred = utils.to_numpy(info["pred"]).argmax(axis=1).astype(np.int)
batch_labels = utils.to_numpy(batch_labels[0]).astype(np.int)
self._accuracy[partition].append(
(pred == batch_labels).astype(np.float).mean())
def run_partition(model, epoch, tensorboard, optimizer, loader, partition,
training, metrics):
if hasattr(model, "on_epoch_start"):
model.on_epoch_start(epoch=epoch,
tensorboard=tensorboard,
partition=partition,
loader=loader)
for metric in metrics:
metric.on_epoch_start(epoch=epoch, partition=partition)
losses = defaultdict(list)
total_number_samples = 0
for (batch_data,
batch_labels) in tqdm(loader, desc="{} batches".format(partition)):
# make the input and labels lists
if isinstance(batch_data, torch.Tensor):
batch_data = [batch_data]
if isinstance(batch_labels, torch.Tensor):
batch_labels = [batch_labels]
# zero gradients in training phase
if training:
optimizer.zero_grad()
# forward pass
batch_losses, info = model.compute_loss(inputs=batch_data,
labels=batch_labels,
grad_enabled=training)
batch_total_loss = sum([loss for name, loss in batch_losses.items()])
if training:
# backward pass
batch_total_loss.backward()
# some models might need to do something before applying gradients
if hasattr(model, "before_weight_update"):
model.before_weight_update()
# update the parameters
optimizer.step()
# call methods on_iteration_end if it is present
# the network can compute some metrics here
if hasattr(model, "on_iteration_end"):
model.on_iteration_end(
info=info,
batch_losses=batch_losses,
batch_labels=batch_labels,
partition=partition,
tensorboard=tensorboard,
)
for metric in metrics:
metric.on_iteration_end(info=info,
batch_labels=batch_labels,
partition=partition)
# collect all losses
if len(batch_losses) > 1:
batch_losses["total"] = batch_total_loss
for k, v in batch_losses.items():
losses["{}_{}".format(partition, k)].append(
len(batch_data) * utils.to_numpy(v))
total_number_samples += len(batch_data)
for k, v in losses.items():
losses[k] = np.sum(v) / total_number_samples
tensorboard.add_scalar("losses/{}".format(k), losses[k], epoch)
if hasattr(model, "on_epoch_end"):
model.on_epoch_end(epoch=epoch,
tensorboard=tensorboard,
partition=partition,
loader=loader)
for metric in metrics:
metric.on_epoch_end(epoch=epoch,
tensorboard=tensorboard,
partition=partition)
return losses
def latent_scatter(model, data_loader, d1=0, d2=1, plt=None):
"""A scatter plot of latent factors on some 2-d subspace, with points colored according to test labels."""
model.eval()
if plt is None:
plt = matplotlib.pyplot
tab = [
"tab:blue",
"tab:orange",
"tab:green",
"tab:red",
"tab:purple",
"tab:brown",
"tab:pink",
"tab:gray",
"tab:olive",
"tab:cyan",
]
z = []
labels = []
for batch_data, batch_labels in data_loader:
z_batch = model(inputs=[batch_data])["z"]
z.append(utils.to_numpy(z_batch))
labels.append(utils.to_numpy(batch_labels))
z = np.concatenate(z, axis=0)
labels = np.concatenate(labels, axis=0)
fig, ax = plt.subplots(1)
legend = []
# if labels are not vectors, take the first coordiante
if len(labels.shape) > 1:
assert len(labels.shape) == 2
labels = labels[:, 0]
# replace labels with integers
possible_labels = list(np.unique(labels))
labels = [possible_labels.index(label) for label in labels]
# select up to 10000 points
cnt = min(10000, len(labels))
indices = np.random.choice(len(labels), cnt)
z = z[indices]
labels = [labels[idx] for idx in indices]
for i in np.unique(labels):
indices = labels == i
ax.scatter(
z[indices, d1],
z[indices, d2],
marker=".",
color=tab[i],
alpha=0.5,
edgecolor="",
label=i,
)
legend.append(str(i))
fig.legend(legend)
ax.set_xlabel("$Z_{}$".format(d1))
ax.set_ylabel("$Z_{}$".format(d2))
L = np.percentile(z, q=5, axis=0)
R = np.percentile(z, q=95, axis=0)
ax.set_xlim(L[d1], R[d1])
ax.set_ylim(L[d2], R[d2])
ax.set_title("Latent space")
return fig, plt
