def t2i2t(images, captions):
image_caption_distances = pairwise_distances(images, captions)
topk_idx = torch.topk(image_caption_distances, 10 , 1, largest=False)[1]
ranks = []
for i, row in enumerate(topk_idx):
rank = np.where(row.cpu().numpy() == i)
ranks.append(rank)
ranks = np.array(ranks)
r1 = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
r5 = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
r10 = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
image_caption_distances = image_caption_distances.t()
topk_idx = torch.topk(image_caption_distances, 10 , 1, largest=False)[1]
ranks = []
for i, row in enumerate(topk_idx):
rank = np.where(row.cpu().numpy() == i)
ranks.append(rank)
ranks = np.array(ranks)
r1i = 100.0 * len(np.where(ranks < 1)[0]) / len(ranks)
r5i = 100.0 * len(np.where(ranks < 5)[0]) / len(ranks)
r10i = 100.0 * len(np.where(ranks < 10)[0]) / len(ranks)
# print((r1, r5, r10, r1i, r5i, r10i))
return (r1, r5, r10, r1i, r5i, r10i)
def forward(self, embeddings, targets):
cuda = embeddings.is_cuda
if cuda:
embeddings = embeddings.cpu()
targets = targets.cpu()
distance_matrix = pairwise_distances(embeddings)
triplets = []
for target in set(targets):
target_mask = (targets == target)
positive_indices = np.where(target_mask)[0]
negative_indices = np.where(np.logical_not(target_mask))[0]
for i in positive_indices:
hard_positive = positive_indices[np.argmax(
distance_matrix[i, j] for j in positive_indices)]
hard_negative = negative_indices[np.argmin(
distance_matrix[i, j] for j in negative_indices)]
triplets.append([i, hard_positive, hard_negative])
triplets = torch.LongTensor(np.array(triplets))
if cuda:
embeddings = embeddings.cuda()
triplets = triplets.cuda()
ap_distances = (embeddings[triplets[:, 0]] -
embeddings[triplets[:, 1]]).pow(2).sum(1)
an_distances = (embeddings[triplets[:, 0]] -
embeddings[triplets[:, 2]]).pow(2).sum(1)
losses = F.relu(ap_distances - an_distances + self.margin)
return losses.mean()
def unitary_vectors():
# get set of random unitary vectors
x1 = np.random.random((100, 32))
x1 = [x / np.linalg.norm(x) for x in x1]
# get origin vector
x2 = np.zeros((1, 32))
# compute pairwise distances
x1 = np.array(x1)
x2 = np.array(x2)
D1 = utils.pairwise_distances(x1, x2)
D2 = _naive_pairwise_distances(x1, x2)
# check shape
if D1.shape != D2.shape:
return False
# compare naive approach with employed
if not _close(D1, D2):
return False
# check if it is non-negative
if not _close(D1, np.abs(D1)):
return False
# check if distance is close to 1
for row in D1:
for d in row:
if not _close(d, 1):
return False
return True
def same_vectors():
# single set of random vectors
x1 = np.random.random((100, 32))
# compute pairwise distances
D1 = utils.pairwise_distances(x1, x1)
D2 = _naive_pairwise_distances(x1, x1)
# check shape
if D1.shape != D2.shape:
return False
# compare naive approach with employed
if not _close(D1, D2):
return False
# check if it is non-negative
if not _close(D1, np.abs(D1)):
return False
# check if main diagonal is zero
for d in np.diag(D1):
if not _close(d, 0):
return False
# check if it is symmetrical
for i, row in enumerate(D1):
for j, d in enumerate(row):
if not _close(d, D1[j, i]):
return False
return True
def orthogonal_vectors():
# get set of random vectors
x1 = np.random.random((100, 32))
norm_x1 = [x / np.linalg.norm(x) for x in x1]
# get set of vectors orthogonal to x1
x2 = np.random.random((100, 32))
x2 = [x - np.dot(x, y) * y for x, y in zip(x2, norm_x1)]
# compute pairwise distances
x1 = np.array(x1)
x2 = np.array(x2)
D1 = utils.pairwise_distances(x1, x2)
D2 = _naive_pairwise_distances(x1, x2)
# check shape
if D1.shape != D2.shape:
return False
# compare naive approach with employed
if not _close(D1, D2):
return False
# check if it is non-negative
if not _close(D1, np.abs(D1)):
return False
# check if distance is close to sum of squares of magnitudes
for i, x in enumerate(x1):
y = x2[i]
dist = np.sum(x**2) + np.sum(y**2)
if not _close(dist, D1[i, i]):
return False
return True
def query(self, index):
current_state = data["all_predictions"][index].view(1, -1)
all_states = data["all_predictions"]
current_all_dist = pairwise_distances(current_state, all_states)
similar_indices = torch.topk(current_all_dist,
opt.selection_radius,
1,
largest=False)[1]
similar_indices = similar_indices.data[0].cpu().numpy()
for idx in similar_indices:
self.add_index(idx)
return similar_indices
def eval(self):
criterion = nn.CrossEntropyLoss()
self.model.eval()
episodic_acc = []
loss = []
with torch.no_grad():
for b_idx, (data, target) in enumerate(self.val_loader):
data = data.to(self.device)
support_input = data[:self.N_way_val * self.N_shot_val,:,:,:]
query_input = data[self.N_way_val * self.N_shot_val:,:,:,:]
label_encoder = {target[i * self.N_shot_val] : i for i in range(self.N_way_val)}
query_label = torch.cuda.LongTensor([label_encoder[class_name] for class_name in target[self.N_way_val * self.N_shot_val:]])
support = self.model(support_input)
queries = self.model(query_input)
prototypes = support.reshape(self.N_way_val, self.N_shot_val, -1).mean(dim=1)
if self.matching_fn == 'parametric':
distances = pairwise_distances(queries, prototypes, self.matching_fn, self.parametric)
else:
distances = pairwise_distances(queries, prototypes, self.matching_fn)
loss.append(criterion(-distances, query_label).item())
y_pred = (-distances).softmax(dim=1).max(1, keepdim=True)[1]
episodic_acc.append(1. * y_pred.eq(query_label.view_as(y_pred)).sum().item() / len(query_label))
loss = np.array(loss)
episodic_acc = np.array(episodic_acc)
loss = loss.mean()
mean = episodic_acc.mean()
std = episodic_acc.std()
print('\nLoss: {:.6f}\tAccuracy: {:.2f} +- {:.2f} %\n'.format(loss,mean * 100, 1.96 * std / (600)**(1/2) * 100))
return loss, mean, std
def query(self, index):
# current_vector = torch.Tensor(data["train_deleted"][0][index]).view(1, -1)
# all_vectors = torch.Tensor(data["train_deleted"][0])
# current_vector = all_vectors[index].view(1, -1)
# if opt.cuda:
# all_vectors, current_vector = all_vectors.cuda(), current_vector.cuda()
current_state = data["all_states"][index].view(1, -1)
all_states = data["all_states"]
current_all_dist = pairwise_distances(current_state, all_states)
similar_indices = torch.topk(current_all_dist, opt.selection_radius * 5, 1, largest=False)[1]
similar_indices = similar_indices.data[0].cpu().numpy()
for idx in similar_indices:
self.add_index(idx)
return similar_indices
def random_vectors():
# random vectors
x1 = np.random.random((100, 32))
x2 = np.random.random((100, 32))
# compute pairwise distances
D1 = utils.pairwise_distances(x1, x2)
D2 = _naive_pairwise_distances(x1, x2)
# check shape
if D1.shape != D2.shape:
return False
# compare naive approach with employed
if not _close(D1, D2):
return False
# check if it is non-negative
if not _close(D1, np.abs(D1)):
return False
return True
def train(self):
criterion = nn.CrossEntropyLoss()
best_mean = 0
iteration = 0
self.sample_idx_val = []
self.noise_val = []
for i in range(self.episodes_per_epoch):
self.sample_idx_val.append(
torch.tensor([
torch.randint(self.N_shot_val * i,
self.N_shot_val * (i + 1),
(self.M_aug_val, )).numpy()
for i in range(self.N_way_val)
]).reshape(-1))
self.noise_val.append(
torch.randn((self.N_way_val * self.M_aug_val, self.nz),
device=self.device))
if self.resume_iter:
print("resuming step %d ..." % self.resume_iter)
iteration = self.resume_iter
self.load_checkpoint(self.resume_iter)
loss, mean, std = self.eval()
if mean > best_mean:
best_mean = mean
episodic_acc = []
for ep in range(self.num_epochs):
self.cnn.train()
self.g.train()
self.mlp.train()
for batch_idx, (data, target) in enumerate(self.train_loader):
data = data.to(self.device)
self.optimizer.zero_grad()
support_input = data[:self.N_way_train *
self.N_shot_train, :, :, :]
query_input = data[self.N_way_train *
self.N_shot_train:, :, :, :]
label_encoder = {
target[i * self.N_shot_train]: i
for i in range(self.N_way_train)
}
query_label = torch.cuda.LongTensor([
label_encoder[class_name]
for class_name in target[self.N_way_train *
self.N_shot_train:]
])
support = self.cnn(support_input)
queries = self.cnn(query_input)
sample_idx = torch.tensor([
torch.randint(self.N_shot_train * i,
self.N_shot_train * (i + 1),
(self.M_aug_train, )).numpy()
for i in range(self.N_way_train)
]).reshape(-1)
sample = support[sample_idx]
noise = torch.randn(
(self.N_way_train * self.M_aug_train, self.nz),
device=self.device)
support_g = self.g(sample,
noise).reshape(self.N_way_train,
self.M_aug_train, -1)
support = support.reshape(self.N_way_train, self.N_shot_train,
-1)
support_aug = torch.cat([support, support_g], dim=1)
support_aug = support_aug.reshape(
self.N_way_train * (self.N_shot_train + self.M_aug_train),
-1)
prototypes = self.mlp(support_aug)
prototypes = prototypes.reshape(
self.N_way_train, self.N_shot_train + self.M_aug_train,
-1).mean(dim=1)
queries = self.mlp(queries)
if self.matching_fn == 'parametric':
distances = pairwise_distances(queries, prototypes,
self.matching_fn,
self.parametric)
else:
distances = pairwise_distances(queries, prototypes,
self.matching_fn)
loss = criterion(-distances, query_label)
loss.backward()
self.optimizer.step()
y_pred = (-distances).softmax(dim=1).max(1, keepdim=True)[1]
episodic_acc.append(
1. * y_pred.eq(query_label.view_as(y_pred)).sum().item() /
len(query_label))
if (iteration + 1) % self.log_interval == 0:
episodic_acc = np.array(episodic_acc)
mean = episodic_acc.mean()
std = episodic_acc.std()
print(
'Epoch: {:3d} [{:d}/{:d}]\tIteration: {:5d}\tLoss: {:.6f}\tAccuracy: {:.2f} +- {:.2f} %'
.format(
ep, (batch_idx + 1), len(self.train_loader),
iteration + 1, loss.item(), mean * 100,
1.96 * std / (self.log_interval)**(1 / 2) * 100))
if self.use_wandb:
import wandb
wandb.log(
{
"loss":
loss.item(),
"acc_mean":
mean * 100,
"acc_ci":
1.96 * std /
(self.log_interval)**(1 / 2) * 100,
'lr':
self.optimizer.param_groups[0]['lr']
},
step=iteration + 1)
episodic_acc = []
if (iteration + 1) % self.ckp_interval == 0:
loss, mean, std = self.eval()
if mean > best_mean:
best_mean = mean
self.save_checkpoint(iteration)
if self.use_wandb:
wandb.run.summary[
"best_accuracy"] = best_mean * 100
if self.use_wandb:
import wandb
wandb.log(
{
"val_loss": loss,
"val_acc_mean": mean * 100,
"val_acc_ci": 1.96 * std / (600)**(1 / 2) * 100
},
step=iteration + 1,
commit=False)
iteration += 1
self.scheduler.step()
self.save_checkpoint(iteration)
def train(self):
task_criterion = nn.CrossEntropyLoss()
adversarial_criterion = nn.BCELoss()
best_mean = 0
iteration = 0
real_label = 1.
fake_label = 0.
self.sample_idx_val = []
self.noise_val = []
for i in range(self.episodes_per_epoch):
self.sample_idx_val.append(
torch.tensor([
torch.randint(self.N_shot_val * i,
self.N_shot_val * (i + 1),
(self.M_aug_val, )).numpy()
for i in range(self.N_way_val)
]).reshape(-1))
self.noise_val.append(
torch.randn((self.N_way_val * self.M_aug_val, self.nz),
device=self.device))
if self.resume_iter:
print("resuming step %d ..." % self.resume_iter)
iteration = self.resume_iter
self.load_checkpoint(self.resume_iter)
loss, mean, std = self.eval()
if mean > best_mean:
best_mean = mean
episodic_acc = []
for ep in range(self.num_epochs):
self.cnn.train()
self.g.train()
self.mlp.train()
self.d.train()
for batch_idx, (data, target) in enumerate(self.train_loader):
data = data.to(self.device)
support_input = data[:self.N_way_train *
self.N_shot_train, :, :, :]
query_input = data[self.N_way_train *
self.N_shot_train:, :, :, :]
label_encoder = {
target[i * self.N_shot_train]: i
for i in range(self.N_way_train)
}
query_label = torch.cuda.LongTensor([
label_encoder[class_name]
for class_name in target[self.N_way_train *
self.N_shot_train:]
])
real_label = torch.full((self.N_way_train * self.N_shot_val, ),
1.,
dtype=torch.float,
device=self.device)
real_label_g = torch.full(
(self.N_way_train * self.M_aug_train, ),
1.,
dtype=torch.float,
device=self.device)
fake_label_g = torch.full(
(self.N_way_train * self.M_aug_train, ),
0.,
dtype=torch.float,
device=self.device)
################
# update D #
################
support = self.cnn(support_input)
queries = self.cnn(query_input)
sample_idx = torch.tensor([
torch.randint(self.N_shot_train * i,
self.N_shot_train * (i + 1),
(self.M_aug_train, )).numpy()
for i in range(self.N_way_train)
]).reshape(-1)
noise = torch.randn(
(self.N_way_train * self.M_aug_train, self.nz),
device=self.device)
sample = support[sample_idx]
support_g = self.g(sample, noise)
if ep >= self.active_adversarial_loss_step:
for _ in range(self.num_d_steps):
self.optimizer_d.zero_grad()
self.optimizer.zero_grad()
d_loss_adv_fake = adversarial_criterion(
self.d(support_g.detach()).view(-1), fake_label_g)
d_loss_adv_real = adversarial_criterion(
self.d(support.detach()).view(-1), real_label)
d_loss = self.alpha_weight * (d_loss_adv_fake +
d_loss_adv_real)
d_loss.backward()
self.optimizer_d.step()
else:
d_loss_adv_fake = torch.tensor(0).cuda()
d_loss_adv_real = torch.tensor(0).cuda()
d_loss = torch.tensor(0).cuda()
d_loss_task = torch.tensor(0).cuda()
################
# update H #
################
self.optimizer_d.zero_grad()
self.optimizer.zero_grad()
if ep >= self.active_adversarial_loss_step:
h_loss_adv = adversarial_criterion(
self.d(support_g).view(-1), real_label_g)
else:
h_loss_adv = torch.tensor(0).cuda()
support_g_r = support_g.reshape(self.N_way_train,
self.M_aug_train, -1)
support_r = support.reshape(self.N_way_train,
self.N_shot_train, -1)
support_aug = torch.cat([support_r, support_g_r], dim=1)
support_aug = support_aug.reshape(
self.N_way_train * (self.N_shot_train + self.M_aug_train),
-1)
prototypes = self.mlp(support_aug)
prototypes = prototypes.reshape(
self.N_way_train, self.N_shot_train + self.M_aug_train,
-1).mean(dim=1)
queries = self.mlp(queries)
if self.matching_fn == 'parametric':
distances = pairwise_distances(queries, prototypes,
self.matching_fn,
self.parametric)
else:
distances = pairwise_distances(queries, prototypes,
self.matching_fn)
h_loss_task = task_criterion(-distances, query_label)
h_loss = self.alpha_weight * h_loss_adv + h_loss_task
h_loss.backward()
self.optimizer.step()
y_pred = (-distances).softmax(dim=1).max(1, keepdim=True)[1]
episodic_acc.append(
1. * y_pred.eq(query_label.view_as(y_pred)).sum().item() /
len(query_label))
if (iteration + 1) % self.log_interval == 0:
episodic_acc = np.array(episodic_acc)
mean = episodic_acc.mean()
std = episodic_acc.std()
episodic_acc = []
print(
'Epoch: {:3d} [{:d}/{:d}] Iteration: {:5d} h_loss: {:.4f} h_loss_adv: {:.4f} h_loss_task: {:.4f} d_loss: {:.4f} d_loss_adv_fake: {:.4f} d_loss_adv_real: {:.4f} Accuracy: {:.2f} +- {:.2f} %'
.format(
ep, (batch_idx + 1), len(self.train_loader),
iteration + 1, h_loss.item(), h_loss_adv.item(),
h_loss_task.item(), d_loss.item(),
d_loss_adv_fake.item(), d_loss_adv_real.item(),
mean * 100,
1.96 * std / (self.log_interval)**(1 / 2) * 100))
if self.use_wandb:
import wandb
wandb.log(
{
'h_loss':
h_loss.item(),
'h_loss_adv':
h_loss_adv.item(),
'h_loss_task':
h_loss_task.item(),
'd_loss':
d_loss.item(),
'd_loss_adv_fake':
d_loss_adv_fake.item(),
'd_loss_adv_real':
d_loss_adv_real.item(),
"acc_mean":
mean * 100,
"acc_ci":
1.96 * std /
(self.log_interval)**(1 / 2) * 100,
'lr':
self.optimizer.param_groups[0]['lr']
},
step=iteration + 1)
if (iteration + 1) % self.ckp_interval == 0:
loss, mean, std = self.eval()
if mean > best_mean:
best_mean = mean
self.save_checkpoint(iteration)
if self.use_wandb:
wandb.run.summary[
"best_accuracy"] = best_mean * 100
if self.use_wandb:
import wandb
wandb.log(
{
"val_loss": loss,
"val_acc_mean": mean * 100,
"val_acc_ci": 1.96 * std / (600)**(1 / 2) * 100
},
step=iteration + 1,
commit=False)
iteration += 1
self.scheduler.step()
self.scheduler_d.step()