def inter_cosine(embedding1_tuple,
embedding2_tuple,
name,
path,
plt,
vmin=None,
vmax=None,
show_value=False):
embedding1 = to_normalized(detach(embedding1_tuple[1]))
embedding2 = to_normalized(detach(embedding2_tuple[1]))
cos_matrix = matmul(embedding1, embedding2.transpose(1, 0))
matshow(cos_matrix, name, path, plt, embedding1_tuple[0],
embedding2_tuple[0], vmin, vmax, show_value)
def PCA_hierarchy(embeddings, concepts, test_concepts, tree,
filename, sub_folder, path, plt):
pca = PCA(n_components=2)
embeddings = detach(embeddings)
with_zero = np.concatenate([
embeddings,
np.zeros_like(embeddings[0])[None]
], axis=0)
result = pca.fit_transform(with_zero)
result_dict = dict(zip(list(concepts) + ['root'], result))
fig, ax = plt.subplots()
def trace_down(root, forest):
root_xy = result_dict[root]
ax.scatter(*root_xy, c='r' if root in test_concepts else 'b')
ax.annotate(root, xy=tuple(root_xy))
if len(forest) != 0:
for kid, sub_forest in forest.items():
kid_xy = result_dict[kid]
ax.arrow(*root_xy, *(kid_xy - root_xy),
head_width=0)
if isinstance(sub_forest, dict):
trace_down(kid, sub_forest)
trace_down('root', tree)
savefig(fig, filename, sub_folder, path, plt)
plt.close('all')
def forward(self, data):
batch_size = data['batch_size']
assert not any(qtype == 'classifiction' for qtype in data['type']),\
'BERT model can not handle classification tasks'
questions_encoded = [
self.tokenizer.encode(question) for question in data['question']
]
max_length = max(len(encoded) for encoded in questions_encoded)
questions_encoded = torch.stack([
torch.tensor(self.pad_to_length(encoded, max_length),
device=self.device) for encoded in questions_encoded
])
answer_encoded = torch.stack(data['answer_encoded']).to(self.device)
batch_size = data['batch_size']
bert_attention, last_state = self.bert(questions_encoded)
logits = self.mlp(last_state)
logs = F.log_softmax(logits, dim=1)
outputs = [
dict(zip(self.tools.answers, detach(one_log.exp())))
for one_log in logs
]
losses = F.nll_loss(logs, answer_encoded, reduction='none')
debugs = [{} for i in range(batch_size)]
return losses, outputs, debugs
def tsne_hierarchy(embeddings, concepts, test_concepts, tree,
filename, sub_folder, path, plt):
tsne = TSNE(n_components=2, perplexity=30,
n_iter=1000000000, learning_rate=200)
embeddings = np.stack(detach(embeddings))
result = tsne.fit_transform(embeddings)
result_dict = dict(zip(concepts, result))
fig, ax = plt.subplots()
def trace_down(root, forest):
root_xy = result_dict[root]
ax.scatter(*root_xy, c='r' if root in test_concepts else 'b')
ax.annotate(root, xy=tuple(root_xy))
if len(forest) != 0:
for kid, sub_forest in forest.items():
kid_xy = result_dict[kid]
ax.arrow(*root_xy, *(kid_xy - root_xy),
head_width=0)
if isinstance(sub_forest, dict):
trace_down(kid, sub_forest)
trace_down(list(tree.keys())[0], list(tree.values())[0])
savefig(fig, filename, sub_folder, path, plt)
plt.close('all')
def calculate_logits(self, objects, program_indexes):
if objects is None:
return None, None
# filter out concept-type arguments
is_concepts = [
i for i, index in enumerate(program_indexes)
if self.tools.arguments_in_concepts[index] != -1
]
length = program_indexes.shape[0]
concept_program_indexes = \
self.tools.arguments_in_concepts[
detach(program_indexes[is_concepts])
]
concept_program_indexes = torch.LongTensor(concept_program_indexes)\
.to(self.device)
# calculating the raw similarity logits
concepts_used = self.concept_embedding(
concept_program_indexes)
logits = self.logit_fn(objects, concepts_used)
logits = self.align(logits, is_concepts, length)
return logits
def bar_graph(values, labels, name, path, plt):
values = detach(values)
fig, ax = plt.subplots()
y_pos = np.arange(len(values))
ax.bar(y_pos, values, align='center', alpha=0.5)
ax.set_xticks(y_pos)
ax.set_xticklabels(labels)
savefig(fig, name, path, plt)
plt.close('all')
def check_PrecRec(coach, test_concepts, images):
model = coach.model
concepts_in_argument = torch.tensor(
[info.tools.arguments[concept] for concept in test_concepts])
num = len(test_concepts)
total = np.zeros(num)
true_positive = np.zeros(num)
positive = np.zeros(num)
with torch.no_grad():
for image_id in tqdm(images, leave=False):
scene = info.visual_dataset[image_id]
data = {
'object_length': torch.tensor([scene['object_length']]),
'image': torch.tensor(scene['image'])[None],
'objects': torch.tensor(scene['objects']),
'batch_size': 1,
}
_, recognized = model.resnet_model(data)
feature = model.feature_mlp(recognized[0][1])
if args.model == 'h_embedding_v1':
conditional_logits, _ = \
model.embedding.calculate_logits(
feature,
concepts_in_argument,
model.scale,
model.inf
)
logits = torch.stack(conditional_logits)
else:
logits = model.embedding.calculate_logits(
feature,
concepts_in_argument,
)
logits = torch.stack(logits)
recog = detach(torch.sigmoid(logits))
gt = info.visual_dataset.get_classification(
info.visual_dataset.sceneGraphs[image_id],
test_concepts).astype(int).transpose(1, 0)
total += gt.sum(1)
true_positive += (gt * recog).sum(1)
positive += recog.sum(1)
output = {
concept: {
'precision': true_positive[i] / positive[i],
'recall': true_positive[i] / total[i],
}
for i, concept in enumerate(test_concepts)
}
return output
def tsne_embeddings(names, embeddings, filename, path, plt, with_origin):
tsne = TSNE(n_components=2, perplexity=10, n_iter=1000000)
embeddings = detach(embeddings)
if with_origin:
names, embeddings = concatenate_origin(names, embeddings)
result = tsne.fit_transform(embeddings)
fig, ax = plt.subplots()
ax.scatter(result[:, 0], result[:, 1])
for i, word in enumerate(names):
ax.annotate(word, xy=tuple(result[i]))
savefig(fig, filename, path, plt)
plt.close('all')
def pca_embeddings(names, embeddings, filename, path, plt, with_origin):
pca = PCA(n_components=2)
embeddings = detach(embeddings)
if with_origin:
names, embeddings = concatenate_origin(names, embeddings)
result = pca.fit_transform(embeddings)
fig, ax = plt.subplots()
ax.scatter(result[:, 0], result[:, 1])
for i, word in enumerate(names):
ax.annotate(word, xy=tuple(result[i]))
savefig(fig, filename, path, plt)
plt.close('all')
def forward(self, data):
# pre-processing
batch_size = data['batch_size']
data['image'] = data['image'].to(self.device)
data['question'] = data['question_encoded'].long().to(self.device)
questions = data['question']
answers = torch.stack(data['answer_encoded']).to(self.device)
is_conceptual = [
data['category'][i] == 'conceptual' for i in range(batch_size)
]
is_conceptual = torch.tensor(is_conceptual,
device=self.device,
dtype=torch.float)
# running language phase
question_length = (questions > 0).int().sum(1)
question_states, last_state = \
self.encode_sentence(questions, question_length)
last_state = last_state.view(batch_size, -1)
# visual phase
if self.use_vision:
features = self.resnet(data['image'])
features = features.mean(dim=-1).mean(dim=-1)
features = features * (1 - is_conceptual).unsqueeze(-1)
logits = self.mlp(torch.cat([features, last_state], dim=-1))
else:
logits = self.mlp(last_state)
logs = F.log_softmax(logits, dim=1)
outputs = [
dict(zip(self.tools.answers, detach(one_log.exp())))
for one_log in logs
]
# calculating losses
losses = self.loss_fn(logs, answers, reduction='none')
if self.use_lm:
lm_losses = self.forward_lm(questions, question_length,
question_states)
losses = losses + lm_losses.sum(1)
return losses, outputs, None
def pr_curve(coach, test_concepts, images, plt):
all_logits = {concept: [] for concept in test_concepts}
model = coach.model
concepts_in_argument = torch.tensor(
[info.tools.arguments[concept] for concept in test_concepts])
for image_id in tqdm(images, leave=False):
scene = info.visual_dataset[image_id]
data = {
'object_length': torch.tensor([scene['object_length']]),
'image': torch.tensor(scene['image'])[None],
'objects': torch.tensor(scene['objects']),
'batch_size': 1,
}
_, recognized = model.resnet_model(data)
feature = model.feature_mlp(recognized[0][1])
if args.model == 'h_embedding_v1':
conditional_logits, _ = \
model.embedding.calculate_logits(
feature,
concepts_in_argument,
model.scale,
model.inf
)
logits = torch.stack(conditional_logits)
else:
logits = model.embedding.calculate_logits(
feature,
concepts_in_argument,
)
logits = torch.stack(logits)
recog = detach(torch.sigmoid(logits))
gt = info.visual_dataset.get_classification(
info.visual_dataset.sceneGraphs[image_id],
test_concepts).astype(int).transpose(1, 0)
for i, concept in enumerate(test_concepts):
all_logits[concept] += list(zip(recog[i], gt[i]))
pr_data = get_pr(all_logits)
return pr_data, all_logits
def matshow(matrix,
name,
path,
plt,
ylabel=None,
xlabel=None,
vmin=None,
vmax=None,
show_value=False):
if isinstance(matrix, torch.Tensor):
matrix = detach(matrix)
fig, ax = plt.subplots()
im = ax.matshow(matrix, vmin=vmin, vmax=vmax)
if xlabel is not None:
ax.set_xticks(np.arange(len(xlabel)))
ax.set_yticks(np.arange(len(ylabel)))
ax.set_xticklabels(xlabel)
ax.set_yticklabels(ylabel)
plt.setp(ax.get_xticklabels(),
rotation=45,
ha="left",
rotation_mode="anchor")
if show_value:
for (i, j), v in np.ndenumerate(matrix):
ax.text(j,
i,
'{:0.1f}'.format(v),
fontsize=6,
ha='center',
va='center')
fig.colorbar(im)
savefig(fig, name, path, plt)
plt.clf()
def max_len(self):
output = detach(
self.embedding.concept_embedding.weight.pow(2).sum(
-1).max().sqrt())
return output
