def inter_cosine(embedding1_tuple, embedding2_tuple, name, path, plt, vmin=None, vmax=None, show_value=False): embedding1 = to_normalized(detach(embedding1_tuple[1])) embedding2 = to_normalized(detach(embedding2_tuple[1])) cos_matrix = matmul(embedding1, embedding2.transpose(1, 0)) matshow(cos_matrix, name, path, plt, embedding1_tuple[0], embedding2_tuple[0], vmin, vmax, show_value)
def PCA_hierarchy(embeddings, concepts, test_concepts, tree, filename, sub_folder, path, plt): pca = PCA(n_components=2) embeddings = detach(embeddings) with_zero = np.concatenate([ embeddings, np.zeros_like(embeddings[0])[None] ], axis=0) result = pca.fit_transform(with_zero) result_dict = dict(zip(list(concepts) + ['root'], result)) fig, ax = plt.subplots() def trace_down(root, forest): root_xy = result_dict[root] ax.scatter(*root_xy, c='r' if root in test_concepts else 'b') ax.annotate(root, xy=tuple(root_xy)) if len(forest) != 0: for kid, sub_forest in forest.items(): kid_xy = result_dict[kid] ax.arrow(*root_xy, *(kid_xy - root_xy), head_width=0) if isinstance(sub_forest, dict): trace_down(kid, sub_forest) trace_down('root', tree) savefig(fig, filename, sub_folder, path, plt) plt.close('all')
def forward(self, data): batch_size = data['batch_size'] assert not any(qtype == 'classifiction' for qtype in data['type']),\ 'BERT model can not handle classification tasks' questions_encoded = [ self.tokenizer.encode(question) for question in data['question'] ] max_length = max(len(encoded) for encoded in questions_encoded) questions_encoded = torch.stack([ torch.tensor(self.pad_to_length(encoded, max_length), device=self.device) for encoded in questions_encoded ]) answer_encoded = torch.stack(data['answer_encoded']).to(self.device) batch_size = data['batch_size'] bert_attention, last_state = self.bert(questions_encoded) logits = self.mlp(last_state) logs = F.log_softmax(logits, dim=1) outputs = [ dict(zip(self.tools.answers, detach(one_log.exp()))) for one_log in logs ] losses = F.nll_loss(logs, answer_encoded, reduction='none') debugs = [{} for i in range(batch_size)] return losses, outputs, debugs
def tsne_hierarchy(embeddings, concepts, test_concepts, tree, filename, sub_folder, path, plt): tsne = TSNE(n_components=2, perplexity=30, n_iter=1000000000, learning_rate=200) embeddings = np.stack(detach(embeddings)) result = tsne.fit_transform(embeddings) result_dict = dict(zip(concepts, result)) fig, ax = plt.subplots() def trace_down(root, forest): root_xy = result_dict[root] ax.scatter(*root_xy, c='r' if root in test_concepts else 'b') ax.annotate(root, xy=tuple(root_xy)) if len(forest) != 0: for kid, sub_forest in forest.items(): kid_xy = result_dict[kid] ax.arrow(*root_xy, *(kid_xy - root_xy), head_width=0) if isinstance(sub_forest, dict): trace_down(kid, sub_forest) trace_down(list(tree.keys())[0], list(tree.values())[0]) savefig(fig, filename, sub_folder, path, plt) plt.close('all')
def calculate_logits(self, objects, program_indexes): if objects is None: return None, None # filter out concept-type arguments is_concepts = [ i for i, index in enumerate(program_indexes) if self.tools.arguments_in_concepts[index] != -1 ] length = program_indexes.shape[0] concept_program_indexes = \ self.tools.arguments_in_concepts[ detach(program_indexes[is_concepts]) ] concept_program_indexes = torch.LongTensor(concept_program_indexes)\ .to(self.device) # calculating the raw similarity logits concepts_used = self.concept_embedding( concept_program_indexes) logits = self.logit_fn(objects, concepts_used) logits = self.align(logits, is_concepts, length) return logits
def bar_graph(values, labels, name, path, plt): values = detach(values) fig, ax = plt.subplots() y_pos = np.arange(len(values)) ax.bar(y_pos, values, align='center', alpha=0.5) ax.set_xticks(y_pos) ax.set_xticklabels(labels) savefig(fig, name, path, plt) plt.close('all')
def check_PrecRec(coach, test_concepts, images): model = coach.model concepts_in_argument = torch.tensor( [info.tools.arguments[concept] for concept in test_concepts]) num = len(test_concepts) total = np.zeros(num) true_positive = np.zeros(num) positive = np.zeros(num) with torch.no_grad(): for image_id in tqdm(images, leave=False): scene = info.visual_dataset[image_id] data = { 'object_length': torch.tensor([scene['object_length']]), 'image': torch.tensor(scene['image'])[None], 'objects': torch.tensor(scene['objects']), 'batch_size': 1, } _, recognized = model.resnet_model(data) feature = model.feature_mlp(recognized[0][1]) if args.model == 'h_embedding_v1': conditional_logits, _ = \ model.embedding.calculate_logits( feature, concepts_in_argument, model.scale, model.inf ) logits = torch.stack(conditional_logits) else: logits = model.embedding.calculate_logits( feature, concepts_in_argument, ) logits = torch.stack(logits) recog = detach(torch.sigmoid(logits)) gt = info.visual_dataset.get_classification( info.visual_dataset.sceneGraphs[image_id], test_concepts).astype(int).transpose(1, 0) total += gt.sum(1) true_positive += (gt * recog).sum(1) positive += recog.sum(1) output = { concept: { 'precision': true_positive[i] / positive[i], 'recall': true_positive[i] / total[i], } for i, concept in enumerate(test_concepts) } return output
def tsne_embeddings(names, embeddings, filename, path, plt, with_origin): tsne = TSNE(n_components=2, perplexity=10, n_iter=1000000) embeddings = detach(embeddings) if with_origin: names, embeddings = concatenate_origin(names, embeddings) result = tsne.fit_transform(embeddings) fig, ax = plt.subplots() ax.scatter(result[:, 0], result[:, 1]) for i, word in enumerate(names): ax.annotate(word, xy=tuple(result[i])) savefig(fig, filename, path, plt) plt.close('all')
def pca_embeddings(names, embeddings, filename, path, plt, with_origin): pca = PCA(n_components=2) embeddings = detach(embeddings) if with_origin: names, embeddings = concatenate_origin(names, embeddings) result = pca.fit_transform(embeddings) fig, ax = plt.subplots() ax.scatter(result[:, 0], result[:, 1]) for i, word in enumerate(names): ax.annotate(word, xy=tuple(result[i])) savefig(fig, filename, path, plt) plt.close('all')
def forward(self, data): # pre-processing batch_size = data['batch_size'] data['image'] = data['image'].to(self.device) data['question'] = data['question_encoded'].long().to(self.device) questions = data['question'] answers = torch.stack(data['answer_encoded']).to(self.device) is_conceptual = [ data['category'][i] == 'conceptual' for i in range(batch_size) ] is_conceptual = torch.tensor(is_conceptual, device=self.device, dtype=torch.float) # running language phase question_length = (questions > 0).int().sum(1) question_states, last_state = \ self.encode_sentence(questions, question_length) last_state = last_state.view(batch_size, -1) # visual phase if self.use_vision: features = self.resnet(data['image']) features = features.mean(dim=-1).mean(dim=-1) features = features * (1 - is_conceptual).unsqueeze(-1) logits = self.mlp(torch.cat([features, last_state], dim=-1)) else: logits = self.mlp(last_state) logs = F.log_softmax(logits, dim=1) outputs = [ dict(zip(self.tools.answers, detach(one_log.exp()))) for one_log in logs ] # calculating losses losses = self.loss_fn(logs, answers, reduction='none') if self.use_lm: lm_losses = self.forward_lm(questions, question_length, question_states) losses = losses + lm_losses.sum(1) return losses, outputs, None
def pr_curve(coach, test_concepts, images, plt): all_logits = {concept: [] for concept in test_concepts} model = coach.model concepts_in_argument = torch.tensor( [info.tools.arguments[concept] for concept in test_concepts]) for image_id in tqdm(images, leave=False): scene = info.visual_dataset[image_id] data = { 'object_length': torch.tensor([scene['object_length']]), 'image': torch.tensor(scene['image'])[None], 'objects': torch.tensor(scene['objects']), 'batch_size': 1, } _, recognized = model.resnet_model(data) feature = model.feature_mlp(recognized[0][1]) if args.model == 'h_embedding_v1': conditional_logits, _ = \ model.embedding.calculate_logits( feature, concepts_in_argument, model.scale, model.inf ) logits = torch.stack(conditional_logits) else: logits = model.embedding.calculate_logits( feature, concepts_in_argument, ) logits = torch.stack(logits) recog = detach(torch.sigmoid(logits)) gt = info.visual_dataset.get_classification( info.visual_dataset.sceneGraphs[image_id], test_concepts).astype(int).transpose(1, 0) for i, concept in enumerate(test_concepts): all_logits[concept] += list(zip(recog[i], gt[i])) pr_data = get_pr(all_logits) return pr_data, all_logits
def matshow(matrix, name, path, plt, ylabel=None, xlabel=None, vmin=None, vmax=None, show_value=False): if isinstance(matrix, torch.Tensor): matrix = detach(matrix) fig, ax = plt.subplots() im = ax.matshow(matrix, vmin=vmin, vmax=vmax) if xlabel is not None: ax.set_xticks(np.arange(len(xlabel))) ax.set_yticks(np.arange(len(ylabel))) ax.set_xticklabels(xlabel) ax.set_yticklabels(ylabel) plt.setp(ax.get_xticklabels(), rotation=45, ha="left", rotation_mode="anchor") if show_value: for (i, j), v in np.ndenumerate(matrix): ax.text(j, i, '{:0.1f}'.format(v), fontsize=6, ha='center', va='center') fig.colorbar(im) savefig(fig, name, path, plt) plt.clf()
def max_len(self): output = detach( self.embedding.concept_embedding.weight.pow(2).sum( -1).max().sqrt()) return output