def _feature_leaf(num): name = list(converter.num2name(abs(num))) for i in range(len(name)): if name[i] in POS_REL_NAMES_FULL.keys(): name[i] = POS_REL_NAMES_FULL[name[i]] embedding = np.array( [word_vectors[tokenizers['vocab2token'][i]] for i in ' '.join([name[1], name[0], name[2]]).split(' ')]) summed_embedding = np.sum(embedding, axis=0) / 3 return summed_embedding if num > 0 else -summed_embedding
def prediction_to_assignment_embedding(softmax_list, info_list, embedder, tokenizers, pres, objs): """ :param softmax_list: a torch.Tensor :param info_list: [(img_name, (label1, bbox1), (label2, bbox2))] :param embedders: (pe, ce, ae) :return: the embedding of assignment """ def _feature(name): embedding = np.array([ word_vectors[tokenizers['vocab2token'][i]] for i in name.split(' ') ]) summed_embedding = np.sum(embedding, axis=0) return summed_embedding embedded_clauses = [] for idx, info in enumerate(info_list): prop = softmax_list[idx] sub = objs[info_list[idx][1][0]] obj = objs[info_list[idx][2][0]] e_p = 0 for pres_idx in range(0, 70): # e_p += prop[pres_idx] * torch.cuda.FloatTensor(_feature(tokenizers['token2vocab'][pres_idx])) e_p += prop[pres_idx] * torch.cuda.FloatTensor( _feature(pres[pres_idx])) e_p = e_p / 70 e_p = (e_p + torch.cuda.FloatTensor(_feature(sub)) + torch.cuda.FloatTensor(_feature(obj))) / 3 embedded_clauses.append(e_p) for idx, info in enumerate(info_list): pos = box_prop_name(info_list[idx][1][1], info_list[idx][2][1]) if pos in POS_REL_NAMES_FULL.keys(): pos = POS_REL_NAMES_FULL[pos] sub = objs[info_list[idx][1][0]] obj = objs[info_list[idx][2][0]] embedded_clauses.append( (torch.cuda.FloatTensor(_feature(pos)) + torch.cuda.FloatTensor( _feature(sub)) + torch.cuda.FloatTensor(_feature(obj))) / 3) adj0, features0, labels0 = assignment_to_gcn_compatible( embedded_clauses, node_features) embedded_clauses = embedder(features0.cuda().squeeze(0), adj0.cuda().squeeze(0), labels0.cuda()) return embedded_clauses
def tree_prediction_to_assignment_embedding(softmax_list, info_list, embedder, tokenizers, pres, objs): def _feature(name): embedding = np.array([ word_vectors[tokenizers['vocab2token'][i]] for i in name.split(' ') ]) summed_embedding = np.sum(embedding, axis=0) return summed_embedding embedded_clauses = [] for idx, info in enumerate(info_list): prop = softmax_list[idx] sub = objs[info_list[idx][1][0]] obj = objs[info_list[idx][2][0]] e_p = 0 for pres_idx in range(0, 70): # e_p += prop[pres_idx] * torch.cuda.FloatTensor(_feature(tokenizers['token2vocab'][pres_idx])) e_p += prop[pres_idx] * torch.cuda.FloatTensor( _feature(pres[pres_idx])) e_p = e_p / 70 e_p = (e_p + torch.cuda.FloatTensor(_feature(sub)) + torch.cuda.FloatTensor(_feature(obj))) / 3 embedded_clauses.append(e_p) for idx, info in enumerate(info_list): pos = box_prop_name(info_list[idx][1][1], info_list[idx][2][1]) if pos in POS_REL_NAMES_FULL.keys(): pos = POS_REL_NAMES_FULL[pos] sub = objs[info_list[idx][1][0]] obj = objs[info_list[idx][2][0]] embedded_clauses.append( (torch.cuda.FloatTensor(_feature(pos)) + torch.cuda.FloatTensor( _feature(sub)) + torch.cuda.FloatTensor(_feature(obj))) / 3) tree0, features0, labels0 = assignment_to_tree_compatible( embedded_clauses, node_features) embedded_clauses = embedder(tree0, features0.cuda()) return embedded_clauses