def create_dataset_from_file(input_file, load_file=True): data_list = list() if load_file: with open(input_file, "r") as f: lines = f.readlines() else: lines = input_file.split("\n") for line in lines: if not "#SEP#" in line: print( "[!] WARNING: Every line in the input file must contain the string \"#SEP#\" in between the premise and the hypothesis. Skipping line \"%s\"" % (line)) continue line_sep = line.replace("\n", "").split("#SEP#") premise = line_sep[0] hypothesis = line_sep[1] data_point = NLIData(premise, hypothesis, -1) data_list.append(data_point) print("Premise: " + data_point.get_premise() + "\nHypothesis: " + data_point.get_hypothesis()) infer_dataset = SNLIDataset(data_type="infer", data_path=None, shuffle_data=False) infer_dataset.set_data_list(data_list) _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) infer_dataset.set_vocabulary(word2id) return infer_dataset
def visualize_single_sentence(model, sentence, file_path=None): _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) id2word = {v: k for k, v in word2id.items()} new_d = SentData(sentence=sentence, label=-1) new_d.translate_to_dict(word2id) dataset = DatasetTemplate("all") dataset.set_data_list([new_d]) embeds, lengths, _ = dataset.get_batch(batch_size=1, loop_dataset=False, toTorch=True) _, pool_indices = model.encode_sentence(embeds, lengths, debug=True) pool_indices = pool_indices.data.cpu().numpy()[0] word_id_embeds = embeds.data.cpu().numpy()[0] word_index = range(word_id_embeds.shape[0]) word_importance = [ np.sum(i == pool_indices) * 100.0 / pool_indices.shape[0] for i in range(word_id_embeds.shape[0]) ] plt.xticks(word_index, [id2word[w] for w in word_id_embeds], rotation=45) plt.bar(word_index, word_importance) plt.ylabel("Proportion of pool indices") plt.title("Word importance for sample sentence") if file_path is None: plt.show() else: plt.savefig(file_path) plt.close()
def visualize_word_distribution(word_frequency, word, file_path=None): _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) word_id = word2id[word] word_dist = word_frequency[word_id] feature_index = range(word_frequency.shape[1]) plt.bar(feature_index, word_dist * 100.0) plt.ylabel("Frequency of having the highest value") plt.xlabel("Features") plt.title("Feature pool distribution for the word \"%s\"" % (word)) if file_path is None: plt.show() else: plt.savefig(file_path) plt.close()
def visualize_embeddings(word_frequency, word_occurences, tensorboard_writer, max_examples=4000): _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) id2word = {v: k for k, v in word2id.items()} most_frequent_words = np.argsort(word_occurences)[::-1] most_frequent_words = most_frequent_words[:max_examples] print("Using all words with a frequency of more than %i." % (word_occurences[most_frequent_words[-1]])) embeddings = word_frequency[most_frequent_words] labels = [id2word[w] for w in most_frequent_words] tensorboard_writer.add_embedding(embeddings, metadata=labels, tag="MaxPoolDistribution", global_step=None)
def print_top_bottom(word_importance, word_occurences, N=25): _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) id2word = {v: k for k, v in word2id.items()} word_importance_sort_ascending = np.array( [x for x in np.argsort(word_importance) if word_occurences[x] > 100]) word_importance_sort_descending = word_importance_sort_ascending[::-1] print("=" * 50 + "\nMost important words\n" + "=" * 50) for i in range(N): word_id = word_importance_sort_descending[i] print("Top %i: %s (%4.2f%% in %i sentences)" % (i, str(id2word[word_id]), 100.0 * word_importance[word_id], word_occurences[word_id])) print("=" * 50 + "\nLeast important words\n" + "=" * 50) for i in range(N): word_id = word_importance_sort_ascending[i] print("Bottom %i: %s (%4.2f%% in %i sentences)" % (i, str(id2word[word_id]), 100.0 * word_importance[word_id], word_occurences[word_id]))
def export_sample_word_per_feature(word_frequency, word_occurences, output_path="", N=10, min_occ=100): _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) id2word = {v: k for k, v in word2id.items()} word_frequency = word_frequency * (word_occurences[:, None] > min_occ ) # Masking rare words s = "=" * 50 + "\nWords to feature mapping\n" + "=" * 50 + "\n" for i in range(word_frequency.shape[1]): print("Processing feature %i..." % (i + 1), end="\r") most_popular_words = np.argsort(word_frequency[:, i])[::-1] s += "Feature %4i: [%s]\n" % (i, ", ".join([ "\"%s\" (%4.2f%%)" % (id2word[most_popular_words[j]], word_frequency[most_popular_words[j], i]) for j in range(N) ])) with open(os.path.join(output_path, "feature_word_mapping.txt"), "w") as f: f.write(s)
def retrieve_word_importance(model, dataset, batch_size=64, output_path="", overwrite=False): if model.model_type != NLIModel.BILSTM_MAX: print( "[!] ERROR: Only support BiLSTM MAX models for word importance visualization." ) return _, _, _, _, word2id, _ = load_SNLI_datasets(debug_dataset=False) number_words = len(list(word2id.keys())) word_occurences = np.zeros(shape=(number_words, ), dtype=np.int32) word_max_index = np.zeros(shape=(number_words, model.model_params["embed_sent_dim"]), dtype=np.int32) np_file_path = os.path.join(output_path, "np_combined.npz") if not overwrite and os.path.isfile(np_file_path): loaded_np_file = np.load(np_file_path) word_occurences = loaded_np_file['occurences'] word_max_index = loaded_np_file['indices'] else: number_batches = int(math.ceil(dataset.get_num_examples() / batch_size)) for i in range(number_batches): print("Processed %4.2f%% of dataset" % (i * 100.0 / number_batches), end="\r") embeds, lengths, _ = dataset.get_batch(batch_size=batch_size, loop_dataset=False, toTorch=True) embeds = [embeds] if not isinstance(embeds, list) else embeds lengths = [lengths] if not isinstance(lengths, list) else lengths for e, l in zip(embeds, lengths): _, pool_indices = model.encode_sentence(e, l, debug=True) pool_indices = pool_indices.data.cpu().numpy() word_id_embeds = e.data.cpu().numpy() for batch_index in range(word_id_embeds.shape[0]): word_occurences[word_id_embeds[batch_index]] += 1 pooled_words = word_id_embeds[batch_index, pool_indices[batch_index]] word_max_index[pooled_words, range(word_max_index.shape[1])] += 1 np.savez_compressed(np_file_path, indices=word_max_index, occurences=word_occurences) word_frequency = word_max_index / (1e-10 + word_occurences[:, None]) word_importance = np.mean(word_frequency, axis=1) print_top_bottom(word_importance, word_occurences) # visualize_word_distribution(word_frequency, "<s>", file_path=os.path.join(output_path, "word_dist_s.pdf")) # visualize_word_distribution(word_frequency, ",", file_path=os.path.join(output_path, "word_dist_comma.pdf")) # visualize_word_distribution(word_frequency, "bikinis", file_path=os.path.join(output_path, "word_dist_bikinis.pdf")) # visualize_word_distribution(word_frequency, "as", file_path=os.path.join(output_path, "word_dist_as.pdf")) # visualize_word_distribution(word_frequency, "like", file_path=os.path.join(output_path, "word_dist_like.pdf")) export_sample_word_per_feature(word_frequency, word_occurences, N=25) return word_frequency, word_occurences
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--checkpoint_path", help="Folder(name) where checkpoints are saved", type=str, required=True) parser.add_argument( "--overwrite", help= "Whether to re-create the distributions if they are already in the folder or not", action="store_true") args = parser.parse_args() model = load_model_from_args(load_args(args.checkpoint_path), args.checkpoint_path) train_dataset, val_dataset, test_dataset, _, _, _ = load_SNLI_datasets( debug_dataset=True) dataset = copy.deepcopy(test_dataset) dataset.set_data_list(train_dataset.data_list + val_dataset.data_list + test_dataset.data_list) if os.path.isfile(args.checkpoint_path): output_path = args.checkpoint_path.rsplit("/", 1)[0] else: output_path = args.checkpoint_path word_freq, word_occ = retrieve_word_importance(model, dataset, output_path=output_path, overwrite=args.overwrite) # visualize_single_sentence(model, "Two women are sleeping in their bikinis on a beach .") # writer = SummaryWriter(args.checkpoint_path) # visualize_embeddings(word_freq, word_occ, writer)