def reload_evaluation_ir(params, training_vectors, validation_vectors, test_vectors, suffix=""): ### Information Retrieval dataset = data.Dataset(params['dataset']) log_dir = os.path.join(params['model'], 'logs') if not os.path.exists(log_dir): os.makedirs(log_dir) ir_ratio_list = [0.0001, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.3, 0.5, 0.8, 1.0] #ir_ratio_list = [0.02] training_labels = np.array( [[y] for y, _ in dataset.rows('training_docnade', num_epochs=1)] ) test_labels = np.array( [[y] for y, _ in dataset.rows('test_docnade', num_epochs=1)] ) test_ir_list = eval.evaluate( training_vectors, test_vectors, training_labels, test_labels, recall=ir_ratio_list, num_classes=params['num_classes'], multi_label=params['multi_label'] ) # logging information with open(os.path.join(log_dir, "reload_info_ir.txt"), "a") as f: f.write("\n\nFractions list: %s" % (ir_ratio_list)) f.write("\nTest IR: %s" % (test_ir_list))
def load_image(image_path: str): if image_path is not None: return torch.from_numpy(cv2.imread(image_path)) dataloader = data_ops.Dataset('data/valid/') random_image = random.choice(dataloader) return random_image
def main(flags): dataset = data.Dataset(flags.data_dir, flags.seed) vocabsize = len(dataset.index2word) if flags.model == 'gsdtm': model = gsdtm.GSDTM(vocabsize, flags) else: model = lmdtm.LMDTM(vocabsize, flags) evaluate(model, dataset, flags, vocabsize)
def main(args): with open(os.path.join(args.model, 'params.json'), 'r') as f: params = json.loads(f.read()) params.update(vars(args)) params = namedtuple('Params', params.keys())(*params.values()) dataset = data.Dataset(args.dataset) x = tf.placeholder(tf.int32, shape=(None, None), name='x') seq_lengths = tf.placeholder(tf.int32, shape=(None), name='seq_lengths') model = m.DocNADE(x, seq_lengths, params) evaluate(model, dataset, params)
def main(args): with open(os.path.join(args.model, 'params.json'), 'r') as f: params = json.loads(f.read()) params.update(vars(args)) params = namedtuple('Params', params.keys())(*params.values()) dataset = data.Dataset(args.dataset) x = tf.placeholder(tf.float32, shape=(None, params.vocab_size), name='x') z = tf.placeholder(tf.float32, shape=(None, params.z_dim), name='z') model = m.DocModel(x, z, params) vectors(model, dataset, params)
def main(args): if not os.path.isdir(args.model): os.mkdir(args.model) with open(os.path.join(args.model, 'params.json'), 'w') as f: f.write(json.dumps(vars(args))) dataset = data.Dataset(args.dataset) x = tf.placeholder(tf.int32, shape=(None, None), name='x') seq_lengths = tf.placeholder(tf.int32, shape=(None), name='seq_lengths') model = m.DocNADE(x, seq_lengths, args) train(model, dataset, args)
def main(flags): dataset = data.Dataset(flags.data_dir, flags.seed) vocabsize = len(dataset.index2word) # Clean log files if tf.gfile.Exists(flags.summaries_dir): tf.gfile.DeleteRecursively(flags.summaries_dir) tf.gfile.MakeDirs(flags.summaries_dir) # Make Tensorboard projector metafile utils.make_metadata_visualization(dataset, flags) model, emb = train(dataset, vocabsize, flags)
def main(args): if not os.path.exists(args.model): os.mkdir(args.model) if args.d_dim == 0: args.d_dim = args.g_dim with open(os.path.join(args.model, args.param_file), 'w') as f: f.write(json.dumps(vars(args))) dataset = data.Dataset(args.dataset) x = tf.placeholder(tf.float32, shape=(None, args.vocab_size), name='x') z = tf.placeholder(tf.float32, shape=(None, args.z_dim), name='z') model = m.DocModel(x, z, args) train(model, dataset, args)
def main(args): with open(os.path.join(args.model, 'params.json'), 'r') as f: params = json.loads(f.read()) params.update(vars(args)) params = namedtuple('Params', params.keys())(*params.values()) dataset = data.Dataset(args.dataset) x = tf.placeholder(tf.float32, shape=(None, params.vocab_size), name='x') z = tf.placeholder(tf.float32, shape=(None, params.z_dim), name='z') mask = tf.placeholder(tf.float32, shape=(None, params.vocab_size), name='mask') model = m.ADM(x, z, mask, params) #evaluate(model, dataset, params) #plot_tsne(model, dataset, params) #topic_word_dist(model, dataset, params, 10) evaluate_loss(model, dataset, params)
def main(args): if not os.path.exists(args.model): os.mkdir(args.model) with open(os.path.join(args.model, 'params.json'), 'w') as f: f.write(json.dumps(vars(args))) dataset = data.Dataset(args.dataset) x = tf.placeholder(tf.float32, shape=(None, args.vocab_size), name='x') z = tf.placeholder(tf.float32, shape=(None, args.z_dim), name='z') mask = tf.placeholder( tf.float32, shape=(None, args.vocab_size), name='mask' ) model = m.ADM(x, z, mask, args) train(model, dataset, args)
def reload_evaluation_f1(params, training_vectors, validation_vectors, test_vectors, suffix=""): ### Classification - F1 dataset = data.Dataset(params['dataset']) log_dir = os.path.join(params['model'], 'logs') if not os.path.exists(log_dir): os.makedirs(log_dir) c_list = [0.0001, 0.001, 0.01, 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 100.0, 500.0, 1000.0, 10000.0] #c_list = [1.0] test_acc = [] test_f1 = [] val_acc = [] val_f1 = [] test_acc_W = [] test_f1_W = [] val_acc_W = [] val_f1_W = [] y_train = np.array( [y for y, _ in dataset.rows('training_docnade', num_epochs=1)] ) y_val = np.array( [y for y, _ in dataset.rows('validation_docnade', num_epochs=1)] ) y_test = np.array( [y for y, _ in dataset.rows('test_docnade', num_epochs=1)] ) if not params['multi_label']: train_data = (training_vectors, np.array(y_train, dtype=np.int32)) validation_data = (validation_vectors, np.array(y_val, dtype=np.int32)) test_data = (test_vectors, np.array(y_test, dtype=np.int32)) test_acc, test_f1 = eval.perform_classification_test(train_data, test_data, c_list) with open(os.path.join(log_dir, "reload_info_ir.txt"), "a") as f: f.write("\n\nTest accuracy with h vector IR: %s" % (test_acc)) with open(os.path.join(log_dir, "reload_info_ir.txt"), "a") as f: f.write("\n\nTest F1 score with h vector IR: %s" % (test_f1)) else: total_labels = [] y_train_new = [label.strip().split(':') for label in y_train] y_val_new = [label.strip().split(':') for label in y_val] y_test_new = [label.strip().split(':') for label in y_test] total_labels.extend(y_train_new) #total_labels.extend(y_val_new) #total_labels.extend(y_test_new) from sklearn.preprocessing import MultiLabelBinarizer mlb = MultiLabelBinarizer() mlb.fit(total_labels) y_train_one_hot = mlb.transform(y_train_new) y_val_one_hot = mlb.transform(y_val_new) y_test_one_hot = mlb.transform(y_test_new) train_data = (training_vectors, y_train_one_hot) validation_data = (validation_vectors, y_val_one_hot) test_data = (test_vectors, y_test_one_hot) test_acc, test_f1 = eval.perform_classification_test_multi(train_data, test_data, c_list) with open(os.path.join(log_dir, "reload_info_ir.txt"), "a") as f: f.write("\n\nTest accuracy with h vector IR: %s" % (test_acc)) with open(os.path.join(log_dir, "reload_info_ir.txt"), "a") as f: f.write("\n\nTest F1 score with h vector IR: %s" % (test_f1))
def reload_evaluation_f1(params, training_vectors, validation_vectors, training_labels, validation_labels, training_ids, validation_ids, val_labels, val_ids, suffix=""): ### Classification - F1 dataset = data.Dataset(params['dataset']) log_dir = os.path.join(params['model'], 'logs') if not os.path.exists(log_dir): os.makedirs(log_dir) #c_list = [0.0001, 0.001, 0.01, 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 100.0, 500.0, 1000.0, 10000.0] #c_list = [1.0, 3.0, 5.0, 10.0, 100.0, 500.0, 1000.0, 10000.0] c_list = [1000.0] test_acc = [] test_f1 = [] val_acc = [] val_f1 = [] test_acc_W = [] test_f1_W = [] val_acc_W = [] val_f1_W = [] #y_train = training_labels #y_test = validation_labels train_data = (np.array(training_vectors, dtype=np.float32), np.array(training_labels, dtype=np.int32)) test_data = (np.array(validation_vectors, dtype=np.float32), np.array(validation_labels, dtype=np.int32)) test_acc, test_f1, pred_probs, pred_labels = perform_classification_test(train_data, test_data, c_list, classification_model="svm", norm_before_classification=False) with open(os.path.join(log_dir, "info.txt"), "w") as f: f.write("\n\nTest accuracy with h vector IR: %s" % (test_acc)) with open(os.path.join(log_dir, "info.txt"), "a") as f: f.write("\n\nTest F1 score with h vector IR: %s" % (test_f1)) print("Test acc: ", test_acc) print("Test f1: ", test_f1) #import pdb; pdb.set_trace() acc_scores = [] best_acc_score = -1.0 best_val_docs_pred_labels = {} best_val_docs_pred_probs = {} for j in range(len(c_list)): temp_pred_labels = np.array(validation_labels, dtype=np.int32) temp_pred_probs = pred_probs[j][:, 1] unique_val_ids = np.unique(validation_ids) val_docs_pred_labels = {id:[] for id in unique_val_ids} val_docs_pred_probs = {id:[] for id in unique_val_ids} for i, id in enumerate(validation_ids): val_docs_pred_labels[id].append(temp_pred_labels[i]) val_docs_pred_probs[id].append(temp_pred_probs[i]) #import pdb; pdb.set_trace() val_pred_list = [] for id in unique_val_ids: max_index = np.argmax(val_docs_pred_probs[id]) val_pred_list.append(val_docs_pred_labels[id][max_index]) #import pdb; pdb.set_trace() #acc_score = accuracy_score(np.ones(len(val_pred_list), dtype=np.int8), np.array(val_pred_list)) acc_score = np.mean(val_pred_list) acc_scores.append(acc_score) if acc_score > best_acc_score: best_acc_score = acc_score best_val_docs_pred_labels = val_docs_pred_labels best_val_docs_pred_probs = val_docs_pred_probs #precision, recall, f1, support = precision_recall_fscore_support(np.ones(len(val_pred_list), dtype=np.int8), np.array(val_pred_list), average='macro') #print("Test acc_score: ", acc_score) print("Validation acc_scores: ", acc_scores) #print("Test precision: ", precision) #print("Test recall: ", recall) #print("Test f1: ", f1) #print("Test support: ", support) with open(os.path.join(log_dir, "info.txt"), "a") as f: #f.write("\n\nBest accuracy score: %s" % (acc_score)) f.write("\n\nValidation accuracy scores: %s" % (acc_scores)) #f.write("\n\nBest precision score: %s" % (precision)) #f.write("\n\nBest recall score: %s" % (recall)) #f.write("\n\nBest f1 score: %s" % (f1)) #f.write("\n\nBest support score: %s" % (support)) #import pdb; pdb.set_trace() val_scores_dict = {} val_ids_dict = {} for label, id in zip(val_labels, val_ids): if not label in val_scores_dict: val_scores_dict[label] = [] val_ids_dict[label] = [] max_index = np.argmax(best_val_docs_pred_probs[id]) pred_label = best_val_docs_pred_labels[id][max_index] val_scores_dict[label].append(pred_label) val_ids_dict[label].append(id) classwise_scores = [] for label in val_scores_dict.keys(): class_val_prec = np.mean(val_scores_dict[label]) classwise_scores.append(class_val_prec) with open(log_dir + "/classwise_scores.txt", "w") as f: for label, scores in val_scores_dict.items(): f.write("Label: " + label + "\n") f.write("ids: " + str(val_ids_dict[label]) + "\n") f.write("scores: " + str(scores) + "\n") f.write("prec: " + str(np.mean(scores)) + "\n") f.write("\n\n") # logging information with open(os.path.join(log_dir, "reload_info_clusters_with_SVM.txt"), "w") as f: for doc_counter, query, id in zip(range(len(val_pred_list)), val_labels, val_ids): sorted_indices = np.argsort(best_val_docs_pred_probs[id])[::-1] relevance_labels = np.array(best_val_docs_pred_labels[id])[sorted_indices] relevance_scores = np.array(best_val_docs_pred_probs[id])[sorted_indices] f.write("Doc " + str(doc_counter) + ": " + str(query) + ": " + str(id) + "\n\n") f.write("Predicted_labels: " + " ".join([str(int(l)) for l in relevance_labels]) + "\n") f.write("Predicted_probs: " + " ".join([str(l) for l in relevance_scores]) + "\n") f.write("Sentence indices: " + " ".join([str(l) for l in sorted_indices]) + "\n") f.write("\n=================================================================================\n\n")
def reload_evaluation_f1(params, training_vectors, validation_vectors, test_vectors, suffix=""): ### Classification - F1 dataset = data.Dataset(params['dataset']) log_dir = os.path.join(params['model'], 'logs') if not os.path.exists(log_dir): os.makedirs(log_dir) #c_list = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0, 10000.0] #c_list = [1.0, 3.0, 5.0, 10.0, 100.0, 500.0, 1000.0, 10000.0] #c_list = [0.05, 0.5] #c_list = [0.001, 0.01, 0.1, 10.0] c_list = [0.01] test_acc = [] test_f1 = [] val_acc = [] val_f1 = [] test_acc_W = [] test_f1_W = [] val_acc_W = [] val_f1_W = [] y_train = np.array( [int(y) for y, _ in dataset.rows('training_docnade', num_epochs=1)] ) y_val = np.array( [int(y) for y, _ in dataset.rows('validation_docnade', num_epochs=1)] ) y_test = np.array( [int(y) for y, _ in dataset.rows('test_docnade', num_epochs=1)] ) #import pdb; pdb.set_trace() queries = [] with open(params['dataset'] + "/labels.txt", "r") as f: for line in f.readlines(): label_tokens = line.lower().strip().split("_") label_tokens = [token for token in label_tokens if token] query = " ".join(label_tokens) queries.append(query) with open(params['dataset'] + "/test_ids.txt", "r") as f: ids = [line.strip() for line in f.readlines()] #id2label = {0:"Arousal", 1:"Circadian_Rhythms", 2:"Acute_Threat_Fear", 3:"Loss", 4:"Sleep_Wakefulness", 5:"Frustrative_Nonreward", 6:"Potential_Threat_Anxiety", 7:"Sustained_Threat"} id2label = {0:"Acute_Threat_Fear", 1:"Arousal", 2:"Circadian_Rhythms", 3:"Frustrative_Nonreward", 4:"Loss", 5:"Potential_Threat_Anxiety", 6:"Sleep_Wakefulness", 7:"Sustained_Threat"} train_data = (training_vectors, np.array(y_train, dtype=np.int32)) validation_data = (validation_vectors, np.array(y_val, dtype=np.int32)) test_data = (test_vectors, np.array(y_test, dtype=np.int32)) #test_acc, test_f1 = perform_classification_test(train_data, test_data, c_list, classification_model="svm", norm_before_classification=False) test_acc, test_f1, pred_probs_list, pred_labels_list = perform_classification_test(train_data, validation_data, test_data, c_list, classification_model="svm", norm_before_classification=False) #import pdb; pdb.set_trace() #try: for c, pred_probs in zip(c_list, pred_probs_list): relevance_score_svm = [] for i, label in enumerate(y_test): relevance_score_svm.append(pred_probs[i, int(label)]) relevance_score_svm = np.array(relevance_score_svm) #except: # import pdb; pdb.set_trace() dict_label = {int(label):[] for label in np.unique(y_test)} for score, id, label in zip(relevance_score_svm, ids, y_test): dict_label[int(label)].append([id,score]) with open("task1_test_svm_classify" + str(c) + ".txt", "w") as f: for key in dict_label.keys(): f.write(id2label[int(key)] + "\n") for id, score in dict_label[key]: f.write(id + "\t" + str(score) + "\n") #sys.exit() if evaluate_on_test: ## Using BM25 Extra for relevance ranking and mAP calculation with open("./datasets/Task1_and_Task2_without_acronym_with_Task1_testdata_OOV_words/test.csv", "r") as f: file_reader = csv.reader(f, delimiter=',') docs = [line[1].strip() for line in file_reader] bm25 = BM25.BM25("./datasets/Task1_and_Task2_without_acronym_with_Task1_testdata_OOV_words/test.csv", delimiter=' ') def get_bm25_ids(tokens): ids = [] for token in tokens: try: ids.append(bm25.dictionary.token2id[token]) except KeyError: pass return ids bm25_extra_scores_list = [] #for query in queries: for value in id2label.values(): query = " ".join(value.lower().split("_")).strip() if query == "frustrative nonreward": query = "reward aggression" if query == "arousal": query += " affective states heart rate" query = query.split() scores = bm25.BM25Score(query) extra_features = [] for doc in docs: doc = doc.split() #doc_ids = [bm25.dictionary.token2id[token] for token in doc] #query_ids = [bm25.dictionary.token2id[token] for token in query] doc_ids = get_bm25_ids(doc) query_ids = get_bm25_ids(query) feats = bm25.query_doc_overlap(query_ids, doc_ids) extra_features.append(np.sum(feats)) #scores = np.stack([np.array(scores), np.array(extra_features)], axis=1) scores = np.add(np.array(scores), np.array(extra_features)) bm25_extra_scores_list.append(scores) bm25_extra_scores_matrix = np.stack(bm25_extra_scores_list, axis=1) #import pdb; pdb.set_trace() relevance_score_bm25_extra = [] for i, label in enumerate(y_test): relevance_score_bm25_extra.append(bm25_extra_scores_matrix[i, int(label)]) relevance_score_bm25_extra = np.array(relevance_score_bm25_extra) dict_label = {label:[] for label in np.unique(y_test)} for score, id, label in zip(relevance_score_bm25_extra, ids, y_test): dict_label[label].append([id,score]) with open("task1_test_bm25extra.txt", "w") as f: for key in dict_label.keys(): f.write(id2label[int(key)] + "\n") for id, score in dict_label[key]: f.write(id + "\t" + str(score) + "\n") #combined_relevance_score = relevance_score_svm + relevance_score_bm25_extra pseudo_prediction = np.argmax(pred_probs_list[0], axis=1) combined_relevance_score = [] counter = 0 for svm_pred, bm25_score, svm_score, true_label in zip(pseudo_prediction, relevance_score_bm25_extra, relevance_score_svm, y_test): #for svm_pred, bm25_score, svm_score, true_label in zip(pred_labels_list[0], relevance_score_bm25_extra, relevance_score_svm, y_test): #import pdb; pdb.set_trace() if true_label == svm_pred: combined_relevance_score.append(svm_score + bm25_score) counter += 1 else: combined_relevance_score.append(svm_score) print(counter) dict_label = {label:[] for label in np.unique(y_test)} for score, id, label in zip(combined_relevance_score, ids, y_test): dict_label[label].append([id,score]) with open("task1_test_svm_classify_with_bm25extra.txt", "w") as f: for key in dict_label.keys(): f.write(id2label[int(key)] + "\n") for id, score in dict_label[key]: f.write(id + "\t" + str(score) + "\n") with codecs.open("./pretrained_embeddings/biggest_vocab.vocab", "r", encoding='utf-8', errors='ignore') as f: total_vocab = [line.strip() for line in f.readlines()] prior_embedding_matrices = [] if params['use_bio_prior']: #bio_embeddings_large = np.load('./pretrained_embeddings/bionlp_embeddings_biggest_vocab.npy') bio_embeddings_large = np.load('./pretrained_embeddings/' + params['bioemb_path']) prior_embedding_matrices.append(bio_embeddings_large) if params['use_fasttext_prior']: #fasttext_embeddings_large = np.load('./pretrained_embeddings/fasttext_embeddings_biggest_vocab.npy') fasttext_embeddings_large = np.load('./pretrained_embeddings/' + params['fttemb_path']) prior_embedding_matrices.append(fasttext_embeddings_large) if params['use_BOW_repesentation']: BOW_representations = np.eye(len(total_vocab), dtype=np.float32) prior_embedding_matrices.append(BOW_representations) total_embedding_matrix = np.concatenate(prior_embedding_matrices, axis=1) similarity_scores_Attention_Based_EmbSum = np.zeros((len(y_test), 8), dtype=np.float32) with open("./datasets/Task1_and_Task2_without_acronym_with_Task1_testdata_OOV_words/test.csv", "r") as f: file_reader = csv.reader(f, delimiter=",") for j, row in enumerate(file_reader): tokens = [total_vocab.index(word) for word in row[1].strip().split()] Embs = total_embedding_matrix[np.array(tokens), :] #for i, query in enumerate(queries): for k, value in enumerate(id2label.values()): query = " ".join(value.lower().split("_")).strip() if query == "frustrative nonreward": query = "reward aggression" if query == "arousal": query += " affective states heart rate" query_tokens = query.split() EmbSum_attns = [] query_vecs_attns = [] for qword in query_tokens: query_vector = total_embedding_matrix[total_vocab.index(qword), :] query_vector = np.expand_dims(query_vector, axis=0) query_attentions = pw.cosine_similarity(query_vector, Embs) #query_attentions[(query_attentions < 0.5)] = 0.0 query_attentions = softmax(query_attentions) EmbSum_attentions = np.dot(query_attentions, Embs) EmbSum_attns.append(EmbSum_attentions) query_vecs_attns.append(query_vector) EmbSum = np.sum(EmbSum_attns, axis=0) #query_EmbSum_vector = np.expand_dims(query_vecs[i], axis=0) query_EmbSum_vector = np.sum(query_vecs_attns, axis=0) similarity_score = pw.cosine_similarity(query_EmbSum_vector, EmbSum) similarity_scores_Attention_Based_EmbSum[j, k] = similarity_score[0][0] relevance_score_att_embsum = [] for i, label in enumerate(y_test): relevance_score_att_embsum.append(similarity_scores_Attention_Based_EmbSum[i, int(label)]) relevance_score_att_embsum = np.array(relevance_score_att_embsum) #combined_relevance_score = relevance_score_svm + relevance_score_bm25_extra dict_label = {label:[] for label in np.unique(y_test)} for score, id, label in zip(relevance_score_att_embsum, ids, y_test): dict_label[label].append([id,score]) with open("task1_test_att_based_embsum.txt", "w") as f: for key in dict_label.keys(): f.write(id2label[int(key)] + "\n") for id, score in dict_label[key]: f.write(id + "\t" + str(score) + "\n") #combined_relevance_score_classify_embsum = relevance_score_att_embsum + relevance_score_svm pseudo_prediction = np.argmax(pred_probs_list[0], axis=1) combined_relevance_score_classify_embsum = [] #positive_half_scores = [] #negative_half_scores = [] counter = 0 for svm_pred, att_score, svm_score, true_label in zip(pseudo_prediction, relevance_score_att_embsum, relevance_score_svm, y_test): #for svm_pred, att_score, svm_score, true_label in zip(pred_labels_list[0], relevance_score_bm25_extra, relevance_score_svm, y_test): #import pdb; pdb.set_trace() if true_label == svm_pred: combined_relevance_score_classify_embsum.append(svm_score + att_score) #positive_half_scores.append() counter += 1 else: combined_relevance_score_classify_embsum.append(svm_score) print(counter) dict_label = {label:[] for label in np.unique(y_test)} for score, id, label in zip(combined_relevance_score_classify_embsum, ids, y_test): dict_label[label].append([id,score]) with open("task1_test_classify_att_based_embsum.txt", "w") as f: for key in dict_label.keys(): f.write(id2label[int(key)] + "\n") for id, score in dict_label[key]: f.write(id + "\t" + str(score) + "\n") combined_relevance_score_classify_embsum_bm25extra = relevance_score_att_embsum + relevance_score_svm + relevance_score_bm25_extra dict_label = {label:[] for label in np.unique(y_test)} for score, id, label in zip(combined_relevance_score_classify_embsum_bm25extra, ids, y_test): dict_label[label].append([id,score]) with open("task1_test_classify_att_based_embsum_bm25extra.txt", "w") as f: for key in dict_label.keys(): f.write(id2label[int(key)] + "\n") for id, score in dict_label[key]: f.write(id + "\t" + str(score) + "\n")