def negated_and_rare(model, isLogLin=True): if isLogLin: data_man = DataManager(batch_size=BATCH_SIZE) data_iter = data_man.get_torch_iterator(data_subset=TEST) print("Log linear test on negated words: ") else: data_man = DataManager(data_type=W2V_SEQUENCE, batch_size=BATCH_SIZE, embedding_dim=W2V_EMBEDDING_DIM) data_iter = data_man.get_torch_iterator(data_subset=TEST) print("W2V test with on negated words: ") sent_list = data_iter.dataset.data preds = get_predictions_for_data(model, data_iter) neg_indices = data_loader.get_negated_polarity_examples(sent_list) rare_indices = data_loader.get_rare_words_examples( sent_list, data_loader.SentimentTreeBank()) rare_preds = np.take(preds, rare_indices) neg_preds = np.take(preds, neg_indices) rare_labels = np.take(get_iter_labels(data_iter), rare_indices) neg_labels = np.take(get_iter_labels(data_iter), neg_indices) neg_acc = binary_accuracy(neg_preds, neg_labels) rare_acc = binary_accuracy(rare_preds, rare_labels) print("Negated words accuracy: ", neg_acc) print("Rare words accuracy: ", rare_acc)
def train_model1(model, data_manager, n_epochs, lr, weight_decay=0.): """ Runs the full training procedure for the given model. The optimization should be done using the Adam optimizer with all parameters but learning rate and weight decay set to default. :param model: module of one of the models implemented in the exercise :param data_manager: the DataManager object :param n_epochs: number of times to go over the whole training set :param lr: learning rate to be used for optimization :param weight_decay: parameter for l2 regularization """ optimizer = torch.optim.Adam(params=model.parameters(), lr=lr, weight_decay=weight_decay) criterion = torch.nn.modules.loss.BCEWithLogitsLoss() tstloss = np.zeros(n_epochs) tstacc = np.zeros(n_epochs) tst1acc = np.zeros(n_epochs) tst2acc = np.zeros(n_epochs) tst_data = np.array(data_manager.sentences[TEST]) labels = np.array(data_manager.get_labels(TEST)) tst1_indexes = data_loader.get_negated_polarity_examples(tst_data) tst2_indexes = data_loader.get_rare_words_examples( tst_data, data_manager.sentiment_dataset) tst1_data = tst_data[tst1_indexes] tst2_data = tst_data[tst2_indexes] word2vec = load_pickle("w2v_dict.pkl") tst1_data = [ sentence_to_embedding(sent, word2vec, SEQ_LEN) for sent in tst1_data ] tst2_data = [ sentence_to_embedding(sent, word2vec, SEQ_LEN) for sent in tst2_data ] tst1_labels = labels[tst1_indexes] tst2_labels = labels[tst2_indexes] tst1_iterator = [[ torch.FloatTensor([tst1_data[i]]).double().cuda(), torch.FloatTensor([tst1_labels[i]]).double().cuda() ] for i in range(len(tst1_indexes))] tst2_iterator = [[ torch.FloatTensor([tst2_data[i]]).double().cuda(), torch.FloatTensor([tst2_labels[i]]).double().cuda() ] for i in range(len(tst2_indexes))] for i in range(n_epochs): print("epoch: ", i) iterator = data_manager.get_torch_iterator() tst_iterator = data_manager.get_torch_iterator(TEST) train_epoch(model, iterator, optimizer, criterion) tstloss[i], tstacc[i] = evaluate(model, tst_iterator, criterion) _, tst1acc[i] = evaluate(model, tst1_iterator, criterion) _, tst2acc[i] = evaluate(model, tst2_iterator, criterion) print("loss", tstloss[i]) print("acc", tstacc[i]) return tstloss, tstacc, tst1acc, tst2acc
def get_special_acc(model, dataset, test_iterator): sentences = test_iterator.dataset.data y = np.array([]) for batch in test_iterator: y = np.concatenate((y, batch[1].numpy())) rare_words_idxs = data_loader.get_rare_words_examples(sentences, dataset) negated_polarity_idxs = data_loader.get_negated_polarity_examples( sentences) preds = get_predictions_for_data(model, test_iterator) rare_words_preds = preds[rare_words_idxs] negated_polarity_preds = preds[negated_polarity_idxs] rare_words_gt = y[rare_words_idxs] negated_polarity_gt = y[negated_polarity_idxs] rare_test_acc = binary_accuracy(rare_words_preds, rare_words_gt) negated_polarity_test_acc = binary_accuracy(negated_polarity_preds, negated_polarity_gt) print("rare words accuracy is {}\nnegated polarity acc is {}".format( rare_test_acc, negated_polarity_test_acc))
def special_cases_acc(data_manager, model): test_sents = data_manager.sentences[TEST] test_batches = list(data_manager.get_torch_iterator(TEST)) test_vectors = [] for batch in test_batches: for vector in batch[0]: test_vectors.append(vector) test_labels = data_manager.get_labels(TEST) NP_sents_idxs = data_loader.get_negated_polarity_examples(test_sents) NP_x = [test_vectors[i].float() for i in NP_sents_idxs] NP_y = [test_labels[i] for i in NP_sents_idxs] pred = model.predict(torch.stack(NP_x)).detach().numpy() NP_acc = binary_accuracy(pred, NP_y) RW_sents_idxs = data_loader.get_rare_words_examples( test_sents, data_manager.sentiment_dataset) RW_x = [test_vectors[i].float() for i in RW_sents_idxs] RW_y = [test_labels[i] for i in RW_sents_idxs] pred = model.predict(torch.stack(RW_x)).detach().numpy() RW_acc = binary_accuracy(pred, RW_y) return NP_acc, RW_acc
def train_model(model, data_manager, n_epochs, lr, weight_decay=0.): """ Runs the full training procedure for the given model. The optimization should be done using the Adam optimizer with all parameters but learning rate and weight decay set to default. :param model: module of one of the models implemented in the exercise :param data_manager: the DataManager object :param n_epochs: number of times to go over the whole training set :param lr: learning rate to be used for optimization :param weight_decay: parameter for l2 regularization """ train_loader = data_manager.get_torch_iterator(TRAIN) validation_loader = data_manager.get_torch_iterator(VAL) test_loader = data_manager.get_torch_iterator(TEST) optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) criterion = nn.BCEWithLogitsLoss() train_losses = [] train_accuracies = [] validation_losses = [] validation_accuracies = [] for epoch in range(n_epochs): print(epoch) train_loss, train_accuracy = train_epoch(model, train_loader, optimizer, criterion, epoch) validation_loss, validation_accuracy = evaluate( model, validation_loader, criterion) train_losses.append(train_loss) train_accuracies.append(train_accuracy) validation_losses.append(validation_loss) validation_accuracies.append(validation_accuracy) test_accuracy = get_predictions_for_data(model, test_loader) print("Test set: Accuracy: {:.0f}%".format(test_accuracy)) test_sentences = data_manager.sentences[TEST] test_labels = data_manager.get_labels(TEST) negated_indexes = get_negated_polarity_examples(test_sentences) rare_words_indexes = get_rare_words_examples( test_sentences, data_manager.sentiment_dataset) all_word_vectors = [] for batch in list(data_manager.get_torch_iterator(TEST)): for vector in batch[0]: all_word_vectors.append(vector) negated_inputs = [all_word_vectors[i].float() for i in negated_indexes] rare_words_inputs = [ all_word_vectors[i].float() for i in rare_words_indexes ] negated_labels = [torch.tensor(test_labels[i]) for i in negated_indexes] rare_words_labels = [ torch.tensor(test_labels[i]) for i in rare_words_indexes ] negated_data = torch.stack(negated_inputs) rare_words_data = torch.stack(rare_words_inputs) negated_labels = torch.stack(negated_labels) rare_words_labels = torch.stack(rare_words_labels) negated_dataset = torch.utils.data.TensorDataset(negated_data, negated_labels) rare_words_dataset = torch.utils.data.TensorDataset( rare_words_data, rare_words_labels) negated_test_loader = torch.utils.data.DataLoader(negated_dataset) rare_words_test_loader = torch.utils.data.DataLoader(rare_words_dataset) negated_test_accuracy = get_predictions_for_data(model, negated_test_loader) rare_words_test_accuracy = get_predictions_for_data( model, rare_words_test_loader) print("Negated test set: Accuracy: {:.0f}%".format(negated_test_accuracy)) print("Rare words test set: Accuracy: {:.0f}%".format( rare_words_test_accuracy)) plt.title("Training Loss Curve (batch_size={}, lr={})".format( train_loader.batch_size, lr)) plt.plot(range(n_epochs), train_losses, label="Train") plt.plot(range(n_epochs), validation_losses, label="Validation") plt.xlabel("Iterations") plt.ylabel("Training Loss") plt.legend(loc='best') plt.show() plt.title("Training Accuracy Curve (batch_size={}, lr={})".format( train_loader.batch_size, lr)) plt.plot(range(n_epochs), train_accuracies, label="Train") plt.plot(range(n_epochs), validation_accuracies, label="Validation") plt.xlabel("Iterations") plt.ylabel("Training Accuracy") plt.legend(loc='best') plt.show()
def __init__(self, data_type=ONEHOT_AVERAGE, use_sub_phrases=True, dataset_path="stanfordSentimentTreebank", batch_size=50, embedding_dim=None): """ builds the data manager used for training and evaluation. :param data_type: one of ONEHOT_AVERAGE, W2V_AVERAGE and W2V_SEQUENCE :param use_sub_phrases: if true, training data will include all sub-phrases plus the full sentences :param dataset_path: path to the dataset directory :param batch_size: number of examples per batch :param embedding_dim: relevant only for the W2V data types. """ # load the dataset self.sentiment_dataset = data_loader.SentimentTreeBank( dataset_path, split_words=True) # map data splits to sentences lists self.sentences = {} self.sentences[NEGATIVE_POLARITY] = [ self.sentiment_dataset.sentences[i] for i in data_loader.get_negated_polarity_examples( self.sentiment_dataset.sentences) ] self.sentences[RARE] = [ self.sentiment_dataset.sentences[i] for i in data_loader.get_rare_words_examples( self.sentiment_dataset.sentences, self.sentiment_dataset, 50) ] if use_sub_phrases: self.sentences[ TRAIN] = self.sentiment_dataset.get_train_set_phrases() else: self.sentences[TRAIN] = self.sentiment_dataset.get_train_set() self.sentences[VAL] = self.sentiment_dataset.get_validation_set() self.sentences[TEST] = self.sentiment_dataset.get_test_set() # map data splits to sentence input preperation functions words_list = list(self.sentiment_dataset.get_word_counts().keys()) if data_type == ONEHOT_AVERAGE: self.sent_func = average_one_hots self.sent_func_kwargs = { "word_to_ind": get_word_to_ind(words_list) } elif data_type == W2V_SEQUENCE: self.sent_func = sentence_to_embedding self.sent_func_kwargs = { "seq_len": SEQ_LEN, "word_to_vec": create_or_load_slim_w2v(words_list), "embedding_dim": embedding_dim } elif data_type == W2V_AVERAGE: self.sent_func = get_w2v_average words_list = list(self.sentiment_dataset.get_word_counts().keys()) self.sent_func_kwargs = { "word_to_vec": create_or_load_slim_w2v(words_list), "embedding_dim": embedding_dim } else: raise ValueError("invalid data_type: {}".format(data_type)) # map data splits to torch datasets and iterators self.torch_datasets = { k: OnlineDataset(sentences, self.sent_func, self.sent_func_kwargs) for k, sentences in self.sentences.items() } self.torch_iterators = { k: DataLoader(dataset, batch_size=batch_size, shuffle=k == TRAIN) for k, dataset in self.torch_datasets.items() }