def bow_baseline_naive_bayes(train_data, train_labels, test_data, test_labels): clf = sklearn.naive_bayes.MultinomialNB() cv = sklearn.feature_extraction.text.CountVectorizer() train_data = strip_triggerword(train_data) test_data = strip_triggerword(test_data) train = cv.fit_transform(train_data) test = cv.transform(test_data) clf.fit(train, train_labels) pred = clf.predict(test) print("\n## Bag of words (Naive Bayes) ##\n") evaluate_iest.calculatePRF(test_labels, pred.tolist())
def tfidf_baseline_svm(train_data, train_labels, test_data, test_labels): cv = sklearn.feature_extraction.text.TfidfVectorizer() clf = sklearn.svm.LinearSVC() # scaler = sklearn.preprocessing.StandardScaler(with_mean=False) # train_data = strip_triggerword(train_data) # test_data = strip_triggerword(test_data) train = cv.fit_transform(train_data) # train = scaler.fit_transform(train) test = cv.transform(test_data) # test = scaler.transform(test) clf.fit(train, train_labels) pred = clf.predict(test) print("\n## Bag of words tf-idf (Linear SVC) ##\n") evaluate_iest.calculatePRF(test_labels, pred.tolist())
def bigrams_svm(train_data, train_labels, test_data, test_labels): cv = sklearn.feature_extraction.text.CountVectorizer(ngram_range=(1, 2)) clf = sklearn.svm.LinearSVC() # scaler = sklearn.preprocessing.StandardScaler(with_mean=False) # train_data = strip_triggerword(train_data) # test_data = strip_triggerword(test_data) train = cv.fit_transform(train_data) # train = scaler.fit_transform(train) test = cv.transform(test_data) # test = scaler.transform(test) clf.fit(train, train_labels) pred = clf.predict(test) print("\n## Bag of uni- and bigrams (Linear SVC) ##\n") evaluate_iest.calculatePRF(test_labels, pred.tolist())
def test(args): # with keras.utils.CustomObjectScope({L1L2_m.__name__: L1L2_m}): model = keras.models.load_model("%s.h5" % args.model) with open("%s.maps" % args.model, encoding="utf-8") as f: word_to_idx, tgt_to_idx, max_len_lw, max_len_rw = json.load(f) idx_to_tgt = {i: c for c, i in tgt_to_idx.items()} test_lw, test_rw, test_tgt, _, _ = read_dataset(args.FILE) if args.lda: topics = topic_distribution(args.dict, args.lda, test_lw, test_rw) test_lw = [vectorize_words(lw, word_to_idx) for lw in test_lw] test_rw = [ vectorize_words(rw, word_to_idx, reverse=True) for rw in test_rw ] test_left_words = keras.preprocessing.sequence.pad_sequences( test_lw, maxlen=max_len_lw, padding="pre", truncating="pre") test_right_words = keras.preprocessing.sequence.pad_sequences( test_rw, maxlen=max_len_rw, padding="pre", truncating="pre") if args.lda: predictions = model.predict( [test_left_words, test_right_words, topics]) else: predictions = model.predict([test_left_words, test_right_words]) predicted = [idx_to_tgt[p] for p in predictions.argmax(axis=1)] evaluate_iest.calculatePRF(list(test_tgt), predicted)
def majority_baseline(train_labels, test_labels): train_freq = collections.Counter(train_labels) most_common = train_freq.most_common(1)[0][0] fake_prediction = [most_common] * len(test_labels) print("\n## Majority baseline ##\n") evaluate_iest.calculatePRF(test_labels, fake_prediction)
def retrain(args): BATCH_SIZE = 160 # with keras.utils.CustomObjectScope({L1L2_m.__name__: L1L2_m}): model = keras.models.load_model("%s.h5" % args.model) # model.optimizer.set_state() with open("%s.maps" % args.model, encoding="utf-8") as f: word_to_idx, tgt_to_idx, max_len_lw, max_len_rw = json.load(f) train_lw, train_rw, train_tgt, _, _ = read_dataset(args.FILE) val_lw, val_rw, val_tgt, _, _ = read_dataset(args.val) if args.lda: train_topics = topic_distribution(args.dict, args.lda, train_lw, train_rw) val_topics = topic_distribution(args.dict, args.lda, val_lw, val_rw) train_lw = [vectorize_words(lw, word_to_idx) for lw in train_lw] train_rw = [ vectorize_words(rw, word_to_idx, reverse=True) for rw in train_rw ] train_tgt = vectorize_words(train_tgt, tgt_to_idx) targets = keras.utils.to_categorical(train_tgt, num_classes=len(tgt_to_idx.values())) train_left_words = keras.preprocessing.sequence.pad_sequences( train_lw, maxlen=max_len_lw, padding="pre", truncating="pre") train_right_words = keras.preprocessing.sequence.pad_sequences( train_rw, maxlen=max_len_rw, padding="pre", truncating="pre") val_tgts = val_tgt val_lw = [vectorize_words(lw, word_to_idx) for lw in val_lw] val_rw = [vectorize_words(rw, word_to_idx, reverse=True) for rw in val_rw] val_tgt = vectorize_words(val_tgt, tgt_to_idx) val_targets = keras.utils.to_categorical(val_tgt, num_classes=len( tgt_to_idx.values())) val_left_words = keras.preprocessing.sequence.pad_sequences( val_lw, maxlen=max_len_lw, padding="pre", truncating="pre") val_right_words = keras.preprocessing.sequence.pad_sequences( val_rw, maxlen=max_len_rw, padding="pre", truncating="pre") early_stopper = keras.callbacks.EarlyStopping(monitor="val_loss", patience=2) if args.lda: model.fit( [train_left_words, train_right_words, train_topics], targets, batch_size=BATCH_SIZE, epochs=args.epochs, callbacks=[early_stopper], validation_data=([val_left_words, val_right_words, val_topics], val_targets)) else: model.fit([train_left_words, train_right_words], targets, batch_size=BATCH_SIZE, epochs=args.epochs, callbacks=[early_stopper], validation_data=([val_left_words, val_right_words], val_targets)) model.save("%s_retrain.h5" % args.model) with open("%s_retrain.maps" % args.model, mode="w", encoding="utf-8") as f: json.dump((word_to_idx, tgt_to_idx, max_len_lw, max_len_rw), f, ensure_ascii=False) # Official score if args.lda: predictions = model.predict( [val_left_words, val_right_words, val_topics]) else: predictions = model.predict([val_left_words, val_right_words]) idx_to_tgt = {i: c for c, i in tgt_to_idx.items()} predicted = [idx_to_tgt[p] for p in predictions.argmax(axis=1)] evaluate_iest.calculatePRF(list(val_tgts), predicted)
def train(args): WORD_LSTM_DIM = 300 DENSE_DIM = WORD_LSTM_DIM DROPOUT = 0.2 RECURRENT_DROPOUT = 0.0 BATCH_SIZE = 160 train_lw, train_rw, train_tgt, vocabulary, classes = read_dataset( args.FILE) val_lw, val_rw, val_tgt, _, _ = read_dataset(args.val) val_tgts = val_tgt embeddings_index, WORD_EMBEDDING_DIM = read_glove(args.embeddings) # mappings word_to_idx = {w: i for i, w in enumerate(sorted(vocabulary), start=1)} tgt_to_idx = {c: i for i, c in enumerate(sorted(classes), start=0)} # create embedding layers embedding_matrix = np.zeros((len(vocabulary) + 2, WORD_EMBEDDING_DIM)) for word, idx in word_to_idx.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: # words not found in embedding index will be all-zeros. embedding_matrix[idx] = embedding_vector # LDA models if args.lda: train_topics = topic_distribution(args.dict, args.lda, train_lw, train_rw) val_topics = topic_distribution(args.dict, args.lda, val_lw, val_rw) # vectorize train_lw = [vectorize_words(lw, word_to_idx) for lw in train_lw] train_rw = [ vectorize_words(rw, word_to_idx, reverse=True) for rw in train_rw ] train_tgt = vectorize_words(train_tgt, tgt_to_idx) targets = keras.utils.to_categorical(train_tgt, num_classes=len(classes)) val_lw = [vectorize_words(lw, word_to_idx) for lw in val_lw] val_rw = [vectorize_words(rw, word_to_idx, reverse=True) for rw in val_rw] val_tgt = vectorize_words(val_tgt, tgt_to_idx) val_targets = keras.utils.to_categorical(val_tgt, num_classes=len(classes)) # pad sequences max_len_lw = int( max((len(lw) for lw in itertools.chain(train_lw, val_lw))) * 1.1) max_len_rw = int( max((len(rw) for rw in itertools.chain(train_rw, val_rw))) * 1.1) train_left_words = keras.preprocessing.sequence.pad_sequences( train_lw, maxlen=max_len_lw, padding="pre", truncating="pre") train_right_words = keras.preprocessing.sequence.pad_sequences( train_rw, maxlen=max_len_rw, padding="pre", truncating="pre") val_left_words = keras.preprocessing.sequence.pad_sequences( val_lw, maxlen=max_len_lw, padding="pre", truncating="pre") val_right_words = keras.preprocessing.sequence.pad_sequences( val_rw, maxlen=max_len_rw, padding="pre", truncating="pre") # with keras.utils.CustomObjectScope({L1L2_m.__name__: L1L2_m}): # input layers input_lw = keras.layers.Input(shape=(train_left_words.shape[1], )) input_rw = keras.layers.Input(shape=(train_right_words.shape[1], )) input_topics = keras.layers.Input(shape=(100, )) # embedding layers embedding_lw = keras.layers.Embedding(len(vocabulary) + 2, WORD_EMBEDDING_DIM, mask_zero=True, weights=[embedding_matrix], trainable=False)(input_lw) embedding_rw = keras.layers.Embedding(len(vocabulary) + 2, WORD_EMBEDDING_DIM, mask_zero=True, weights=[embedding_matrix], trainable=False)(input_rw) # LSTMs # lstm_lw = keras.layers.LSTM(WORD_LSTM_DIM, dropout=DROPOUT, recurrent_dropout=RECURRENT_DROPOUT, # kernel_regularizer=L1L2_m(l1=0, l2=0, prior_shape=(WORD_EMBEDDING_DIM, WORD_LSTM_DIM * 4)))(embedding_lw) # lstm_rw = keras.layers.LSTM(WORD_LSTM_DIM, dropout=DROPOUT, recurrent_dropout=RECURRENT_DROPOUT, # kernel_regularizer=L1L2_m(l1=0, l2=0, prior_shape=(WORD_EMBEDDING_DIM, WORD_LSTM_DIM * 4)))(embedding_rw) lstm_lw = keras.layers.LSTM( WORD_LSTM_DIM, dropout=DROPOUT, recurrent_dropout=RECURRENT_DROPOUT)(embedding_lw) lstm_rw = keras.layers.LSTM( WORD_LSTM_DIM, dropout=DROPOUT, recurrent_dropout=RECURRENT_DROPOUT)(embedding_rw) # concatenate lstm_out = keras.layers.Concatenate(axis=1)([lstm_lw, lstm_rw]) # dense layers # dense01 = keras.layers.Dense(DENSE_DIM, activation="tanh", kernel_regularizer=L1L2_m(l1=0, l2=0, prior_shape=(LSTM_DIM, DENSE_DIM)))(lstm_out) if args.lda: if args.lda_mode == "feature": # TOPICS AS FEATURES # topics = keras.layers.Concatenate(axis=1)([lstm_out, input_topics]) dense01 = keras.layers.Dense(DENSE_DIM, activation="tanh")(topics) dropout01 = keras.layers.Dropout(DROPOUT)(dense01) predictions = keras.layers.Dense(len(classes), activation="softmax")(dropout01) elif args.lda_mode == "filter": # MULTIPLY WITH TOPICS # dense01 = keras.layers.Dense(DENSE_DIM, activation="tanh")(lstm_out) # dropout01 = keras.layers.Dropout(DROPOUT)(dense01) topic_dense = keras.layers.Dense( DENSE_DIM, activation="softmax")(input_topics) # element-wise multiplication with topics # topic_filter = keras.layers.Multiply()([dropout01, topic_dense]) topic_filter = keras.layers.Multiply()([dense01, topic_dense]) predictions = keras.layers.Dense( len(classes), activation="softmax")(topic_filter) else: # NOTHING # dense01 = keras.layers.Dense(DENSE_DIM, activation="tanh")(lstm_out) dropout01 = keras.layers.Dropout(DROPOUT)(dense01) predictions = keras.layers.Dense(len(classes), activation="softmax")(dropout01) # dense02 = keras.layers.Dense(DENSE_DIM // 2, activation="tanh", kernel_regularizer=L1L2_m(l1=0, l2=0, prior_shape=(LSTM_DIM, DENSE_DIM)))(dropout01) # predictions = keras.layers.Dense(len(classes), activation="softmax", kernel_regularizer=L1L2_m(l1=0, l2=0, prior_shape=(DENSE_DIM, len(classes))))(dropout02) # predictions = keras.layers.Dense(len(classes), activation="softmax")(dropout01) if args.lda: model = keras.models.Model(inputs=[input_lw, input_rw, input_topics], outputs=predictions) else: model = keras.models.Model(inputs=[input_lw, input_rw], outputs=predictions) model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]) model.summary() early_stopper = keras.callbacks.EarlyStopping(monitor="val_loss", patience=2) if args.lda: model.fit( [train_left_words, train_right_words, train_topics], targets, batch_size=BATCH_SIZE, epochs=args.epochs, callbacks=[early_stopper], validation_data=([val_left_words, val_right_words, val_topics], val_targets)) else: model.fit([train_left_words, train_right_words], targets, batch_size=BATCH_SIZE, epochs=args.epochs, callbacks=[early_stopper], validation_data=([val_left_words, val_right_words], val_targets)) model.save("%s.h5" % args.model) with open("%s.maps" % args.model, mode="w", encoding="utf-8") as f: json.dump((word_to_idx, tgt_to_idx, max_len_lw, max_len_rw), f, ensure_ascii=False) # Official score if args.lda: predictions = model.predict( [val_left_words, val_right_words, val_topics]) else: predictions = model.predict([val_left_words, val_right_words]) idx_to_tgt = {i: c for c, i in tgt_to_idx.items()} predicted = [idx_to_tgt[p] for p in predictions.argmax(axis=1)] evaluate_iest.calculatePRF(list(val_tgts), predicted)