import pandas as pd

import numpy as np

from sentence_transformers import SentenceTransformer

import gensim

from gensim.utils import tokenize

from gensim.models import FastText

from gensim.models.phrases import Phrases, Phraser

from gensim.parsing.preprocessing import remove_stopwords, strip_punctuation, strip_non_alphanum

from sklearn.feature_extraction.text import TfidfVectorizer

from sklearn.metrics.pairwise import cosine_similarity

from keras.models import Sequential

from keras.layers import Dense, Activation, Embedding, Flatten, MaxPooling1D, GlobalMaxPool1D, Dropout, Conv1D

from keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint

from keras.losses import binary_crossentropy

from keras.optimizers import Adam

from keras.utils import to_categorical

from keras import models

# Config Variables

tfidf_based = True

from_scratch = True

FT_VECTOR_SIZE = 300

f = open('goemotions/data/emotions.txt')

lines = f.readlines()

emotions = {}

for idx, label in enumerate(lines):

emotions[idx] = label.strip()

emotions

dfTrain = pd.read_csv('goemotions/data/train.tsv', sep='\t', header=None, names=['text', 'label', 'rater'])

dfDev = pd.read_csv('goemotions/data/dev.tsv', sep='\t', header=None, names=['text', 'label', 'rater'])

dfTest = pd.read_csv('goemotions/data/test.tsv', sep='\t', header=None, names=['text', 'label', 'rater'])

data = pd.concat([dfTrain, dfTest, dfDev], ignore_index=True)

data.text = data.text.apply(lambda s: s.lower())

def fasttext_model_train(data, from_scratch):

# There is an option using the original fastext library in python where the sentence vecotr is available but can be formolated here simply by adding two more functions

# TF-IDF Using SK_learn # needs a list of lists for words and docs along with a fasttext 'model'

def tfidf_model_train(data):

return tfidf_feat, final_tf_idf

# @TODO new document should be concatenated to the DATA and then the tfidf matrix should be computed again we need another method

# to do so >>> https://stackoverflow.com/questions/40112373/how-to-classify-new-documents-with-tf-idf4444444466444

fastText_model = fasttext_model_train(data, from_scratch) # training fastext

dictionary, tfidf_model = tfidf_model_train(data["text"].to_list())

# Sum( tfidf[word] * wvector) / sum(tfidf(words))

# @TODO maybe later develop the BM25 alg.

def get_sentence_vector(row, sent_tokens, dictionary, tfidf_model, ft_model, tfidf_based=False):

return vec_sum / weight

from tqdm import tqdm

# Fasttext word embedding

def fasttext_embedding(model, data, dictionary, tfidf_model, tfidf_based):

return sentence_vectors

# fastext_raw = fasttext_embedding(fastText_model, data.text.to_list(), dictionary, tfidf_model,False)

fasttext_tfidf = fasttext_embedding(fastText_model, data.text.to_list(), dictionary, tfidf_model, True)

# data['fasttext_raw'] = fastext_raw

data['fasttext_tfidf'] = fasttext_tfidf

from sklearn.preprocessing import MultiLabelBinarizer

mlb = MultiLabelBinarizer()

x = mlb.fit_transform([tuple(int(x) for x in i.split(',')) for i in data.label.to_list()])

data['new_label'] = list(x)

len(data.new_label[0])

###Bert

modelBert = SentenceTransformer('monologg/bert-base-cased-goemotions-original')

def bert_embedding(sentences):

sentence_embeddings = modelBert.encode(sentences)

return sentence_embeddings

bert_features = bert_embedding(data.text.to_list())

fabeec = []

for i in range(len(bert_features)):

fabeec.append(bert_features[i].tolist() + fasttext_tfidf[i].tolist())

# imbalance

data['fabeec'] = fabeec

start = 0

end = dfTrain.shape[0]

dfTrain['fabeec'] = data.fabeec[start:end]

dfTrain['new_label'] = data.new_label[start:end]

start = dfTrain.shape[0]

end = start + dfTest.shape[0]

dfTest['fabeec'] = data.fabeec[start:end].values

dfTest['new_label'] = data.new_label[start:end].values

start = end

dfDev['fabeec'] = data.fabeec[start:].values

dfDev['new_label'] = data.new_label[start:].values

def get_nan_ids(df):

return to_drop

# remove nan for Train set

to_drop = get_nan_ids(dfTrain)

dfTrain = dfTrain.drop(to_drop)

print("Number of removed items in trainin set that contains Nan : {}, the removed inexes {}".format(len(to_drop),

to_drop))

# remove nan for dev test

to_drop = get_nan_ids(dfDev)

dfDev = dfDev.drop(to_drop)

print("Number of removed items in Dev set that contains Nan : {}, the removed inexes {}".format(len(to_drop), to_drop))

# remove nan for test data

to_drop = get_nan_ids(dfTest)

dfTest = dfTest.drop(to_drop)

print(

"Number of removed items in Test set that contains Nan : {}, the removed inexes {}".format(len(to_drop), to_drop))

from sklearn import tree

from sklearn.multioutput import MultiOutputClassifier

from sklearn.svm import LinearSVC

from sklearn.neighbors import KNeighborsClassifier

from sklearn.ensemble import RandomForestRegressor

from sklearn.metrics import precision_recall_fscore_support

from sklearn.model_selection import GridSearchCV

import xgboost as xgb

from sklearn.ensemble import RandomForestClassifier

def classifier_pipline(X_train, Y_train, X_test):

return preds

X_train = np.array(dfTrain.fabeec.to_list() + dfDev.fabeec.to_list())

X_test = np.array(dfTest.fabeec.to_list())

y_train = np.array(dfTrain.new_label.to_list() + dfDev.new_label.to_list())

y_test = np.array(dfTest.new_label.to_list())

np.savetxt('X_train.csv', X_train, delimiter=',')

np.savetxt('X_test.csv', X_test, delimiter=',')

np.savetxt('y_train.csv', y_train, delimiter=',')

np.savetxt('y_test.csv', y_test, delimiter=',')

y_pred = classifier_pipline(X_train, y_train, X_test)

def Calculate_metric(y_test, y_pred):

return results

met_results = []

for met in y_pred:

temp = Calculate_metric(y_test, met)

met_results.append(temp)

print(temp)

# CNN

X_train_cnn = X_train.reshape(X_train.shape[0], X_train.shape[1], 1)

y_train_cnn = y_train.reshape(y_train.shape[0], y_train.shape[1])

X_test_cnn = X_test.reshape(X_test.shape[0], X_test.shape[1], 1)

y_test_cnn = y_test.reshape(y_test.shape[0], y_test.shape[1], 1)

model = Sequential()

model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(1068, 1)))

model.add(Dropout(0.5))

model.add(MaxPooling1D(pool_size=2))

model.add(Conv1D(filters=32, kernel_size=3, activation='relu'))

model.add(Dropout(0.5))

model.add(MaxPooling1D(pool_size=2))

model.add(Flatten())

model.add(Dense(100, activation='relu'))

model.add(Dense(28, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['categorical_accuracy'])

callbacks = [

]

history = model.fit(X_train_cnn, y_train_cnn,

callbacks=callbacks)

y_pred = model.predict(X_test_cnn)

y_pred[y_pred > .3] = 1

y_pred[y_pred <= .3] = 0

print(y_pred)

print(y_test)

results = Calculate_metric(y_test, y_pred)

print(results)