from keras.layers import Bidirectional, Input, LSTM, Dense, Activation, Conv1D, Flatten, Embedding, GlobalMaxPooling1D, Dropout

from keras.layers import Add, Concatenate, Lambda, Reshape, Permute, Average, Layer, TimeDistributed, Multiply, GRU, BatchNormalization, CuDNNGRU, SpatialDropout1D, GlobalAveragePooling1D

#from keras.layers.embeddings import Embedding

from keras.preprocessing.sequence import pad_sequences

from keras import optimizers

from keras.models import Sequential, Model

import pandas as pd

import numpy as np

from keras.callbacks import Callback

from keras.preprocessing.text import Tokenizer, text_to_word_sequence

from sklearn.utils import shuffle

import pickle

from sklearn.model_selection import train_test_split

import re

from sklearn.utils import shuffle

import keras

import joblib

from keras.utils.vis_utils import plot_model

import keras.backend as K

from nltk.tokenize import sent_tokenize, word_tokenize

from collections import defaultdict

import tokenizer_util as tu

import os

from sklearn.metrics import roc_auc_score

from keras import regularizers

os.environ["CUDA_VISIBLE_DEVICES"]="1"

#nohup python -u han_classifier_tox_cnn_lstm.py > class_output.log &

TRAIN_FILE_PATH = 'train.csv'#'/data/train.csv'

TEST_FILE = 'test.csv'#'/data/test.csv'

TIME_STEPS = 300

BATCH_SIZE = 256

LEARNING_RATE = 0.001

DECAY = 0.001

EPOCH_SIZE = 100

TOKENIZER_FILE = 'tokenizer'

EMBEDDING_FILE = 'embedding'

TENSORFLOW_LOGDIR = 'logs'#'/output/tensorboard_logs'

MODEL_SAVE_PATH = 'models/best_model_new_reg_2.h5' #'/output/best_model.h5'

OUTPUT_FILENAME = 'sub_h_n_consolidated_filtered.csv'

SENTENCE = 300

WORDS = 1

df = pd.read_csv(TRAIN_FILE_PATH)

pred_cols = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']

df['total_classes'] = df['toxic']+df['severe_toxic']+df['obscene']+df['threat']+df['insult']+df['identity_hate']

comment_col = 'comment_text'

#df[comment_col] = df[comment_col].astype(str).apply(lambda x : x.replace("'", "").replace('"',''))

df[comment_col] = df[comment_col].apply(lambda x: re.sub('[0-9]','',x))

comment_list = df[comment_col].tolist()

n_classes = 1

tokenizer = joblib.load(TOKENIZER_FILE)

final_emb_matrix = joblib.load(EMBEDDING_FILE)

print('Total vocabulary is {0}'.format(final_emb_matrix.shape[0]))

train, test = train_test_split(df, test_size=0.10, random_state=42)

XVal = tokenizer.texts_to_sequences(test.astype(str)[comment_col].tolist())

def ys(dftox, predcols):

return ys

def ys_unified(dftox, predcols):

ys = dftox[predcols].values

return ys

YTrain = ys_unified(train, pred_cols)

YVal = ys_unified(test, pred_cols)

"""

Attention Layer with works follows the math from https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf

This layer only computes the weights, does not multiply the RNN output with the weights. This layer has to be

followed by a Multiply layer, followed by Reshape, followed by a Lambda for summing.

"""

class ATTNWORD(Layer):

return super(ATTNWORD, self).get_config()

"""

A attenion layer, built on the basis of https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf.

Takes care of all the atention compute, Takes array of input - Bidirectional RNN output and the TanH layer output.

Usage ATTNWORD_COMPLETE(1)([tanh_output, rnn_output])

"""

class ATTNWORD_COMPLETE(Layer):

return super(ATTNWORD_COMPLETE, self).get_config()

"""

This method creates a model with an input of word length, followed by embedding layer and finally GRU,

with output dim as passed in the argument.

"""

def get_word_attention(emb_matrix, word_length, optimizer, nclasses, gru_output_dim=128):

return model

"""

This method applies attention only at the word level. The last layer is a sigmoid layer with output of 1.

The output is going to be an array, the number of output is determined by n_classes.

Here the labels are assumed to be independent of each other and probability for each label is independently calculated

using dedicated Attention layer for each.

"""

def attention_words_only(emb_matrix, word_length, n_classes, trainable=True):

return model

def get_sentence_attention(word_model , word_length, sent_length, n_classes):

return model

def get_sentence_attention_combined_output(word_model , word_length, sent_length, n_classes):

return model

"""

This returns LSTM based model. There are 6 output classes, all soft sharing the parameters of a common network.

"""

def get_model_soft_sharing_lstm_singleoutput(emb_matrix, sentence_length, word_length, learning_rate=0.001, n_classes=1, decay=0.1, combined_model=False):

return model

class RocAucEvaluation(Callback):

print("\n ROC-AUC - epoch: %d - score: %.6f \n" % (epoch+1, score))

lookup_words = tu.get_word_reverse_lookup(tokenizer)

comment_list = train.astype(str)[comment_col].tolist()

xtrain = tu.pad_sentences(comment_list,SENTENCE,WORDS, tokenizer)

test_comment_list = test.astype(str)[comment_col].tolist()

xval = tu.pad_sentences(test_comment_list,SENTENCE,WORDS, tokenizer)

print (xval.shape)

# Callbacks are passed to the model fit the `callbacks` argument in `fit`,

# which takes a list of callbacks. You can pass any number of callbacks.

callbacks_list = [

]

model = get_model_soft_sharing_lstm_singleoutput(final_emb_matrix, SENTENCE, WORDS, learning_rate=LEARNING_RATE, n_classes=6, decay=DECAY, combined_model=False)

#plot_model(model,to_file='attn_model_multi_rework_sent_allclasses_bn.png')

tu.sentence_tokenizer("This is a great day!", 25)

def get_label_stat(y):

return total_count, y1, y0

total_count_train, y1, y0 = get_label_stat(YTrain[0])

print ('Training State - Total Records: {0}, Toxic percent: {1}, Normal percent: {2}'.format(total_count_train, y1, y0))

total_count_val, y1, y0 = get_label_stat(YVal[0])

print ('Validation State - Total Records: {0}, Toxic percent: {1}, Normal percent: {2}'.format(total_count_val, y1, y0))

#model.load_weights(MODEL_SAVE_PATH)

model.fit(xtrain,YTrain ,batch_size=BATCH_SIZE, epochs=EPOCH_SIZE, verbose=1, validation_data=(xval, YVal), shuffle=True, callbacks=callbacks_list)#, callbacks=callbacks_list

test_df = pd.read_csv(TEST_FILE)

test_df = tu.clean_up(test_df)

#test_df['comment_text'] = test_df['comment_text'].apply(lambda x: tu.replace_unknown_words_with_UNK(x, tokenizer))

test_comments = test_df.astype(str)['comment_text'].tolist()

xtrain = tu.pad_sentences(test_comments,SENTENCE,WORDS, tokenizer)

test_df.head()

predictions = model.predict(xtrain)

predicted_df = pd.DataFrame(columns=['id','toxic','severe_toxic','obscene','threat','insult','identity_hate'])

predicted_df['id'] = test_df['id']

for i, k in enumerate(pred_cols):

predicted_df[k] = predictions[:,i]

predicted_df.head()

predicted_df.to_csv(OUTPUT_FILENAME,index=False, header=True)