# -*- coding: utf-8 -*-

"""

Created on Sat Apr 21 15:58:50 2018

@author: mgungor

dir = r'C:\Program Files\mingw-w64\x86_64-7.2.0-posix-seh-rt_v5-rev0\mingw64\bin'

import os

os.environ['PATH'].count(dir)

os.environ['PATH'].find(dir)

os.environ['PATH'] = dir + ';' + os.environ['PATH']

"""

import numpy as np # linear algebra

import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

import matplotlib.pyplot as plt

import seaborn as sns

import nltk

from nltk.corpus import stopwords

import string

import xgboost as xgb

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

from sklearn.decomposition import TruncatedSVD

from sklearn import ensemble, metrics, model_selection, naive_bayes

color = sns.color_palette()

%matplotlib inline

eng_stopwords = set(stopwords.words("english"))

pd.options.mode.chained_assignment = None

## Read the train and test dataset and check the top few lines ##

train_df = pd.read_csv("train.csv")

test_df = pd.read_csv("test.csv")

test_author = pd.read_csv("test_author.txt", header=None, names=["author"])

## Number of words in the text ##

train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x).split()))

test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x).split()))

## Number of unique words in the text ##

train_df["num_unique_words"] = train_df["text"].apply(lambda x: len(set(str(x).split())))

test_df["num_unique_words"] = test_df["text"].apply(lambda x: len(set(str(x).split())))

## Number of characters in the text ##

train_df["num_chars"] = train_df["text"].apply(lambda x: len(str(x)))

test_df["num_chars"] = test_df["text"].apply(lambda x: len(str(x)))

## Number of stopwords in the text ##

train_df["num_stopwords"] = train_df["text"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))

test_df["num_stopwords"] = test_df["text"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))

## Number of punctuations in the text ##

train_df["num_punctuations"] =train_df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

test_df["num_punctuations"] =test_df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

## Number of title case words in the text ##

train_df["num_words_upper"] = train_df["text"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

test_df["num_words_upper"] = test_df["text"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

## Number of title case words in the text ##

train_df["num_words_title"] = train_df["text"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

test_df["num_words_title"] = test_df["text"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

## Average length of the words in the text ##

train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))

test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))

## Prepare the data for modeling ###

author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}

train_y = train_df['author'].map(author_mapping_dict)

test_y = test_author['author'].map(author_mapping_dict)

train_id = train_df['id'].values

test_id = test_df['id'].values

### recompute the trauncated variables again ###

train_df["num_words"] = train_df["text"].apply(lambda x: len(str(x).split()))

test_df["num_words"] = test_df["text"].apply(lambda x: len(str(x).split()))

train_df["mean_word_len"] = train_df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))

test_df["mean_word_len"] = test_df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))

cols_to_drop = ['id', 'text']

train_X = train_df.drop(cols_to_drop+['author'], axis=1)

test_X = test_df.drop(cols_to_drop, axis=1)

def runXGB(train_X, train_y, test_X, test_y=None, test_X2=None, seed_val=0, child=1, colsample=0.3):

return pred_test_y, pred_test_y2, model

kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)

cv_scores = []

pred_full_test = 0

pred_train = np.zeros([train_df.shape[0], 3])

for dev_index, val_index in kf.split(train_X):

dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]

dev_y, val_y = train_y[dev_index], train_y[val_index]

pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0)

pred_full_test = pred_full_test + pred_test_y

pred_train[val_index,:] = pred_val_y

cv_scores.append(metrics.log_loss(val_y, pred_val_y))

break

print("cv scores : ", cv_scores)

### Plot the important variables ###

fig, ax = plt.subplots(figsize=(12,12))

xgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)

plt.show()

### Fit transform the tfidf vectorizer ###

tfidf_vec = TfidfVectorizer(stop_words='english', ngram_range=(1,3))

full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())

train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())

test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())

def runMNB(train_X, train_y, test_X, test_y, test_X2):

return pred_test_y, pred_test_y2, model

cv_scores = []

pred_full_test = 0

pred_train = np.zeros([train_df.shape[0], 3])

kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)

for dev_index, val_index in kf.split(train_X):

dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]

dev_y, val_y = train_y[dev_index], train_y[val_index]

pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)

pred_full_test = pred_full_test + pred_test_y

pred_train[val_index,:] = pred_val_y

cv_scores.append(metrics.log_loss(val_y, pred_val_y))

print("Mean cv score : ", np.mean(cv_scores))

pred_full_test = pred_full_test / 5.

n_comp = 20

svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')

svd_obj.fit(full_tfidf)

train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))

test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))

train_svd.columns = ['svd_word_'+str(i) for i in range(n_comp)]

test_svd.columns = ['svd_word_'+str(i) for i in range(n_comp)]

train_df = pd.concat([train_df, train_svd], axis=1)

test_df = pd.concat([test_df, test_svd], axis=1)

del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd

### Fit transform the count vectorizer ###

tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,3))

tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())

train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())

test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())

cv_scores = []

pred_full_test = 0

pred_train = np.zeros([train_df.shape[0], 3])

kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)

for dev_index, val_index in kf.split(train_X):

dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]

dev_y, val_y = train_y[dev_index], train_y[val_index]

pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)

pred_full_test = pred_full_test + pred_test_y

pred_train[val_index,:] = pred_val_y

cv_scores.append(metrics.log_loss(val_y, pred_val_y))

print("Mean cv score : ", np.mean(cv_scores))

pred_full_test = pred_full_test / 5.

# add the predictions as new features #

train_df["nb_cvec_eap"] = pred_train[:,0]

train_df["nb_cvec_hpl"] = pred_train[:,1]

train_df["nb_cvec_mws"] = pred_train[:,2]

test_df["nb_cvec_eap"] = pred_full_test[:,0]

test_df["nb_cvec_hpl"] = pred_full_test[:,1]

test_df["nb_cvec_mws"] = pred_full_test[:,2]

### Fit transform the tfidf vectorizer ###

tfidf_vec = CountVectorizer(ngram_range=(1,7), analyzer='char')

tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())

train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())

test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())

cv_scores = []

pred_full_test = 0

pred_train = np.zeros([train_df.shape[0], 3])

kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)

for dev_index, val_index in kf.split(train_X):

dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]

dev_y, val_y = train_y[dev_index], train_y[val_index]

pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)

pred_full_test = pred_full_test + pred_test_y

pred_train[val_index,:] = pred_val_y

cv_scores.append(metrics.log_loss(val_y, pred_val_y))

print("Mean cv score : ", np.mean(cv_scores))

pred_full_test = pred_full_test / 5.

# add the predictions as new features #

train_df["nb_cvec_char_eap"] = pred_train[:,0]

train_df["nb_cvec_char_hpl"] = pred_train[:,1]

train_df["nb_cvec_char_mws"] = pred_train[:,2]

test_df["nb_cvec_char_eap"] = pred_full_test[:,0]

test_df["nb_cvec_char_hpl"] = pred_full_test[:,1]

test_df["nb_cvec_char_mws"] = pred_full_test[:,2]

### Fit transform the tfidf vectorizer ###

tfidf_vec = TfidfVectorizer(ngram_range=(1,5), analyzer='char')

full_tfidf = tfidf_vec.fit_transform(train_df['text'].values.tolist() + test_df['text'].values.tolist())

train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())

test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist())

cv_scores = []

pred_full_test = 0

pred_train = np.zeros([train_df.shape[0], 3])

kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)

for dev_index, val_index in kf.split(train_X):

dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]

dev_y, val_y = train_y[dev_index], train_y[val_index]

pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)

pred_full_test = pred_full_test + pred_test_y

pred_train[val_index,:] = pred_val_y

cv_scores.append(metrics.log_loss(val_y, pred_val_y))

print("Mean cv score : ", np.mean(cv_scores))

pred_full_test = pred_full_test / 5.

# add the predictions as new features #

train_df["nb_tfidf_char_eap"] = pred_train[:,0]

train_df["nb_tfidf_char_hpl"] = pred_train[:,1]

train_df["nb_tfidf_char_mws"] = pred_train[:,2]

test_df["nb_tfidf_char_eap"] = pred_full_test[:,0]

test_df["nb_tfidf_char_hpl"] = pred_full_test[:,1]

test_df["nb_tfidf_char_mws"] = pred_full_test[:,2]

n_comp = 20

svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')

svd_obj.fit(full_tfidf)

train_svd = pd.DataFrame(svd_obj.transform(train_tfidf))

test_svd = pd.DataFrame(svd_obj.transform(test_tfidf))

train_svd.columns = ['svd_char_'+str(i) for i in range(n_comp)]

test_svd.columns = ['svd_char_'+str(i) for i in range(n_comp)]

train_df = pd.concat([train_df, train_svd], axis=1)

test_df = pd.concat([test_df, test_svd], axis=1)

del full_tfidf, train_tfidf, test_tfidf, train_svd, test_svd

cols_to_drop = ['id', 'text']

train_X = train_df.drop(cols_to_drop+['author'], axis=1)

test_X = test_df.drop(cols_to_drop, axis=1)

kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)

cv_scores = []

pred_full_test = 0

pred_train = np.zeros([train_df.shape[0], 3])

for dev_index, val_index in kf.split(train_X):

dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]

dev_y, val_y = train_y[dev_index], train_y[val_index]

pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0, colsample=0.7)

pred_full_test = pred_full_test + pred_test_y

pred_train[val_index,:] = pred_val_y

cv_scores.append(metrics.log_loss(val_y, pred_val_y))

break

print("cv scores : ", cv_scores)

out_df = pd.DataFrame(pred_full_test)

out_df.columns = ['EAP', 'HPL', 'MWS']

import itertools

from sklearn.metrics import confusion_matrix

### From http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py #

def plot_confusion_matrix(cm, classes,

plt.xlabel('Predicted label')

cnf_matrix = confusion_matrix(val_y, np.argmax(pred_val_y,axis=1))

np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix

plt.figure(figsize=(8,8))

plot_confusion_matrix(cnf_matrix, classes=['EAP', 'HPL', 'MWS'],

title='Confusion matrix of XGB, without normalization')

plt.show()

predicted_result = pd.DataFrame(data = out_df.idxmax(axis=1).tolist(), columns = ["author"])

author_mapping_dict = {'EAP':0, 'HPL':1, 'MWS':2}

predicted_result_2 = predicted_result['author'].map(author_mapping_dict)

from sklearn.metrics import f1_score, accuracy_score

print("f1 Score")

print(f1_score(test_y, predicted_result_2, average='macro'))

print("Accuracy")

print(accuracy_score(test_y, predicted_result_2))

fig, ax = plt.subplots(figsize=(12,12))

xgb.plot_importance(model, max_num_features=50, height=0.8, ax=ax)

plt.show()

#

principalComponents = train_X[["svd_char_3", "svd_char_1", "svd_char_5"]]

principalDf = pd.DataFrame(data = principalComponents.values

, columns = ['pca 1', 'pca 2','pca 3'])

author_id = pd.DataFrame(data = train_df.author.values, columns = ['author'])

finalDf = pd.concat([principalDf, author_id], axis = 1)

#Now 3-D gif

import pynamical

from pynamical import simulate, phase_diagram_3d

import pandas as pd, numpy as np, matplotlib.pyplot as plt, random, glob, os, IPython.display as IPdisplay

from PIL import Image

%matplotlib inline

title_font = pynamical.get_title_font()

label_font = pynamical.get_label_font()

save_folder = 'images/phase-animate'

import os

# set a filename, run the logistic model, and create the plot

gif_filename = '02-pan-rotate-logistic-phase-diagram'

working_folder = '{}/{}'.format(save_folder, gif_filename)

if not os.path.exists(working_folder):

os.makedirs(working_folder)

import matplotlib.pyplot as plt

from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure(figsize = (15,15))

ax = Axes3D(fig)

targets =np.unique(train_df.author.values)

for target in targets:

indicesToKeep = finalDf['author'] == target

ax.scatter(finalDf.loc[indicesToKeep, 'pca 1']

, finalDf.loc[indicesToKeep, 'pca 2']

, finalDf.loc[indicesToKeep, 'pca 3'])

ax.set_title('Authorship Attribution', fontsize = 20)

ax.legend(targets)

ax.grid()

# look straight down at the x-y plane to start off

ax.elev = 89.9

ax.azim = 270.1

ax.dist = 11.0

# sweep the perspective down and rotate to reveal the 3-D structure of the strange attractor

for n in range(0, 100):

if n > 19 and n < 23:

ax.set_xlabel('')

ax.set_ylabel('') #don't show axis labels while we move around, it looks weird

ax.elev = ax.elev-0.5 #start by panning down slowly

if n > 22 and n < 37:

ax.elev = ax.elev-1.0 #pan down faster

if n > 36 and n < 61:

ax.elev = ax.elev-1.5

ax.azim = ax.azim+1.1 #pan down faster and start to rotate

if n > 60 and n < 65:

ax.elev = ax.elev-1.0

ax.azim = ax.azim+1.1 #pan down slower and rotate same speed

if n > 64 and n < 74:

ax.elev = ax.elev-0.5

ax.azim = ax.azim+1.1 #pan down slowly and rotate same speed

if n > 73 and n < 77:

ax.elev = ax.elev-0.2

ax.azim = ax.azim+0.5 #end by panning/rotating slowly to stopping position

if n > 76: #add axis labels at the end, when the plot isn't moving around

ax.set_xlabel('tsne 1', fontsize = 15)

ax.set_ylabel('tsne 2', fontsize = 15)

ax.set_zlabel('tsne 3', fontsize = 15)

# add a figure title to each plot then save the figure to the disk

#fig.suptitle('Benchmarking Authorship Attribution', fontsize=16, x=0.5, y=0.85)

plt.savefig('{}/{}/img{:03d}.png'.format(save_folder, gif_filename, n), bbox_inches='tight')

# don't display the static plot

plt.close()

# load all the static images into a list then save as an animated gif

gif_filepath = '{}/{}.gif'.format(save_folder, gif_filename)

images = [Image.open(image) for image in glob.glob('{}/*.png'.format(working_folder))]

gif = images[0]

gif.info['duration'] = 10 #milliseconds per frame

gif.info['loop'] = 0 #how many times to loop (0=infinite)

gif.save(fp=gif_filepath, format='gif', save_all=True, append_images=images[1:])

IPdisplay.Image(url=gif_filepath)