# -*- coding: utf-8 -*-

"""fake_content_detector.ipynb

Automatically generated by Colaboratory.

Original file is located at

https://colab.research.google.com/drive/1bRmry7ZLL-WHKP0FDDdolvr-duKX4nbK

"""

# ignoring warnings to remove clutter

import warnings

warnings.filterwarnings('ignore')

# Commented out IPython magic to ensure Python compatibility.

import os

import json

import string

import numpy as np

import pandas as pd

from pandas.io.json import json_normalize

import matplotlib.pyplot as plt

import seaborn as sns

color = sns.color_palette()

!pip install gensim

!pip install lightgbm

import re

import nltk

nltk.download('stopwords')

from nltk.corpus import stopwords # stop words are, is, the etc. which are not needed for model

from nltk.stem.porter import PorterStemmer

from gensim.models import word2vec

from sklearn.manifold import TSNE

# %matplotlib inline

from plotly import tools # to install $ pip install plotly

import plotly.offline as py

py.init_notebook_mode(connected=True)

import plotly.graph_objs as go

from sklearn.tree import DecisionTreeClassifier

from sklearn import model_selection, preprocessing, metrics, ensemble, naive_bayes, linear_model

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

from sklearn.decomposition import TruncatedSVD

import lightgbm as lgb # to install $ pip install lightgbm

from PIL import Image

pd.options.mode.chained_assignment = None

pd.options.display.max_columns = 999

!wget https://raw.githubusercontent.com/manojknit/PySpark_Python-ML-Models/master/dataset/fake_real_dataset_spam_category_clickbait_toxicity_politafln_sentiment_stance.csv

from io import BytesIO

df = pd.read_csv('fake_real_dataset_spam_category_clickbait_toxicity_politafln_sentiment_stance.csv')

print(df.shape)

#print(df.describe())

df.head()

df.drop(['Unnamed: 0', 'Unnamed: 0.1', 'Unnamed: 0.1.1', 'ord_in_thread', 'replies_count', 'participants_count', 'country','likes', 'comments', 'site_url', 'language','content', 'ord_in_thread', 'uuid', 'crawled'], axis=1, inplace=True)

df.head()

# Fix for plotly

##only for colab

def configure_plotly_browser_state():

'''))

configure_plotly_browser_state() # should be called in each cell https://stackoverflow.com/questions/47230817/plotly-notebook-mode-with-google-colaboratory

# Handel if title or news content is blank

import math

def title_column(tuple1):

return re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", tuple1[0])

def text_column(tuple1):

return re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", tuple1[1])

df['title'] = df[['title', 'text']].apply(title_column, axis=1)

df['text'] = df[['title', 'text' ]].apply(text_column, axis=1)

nltk.download('punkt')

from nltk import word_tokenize

import pdb

def CleaningText(txt):

return review

temp_published = df['published'].apply(lambda x: x[slice(10)])

df['published']=pd.to_datetime(temp_published,format="%Y-%M-%d")

#df['year']=df['published'].dt.year #df.year.unique()

# to check missing values

sns.heatmap(df.isnull(),yticklabels=False,cbar=False,cmap='viridis')

#dft = df.loc[:, ['domain_rank', 'shares', 'type']]

#print(dft.shape)

# clean text

df['text'] = df['text'].apply(lambda x: CleaningText(x))

df.text.fillna(df.title, inplace=True)

df.columns

from sklearn.preprocessing import LabelEncoder

lb_encode = LabelEncoder()

#df['type_num']= lb_encode.fit_transform(df['type'])

df['category_factor_num'] = lb_encode.fit_transform(df['category_factor'])

#df['stance_factor_num'] = lb_encode.fit_transform(df['stance'])

df['stance_factor_num'] = 0

df.loc[df.stance.str.contains('discuss'),'stance_factor_num'] = 0.3

df.loc[df.stance.str.contains('unrelated'),'stance_factor_num'] = 1.0

df.loc[df.stance.str.contains('agree'),'stance_factor_num'] = 0

df.loc[df.stance.str.contains('disagree'),'stance_factor_num'] = 0.8

df.head()

df.fake.unique()

tc = df.corr() #shows corelation in matrix form

tc

plt.figure(figsize = (16,5))

sns.heatmap(tc, annot=True, cmap='coolwarm')

dk= df[[ 'shares','domain_rank','spam_score_fector','fake','category_factor_num','click_bait_score','toxicity_factor','src_url_polarity','stance','sentiment_score']]

dk.head()

sns.pairplot(data=dk, hue="fake", dropna='true')

#plotly fix

configure_plotly_browser_state()

## target count ##

cnt_srs = df['type'].value_counts()

trace = go.Bar(

)

layout = go.Layout(

title='Target Count',

font=dict(size=18)

)

data = [trace]

fig = go.Figure(data=data, layout=layout)

py.iplot(fig, filename="TargetCount")

## target distribution ##

labels = (np.array(cnt_srs.index))

sizes = (np.array((cnt_srs / cnt_srs.sum())*100))

trace = go.Pie(labels=labels, values=sizes)

layout = go.Layout(

)

data = [trace]

fig = go.Figure(data=data, layout=layout)

py.iplot(fig, filename="usertype")

!pip install wordcloud

import requests

from wordcloud import WordCloud, STOPWORDS #install $ pip install wordcloud

# Thanks : https://www.kaggle.com/aashita/word-clouds-of-various-shapes ##

def plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0),

plt.tight_layout()

#Word cloud for real words

response = requests.get('https://raw.githubusercontent.com/manojknit/Natural_Language_Processing/master/images/upvote.png')

upvote_mask = np.array(Image.open(BytesIO(response.content)))#https://raw.githubusercontent.com/manojknit/Natural_Language_Processing/master/images/upvote.png

plot_wordcloud(df[df["fake"]==0]["text"], upvote_mask, max_words=300000, max_font_size=300, title="Word Cloud of Questions")

#Word cloud for Fake words

response = requests.get('https://image.freepik.com/free-icon/thumbs-down-silhouette_318-41911.jpg')

upvote_mask = np.array(Image.open(BytesIO(response.content)))#https://raw.githubusercontent.com/manojknit/Natural_Language_Processing/master/images/upvote.png

plot_wordcloud(df[df["fake"]==1]["text"], upvote_mask, max_words=300000, max_font_size=300, title="Word Cloud of Questions")

"""Word Frequency plot of real and fake news:"""

from collections import defaultdict

df_real = df[df["type"]=='news']

df_fake = df[df["type"]!='news']

## custom function for ngram generation ##

def generate_ngrams(text, n_gram=1):

token = [token for token in text.lower().split(" ") if token != "" if token not in STOPWORDS]

ngrams = zip(*[token[i:] for i in range(n_gram)])

return [" ".join(ngram) for ngram in ngrams]

## custom function for horizontal bar chart ##

def horizontal_bar_chart(df, color):

return trace

# for colab plotly

configure_plotly_browser_state()

## Get the bar chart from sincere questions ##

freq_dict = defaultdict(int)

for sent in df_real["text"]:

for word in generate_ngrams(sent, 1):

freq_dict[word] += 1

fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])

fd_sorted.columns = ["word", "wordcount"]

trace0 = horizontal_bar_chart(fd_sorted.head(50), 'blue')

## Get the bar chart from insincere questions ##

freq_dict = defaultdict(int)

for sent in df_fake["text"]:

for word in generate_ngrams(sent, 1):

freq_dict[word] += 1

fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])

fd_sorted.columns = ["word", "wordcount"]

trace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')

# Creating two subplots

fig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,

subplot_titles=["Frequent words of real news",

"Frequent words of fake news"])

fig.append_trace(trace0, 1, 1)

fig.append_trace(trace1, 1, 2)

fig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title="Word Count Plots")

py.iplot(fig, filename='word-plots')

#plt.figure(figsize=(10,16))

#sns.barplot(x="ngram_count", y="ngram", data=fd_sorted.loc[:50,:], color="b")

#plt.title("Frequent words for Insincere Questions", fontsize=16)

#plt.show()

"""Now let us also create bigram frequency plots for both the classes separately to get more idea."""

# for colab plotly

configure_plotly_browser_state()

freq_dict = defaultdict(int)

for sent in df_real["text"]:

for word in generate_ngrams(sent,2):

freq_dict[word] += 1

fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])

fd_sorted.columns = ["word", "wordcount"]

trace0 = horizontal_bar_chart(fd_sorted.head(50), 'orange')

freq_dict = defaultdict(int)

for sent in df_fake["text"]:

for word in generate_ngrams(sent,2):

freq_dict[word] += 1

fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])

fd_sorted.columns = ["word", "wordcount"]

trace1 = horizontal_bar_chart(fd_sorted.head(50), 'orange')

# Creating two subplots

fig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,horizontal_spacing=0.15,

subplot_titles=["Frequent bigrams of real news",

"Frequent bigrams of fake news"])

fig.append_trace(trace0, 1, 1)

fig.append_trace(trace1, 1, 2)

fig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title="Bigram Count Plots")

py.iplot(fig, filename='word-plots')

"""Now let's usl look at the trigram plots as well."""

# for colab plotly

configure_plotly_browser_state()

freq_dict = defaultdict(int)

for sent in df_real["text"]:

for word in generate_ngrams(sent,3):

freq_dict[word] += 1

fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])

fd_sorted.columns = ["word", "wordcount"]

trace0 = horizontal_bar_chart(fd_sorted.head(50), 'green')

freq_dict = defaultdict(int)

for sent in df_fake["text"]:

for word in generate_ngrams(sent,3):

freq_dict[word] += 1

fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])

fd_sorted.columns = ["word", "wordcount"]

trace1 = horizontal_bar_chart(fd_sorted.head(50), 'green')

# Creating two subplots

fig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04, horizontal_spacing=0.2,

subplot_titles=["Frequent trigrams of real news",

"Frequent trigrams of fake news"])

fig.append_trace(trace0, 1, 1)

fig.append_trace(trace1, 1, 2)

fig['layout'].update(height=1200, width=1200, paper_bgcolor='rgb(233,233,233)', title="Trigram Count Plots")

py.iplot(fig, filename='word-plots')

## Number of words in the text ##

df["num_words"] = df["text"].apply(lambda x: len(str(x).split()))

## Number of unique words in the text ##

df["num_unique_words"] = df["text"].apply(lambda x: len(set(str(x).split())))

## Number of characters in the text ##

df["num_chars"] = df["text"].apply(lambda x: len(str(x)))

## Number of stopwords in the text ##

df["num_stopwords"] = df["text"].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))

## Number of punctuations in the text ##

df["num_punctuations"] =df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

## Number of title case words in the text ##

df["num_words_upper"] = df["text"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

## Number of title case words in the text ##

df["num_words_title"] = df["text"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

## Average length of the words in the text ##

df["mean_word_len"] = df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))

"""## Number of words in the text ##

df["num_words"] = df["text"].apply(lambda x: len(str(x).split()))

## Number of unique words in the text ##

df["num_unique_words"] = df["text"].apply(lambda x: len(set(str(x).split())))

## Number of characters in the text ##

df["num_chars"] = df["text"].apply(lambda x: len(str(x)))

## Number of stopwords in the text ##

df["num_stopwords"] = df["text"].apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))

## Number of punctuations in the text ##

df["num_punctuations"] =df['text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )

## Number of title case words in the text ##

df["num_words_upper"] = df["text"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))

## Number of title case words in the text ##

df["num_words_title"] = df["text"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))

## Average length of the words in the text ##

df["mean_word_len"] = df["text"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))

"""

## Truncate some extreme values for better visuals ##

df['num_words'].loc[df['num_words']>60] = 60 #truncation for better visuals

df['num_punctuations'].loc[df['num_punctuations']>10] = 10 #truncation for better visuals

df['num_chars'].loc[df['num_chars']>350] = 350 #truncation for better visuals

f, axes = plt.subplots(3, 1, figsize=(10,20))

sns.violinplot(x='type', y='num_words', data=df, ax=axes[0])

axes[0].set_xlabel('Target', fontsize=12)

axes[0].set_title("Number of words in each class", fontsize=15)

sns.violinplot(x='type', y='num_chars', data=df, ax=axes[1])

axes[1].set_xlabel('Target', fontsize=12)

axes[1].set_title("Number of characters in each class", fontsize=15)

sns.violinplot(x='type', y='num_punctuations', data=df, ax=axes[2])

axes[2].set_xlabel('Target', fontsize=12)

#plt.ylabel('Number of punctuations in text', fontsize=12)

axes[2].set_title("Number of punctuations in each class", fontsize=15)

plt.show()

"""Visualizing Word Vectors"""

import nltk

import re

from gensim.models import word2vec

from sklearn.manifold import TSNE

def build_corpus(data):

return corpus

!unzip '/content/fake.csv.zip'

data=pd.read_csv('/content/fake.csv')

corpus = build_corpus(data)

def tsne_plot(model):

plt.show()

model = word2vec.Word2Vec(corpus, size=100, window=20, min_count=500, workers=4)

tsne_plot(model)

model.most_similar('trump')

tc = df.corr() #shows corelation in matrix form

plt.figure(figsize = (16,5))

sns.heatmap(tc, annot=True, cmap='coolwarm')

"""As per importance we can say our NLP non scalar features are quite valuable along with Word count and domain rank.

Encoding and Train Test Split

It seems domain rank, num words are very important features which can creating overfitting. Let's devide features in two model to get uniform representation of all features.

"""

# dropping columns which are not relevant or similar columns.

X = df[[ 'spam_score_fector','click_bait_score', 'category_factor_num', 'toxicity_factor','src_url_polarity','sentiment_score','stance_factor_num']]

#title text language site_url country domain_rank thread_title spam_score main_img_url shares type stance_factor_num' spam_score_fector category_factor fake click_bait_score toxicity_factor src_url_polarity type_num

y = df['fake']

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.25, random_state = 100)

# dropping columns which are not relevant or similar columns.

Xr = df[[ 'domain_rank','num_words']]

# target

yr = df['fake']

Xr_train, Xr_test, yr_train, yr_test = train_test_split(Xr, yr, test_size=.25, random_state = 100)

"""Model

We will run only models which can help to drive the importance of the factors and weight for prediction equation. We will visualize equation at the end as per model driven weight. We believe model driven equation will be more accurate.

Model Validation Metrics

ROC

Receiver Operating Characteristic (ROC) metric is used to evaluate the classifier's output quality.ROC curves typically feature true positive rate on the Y axis, and false positive rate on the X axis.

This means that the top left corner of the plot is the “ideal” point - a false positive rate of zero, and a true positive rate of one. This is not very realistic, but it does mean that a larger area under the curve (AUC) is usually better.

The “steepness” of ROC curves is also important, since it is ideal to maximize the true positive rate while minimizing the false positive rate.

Precision-Recall

Precision-Recall is a useful measure of success of prediction when the classes are very imbalanced. In information retrieval, precision is a measure of result relevancy, while recall is a measure of how many truly relevant results are returned.

The precision-recall curve shows the tradeoff between precision and recall for different threshold. A high area under the curve represents both high recall and high precision, where high precision relates to a low false positive rate, and high recall relates to a low false negative rate. High scores for both show that the classifier is returning accurate results (high precision), as well as returning a majority of all positive results (high recall).

The precision is the ratio tp / (tp + fp) where tp is the number of true positives and fp the number of false positives. The precision is intuitively the ability of the classifier not to label as positive a sample that is negative.

The recall is the ratio tp / (tp + fn) where tp is the number of true positives and fn the number of false negatives. The recall is intuitively the ability of the classifier to find all the positive samples.

"""

from sklearn.metrics import roc_curve, roc_auc_score, auc

# Function to get roc curve

def get_roc (y_test,y_pred):

return

from sklearn.metrics import average_precision_score, precision_recall_curve

# Function to get Precision recall curve

def get_prec_recall (y_test,y_pred):

return

from sklearn.linear_model import LogisticRegression

logmodel = LogisticRegression()

logmodel.fit(X_train,y_train)

predictions = logmodel.predict(X_test)

#Getting feature importances

print(logmodel.coef_)

from sklearn.metrics import classification_report

print(classification_report(y_test,predictions))

# Applying k-Fold Cross Validation

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = logmodel, X = X_train, y = y_train, cv = 10)

accuracies.mean()

from sklearn.metrics import confusion_matrix

#print(confusion_matrix(y_test,predictions))

cnf_matrix_logreg = metrics.confusion_matrix(y_test, predictions)

# create heatmap

sns.heatmap(pd.DataFrame(cnf_matrix_logreg), annot=True, cmap="YlGnBu" ,fmt='g')

plt.tight_layout()

plt.title('Confusion matrix for Logistic Regression', y=1.1)

plt.ylabel('Actual label')

plt.xlabel('Predicted label')

print("Model Accuracy for Logistic Regression:",metrics.accuracy_score(y_test, predictions))

#from sklearn.metrics import accuracy_score

from sklearn.metrics import accuracy_score

print ("Accuracy : ", accuracy_score(y_test,predictions)*100)

#MAE L1 loss function - Should be close to 0

from sklearn.metrics import mean_absolute_error

mean_absolute_error(y_test,predictions) #y_target, y_pred

#MAE L2 loss function - Should be close to 0

from sklearn.metrics import mean_squared_error

mean_squared_error(y_test,predictions) #y_target, y_pred

# Log Loss - Should be close to 0 - Only for classification models

from sklearn.metrics import log_loss

log_loss(y_test,predictions)

# Get ROC curve for Logistic Regression

get_roc(y_test,predictions)

get_prec_recall(y_test,predictions)

"""Logistic Regression Model evaluation based on K-fold cross-validation using cross_validate() function"""

from sklearn.model_selection import cross_validate

scoring = {'accuracy': 'accuracy', 'log_loss': 'neg_log_loss', 'auc': 'roc_auc'}

results = cross_validate(logmodel, X, y, cv=10, scoring=list(scoring.values()),

return_train_score=False)

print('K-fold cross-validation results:')

for sc in range(len(scoring)):

print(logmodel.__class__.__name__+" average %s: %.3f (+/-%.3f)" % (list(scoring.keys())[sc], -results['test_%s' % list(scoring.values())[sc]].mean()

if list(scoring.values())[sc]=='neg_log_loss'

else results['test_%s' % list(scoring.values())[sc]].mean(),

results['test_%s' % list(scoring.values())[sc]].std()))

from sklearn.naive_bayes import GaussianNB

classifier = GaussianNB()

classifier.fit(X_train, y_train)

# Predicting the Test set results

y_pred = classifier.predict(X_test)

from sklearn.metrics import classification_report

print(classification_report(y_test,y_pred))

# Applying k-Fold Cross Validation

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10)

accuracies.mean()

# Making the Confusion Matrix

from sklearn.metrics import confusion_matrix

#print(confusion_matrix(y_test,predictions))

cnf_matrix_logreg = metrics.confusion_matrix(y_test, y_pred)

# create heatmap

sns.heatmap(pd.DataFrame(cnf_matrix_logreg), annot=True, cmap="YlGnBu" ,fmt='g')

plt.tight_layout()

plt.title('Confusion matrix for Naive Bayes', y=1.1)

plt.ylabel('Actual label')

plt.xlabel('Predicted label')

print("Model Accuracy for Naive Bayes:",metrics.accuracy_score(y_test, y_pred))

#from sklearn.metrics import accuracy_score

from sklearn.metrics import accuracy_score

print ("Accuracy : ", accuracy_score(y_test,y_pred)*100)

#MAE L1 loss function - Should be close to 0

from sklearn.metrics import mean_absolute_error

mean_absolute_error(y_test,y_pred) #y_target, y_pred

#MAE L2 loss function - Should be close to 0

from sklearn.metrics import mean_squared_error

mean_squared_error(y_test,y_pred) #y_target, y_pred

# Log Loss - Should be close to 0 - Only for classification models

from sklearn.metrics import log_loss

log_loss(y_test,y_pred)

# Get ROC curve for Naive Bayes

get_roc(y_test,y_pred)

get_prec_recall(y_test,y_pred)

# Applying k-Fold Cross Validation for test set

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = logmodel, X = X_test, y = y_test, cv = 10)

accuracies.mean()

from sklearn.model_selection import cross_validate

scoring = {'accuracy': 'accuracy', 'log_loss': 'neg_log_loss', 'auc': 'roc_auc'}

results = cross_validate(classifier, X_train, y_train, cv=10, scoring=list(scoring.values()),

return_train_score=False)

print('K-fold cross-validation results:')

for sc in range(len(scoring)):

print(classifier.__class__.__name__+" average %s: %.3f (+/-%.3f)" % (list(scoring.keys())[sc], -results['test_%s' % list(scoring.values())[sc]].mean()

if list(scoring.values())[sc]=='neg_log_loss'

else results['test_%s' % list(scoring.values())[sc]].mean(),

results['test_%s' % list(scoring.values())[sc]].std()))

decclassifier = DecisionTreeClassifier(criterion ='entropy')

decclassifier.fit(X_train, y_train)

y_pred = decclassifier.predict(X_test)

decclassifier.feature_importances_

feats = {} # a dict to hold feature_name: feature_importance

for feature, importance in zip(X.columns, decclassifier.feature_importances_):

feats[feature] = importance #add the name/value pair

importances = pd.DataFrame.from_dict(feats, orient='index').rename(columns={0: 'Gini-importance'})

importances.sort_values(by='Gini-importance').plot(kind='bar', rot=45)

#Validation

from sklearn.metrics import confusion_matrix

confusion_matrix(y_test, y_pred)

from sklearn.metrics import classification_report

print(classification_report(y_test,y_pred))

# Applying k-Fold Cross Validation

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = decclassifier, X = X_train, y = y_train, cv = 10)

accuracies.mean()

# Making the Confusion Matrix

from sklearn.metrics import confusion_matrix

#print(confusion_matrix(y_test,predictions))

cnf_matrix_dectree = metrics.confusion_matrix(y_test, y_pred)

# create heatmap

sns.heatmap(pd.DataFrame(cnf_matrix_dectree), annot=True, cmap="YlGnBu" ,fmt='g')

plt.tight_layout()

plt.title('Confusion matrix for Decision Tree', y=1.1)

plt.ylabel('Actual label')

plt.xlabel('Predicted label')

print("Model Accuracy for Decision tree:",metrics.accuracy_score(y_test, y_pred))

#from sklearn.metrics import accuracy_score

from sklearn.metrics import accuracy_score

print ("Accuracy : ", accuracy_score(y_test,y_pred)*100)

#MAE L1 loss function - Should be close to 0

from sklearn.metrics import mean_absolute_error

mean_absolute_error(y_test,y_pred) #y_target, y_pred

#MAE L2 loss function - Should be close to 0

from sklearn.metrics import mean_squared_error

mean_squared_error(y_test,y_pred) #y_target, y_pred

# Log Loss - Should be close to 0 - Only for classification models

from sklearn.metrics import log_loss

log_loss(y_test,y_pred)

# Get ROC curve for Decision Tree

get_roc(y_test,y_pred)

get_prec_recall(y_test,y_pred)

# Applying k-Fold Cross Validation

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = decclassifier, X = X_train, y = y_train, cv = 10)

accuracies.mean()

from sklearn.model_selection import cross_validate

scoring = {'accuracy': 'accuracy', 'log_loss': 'neg_log_loss', 'auc': 'roc_auc'}

results = cross_validate(decclassifier, X_train, y_train, cv=10, scoring=list(scoring.values()),

return_train_score=False)

print('K-fold cross-validation results:')

for sc in range(len(scoring)):

print(decclassifier.__class__.__name__+" average %s: %.3f (+/-%.3f)" % (list(scoring.keys())[sc], -results['test_%s' % list(scoring.values())[sc]].mean()

if list(scoring.values())[sc]=='neg_log_loss'

else results['test_%s' % list(scoring.values())[sc]].mean(),

results['test_%s' % list(scoring.values())[sc]].std()))

from sklearn.ensemble import RandomForestClassifier

rfc = RandomForestClassifier(n_estimators=1000)

rfc.fit(X_train, y_train)

rfc_pred = rfc.predict(X_test)

rfc.feature_importances_

feats = {} # a dict to hold feature_name: feature_importance

for feature, importance in zip(X.columns, rfc.feature_importances_):

feats[feature] = importance #add the name/value pair

importances = pd.DataFrame.from_dict(feats, orient='index').rename(columns={0: 'Gini-importance'})

ax = importances.sort_values(by='Gini-importance').plot(kind='bar', rot=45)

for p in ax.patches:

ax.annotate(str(np.round(p.get_height(),decimals=2)), (p.get_x(), p.get_height()))

print(confusion_matrix(y_test,rfc_pred))

print(classification_report(y_test,rfc_pred))

#from sklearn.metrics import accuracy_score

print ("Accuracy : ", metrics.accuracy_score(y_test,rfc_pred)*100 )

#MAE L1 loss function - Should be close to 0

from sklearn.metrics import mean_absolute_error

mean_absolute_error(y_test,rfc_pred) #y_target, y_pred

#MAE L2 loss function - Should be close to 0

from sklearn.metrics import mean_squared_error

mean_squared_error(y_test,rfc_pred) #y_target, y_pred

# Log Loss - Should be close to 0 - Only for classification models

from sklearn.metrics import log_loss

log_loss(y_test,rfc_pred)

# Making the Confusion Matrix

from sklearn.metrics import confusion_matrix

#print(confusion_matrix(y_test,predictions))

cnf_matrix_rf = metrics.confusion_matrix(y_test, rfc_pred)

# create heatmap

sns.heatmap(pd.DataFrame(cnf_matrix_rf), annot=True, cmap="YlGnBu" ,fmt='g')

plt.tight_layout()

plt.title('Confusion matrix for Random Forest', y=1.1)

plt.ylabel('Actual label')

plt.xlabel('Predicted label')

print("Model Accuracy for Decision tree:",metrics.accuracy_score(y_test, y_pred) * 100)

get_roc(y_test,rfc_pred)

get_prec_recall(y_test,rfc_pred)

# Applying k-Fold Cross Validation

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = rfc, X = X_train, y = y_train, cv = 10)

accuracies.mean()

from sklearn.model_selection import cross_validate

scoring = {'accuracy': 'accuracy', 'log_loss': 'neg_log_loss', 'auc': 'roc_auc'}

results = cross_validate(rfc, X_train, y_train, cv=10, scoring=list(scoring.values()),

return_train_score=False)

print('K-fold cross-validation results:')

for sc in range(len(scoring)):

print(rfc.__class__.__name__+" average %s: %.3f (+/-%.3f)" % (list(scoring.keys())[sc], -results['test_%s' % list(scoring.values())[sc]].mean()

if list(scoring.values())[sc]=='neg_log_loss'

else results['test_%s' % list(scoring.values())[sc]].mean(),

results['test_%s' % list(scoring.values())[sc]].std()))

# Fitting Kernel SVM to the Training set

from sklearn.svm import SVC

svcclassifier = SVC(kernel = 'rbf', random_state = 0, gamma=0.8, C=100, probability=True)

svcclassifier.fit(X_train, y_train)

svc_pred = svcclassifier.predict(X_test)

#print (svcclassifier.get_feature_names())

print(classification_report(y_test,svc_pred))

#from sklearn.metrics import accuracy_score

print ("Accuracy : ", accuracy_score(y_test,svc_pred)*100)

#MAE L1 loss function - Should be close to 0

from sklearn.metrics import mean_absolute_error

mean_absolute_error(y_test,svc_pred) #y_target, y_pred

#MAE L2 loss function - Should be close to 0

from sklearn.metrics import mean_squared_error

mean_squared_error(y_test,svc_pred) #y_target, y_pred

# Log Loss - Should be close to 0 - Only for classification models

from sklearn.metrics import log_loss

log_loss(y_test,svc_pred)

# Making the Confusion Matrix

cm = confusion_matrix(y_test, svc_pred)

#print(cm)

# create heatmap

sns.heatmap(pd.DataFrame(cm), annot=True, cmap="YlGnBu" ,fmt='g')

plt.tight_layout()

plt.title('Confusion matrix for SVM', y=1.1)

plt.ylabel('Actual label')

plt.xlabel('Predicted label')

print("Model Accuracy for SVM:",metrics.accuracy_score(y_test, svc_pred) * 100)

get_roc(y_test,svc_pred)

get_prec_recall(y_test,svc_pred)

# Applying k-Fold Cross Validation

from sklearn.model_selection import cross_val_score

accuracies = cross_val_score(estimator = svcclassifier, X = X_train, y = y_train, cv = 10)

accuracies.mean()

from sklearn.model_selection import cross_validate

scoring = {'accuracy': 'accuracy', 'log_loss': 'neg_log_loss', 'auc': 'roc_auc'}

results = cross_validate(svcclassifier, X_train, y_train, cv=10, scoring=list(scoring.values()),

return_train_score=False)

print('K-fold cross-validation results:')

for sc in range(len(scoring)):

print(svcclassifier.__class__.__name__+" average %s: %.3f (+/-%.3f)" % (list(scoring.keys())[sc], -results['test_%s' % list(scoring.values())[sc]].mean()

if list(scoring.values())[sc]=='neg_log_loss'

else results['test_%s' % list(scoring.values())[sc]].mean(),

results['test_%s' % list(scoring.values())[sc]].std()))

# Applying PCA

from sklearn.decomposition import PCA

pca = PCA(n_components = 5)

X_train_pca = pca.fit_transform(X_train)

X_test_pca = pca.transform(X_test)

explained_variance = pca.explained_variance_ratio_

from sklearn.neighbors import KNeighborsClassifier

knnclassifier = KNeighborsClassifier(n_neighbors = 5, metric = 'minkowski', p = 2)

knnclassifier.fit(X_train_pca, y_train)

# Predicting the Test set results

knn_pred = knnclassifier.predict(X_test_pca)

from sklearn.metrics import classification_report

print(classification_report(y_test,knn_pred))

#from sklearn.metrics import accuracy_score

print ("Accuracy : ", accuracy_score(y_test,knn_pred)*100)

#MAE L1 loss function - Should be close to 0

from sklearn.metrics import mean_absolute_error

mean_absolute_error(y_test,knn_pred) #y_target, y_pred

#MAE L2 loss function - Should be close to 0

from sklearn.metrics import mean_squared_error

mean_squared_error(y_test,knn_pred) #y_target, y_pred

# Log Loss - Should be close to 0 - Only for classification models

from sklearn.metrics import log_loss

log_loss(y_test,knn_pred)

# Making the Confusion Matrix

cm = confusion_matrix(y_test, knn_pred)

#print(cm)

# create heatmap

sns.heatmap(pd.DataFrame(cm), annot=True, cmap="YlGnBu" ,fmt='g')

plt.tight_layout()

plt.title('Confusion matrix for KNN', y=1.1)

plt.ylabel('Actual label')

plt.xlabel('Predicted label')

print("Model Accuracy for KNN:",metrics.accuracy_score(y_test, knn_pred) * 100)

get_roc(y_test,knn_pred)

get_prec_recall(y_test,knn_pred)

from sklearn.model_selection import cross_validate

scoring = {'accuracy': 'accuracy', 'log_loss': 'neg_log_loss', 'auc': 'roc_auc'}

results = cross_validate(knnclassifier, X_train, y_train, cv=10, scoring=list(scoring.values()),

return_train_score=False)

print('K-fold cross-validation results:')

for sc in range(len(scoring)):