import pandas as pd

import scipy as sp

import numpy as np

import matplotlib.pyplot as plt

import seaborn as sns

import numpy.ma as ma

import datetime as dt

from sklearn.decomposition import PCA

from sklearn.preprocessing import scale, PolynomialFeatures

from sklearn import linear_model, datasets

from sklearn.cross_validation import train_test_split, cross_val_score

from sklearn.linear_model import LogisticRegression

from sklearn import metrics, grid_search

from sklearn.svm import SVC

from sklearn.metrics import roc_curve, auc

from sklearn.learning_curve import learning_curve

plt.style.use('ggplot')

#Functions to clean up data

def ConvPercent(x):

return float(x.strip('%').strip(' '))/100.0

def ConvertEmpYears(x):

return int(y[0])

def ConvertOwnership(x):

return np.nan

def ConvHist(x):

return 12.*(2015-x.year)+11-x.month

def ConvTerm(x):

return float(x.split(' ')[1])

def ConvLoanStatus(x, Trouble=[], Success=[]):

return 2

def IntQuartiles(x, q1=.25, q2=.5, q3=.75):

return 'Tier 4'

#Import data function

def GetData(cols, files):

return pd.concat(frames, ignore_index=True).dropna()

#Clean Data

def CleanData(df):

return df

#Plot the distribution of lost principal

def PlotLostPrin(df):

plt.show()

#Function that plots the portion of defaulted and successful loans in each interest rate quartile

def PlotFractions(df):

plt.show()

#This function plots the distribution of defaults versus successful loans for each of the

#seven features of interest in this dataset

def Visualize2(df):

plt.show()

#Grab relevant features and labels from the data set

def GetFeatures(df, RelFeats, conditions, polyorder=1):

return scale(PolynomialFeatures(degree=polyorder, include_bias=False).fit_transform(scale(temp[RelFeats]))), temp.loan_status.as_matrix(), temp.index

def GetReport(model, PlotROC, X_test, y_test):

return accur, ConfMat, Rep, roc_auc

#Function that fits a logistic regression model and summarizes results

def FitLogReg(features, labels, RelFeats,c=10, split=.2, Reports=False, PlotROC=False):

return model, accur, ConfMat, Rep, roc_auc

#Function that fits a logistic regression model and summarizes results

def FitSVMReg(features, labels, RelFeats, c=10, split=.2, Reports=False, PlotROC=False):

return model, accur, ConfMat, Rep, roc_auc

#Perform a gridsearch on a particular model, print reports

def GridSearchModel(features, labels, model=LogisticRegression(penalty='l2', class_weight='auto'), parameters = {'C':[ .01, .1, 10]}, Reports=False, PlotROC=False):

return clf, accur, ConfMat, Rep, roc_auc

#Function to plot learning curves

def PlotLearningCurves(model, features, labels):

return

#Backtest the model on Tier 4 loans

def BackTest(Feats, df, GSCV=False):

return pd.DataFrame(datmodels, columns=['AvePerLoss','AveLoss','DefaultRate'], index=['Before']+(ModNames)), pd.DataFrame(Full, columns=['DefaultRate','AverageInterest', 'AnticipatedReturn', 'ChangeInNumberofLoans'], index=['Before']+(ModNames))

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#Begin main part of the program

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#Define data location

file1='.Data/LoanStats3a.csv'

file2='./Data/LoanStats3b.csv'

file3='./Data/LoanStats3c.csv'

file4='./Data/LoanStats3d.csv'

files=[file1,file2,file3,file4]

#Interesting Columns

cols=[ 'loan_amnt', 'annual_inc', 'int_rate', 'emp_length', 'home_ownership', 'earliest_cr_line', 'revol_util', 'loan_status', 'term', 'total_pymnt_inv', 'funded_amnt_inv']

#Import Data

df=CleanData(GetData(cols, files))

#Summary of defaulted loans

print pd.Series([100.*df.groupby('loan_status').count().loan_amnt.ix['0b0']/float(df.loan_amnt.count()), df.funded_amnt_inv.sum()/10**9, df.groupby('loan_status').sum().Loss.ix['0b0']/10**6], index=['Default Percentage', 'Total Funded Loans (billions)', 'Total Lost Principal (millions'])

#Plot distribution of principal lossses

PlotLostPrin(df)

#Summary of loans by interest category

print pd.DataFrame([df.groupby('IntCat').mean().int_rate*100., 100.*df.groupby(['loan_status','IntCat']).count().int_rate.ix['0b0']/df.groupby('IntCat').count().int_rate, df.groupby(['loan_status','IntCat']).mean().Loss.ix['0b0']]).T

#Look at distribution of loans by interest rate category

PlotFractions(df)

#Visualize Data for 4th quartile

Visualize2(df)

#Get ready to fit classifier. First define features and labels

RelFeats=[ 'loan_amnt', 'annual_inc', 'int_rate', 'emp_length', 'home_ownership', 'earliest_cr_line', 'revol_util', 'term']

#Remove outlierts and non-completed loans

conditions=(df.annual_inc<200000) & (df.int_rate>.15) & (df.earliest_cr_line<500) & (df.earliest_cr_line>50) & (df.revol_util<1.0) & (df['loan_status']!=2) & (df.IntCat=='Tier 4')

features, labels, _=GetFeatures(df, RelFeats, conditions, polyorder=1)

feats={}

#Uncomment to fit without gridsearch

'''

#Fit Logistic Regression Model

model1, accur, ConfMat, Rep, roc_auc = FitLogReg(features, labels, RelFeats,c=10, split=.2, Reports=True, PlotROC=False)

feats['Logit']=[model1, RelFeats,1]

#Fit SVM Model

model2, accur, ConfMat, Rep, roc_auc = FitSVMReg(features, labels, RelFeats, c=10, split=.2, Reports=True, PlotROC=True)

feats['SVM']=[model2, RelFeats,1]

r1,r2=BackTest(feats, df)

print r1

print r2

'''

#Use grid search on logistic regression and SVM to search for best regularization parameter C

clf, accur, ConfMat, Rep, roc_auc= GridSearchModel(features, labels, model=LogisticRegression(penalty='l2', class_weight='auto'), parameters = {'C':[ 0.001,.01, .1, 10, 100]}, Reports=True, PlotROC=False)

feats['Logit']=[clf, RelFeats, 1]

clf2, accur, ConfMat, Rep, roc_auc= GridSearchModel(features, labels, model=SVC( class_weight='auto', probability=True), parameters = {'C':[ 0.001,.01, .1, 10, 100]}, Reports=True, PlotROC=False)

feats['SVM']=[clf2, RelFeats, 1]

#Now backtest the above results

r1,r2=BackTest(feats, df, GSCV=True)

print r1

print r2

#Plot learning curves

PlotLearningCurves(LogisticRegression(penalty='l2', class_weight='auto', C=10), features, labels)