def explore_categorical_imputation (self, variable): """ Compares the results from various imputation methods so that you can choose the best suited one # 1st chart => existing categories and avg target value # 2nd chart => missing value replaced by frequent category ; then plot a chart with target value # 3rd chart => missing value replaced by 'Missing' category ; then plot a chart with target value # 4th chart => missing value replaced by random distribution ; then plot a chart with target value """ df = self.__df__ c = variable printmd ('**<u>Missing Values :</u>**') print (' Number :', df[c].isnull().sum()) print (' Percentage :', df[c].isnull().mean()*100, '%') print () printmd(f'**<u>We have following options for Imputing the Missing Value for Categorical Variable, {c} :</u>**') print (' 1. Imputing missing values by Frequent Category' ) print (' 2. Imputing missing values by Missing Label' ) print (' 3. Imputing missing values by Randomly selected value' ) print () print ("Let's visualize the impact of each imputation and compare it with original distribution") print () printmd ('**<u>1. Original Distribution of all Categories :</u>**') plot_categories_with_target(df, c, target = self.__target__) printmd ('**<u>2. All Categories after Frequent Category Imputation :</u>**') # Frequent value print ('Look at the Distibution of Frequent Category and Missing Data. Are there some major differences') fig = plt.figure(figsize = (8,4)) ax = fig.add_subplot(111) value = df[c].mode().item() print ('\n\nMost Frequent Category: ', value) df[df[c] == value][self.__target__].plot(kind = 'kde', ax = ax, color = 'blue') # NA Value df[df[c].isnull()][self.__target__].plot(kind = 'kde', ax = ax, color = 'red') # Add the legend labels = ['Most Frequent category', 'with NA'] ax.legend(labels, loc = 'best') plt.show() df[c+'_freq'] = df[c].fillna(value) plot_categories_with_target(df, c+'_freq', target = self.__target__) print ("3. All Categories after Missing Label Imputation") value = 'Missing' df[c+'_miss'] = df[c].fillna(value) plot_categories_with_target(df, c+'_miss', target = self.__target__) print ("4. All Categories after Randomly Selected Value Imputation") temp = self.__random_category_imputation__(c) plot_categories_with_target(temp, c+'_random', target = self.__target__)
def eda_numerical_variable(self, variable): ''' Parameter: variable: pass the variable for which EDA is required provides basic statistcs, missing values, distribution, spread statistics, Q-Q plot, Box plot, outliers using IQR, various variable transformations''' c = variable s = self.__df__[variable] # 1. Basic Statistics print ('Total Number of observations : ', len(s)) print () print ('Datatype :', (s.dtype)) print () printmd ('**<u>5 Point Summary :</u>**') print (' Minimum :\t\t', s.min(), '\n 25th Percentile :\t', s.quantile(0.25), '\n Median :\t\t', s.median(), '\n 75th Percentile :\t', s.quantile(0.75), '\n Maximum :\t\t', s.max()) print () # 2. Missing values printmd ('**<u>Missing Values :</u>**') print (' Number :', s.isnull().sum()) print (' Percentage :', s.isnull().mean()*100, '%') # 3. Histogram printmd ('**<u>Variable distribution and Spread statistics :</u>**') sns.distplot(s.dropna(), hist = True, fit = norm, kde = True) plt.show() # 4. Spread Statistics print ('Skewness :' , s.skew()) print ('Kurtosis :', s.kurt()) print () # 5. Q-Q plot printmd ('**<u>Normality Check :</u>**') res = stats.probplot(s.dropna(), dist = 'norm', plot = plt) plt.show() # 6. Box plot to check the spread outliers print () printmd ('**<u>Box Plot and Visual check for Outlier :</u>**') sns.boxplot(s.dropna(), orient = 'v') plt.show() # 7. Get outliers. Here distance could be a user defined parameter which defaults to 1.5 print () printmd ('**<u>Outliers (using IQR):</u>**') IQR = np.quantile(s, .75) - np.quantile(s, .25) upper_boundary = np.quantile(s, .75) + 1.5 * IQR lower_boundary = np.quantile(s, .25) - 1.5 * IQR print (' Right end outliers :', np.sum(s>upper_boundary)) print (' Left end outliers :', np.sum(s < lower_boundary)) # 8. Various Variable Transformations print () printmd (f'**<u>Explore various transformations for {c}</u>**') print () print ('1. Logarithmic Transformation') s_log = np.log(s) normality_diagnostic(s_log) print ('2. Exponential Transformation') s_exp = np.exp(s) normality_diagnostic(s_exp) print ('3. Square Transformation') s_sqr = np.square(s) normality_diagnostic(s_sqr) print ('4. Square-root Transformation') s_sqrt = np.sqrt(s) normality_diagnostic(s_sqrt) print ('5. Box-Cox Transformation') s_boxcox, lambda_param = stats.boxcox(s) normality_diagnostic(s_boxcox) print ('Optimal Lambda for Box-Cox transformation is :', lambda_param ) print () print ('6. Yeo Johnson Transformation') s = s.astype('float') s_yeojohnson, lambda_param = stats.yeojohnson(s) normality_diagnostic(s_yeojohnson) print ('Optimal Lambda for Yeo Johnson transformation is :', lambda_param ) print ()
def eda_categorical_variable(self, variable, add_missing=False, add_rare=False, tol=0.05): """ """ c = variable df = self.__df__ s = self.__df__[variable] target = self.__target__ model = self.__model__ # 1. Basic Statistics printmd ('**<u>Basic Info :</u>**') print ('Total Number of observations : ', len(s)) print () # 2. Cardinality printmd ('**<u>Cardinality of the variable :</u>**') print ('Number of Distinct Categories (Cardinality): ', len(s.unique())) print ('Distinct Values : ', s.unique()) print () # 3. Missing Values printmd ('**<u>Missing Values :</u>**') nmiss = s.isnull().sum() print (' Number :', s.isnull().sum()) print (' Percentage :', s.isnull().mean()*100, '%') # 4. Plot Categories printmd ('**<u>Category Plots :</u>**') plot_categories(df, c) # 5. Plot Categories by including Missing Values if nmiss: printmd ('**<u>Category plot by including Missing Values**') plot_categories(df, c, add_missing = True) # 6. Plot categories by combining Rare label printmd ('**<u>Category plot by including missing (if any) and Rare labels**') print (f'Categories less than {tol} value are clubbed in Rare label') plot_categories(df, c, add_missing = True, add_rare = True) #7. Plot categories with target if target: printmd ('**<u>Category Plot and Mean Target value:</u>**') plot_categories_with_target(df, c, target) #8. Plot distribution of target variable for each categories if target: printmd ('**<u>Distribution of Target variable for all categories:</u>**') plot_target_with_categories(df, c, target)
def explore_categorical_imputation(self, variable): ''' Parameters: ----------- df :Dataset we are working on for Analysis. model : default is None. Most of the encoding methods can be used for both classification and regression problems. variables : list of all the categorical variables target : target variable if any target Methods for Imputation: method : 'Mode' 'Random_Imputation' 'Rare_Encding' 'constant' 'Frequency_Encoding' Returns: -------- plots/graph Histograms, KDE, CountPlots for categorical variables depecting their distribution, counts, corelations among themselves/the target before and after imputation for missing value are done via different methods(strategy) along with columns for imputed value. Compares the results from various imputation methods so that you can choose the best suited one # 1st chart => existing categories and avg target value # 2nd chart => missing value replaced by frequent category ; then plot a chart with target value # 3rd chart => missing value replaced by 'Missing' category ; then plot a chart with target value # 4th chart => missing value replaced by random distribution ; then plot a chart with target value ''' df = self.__df__ c = variable printmd('**<u>Missing Values :</u>**') print(' Number :', df[c].isnull().sum()) print(' Percentage :', df[c].isnull().mean() * 100, '%') print() printmd( f'**<u>We have following options for Imputing the Missing Value for Categorical Variable, {c} :</u>**' ) print(' 1. Imputing missing values by Frequent Category') print(' 2. Imputing missing values by Missing Label') print(' 3. Imputing missing values by Randomly selected value') print() print( "Let's visualize the impact of each imputation and compare it with original distribution" ) print() printmd('**<u>1. Original Distribution of all Categories :</u>**') plot_categories_with_target(df, c, target=self.__target__) printmd( '**<u>2. All Categories after Frequent Category Imputation :</u>**' ) # Frequent value print( 'Look at the Distibution of Frequent Category and Missing Data. Are there some major differences' ) fig = plt.figure(figsize=(8, 4)) ax = fig.add_subplot(111) value = df[c].mode().item() print('\n\nMost Frequent Category: ', value) df[df[c] == value][self.__target__].plot(kind='kde', ax=ax, color='blue') # NA Value df[df[c].isnull()][self.__target__].plot(kind='kde', ax=ax, color='red') # Add the legend labels = ['Most Frequent category', 'with NA'] ax.legend(labels, loc='best') plt.show() df[c + '_freq'] = df[c].fillna(value) plot_categories_with_target(df, c + '_freq', target=self.__target__) print("3. All Categories after Missing Label Imputation") value = 'Missing' df[c + '_miss'] = df[c].fillna(value) plot_categories_with_target(df, c + '_miss', target=self.__target__) print("4. All Categories after Randomly Selected Value Imputation") temp = self.__random_category_imputation__(c) plot_categories_with_target(temp, c + '_random', target=self.__target__)
def eda_categorical_variable(self, variable, add_missing=False, add_rare=False, tol=0.05): """ This function provides EDA for Categorical variable, this includes - Counts - Cardinality, number of Categories in each Varaible - Missing values counts and percentages Also Category wise basic plots will be generated for the given variable - Plot Categories - Plot Categories by including Missing Values - Plot categories by combining Rare label - Plot categories with target - Plot distribution of target variable for each categories (If Target Variable is passed) Parameters : ----------- variable: Pass the variable(s) for which EDA is required tol : Threshold limit to combine the rare occurrence categories, (tol=0.05) i.e., less than 5% occurance categories will be grouped and forms a rare category Optional Arguments : ------------------- target : Define the target variable, default None model : Specify the model either regression OR classification Return : ------- Returns summary & plots of given variable """ c = variable df = self.__df__ s = self.__df__[variable] target = self.__target__ model = self.__model__ # 1. Basic Statistics printmd ('**<u>Basic Info :</u>**') print ('Total Number of observations : ', len(s)) print () # 2. Cardinality printmd ('**<u>Cardinality of the variable :</u>**') print ('Number of Distinct Categories (Cardinality): ', len(s.unique())) print ('Distinct Values : ', s.unique()) print () # 3. Missing Values printmd ('**<u>Missing Values :</u>**') nmiss = s.isnull().sum() print (' Number :', s.isnull().sum()) print (' Percentage :', s.isnull().mean()*100, '%') # 4. Plot Categories printmd ('**<u>Category Plots :</u>**') plot_categories(df, c) # 5. Plot Categories by including Missing Values if nmiss: printmd ('**<u>Category plot by including Missing Values**') plot_categories(df, c, add_missing = True) # 6. Plot categories by combining Rare label printmd ('**<u>Category plot by including missing (if any) and Rare labels**') print (f'Categories less than {tol} value are clubbed in Rare label') plot_categories(df, c, add_missing = True, add_rare = True) #7. Plot categories with target if target: printmd ('**<u>Category Plot and Mean Target value:</u>**') plot_categories_with_target(df, c, target) #8. Plot distribution of target variable for each categories if target: printmd ('**<u>Distribution of Target variable for all categories:</u>**') plot_target_with_categories(df, c, target)
def explore_categorical_imputation(self, variable): """ In this function you just pass each variable one-by-one to explore the various missing value treatments Parameters: ----------- variables : single categorical variable Methods for Imputation: method : 'Mode' 'Random_Imputation' 'Rare_Encding' 'constant' 'Frequency_Encoding' Returns: -------- plots/graph Histograms, KDE, CountPlots for categorical variables depecting their distribution, counts, corelations among themselves/the target before and after imputation for missing value are done via different methods(strategy) along with columns for imputed value. Compares the results from various imputation methods so that you can choose the best suited one # 1st chart => existing categories and avg target value # 2nd chart => missing value replaced by frequent category ; then plot a chart with target value # 3rd chart => missing value replaced by 'Missing' category ; then plot a chart with target value # 4th chart => missing value replaced by random distribution ; then plot a chart with target value """ miss_df = self.__miss_df__ c = variable printmd('**<u>Missing Values :</u>**') n_miss = miss_df[c].isnull().sum() n_miss_perc = miss_df[c].isnull().mean() * 100 print(' Number :', n_miss) print(' Percentage :', n_miss_perc, '%') print() if n_miss == 0: print('No Missing values... ') print('\n Stopping the process') else: printmd( f'**<u>We have following options for Imputing the Missing Value for Categorical Variable, {c} :</u>**' ) print(' 1. Imputing missing values by Frequent Category') print(' 2. Imputing missing values by Missing Label') print(' 3. Imputing missing values by Randomly selected value') print() print( "Let's visualize the impact of each imputation and compare it with original distribution" ) print() printmd('**<u>1. Original Distribution of all Categories :</u>**') plot_categories_with_target(miss_df, c, target=self.__target__) printmd( '**<u>2. All Categories after Frequent Category Imputation :</u>**' ) # Frequent value print( 'Look at the Distibution of Frequent Category and Missing Data. Are there some major differences' ) if n_miss > 10: fig = plt.figure(figsize=(8, 4)) ax = fig.add_subplot(111) value = miss_df[c].mode().item() print('\n\nMost Frequent Category: ', value) miss_df[miss_df[c] == value][self.__target__].plot( kind='kde', ax=ax, color='blue') # NA Value miss_df[miss_df[c].isnull()][self.__target__].plot(kind='kde', ax=ax, color='red') # Add the legend labels = ['Most Frequent category', 'with NA'] ax.legend(labels, loc='best') plt.show() else: print( 'Not plotting the KDE plot because number of missing values is less than 10' ) miss_df[c + '_freq'] = miss_df[c].fillna(value) plot_categories_with_target(miss_df, c + '_freq', target=self.__target__) print("3. All Categories after Missing Label Imputation") value = 'Missing' miss_df[c + '_miss'] = miss_df[c].fillna(value) plot_categories_with_target(miss_df, c + '_miss', target=self.__target__) print("4. All Categories after Randomly Selected Value Imputation") if n_miss_perc < 50: temp = self.__random_category_imputation__(c) plot_categories_with_target(temp, c + '_random', target=self.__target__) else: print( 'Since more than 50% Missing Value... Random value transformation is not advisable ' )