def getStatisticsOfData (dataSet): print("***** Start checking the statistics of the dataSet *****\n") labelName = getLabelName() #Number of rows and columns in the dataset print("***** Shape (number of rows and columns) in the dataset: ", dataSet.shape) #Total number of features in the dataset numberOfColumnsInTheDataset = len(dataSet.columns) print("***** Total number of features in the dataset: ",numberOfColumnsInTheDataset) #Total number of categorical featuers in the dataset categoricalFeaturesInTheDataset = list(set(dataSet.columns) - set(dataSet._get_numeric_data().columns)) print("***** Number of categorical features in the dataset: ",len(categoricalFeaturesInTheDataset)) #Total number of numerical features in the dataset numericalFeaturesInTheDataset = list(dataSet._get_numeric_data().columns) print("***** Number of numerical features in the dataset: ",len(numericalFeaturesInTheDataset)) #Names of categorical features in the dataset print("\n***** Names of categorical features in dataset *****\n") printList(categoricalFeaturesInTheDataset,'Categorical features in dataset') #Names of numerical features in the dataset print("\n***** Names of numerical features in dataset *****\n") printList(numericalFeaturesInTheDataset,'Numerical features in the dataset') #Checking for any missing values in the data set anyMissingValuesInTheDataset = checkForMissingValues(dataSet) print("\n***** Are there any missing values in the data set: ", anyMissingValuesInTheDataset) anyDuplicateRecordsInTheDataset = checkForDulicateRecords(dataSet) print("\n***** Are there any duplicate records in the data set: ", anyDuplicateRecordsInTheDataset) #Check if there are any duplicate records in the data set if (anyDuplicateRecordsInTheDataset): dataSet = dataSet.drop_duplicates() print("Number of records in the dataSet after removing the duplicates: ", len(dataSet.index)) #How many number of different values for label that are present in the dataset print('\n****** Number of different values for label that are present in the dataset: ',dataSet[labelName].nunique()) #What are the different values for label in the dataset print('\n****** Here is the list of unique label types present in the dataset ***** \n') printList(list(dataSet[getLabelName()].unique()),'Unique label types in the dataset') #What are the different values in each of the categorical features in the dataset print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n') categoricalFeaturesInTheDataset = list(set(dataSet.columns) - set(dataSet._get_numeric_data().columns)) numericalFeaturesInTheDataset = list(dataSet._get_numeric_data().columns) for feature in categoricalFeaturesInTheDataset: uniq = np.unique(dataSet[feature]) print('\n{}: {} '.format(feature,len(uniq))) printList(dataSet[feature].unique(),'distinct values') print('\n****** Label distribution in the dataset *****\n') print(dataSet[labelName].value_counts()) print() print("\n***** End checking the statistics of the dataSet *****")
def featureEncodingUsingBinaryEncoder(dataSetForFeatureEncoding): print("****** Start binary encoding on the categorical features in the given dataset *****") labelName = getLabelName() #Extract the categorical features, leave the label categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object']) #Get the names of the categorical features categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values print("****** Number of features before binary encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames)) print("****** Categorical feature names in the dataset: ",categoricalColumnNames) print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n') label = dataSetForFeatureEncoding.drop(dataSetForFeatureEncoding.loc[:, ~dataSetForFeatureEncoding.columns.isin([labelName])].columns, axis = 1) for feature in categoricalColumnNames: uniq = np.unique(dataSetForFeatureEncoding[feature]) print('\n{}: {} '.format(feature,len(uniq))) printList(dataSetForFeatureEncoding[feature].unique(),'distinct values') featureColumns = dataSetForFeatureEncoding.drop(dataSetForFeatureEncoding.loc[:, ~dataSetForFeatureEncoding.columns.isin([feature])].columns, axis = 1) binaryEncoder = ce.BinaryEncoder(cols = [feature]) binaryEncodedFeature = binaryEncoder.fit_transform(featureColumns, label) dataSetForFeatureEncoding = dataSetForFeatureEncoding.join(binaryEncodedFeature) dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(feature, axis=1) dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(labelName, axis=1) dataSetForFeatureEncoding[labelName] = label print("****** Number of features after binary encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** End binary encoding on the categorical features in the given dataset *****\n") return dataSetForFeatureEncoding
def featureEncodingUsingOneHotEncoder(dataSetForFeatureEncoding): print("****** Start one hot encoding on the categorical features in the given dataset *****") labelName = getLabelName() #Extract the categorical features, leave the label categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object']) #Get the names of the categorical features categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values print("****** Number of features before one hot encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames)) print("****** Categorical feature names in the dataset: ",categoricalColumnNames) print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n') categoricalFeaturesInTheDataset = list(set(dataSetForFeatureEncoding.columns) - set(dataSetForFeatureEncoding._get_numeric_data().columns)) numericalFeaturesInTheDataset = list(dataSetForFeatureEncoding._get_numeric_data().columns) for feature in categoricalFeaturesInTheDataset: uniq = np.unique(dataSetForFeatureEncoding[feature]) print('\n{}: {} '.format(feature,len(uniq))) printList(dataSetForFeatureEncoding[feature].unique(),'distinct values') #Using get_dummies function to get the dummy variables for the categorical columns onHotEncodedDataSet=pd.get_dummies(dataSetForFeatureEncoding, columns=categoricalColumnNames, prefix=categoricalColumnNames) #Move the label column to the end label = onHotEncodedDataSet.pop(labelName) onHotEncodedDataSet[labelName] = label numberOfColumnsInOneHotEncodedDataset = len(onHotEncodedDataSet.columns) print("****** Number of features after one hot encoding: ",numberOfColumnsInOneHotEncodedDataset) print("****** End one hot encoding on the categorical features in the given dataset *****\n") return onHotEncodedDataSet
def featureEncodingUsingFrequencyEncoder(dataSetForFeatureEncoding): print("****** Start frequency encoding on the categorical features in the given dataset *****") labelName = getLabelName() #Extract the categorical features, leave the label categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object']) #Get the names of the categorical features categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values print("****** Number of features before label encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames)) print("****** Categorical feature names in the dataset: ",categoricalColumnNames) print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n') label = dataSetForFeatureEncoding.drop(dataSetForFeatureEncoding.loc[:, ~dataSetForFeatureEncoding.columns.isin([labelName])].columns, axis = 1) for feature in categoricalColumnNames: uniq = np.unique(dataSetForFeatureEncoding[feature]) print('\n{}: {} '.format(feature,len(uniq))) printList(dataSetForFeatureEncoding[feature].unique(),'distinct values') frequencyEncoder = dataSetForFeatureEncoding.groupby(feature).size()/len(dataSetForFeatureEncoding) dataSetForFeatureEncoding.loc[:,feature+"_Encoded"] = dataSetForFeatureEncoding[feature].map(frequencyEncoder) dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(feature, axis=1) dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(labelName, axis=1) dataSetForFeatureEncoding[labelName] = label print("****** Number of features after frequency encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** End frequency encoding on the categorical features in the given dataset *****\n") return dataSetForFeatureEncoding
def featureSelectionUsingTheilU(dataSetForFeatureSelection): print("\n****** Start performing feature selection using TheilU *****") print("****** Falls under the group of techniques that use correlation matrix with Heatmap *****") labelName = getLabelName() label = dataSetForFeatureSelection[labelName] theilu = pd.DataFrame(index=[labelName],columns=dataSetForFeatureSelection.columns) columns = dataSetForFeatureSelection.columns dataSetAfterFeatuerSelection = dataSetForFeatureSelection for j in range(0,len(columns)): u = theil_u(label.tolist(),dataSetForFeatureSelection[columns[j]].tolist()) theilu.loc[:,columns[j]] = u if u < 0.50: dataSetAfterFeatuerSelection.pop(columns[j]) print('***** Ploting the uncertainty coefficient between the target and each feature *****') theilu.fillna(value=np.nan,inplace=True) plt.figure(figsize=(30,1)) sns.heatmap(theilu,annot=True,fmt='.2f') plt.show() numberOfFeaturesInTheDatasetAfterFeatureSelection = len(dataSetAfterFeatuerSelection.columns) print('***** Number of columns in the dataSet after feature selection: ', len(dataSetAfterFeatuerSelection.columns)) print('***** Columns in the dataSet after feature selection: \n', dataSetAfterFeatuerSelection.columns) print("****** End performing feature selection using TheilU *****") return dataSetAfterFeatuerSelection
def featureScalingUsingStandardScalar(dataSetForFeatureScaling): print("****** Start feature scaling of the features present in the dataset using StandardScalar *****") numberOfColumnsInEncodedDataset = len(dataSetForFeatureScaling.columns) dataSetInArrayFormat = dataSetForFeatureScaling.values features = dataSetInArrayFormat[:,0:numberOfColumnsInEncodedDataset-1] print("\n****** Number of features in the dataset before performing scaling: ",np.size(features,1)) print("\n****** Features in the dataset before performing scaling ***** \n",features) #Perform feature scaling scaler=StandardScaler() scaledFeatures=scaler.fit_transform(features) print("\n****** Number of features in the dataset after performing scaling: ",np.size(scaledFeatures,1)) print("\n****** Features in the dataset after performing scaling ***** \n",scaledFeatures) #Remove the label column from the dataset labelName = getLabelName() label = dataSetForFeatureScaling.pop(labelName) #Convert from array format to dataframe scaledFeatures = pd.DataFrame(scaledFeatures, columns=dataSetForFeatureScaling.columns) scaledFeatures[labelName]=label print("\n****** End of feature scaling of the features present in the dataset using StandardScalar *****\n") return scaledFeatures
def featureSelectionUsingExtraTreesClassifier(dataSetForFeatureSelection): print("\n****** Start performing feature selection using ExtraTreesClassifier *****") print("****** Falls under wrapper methods (feature importance) *****") labelName = getLabelName() #Applying feature encoding before applying the ExtraTreesClassification dataSetForFeatureSelection = featureEncodingUsingLabelEncoder(dataSetForFeatureSelection) dataSetAfterFeatuerSelection = dataSetForFeatureSelection features = dataSetForFeatureSelection.iloc[:,0:len(dataSetForFeatureSelection.columns)-1] label = dataSetForFeatureSelection[labelName] print("****** ExtraTreesClassification is in progress *****") #Train using ExtraTreesClassifier trainedforest = ExtraTreesClassifier(n_estimators=700).fit(features,label) importances = trainedforest.feature_importances_ #array with importances of each feature idx = np.arange(0, features.shape[1]) #create an index array, with the number of features features_to_keep = idx[importances > np.mean(importances)] #only keep features whose importance is greater than the mean importance featureImportances = pd.Series(importances, index= features.columns) selectedFeatures = featureImportances.nlargest(len(features_to_keep)) print("\n selectedFeatures after ExtraTreesClassification: ", selectedFeatures) print("****** Completed ExtraTreesClassification *****") #Plot the feature Importance to see which features have been considered as most important for our model to make its predictions #figure(num=None, figsize=(20, 22), dpi=80, facecolor='w', edgecolor='k') #selectedFeatures.plot(kind='barh') selectedFeaturesNames = selectedFeatures.keys() dataSetForFeatureSelection = dataSetForFeatureSelection.drop(selectedFeaturesNames,axis=1) dataSetAfterFeatuerSelection = dataSetAfterFeatuerSelection.drop(dataSetForFeatureSelection.columns, axis=1) dataSetAfterFeatuerSelection[labelName] = label numberOfFeaturesInTheDatasetAfterFeatureSelection = len(dataSetAfterFeatuerSelection.columns) print('\n***** Number of columns in the dataSet after feature selection: ', len(dataSetAfterFeatuerSelection.columns)) print('***** Columns in the dataSet after feature selection: \n', dataSetAfterFeatuerSelection.columns) print("****** End performing feature selection using ExtraTreesClassifier *****") return dataSetAfterFeatuerSelection
def featureEncodingUsingLabelEncoder(dataSetForFeatureEncoding): print("****** Start label encoding on the categorical features in the given dataset *****") labelName = getLabelName() #Extract the categorical features, leave the label categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object']) #Get the names of the categorical features categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values print("****** Number of features before label encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames)) print("****** Categorical feature names in the dataset: ",categoricalColumnNames) print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n') labelEncoder = LabelEncoder() for feature in categoricalColumnNames: uniq = np.unique(dataSetForFeatureEncoding[feature]) print('\n{}: {} '.format(feature,len(uniq))) printList(dataSetForFeatureEncoding[feature].unique(),'distinct values') dataSetForFeatureEncoding[feature] = labelEncoder.fit_transform(dataSetForFeatureEncoding[feature]) print("****** Number of features after label encoding: ",len(dataSetForFeatureEncoding.columns)) print("****** End label encoding on the categorical features in the given dataset *****\n") return dataSetForFeatureEncoding
def featureSelectionUsingChisquaredTest(dataSetForFeatureSelection): print("\n****** Start performing feature selection using ChisquaredTest *****") print("****** Falls under filter methods (univariate selection) *****") numberOfFeatureToBeSelected = 10 labelName = getLabelName() #To be able to apply Chi-squared test dataSetForFeatureSelection = featureEncodingUsingLabelEncoder(dataSetForFeatureSelection) dataSetAfterFeatuerSelection = dataSetForFeatureSelection features = dataSetForFeatureSelection.iloc[:,0:len(dataSetForFeatureSelection.columns)-1] label = dataSetForFeatureSelection[labelName] #Apply SelectKBest class to extract top 10 best features bestfeatures = SelectKBest(score_func=chi2, k=numberOfFeatureToBeSelected) fitBestfeatures = bestfeatures.fit(features,label) columns = pd.DataFrame(features.columns) scores = pd.DataFrame(fitBestfeatures.scores_) #concat two dataframes for better visualization scoresOfBestFeatures = pd.concat([columns,scores],axis=1) scoresOfBestFeatures.columns = ['Features','Score'] print("\n***** Scores for each feature in the dataset are *****") print(scoresOfBestFeatures.nlargest(numberOfFeatureToBeSelected,'Score')) mask = fitBestfeatures.get_support() for j in range(0,len(mask)): if (mask[j] == False): dataSetAfterFeatuerSelection.pop(features.columns[j]) numberOfFeaturesInTheDatasetAfterFeatureSelection = len(dataSetAfterFeatuerSelection.columns) print('***** Number of columns in the dataSet after feature selection: ', len(dataSetAfterFeatuerSelection.columns)) print('***** Columns in the dataSet after feature selection: \n', dataSetAfterFeatuerSelection.columns) print("****** End performing feature selection using ChisquaredTest *****") return dataSetAfterFeatuerSelection