def getStatisticsOfData (dataSet):
print("***** Start checking the statistics of the dataSet *****\n")
labelName = getLabelName()
#Number of rows and columns in the dataset
print("***** Shape (number of rows and columns) in the dataset: ", dataSet.shape)
#Total number of features in the dataset
numberOfColumnsInTheDataset = len(dataSet.columns)
print("***** Total number of features in the dataset: ",numberOfColumnsInTheDataset)
#Total number of categorical featuers in the dataset
categoricalFeaturesInTheDataset = list(set(dataSet.columns) - set(dataSet._get_numeric_data().columns))
print("***** Number of categorical features in the dataset: ",len(categoricalFeaturesInTheDataset))
#Total number of numerical features in the dataset
numericalFeaturesInTheDataset = list(dataSet._get_numeric_data().columns)
print("***** Number of numerical features in the dataset: ",len(numericalFeaturesInTheDataset))
#Names of categorical features in the dataset
print("\n***** Names of categorical features in dataset *****\n")
printList(categoricalFeaturesInTheDataset,'Categorical features in dataset')
#Names of numerical features in the dataset
print("\n***** Names of numerical features in dataset *****\n")
printList(numericalFeaturesInTheDataset,'Numerical features in the dataset')
#Checking for any missing values in the data set
anyMissingValuesInTheDataset = checkForMissingValues(dataSet)
print("\n***** Are there any missing values in the data set: ", anyMissingValuesInTheDataset)
anyDuplicateRecordsInTheDataset = checkForDulicateRecords(dataSet)
print("\n***** Are there any duplicate records in the data set: ", anyDuplicateRecordsInTheDataset)
#Check if there are any duplicate records in the data set
if (anyDuplicateRecordsInTheDataset):
dataSet = dataSet.drop_duplicates()
print("Number of records in the dataSet after removing the duplicates: ", len(dataSet.index))
#How many number of different values for label that are present in the dataset
print('\n****** Number of different values for label that are present in the dataset: ',dataSet[labelName].nunique())
#What are the different values for label in the dataset
print('\n****** Here is the list of unique label types present in the dataset ***** \n')
printList(list(dataSet[getLabelName()].unique()),'Unique label types in the dataset')
#What are the different values in each of the categorical features in the dataset
print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n')
categoricalFeaturesInTheDataset = list(set(dataSet.columns) - set(dataSet._get_numeric_data().columns))
numericalFeaturesInTheDataset = list(dataSet._get_numeric_data().columns)
for feature in categoricalFeaturesInTheDataset:
uniq = np.unique(dataSet[feature])
print('\n{}: {} '.format(feature,len(uniq)))
printList(dataSet[feature].unique(),'distinct values')
print('\n****** Label distribution in the dataset *****\n')
print(dataSet[labelName].value_counts())
print()
print("\n***** End checking the statistics of the dataSet *****")
def featureEncodingUsingBinaryEncoder(dataSetForFeatureEncoding):
print("****** Start binary encoding on the categorical features in the given dataset *****")
labelName = getLabelName()
#Extract the categorical features, leave the label
categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object'])
#Get the names of the categorical features
categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values
print("****** Number of features before binary encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames))
print("****** Categorical feature names in the dataset: ",categoricalColumnNames)
print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n')
label = dataSetForFeatureEncoding.drop(dataSetForFeatureEncoding.loc[:, ~dataSetForFeatureEncoding.columns.isin([labelName])].columns, axis = 1)
for feature in categoricalColumnNames:
uniq = np.unique(dataSetForFeatureEncoding[feature])
print('\n{}: {} '.format(feature,len(uniq)))
printList(dataSetForFeatureEncoding[feature].unique(),'distinct values')
featureColumns = dataSetForFeatureEncoding.drop(dataSetForFeatureEncoding.loc[:, ~dataSetForFeatureEncoding.columns.isin([feature])].columns, axis = 1)
binaryEncoder = ce.BinaryEncoder(cols = [feature])
binaryEncodedFeature = binaryEncoder.fit_transform(featureColumns, label)
dataSetForFeatureEncoding = dataSetForFeatureEncoding.join(binaryEncodedFeature)
dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(feature, axis=1)
dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(labelName, axis=1)
dataSetForFeatureEncoding[labelName] = label
print("****** Number of features after binary encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** End binary encoding on the categorical features in the given dataset *****\n")
return dataSetForFeatureEncoding
def featureEncodingUsingOneHotEncoder(dataSetForFeatureEncoding):
print("****** Start one hot encoding on the categorical features in the given dataset *****")
labelName = getLabelName()
#Extract the categorical features, leave the label
categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object'])
#Get the names of the categorical features
categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values
print("****** Number of features before one hot encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames))
print("****** Categorical feature names in the dataset: ",categoricalColumnNames)
print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n')
categoricalFeaturesInTheDataset = list(set(dataSetForFeatureEncoding.columns) - set(dataSetForFeatureEncoding._get_numeric_data().columns))
numericalFeaturesInTheDataset = list(dataSetForFeatureEncoding._get_numeric_data().columns)
for feature in categoricalFeaturesInTheDataset:
uniq = np.unique(dataSetForFeatureEncoding[feature])
print('\n{}: {} '.format(feature,len(uniq)))
printList(dataSetForFeatureEncoding[feature].unique(),'distinct values')
#Using get_dummies function to get the dummy variables for the categorical columns
onHotEncodedDataSet=pd.get_dummies(dataSetForFeatureEncoding, columns=categoricalColumnNames, prefix=categoricalColumnNames)
#Move the label column to the end
label = onHotEncodedDataSet.pop(labelName)
onHotEncodedDataSet[labelName] = label
numberOfColumnsInOneHotEncodedDataset = len(onHotEncodedDataSet.columns)
print("****** Number of features after one hot encoding: ",numberOfColumnsInOneHotEncodedDataset)
print("****** End one hot encoding on the categorical features in the given dataset *****\n")
return onHotEncodedDataSet
def featureEncodingUsingFrequencyEncoder(dataSetForFeatureEncoding):
print("****** Start frequency encoding on the categorical features in the given dataset *****")
labelName = getLabelName()
#Extract the categorical features, leave the label
categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object'])
#Get the names of the categorical features
categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values
print("****** Number of features before label encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames))
print("****** Categorical feature names in the dataset: ",categoricalColumnNames)
print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n')
label = dataSetForFeatureEncoding.drop(dataSetForFeatureEncoding.loc[:, ~dataSetForFeatureEncoding.columns.isin([labelName])].columns, axis = 1)
for feature in categoricalColumnNames:
uniq = np.unique(dataSetForFeatureEncoding[feature])
print('\n{}: {} '.format(feature,len(uniq)))
printList(dataSetForFeatureEncoding[feature].unique(),'distinct values')
frequencyEncoder = dataSetForFeatureEncoding.groupby(feature).size()/len(dataSetForFeatureEncoding)
dataSetForFeatureEncoding.loc[:,feature+"_Encoded"] = dataSetForFeatureEncoding[feature].map(frequencyEncoder)
dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(feature, axis=1)
dataSetForFeatureEncoding = dataSetForFeatureEncoding.drop(labelName, axis=1)
dataSetForFeatureEncoding[labelName] = label
print("****** Number of features after frequency encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** End frequency encoding on the categorical features in the given dataset *****\n")
return dataSetForFeatureEncoding
def featureSelectionUsingTheilU(dataSetForFeatureSelection):
print("\n****** Start performing feature selection using TheilU *****")
print("****** Falls under the group of techniques that use correlation matrix with Heatmap *****")
labelName = getLabelName()
label = dataSetForFeatureSelection[labelName]
theilu = pd.DataFrame(index=[labelName],columns=dataSetForFeatureSelection.columns)
columns = dataSetForFeatureSelection.columns
dataSetAfterFeatuerSelection = dataSetForFeatureSelection
for j in range(0,len(columns)):
u = theil_u(label.tolist(),dataSetForFeatureSelection[columns[j]].tolist())
theilu.loc[:,columns[j]] = u
if u < 0.50:
dataSetAfterFeatuerSelection.pop(columns[j])
print('***** Ploting the uncertainty coefficient between the target and each feature *****')
theilu.fillna(value=np.nan,inplace=True)
plt.figure(figsize=(30,1))
sns.heatmap(theilu,annot=True,fmt='.2f')
plt.show()
numberOfFeaturesInTheDatasetAfterFeatureSelection = len(dataSetAfterFeatuerSelection.columns)
print('***** Number of columns in the dataSet after feature selection: ', len(dataSetAfterFeatuerSelection.columns))
print('***** Columns in the dataSet after feature selection: \n', dataSetAfterFeatuerSelection.columns)
print("****** End performing feature selection using TheilU *****")
return dataSetAfterFeatuerSelection
def featureScalingUsingStandardScalar(dataSetForFeatureScaling):
print("****** Start feature scaling of the features present in the dataset using StandardScalar *****")
numberOfColumnsInEncodedDataset = len(dataSetForFeatureScaling.columns)
dataSetInArrayFormat = dataSetForFeatureScaling.values
features = dataSetInArrayFormat[:,0:numberOfColumnsInEncodedDataset-1]
print("\n****** Number of features in the dataset before performing scaling: ",np.size(features,1))
print("\n****** Features in the dataset before performing scaling ***** \n",features)
#Perform feature scaling
scaler=StandardScaler()
scaledFeatures=scaler.fit_transform(features)
print("\n****** Number of features in the dataset after performing scaling: ",np.size(scaledFeatures,1))
print("\n****** Features in the dataset after performing scaling ***** \n",scaledFeatures)
#Remove the label column from the dataset
labelName = getLabelName()
label = dataSetForFeatureScaling.pop(labelName)
#Convert from array format to dataframe
scaledFeatures = pd.DataFrame(scaledFeatures, columns=dataSetForFeatureScaling.columns)
scaledFeatures[labelName]=label
print("\n****** End of feature scaling of the features present in the dataset using StandardScalar *****\n")
return scaledFeatures
def featureSelectionUsingExtraTreesClassifier(dataSetForFeatureSelection):
print("\n****** Start performing feature selection using ExtraTreesClassifier *****")
print("****** Falls under wrapper methods (feature importance) *****")
labelName = getLabelName()
#Applying feature encoding before applying the ExtraTreesClassification
dataSetForFeatureSelection = featureEncodingUsingLabelEncoder(dataSetForFeatureSelection)
dataSetAfterFeatuerSelection = dataSetForFeatureSelection
features = dataSetForFeatureSelection.iloc[:,0:len(dataSetForFeatureSelection.columns)-1]
label = dataSetForFeatureSelection[labelName]
print("****** ExtraTreesClassification is in progress *****")
#Train using ExtraTreesClassifier
trainedforest = ExtraTreesClassifier(n_estimators=700).fit(features,label)
importances = trainedforest.feature_importances_ #array with importances of each feature
idx = np.arange(0, features.shape[1]) #create an index array, with the number of features
features_to_keep = idx[importances > np.mean(importances)] #only keep features whose importance is greater than the mean importance
featureImportances = pd.Series(importances, index= features.columns)
selectedFeatures = featureImportances.nlargest(len(features_to_keep))
print("\n selectedFeatures after ExtraTreesClassification: ", selectedFeatures)
print("****** Completed ExtraTreesClassification *****")
#Plot the feature Importance to see which features have been considered as most important for our model to make its predictions
#figure(num=None, figsize=(20, 22), dpi=80, facecolor='w', edgecolor='k')
#selectedFeatures.plot(kind='barh')
selectedFeaturesNames = selectedFeatures.keys()
dataSetForFeatureSelection = dataSetForFeatureSelection.drop(selectedFeaturesNames,axis=1)
dataSetAfterFeatuerSelection = dataSetAfterFeatuerSelection.drop(dataSetForFeatureSelection.columns, axis=1)
dataSetAfterFeatuerSelection[labelName] = label
numberOfFeaturesInTheDatasetAfterFeatureSelection = len(dataSetAfterFeatuerSelection.columns)
print('\n***** Number of columns in the dataSet after feature selection: ', len(dataSetAfterFeatuerSelection.columns))
print('***** Columns in the dataSet after feature selection: \n', dataSetAfterFeatuerSelection.columns)
print("****** End performing feature selection using ExtraTreesClassifier *****")
return dataSetAfterFeatuerSelection
def featureEncodingUsingLabelEncoder(dataSetForFeatureEncoding):
print("****** Start label encoding on the categorical features in the given dataset *****")
labelName = getLabelName()
#Extract the categorical features, leave the label
categoricalColumnsInTheDataSet = dataSetForFeatureEncoding.drop([labelName],axis=1).select_dtypes(['object'])
#Get the names of the categorical features
categoricalColumnNames = categoricalColumnsInTheDataSet.columns.values
print("****** Number of features before label encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** Number of categorical features in the dataset: ",len(categoricalColumnNames))
print("****** Categorical feature names in the dataset: ",categoricalColumnNames)
print('\n****** Here is the list of unique values present in each categorical feature in the dataset *****\n')
labelEncoder = LabelEncoder()
for feature in categoricalColumnNames:
uniq = np.unique(dataSetForFeatureEncoding[feature])
print('\n{}: {} '.format(feature,len(uniq)))
printList(dataSetForFeatureEncoding[feature].unique(),'distinct values')
dataSetForFeatureEncoding[feature] = labelEncoder.fit_transform(dataSetForFeatureEncoding[feature])
print("****** Number of features after label encoding: ",len(dataSetForFeatureEncoding.columns))
print("****** End label encoding on the categorical features in the given dataset *****\n")
return dataSetForFeatureEncoding
def featureSelectionUsingChisquaredTest(dataSetForFeatureSelection):
print("\n****** Start performing feature selection using ChisquaredTest *****")
print("****** Falls under filter methods (univariate selection) *****")
numberOfFeatureToBeSelected = 10
labelName = getLabelName()
#To be able to apply Chi-squared test
dataSetForFeatureSelection = featureEncodingUsingLabelEncoder(dataSetForFeatureSelection)
dataSetAfterFeatuerSelection = dataSetForFeatureSelection
features = dataSetForFeatureSelection.iloc[:,0:len(dataSetForFeatureSelection.columns)-1]
label = dataSetForFeatureSelection[labelName]
#Apply SelectKBest class to extract top 10 best features
bestfeatures = SelectKBest(score_func=chi2, k=numberOfFeatureToBeSelected)
fitBestfeatures = bestfeatures.fit(features,label)
columns = pd.DataFrame(features.columns)
scores = pd.DataFrame(fitBestfeatures.scores_)
#concat two dataframes for better visualization
scoresOfBestFeatures = pd.concat([columns,scores],axis=1)
scoresOfBestFeatures.columns = ['Features','Score']
print("\n***** Scores for each feature in the dataset are *****")
print(scoresOfBestFeatures.nlargest(numberOfFeatureToBeSelected,'Score'))
mask = fitBestfeatures.get_support()
for j in range(0,len(mask)):
if (mask[j] == False):
dataSetAfterFeatuerSelection.pop(features.columns[j])
numberOfFeaturesInTheDatasetAfterFeatureSelection = len(dataSetAfterFeatuerSelection.columns)
print('***** Number of columns in the dataSet after feature selection: ', len(dataSetAfterFeatuerSelection.columns))
print('***** Columns in the dataSet after feature selection: \n', dataSetAfterFeatuerSelection.columns)
print("****** End performing feature selection using ChisquaredTest *****")
return dataSetAfterFeatuerSelection
