def features_correlation(df, cols, target, fig_size=(6, 6), path=None): """ Correlation of variables in the dataframe with respect to the target Parameters ---------- df : pd.Dataframe dataframe with the data to calculate the correlation cols : array columns to be correlated with the target target : str target name fig_size : tuple figure size path : str path where the graphics will be saved Returns ------- None """ f, ax = plt.subplots(1, figsize=fig_size) ax.set_xlabel("Feature Correlation") visualizer = FeatureCorrelation(labels=list(cols)) visualizer.fit(df[cols], df[target]) f.tight_layout() if (path != None): f.savefig(path + '/features_correlation.png')
def test_feature_correlation_labels(self): """ Test labels as feature labels """ viz = FeatureCorrelation(labels=self.labels) viz.fit(self.X, self.y) npt.assert_array_equal(viz.features_, self.labels)
def test_feature_correlation_select_feature_by_index_out_of_range(self): """ Test selecting feature by feature index but index is out of range """ e = 'Feature index is out of range' with pytest.raises(YellowbrickValueError, match=e): viz = FeatureCorrelation(feature_index=[0, 2, 10]) viz.fit(self.X, self.y)
def test_feature_correlation_select_feature_by_index(self): """ Test selecting feature by index """ viz = FeatureCorrelation(feature_index=[0, 2, 3]) viz.fit(self.X, self.y) assert viz.scores_.shape[0] == 3
def test_feature_correlation_labels_from_index(self): """ Test getting feature labels from index """ viz = FeatureCorrelation() viz.fit(self.X, self.y) npt.assert_array_equal(viz.features_, np.arange(self.X.shape[1]))
def test_feature_correlation_sort(self): """ Test sorting of correlation """ viz = FeatureCorrelation(sort=True) viz.fit(self.X, self.y) assert np.all(viz.scores_[:-1] <= viz.scores_[1:])
def test_feature_correlation_select_feature_by_index(self): """ Test selecting feature by index """ viz = FeatureCorrelation(feature_index=[0, 2, 3]) viz.fit(self.X, self.y) assert viz.scores_.shape[0] == 3
def test_feature_correlation_labels(self): """ Test labels as feature labels """ viz = FeatureCorrelation(labels=self.labels) viz.fit(self.X, self.y) npt.assert_array_equal(viz.features_, self.labels)
def test_feature_correlation_select_feature_by_index_out_of_range(self): """ Test selecting feature by feature index but index is out of range """ e = "Feature index is out of range" with pytest.raises(YellowbrickValueError, match=e): viz = FeatureCorrelation(feature_index=[0, 2, 10]) viz.fit(self.X, self.y)
def test_feature_correlation_sort(self): """ Test sorting of correlation """ viz = FeatureCorrelation(sort=True) viz.fit(self.X, self.y) assert np.all(viz.scores_[:-1] <= viz.scores_[1:])
def test_feature_correlation_labels_from_index(self): """ Test getting feature labels from index """ viz = FeatureCorrelation() viz.fit(self.X, self.y) npt.assert_array_equal(viz.features_, np.arange(self.X.shape[1]))
def test_feature_correlation_integrated_mutual_info_regression(self): """ Test FeatureCorrelation visualizer with mutual information regression """ viz = FeatureCorrelation(method='mutual_info-regression') viz.fit(self.X, self.y, random_state=23456) viz.poof() self.assert_images_similar(viz)
def feature_correlation_pearson( path="images/feature_correlation_pearson.png"): data = datasets.load_diabetes() X, y = data['data'], data['target'] feature_names = np.array(data['feature_names']) visualizer = FeatureCorrelation(labels=feature_names) visualizer.fit(X, y) visualizer.poof(outpath=path, clear_figure=True)
def test_feature_correlation_select_feature_by_name(self): """ Test selecting feature by feature names """ feature_names = ["age", "sex", "bp", "s5"] viz = FeatureCorrelation(labels=self.labels, feature_names=feature_names) viz.fit(self.X, self.y) npt.assert_array_equal(viz.features_, feature_names)
def test_feature_correlation_select_feature_by_name_no_labels(self): """ Test selecting feature by feature names with labels is not supplied """ feature_names = ["age"] e = "age not in labels" with pytest.raises(YellowbrickValueError, match=e): viz = FeatureCorrelation(feature_names=feature_names) viz.fit(self.X, self.y)
def test_feature_correlation_labels_from_dataframe(self): """ Test getting feature labels from DataFrame """ X_pd = pd.DataFrame(self.X, columns=self.labels) viz = FeatureCorrelation() viz.fit(X_pd, self.y) npt.assert_array_equal(viz.features_, self.labels)
def test_feature_correlation_integrated_pearson(self): """ Test FeatureCorrelation visualizer with pearson correlation coefficient """ viz = FeatureCorrelation() viz.fit(self.X, self.y) viz.poof() self.assert_images_similar(viz)
def test_feature_correlation_select_feature_by_name_no_labels(self): """ Test selecting feature by feature names with labels is not supplied """ feature_names = ['age'] e = 'age not in labels' with pytest.raises(YellowbrickValueError, match=e): viz = FeatureCorrelation(feature_names=feature_names) viz.fit(self.X, self.y)
def test_feature_correlation_labels_from_dataframe(self): """ Test getting feature labels from DataFrame """ X_pd = pd.DataFrame(self.X, columns=self.labels) viz = FeatureCorrelation() viz.fit(X_pd, self.y) npt.assert_array_equal(viz.features_, self.labels)
def test_feature_correlation_select_feature_by_name(self): """ Test selecting feature by feature names """ feature_names = ['age', 'sex', 'bp', 's5'] viz = FeatureCorrelation(labels=self.labels, feature_names=feature_names) viz.fit(self.X, self.y) npt.assert_array_equal(viz.features_, feature_names)
def target_visualizer(self, classes=None, params={'BalancedBinningReference': { 'bins': 5 }}): LOGGER.info('Initializing target visualizer') if os.path.isdir(os.path.join(os.getcwd(), 'visualizer/')) == False: os.makedirs(os.path.join(os.getcwd(), 'visualizer/')) visualizers = [] y = self.y.squeeze() try: LOGGER.info('Visualizer BalancedBinningReference') visualizer = BalancedBinningReference() if visualizer.__class__.__name__ in params.keys(): visualizer = BalancedBinningReference( **params[visualizer.__class__.__name__]) visualizer.fit(y) visualizer.show(outpath=os.path.join( os.getcwd(), f"visualizer/{visualizer.__class__.__name__}.png")) plt.cla() except: LOGGER.warn('ERROR BalancedBinning') try: LOGGER.info('Visualizer CLassBalance') visualizer = ClassBalance() if visualizer.__class__.__name__ in params.keys(): visualizer = ClassBalance( **params[visualizer.__class__.__name__]) visualizer.fit(y) visualizer.show(outpath=os.path.join( os.getcwd(), f"visualizer/{visualizer.__class__.__name__}.png")) plt.cla() except: LOGGER.warn('ERROR ClassBalance') try: LOGGER.info('Visualizer Feature Correlation') visualizer = FeatureCorrelation( method='mutual_info-classification', feature_names=self.X.columns.tolist(), sort=True) if visualizer.__class__.__name__ in params.keys(): visualizer = FeatureCorrelation( **params[visualizer.__class__.__name__]) visualizer.fit(self.X, y) visualizer.show(outpath=os.path.join( os.getcwd(), f"visualizer/{visualizer.__class__.__name__}.png")) plt.cla() except: LOGGER.warn('ERROR FeatureCorrelation')
def test_feature_correlation_integrated_mutual_info_classification(self): """ Test FeatureCorrelation visualizer with mutual information on wine dataset (classification) """ data = datasets.load_wine() X, y = data['data'], data['target'] viz = FeatureCorrelation(method='mutual_info-classification') viz.fit(X, y, random_state=12345) viz.poof() self.assert_images_similar(viz)
def feature_correlation_mutual_info_regression( path="images/feature_correlation_mutual_info_regression.png"): data = datasets.load_diabetes() X, y = data['data'], data['target'] feature_names = np.array(data['feature_names']) discrete_features = [False for _ in range(len(feature_names))] discrete_features[1] = True visualizer = FeatureCorrelation(method='mutual_info-regression', labels=feature_names, sort=True) visualizer.fit(X, y, discrete_features=discrete_features, random_state=0) visualizer.poof(outpath=path, clear_figure=True)
def feature_correlation_mutual_info_classification( path="images/feature_correlation_mutual_info_classification.png"): data = datasets.load_wine() X, y = data['data'], data['target'] feature_names = np.array(data['feature_names']) X_pd = pd.DataFrame(X, columns=feature_names) feature_to_plot = ['alcohol', 'ash', 'hue', 'proline', 'total_phenols'] visualizer = FeatureCorrelation(method='mutual_info-classification', feature_names=feature_to_plot) visualizer.fit(X_pd, y, random_state=0) visualizer.poof(outpath=path, clear_figure=True)
def test_feature_correlation_select_feature_by_index_and_name(self): """ Test selecting feature warning when both index and names are provided """ feature_index = [0, 2, 3] feature_names = ['age'] e = ('Both feature_index and feature_names are specified. ' 'feature_names is ignored') with pytest.raises(YellowbrickWarning, match=e): viz = FeatureCorrelation(feature_index=feature_index, feature_names=feature_names) viz.fit(self.X, self.y) assert viz.scores_.shape[0] == 3
def test_feature_correlation_select_feature_by_index_and_name(self): """ Test selecting feature warning when both index and names are provided """ feature_index = [0, 2, 3] feature_names = ['age'] e = ('Both feature_index and feature_names are specified. ' 'feature_names is ignored') with pytest.raises(YellowbrickWarning, match=e): viz = FeatureCorrelation(feature_index=feature_index, feature_names=feature_names) viz.fit(self.X, self.y) assert viz.scores_.shape[0] == 3
def test_feature_correlation_method_not_implemented(self): """ Test FeatureCorrelation visualizer with unknown method """ method = "foo" e = "Method foo not implement; choose from *" with pytest.raises(YellowbrickValueError, match=e): FeatureCorrelation(method=method)
def showCorrelation(gripperjack_nr, part): data = DynamicCsvConverter(gripperjack_nr, part, '5min', 'max', pd.read_csv( 'C:\\Users\\Lukassen\\PycharmProjects\\GelredomeVeldErrorVoorspellen\\Recources\\Volledige_Gelredome_Data_CSV.csv', index_col=False)) data = data.make_file() # to see correlation with the to be predicted remove 'to_be_predicted' from drop columns and put the 'to_be_predicted variable in the data.pop method' data = data.drop(columns=['Timestamp']) data = data.dropna() y = data.pop('to_be_predicted') X = data # Create a list of the feature names features = np.array(data.columns) # Create a list of the discrete features discrete = [False for _ in range(len(features))] discrete[1] = True # Instantiate the visualizer visualizer = FeatureCorrelation(labels=features, size=(1200, 700)) visualizer.title = part visualizer.fit(X, y) values.append(visualizer.scores_) visualizer.show()
def create_correlation_matrix(data): encoded = encode_data(data) kendall = encoded.corr(method ='kendall')['class'].to_frame() pearson = encoded.corr(method ='pearson')['class'].to_frame() spearman = encoded.corr(method ='spearman')['class'].to_frame() kendall.columns = ['kendall'] pearson.columns = ['pearson'] spearman.columns = ['spearman'] correlation_matrix = pd.concat([kendall, pearson, spearman], axis=1, sort=False) X, y = encoded.drop(columns = ['class']), encoded['class'] visualizer = FeatureCorrelation(method='mutual_info-classification', labels=X.columns) visualizer.fit(X, y) correlation_matrix = correlation_matrix.drop('class', axis = 0) correlation_matrix['mutual_info-classification'] = visualizer.scores_.tolist() return correlation_matrix
def feature_correlation_pearson(path="images/feature_correlation_pearson.png"): data = datasets.load_diabetes() X, y = data['data'], data['target'] feature_names = np.array(data['feature_names']) visualizer = FeatureCorrelation(labels=feature_names) visualizer.fit(X, y) visualizer.poof(outpath=path, clear_figure=True)
def test_feature_correlation_integrated_mutual_info_regression(self): """ Test FeatureCorrelation visualizer with mutual information regression """ viz = FeatureCorrelation(method="mutual_info-regression") viz.fit(self.X, self.y, random_state=23456) viz.finalize() self.assert_images_similar(viz)
def test_feature_correlation_integrated_pearson(self): """ Test FeatureCorrelation visualizer with pearson correlation coefficient """ viz = FeatureCorrelation() viz.fit(self.X, self.y) viz.finalize() self.assert_images_similar(viz)
def mutual_info_classification(classes, feature_names, X, y): from sklearn import datasets from yellowbrick.target import FeatureCorrelation # Load the regression data set visualizer = FeatureCorrelation(method='mutual_info-classification', feature_names=feature_names, sort=True) visualizer.fit(X, y, random_state=0) visualizer.poof()
def pearson_correlation(classes, fetures, X, Y): from sklearn import datasets from yellowbrick.target import FeatureCorrelation # Load the regression data set # data = datasets.load_diabetes() # X, y = data['data'], data['target'] # feature_names = np.array(data['feature_names']) visualizer = FeatureCorrelation(labels=fetures) visualizer.fit(X, Y) visualizer.poof()
def report(self, pipeline: AbstractPipeline): folder = get_cache_path() path = pkg_resources.resource_filename( 'crcdal', 'cache/' + folder + '/' + self.sub_folder + '/') pkg_resources.ensure_directory(path) feature_names = list(pipeline.train.columns()) visualizer = FeatureCorrelation(labels=feature_names) visualizer.fit(pipeline.train, pipeline.train_y) visualizer.poof(outpath=path + pipeline.dataset_tag + '_model_feature_correlation_report.csv')
def mutual_info_regress(classes, feature_names, X, y): from sklearn import datasets from yellowbrick.target import FeatureCorrelation # Load the regression data set discrete_features = [False for _ in range(len(feature_names))] discrete_features[1] = True visualizer = FeatureCorrelation(method='mutual_info-regression', labels=feature_names) visualizer.fit(X, y, discrete_features=discrete_features, random_state=0) visualizer.poof()
def feature_correlation_mutual_info_classification( path="images/feature_correlation_mutual_info_classification.png"): data = datasets.load_wine() X, y = data['data'], data['target'] feature_names = np.array(data['feature_names']) X_pd = pd.DataFrame(X, columns=feature_names) feature_to_plot = ['alcohol', 'ash', 'hue', 'proline', 'total_phenols'] visualizer = FeatureCorrelation(method='mutual_info-classification', feature_names=feature_to_plot) visualizer.fit(X_pd, y, random_state=0) visualizer.poof(outpath=path, clear_figure=True)
def test_feature_correlation_integrated_mutual_info_classification(self): """ Test FeatureCorrelation visualizer with mutual information on wine dataset (classification) """ data = datasets.load_wine() X, y = data["data"], data["target"] viz = FeatureCorrelation(method="mutual_info-classification") viz.fit(X, y, random_state=12345) viz.finalize() self.assert_images_similar(viz)
def feature_correlation_mutual_info_regression( path="images/feature_correlation_mutual_info_regression.png"): data = datasets.load_diabetes() X, y = data['data'], data['target'] feature_names = np.array(data['feature_names']) discrete_features = [False for _ in range(len(feature_names))] discrete_features[1] = True visualizer = FeatureCorrelation(method='mutual_info-regression', labels=feature_names, sort=True) visualizer.fit(X, y, discrete_features=discrete_features, random_state=0) visualizer.poof(outpath=path, clear_figure=True)
import pandas as pd import numpy as np import math import seaborn as sns import matplotlib.pyplot as plt from yellowbrick.target import FeatureCorrelation dataset = pd.read_csv('house_prices.csv') #removendo atributos que nao serao utilizados para analise dataset.drop(labels=['id', 'date', 'sqft_living15', 'sqft_lot15'], axis=1, inplace=True) print(dataset.columns) grafico = FeatureCorrelation(labels=dataset.columns[1:]) grafico.fit(dataset.iloc[:, 1:16].values, dataset.iloc[:, 0].values) plt.show()
visualizer.fit_transform(X, y) visualizer.poof() # %% visualizer = Rank1D() visualizer.fit(X, y) visualizer.transform(X) visualizer.poof() # %% visualizer = Rank2D() visualizer.fit_transform(X) visualizer.poof() # %% visualizer = FeatureCorrelation() visualizer.fit(X, y) visualizer.poof() # %% visualizer = FeatureCorrelation(method='mutual_info-classification') visualizer.fit(X, y) visualizer.poof() # %% visualizer = RadViz(classes=class_names) visualizer.fit(X, y) visualizer.transform(X) visualizer.poof() # %%
correlated_features.add(colname) ## yellowbrick from sklearn import datasets from yellowbrick.target import FeatureCorrelation # Load the regression dataset data = datasets.load_diabetes() X, y = data['data'], data['target'] # Create a list of the feature names features = np.array(data['feature_names']) # Instantiate the visualizer visualizer = FeatureCorrelation(labels=features) visualizer.fit(X, y) # Fit the data to the visualizer visualizer.show() ## PCA - Principal Component Analysis https://www.kaggle.com/ryanholbrook/principal-component-analysis from sklearn.decomposition import PCA # Create principal components pca = PCA() X_pca = pca.fit_transform(X_scaled) # Convert to dataframe component_names = [f"PC{i+1}" for i in range(X_pca.shape[1])] X_pca = pd.DataFrame(X_pca, columns=component_names)
def featcorr(): data = load_diabetes() oz = FeatureCorrelation(ax=newfig()) oz.fit(data.data, data.target) savefig(oz, "feature_correlation")
from matplotlib import rcParams as rc from yellowbrick.target import FeatureCorrelation rc['xtick.labelsize'] = 15.0 rc['ytick.labelsize'] = 15.0 rc['xtick.direction'] = 'out' rc['axes.labelsize'] = 15.0 rc['axes.titlesize'] = 18.0 rc['savefig.format'] = 'png' rc['savefig.dpi'] = 600 rc['legend.fontsize'] = 15 x = df.drop('Death_Event', axis=1) y = df['Death_Event'] fig = plt.figure(figsize=(8, 6)) corr = FeatureCorrelation(method='pearson', label=x.columns, sort=True).fit(x, y) plt.savefig('../../outputs/visuals/correlations') corr.show() fig, ax = plt.subplots(figsize=(20, 10)) sns.heatmap(df.corr(), annot=True, square=False, ax=ax) ax.set_title('Correlations between features') plt.savefig('../../outputs/visuals/correlations_all') plt.show() # Age distribution of Patients fig, ax = plt.subplots(figsize=(8, 6)) sns.kdeplot(df['Age'], legend=False, shade=True, ax=ax) plt.savefig('../../outputs/visuals/age_distribution') ax.set_title('Age Distribution of Patients')
cv=cv) clf = scores['estimator'][np.argmax(scores['test_score'])] print(np.max(scores['test_score'])) # %% explainer = shap.TreeExplainer(clf) shap_values = explainer.shap_values(Xv) # %% shap.summary_plot(shap_values, Xv, plot_type="bar") # %% feat = feature_names[feat][np.mean(abs(shap_values), axis=0) > 0.55] print(feat) X = X[feat] # %% visualizer = FeatureCorrelation(method='mutual_info-classification') visualizer.fit(X, y) visualizer.poof() # %% # This step doesn't always produce the same result, idk why. feat = visualizer.features_[visualizer.scores_ > 0.04] X = X[feat] # %% # Our final 10 features: # [263, 268, 287, 288, 300, 302, 307, 308, 313, 315] print(feat)