def pearson_correlation(data, verbose=True):
plt.rcdefaults()
#----------------
norm_data = norm.normalization_with_minmax(data)
# 2. Principal Component Analysis
estimator, X_pca = norm.pca(norm_data)
#----------------
features = data.columns.tolist()[:-1]
# 1. Correlation
corr = map(lambda x1: pearsonr(x1,
norm_data.tolist()[-1])[0],
[norm_data[x].tolist() for x in range(len(features))])
features_corr = zip(features, corr)
if verbose:
y_pos = np.arange(len(features))
plt.bar(y_pos, corr, align='center')
plt.xticks(y_pos, features, rotation=90)
plt.ylabel('Correlation')
plt.title('Correlation features vs \'total_cases\'')
plt.show()
print ''
print tabulate(features_corr, headers=['Feature', 'R value'])
return features_corr
def hierarchical_clustering_features(data, verbose=False):
names = list(data)
data_aux = data
data_droped = data_aux.dropna()
data_transpose = data_droped.transpose()
#1. Normalization of the data
data_transpose_norm = norm.normalization_with_minmax(data_transpose)
#1.2. Principal Component Analysis
estimator, X_pca = norm.pca(data_transpose_norm)
if verbose:
plt.plot(X_pca[:, 0], X_pca[:, 1], 'x')
print("Variance Ratio: ", estimator.explained_variance_ratio_)
fig, ax = plt.subplots()
for i in range(len(data_transpose)):
plt.text(X_pca[i][0], X_pca[i][1], i + 1)
plt.xlim(min(X_pca[:, 0] - 0.2), max(X_pca[:, 0]) + 0.2)
plt.ylim(min(X_pca[:, 1] - 0.2), max(X_pca[:, 1]) + 0.2)
ax.grid(True)
fig.tight_layout()
plt.show()
data_transpose_name = zip(range(1, len(names) + 1), names)
print tabulate(data_transpose_name, headers=['# ', 'Feature name'])
# 2. Compute the similarity matrix
dist = sklearn.neighbors.DistanceMetric.get_metric('euclidean')
matsim = dist.pairwise(data_transpose_norm)
avSim = np.average(matsim)
if verbose:
print "%s\t%6.2f" % ('Average Distance', avSim)
plt.figure(figsize=(7, 7))
# 3. Building the Dendrogram
# http://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
clusters = cluster.hierarchy.linkage(matsim, method='complete')
# http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.cluster.hierarchy.dendrogram.html
cluster.hierarchy.dendrogram(clusters,
color_threshold=5,
labels=names,
leaf_rotation=90)
plt.show()
def pearson_correlation(data, verbose=False):
plt.rcdefaults()
#----------------
norm_data = norm.normalization_with_minmax(data)
# 2. Principal Component Analysis
estimator, X_pca = norm.pca(norm_data)
#----------------
features = data.columns.tolist()[:-1]
# 1. Correlation
corr_SanJuan = map(lambda x1: pearsonr(x1,
norm_data.tolist()[-1])[0],
[norm_data[x].tolist() for x in range(len(features))])
features_corr = zip(features, corr_SanJuan)
if verbose:
y_pos = np.arange(len(features))
plt.bar(y_pos, corr_SanJuan, align='center')
plt.xticks(y_pos, features, rotation=90)
plt.ylabel('Correlation')
plt.title('Correlation features vs target')
plt.show()
print ''
print tabulate(features_corr, headers=['Feature', 'R value'])
#Selection of features with correlation greater than 0.7.
features_selected = [
features_corr[i][0] for i in range(len(features_corr))
if abs(features_corr[i][1]) > 0.7
]
#Else, select features with high correlation.
if len(features_selected) == 0:
features_selected = [
features_corr[i][0] for i in range(len(features_corr))
if abs(features_corr[i][1]) >= 0.41
]
return features_selected
def kMeans_clustering(data, verbose=False):
# 1. Data normalization
no_nan_data = data.dropna(how='any')
norm_data = norm.normalization_with_minmax(data)
# 2. Principal Component Analysis
estimator, X_pca = norm.pca(norm_data)
#plt.plot(X_pca[:,0], X_pca[:,1],'x')
# 3. Setting parameters (ad-hoc)
# parameters
init = 'k-means++' # initialization method
iterations = 10 # to run 10 times with different random centroids to choose the final model as the one with the lowest SSE
max_iter = 300 # maximum number of iterations for each single run
tol = 1e-04 # controls the tolerance with regard to the changes in the within-cluster sum-squared-error to declare convergence
random_state = 0 # random
distortions = []
silhouettes = []
for i in range(2, 11):
km = KMeans(i,
init,
n_init=iterations,
max_iter=max_iter,
tol=tol,
random_state=random_state)
labels = km.fit_predict(norm_data)
distortions.append(km.inertia_)
silhouettes.append(metrics.silhouette_score(norm_data, labels))
if verbose:
# 4. Plot results to know which K set
# Plot distoritions
plt.plot(range(2, 11), distortions, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.show()
# Plot Silhouette
plt.plot(range(2, 11), silhouettes, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Silohouette')
plt.show()
# Set K value
k = distortions.index(max(distortions)) + 2
### 5. Execute clustering
km = KMeans(k,
init,
n_init=iterations,
max_iter=max_iter,
tol=tol,
random_state=random_state)
labels = km.fit_predict(norm_data)
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
### 6. Plot the results
if verbose:
print '\n\n K value is: ' + str(k) + '\n\n'
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=labels)
plt.grid()
plt.show()
print('Estimated number of clusters: %d' % n_clusters_)
numbers = no_nan_data.index.values
n_total = len(numbers)
groups = {}
outliers = []
for c in range(1, n_clusters_ + 1):
elements = []
for i in range(len(norm_data)):
if labels[i] == c - 1:
elements.append(numbers[i])
groups[c] = elements
n_elements = len(elements)
percent = (float(n_elements) / float(n_total)) * 100.0
if (percent <= 2.0):
outliers.append(elements)
if verbose:
print '\nGroup %2d, length: %d, total %d, percent %5.2f ' % (
c, n_elements, n_total, percent)
if (len(outliers) == 0):
outliers = None
return groups, outliers
def hierarchical_clustering(data, cut=None, first_total=None, verbose=False):
#Normalization Data
no_nan_data = data.dropna(how='any')
norm_data = norm.normalization_with_minmax(data)
estimator, X_pca = norm.pca(norm_data)
if verbose:
norm.pca_plots(estimator, X_pca, no_nan_data.index.values)
# 1. Hierarchical Clustering
# 1.1. Compute the similarity matrix
dist = sklearn.neighbors.DistanceMetric.get_metric('euclidean')
matsim = dist.pairwise(X_pca)
avSim = np.average(matsim)
if verbose:
print "%s\t%6.2f" % ('Average Distance', avSim)
# 1.2. Building the Dendrogram
methods = [
"single", "complete", "average", "weighted", "centroid", "median",
"ward"
]
selec_meth = methods[0]
criterions = ["inconsistent", "distance", "maxclust"]
sel_crit = criterions[1]
#cut = 3.7 # ad-hoc
clusters = cluster.hierarchy.linkage(matsim, method=selec_meth)
# http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.cluster.hierarchy.dendrogram.html
if verbose:
plt.figure(figsize=(10, 10))
dendrogram_data = cluster.hierarchy.dendrogram(clusters,
no_plot=(not verbose),
color_threshold=7)
flatten = lambda l: [item for sublist in l for item in sublist]
dendrogram_flat = flatten(dendrogram_data.get('dcoord'))
if not cut:
cut = max(dendrogram_flat)
dendrogram_flat.remove(cut)
max_distance = cut - 0.1
else:
max_distance = max(dendrogram_flat) - 0.1
if verbose:
plt.title('%s color_threshold: %d' % (selec_meth, 7))
plt.show()
labels = cluster.hierarchy.fcluster(clusters,
max_distance,
criterion=sel_crit)
# 4. plot
numbers = no_nan_data.index.values
n_total = len(numbers)
if not first_total:
first_total = n_total
if verbose:
colors = np.array([x for x in 'bgrcmykbgrcmykbgrcmykbgrcmyk'])
colors = np.hstack([colors] * 20)
fig, ax = plt.subplots()
for i in range(n_total):
plt.text(X_pca[i][0],
X_pca[i][1],
numbers[i],
color=colors[labels[i]])
plt.xlim(min(X_pca[:, 0] - 0.2), max(X_pca[:, 0]) + 0.2)
plt.ylim(min(X_pca[:, 1] - 0.2), max(X_pca[:, 1]) + 0.2)
ax.grid(True)
fig.tight_layout()
plt.title('Method: %s, Cut: %5.2f, Criterion: %s' %
(selec_meth, max_distance, sel_crit))
plt.show()
# 5. characterization
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
if verbose:
print('Estimated number of clusters: %d' % n_clusters_)
#print 'cut: %.2f' % cut
groups = {}
outliers = []
cut_percent = cut * (1 - avSim + 0.2)
max_cut = cut - cut_percent
for c in range(1, n_clusters_ + 1):
elements = []
for i in range(len(norm_data)):
if labels[i] == c:
elements.append(numbers[i])
groups[c] = elements
n_elements = len(elements)
percent = (float(n_elements +
(first_total - n_total)) / float(n_total)) * 100.0
if (percent <= 2.0 and max_cut <= max_distance):
outliers.append(elements)
del groups[c]
if verbose:
print 'Para eliminar 2%: \t'
print elements
elif (percent <= 10.0 and max_cut <= max_distance):
df = pd.DataFrame(data, index=elements)
sub_elements, sub_outliers, cut = hierarchical_clustering(
df, cut=cut, first_total=first_total, verbose=False)
if verbose and sub_outliers:
print 'Para eliminar 10%: \t'
print sub_outliers
if sub_outliers:
outliers.append(sub_outliers)
else:
outliers = None
flatten = lambda l: [item for sublist in l for item in sublist]
if outliers:
final_outliers = flatten(outliers)
else:
final_outliers = None
return groups, final_outliers, cut
def hierarchical_clustering_features(data, verbose = True):
names = list(data)
data_transpose = data.transpose();
#1. Normalization of the data
data_transpose_norm = norm.normalization_with_minmax(data_transpose)
#1.2. Principal Component Analysis
estimator, X_pca = norm.pca(data_transpose_norm)
#------------------------
if verbose:
plt.plot(X_pca[:,0], X_pca[:,1],'x')
print("Variance Ratio: ", estimator.explained_variance_ratio_)
fig, ax = plt.subplots()
for i in range(len(data_transpose)):
plt.text(X_pca[i][0], X_pca[i][1], i+1)
plt.xlim(min(X_pca[: , 0] - 0.2), max(X_pca[: , 0]) + 0.2)
plt.ylim(min(X_pca[: , 1] - 0.2), max(X_pca[: , 1]) + 0.2)
ax.grid(True)
fig.tight_layout()
plt.show()
data_transpose_name = zip(range(1,len(names) + 1), names)
print tabulate(data_transpose_name, headers = ['# ','Feature name'])
# 2. Compute the similarity matrix
dist = sklearn.neighbors.DistanceMetric.get_metric('euclidean')
matsim = dist.pairwise(data_transpose_norm)
avSim = np.average(matsim)
clusters = cluster.hierarchy.linkage(matsim, method = 'complete')
labels = cluster.hierarchy.fcluster(clusters ,6, criterion = 'maxclust')
feature_name = data_transpose.index.values
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
groups = {}
for c in range(1, n_clusters_+1):
elements = []
for i in range(len(data_transpose_norm)):
if labels[i] == c:
elements.append(feature_name[i])
groups[c] = elements
if verbose:
print "%s\t%6.2f" % ('Average Distance', avSim)
plt.figure(figsize = (7,7));
# 3. Building the Dendrogram
# http://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
# http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.cluster.hierarchy.dendrogram.html
cluster.hierarchy.dendrogram(clusters, color_threshold = 5, labels = names, leaf_rotation=90)
print('Estimated number of clusters: %d' % n_clusters_)
plt.show()
return groups
