def silhouette_samples(X, labels, metric='euclidean', **kwds): """Compute the Silhouette Coefficient for each sample. The Silhouette Coefficient is a measure of how well samples are clustered with samples that are similar to themselves. Clustering models with a high Silhouette Coefficient are said to be dense, where samples in the same cluster are similar to each other, and well separated, where samples in different clusters are not very similar to each other. The Silhouette Coefficient is calculated using the mean intra-cluster distance (``a``) and the mean nearest-cluster distance (``b``) for each sample. The Silhouette Coefficient for a sample is ``(b - a) / max(a, b)``. Note that Silhouette Coefficient is only defined if number of labels is 2 <= n_labels <= n_samples - 1. This function returns the Silhouette Coefficient for each sample. The best value is 1 and the worst value is -1. Values near 0 indicate overlapping clusters. Read more in the :ref:`User Guide <silhouette_coefficient>`. Args: X : array [n_samples_a, n_samples_a] if metric == “precomputed”, or, [n_samples_a, n_features] otherwise Array of pairwise distances between samples, or a feature array. labels : array, shape = [n_samples] label values for each sample metric: string, or callable The metric to use when calculating distance between instances in a feature array. If metric is a string, it must be one of the options allowed by sklearn.metrics.pairwise.pairwise_distances. If X is the distance array itself, use “precomputed” as the metric. `**kwds` : optional keyword parameters Any further parameters are passed directly to the distance function. If using a scipy.spatial.distance metric, the parameters are still metric dependent. See the scipy docs for usage examples. Returns: silhouette : array, shape = [n_samples] Silhouette Coefficient for each samples. """ return unsupervised.silhouette_samples(X, labels, metric, **kwds)
def compute_silhouette(self): result = unsupervised.silhouette_samples(self.D, self.labels, metric='precomputed') mean_silhouette_labels = dict() for l in set(self.labels): if l != -1: label_mask = np.asarray(self.labels) != l masked_result = ma.masked_array(result, mask=label_mask) mean_silhouette_labels[l] = masked_result.mean() self.mean_silhouette_labels = mean_silhouette_labels return self.mean_silhouette_labels, result
def determine_anomaly_tweets_kmeans(top_n, pca_vectors, cluster_labels): #NOTE: #pca_vectors = PCA vectors of the top 2 PCA components only #cluster_labels = labels of 2 clusters only silhouette_values = silhouette_samples(X=pca_vectors, labels=cluster_labels, metric='cosine') #My understanding of silhouette_samples() is that it returns the silhouette score of an individual data point in a cluster tweet_index_silhouette_scores = [ ] #this will hold the tweet index & its actual sh_score absolute_silhouette_scores_tweet_index = [ ] #this will hold the tweet index & its absolute sh_score for index, sh_score in enumerate(silhouette_values): absolute_silhouette_scores_tweet_index.append((abs(sh_score), index)) #We'll need to sort the tweet indices and their sh_scores by the abs sh_score hence the abs sh_score comes first in the tuple tweet_index_silhouette_scores.append((index, sh_score)) #We'll need to retrieve tweet indices and their actual sh_score hence index comes first sorted_scores = sorted(absolute_silhouette_scores_tweet_index) #sorted() is for use in lists/tuples/strings (they're similar) when not using data frame columns top_n_silhouette_scores = [] pca_vectors_anomalies = [] print('Top ', top_n, ' anomalies:') for i in range(top_n): abs_sh_score, index = sorted_scores[ i] #get top n of sorted absolute silhouette score i.e. those approx 0 index_1, sh_score = tweet_index_silhouette_scores[ index] #get actual sh_score of top n sorted abs #top_n_silhouette_scores.append((index, sh_score)) #Results are the same with this commented out. Maybe it's for plotting? print(tweets_dict[index] ) #print actual tweet referenced by the index value print('PCA Vector: ', pca_vectors[index]) #print the tweet's PCA vectors #pca_vectors_anomalies.append(pca_vectors[index]) #Results are the same with this commented out. Maybe it's for plotting? print('Silhouette score: ', sh_score) print('.................')
def _kmeans_silhouette_train_predict(table, input_cols, n_clusters_list=range(2, 10), prediction_col='prediction', init='k-means++', n_init=10, max_iter=300, tol=1e-4, precompute_distances='auto', seed=None, n_jobs=1, algorithm='auto', n_samples=None): feature_names, features = check_col_type(table, input_cols) if n_samples is None: n_samples = len(table) inputarr = features pca2_model = PCA(n_components=min(2, len(feature_names))).fit(inputarr) pca2 = pca2_model.transform(inputarr) silhouette_list = [] models = [] centers_list = [] images = [] for k in n_clusters_list: k_means = SKKMeans(n_clusters=k, init=init, n_init=n_init, max_iter=max_iter, tol=tol, precompute_distances=precompute_distances, verbose=0, random_state=seed, copy_x=True, n_jobs=n_jobs, algorithm=algorithm) k_means.fit(inputarr) models.append(k_means) predict = k_means.labels_ centersk = k_means.cluster_centers_ centers_list.append(centersk) score = silhouette_score(inputarr, predict) silhouette_list.append(score) samples = silhouette_samples(inputarr, predict) # silhouette_samples_list.append(samples) pca2_centers = pca2_model.transform(centersk) _, (ax1, ax2) = plt.subplots(1, 2) colors = cm.nipy_spectral(np.arange(k).astype(float) / k) y_lower = 0 for i, color in zip(range(k), colors): si = samples[predict == i] si.sort() sizei = si.shape[0] y_upper = y_lower + sizei ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, si, facecolor=color, edgecolor=color, alpha=0.7) # cluster label ax1.text(0.9, y_lower + 0.45 * sizei, str(i)) y_lower = y_upper if pca2.shape[1] == 1: ax2.scatter(pca2[:, 0][predict == i], pca2[:, 0][predict == i], color=color) else: ax2.scatter(pca2[:, 0][predict == i], pca2[:, 1][predict == i], color=color) ax1.axvline(x=score, color="red") ax1.set_xlim(right=1.0) ax1.set_yticks([]) ax1.set_xlabel("Silhouette coefficient values") ax1.set_ylabel("Cluster label") if pca2.shape[1] == 1: ax2.scatter(pca2_centers[:, 0], pca2_centers[:, 0], marker='x', edgecolors=1, s=200, color=colors) ax2.set_xlabel("Feature space for the 1st feature") ax2.set_ylabel("Feature space for the 1st feature") else: ax2.scatter(pca2_centers[:, 0], pca2_centers[:, 1], marker='x', edgecolors=1, s=200, color=colors) ax2.set_xlabel("Feature space for the 1st feature") ax2.set_ylabel("Feature space for the 2nd feature") plt.tight_layout() imagek = plt2MD(plt) plt.clf() images.append(imagek) argmax = np.argmax(silhouette_list) best_k = n_clusters_list[argmax] best_model = models[argmax] predict = best_model.predict(inputarr) best_centers = best_model.cluster_centers_ best_labels = best_model.labels_ best_sse = best_model.inertia_ n_clusters = len(best_centers) colors = cm.nipy_spectral(np.arange(n_clusters).astype(float) / n_clusters) fig_centers = _kmeans_centers_plot(feature_names, best_centers, colors) fig_samples = _kmeans_samples_plot(table, input_cols, n_samples, best_centers, seed, colors) fig_pca = _kmeans_pca_plot(predict, best_centers, pca2_model, pca2, colors) x_clusters = range(len(n_clusters_list)) plt.xticks(x_clusters, n_clusters_list) plt.plot(x_clusters, silhouette_list, '.-') plt.xlabel("Number of Clusters k") plt.tight_layout() fig_silhouette = plt2MD(plt) plt.clf() rb = BrtcReprBuilder() rb.addMD(strip_margin(""" | ## Kmeans Silhouette Result | - silhoutte metrics: | {fig_silhouette} | - best K: {best_k} | - Sum of square error: {best_sse}. | - best centers: | {fig_pca} | {fig_centers} | {fig_samples} | """.format(fig_silhouette=fig_silhouette, best_k=best_k, best_sse=best_sse, fig_pca=fig_pca, fig_centers=fig_centers, fig_samples=fig_samples))) for k, image in zip(n_clusters_list, images): rb.addMD(strip_margin(""" | ### k = {k} | {image} | """.format(k=k, image=image))) model = _model_dict('kmeans_silhouette') model['best_k'] = best_k model['best_centers'] = best_centers model['best_model'] = best_model model['input_cols'] = input_cols model['_repr_brtc_'] = rb.get() out_table = table.copy() out_table[prediction_col] = predict # out_table['silhouette'] = silhouette_samples_list[best_k-2] # out_table = out_table.sort_values(by=['prediction','silhouette']) # out_table = out_table.reset_index(drop=True) return {'out_table': out_table, 'model': model}
def _kmeans_silhouette_train_predict(table, input_cols, n_clusters_list=range(2, 10), prediction_col='prediction', init='k-means++', n_init=10, max_iter=300, tol=1e-4, precompute_distances='auto', seed=None, n_jobs=1, algorithm='auto', n_samples=None): if n_samples is None: n_samples = len(table) inputarr = table[input_cols] validate(all_elements_greater_than(n_clusters_list, 1, 'n_clusters_list'), greater_than_or_equal_to(n_init, 1, 'n_init'), greater_than_or_equal_to(max_iter, 1, 'max_iter'), greater_than(tol, 0.0, 'tol'), greater_than_or_equal_to(n_jobs, 1, 'n_jobs'), greater_than_or_equal_to(n_samples, 0, 'n_samples')) pca2_model = PCA(n_components=2).fit(inputarr) pca2 = pca2_model.transform(inputarr) silhouette_list = [] silouette_samples_list = [] models = [] centers_list = [] images = [] for k in n_clusters_list: k_means = SKKMeans(n_clusters=k, init=init, n_init=n_init, max_iter=max_iter, tol=tol, precompute_distances=precompute_distances, verbose=0, random_state=seed, copy_x=True, n_jobs=n_jobs, algorithm=algorithm) k_means.fit(inputarr) models.append(k_means) predict = k_means.labels_ centersk = k_means.cluster_centers_ centers_list.append(centersk) score = silhouette_score(inputarr, predict) silhouette_list.append(score) samples = silhouette_samples(inputarr, predict) silouette_samples_list.append(samples) pca2_centers = pca2_model.transform(centersk) _, (ax1, ax2) = plt.subplots(1, 2) colors = cm.nipy_spectral(np.arange(k).astype(float) / k) y_lower = 0 for i, color in zip(range(k), colors): si = samples[predict == i] si.sort() sizei = si.shape[0] y_upper = y_lower + sizei ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, si, facecolor=color, edgecolor=color, alpha=0.7) y_lower = y_upper ax2.scatter(pca2[:, 0][predict == i], pca2[:, 1][predict == i], color=color) ax1.axvline(x=score, color="red") ax2.scatter(pca2_centers[:, 0], pca2_centers[:, 1], marker='x', edgecolors=1, s=200, color=colors) imagek = plt2MD(plt) plt.clf() images.append(imagek) argmax = np.argmax(silhouette_list) best_k = n_clusters_list[argmax] best_model = models[argmax] predict = best_model.predict(inputarr) best_centers = best_model.cluster_centers_ best_labels = best_model.labels_ fig_centers = _kmeans_centers_plot(input_cols, best_centers) fig_samples = _kmeans_samples_plot(table, input_cols, n_samples, best_centers) fig_pca = _kmeans_pca_plot(predict, best_centers, pca2_model, pca2) x_clusters = range(len(n_clusters_list)) plt.xticks(x_clusters, n_clusters_list) plt.plot(x_clusters, silhouette_list, '.-') fig_silhouette = plt2MD(plt) plt.clf() rb = BrtcReprBuilder() rb.addMD( strip_margin(""" | ## Kmeans Silhouette Result | - silloutte metrics: | {fig_silhouette} | - best K: {best_k} | - best centers: | {fig_pca} | {fig_centers} | {fig_samples} | """.format(fig_silhouette=fig_silhouette, best_k=best_k, fig_pca=fig_pca, fig_centers=fig_centers, fig_samples=fig_samples))) for k, image in zip(n_clusters_list, images): rb.addMD( strip_margin(""" | ### k = {k} | {image} | """.format(k=k, image=image))) model = _model_dict('kmeans_silhouette') model['best_k'] = best_k model['best_centers'] = best_centers model['best_model'] = best_model model['input_cols'] = input_cols model['_repr_brtc_'] = rb.get() out_table = table.copy() out_table[prediction_col] = predict return {'out_table': out_table, 'model': model}
def silhouette_plot(X, labels, metric='euclidean', fig_size=None, cluster=None, ID=None, index=False, **kwds): """Makes a silhouette bar graph. Calculate the silhouette of the samples and then plot a graph of the chosen cluster and the silhouettes of the samples or plot a graph with all the means of the silhouettes of the clusters Args: X : array [n_samples_a, n_samples_a] if metric == "precomputed", or, \ [n_samples_a, n_features] otherwise Array of pairwise distances between samples, or a feature array. labels : array, shape = [n_samples] Predicted labels for each sample. metric : string, or callable The metric to use when calculating distance between instances in a feature array. If metric is a string, it must be one of the options allowed by :func:`metrics.pairwise.pairwise_distances <sklearn.metrics.pairwise.pairwise_distances>`. If X is the distance array itself, use ``metric="precomputed"``. fig_size: number, type int Number for the figsize of the plot if fig_size == None then a calculation is made to leave a suitable size for the quantity of samples. cluster: number do cluster,type int Cluster number to generate the silhouettes graph of its samples if cluster == None will then generate a plot of the mean silhouettes of all clusters. y : Smple ID ID of the samples, used to label the bars on the y-axis **kwds : optional keyword parameters Any further parameters are passed directly to the distance function. If using a scipy.spatial.distance metric, the parameters are still metric dependent. See the scipy docs for usage examples. Returns: Void """ silhouette_samples = unsupervised.silhouette_samples( X, labels, metric, **kwds) df = pd.DataFrame(silhouette_samples) df['cluster'] = labels if cluster == None: cluster_means = df.groupby('cluster').mean() dit = dict(zip(cluster_means.index, cluster_means[0])) df2 = pd.DataFrame(list(dit.items())) df2.columns = ['Cluster', 'silhouette_mean'] if fig_size == None: if len(df2) > 64: fig = plt.figure(figsize=(len(df2) / 7, len(df2) / 3)) else: fig = plt.figure(figsize=(10, 8)) else: fig = plt.figure(figsize=fig_size) df2 = df2.sort_values(['silhouette_mean'], ascending=False).reset_index(drop=True) ax = sns.barplot(df2['silhouette_mean'], y=df2.index, orient='h') ax.set_yticklabels(df2['Cluster']) plt.ylabel('Cluster') plt.show() elif cluster != None: df['ID'] = ID cluster = df[df['cluster'] == cluster] cluster.columns = ['silhouette', 'Cluster', 'ID'] cluster = cluster.sort_values(['silhouette'], ascending=False).reset_index(drop=True) if fig_size == None: if len(cluster) > 64: fig = plt.figure(figsize=(len(cluster) / 7, len(cluster) / 3)) else: fig = plt.figure(figsize=(12, 10)) else: fig = plt.figure(figsize=fig_size) ax = sns.barplot(cluster['silhouette'], y=cluster.index, orient='h') if index != None: ax.set_yticklabels(cluster['ID']) plt.ylabel('ID samples') plt.show()
# int(max(y_pred) + 1)))) #import random r = lambda: random.randint(255) colours = numpy.array([ '#%02X%02X%02X'%(r(),r(),r()) for i in range(max(y_pred)+1) ]) print(colours) c = Counter(y_pred) print(c.most_common()) # ixs_over_thresh = [k for (k,v) in c.items() if v > 1] # print(ixs_over_thresh) # sil_X = X[ixs_over_thresh, :] # sil_y = y_pred[ixs_over_thresh] # ss = silhouette_samples(sil_X,sil_y, metric='euclidean') ss = silhouette_samples(X,y_pred, metric="euclidean") ss = [s for s in ss if not isnan(s)] print("Sil=", numpy.mean(ss)) print("CHS=",calinski_harabaz_score(X, y_pred)) pca = PCA() to_plot = pca.fit_transform(X) print(X.shape) print(y_pred.shape) w_clusters = numpy.column_stack((X,y_pred)) numpy.savetxt("w_clusters.csv", w_clusters, delimiter=",")