def f1(): model = AgglomerativeClustering(n_clusters=claster_number, affinity='euclidean', linkage='complete') # model.fit(matrix.toarray()) # print (model.distance) # print(linkage(matrix.toarray(), method='single', metric='euclidean')) preds = model.fit_predict(matrix.toarray()) res = dict() for i, p in enumerate(preds): # print(p) res[basename(dataset['filenames'][i])] = dataset['target_names'][p] prev = None for k, v in sorted(res.items(), key=operator.itemgetter(1)): if prev != v: print(v, ':') prev = v print('\t\t', k) dist = 1 - cosine_similarity(matrix.toarray()) row_sums = dist.sum(axis=1) new_matrix = dist / row_sums[:, np.newaxis] plt.figure(figsize=(20, 20), dpi=300) sb.heatmap(new_matrix) lbls = list() for fn in dataset['filenames']: lbls.append(basename(fn)[:-4]) plt.xticks(np.arange(0, article_number), lbls, rotation='vertical') plt.yticks(np.arange(0, article_number), lbls, rotation='horizontal') plt.show() print(())
def _global_clustering(self, X=None): #对fitting之后获得的subclusters进行global_clustering clusterer = self.n_clusters centroids = self.subcluster_centers_ compute_labels = (X is not None) and self.compute_labels # 预处理 not_enough_centroids = False if isinstance(clusterer, int): clusterer = AgglomerativeClustering(n_clusters=self.n_clusters) if len(centroids) < self.n_clusters: not_enough_centroids = True elif (clusterer is not None): raise ValueError("n_clusters should be an instance of " "ClusterMixin or an int") # 避免predict环节,重复运算 self._subcluster_norms = row_norms( self.subcluster_centers_, squared=True) if clusterer is None or not_enough_centroids: self.subcluster_labels_ = np.arange(len(centroids)) if not_enough_centroids: warnings.warn( "Number of subclusters found (%d) by Birch is less than (%d). Decrease the threshold."% (len(centroids), self.n_clusters)) else: # 对所有叶子节点的subcluster进行聚类,它将subcluster的centroids作为样本,并且找到最终的centroids. self.subcluster_labels_ = clusterer.fit_predict( self.subcluster_centers_) if compute_labels: self.labels_ = self.predict(X)
def agglomerative_clustering (matrix): print "====== Agglomerative Clustering ===============" model = AgglomerativeClustering(n_clusters=3, affinity='cosine', linkage='complete') preds = model.fit_predict(matrix) clusters = model.labels_.tolist() return (preds, clusters)
def test_n_clusters(): # Test that n_clusters param works properly X, y = make_blobs(n_samples=100, centers=10) brc1 = Birch(n_clusters=10) brc1.fit(X) assert len(brc1.subcluster_centers_) > 10 assert len(np.unique(brc1.labels_)) == 10 # Test that n_clusters = Agglomerative Clustering gives # the same results. gc = AgglomerativeClustering(n_clusters=10) brc2 = Birch(n_clusters=gc) brc2.fit(X) assert_array_equal(brc1.subcluster_labels_, brc2.subcluster_labels_) assert_array_equal(brc1.labels_, brc2.labels_) # Test that the wrong global clustering step raises an Error. clf = ElasticNet() brc3 = Birch(n_clusters=clf) with pytest.raises(ValueError): brc3.fit(X) # Test that a small number of clusters raises a warning. brc4 = Birch(threshold=10000.) assert_warns(ConvergenceWarning, brc4.fit, X)
def classification_clustering(X, n_clusters=1): X = tfidf(X) clustering = AgglomerativeClustering(n_clusters=n_clusters, compute_full_tree=True).fit(X) print(X.shape) print(clustering.labels_) print(clustering.children_) print(len(clustering.children_))
def _global_clustering(self, X=None): """ Global clustering for the subclusters obtained after fitting """ clusterer = self.n_clusters centroids = self.subcluster_centers_ compute_labels = (X is not None) and self.compute_labels # Preprocessing for the global clustering. not_enough_centroids = False if isinstance(clusterer, int): clusterer = AgglomerativeClustering( n_clusters=self.n_clusters) # There is no need to perform the global clustering step. if len(centroids) < self.n_clusters: not_enough_centroids = True elif (clusterer is not None and not hasattr(clusterer, 'fit_predict')): raise ValueError("n_clusters should be an instance of " "ClusterMixin or an int") # To use in predict to avoid recalculation. self._subcluster_norms = row_norms( self.subcluster_centers_, squared=True) if clusterer is None or not_enough_centroids: self.subcluster_labels_ = np.arange(len(centroids)) if not_enough_centroids: warnings.warn( "Number of subclusters found (%d) by Birch is less " "than (%d). Decrease the threshold." % (len(centroids), self.n_clusters)) else: # The global clustering step that clusters the subclusters of # the leaves. It assumes the centroids of the subclusters as # samples and finds the final centroids. self.subcluster_labels_ = clusterer.fit_predict( self.subcluster_centers_) if compute_labels: self.labels_ = self.predict(X)
def cluster_faces(self, eucl_dist_vecs, n_clusters=30): """ :param eucl_dist_vecs: A list of euclidian distances as returned from FaceRecognitionModel.get_vector() :param cluster_range_min: Minimal number of clusters to produce :param cluster_range_max: Maximal number of clusters to produce :return: A dict(key:n_clusters, val=list[label_idx][data_idx]) """ X = np.array(eucl_dist_vecs) clustering = AgglomerativeClustering(n_clusters=n_clusters).fit(X) return clustering.labels_
X=np.array([[0.697,0.460],[0.774,0.376],[0.634,0.264],[0.608,0.318],[0.556,0.215], [0.403,0.237],[0.481,0.149],[0.437,0.211],[0.666,0.091],[0.243,0.267], [0.245,0.057],[0.343,0.099],[0.639,0.161],[0.657,0.198],[0.360,0.370], [0.593,0.042],[0.719,0.103],[0.359,0.188],[0.339,0.241],[0.282,0.257], [0.748,0.232],[0.714,0.346],[0.483,0.312],[0.478,0.437],[0.525,0.369], [0.751,0.489],[0.532,0.472],[0.473,0.376],[0.725,0.445],[0.446,0.459]]) X_test=X agnes=AGNES() agnes.fit(X) print('C:', agnes.C) print(agnes.labels_) plt.figure(12) plt.subplot(121) plt.scatter(X[:, 0], X[:, 1], c=agnes.labels_) plt.title('tinyml') from sklearn.cluster.hierarchical import AgglomerativeClustering sklearn_agnes=AgglomerativeClustering(n_clusters=7,affinity='l2',linkage='average') sklearn_agnes.fit(X) print(sklearn_agnes.labels_) plt.subplot(122) plt.scatter(X[:,0],X[:,1],c=sklearn_agnes.labels_) plt.title('sklearn') plt.show()
def run_concurrent(self, args, sign_progress): idx = 0 movie_path = args[0] resolution = args[1] in_hists = args[3] fps = args[2] indices = args[4] frame_resolution = args[5] n_cluster_range = args[6] alt_resolution = args[7] cluster_sizes = range(n_cluster_range[0], n_cluster_range[1], 1) histograms = [] frames = [] cap = cv2.VideoCapture(movie_path) length = cap.get(cv2.CAP_PROP_FRAME_COUNT) resize_f = 192.0 / cap.get(cv2.CAP_PROP_FRAME_WIDTH) data_idx = 0 read_img = -1 # We only want to read every second image if indices is None: indices = [] resolution = alt_resolution for i in range(int(length)): if self.aborted: return None if i % resolution == 0: read_img += 1 if in_hists is not None and data_idx >= len(in_hists): break if read_img % frame_resolution == 0: cap.set(cv2.CAP_PROP_POS_FRAMES, i) ret, frame = cap.read() frames.append( cv2.resize(frame, None, None, resize_f, resize_f, cv2.INTER_CUBIC)) read_img = 0 sign_progress(i / length) if in_hists is not None: histograms.append( np.resize(in_hists[data_idx], new_shape=16**3)) else: cap.set(cv2.CAP_PROP_POS_FRAMES, i) ret, frame = cap.read() if frame is None: break frame = cv2.cvtColor(floatify_img(frame), cv2.COLOR_BGR2LAB) data = np.resize(frame, (frame.shape[0] * frame.shape[1], 3)) hist = cv2.calcHist([data[:, 0], data[:, 1], data[:, 2]], [0, 1, 2], None, [16, 16, 16], [0, 100, -128, 128, -128, 128]) indices.append(i) histograms.append(np.resize(hist, new_shape=16**3)) data_idx += 1 connectivity = np.zeros(shape=(len(histograms), len(histograms)), dtype=np.uint8) for i in range(1, len(histograms) - 1, 1): connectivity[i][i - 1] = 1 connectivity[i][i] = 1 connectivity[i][i + 1] = 1 clusterings = [] for i, n_cluster in enumerate(cluster_sizes): sign_progress(i / len(cluster_sizes)) if len(histograms) > n_cluster: model = AgglomerativeClustering(linkage="ward", connectivity=connectivity, n_clusters=n_cluster, compute_full_tree=True) model.fit(histograms) clusterings.append(model.labels_) return [ clusterings, frames, indices, fps, frame_resolution, n_cluster_range ]
def cluster(diss_matrix, n_clusters): agglo_clusterer = AgglomerativeClustering(affinity='precomputed', linkage='average', n_clusters=n_clusters) return agglo_clusterer.fit(diss_matrix).labels_
from sklearn.preprocessing.data import StandardScaler from sklearn.manifold.t_sne import TSNE from sklearn.linear_model.theil_sen import TheilSenRegressor from sklearn.mixture.dpgmm import VBGMM from sklearn.feature_selection.variance_threshold import VarianceThreshold import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) clf_dict = {'ARDRegression':ARDRegression(), 'AdaBoostClassifier':AdaBoostClassifier(), 'AdaBoostRegressor':AdaBoostRegressor(), 'AdditiveChi2Sampler':AdditiveChi2Sampler(), 'AffinityPropagation':AffinityPropagation(), 'AgglomerativeClustering':AgglomerativeClustering(), 'BaggingClassifier':BaggingClassifier(), 'BaggingRegressor':BaggingRegressor(), 'BayesianGaussianMixture':BayesianGaussianMixture(), 'BayesianRidge':BayesianRidge(), 'BernoulliNB':BernoulliNB(), 'BernoulliRBM':BernoulliRBM(), 'Binarizer':Binarizer(), 'Birch':Birch(), 'CCA':CCA(), 'CalibratedClassifierCV':CalibratedClassifierCV(), 'DBSCAN':DBSCAN(), 'DPGMM':DPGMM(), 'DecisionTreeClassifier':DecisionTreeClassifier(), 'DecisionTreeRegressor':DecisionTreeRegressor(), 'DictionaryLearning':DictionaryLearning(),
def run_experiment(ae_model_path): logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info(f"Working now on {ae_model_path.name}") logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) logger.info( f"++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" ) new_seed = random.randint(0, 1000) logger.info(f"Seed value for this is: {new_seed}") set_random_seed(new_seed) ae_module = stacked_ae(pt_data.shape[1], [500, 500, 2000, 10], weight_initalizer=torch.nn.init.xavier_normal_, activation_fn=lambda x: F.relu(x), loss_fn=None, optimizer_fn=None) model_data = torch.load(ae_model_path, map_location='cpu') ae_module.load_state_dict(model_data) ae_module = ae_module.cuda() # Get embedded data embedded_data = None for batch_data in torch.utils.data.DataLoader(pt_data, batch_size=256, shuffle=False): embedded_batch_np = ae_module.forward( batch_data.cuda())[0].detach().cpu().numpy() if embedded_data is None: embedded_data = embedded_batch_np else: embedded_data = np.concatenate([embedded_data, embedded_batch_np], 0) del ae_module sl_cl = AgglomerativeClustering(compute_full_tree=True, n_clusters=n_clusters, linkage="single").fit(embedded_data) sl_labels = sl_cl.labels_ sl_purity_tree = prune_dendrogram_purity_tree( to_dendrogram_purity_tree(sl_cl.children_), n_leaf_nodes_final) sl_nmi = nmi(gold_labels, sl_labels, average_method='arithmetic') sl_acc = cluster_acc(sl_labels, gold_labels)[0] sl_purity = dendrogram_purity(sl_purity_tree, gold_labels) sl_lp = leaf_purity(sl_purity_tree, gold_labels) sl_leaf_purity_value = f"{sl_lp[0]:1.3}\t({sl_lp[1]:1.3})" result_file_sl = Path( f"{result_dir}/results_ae_agglo_single_{dataset_name}.txt") result_file_sl_exists = result_file_sl.exists() f = open(result_file_sl, "a+") if not result_file_sl_exists: f.write( "#\"ae_model_name\"\t\"NMI\"\t\"ACC\"\t\"Dendrogram_Purity\"\t\"Leaf_Purity\t(Std)\"\n" ) f.write( f"{ae_model_path.name}\t{sl_nmi}\t{sl_acc}\t{sl_purity}\t{sl_leaf_purity_value}\n" ) f.close() del sl_cl, sl_labels, sl_purity_tree cl_cl = AgglomerativeClustering(compute_full_tree=True, n_clusters=n_clusters, linkage="complete").fit(embedded_data) cl_labels = cl_cl.labels_ cl_purity_tree = prune_dendrogram_purity_tree( to_dendrogram_purity_tree(cl_cl.children_), n_leaf_nodes_final) cl_nmi = nmi(gold_labels, cl_labels, average_method='arithmetic') cl_acc = cluster_acc(cl_labels, gold_labels)[0] cl_purity = dendrogram_purity(cl_purity_tree, gold_labels) cl_lp = leaf_purity(cl_purity_tree, gold_labels) cl_leaf_purity_value = f"{cl_lp[0]:1.3}\t({cl_lp[1]:1.3})" result_file_cl = Path( f"{result_dir}/results_ae_agglo_complete_{dataset_name}.txt", ) result_file_cl_exists = result_file_cl.exists() f = open(result_file_cl, "a+") if not result_file_cl_exists: f.write( "#\"ae_model_name\"\t\"NMI\"\t\"ACC\"\t\"Dendrogram_Purity\"\t\"Leaf_Purity\t(Std)\"\n" ) f.write( f"{ae_model_path.name}\t{cl_nmi}\t{cl_acc}\t{cl_purity}\t{cl_leaf_purity_value}\n" ) f.close() del cl_cl, cl_labels, cl_purity_tree
z = linkage(df_norm1, method='complete', metric='euclidean') from sklearn.cluster.hierarchical import AgglomerativeClustering import sklearn.cluster.hierarchical as shch plt.figure(figsize=(15, 5)) plt.title('Hierarchical Clustering Dendogram') plt.xlabel('Index') plt.ylabel('Distance') sch.dendrogram(z, leaf_rotation=0, leaf_font_size=8) plt.show() h_clustering = AgglomerativeClustering(n_clusters=6, affinity="euclidean", linkage="complete").fit(df_norm1) h_clustering h = pd.Series(h_clustering.labels_) h crime2['clust'] = h crime2 = crime2.iloc[:, [5, 0, 1, 2, 3, 4]] crime2.iloc[:, 2:].groupby(crime2.clust).median() crime2.to_csv("crime2.csv", encoding="utf-8") import os os.getcwd()
plt.scatter(X[:, 0], X[:, 1]) plt.show() #kMeans clustering from sklearn.cluster import KMeans km = KMeans(init='random', max_iter=150, n_clusters=2, random_state=0) y_km = km.fit_predict(X) plt.scatter(X[y_km == 0, 0], X[y_km == 0, 1], c='green') plt.scatter(X[y_km == 1, 0], X[y_km == 1, 1], c='red') plt.title("KMeans") plt.show() #Agglomerative Clustering with complete linkage from sklearn.cluster.hierarchical import AgglomerativeClustering aggcl = AgglomerativeClustering(n_clusters=2, linkage='complete') y_agcl = aggcl.fit_predict(X) plt.scatter(X[y_agcl == 0, 0], X[y_agcl == 0, 1], c='green') plt.scatter(X[y_agcl == 1, 0], X[y_agcl == 1, 1], c='red') plt.title("Aggolomerative Clustering") plt.show() #Demonstaring clustering using density-based approach from sklearn.cluster import DBSCAN dbs = DBSCAN(eps=0.2, min_samples=5) y_dbs = dbs.fit_predict(X) plt.scatter(X[y_dbs == 0, 0], X[y_dbs == 0, 1], c='green') plt.scatter(X[y_dbs == 1, 0], X[y_dbs == 1, 1], c='red') plt.title("Density based(DBSCAN) Clustering")
print(len(i)) # In[24]: for cluster in dbscan_clusters: print(data.loc[cluster].mean()) # In[11]: from sklearn.cluster.hierarchical import AgglomerativeClustering model = AgglomerativeClustering(n_clusters=4, linkage='average', affinity='manhattan') aggl_preds = model.fit_predict(scaled_features) # In[12]: clusters_aggl = [] for lbl in np.unique(aggl_preds): indices = [i for i, x in enumerate(aggl_preds) if x == lbl] clusters_aggl.append(indices) # In[25]:
vectors = np.loadtxt(config("path.article-embeds")) # checking vectors length: mean ~0.46, std ~ 0.05, max = 1, min ~ 0.29 # stats = np.array([np.linalg.norm(vectors[i, :]) for i in range(vectors.shape[0])]) pick_recent = 3 * 25 * 365 clusters = model.fit_predict(vectors[-pick_recent:]) clust_series = pd.Series(clusters, name='Cluster') tagged = pd.concat( [clust_series, df.iloc[-pick_recent:].reset_index(drop=True)], axis=1, sort=False) tagged[["News", "Cluster"]].iloc[-pick_recent:].to_csv("for_visualization.tsv", sep="\t", index=False, header=False) tagged.sort_values(by='Cluster').to_csv( "explore_topics({}).tsv".format(label), sep="\t", index=False) print(tagged.Cluster.value_counts()) if __name__ == '__main__': cluster_and_dump( AgglomerativeClustering(n_clusters=100, affinity="cosine", linkage="complete"), "100,cos,complete") # cluster_and_dump(AgglomerativeClustering(n_clusters=100), "ward") # cluster_and_dump(DBSCAN(eps=0.19), "dbscan")
def process(self, args, sign_progress): """ This is the actual analysis, which takes place in a WorkerThread. Do NOT and NEVER modify the project within this function. We want to read though the movie and get the Average Colors from each Segment. Once done, we create an Analysis Object from it. """ args, sign_progress = super(ShotSegmentationAnalysis, self).process(args, sign_progress) # Signal the Progress sign_progress(0.0) video_capture = cv2.VideoCapture(args['movie_path']) duration = video_capture.get(cv2.CAP_PROP_FRAME_COUNT) resize_f = 192.0 / video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) resize_clamp = self.frame_width_clamp / video_capture.get( cv2.CAP_PROP_FRAME_WIDTH) width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH) height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT) n = int(np.floor(duration / self.resolution)) X = np.zeros(shape=(n, 16**3), dtype=np.float16) frame_pos = np.zeros(shape=n, dtype=np.int32) frames = [] for i in range(int(n)): if self.aborted: return None idx = int(i * self.resolution) video_capture.set(cv2.CAP_PROP_POS_FRAMES, idx) ret, frame = video_capture.read() if frame is None: continue if self.return_frames and i % self.frame_resolution == 0: frames.append( cv2.resize(frame, None, None, resize_f, resize_f, cv2.INTER_CUBIC)) if resize_clamp < 1.0: frame = cv2.resize(frame, None, None, resize_clamp, resize_clamp, cv2.INTER_CUBIC) frame = cv2.cvtColor(floatify_img(frame), cv2.COLOR_BGR2LAB) X[i] = np.resize(calculate_histogram(frame, lab_mode=True), new_shape=16**3) / (width * height) frame_pos[i] = idx sign_progress(round(i / n, 4)) connectivity = np.zeros(shape=(n, n), dtype=np.uint8) for i in range(1, n - 1, 1): connectivity[i][i - 1] = 1 connectivity[i][i] = 1 connectivity[i][i + 1] = 1 clusterings = [] cluster_sizes = range(self.cluster_range[0], self.cluster_range[1], 1) for i, n_cluster in enumerate(cluster_sizes): sign_progress(i / len(cluster_sizes)) if X.shape[0] > n_cluster: model = AgglomerativeClustering(linkage="ward", connectivity=connectivity, n_clusters=n_cluster, compute_full_tree=True) model.fit(X) timestamps = self._generate_segments( model.labels_, frame_pos, video_capture.get(cv2.CAP_PROP_FPS)) clusterings.append(timestamps) if self.return_hdf5_compatible: result = np.zeros(shape=self.dataset_shape, dtype=self.dataset_dtype) for i in range(len(clusterings)): result[i][0:len(clusterings[i])] = [[ c['label'], c['f_start'], c['f_stop'] ] for c in clusterings[i]] # Creating an IAnalysisJobAnalysis Object that will be handed back to the Main-Thread analysis = IAnalysisJobAnalysis( name="My Analysis", results=result, analysis_job_class=self.__class__, parameters=dict(resolution=self.resolution), container=args['movie_descriptor']) else: analysis = AnalysisContainer(name=self.name, data=clusterings) sign_progress(1.0) return analysis