def test_cluster_perfect_center(self): centroids = np.zeros((self.cluster_count,dna.DNA.kmer_hash_count)) centroids[0,:] = multinomial.fit_nonzero_parameters([self.contigs[0],self.contigs[1]]) centroids[1,:] = multinomial.fit_nonzero_parameters([self.contigs[2],self.contigs[3]]) correct_clusters = kmeans._expectation(self.contigs,multinomial.log_probabilities,centroids) self.params["centroids"] = centroids (clusters, clust_prob,new_centroids) = kmeans._clustering(self.cluster_count, self.max_iter, self.run, self.epsilon, self.verbose, multinomial.log_probabilities, multinomial.fit_nonzero_parameters, **self.params) assert_equal(kmeans._evaluate_clustering(multinomial.log_probabilities, correct_clusters, centroids),clust_prob)
def test_generate_kplusplus_centroids10000(self): centroids = kmeans._generate_kplusplus(self.contigs,multinomial.log_probabilities,multinomial.fit_nonzero_parameters,self.cluster_count,dna.DNA.kmer_hash_count,self.rs) print>>sys.stderr, centroids.shape print>>sys.stderr, len(self.contigs) print>>sys.stderr, self.cluster_count self.params["centroids"] = centroids (clusters,clust_prob,new_centroids) = kmeans._clustering(self.cluster_count,self.max_iter, self.run, self.epsilon, self.verbose, multinomial.log_probabilities,multinomial.fit_nonzero_parameters,**self.params) assert_equal(len(centroids), self.cluster_count) assert_equal(len(centroids[0]),dna.DNA.kmer_hash_count ) assert_equal(np.sum(centroids,axis=1).all(),1)
def test_generate_kplusplus_centroids(self): centroids = kmeans._generate_kplusplus(self.contigs,multinomial.log_probabilities, multinomial.fit_nonzero_parameters,self.cluster_count,dna.DNA.kmer_hash_count,self.rs) assert_equal(len(centroids), self.cluster_count) assert_equal(len(centroids[0]),dna.DNA.kmer_hash_count ) assert_equal(np.sum(centroids,axis=1).all(),1) correct_centroids = np.zeros((self.cluster_count,dna.DNA.kmer_hash_count)) correct_centroids[0,:] = multinomial.fit_nonzero_parameters([self.contigs[0], self.contigs[1]]) correct_centroids[1,:] = multinomial.fit_nonzero_parameters([self.contigs[2], self.contigs[3]]) correct_clusters = kmeans._expectation(self.contigs,multinomial.log_probabilities,correct_centroids) correct_clust_prob = kmeans._evaluate_clustering(multinomial.log_probabilities, correct_clusters,correct_centroids) self.params["centroids"] = correct_centroids (clusters, clust_prob,new_centroids) = kmeans._clustering(self.cluster_count, self.max_iter, self.run, self.epsilon, self.verbose, multinomial.log_probabilities, multinomial.fit_nonzero_parameters, **self.params) print clust_prob assert_almost_equal(0, min(np.abs(clust_prob - np.array([-1659.9510320847476, -1652.322663414292, -1658.28785337, -1665.52431153]))))
def _clustering_XXXXX_NOT_USED(cluster_count, max_iter, run, epsilon, verbose, log_probabilities_func, fit_nonzero_parameters_func, **kwargs): contigs = kwargs["contigs"] p = kwargs["centroids"] if 'model_coverage' in kwargs and kwargs['model_coverage'] is not None: print >> sys.stderr, "Model coverage in em" sys.exit(-1) if not np.any(p): clustering,_, p = kmeans._clustering(cluster_count, max_iter=3, run=run, epsilon=epsilon, verbose=verbose, log_probabilities_func=log_probabilities_func, fit_nonzero_parameters_func=fit_nonzero_parameters_func, **kwargs) n = np.array([len(cluster) for cluster in clustering]) exp_log_qs, max_log_qs = _get_exp_log_qs(contigs,log_probabilities_func,p) z = _expectation(contigs,n,exp_log_qs) prev_prob,_,_ = _evaluate_clustering(contigs, log_probabilities_func, p, z) else: print >> sys.stderr, "Not implemented for EM to start with fixed p (centroids)" sys.exit(-1) if verbose: _log_current_status(contigs,prev_prob,p,z,run) prob_diff = np.inf prev_prob = -np.inf iteration = 0 while(max_iter - iteration > 0 and prob_diff >= epsilon): z = _expectation(contigs,n,exp_log_qs) p = _maximization(contigs,fit_nonzero_parameters_func,z) curr_prob, exp_log_qs, max_log_qs = _evaluate_clustering(contigs,log_probabilities_func,p,z) n = np.sum(z,axis=0,keepdims=True) prob_diff = curr_prob - prev_prob if verbose: _log_current_status(contigs,curr_prob,p,z,run) (curr_prob,prev_prob) = (prev_prob,curr_prob) iteration += 1 #Change back so curr_prob represents the highest probability (curr_prob,prev_prob) = (prev_prob,curr_prob) print >> sys.stderr, "EM iterations: {0}, difference: {1}".format(iteration, prob_diff) if prob_diff < 0: print >> sys.stderr, "EM got worse, diff: {0}".format(prob_diff) return (clustering, curr_prob, p)