def cluster_spectral(self, W, num_clusters, spectral_cluster=None):
if spectral_cluster is None:
spectral_cluster = SpectralClustering()
spectral_cluster.set_params(
n_clusters=num_clusters,
affinity='precomputed',
n_init=10
)
try:
I = spectral_cluster.fit_predict(W)
except:
print 'Spectral clustering failed, clustering on identity matrix'
try:
#I = spectral_cluster.fit_predict(np.eye(W.shape[0]))
I = np.random.choice(range(0, num_clusters), W.shape[0])
except:
I = np.random.choice(range(0, num_clusters), W.shape[0])
return spectral_cluster, I
def cluster_reproducibility(self, repeats=None, clusters=50):
""" Given the tag co-occurence arrays generated by the train
method, use the spectral clustering method in sklearn and the
known (or desired) number of clusters to assign tags to
specific clusters.
Required input:
None
Optional input:
repeats - a set of co-occurence arrays to cluster using
spectral methods. If not supplied, this method
defaults to self.repeats which is the data generated
by the train() method.
labels - the tags corresponding to the feature vectors.
Labels must be correctly ordered, obviously.
Returns:
None ----BUT---- generates the following analysis in the
self namespace.'
1. self.reproduction_matrices: a reorganization of the
repeats data into block diagonal form.
2. self.reproduction_analysis: a list of dictionaries.
Each dictionary has two keys: 'members' and 'sizes'.
'members' lists the tag membership of each cluster
in terms of the indices of the feature vectors represented
by samples in train(),arranged by size.
'sizes' gives the size of each
cluster. The index of the self.reproduction_analysis
list gives the number of clusters remainging from
the agglomeration. For example,
self.reproduction_analysis[10][4]['members'] lists the
tag indices of the 5th largest cluster when there are
11 clusters remaining from the agglomeration.
"""
def _find(where, what):
""" Helper """
return np.where(where == what[0])[0].tolist()
from sklearn.cluster import SpectralClustering
from collections import Counter
if repeats == None:
repeats = self.repeats
spectral = SpectralClustering(n_clusters=1, affinity="precomputed")
cluster = 0
shape = (clusters,)+repeats.shape[1:]
self.reproduction_matrices = np.zeros(shape, np.uint8)
self.reproduction_analysis = []
for idx, repeat in enumerate(repeats[:clusters]):
# run the spectral clustering on the current repeat array.
# this is the rate limiting step, and already uses all
# available cpu cores.
spectral.set_params(n_clusters=idx+1)
spectral.fit(repeat)
labels = spectral.labels_
# order the clusters by size. keys in members are strings
# as required for json dumps
count = Counter(spectral.labels_)
by_size = [(k, v) for k, v in count.items()]
by_size.sort(key=lambda x: -x[1])
members = {str(t[0]+cluster):_find(labels, t) for t in by_size}
order = np.hstack([members[str(t[0]+cluster)] for t in by_size])
#rearrange
rearr = repeat[order].transpose()[order]
sizes = [[str(k), len(v)] for k, v in members.items()]
sizes.sort(key=lambda x: -x[1])
# m gives the counts for each pair of tags. 3d array.
# shape: [nclusters-1,ntags,ntags]. members are the tag
# indices; self.graph.graph.nodes()[members] gives members as words.
# sizes are the number of tags in each cluster, sorted by size
tmp = {'members':members, 'sizes':sizes}
rescale = (rearr*255./rearr.max()).astype(np.uint8)
self.reproduction_matrices[idx] = rescale
self.reproduction_analysis.append(tmp)
cluster += idx+1
def cluster_reproducibility(self, repeats=None, clusters=50):
""" Given the tag co-occurence arrays generated by the train
method, use the spectral clustering method in sklearn and the
known (or desired) number of clusters to assign tags to
specific clusters.
Required input:
None
Optional input:
repeats - a set of co-occurence arrays to cluster using
spectral methods. If not supplied, this method
defaults to self.repeats which is the data generated
by the train() method.
labels - the tags corresponding to the feature vectors.
Labels must be correctly ordered, obviously.
Returns:
None ----BUT---- generates the following analysis in the
self namespace.'
1. self.reproduction_matrices: a reorganization of the
repeats data into block diagonal form.
2. self.reproduction_analysis: a list of dictionaries.
Each dictionary has two keys: 'members' and 'sizes'.
'members' lists the tag membership of each cluster
in terms of the indices of the feature vectors represented
by samples in train(),arranged by size.
'sizes' gives the size of each
cluster. The index of the self.reproduction_analysis
list gives the number of clusters remainging from
the agglomeration. For example,
self.reproduction_analysis[10][4]['members'] lists the
tag indices of the 5th largest cluster when there are
11 clusters remaining from the agglomeration.
"""
def _find(where, what):
""" Helper """
return np.where(where == what[0])[0].tolist()
from sklearn.cluster import SpectralClustering
from collections import Counter
if repeats == None:
repeats = self.repeats
spectral = SpectralClustering(n_clusters=1, affinity="precomputed")
cluster = 0
shape = (clusters, ) + repeats.shape[1:]
self.reproduction_matrices = np.zeros(shape, np.uint8)
self.reproduction_analysis = []
for idx, repeat in enumerate(repeats[:clusters]):
# run the spectral clustering on the current repeat array.
# this is the rate limiting step, and already uses all
# available cpu cores.
spectral.set_params(n_clusters=idx + 1)
spectral.fit(repeat)
labels = spectral.labels_
# order the clusters by size. keys in members are strings
# as required for json dumps
count = Counter(spectral.labels_)
by_size = [(k, v) for k, v in count.items()]
by_size.sort(key=lambda x: -x[1])
members = {str(t[0] + cluster): _find(labels, t) for t in by_size}
order = np.hstack([members[str(t[0] + cluster)] for t in by_size])
#rearrange
rearr = repeat[order].transpose()[order]
sizes = [[str(k), len(v)] for k, v in members.items()]
sizes.sort(key=lambda x: -x[1])
# m gives the counts for each pair of tags. 3d array.
# shape: [nclusters-1,ntags,ntags]. members are the tag
# indices; self.graph.graph.nodes()[members] gives members as words.
# sizes are the number of tags in each cluster, sorted by size
tmp = {'members': members, 'sizes': sizes}
rescale = (rearr * 255. / rearr.max()).astype(np.uint8)
self.reproduction_matrices[idx] = rescale
self.reproduction_analysis.append(tmp)
cluster += idx + 1
