def _get_components(x_in, connectivity, return_list=True):
"""get connected components from a mask and a connectivity matrix"""
try:
from sklearn.utils._csgraph import cs_graph_components
except:
from scikits.learn.utils._csgraph import cs_graph_components
mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
data = connectivity.data[mask]
row = connectivity.row[mask]
col = connectivity.col[mask]
shape = connectivity.shape
idx = np.where(x_in)[0]
row = np.concatenate((row, idx))
col = np.concatenate((col, idx))
data = np.concatenate((data, np.ones(len(idx), dtype=data.dtype)))
connectivity = sparse.coo_matrix((data, (row, col)), shape=shape)
_, components = cs_graph_components(connectivity)
if return_list:
labels = np.unique(components)
clusters = list()
for l in labels:
c = np.where(components == l)[0]
if np.any(x_in[c]):
clusters.append(c)
# logger.info("-- number of components : %d"
# % np.unique(components).size)
return clusters
else:
return components
def _get_components(x_in, connectivity, return_list=True):
"""get connected components from a mask and a connectivity matrix"""
try:
from sklearn.utils._csgraph import cs_graph_components
except:
from scikits.learn.utils._csgraph import cs_graph_components
mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
data = connectivity.data[mask]
row = connectivity.row[mask]
col = connectivity.col[mask]
shape = connectivity.shape
idx = np.where(x_in)[0]
row = np.concatenate((row, idx))
col = np.concatenate((col, idx))
data = np.concatenate((data, np.ones(len(idx), dtype=data.dtype)))
connectivity = sparse.coo_matrix((data, (row, col)), shape=shape)
_, components = cs_graph_components(connectivity)
if return_list:
labels = np.unique(components)
clusters = list()
for l in labels:
c = np.where(components == l)[0]
if np.any(x_in[c]):
clusters.append(c)
# logger.info("-- number of components : %d"
# % np.unique(components).size)
return clusters
else:
return components
def _get_components(x_in, connectivity):
"""get connected components from a mask and a connectivity matrix"""
try:
from sklearn.utils._csgraph import cs_graph_components
except:
from scikits.learn.utils._csgraph import cs_graph_components
mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
data = connectivity.data[mask]
row = connectivity.row[mask]
col = connectivity.col[mask]
shape = connectivity.shape
idx = np.where(x_in)[0]
row = np.concatenate((row, idx))
col = np.concatenate((col, idx))
data = np.concatenate((data, np.ones(len(idx), dtype=data.dtype)))
connectivity = sparse.coo_matrix((data, (row, col)), shape=shape)
_, components = cs_graph_components(connectivity)
# print "-- number of components : %d" % np.unique(components).size
return components
def _get_components(x_in, connectivity, return_list=True):
"""get connected components from a mask and a connectivity matrix"""
try:
from sklearn.utils._csgraph import cs_graph_components
except:
try:
from scikits.learn.utils._csgraph import cs_graph_components
except:
# in theory we might be able to shoehorn this into using
# _get_clusters_spatial if we transform connectivity into
# a neighbor list, and it might end up being faster anyway,
# but for now:
raise ValueError('scikits-learn must be installed')
mask = np.logical_and(x_in[connectivity.row], x_in[connectivity.col])
data = connectivity.data[mask]
row = connectivity.row[mask]
col = connectivity.col[mask]
shape = connectivity.shape
idx = np.where(x_in)[0]
row = np.concatenate((row, idx))
col = np.concatenate((col, idx))
data = np.concatenate((data, np.ones(len(idx), dtype=data.dtype)))
connectivity = sparse.coo_matrix((data, (row, col)), shape=shape)
_, components = cs_graph_components(connectivity)
if return_list:
labels = np.unique(components)
clusters = list()
for l in labels:
c = np.where(components == l)[0]
if np.any(x_in[c]):
clusters.append(c)
# logger.info("-- number of components : %d"
# % np.unique(components).size)
return clusters
else:
return components
def ward_tree(X, connectivity=None, n_components=None, copy=True,
n_clusters=None):
"""Ward clustering based on a Feature matrix.
The inertia matrix uses a Heapq-based representation.
This is the structured version, that takes into account a some topological
structure between samples.
Parameters
----------
X : array of shape (n_samples, n_features)
feature matrix representing n_samples samples to be clustered
connectivity : sparse matrix.
connectivity matrix. Defines for each sample the neigbhoring samples
following a given structure of the data. The matrix is assumed to
be symmetric and only the upper triangular half is used.
Default is None, i.e, the Ward algorithm is unstructured.
n_components : int (optional)
Number of connected components. If None the number of connected
components is estimated from the connectivity matrix.
copy : bool (optional)
Make a copy of connectivity or work inplace. If connectivity
is not of LIL type there will be a copy in any case.
n_clusters : int (optional)
Stop early the construction of the tree at n_clusters. This is
useful to decrease computation time if the number of clusters is
not small compared to the number of samples. In this case, the
complete tree is not computed, thus the 'children' output is of
limited use, and the 'parents' output should rather be used.
This option is valid only when specifying a connectivity matrix.
Returns
-------
children : 2D array, shape (n_nodes, 2)
list of the children of each nodes.
Leaves of the tree have empty list of children.
n_components : sparse matrix.
The number of connected components in the graph.
n_leaves : int
The number of leaves in the tree
parents : 1D array, shape (n_nodes, ) or None
The parent of each node. Only returned when a connectivity matrix
is specified, elsewhere 'None' is returned.
"""
X = np.asarray(X)
if X.ndim == 1:
X = np.reshape(X, (-1, 1))
n_samples, n_features = X.shape
if connectivity is None:
if n_clusters is not None:
warnings.warn('Early stopping is implemented only for '
'structured Ward clustering (i.e. with '
'explicit connectivity.', stacklevel=2)
out = hierarchy.ward(X)
children_ = out[:, :2].astype(np.int)
return children_, 1, n_samples, None
# Compute the number of nodes
if n_components is None:
n_components, labels = cs_graph_components(connectivity)
# Convert connectivity matrix to LIL with a copy if needed
if sparse.isspmatrix_lil(connectivity) and copy:
connectivity = connectivity.copy()
elif not sparse.isspmatrix(connectivity):
connectivity = sparse.lil_matrix(connectivity)
else:
connectivity = connectivity.tolil()
if n_components > 1:
warnings.warn("the number of connected components of the "
"connectivity matrix is %d > 1. Completing it to avoid "
"stopping the tree early." % n_components)
connectivity = _fix_connectivity(X, connectivity, n_components, labels)
n_components = 1
if n_clusters is None:
n_nodes = 2 * n_samples - n_components
else:
assert n_clusters <= n_samples
n_nodes = 2 * n_samples - n_clusters
if (connectivity.shape[0] != n_samples
or connectivity.shape[1] != n_samples):
raise ValueError('Wrong shape for connectivity matrix: %s '
'when X is %s' % (connectivity.shape, X.shape))
# create inertia matrix
coord_row = []
coord_col = []
A = []
for ind, row in enumerate(connectivity.rows):
A.append(row)
# We keep only the upper triangular for the moments
# Generator expressions are faster than arrays on the following
row = [i for i in row if i < ind]
coord_row.extend(len(row) * [ind, ])
coord_col.extend(row)
coord_row = np.array(coord_row, dtype=np.int)
coord_col = np.array(coord_col, dtype=np.int)
# build moments as a list
moments_1 = np.zeros(n_nodes)
moments_1[:n_samples] = 1
moments_2 = np.zeros((n_nodes, n_features))
moments_2[:n_samples] = X
inertia = np.empty(len(coord_row), dtype=np.float)
_hierarchical.compute_ward_dist(moments_1, moments_2, coord_row, coord_col,
inertia)
inertia = zip(inertia, coord_row, coord_col)
heapify(inertia)
# prepare the main fields
parent = np.arange(n_nodes, dtype=np.int)
heights = np.zeros(n_nodes)
used_node = np.ones(n_nodes, dtype=bool)
children = []
not_visited = np.empty(n_nodes, dtype=np.int8)
# recursive merge loop
for k in xrange(n_samples, n_nodes):
# identify the merge
while True:
inert, i, j = heappop(inertia)
if used_node[i] and used_node[j]:
break
parent[i], parent[j], heights[k] = k, k, inert
children.append([i, j])
used_node[i] = used_node[j] = False
# update the moments
moments_1[k] = moments_1[i] + moments_1[j]
moments_2[k] = moments_2[i] + moments_2[j]
# update the structure matrix A and the inertia matrix
coord_col = []
not_visited.fill(1)
not_visited[k] = 0
_hierarchical._get_parents(A[i], coord_col, parent, not_visited)
_hierarchical._get_parents(A[j], coord_col, parent, not_visited)
# List comprehension is faster than a for loop
[A[l].append(k) for l in coord_col]
A.append(coord_col)
coord_col = np.array(coord_col, dtype=np.int)
coord_row = np.empty_like(coord_col)
coord_row.fill(k)
n_additions = len(coord_row)
ini = np.empty(n_additions, dtype=np.float)
_hierarchical.compute_ward_dist(moments_1, moments_2,
coord_row, coord_col, ini)
# List comprehension is faster than a for loop
[heappush(inertia, (ini[idx], k, coord_col[idx]))
for idx in xrange(n_additions)]
# Separate leaves in children (empty lists up to now)
n_leaves = n_samples
children = np.array(children) # return numpy array for efficient caching
parent = np.array(parent)
#print 'parent = ', parent
#--------moi them vao 04-04-2013 -----------------------
#heights = np.array(heights) # return numpy array for efficient caching
return children, n_components, n_leaves, parent
