def fit(self, X, y=None):
"""Learn the idf vector (global term weights).
Parameters
----------
X : sparse matrix of shape n_samples, n_features)
A matrix of term/token counts.
"""
X = check_array(X, accept_sparse=('csr', 'csc'))
if not sp.issparse(X):
X = sp.csr_matrix(X)
dtype = X.dtype if X.dtype in FLOAT_DTYPES else np.float64
if self.use_idf:
n_samples, n_features = X.shape
df = _document_frequency(X)
df = df.astype(dtype, **_astype_copy_false(df))
# perform idf smoothing if required
df += int(self.smooth_idf)
n_samples += int(self.smooth_idf)
# log+1 instead of log makes sure terms with zero idf don't get suppressed entirely.
# original idf = np.log(n_samples / df) + 1
# BM25 idf = np.log((n_samples - df + 0.5)/(df + 0.5) + 1)
# = np.log(n_samples - df + 0.5 + df + 0.5) - np.log(df + 0.5)
idf = np.log(n_samples + 1) - np.log(df + 0.5)
self._idf_diag = sp.diags(idf,
offsets=0,
shape=(n_features, n_features),
format='csr',
dtype=dtype)
return self
def test_float_precision(Estimator, data):
# Check that the results are the same for single and double precision.
km = Estimator(n_init=1, random_state=0)
inertia = {}
Xt = {}
centers = {}
labels = {}
for dtype in [np.float64, np.float32]:
X = data.astype(dtype, **_astype_copy_false(data))
km.fit(X)
inertia[dtype] = km.inertia_
Xt[dtype] = km.transform(X)
centers[dtype] = km.cluster_centers_
labels[dtype] = km.labels_
# dtype of cluster centers has to be the dtype of the input data
assert km.cluster_centers_.dtype == dtype
# same with partial_fit
if Estimator is MiniBatchKMeans:
km.partial_fit(X[0:3])
assert km.cluster_centers_.dtype == dtype
# compare arrays with low precision since the difference between 32 and
# 64 bit comes from an accumulation of rounding errors.
assert_allclose(inertia[np.float32], inertia[np.float64], rtol=1e-5)
assert_allclose(Xt[np.float32], Xt[np.float64], rtol=1e-5)
assert_allclose(centers[np.float32], centers[np.float64], rtol=1e-5)
assert_array_equal(labels[np.float32], labels[np.float64])
def fit(self,
X: scipy.sparse.csr_matrix,
y: Any = None) -> "BM25Transformer":
"""Learn the idf vector (global term weights).
Parameters
----------
X : sparse matrix of shape n_samples, n_features)
A matrix of term/token counts.
"""
X = check_array(X, accept_sparse=("csr", "csc"))
if not scipy.sparse.issparse(X):
X = scipy.sparse.csr_matrix(X)
dtype = X.dtype if X.dtype in FLOAT_DTYPES else np.float64
if self.use_idf:
n_samples, n_features = X.shape
df = _document_frequency(X)
df = df.astype(dtype, **_astype_copy_false(df))
idf = np.log(1 + (n_samples - df + 0.5) / (df + 0.5))
self._idf_diag = scipy.sparse.diags(idf,
offsets=0,
shape=(n_features, n_features),
format="csr",
dtype=dtype)
return self
def fit(self, X, y=None):
"""Learn the idf vector (global term weights).
Parameters
----------
X : sparse matrix of shape n_samples, n_features)
A matrix of term/token counts.
"""
# 计算所有文档的平均长度
# self.avdl = np.mean(np.sum(X.toarray(), axis=-1))
self.avdl = np.sum(X.toarray()) / X.toarray().shape[0]
X = check_array(X, accept_sparse=('csr', 'csc'))
if not sp.issparse(X):
X = sp.csr_matrix(X)
dtype = X.dtype if X.dtype in FLOAT_DTYPES else np.float64
if self.use_idf:
n_samples, n_features = X.shape
df = _document_frequency(X)
df = df.astype(dtype, **_astype_copy_false(df))
# perform idf smoothing if required
df += int(self.smooth_idf)
n_samples += int(self.smooth_idf)
# log+1 instead of log makes sure terms with zero idf don't get
# suppressed entirely.
idf = np.log(n_samples / df) + 1
self._idf_diag = sp.diags(idf, offsets=0,
shape=(n_features, n_features),
format='csr',
dtype=dtype)
return self
def _estimate_mi(X,
y,
discrete_features='auto',
discrete_target=False,
n_neighbors=3,
copy=True,
random_state=None):
X, y = check_X_y(X, y, accept_sparse='csc', y_numeric=not discrete_target)
n_samples, n_features = X.shape
if isinstance(discrete_features, (str, bool)):
if isinstance(discrete_features, str):
if discrete_features == 'auto':
discrete_features = issparse(X)
else:
raise ValueError("Invalid string value for discrete_features.")
discrete_mask = np.empty(n_features, dtype=bool)
discrete_mask.fill(discrete_features)
else:
discrete_features = check_array(discrete_features, ensure_2d=False)
if discrete_features.dtype != 'bool':
discrete_mask = np.zeros(n_features, dtype=bool)
discrete_mask[discrete_features] = True
else:
discrete_mask = discrete_features
continuous_mask = ~discrete_mask
if np.any(continuous_mask) and issparse(X):
raise ValueError("Sparse matrix `X` can't have continuous features.")
rng = check_random_state(random_state)
if np.any(continuous_mask):
if copy:
X = X.copy()
if not discrete_target:
X[:, continuous_mask] = scale(X[:, continuous_mask],
with_mean=False,
copy=False)
# Add small noise to continuous features as advised in Kraskov et. al.
X = X.astype(float, **_astype_copy_false(X))
means = np.maximum(1, np.mean(np.abs(X[:, continuous_mask]), axis=0))
X[:, continuous_mask] += 1e-10 * means * rng.randn(
n_samples, np.sum(continuous_mask))
if not discrete_target:
y = scale(y, with_mean=False)
y += 1e-10 * np.maximum(1, np.mean(np.abs(y))) * rng.randn(n_samples)
mi = [
_compute_mi(x, y, discrete_feature, discrete_target, n_neighbors)
for x, discrete_feature in zip(_iterate_columns(X), discrete_mask)
]
return np.array(mi)
def _single_linkage_tree(connectivity, n_samples, n_nodes, n_clusters,
n_connected_components, return_distance):
"""
Perform single linkage clustering on sparse data via the minimum
spanning tree from scipy.sparse.csgraph, then using union-find to label.
The parent array is then generated by walking through the tree.
"""
from scipy.sparse.csgraph import minimum_spanning_tree
# explicitly cast connectivity to ensure safety
connectivity = connectivity.astype('float64',
**_astype_copy_false(connectivity))
# Ensure zero distances aren't ignored by setting them to "epsilon"
epsilon_value = np.finfo(dtype=connectivity.data.dtype).eps
connectivity.data[connectivity.data == 0] = epsilon_value
# Use scipy.sparse.csgraph to generate a minimum spanning tree
mst = minimum_spanning_tree(connectivity.tocsr())
# Convert the graph to scipy.cluster.hierarchy array format
mst = mst.tocoo()
# Undo the epsilon values
mst.data[mst.data == epsilon_value] = 0
mst_array = np.vstack([mst.row, mst.col, mst.data]).T
# Sort edges of the min_spanning_tree by weight
mst_array = mst_array[np.argsort(mst_array.T[2]), :]
# Convert edge list into standard hierarchical clustering format
single_linkage_tree = _hierarchical._single_linkage_label(mst_array)
children_ = single_linkage_tree[:, :2].astype(np.int)
# Compute parents
parent = np.arange(n_nodes, dtype=np.intp)
for i, (left, right) in enumerate(children_, n_samples):
if n_clusters is not None and i >= n_nodes:
break
if left < n_nodes:
parent[left] = i
if right < n_nodes:
parent[right] = i
if return_distance:
distances = single_linkage_tree[:, 2]
return children_, n_connected_components, n_samples, parent, distances
return children_, n_connected_components, n_samples, parent
def normalized_mutual_info_score(labels_true, labels_pred, *,
average_method='geometric'):
labels_true, labels_pred = check_clusterings(labels_true, labels_pred)
classes = np.unique(labels_true)
clusters = np.unique(labels_pred)
# Special limit cases: no clustering since the data is not split.
# This is a perfect match hence return 1.0.
if (classes.shape[0] == clusters.shape[0] == 1 or
classes.shape[0] == clusters.shape[0] == 0):
if classes[0] == 0 or clusters[0] == 0:
return 0.0
else:
return 1.0
contingency = contingency_matrix(labels_true, labels_pred, sparse=True)
contingency = contingency.astype(np.float64,
**_astype_copy_false(contingency))
# Calculate the MI for the two clusterings
mi = mutual_info_score(labels_true, labels_pred,
contingency=contingency)
# Calculate the expected value for the mutual information
# Calculate entropy for each labeling
h_true, h_pred = entropy(labels_true), entropy(labels_pred)
normalizer = _generalized_average(h_true, h_pred, average_method)
# Avoid 0.0 / 0.0 when either entropy is zero.
normalizer = max(normalizer, np.finfo('float64').eps)
nmi = mi / normalizer
# Avoid problems associated with low entropy regions
# not normalizing correctly
if nmi > 1.0:
nmi = 0.0
return nmi
def transform_selected(X,
transform,
dtype,
selected="all",
copy=True,
retain_order=False):
"""Apply a transform function to portion of selected features.
Returns an array Xt, where the non-selected features appear on the right
side (largest column indices) of Xt.
Retained from https://github.com/scikit-learn/scikit-learn/pull/14866
Parameters
----------
X : {array-like, sparse matrix}, shape [n_samples, n_features]
Dense array or sparse matrix.
transform : callable
A callable transform(X) -> X_transformed
dtype : number type
Desired dtype of output.
copy : boolean, default=True
Copy X even if it could be avoided.
selected : "all" or array of indices or mask
Specify which features to apply the transform to.
retain_order : boolean, default=False
If True, the non-selected features will not be displaced to the right
side of the transformed array. The number of features in Xt must
match the number of features in X. Furthermore, X and Xt cannot be
sparse.
Returns
-------
Xt : array or sparse matrix, shape=(n_samples, n_features_new)
"""
X = check_array(X, accept_sparse='csc', copy=copy, dtype=FLOAT_DTYPES)
if sparse.issparse(X) and retain_order:
raise ValueError("The retain_order option can only be set to True "
"for dense matrices.")
if isinstance(selected, str) and selected == "all":
return transform(X)
if len(selected) == 0:
return X
n_features = X.shape[1]
ind = np.arange(n_features)
sel = np.zeros(n_features, dtype=bool)
sel[np.asarray(selected)] = True
not_sel = np.logical_not(sel)
n_selected = np.sum(sel)
if n_selected == 0:
# No features selected.
return X
elif n_selected == n_features:
# All features selected.
return transform(X)
else:
X_sel = transform(X[:, ind[sel]])
# The columns of X which are not transformed need
# to be casted to the desire dtype before concatenation.
# Otherwise, the stacking will cast to the higher-precision dtype.
X_not_sel = X[:, ind[not_sel]].astype(dtype, **_astype_copy_false(X))
if retain_order:
if X_sel.shape[1] + X_not_sel.shape[1] != n_features:
raise ValueError("The retain_order option can only be set to True "
"if the dimensions of the input array match the "
"dimensions of the transformed array.")
# Fancy indexing not supported for sparse matrices
X[:, ind[sel]] = X_sel
return X
if sparse.issparse(X_sel) or sparse.issparse(X_not_sel):
return sparse.hstack((X_sel, X_not_sel))
else:
return np.hstack((X_sel, X_not_sel))
def linkage_tree(X,
connectivity=None,
n_clusters=None,
linkage='complete',
affinity="euclidean",
return_distance=False):
"""Linkage agglomerative clustering based on a Feature matrix.
The inertia matrix uses a Heapq-based representation.
This is the structured version, that takes into account some topological
structure between samples.
Read more in the :ref:`User Guide <hierarchical_clustering>`.
Parameters
----------
X : array, shape (n_samples, n_features)
feature matrix representing n_samples samples to be clustered
connectivity : sparse matrix (optional).
connectivity matrix. Defines for each sample the neighboring samples
following a given structure of the data. The matrix is assumed to
be symmetric and only the upper triangular half is used.
Default is None, i.e, the Ward algorithm is unstructured.
n_clusters : int (optional)
Stop early the construction of the tree at n_clusters. This is
useful to decrease computation time if the number of clusters is
not small compared to the number of samples. In this case, the
complete tree is not computed, thus the 'children' output is of
limited use, and the 'parents' output should rather be used.
This option is valid only when specifying a connectivity matrix.
linkage : {"average", "complete", "single"}, optional, default: "complete"
Which linkage criteria to use. The linkage criterion determines which
distance to use between sets of observation.
- average uses the average of the distances of each observation of
the two sets
- complete or maximum linkage uses the maximum distances between
all observations of the two sets.
- single uses the minimum of the distances between all observations
of the two sets.
affinity : string or callable, optional, default: "euclidean".
which metric to use. Can be "euclidean", "manhattan", or any
distance know to paired distance (see metric.pairwise)
return_distance : bool, default False
whether or not to return the distances between the clusters.
Returns
-------
children : 2D array, shape (n_nodes-1, 2)
The children of each non-leaf node. Values less than `n_samples`
correspond to leaves of the tree which are the original samples.
A node `i` greater than or equal to `n_samples` is a non-leaf
node and has children `children_[i - n_samples]`. Alternatively
at the i-th iteration, children[i][0] and children[i][1]
are merged to form node `n_samples + i`
n_connected_components : int
The number of connected components in the graph.
n_leaves : int
The number of leaves in the tree.
parents : 1D array, shape (n_nodes, ) or None
The parent of each node. Only returned when a connectivity matrix
is specified, elsewhere 'None' is returned.
distances : ndarray, shape (n_nodes-1,)
Returned when return_distance is set to True.
distances[i] refers to the distance between children[i][0] and
children[i][1] when they are merged.
See also
--------
ward_tree : hierarchical clustering with ward linkage
"""
X = np.asarray(X)
if X.ndim == 1:
X = np.reshape(X, (-1, 1))
n_samples, n_features = X.shape
linkage_choices = {
'complete': _hierarchical.max_merge,
'average': _hierarchical.average_merge,
'single': None
} # Single linkage is handled differently
try:
join_func = linkage_choices[linkage]
except KeyError:
raise ValueError('Unknown linkage option, linkage should be one '
'of %s, but %s was given' %
(linkage_choices.keys(), linkage))
if affinity == 'cosine' and np.any(~np.any(X, axis=1)):
raise ValueError(
'Cosine affinity cannot be used when X contains zero vectors')
if connectivity is None:
from scipy.cluster import hierarchy # imports PIL
if n_clusters is not None:
warnings.warn(
'Partial build of the tree is implemented '
'only for structured clustering (i.e. with '
'explicit connectivity). The algorithm '
'will build the full tree and only '
'retain the lower branches required '
'for the specified number of clusters',
stacklevel=2)
if affinity == 'precomputed':
# for the linkage function of hierarchy to work on precomputed
# data, provide as first argument an ndarray of the shape returned
# by pdist: it is a flat array containing the upper triangular of
# the distance matrix.
i, j = np.triu_indices(X.shape[0], k=1)
X = X[i, j]
elif affinity == 'l2':
# Translate to something understood by scipy
affinity = 'euclidean'
elif affinity in ('l1', 'manhattan'):
affinity = 'cityblock'
elif callable(affinity):
X = affinity(X)
i, j = np.triu_indices(X.shape[0], k=1)
X = X[i, j]
out = hierarchy.linkage(X, method=linkage, metric=affinity)
children_ = out[:, :2].astype(np.int, copy=False)
if return_distance:
distances = out[:, 2]
return children_, 1, n_samples, None, distances
return children_, 1, n_samples, None
connectivity, n_connected_components = _fix_connectivity(X,
connectivity,
affinity=affinity)
connectivity = connectivity.tocoo()
# Put the diagonal to zero
diag_mask = (connectivity.row != connectivity.col)
connectivity.row = connectivity.row[diag_mask]
connectivity.col = connectivity.col[diag_mask]
connectivity.data = connectivity.data[diag_mask]
del diag_mask
if affinity == 'precomputed':
distances = X[connectivity.row,
connectivity.col].astype('float64',
**_astype_copy_false(X))
else:
# FIXME We compute all the distances, while we could have only computed
# the "interesting" distances
distances = paired_distances(X[connectivity.row],
X[connectivity.col],
metric=affinity)
connectivity.data = distances
if n_clusters is None:
n_nodes = 2 * n_samples - 1
else:
assert n_clusters <= n_samples
n_nodes = 2 * n_samples - n_clusters
if linkage == 'single':
return _single_linkage_tree(connectivity, n_samples, n_nodes,
n_clusters, n_connected_components,
return_distance)
if return_distance:
distances = np.empty(n_nodes - n_samples)
# create inertia heap and connection matrix
A = np.empty(n_nodes, dtype=object)
inertia = list()
# LIL seems to the best format to access the rows quickly,
# without the numpy overhead of slicing CSR indices and data.
connectivity = connectivity.tolil()
# We are storing the graph in a list of IntFloatDict
for ind, (data, row) in enumerate(zip(connectivity.data,
connectivity.rows)):
A[ind] = IntFloatDict(np.asarray(row, dtype=np.intp),
np.asarray(data, dtype=np.float64))
# We keep only the upper triangular for the heap
# Generator expressions are faster than arrays on the following
inertia.extend(
_hierarchical.WeightedEdge(d, ind, r) for r, d in zip(row, data)
if r < ind)
del connectivity
heapify(inertia)
# prepare the main fields
parent = np.arange(n_nodes, dtype=np.intp)
used_node = np.ones(n_nodes, dtype=np.intp)
children = []
# recursive merge loop
for k in range(n_samples, n_nodes):
# identify the merge
while True:
edge = heappop(inertia)
if used_node[edge.a] and used_node[edge.b]:
break
i = edge.a
j = edge.b
if return_distance:
# store distances
distances[k - n_samples] = edge.weight
parent[i] = parent[j] = k
children.append((i, j))
# Keep track of the number of elements per cluster
n_i = used_node[i]
n_j = used_node[j]
used_node[k] = n_i + n_j
used_node[i] = used_node[j] = False
# update the structure matrix A and the inertia matrix
# a clever 'min', or 'max' operation between A[i] and A[j]
coord_col = join_func(A[i], A[j], used_node, n_i, n_j)
for l, d in coord_col:
A[l].append(k, d)
# Here we use the information from coord_col (containing the
# distances) to update the heap
heappush(inertia, _hierarchical.WeightedEdge(d, k, l))
A[k] = coord_col
# Clear A[i] and A[j] to save memory
A[i] = A[j] = 0
# Separate leaves in children (empty lists up to now)
n_leaves = n_samples
# # return numpy array for efficient caching
children = np.array(children)[:, ::-1]
if return_distance:
return children, n_connected_components, n_leaves, parent, distances
return children, n_connected_components, n_leaves, parent
