def test_internals_api(n_components): callback = CustomCallback() reducer = UMAP(n_components=n_components, callback=callback) reducer.fit(data) callback.check() # Make sure super().__init__ is called callback = CustomCallback(skip_init=True) model = UMAP(n_epochs=10, callback=callback) with pytest.raises(ValueError): model.fit_transform(data)
def test_trustworthiness(input_type, n_samples, n_features, n_components): centers = round(n_samples * 0.4) X, y = make_blobs(n_samples=n_samples, centers=centers, n_features=n_features) X_embedded = UMAP(n_components=n_components).fit_transform(X) X = X.astype(np.float32) X_embedded = X_embedded.astype(np.float32) if input_type == 'dataframe': gdf = cudf.DataFrame() for i in range(X.shape[1]): gdf[str(i)] = np.asarray(X[:, i], dtype=np.float32) gdf_embedded = cudf.DataFrame() for i in range(X_embedded.shape[1]): gdf_embedded[str(i)] = np.asarray(X_embedded[:, i], dtype=np.float32) cu_score = cuml_trustworthiness(gdf, gdf_embedded) else: cu_score = cuml_trustworthiness(X, X_embedded) sk_score = sklearn_trustworthiness(X, X_embedded) eps = 0.0001 assert (sk_score * (1 - eps) <= cu_score and cu_score <= sk_score * (1 + eps))
def _build_mnmg_umap(m, data, args, tmpdir): client = args['client'] del args['client'] local_model = UMAP(**args) if isinstance(data, (tuple, list)): local_data = [x.compute() for x in data if x is not None] if len(local_data) == 2: X, y = local_data local_model.fit(X, y) else: X = local_data local_model.fit(X) return m(client=client, model=local_model, **args)
def _local_umap_trustworthiness(local_X, local_y, n_neighbors, supervised): """ Train model on all data, report trustworthiness """ from cuml.manifold import UMAP local_model = UMAP(n_neighbors=n_neighbors, random_state=42) y_train = None if supervised: y_train = local_y local_model.fit(local_X, y=y_train) embedding = local_model.transform(local_X) return trustworthiness(local_X, embedding, n_neighbors=n_neighbors, batch_size=5000)
def reduce_dimensionality(self, embeddings): """Reduce dimensionality of embeddings using UMAP and train a UMAP model Args: embeddings (cupy.ndarray): The extracted embeddings using the sentence transformer module. Returns: umap_embeddings: The reduced embeddings """ m_cos = NearestNeighbors(n_neighbors=15, metric="cosine") m_cos.fit(embeddings) knn_graph_cos = m_cos.kneighbors_graph(embeddings, mode="distance") u1 = UMAP(n_neighbors=15, n_components=5, min_dist=0.0) umap_embeddings = u1.fit_transform(embeddings, knn_graph=knn_graph_cos) return umap_embeddings
def _umap_mnmg_trustworthiness(local_X, local_y, n_neighbors, supervised, n_parts, sampling_ratio): """ Train model on random sample of data, transform in parallel, report trustworthiness """ import dask.array as da from cuml.dask.manifold import UMAP as MNMG_UMAP from cuml.manifold import UMAP local_model = UMAP(n_neighbors=n_neighbors, random_state=42) n_samples = local_X.shape[0] n_samples_per_part = math.ceil(n_samples / n_parts) selection = np.random.RandomState(42).choice( [True, False], n_samples, replace=True, p=[sampling_ratio, 1.0 - sampling_ratio]) X_train = local_X[selection] X_transform = local_X X_transform_d = da.from_array(X_transform, chunks=(n_samples_per_part, -1)) y_train = None if supervised: y_train = local_y[selection] local_model.fit(X_train, y=y_train) distributed_model = MNMG_UMAP(local_model) embedding = distributed_model.transform(X_transform_d) embedding = embedding.compute() return trustworthiness(X_transform, embedding, n_neighbors=n_neighbors, batch_size=5000)
def umap_mnmg_trustworthiness(): n_samples = local_X.shape[0] n_sampling = int(n_samples * sampling_ratio) n_samples_per_part = int(n_samples / n_parts) local_model = UMAP(n_neighbors=n_neighbors) selection = np.random.choice(n_samples, n_sampling) X_train = local_X[selection] X_transform = local_X[~selection] X_transform_d = da.from_array(X_transform, chunks=(n_samples_per_part, -1)) y_train = None if supervised: y_train = local_y[selection] local_model.fit(X_train, y=y_train) distributed_model = MNMG_UMAP(local_model) embedding = distributed_model.transform(X_transform_d) embedding = cp.asnumpy(embedding.compute()) return trustworthiness(X_transform, embedding, n_neighbors)
def local_umap_trustworthiness(): local_model = UMAP(n_neighbors=n_neighbors) local_model.fit(local_X, local_y) embedding = local_model.transform(local_X) return trustworthiness(local_X, embedding, n_neighbors)
def test_internals_api(n_components): callback = CustomCallback() reducer = UMAP(n_components=n_components, callback=callback) reducer.fit(data) callback.check()
from cuml.manifold import UMAP from sklearn.datasets import load_digits from numba import cuda from pyrr import Matrix44 import numpy as np import cudatashader as ds from cudatashader import transfer_functions as tf from cudatashader.colors import Hot from IPython.core.display import display, HTML, clear_output digits = load_digits() data, target_classes = digits.data, digits.target n_samples = target_classes.shape[0] reducer = UMAP(n_components=3) reducer.fit(data) embedding = reducer.transform(data) maxThreadsPerBlock = cuda.get_current_device().MAX_THREADS_PER_BLOCK @cuda.jit('void(int64[:], float64[:,:])') def fill_agg_value(target_classes, result): i = cuda.grid(1) result[i, 2] = target_classes[i] @cuda.jit('void(float64[:,:], float64[:,:], float64[:,:])') def apply_projection(MVP, embedding, result): i = cuda.grid(1) x, y, z = embedding[i, 0], embedding[i, 1], embedding[i, 2]
data, target = digits.data, digits.target class CustomCallback(GraphBasedDimRedCallback): preprocess_event, epoch_event, train_event = False, 0, False def check(self): assert (self.preprocess_event) assert (self.epoch_event > 10) assert (self.train_event) def on_preprocess_end(self, embeddings): self.preprocess_event = True def on_epoch_end(self, embeddings): self.epoch_event += 1 def on_train_end(self, embeddings): self.train_event = True reducer = UMAP(n_components=2, callback=CustomCallback()) @pytest.mark.parametrize('n_components', [2, 4, 8]) def test_internals_api(n_components): callback = CustomCallback() reducer = UMAP(n_components=n_components, callback=callback) reducer.fit(data) callback.check()