def kneighbors(self, X=None, n_neighbors=None, return_distance=True, _return_futures=False): """ Query the distributed nearest neighbors index :param X : dask_cudf.Dataframe Vectors to query. If not provided, neighbors of each indexed point are returned. :param n_neighbors : Number of neighbors to query for each row in X. If not provided, the n_neighbors on the model are used. :param return_distance : If false, only indices are returned :return : dask_cudf.DataFrame containing distances :return : dask_cudf.DataFrame containing indices """ n_neighbors = self.get_neighbors(n_neighbors) query_futures = self.X if X is None else \ self.client.sync(extract_ddf_partitions, X) if X is None: raise ValueError("Model needs to be trained using fit() " "before calling kneighbors()") """ Create communicator clique """ comms = NearestNeighbors._build_comms(self.X, query_futures, self.streams_per_handle, self.verbose) """ Initialize models on workers """ nn_models = self._create_models(comms) """ Perform model query """ nn_fit, out_d_futures, out_i_futures = \ self._query_models(n_neighbors, comms, nn_models, self.X, query_futures) comms.destroy() if _return_futures: ret = nn_fit, out_i_futures if not return_distance else \ (nn_fit, out_d_futures, out_i_futures) else: ret = to_dask_cudf(out_i_futures) \ if not return_distance else (to_dask_cudf(out_d_futures), to_dask_cudf(out_i_futures)) return ret
def _parallel_func(self, X, func): """ Internal function that predicts the labels using a distributed KMeans model. Parameters ---------- X : dask_cudf.Dataframe Dataframe to predict Returns ------- result: dask_cudf.Dataframe Dataframe containing label predictions """ key = uuid1() gpu_futures = self.client.sync(extract_ddf_partitions, X) worker_to_parts = workers_to_parts(gpu_futures) kmeans_predict = [ self.client.submit(func, self.local_model, wf[1], workers=[wf[0]], key="%s-%s" % (key, idx)) for idx, wf in enumerate(worker_to_parts.items()) ] self.raise_exception_from_futures(kmeans_predict) return to_dask_cudf(kmeans_predict)
def predict(self, X): """ Make predictions for X and returns a y_pred. Parameters ---------- X : dask cuDF dataframe (n_rows, n_features) Returns ------- y : dask cuDF (n_rows, 1) """ gpu_futures = self.client.sync(extract_ddf_partitions, X) worker_to_parts = OrderedDict() for w, p in gpu_futures: if w not in worker_to_parts: worker_to_parts[w] = [] worker_to_parts[w].append(p) key = uuid1() partsToSizes = [(self.rnks[wf[0]], self.client.submit( Ridge._func_get_size, wf[1], workers=[wf[0]], key="%s-%s" % (key, idx)).result()) for idx, wf in enumerate(gpu_futures)] n_cols = X.shape[1] n_rows = reduce(lambda a, b: a+b, map(lambda x: x[1], partsToSizes)) key = uuid1() linear_pred = dict([(self.rnks[wf[0]], self.client.submit( Ridge._func_predict, wf[1], worker_to_parts[wf[0]], n_rows, n_cols, partsToSizes, self.rnks[wf[0]], key="%s-%s" % (key, idx), workers=[wf[0]])) for idx, wf in enumerate(self.linear_models)]) wait(list(linear_pred.values())) raise_exception_from_futures(list(linear_pred.values())) out_futures = [] completed_part_map = {} for rank, size in partsToSizes: if rank not in completed_part_map: completed_part_map[rank] = 0 f = linear_pred[rank] out_futures.append(self.client.submit( Ridge._func_get_idx, f, completed_part_map[rank])) completed_part_map[rank] += 1 return to_dask_cudf(out_futures)
def _inverse_transform(self, X): gpu_futures = self.client.sync(extract_ddf_partitions, X) worker_to_parts = OrderedDict() for w, p in gpu_futures: if w not in worker_to_parts: worker_to_parts[w] = [] worker_to_parts[w].append(p) key = uuid1() partsToRanks = [(self.rnks[wf[0]], self.client.submit(TruncatedSVD._func_get_size, wf[1], workers=[wf[0]], key="%s-%s" % (key, idx)).result()) for idx, wf in enumerate(gpu_futures)] N = X.shape[1] M = reduce(lambda a, b: a + b, map(lambda x: x[1], partsToRanks)) key = uuid1() tsvd_inverse_transform = dict([ (self.rnks[wf[0]], self.client.submit(TruncatedSVD._func_inverse_transform, wf[1], worker_to_parts[wf[0]], M, N, partsToRanks, self.rnks[wf[0]], key="%s-%s" % (key, idx), workers=[wf[0]])) for idx, wf in enumerate(self.tsvd_models) ]) wait(list(tsvd_inverse_transform.values())) raise_exception_from_futures(list(tsvd_inverse_transform.values())) out_futures = [] completed_part_map = {} for rank, size in partsToRanks: if rank not in completed_part_map: completed_part_map[rank] = 0 f = tsvd_inverse_transform[rank] out_futures.append( self.client.submit(TruncatedSVD._func_get_idx, f, completed_part_map[rank])) completed_part_map[rank] += 1 return to_dask_cudf(out_futures)
def post_etl_processing(client, train_data, test_data): import cudf from cuml.dask.naive_bayes import MultinomialNB as DistMNB from cuml.dask.common import to_dask_cudf from cuml.dask.common.input_utils import DistributedDataHandler # Feature engineering X_train = build_features(train_data) X_test = build_features(test_data) y_train = build_labels(train_data) y_test = build_labels(test_data) # Perform ML model = DistMNB(client=client, alpha=0.001) model.fit(X_train, y_train) ### this regression seems to be coming from here test_pred_st = time.time() y_hat = model.predict(X_test).persist() # Compute distributed performance metrics acc = accuracy_score(client, y_test, y_hat) print("Accuracy: " + str(acc)) prec = precision_score(client, y_test, y_hat, average="macro") print("Precision: " + str(prec)) cmat = confusion_matrix(client, y_test, y_hat) print("Confusion Matrix: " + str(cmat)) metric_et = time.time() # Place results back in original Dataframe ddh = DistributedDataHandler.create(y_hat) test_preds = to_dask_cudf( [client.submit(cudf.Series, part) for w, part in ddh.gpu_futures]) test_preds = test_preds.map_partitions(categoricalize) test_data["prediction"] = test_preds final_data = test_data[["pr_review_sk", "pr_review_rating", "prediction"]].persist() final_data = final_data.sort_values("pr_review_sk").reset_index(drop=True) wait(final_data) return final_data, acc, prec, cmat
def parallel_func(self, X, func): """ Predicts the labels using a distributed KMeans model :param X: dask_cudf.Dataframe to predict :return: A dask_cudf.Dataframe containing label predictions """ gpu_futures = self.client.sync(extract_ddf_partitions, X) kmeans_predict = [ self.client.submit(func, self.local_model, f, random.random(), workers=[w]) for w, f in gpu_futures.items() ] return to_dask_cudf(kmeans_predict)
def parallel_func(self, X, func): """ Predicts the labels using a distributed KMeans model :param X: dask_cudf.Dataframe to predict :return: A dask_cudf.Dataframe containing label predictions """ key = uuid1() gpu_futures = self.client.sync(extract_ddf_partitions, X) kmeans_predict = [ self.client.submit(func, self.local_model, wf[1], workers=[wf[0]], key="%s-%s" % (key, idx)) for idx, wf in enumerate(gpu_futures.items()) ] self.raise_exception_from_futures(kmeans_predict) return to_dask_cudf(kmeans_predict)
def fit(self, X, _transform=False): """ Fit the model with X. Parameters ---------- X : dask cuDF input """ gpu_futures = self.client.sync(extract_ddf_partitions, X) self.rnks = dict() rnk_counter = 0 worker_to_parts = OrderedDict() for w, p in gpu_futures: if w not in worker_to_parts: worker_to_parts[w] = [] if w not in self.rnks.keys(): self.rnks[w] = rnk_counter rnk_counter = rnk_counter + 1 worker_to_parts[w].append(p) workers = list(map(lambda x: x[0], gpu_futures)) comms = CommsContext(comms_p2p=False) comms.init(workers=workers) worker_info = comms.worker_info(comms.worker_addresses) key = uuid1() partsToRanks = [(worker_info[wf[0]]["r"], self.client.submit(TruncatedSVD._func_get_size, wf[1], workers=[wf[0]], key="%s-%s" % (key, idx)).result()) for idx, wf in enumerate(gpu_futures)] N = X.shape[1] M = reduce(lambda a, b: a + b, map(lambda x: x[1], partsToRanks)) key = uuid1() self.tsvd_models = [ (wf[0], self.client.submit(TruncatedSVD._func_create_model, comms.sessionId, wf[1], **self.kwargs, workers=[wf[0]], key="%s-%s" % (key, idx))) for idx, wf in enumerate(worker_to_parts.items()) ] key = uuid1() tsvd_fit = dict([(worker_info[wf[0]]["r"], self.client.submit(TruncatedSVD._func_fit, wf[1], M, N, partsToRanks, worker_info[wf[0]]["r"], _transform, key="%s-%s" % (key, idx), workers=[wf[0]])) for idx, wf in enumerate(self.tsvd_models)]) wait(list(tsvd_fit.values())) raise_exception_from_futures(list(tsvd_fit.values())) comms.destroy() self.local_model = self.client.submit(TruncatedSVD._func_get_first, self.tsvd_models[0][1]).result() self.components_ = self.local_model.components_ self.explained_variance_ = self.local_model.explained_variance_ self.explained_variance_ratio_ = \ self.local_model.explained_variance_ratio_ self.singular_values_ = self.local_model.singular_values_ out_futures = [] if _transform: completed_part_map = {} for rank, size in partsToRanks: if rank not in completed_part_map: completed_part_map[rank] = 0 f = tsvd_fit[rank] out_futures.append( self.client.submit(TruncatedSVD._func_get_idx, f, completed_part_map[rank])) completed_part_map[rank] += 1 return to_dask_cudf(out_futures)