def test_tree_stats(self) -> None:
with LocalCUDACluster(n_workers=1) as cluster:
with Client(cluster) as client:
local = run_tree_stats(client, "gpu_hist")
with LocalCUDACluster(n_workers=2) as cluster:
with Client(cluster) as client:
distributed = run_tree_stats(client, "gpu_hist")
assert local == distributed
async def test_local_cuda_cluster():
async with LocalCUDACluster(asynchronous=True) as cluster:
async with Client(cluster, asynchronous=True) as client:
assert len(cluster.workers) == utils.get_n_gpus()
# CUDA_VISIBLE_DEVICES cycles properly
def get_visible_devices():
return os.environ["CUDA_VISIBLE_DEVICES"]
result = await client.run(get_visible_devices)
assert all(
len(v.split(",")) == utils.get_n_gpus()
for v in result.values())
for i in range(utils.get_n_gpus()):
assert {int(v.split(",")[i])
for v in result.values()
} == set(range(utils.get_n_gpus()))
# Use full memory
assert sum(w.memory_limit
for w in cluster.workers.values()) == TOTAL_MEMORY
for w, devices in result.items():
ident = devices[0]
assert int(ident) == cluster.scheduler.workers[w].name
with pytest.raises(ValueError):
cluster.scale(1000)
def test_consolidation(graph_file):
gc.collect()
cluster = LocalCUDACluster()
client = Client(cluster)
chunksize = dcg.get_chunksize(graph_file)
M = utils.read_csv_for_nx(graph_file)
df = pd.DataFrame()
df['source'] = pd.Series(M['0'])
df['target'] = pd.Series(M['1'])
ddf = dask_cudf.read_csv(graph_file,
chunksize=chunksize,
delimiter=' ',
names=['source', 'target', 'weight'],
dtype=['int32', 'int32', 'float32'],
header=None)
Gnx = nx.from_pandas_edgelist(df,
source='source',
target='target',
create_using=nx.DiGraph)
G = cugraph.from_cudf_edgelist(ddf,
source='source',
destination='target',
create_using=cugraph.DiGraph)
assert compare_graphs(Gnx, G)
Gnx.clear()
G.clear()
client.close()
cluster.close()
async def run():
initialize(
create_cuda_context=True,
enable_tcp_over_ucx=enable_tcp_over_ucx,
enable_infiniband=enable_infiniband,
enable_nvlink=enable_nvlink,
)
async with LocalCUDACluster(
interface="enp1s0f0",
protocol="ucx",
enable_tcp_over_ucx=enable_tcp_over_ucx,
enable_infiniband=enable_infiniband,
enable_nvlink=enable_nvlink,
asynchronous=True,
) as cluster:
async with Client(cluster, asynchronous=True) as client:
rs = da.random.RandomState(RandomState=cupy.random.RandomState)
a = rs.normal(10, 1, (int(4e3), int(4e3)), chunks=(int(1e3), int(1e3)))
x = a + a.T
for i in range(100):
print("Running iteration:", i)
start = time.time()
await client.compute(x)
print("Time for iteration", i, ":", time.time() - start)
def test_empty_dmatrix(self):
with LocalCUDACluster() as cluster:
with Client(cluster) as client:
parameters = {'tree_method': 'gpu_hist',
'debug_synchronize': True}
run_empty_dmatrix_reg(client, parameters)
run_empty_dmatrix_cls(client, parameters)
def setup(dask_scheduler_file=None, rmm_pool_size=None):
if dask_scheduler_file:
cluster = None
# Env var UCX_MAX_RNDV_RAILS=1 must be set too.
initialize(
enable_tcp_over_ucx=True,
enable_nvlink=True,
enable_infiniband=False,
enable_rdmacm=False,
#net_devices="mlx5_0:1",
)
client = Client(scheduler_file=dask_scheduler_file)
else:
tempdir_object = tempfile.TemporaryDirectory()
cluster = LocalCUDACluster(local_directory=tempdir_object.name,
rmm_pool_size=rmm_pool_size)
client = Client(cluster)
# add the obj to the client so it doesn't get deleted until
# the 'client' obj gets cleaned up
client.tempdir_object = tempdir_object
client.wait_for_workers(len(get_visible_devices()))
Comms.initialize(p2p=True)
return (client, cluster)
async def test_with_subset_of_cuda_visible_devices():
os.environ["CUDA_VISIBLE_DEVICES"] = "2,3,6,7"
try:
async with LocalCUDACluster(scheduler_port=0,
asynchronous=True,
device_memory_limit=1) as cluster:
async with Client(cluster, asynchronous=True) as client:
assert len(cluster.workers) == 4
# CUDA_VISIBLE_DEVICES cycles properly
def get_visible_devices():
return os.environ["CUDA_VISIBLE_DEVICES"]
result = await client.run(get_visible_devices)
assert all(len(v.split(",")) == 4 for v in result.values())
for i in range(4):
assert {int(v.split(",")[i])
for v in result.values()} == {
2,
3,
6,
7,
}
finally:
del os.environ["CUDA_VISIBLE_DEVICES"]
def fit(self, data, args):
params = self.configure(data, args)
cluster = LocalCUDACluster(
n_workers=None if args.gpus < 0 else args.gpus,
local_directory=args.root,
threads_per_worker=1)
client = Client(cluster)
partition_size = 10000
if isinstance(data.X_train, np.ndarray):
X = da.from_array(data.X_train,
(partition_size, data.X_train.shape[1]))
y = da.from_array(data.y_train, partition_size)
else:
X = dd.from_pandas(data.X_train, chunksize=partition_size)
y = dd.from_pandas(data.y_train, chunksize=partition_size)
dtrain = xgb.dask.DaskDMatrix(client, X, y)
with Timer() as t:
output = xgb.dask.train(client,
params,
dtrain,
num_boost_round=args.ntrees)
self.model = output['booster']
client.close()
cluster.close()
return t.interval
def cluster_initialize(self):
""" Initialize dask GPU cluster"""
cluster = None
client = None
self.n_workers = cupy.cuda.runtime.getDeviceCount()
cluster = LocalCUDACluster(n_workers=self.n_workers)
client = Client(cluster)
hpo_log.info(f'dask multi-GPU cluster with {self.n_workers} workers ')
dask.config.set({
'temporary_directory': self.hpo_config.output_artifacts_directory,
'logging': {
'loggers': {
'distributed.nanny': {
'level': 'CRITICAL'
}
}
} # noqa
})
return cluster, client
async def run():
initialize(
create_cuda_context=True,
enable_tcp_over_ucx=enable_tcp_over_ucx,
enable_infiniband=enable_infiniband,
enable_nvlink=enable_nvlink,
)
async with LocalCUDACluster(
interface="enp1s0f0",
protocol="ucx",
enable_tcp_over_ucx=enable_tcp_over_ucx,
enable_infiniband=enable_infiniband,
enable_nvlink=enable_nvlink,
asynchronous=True,
) as cluster:
async with Client(cluster, asynchronous=True) as client:
d = dask_cudf.from_cudf(
cudf.DataFrame({"a": range(2 ** 16)}), npartitions=2
)
r = d.sum()
for i in range(100):
print("Running iteration:", i)
start = time.time()
await client.compute(r)
print("Time for iteration", i, ":", time.time() - start)
def __init__(self,
model_type='RandomForest',
compute_type='multi-GPU',
CSP_paths=default_sagemaker_paths):
self.CSP_paths = CSP_paths
self.model_type = model_type
self.compute_type = compute_type
# CPU or GPU cluster
if 'multi-GPU' in self.compute_type:
self.n_workers = cupy.cuda.runtime.getDeviceCount()
self.cluster = LocalCUDACluster(n_workers=self.n_workers)
self.client = Client(self.cluster)
print(f'dask multi-GPU cluster with {self.n_workers} workers ')
elif 'multi-CPU' in self.compute_type:
self.n_workers = os.cpu_count()
self.cluster = LocalCluster(n_workers=self.n_workers,
threads_per_worker=1)
self.client = Client(self.cluster)
print(f'dask multi-CPU cluster with {self.n_workers} workers')
else:
self.cluster = None
self.client = None
def launch_dask(n_gpus, min_gpus, k8s, adapt, worker_spec):
if k8s:
if worker_spec is None:
worker_spec = default_worker_spec_fname
print(f'Creating a default K8S worker spec at {worker_spec}')
with open(worker_spec, "w") as yaml_file:
yaml_file.write(default_worker_spec)
cluster = KubeCluster.from_yaml(worker_spec)
if adapt:
cluster.adapt(minimum=min_gpus, maximum=n_gpus)
print(
f'Launching Adaptive K8S Dask cluster with [{min_gpus}, {n_gpus}] workers'
)
else:
cluster.scale(n_gpus)
print(f'Launching K8S Dask cluster with {n_gpus} workers')
sleep(10)
else:
cluster = LocalCUDACluster(ip="", n_workers=n_gpus)
print(f'Launching Local Dask cluster with {n_gpus} GPUs')
client = Client(cluster)
print(client)
print(cluster)
return client, cluster
def cluster_initialize(self):
"""Initialize dask GPU cluster"""
cluster = None
client = None
self.n_workers = cupy.cuda.runtime.getDeviceCount()
cluster = LocalCUDACluster(n_workers=self.n_workers)
client = Client(cluster)
hpo_log.info(f"dask multi-GPU cluster with {self.n_workers} workers ")
dask.config.set({
"temporary_directory": self.hpo_config.output_artifacts_directory,
"logging": {
"loggers": {
"distributed.nanny": {
"level": "CRITICAL"
}
}
}, # noqa
})
return cluster, client
def _init(self):
"""Allocate cluster resources
"""
if self.memory_limit > 0:
memory_limit = int(self.memory_limit * 1024 * 1024 * 1024)
else:
memory_limit = None
if self.n_cpu == 0:
n_cpu = None
else:
n_cpu = self.n_cpu
if self.n_visible_gpu:
cluster = LocalCUDACluster(
n_workers=n_cpu,
memory_limit=memory_limit,
dashboard_address=f":{self._get_port()}",
CUDA_VISIBLE_DEVICES=self.n_visible_gpu,
)
else:
cluster = DaskLocalClutster(
n_workers=n_cpu,
memory_limit=memory_limit,
dashboard_address=f":{self._get_port()}",
)
self.client = Client(cluster)
self.initialized = True
async def test_available_mig_workers():
uuids = get_gpu_count_mig(return_uuids=True)[1]
if len(uuids) > 0:
cuda_visible_devices = ",".join([i.decode("utf-8") for i in uuids])
else:
pytest.skip("No MIG devices found")
with patch.dict(os.environ,
{"CUDA_VISIBLE_DEVICES": cuda_visible_devices}):
async with LocalCUDACluster(CUDA_VISIBLE_DEVICES=cuda_visible_devices,
asynchronous=True) as cluster:
async with Client(cluster, asynchronous=True) as client:
len(cluster.workers) == len(uuids)
# Check to see if CUDA_VISIBLE_DEVICES cycles properly
def get_visible_devices():
return os.environ["CUDA_VISIBLE_DEVICES"]
result = await client.run(get_visible_devices)
assert all(
len(v.split(",")) == len(uuids) for v in result.values())
for i in range(len(uuids)):
assert set(v.split(",")[i]
for v in result.values()) == set(uuids)
def test_pagerank():
gc.collect()
input_data_path = r"../datasets/karate.csv"
# Networkx Call
pd_df = pd.read_csv(input_data_path,
delimiter=' ',
names=['src', 'dst', 'value'])
G = nx.Graph()
for i in range(0, len(pd_df)):
G.add_edge(pd_df['src'][i], pd_df['dst'][i])
nx_pr = nx.pagerank(G, alpha=0.85)
nx_pr = sorted(nx_pr.items(), key=lambda x: x[0])
# Cugraph snmg pagerank Call
cluster = LocalCUDACluster(threads_per_worker=1)
client = Client(cluster)
chunksize = dcg.get_chunksize(input_data_path)
ddf = dask_cudf.read_csv(input_data_path,
chunksize=chunksize,
delimiter=' ',
names=['src', 'dst', 'value'],
dtype=['int32', 'int32', 'float32'])
pr = dcg.pagerank(ddf, alpha=0.85, max_iter=50)
res_df = pr.compute()
err = 0
tol = 1.0e-05
for i in range(len(res_df)):
if (abs(res_df['pagerank'][i] - nx_pr[i][1]) > tol * 1.1):
err = err + 1
print("Mismatches:", err)
assert err < (0.01 * len(res_df))
client.close()
cluster.close()
def test_send_recv(n_trials):
cluster = LocalCUDACluster(threads_per_worker=1)
client = Client(cluster)
cb = CommsContext(comms_p2p=True)
cb.init()
cb = default_comms()
start = time.time()
dfs = [client.submit(func_test_send_recv,
cb.sessionId,
n_trials,
random.random(),
workers=[w])
for wid, w in zip(range(len(cb.worker_addresses)),
cb.worker_addresses)]
wait(dfs)
print("Time: " + str(time.time() - start))
result = list(map(lambda x: x.result(), dfs))
print(str(result))
assert(result)
cb.destroy()
client.close()
cluster.close()
def test_dask_dataframe(self):
with LocalCUDACluster() as cluster:
with Client(cluster) as client:
import cupy
X, y = generate_array()
X = dd.from_dask_array(X)
y = dd.from_dask_array(y)
X = X.map_partitions(cudf.from_pandas)
y = y.map_partitions(cudf.from_pandas)
dtrain = dxgb.DaskDMatrix(client, X, y)
out = dxgb.train(client, {'tree_method': 'gpu_hist'},
dtrain=dtrain,
evals=[(dtrain, 'X')],
num_boost_round=2)
assert isinstance(out['booster'], dxgb.Booster)
assert len(out['history']['X']['rmse']) == 2
predictions = dxgb.predict(client, out, dtrain).compute()
assert isinstance(predictions, np.ndarray)
series_predictions = dxgb.inplace_predict(client, out, X)
assert isinstance(series_predictions, dd.Series)
series_predictions = series_predictions.compute()
single_node = out['booster'].predict(
xgboost.DMatrix(X.compute()))
cupy.testing.assert_allclose(single_node, predictions)
cupy.testing.assert_allclose(single_node, series_predictions)
async def test_local_cuda_cluster():
async with LocalCUDACluster(scheduler_port=0,
asynchronous=True,
device_memory_limit=1) as cluster:
async with Client(cluster, asynchronous=True) as client:
assert len(cluster.workers) == utils.get_n_gpus()
# CUDA_VISIBLE_DEVICES cycles properly
def get_visible_devices():
return os.environ["CUDA_VISIBLE_DEVICES"]
result = await client.run(get_visible_devices)
assert all(
len(v.split(",")) == utils.get_n_gpus()
for v in result.values())
for i in range(utils.get_n_gpus()):
assert {int(v.split(",")[i])
for v in result.values()
} == set(range(utils.get_n_gpus()))
# Use full memory, checked with some buffer to ignore rounding difference
full_mem = sum(w.memory_limit for w in cluster.workers.values())
assert full_mem >= MEMORY_LIMIT - 1024 and full_mem < MEMORY_LIMIT + 1024
for w, devices in result.items():
ident = devices.split(",")[0]
assert int(ident) == cluster.scheduler.workers[w].name
with pytest.raises(ValueError):
cluster.scale(1000)
def publish_dataset_to_cluster():
census_data_url = 'https://s3.us-east-2.amazonaws.com/rapidsai-data/viz-data/census_data.parquet.tar.gz'
data_path = "../data/census_data.parquet"
check_dataset(census_data_url, data_path)
# Note: The creation of a Dask LocalCluster must happen inside the `__main__` block,
cluster = LocalCUDACluster(CUDA_VISIBLE_DEVICES="0")
client = Client(cluster)
print(f"Dask status: {cluster.dashboard_link}")
# Load dataset and persist dataset on cluster
def load_and_publish_dataset():
# cudf DataFrame
c_df_d = delayed(load_dataset)(data_path).persist()
# pandas DataFrame
pd_df_d = delayed(c_df_d.to_pandas)().persist()
# print(type(c_df_d))
# Unpublish datasets if present
for ds_name in ['pd_df_d', 'c_df_d']:
if ds_name in client.datasets:
client.unpublish_dataset(ds_name)
# Publish datasets to the cluster
client.publish_dataset(pd_df_d=pd_df_d)
client.publish_dataset(c_df_d=c_df_d)
load_and_publish_dataset()
# Precompute field bounds
c_df_d = client.get_dataset('c_df_d')
# Register top-level callback that updates plots
register_update_plots_callback(client)
def create_cuml_distributed(X_train, y_train):
start_time = datetime.now()
print('init dask cluster')
cluster = LocalCUDACluster(threads_per_worker=1)
client = Client(cluster)
workers = client.has_what().keys()
n_workers = len(workers)
X_train_cudf = cudf.DataFrame.from_pandas(pd.DataFrame(X_train))
y_train_cudf = cudf.Series(y_train)
X_train_dask = dask_cudf.from_cudf(X_train_cudf, npartitions=n_workers)
y_train_dask = dask_cudf.from_cudf(y_train_cudf, npartitions=n_workers)
X_train_ddask, y_train_ddask = dask_utils.persist_across_workers(
client, [X_train_dask, y_train_dask], workers=workers)
print('cuml distributed initialized', datetime.now() - start_time)
model = distributed_cuml_Rf(n_estimators=500, n_streams=64)
model.fit(X_train, y_train)
wait(model.rfs)
print('cuml distributed finished', datetime.now() - start_time)
client.close()
cluster.close()
return model
def test_gpu_hist(self, params, num_rounds, dataset):
with LocalCUDACluster(n_workers=2) as cluster:
with Client(cluster) as client:
params['tree_method'] = 'gpu_hist'
params = dataset.set_params(params)
# multi class doesn't handle empty dataset well (empty
# means at least 1 worker has data).
if params['objective'] == "multi:softmax":
return
# It doesn't make sense to distribute a completely
# empty dataset.
if dataset.X.shape[0] == 0:
return
chunk = 128
X = da.from_array(dataset.X,
chunks=(chunk, dataset.X.shape[1]))
y = da.from_array(dataset.y, chunks=(chunk, ))
if dataset.w is not None:
w = da.from_array(dataset.w, chunks=(chunk, ))
else:
w = None
m = dxgb.DaskDMatrix(client, data=X, label=y, weight=w)
history = dxgb.train(client,
params=params,
dtrain=m,
num_boost_round=num_rounds,
evals=[(m, 'train')])['history']
note(history)
assert tm.non_increasing(history['train'][dataset.metric])
def test_gpu_hist(self, params, num_rounds, dataset):
with LocalCUDACluster(n_workers=2) as cluster:
with Client(cluster) as client:
run_gpu_hist(params, num_rounds, dataset, dxgb.DaskDMatrix,
client)
run_gpu_hist(params, num_rounds, dataset,
dxgb.DaskDeviceQuantileDMatrix, client)
def gpu_cluster():
if LocalCUDACluster is None:
pytest.skip("dask_cuda not installed")
return None
with LocalCUDACluster(protocol="tcp") as cluster:
yield cluster
def test_with_asyncio():
with LocalCUDACluster() as cluster:
with Client(cluster) as client:
address = client.scheduler.address
output = asyncio.run(run_from_dask_array_asyncio(address))
assert isinstance(output['booster'], xgboost.Booster)
assert isinstance(output['history'], dict)
def test_dask_array(self):
with LocalCUDACluster() as cluster:
with Client(cluster) as client:
import cupy as cp
cp.cuda.runtime.setDevice(0)
X, y = generate_array()
X = X.map_blocks(cp.asarray)
y = y.map_blocks(cp.asarray)
dtrain = dxgb.DaskDMatrix(client, X, y)
out = dxgb.train(client, {'tree_method': 'gpu_hist'},
dtrain=dtrain,
evals=[(dtrain, 'X')],
num_boost_round=2)
from_dmatrix = dxgb.predict(client, out, dtrain).compute()
inplace_predictions = dxgb.inplace_predict(client, out,
X).compute()
single_node = out['booster'].predict(
xgboost.DMatrix(X.compute()))
np.testing.assert_allclose(single_node, from_dmatrix)
device = cp.cuda.runtime.getDevice()
assert device == inplace_predictions.device.id
single_node = cp.array(single_node)
assert device == single_node.device.id
cp.testing.assert_allclose(single_node, inplace_predictions)
def __init__(
self,
cloud_type="Azure",
model_type="RandomForest",
data_type="Parquet",
compute_type="single-GPU",
verbose_estimator=False,
CSP_paths=default_azureml_paths,
):
self.CSP_paths = CSP_paths
self.cloud_type = cloud_type
self.model_type = model_type
self.data_type = data_type
self.compute_type = compute_type
self.verbose_estimator = verbose_estimator
self.log_to_file(
f"\n> RapidsCloudML\n\tCompute, Data , Model, Cloud types {self.compute_type, self.data_type, self.model_type, self.cloud_type}"
)
# Setting up client for multi-GPU option
if "multi" in self.compute_type:
self.log_to_file("\n\tMulti-GPU selected")
# This will use all GPUs on the local host by default
cluster = LocalCUDACluster(threads_per_worker=1)
self.client = Client(cluster)
# Query the client for all connected workers
self.workers = self.client.has_what().keys()
self.n_workers = len(self.workers)
self.log_to_file(f"\n\tClient information {self.client}")
def setup_cluster(use_gpus: Optional[int] = None, worker_space: Optional[str] = None) -> Client:
"""Setup a dask distributed cuda cluster on GPUs. Sometimes
if this has been done before, the legacy data needs to be
removed.
Args:
use_gpus (int, optional): Number of gpus to use
worker_space (str, optional): The location of the
dask-worker-space director. Defaults to None.
Returns:
Client: A distributed cluster to contain dask or dask_cudf objects.
"""
if worker_space is not None and os.path.exists(worker_space):
os.system(f"rm -r {worker_space}/*")
if use_gpus is None:
use_gpus = gpus()
if bool(use_gpus):
cluster = LocalCUDACluster(n_workers=use_gpus,
local_directory=worker_space)
logger.info("GPU cluster has been established")
else:
cluster = None
logger.info("No GPUs found - running cluster on CPU.")
return Client(address=cluster)
def fit(self, data, args):
params = self.configure(data, args)
n_workers = None if args.gpus < 0 else args.gpus
cluster = LocalCUDACluster(n_workers=n_workers,
local_directory=args.root)
client = Client(cluster)
n_partitions = len(client.scheduler_info()['workers'])
X_sliced, y_sliced = self.get_slices(n_partitions, data.X_train,
data.y_train)
X = da.concatenate(
[da.from_array(sub_array) for sub_array in X_sliced])
X = X.rechunk((X_sliced[0].shape[0], data.X_train.shape[1]))
y = da.concatenate(
[da.from_array(sub_array) for sub_array in y_sliced])
y = y.rechunk(X.chunksize[0])
dtrain = xgb.dask.DaskDMatrix(client, X, y)
with Timer() as t:
output = xgb.dask.train(client,
params,
dtrain,
num_boost_round=args.ntrees)
self.model = output['booster']
client.close()
cluster.close()
return t.interval
def test_pca_fit_transform_fp32(nrows, ncols, n_parts, client=None):
owns_cluster = False
if client is None:
owns_cluster = True
cluster = LocalCUDACluster(threads_per_worker=1)
client = Client(cluster)
from cuml.dask.decomposition import PCA as daskPCA
from cuml.dask.datasets import make_blobs
X_cudf, _ = make_blobs(nrows,
ncols,
1,
n_parts,
cluster_std=1.5,
verbose=False,
random_state=10,
dtype=np.float32)
wait(X_cudf)
cupca = daskPCA(n_components=20, whiten=True)
cupca.fit_transform(X_cudf)
if owns_cluster:
client.close()
cluster.close()