def test_fast(client):
L = client.map(inc, range(100))
L2 = client.map(dec, L)
L3 = client.map(add, L, L2)
p = progress(L3, multi=True, complete=True, notebook=True)
client.sync(p.listen)
assert set(p._last_response["all"]) == {"inc", "dec", "add"}
def test_fast(client):
L = client.map(inc, range(100))
L2 = client.map(dec, L)
L3 = client.map(add, L, L2)
p = progress(L3, multi=True, complete=True, notebook=True)
client.sync(p.listen)
assert set(p._last_response['all']) == {'inc', 'dec', 'add'}
def test_progressbar_cancel(client):
import time
L = [client.submit(lambda: time.sleep(0.3), i) for i in range(5)]
p = ProgressWidget(L)
client.sync(p.listen)
L[-1].cancel()
wait(L[:-1])
assert p.status == 'error'
assert p.bar.value == 0 # no tasks finish before cancel is called
def test_progressbar_cancel(client):
import time
L = [client.submit(lambda: time.sleep(0.3), i) for i in range(5)]
p = ProgressWidget(L)
client.sync(p.listen)
L[-1].cancel()
wait(L[:-1])
assert p.status == 'error'
assert p.bar.value == 0 # no tasks finish before cancel is called
def test_values(client):
L = [client.submit(inc, i) for i in range(5)]
wait(L)
p = MultiProgressWidget(L)
client.sync(p.listen)
assert set(p.bars) == {"inc"}
assert p.status == "finished"
assert p.comm.closed()
assert "5 / 5" in p.bar_texts["inc"].value
assert p.bars["inc"].value == 1.0
x = client.submit(throws, 1)
p = MultiProgressWidget([x])
client.sync(p.listen)
assert p.status == "error"
def test_values(client):
L = [client.submit(inc, i) for i in range(5)]
wait(L)
p = MultiProgressWidget(L)
client.sync(p.listen)
assert set(p.bars) == {'inc'}
assert p.status == 'finished'
assert p.comm.closed()
assert '5 / 5' in p.bar_texts['inc'].value
assert p.bars['inc'].value == 1.0
x = client.submit(throws, 1)
p = MultiProgressWidget([x])
client.sync(p.listen)
assert p.status == 'error'
def test_values(client):
L = [client.submit(inc, i) for i in range(5)]
wait(L)
p = MultiProgressWidget(L)
client.sync(p.listen)
assert set(p.bars) == {'inc'}
assert p.status == 'finished'
assert p.comm.closed()
assert '5 / 5' in p.bar_texts['inc'].value
assert p.bars['inc'].value == 1.0
x = client.submit(throws, 1)
p = MultiProgressWidget([x])
client.sync(p.listen)
assert p.status == 'error'
def test_progressbar_done(client):
L = [client.submit(inc, i) for i in range(5)]
wait(L)
p = ProgressWidget(L)
client.sync(p.listen)
assert p.status == "finished"
assert p.bar.value == 1.0
assert p.bar.bar_style == "success"
assert "Finished" in p.elapsed_time.value
f = client.submit(throws, L)
wait([f])
p = ProgressWidget([f])
client.sync(p.listen)
assert p.status == "error"
assert p.bar.value == 0.0
assert p.bar.bar_style == "danger"
assert "Exception" in p.elapsed_time.value
try:
throws(1)
except Exception as e:
assert repr(e) in p.elapsed_time.value
def test_progressbar_done(client):
L = [client.submit(inc, i) for i in range(5)]
wait(L)
p = ProgressWidget(L)
client.sync(p.listen)
assert p.status == 'finished'
assert p.bar.value == 1.0
assert p.bar.bar_style == 'success'
assert 'Finished' in p.elapsed_time.value
f = client.submit(throws, L)
wait([f])
p = ProgressWidget([f])
client.sync(p.listen)
assert p.status == 'error'
assert p.bar.value == 0.0
assert p.bar.bar_style == 'danger'
assert 'Exception' in p.elapsed_time.value
try:
throws(1)
except Exception as e:
assert repr(e) in p.elapsed_time.value
def run_quantile(self, name: str) -> None:
if sys.platform.startswith("win"):
pytest.skip("Skipping dask tests on Windows")
exe: Optional[str] = None
for possible_path in {
'./testxgboost', './build/testxgboost', '../build/testxgboost',
'../cpu-build/testxgboost'
}:
if os.path.exists(possible_path):
exe = possible_path
if exe is None:
return
test = "--gtest_filter=Quantile." + name
def runit(worker_addr: str,
rabit_args: List[bytes]) -> subprocess.CompletedProcess:
port_env = ''
# setup environment for running the c++ part.
for arg in rabit_args:
if arg.decode('utf-8').startswith('DMLC_TRACKER_PORT'):
port_env = arg.decode('utf-8')
port = port_env.split('=')
env = os.environ.copy()
env[port[0]] = port[1]
return subprocess.run([str(exe), test],
env=env,
capture_output=True)
with LocalCluster(n_workers=4) as cluster:
with Client(cluster) as client:
workers = list(_get_client_workers(client).keys())
rabit_args = client.sync(xgb.dask._get_rabit_args,
len(workers), client)
futures = client.map(runit,
workers,
pure=False,
workers=workers,
rabit_args=rabit_args)
results = client.gather(futures)
for ret in results:
msg = ret.stdout.decode('utf-8')
assert msg.find('1 test from Quantile') != -1, msg
assert ret.returncode == 0, msg
def test_data_initialization(self) -> None:
'''Assert each worker has the correct amount of data, and DMatrix initialization doesn't
generate unnecessary copies of data.
'''
with LocalCluster(n_workers=2) as cluster:
with Client(cluster) as client:
X, y, _ = generate_array()
n_partitions = X.npartitions
m = xgb.dask.DaskDMatrix(client, X, y)
workers = list(_get_client_workers(client).keys())
rabit_args = client.sync(xgb.dask._get_rabit_args,
len(workers), client)
n_workers = len(workers)
def worker_fn(worker_addr: str, data_ref: Dict) -> None:
with xgb.dask.RabitContext(rabit_args):
local_dtrain = xgb.dask._dmatrix_from_list_of_parts(
**data_ref)
total = np.array([local_dtrain.num_row()])
total = xgb.rabit.allreduce(total, xgb.rabit.Op.SUM)
assert total[0] == kRows
futures = []
for i in range(len(workers)):
futures.append(
client.submit(worker_fn,
workers[i],
m.create_fn_args(workers[i]),
pure=False,
workers=[workers[i]]))
client.gather(futures)
has_what = client.has_what()
cnt = 0
data = set()
for k, v in has_what.items():
for d in v:
cnt += 1
data.add(d)
assert len(data) == cnt
# Subtract the on disk resource from each worker
assert cnt - n_workers == n_partitions
def test_data_initialization(self):
'''Assert each worker has the correct amount of data, and DMatrix initialization doesn't
generate unnecessary copies of data.
'''
with LocalCluster(n_workers=2) as cluster:
with Client(cluster) as client:
X, y = generate_array()
n_partitions = X.npartitions
m = xgb.dask.DaskDMatrix(client, X, y)
workers = list(xgb.dask._get_client_workers(client).keys())
rabit_args = client.sync(xgb.dask._get_rabit_args, workers,
client)
n_workers = len(workers)
def worker_fn(worker_addr, data_ref):
with xgb.dask.RabitContext(rabit_args):
local_dtrain = xgb.dask._dmatrix_from_worker_map(
**data_ref)
assert local_dtrain.num_row() == kRows / n_workers
futures = client.map(worker_fn,
workers,
[m.create_fn_args()] * len(workers),
pure=False,
workers=workers)
client.gather(futures)
has_what = client.has_what()
cnt = 0
data = set()
for k, v in has_what.items():
for d in v:
cnt += 1
data.add(d)
assert len(data) == cnt
# Subtract the on disk resource from each worker
assert cnt - n_workers == n_partitions
