def test_fast(client): L = client.map(inc, range(100)) L2 = client.map(dec, L) L3 = client.map(add, L, L2) p = progress(L3, multi=True, complete=True, notebook=True) client.sync(p.listen) assert set(p._last_response["all"]) == {"inc", "dec", "add"}
def test_fast(client): L = client.map(inc, range(100)) L2 = client.map(dec, L) L3 = client.map(add, L, L2) p = progress(L3, multi=True, complete=True, notebook=True) client.sync(p.listen) assert set(p._last_response['all']) == {'inc', 'dec', 'add'}
def test_progressbar_cancel(client): import time L = [client.submit(lambda: time.sleep(0.3), i) for i in range(5)] p = ProgressWidget(L) client.sync(p.listen) L[-1].cancel() wait(L[:-1]) assert p.status == 'error' assert p.bar.value == 0 # no tasks finish before cancel is called
def test_progressbar_cancel(client): import time L = [client.submit(lambda: time.sleep(0.3), i) for i in range(5)] p = ProgressWidget(L) client.sync(p.listen) L[-1].cancel() wait(L[:-1]) assert p.status == 'error' assert p.bar.value == 0 # no tasks finish before cancel is called
def test_values(client): L = [client.submit(inc, i) for i in range(5)] wait(L) p = MultiProgressWidget(L) client.sync(p.listen) assert set(p.bars) == {"inc"} assert p.status == "finished" assert p.comm.closed() assert "5 / 5" in p.bar_texts["inc"].value assert p.bars["inc"].value == 1.0 x = client.submit(throws, 1) p = MultiProgressWidget([x]) client.sync(p.listen) assert p.status == "error"
def test_values(client): L = [client.submit(inc, i) for i in range(5)] wait(L) p = MultiProgressWidget(L) client.sync(p.listen) assert set(p.bars) == {'inc'} assert p.status == 'finished' assert p.comm.closed() assert '5 / 5' in p.bar_texts['inc'].value assert p.bars['inc'].value == 1.0 x = client.submit(throws, 1) p = MultiProgressWidget([x]) client.sync(p.listen) assert p.status == 'error'
def test_values(client): L = [client.submit(inc, i) for i in range(5)] wait(L) p = MultiProgressWidget(L) client.sync(p.listen) assert set(p.bars) == {'inc'} assert p.status == 'finished' assert p.comm.closed() assert '5 / 5' in p.bar_texts['inc'].value assert p.bars['inc'].value == 1.0 x = client.submit(throws, 1) p = MultiProgressWidget([x]) client.sync(p.listen) assert p.status == 'error'
def test_progressbar_done(client): L = [client.submit(inc, i) for i in range(5)] wait(L) p = ProgressWidget(L) client.sync(p.listen) assert p.status == "finished" assert p.bar.value == 1.0 assert p.bar.bar_style == "success" assert "Finished" in p.elapsed_time.value f = client.submit(throws, L) wait([f]) p = ProgressWidget([f]) client.sync(p.listen) assert p.status == "error" assert p.bar.value == 0.0 assert p.bar.bar_style == "danger" assert "Exception" in p.elapsed_time.value try: throws(1) except Exception as e: assert repr(e) in p.elapsed_time.value
def test_progressbar_done(client): L = [client.submit(inc, i) for i in range(5)] wait(L) p = ProgressWidget(L) client.sync(p.listen) assert p.status == 'finished' assert p.bar.value == 1.0 assert p.bar.bar_style == 'success' assert 'Finished' in p.elapsed_time.value f = client.submit(throws, L) wait([f]) p = ProgressWidget([f]) client.sync(p.listen) assert p.status == 'error' assert p.bar.value == 0.0 assert p.bar.bar_style == 'danger' assert 'Exception' in p.elapsed_time.value try: throws(1) except Exception as e: assert repr(e) in p.elapsed_time.value
def run_quantile(self, name: str) -> None: if sys.platform.startswith("win"): pytest.skip("Skipping dask tests on Windows") exe: Optional[str] = None for possible_path in { './testxgboost', './build/testxgboost', '../build/testxgboost', '../cpu-build/testxgboost' }: if os.path.exists(possible_path): exe = possible_path if exe is None: return test = "--gtest_filter=Quantile." + name def runit(worker_addr: str, rabit_args: List[bytes]) -> subprocess.CompletedProcess: port_env = '' # setup environment for running the c++ part. for arg in rabit_args: if arg.decode('utf-8').startswith('DMLC_TRACKER_PORT'): port_env = arg.decode('utf-8') port = port_env.split('=') env = os.environ.copy() env[port[0]] = port[1] return subprocess.run([str(exe), test], env=env, capture_output=True) with LocalCluster(n_workers=4) as cluster: with Client(cluster) as client: workers = list(_get_client_workers(client).keys()) rabit_args = client.sync(xgb.dask._get_rabit_args, len(workers), client) futures = client.map(runit, workers, pure=False, workers=workers, rabit_args=rabit_args) results = client.gather(futures) for ret in results: msg = ret.stdout.decode('utf-8') assert msg.find('1 test from Quantile') != -1, msg assert ret.returncode == 0, msg
def test_data_initialization(self) -> None: '''Assert each worker has the correct amount of data, and DMatrix initialization doesn't generate unnecessary copies of data. ''' with LocalCluster(n_workers=2) as cluster: with Client(cluster) as client: X, y, _ = generate_array() n_partitions = X.npartitions m = xgb.dask.DaskDMatrix(client, X, y) workers = list(_get_client_workers(client).keys()) rabit_args = client.sync(xgb.dask._get_rabit_args, len(workers), client) n_workers = len(workers) def worker_fn(worker_addr: str, data_ref: Dict) -> None: with xgb.dask.RabitContext(rabit_args): local_dtrain = xgb.dask._dmatrix_from_list_of_parts( **data_ref) total = np.array([local_dtrain.num_row()]) total = xgb.rabit.allreduce(total, xgb.rabit.Op.SUM) assert total[0] == kRows futures = [] for i in range(len(workers)): futures.append( client.submit(worker_fn, workers[i], m.create_fn_args(workers[i]), pure=False, workers=[workers[i]])) client.gather(futures) has_what = client.has_what() cnt = 0 data = set() for k, v in has_what.items(): for d in v: cnt += 1 data.add(d) assert len(data) == cnt # Subtract the on disk resource from each worker assert cnt - n_workers == n_partitions
def test_data_initialization(self): '''Assert each worker has the correct amount of data, and DMatrix initialization doesn't generate unnecessary copies of data. ''' with LocalCluster(n_workers=2) as cluster: with Client(cluster) as client: X, y = generate_array() n_partitions = X.npartitions m = xgb.dask.DaskDMatrix(client, X, y) workers = list(xgb.dask._get_client_workers(client).keys()) rabit_args = client.sync(xgb.dask._get_rabit_args, workers, client) n_workers = len(workers) def worker_fn(worker_addr, data_ref): with xgb.dask.RabitContext(rabit_args): local_dtrain = xgb.dask._dmatrix_from_worker_map( **data_ref) assert local_dtrain.num_row() == kRows / n_workers futures = client.map(worker_fn, workers, [m.create_fn_args()] * len(workers), pure=False, workers=workers) client.gather(futures) has_what = client.has_what() cnt = 0 data = set() for k, v in has_what.items(): for d in v: cnt += 1 data.add(d) assert len(data) == cnt # Subtract the on disk resource from each worker assert cnt - n_workers == n_partitions