def test_keep_results(): with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) assert c.get({'x': (inc, 1)}, 'x', keep_results=True) == 2 assert 'x' in a.data or 'x' in b.data
def test_error(): with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) assert raises(TypeError, lambda: c.get({'x': 1, 'y': (inc, 'x', 'x')}, 'y')) assert 'y' not in s.data c.close()
def main(): client = Client('localhost:8786') A = client.map(set_value, range(100)) B = client.map(square, A) C = client.map(neg, B) total = client.submit(sum, C) print(progress(total)) print(total.result())
def test_get(): with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) dsk = {'x': 1, 'y': (add, 'x', 'x'), 'z': (inc, 'y')} keys = ['y', 'z'] assert c.get(dsk, keys) == [2, 3] c.close()
def test_get_with_dill(): with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) dsk = {'x': 1, 'y': (partial(add, 1), 'x')} keys = 'y' assert c.get(dsk, keys) == 2 c.close()
def test_get_workers(): with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) pid = os.getpid() workers = c.get_registered_workers() assert set(workers) == set([a.address, b.address]) assert workers[a.address]['pid'] == pid assert workers[b.address]['pid'] == pid s.close_workers() assert c.get_registered_workers() == {}
def main(): # Setup logging on the main process: _start_logging() # Start three worker processes on the local machine: client = Client(n_workers=3, threads_per_worker=1) # Setup Eliot logging on each worker process: client.run(_start_logging) # Run the Dask computation in the worker processes: result = main_computation() print("Result:", result)
def main(): """.""" host = os.getenv('DASK_SCHEDULER_HOST', default='localhost') port = os.getenv('DASK_SCHEDULER_PORT', default=8786) print(host, port) client = Client('{}:{}'.format(host, port)) # client.run(init_logging) # client.run_on_scheduler(init_logging) # Run some mock functions and gather a result data = client.map(print_listdir, range(10)) future = client.submit(print_values, data) progress(future) print('') result = client.gather(future) print(result) # Run a second stage which runs some additional processing. print('here A') data_a = client.map(set_value, range(100)) print('here B') data_b = client.map(square, data_a) print('here C') data_c = client.map(neg, data_b) print('here D') # Submit a function application to the scheduler total = client.submit(sum, data_c) print('here E') progress(total) print(total.result()) print('here F')
def cli(scheduler_file, jlab_port, dash_port, notebook_dir, hostname, log_level): logger = get_logger(log_level) logger.info('getting client with scheduler file: %s' % scheduler_file) client = Client(scheduler_file=scheduler_file, timeout=30) logger.debug('Client: %s' % client) logger.debug('Getting hostname where scheduler is running') host = client.run_on_scheduler(socket.gethostname) logger.info('host is %s' % host) logger.info('Starting jupyter lab on host') client.run_on_scheduler(start_jlab, host=host, port=jlab_port, notebook_dir=notebook_dir) logger.debug('Done.') user = os.environ['USER'] print('Run the following command from your local machine:') #print('ssh -N -L {}:{}:{} -L {}:{}:8787 {}@{}'.format(jlab_port, host, jlab_port, dash_port, host, user, hostname)) print('ssh -N -L {}:{}:{} -L {}:{}:8787 {}'.format(jlab_port, host, jlab_port, dash_port, host, hostname)) #Modification for existing ssh key print('Then open the following URLs:') print('\tJupyter lab: http://localhost:{}'.format(jlab_port)) print('\tDask dashboard: http://localhost:{}'.format(dash_port))
def test_register_collections(): try: import dask.bag as db except ImportError: return with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) b = db.from_sequence(range(5), npartitions=2).map(inc) assert not s.collections c.set_collection('mybag', b) assert 'mybag' in s.collections d = Client(s.address_to_clients) b2 = d.get_collection('mybag') assert (type(b) == type(b2) and b.npartitions == b2.npartitions) assert list(b) == list(b2) c.close() d.close()
def test_multiple_clients(): with scheduler_and_workers(1) as (s, (a,)): c = Client(s.address_to_clients) d = Client(s.address_to_clients) assert c.get({'x': (inc, 1)}, 'x') == d.get({'x': (inc, 1)}, 'x') pool = ThreadPool(2) future1 = pool.apply_async(c.get, args=({'x': 1, 'y': (inc, 'x')}, 'y')) future2 = pool.apply_async(d.get, args=({'a': 1, 'b': (inc, 'a')}, 'b')) while not (future1.ready() and future2.ready()): sleep(1e-6) assert future1.get() == future2.get() c.close() d.close()
def test_multiple_clients(): with scheduler_and_workers(1) as (s, (a,)): c = Client(s.address_to_clients) d = Client(s.address_to_clients) assert c.get({'x': (inc, 1)}, 'x') == d.get({'x': (inc, 1)}, 'x') def sleep_inc(x): sleep(0.5) return x + 1 pool = ThreadPool(2) future1 = pool.apply_async(c.get, args=({'x': 1, 'y': (sleep_inc, 'x')}, 'y')) future2 = pool.apply_async(d.get, args=({'a': 1, 'b': (sleep_inc, 'a')}, 'b')) assert future1.get() == future2.get() c.close() d.close()
def _merge_feature_and_label(self, feature_result_df, label_result_df, is_train=True): # 获取 x_train, x_test, y_train, y_test feature_result_df[self.default_time_col_name] = pd.to_datetime( feature_result_df[self.default_time_col_name]) label_result_df[self.default_time_col_name] = pd.to_datetime( label_result_df[self.default_time_col_name]) cutoff_time_column = self.default_time_col_name label_column = self.problem_type self.customer_entity_index = list( set(label_result_df.columns) - set([cutoff_time_column, label_column]))[0] try: train_data_df = pd.merge( feature_result_df, label_result_df, on=[self.default_time_col_name, self.customer_entity_index]) except Exception as ex: with LocalCluster( processes=True, threads_per_worker=1, memory_limit='20GB', # ip='tcp://localhost:9895', ) as cluster, Client(cluster) as client: client.cluster.scale(int( 0.8 * mp.cpu_count())) # ask for ten 4-thread workers # MERGE IN DASK logging.warning( '=============dd.merge(feature_result_df_dd, label_result_df_dd) begin' ) feature_result_df_dd = dd.from_pandas(feature_result_df, npartitions=10) label_result_df_dd = dd.from_pandas(label_result_df, npartitions=2) train_data_df_dd = dd.merge(feature_result_df_dd, label_result_df_dd) train_data_df = train_data_df_dd.compute() logging.warning( '=============feature_result_df:{}, label_result_df:{}, train_data_df:{}' .format(feature_result_df.shape, label_result_df.shape, train_data_df.shape)) id_df = train_data_df[self.customer_entity_index] cutoff_time_df = train_data_df[cutoff_time_column] label_df = train_data_df[[ label_column, self.customer_entity_index, cutoff_time_column ]] train_data_df.drop(columns=[label_column], inplace=True) feature_df = train_data_df # .drop(columns=[self.customer_entity_index, cutoff_time_column]) feature_df.set_index([self.customer_entity_index, cutoff_time_column], inplace=True) label_df.set_index([self.customer_entity_index, cutoff_time_column], inplace=True) feature_df = self._deal_feature_for_data_type(feature_df) return feature_df, label_df, id_df, cutoff_time_df
def test_dask_array(self, local_cuda_cluster): with Client(local_cuda_cluster) as client: run_with_dask_array(dxgb.DaskDMatrix, client) run_with_dask_array(dxgb.DaskDeviceQuantileDMatrix, client)
shared_scratch_root.mkdir(parents=True, exist_ok=True) assert shared_scratch_root.is_dir() local_scratch_root = Path(config.cluster.local_scratch) local_scratch_root.mkdir(parents=True, exist_ok=True) assert local_scratch_root.is_dir() # Connect if config.cluster.run_locally: print("Running on local machine!") # Changes to dpg allow for a "none" dask client dask_client = None else: cluster_address = f"{config.cluster.address}:{config.cluster.port}" print("Configuring Dask, attaching to cluster") print(f"\t- {cluster_address}") dask_client = Client(address=cluster_address) if config.cluster.restart: print("\t- Restarting cluster...") dask_client.restart() print(f"\t- Running on {len(dask_client.nthreads())} machines.") # Prepping all scratch dirs ### def scratch(task_name): "Creates a local / global scratch dir with the give name" return file_util.prep_scratches( local_scratch_root=local_scratch_root, shared_scratch_root=shared_scratch_root, task_name=task_name, ) print("Prepping scratch directories")
# Merge ds_out = xr.concat([da_panE, da_RegE], dim='nregions') ds_out.name = 'Extent' ds_out.load( ) # This prevents many errors in the dask graph (I don't know why) # # Save regridded to netcdf file ds_out.to_netcdf(f_out) ds_out = None # Memory clean up da_panE = None da_RegE = None ds = None print('Saved ', f_out) print("Finished...") # In[4]: if __name__ == '__main__': # Start up Client client = Client(n_workers=8) print(client) # Call function Update_Model_Aggs() # Close it down client.close()
if __name__ == '__main__': np.warnings.filterwarnings('ignore') use_dask = False if use_dask is False: main() else: # https://docs.dask.org/en/latest/diagnostics-distributed.html # https://docs.dask.org/en/latest/setup/single-distributed.html from dask.distributed import Client os.environ['NUMEXPR_MAX_THREADS'] = '8' dask.config.set(scheduler='processes') # dask.config.set({'array.slicing.split_large_chunks': True}) with Client( ) as client: # (n_workers=4, threads_per_worker=4, processes=True, memory_limit='15GB') as client: status = client.scheduler_info()['services'] assert client.status == "running" logging.info("[CMIP6_light] client {}".format(client)) logging.info( "Dask started with status at: http://localhost:{}/status". format(status["dashboard"])) main() client.close() assert client.status == "closed" logging.info("[CMIP6_light] Execution of downscaling completed")
def test_status(): with scheduler_and_workers() as (s, (a, b)): c = Client(s.address_to_clients) assert c.scheduler_status() == 'OK' c.close()
def test_score(nrows, ncols, nclusters, n_parts, input_type, cluster): client = None try: client = Client(cluster) from cuml.dask.cluster import KMeans as cumlKMeans from cuml.dask.datasets import make_blobs X, y = make_blobs(n_samples=int(nrows), n_features=ncols, centers=nclusters, n_parts=n_parts, cluster_std=0.01, shuffle=False, random_state=10) wait(X) if input_type == "dataframe": X_train = to_dask_cudf(X) y_train = to_dask_cudf(y) y = y_train elif input_type == "array": X_train, y_train = X, y cumlModel = cumlKMeans(init="k-means||", n_clusters=nclusters, random_state=10) cumlModel.fit(X_train) actual_score = cumlModel.score(X_train) predictions = cumlModel.predict(X_train).compute() if input_type == "dataframe": X = cp.array(X_train.compute().as_gpu_matrix()) predictions = cp.array(predictions) centers = cp.array(cumlModel.cluster_centers_.as_gpu_matrix()) elif input_type == "array": X = X_train.compute() centers = cumlModel.cluster_centers_ expected_score = 0 for idx, label in enumerate(predictions): x = X[idx] y = centers[label] dist = cp.sqrt(cp.sum((x - y)**2)) expected_score += dist**2 assert actual_score + SCORE_EPS \ >= (-1 * expected_score) \ >= actual_score - SCORE_EPS finally: client.close()
def test_auc(self, local_cuda_cluster: LocalCUDACluster) -> None: with Client(local_cuda_cluster) as client: run_auc(client, "gpu_hist")
def test_empty_dmatrix(self, local_cuda_cluster): with Client(local_cuda_cluster) as client: parameters = {'tree_method': 'gpu_hist', 'debug_synchronize': True} run_empty_dmatrix_reg(client, parameters) run_empty_dmatrix_cls(client, parameters)
X = da.random.random((batch_size, 3 ** 2 * input_channels, input_size ** 2)) kernels_mean = da.random.random((total_kernels, 3 ** 2 * input_channels)) cov_list = [random_cov_DASK(3 ** 2 * input_channels) for number in range(total_kernels)] kernels_cov = da.stack(cov_list) # validate = True validate = False if validate: numpy_validation_list = va.single_mean_single_covariance_validator(X.compute(), kernels_mean.compute(), kernels_cov.compute(), batch_size, total_kernels, input_size) times = [] with Client() as client: for n in range(5): client.restart() # resets cluster # Do something using 'client' start = time.time() batches = [] for i in range(batch_size): kernel_out = [] for j in range(total_kernels): mean = da.matmul(kernels_mean[j, :], X[i, :, :]) cov = da.matmul(da.transpose(X[i, :, :]), da.matmul(kernels_cov[j, :, :], X[i, :, :])) z = mvn_random_DASK(mean, cov, total_samples, input_size ** 2) g = relu(z) mean_g = da.mean(g, axis=1) kernel_out.append(mean_g.compute())
class dask_controller: #adapted from Charles' code def __init__(self,n_workers=6,local=True,queue="short",\ walltime='01:30:00',cores=1,processes=1,memory='6GB',job_extra=[]): self.local = local self.n_workers = n_workers self.walltime = walltime self.queue = queue self.processes = processes self.memory = memory self.cores = cores self.job_extra = job_extra def writedir(self, directory): if not os.path.exists(directory): os.makedirs(directory) def startdask(self): if self.local: self.daskclient = Client() self.daskclient.cluster.scale(self.n_workers) else: self.daskcluster = SLURMCluster(queue=self.queue,walltime=self.walltime,\ processes=self.processes,memory=self.memory, cores=self.cores,job_extra=self.job_extra) self.workers = self.daskcluster.start_workers(self.n_workers) self.daskclient = Client(self.daskcluster) def shutdown(self): self.daskcluster.stop_all_jobs() for item in os.listdir("./"): if "worker-" in item or "slurm-" in item or ".lock" in item: path = "./" + item if os.path.isfile(path): os.remove(path) elif os.path.isdir(path): shutil.rmtree(path) def printprogress(self): complete = len( [item for item in self.futures if item.status == "finished"]) print(str(complete) + "/" + str(len(self.futures))) def mapfovs(self, function, fov_list, retries=0): self.function = function self.retries = retries def mapallfovs(fov_number, function=function): function(fov_number) self.futures = {} for fov in fov_list: future = self.daskclient.submit(mapallfovs, fov, retries=retries) self.futures[fov] = future def retry_failed(self): self.failed_fovs = [ fov for fov, future in self.futures.items() if future.status != 'finished' ] self.daskclient.restart() time.sleep(5) self.mapfovs(self.function, self.failed_fovs, retries=self.retries) def retry_processing(self): self.proc_fovs = [ fov for fov, future in self.futures.items() if future.status == 'pending' ] self.daskclient.restart() time.sleep(5) self.mapfovs(self.function, self.proc_fovs, retries=self.retries)
from dask.distributed import Client, LocalCluster, progress import dask import dask.dataframe as dd import dask.array as da if __name__ == '__main__': start=time.time() sink=sys.argv[1] path_sink=sys.argv[2] path_missing_kmers=sys.argv[3] mem=sys.argv[4] t=sys.argv[5] cluster = LocalCluster(memory_limit=mem,n_workers=int(t)) client = Client(cluster) client = Client() print("Client was set:") print(client) #Load metadata from sources metadata=pd.read_csv("metadata.csv") #Load run accession codes for sources from k-mer matrix .fof files colnames=pd.read_csv("kmtricks.fof", sep=" : ",\ header=None,names=["Run_accession","to_drop"],engine="python") colnames.drop(columns="to_drop",inplace=True) #Parse sources and classes sources=list(colnames["Run_accession"])
def init(self): ### initializing required classes self._execute_notebook_class = ExecuteNotebookWriter(self) self._make_site_class = MakeSiteWriter(self) self.executedir = self.outdir + '/executed' self.reportdir = self.outdir + '/reports/' self.errordir = self.outdir + "/reports/{}" self.downloadsdir = self.outdir + "/_downloads" self.downloadsExecutedir = self.downloadsdir + "/executed" self.client = None self.execution_status_code = 0 # Check default language is defined in the jupyter kernels def_lng = self.config["jupyter_default_lang"] if def_lng not in self.config["jupyter_kernels"]: self.logger.warning( "Default language defined in conf.py ({}) is not " "defined in the jupyter_kernels in conf.py. " "Set default language to python3".format(def_lng)) self.config["jupyter_default_lang"] = "python3" # If the user has overridden anything on the command line, set these things which have been overridden. instructions = [] overrides = self.config['jupyter_options'] if overrides: instructions = overrides.split(",") for instruction in instructions: if instruction: if instruction == 'code_only': self.config["jupyter_conversion_mode"] = "code" else: # Fail on unrecognised command. self.logger.warning("Unrecognise command line parameter " + instruction + ", ignoring.") #threads per worker for dask distributed processing if "jupyter_threads_per_worker" in self.config: self.threads_per_worker = self.config["jupyter_threads_per_worker"] #number of workers for dask distributed processing if "jupyter_number_workers" in self.config: self.n_workers = self.config["jupyter_number_workers"] # start a dask client to process the notebooks efficiently. # processes = False. This is sometimes preferable if you want to avoid inter-worker communication and your computations release the GIL. This is common when primarily using NumPy or Dask Array. if (self.config["jupyter_execute_notebooks"]): self.client = Client(processes=False, threads_per_worker=self.threads_per_worker, n_workers=self.n_workers) self.execution_vars = { 'target': 'website', 'dependency_lists': self.config["jupyter_dependency_lists"], 'executed_notebooks': [], 'delayed_notebooks': dict(), 'futures': [], 'delayed_futures': [], 'destination': self.executedir } if (self.config["jupyter_download_nb_execute"]): if self.client is None: self.client = Client( processes=False, threads_per_worker=self.threads_per_worker, n_workers=self.n_workers) self.download_execution_vars = { 'target': 'downloads', 'dependency_lists': self.config["jupyter_dependency_lists"], 'executed_notebooks': [], 'delayed_notebooks': dict(), 'futures': [], 'delayed_futures': [], 'destination': self.downloadsExecutedir }
def test_end_to_end(nrows, ncols, nclusters, n_parts, delayed_predict, input_type, cluster): client = None try: client = Client(cluster) from cuml.dask.cluster import KMeans as cumlKMeans from cuml.dask.datasets import make_blobs X, y = make_blobs(n_samples=int(nrows), n_features=ncols, centers=nclusters, n_parts=n_parts, cluster_std=0.01, random_state=10) wait(X) if input_type == "dataframe": X_train = to_dask_cudf(X) y_train = to_dask_cudf(y) elif input_type == "array": X_train, y_train = X, y cumlModel = cumlKMeans(init="k-means||", n_clusters=nclusters, random_state=10) cumlModel.fit(X_train) cumlLabels = cumlModel.predict(X_train, delayed_predict) n_workers = len(list(client.has_what().keys())) # Verifying we are grouping partitions. This should be changed soon. if n_parts is not None and n_parts < n_workers: parts_len = n_parts else: parts_len = n_workers if input_type == "dataframe": assert cumlLabels.npartitions == parts_len cumlPred = cp.array(cumlLabels.compute().to_pandas().values) labels = cp.squeeze(y_train.compute().to_pandas().values) elif input_type == "array": assert len(cumlLabels.chunks[0]) == parts_len cumlPred = cp.array(cumlLabels.compute()) labels = cp.squeeze(y_train.compute()) assert cumlPred.shape[0] == nrows assert cp.max(cumlPred) == nclusters - 1 assert cp.min(cumlPred) == 0 score = adjusted_rand_score(labels, cumlPred) print(str(score)) assert 1.0 == score finally: client.close()
def test_with_asyncio(local_cuda_cluster): with Client(local_cuda_cluster) as client: address = client.scheduler.address output = asyncio.run(run_from_dask_array_asyncio(address)) assert isinstance(output['booster'], xgboost.Booster) assert isinstance(output['history'], dict)
class MGContext: """Utility Context Manager to start a multi GPU context using dask_cuda Parameters: ----------- number_of_devices : int Number of devices to use, verification must be done prior to call to ensure that there are enough devices available. If not specified, the cluster will be initialized to use all visible devices. rmm_managed_memory : bool True to enable managed memory (UVM) in RMM as part of the cluster. Default is False. p2p : bool Initialize UCX endpoints if True. Default is False. """ def __init__(self, number_of_devices=None, rmm_managed_memory=False, p2p=False): self._number_of_devices = number_of_devices self._rmm_managed_memory = rmm_managed_memory self._client = None self._p2p = p2p self._cluster = CUDACluster( n_workers=self._number_of_devices, rmm_managed_memory=self._rmm_managed_memory ) @property def client(self): return self._client @property def cluster(self): return self._cluster def __enter__(self): self._prepare_mg() return self def _prepare_mg(self): self._prepare_client() self._prepare_comms() def _prepare_client(self): self._client = Client(self._cluster) self._client.wait_for_workers(self._number_of_devices) def _prepare_comms(self): Comms.initialize(p2p=self._p2p) def _close(self): Comms.destroy() if self._client is not None: self._client.close() if self._cluster is not None: self._cluster.close() def __exit__(self, type, value, traceback): self._close()
def _prepare_client(self): self._client = Client(self._cluster) self._client.wait_for_workers(self._number_of_devices)
from dask.distributed import Client import socket client = Client(scheduler_file='scheduler.json') print(client) host = client.run_on_scheduler(socket.gethostname) def start_jlab(dask_scheduler): import subprocess proc = subprocess.Popen(['jupyter', 'notebook', '--ip', host]) dask_scheduler.jlab_proc = proc client.run_on_scheduler(start_jlab) print("HOST : %s" % host)
import dask import dask.threaded import dask.multiprocessing from dask.distributed import Client c = Client() import xarray as xr import numpy as np import glob import time import xscale data_dir = '/store/CT1/hmg2840/lbrodeau/eNATL60/eNATL60-BLBT02-S/' def compute_buoy(t,s): rau0 = 1000 grav = 9.81 buoy= -1*(grav/rau0)*sigma0(t,s) return buoy def sigma0(t,s): zrau0=1000 zsr=np.sqrt(np.abs(s)) zs=s zt=t zr1 = ( ( ( ( 6.536332e-9*zt-1.120083e-6 )*zt+1.001685e-4)*zt - 9.095290e-3 )*zt+6.793952e-2 )*zt+999.842594 zr2= ( ( ( 5.3875e-9*zt-8.2467e-7 )*zt+7.6438e-5 ) *zt - 4.0899e-3 ) *zt+0.824493 zr3= ( -1.6546e-6*zt+1.0227e-4 ) *zt-5.72466e-3
def test_empty_dmatrix_auc(self, local_cuda_cluster: LocalCUDACluster) -> None: with Client(local_cuda_cluster) as client: n_workers = len(_get_client_workers(client)) run_empty_dmatrix_auc(client, "gpu_hist", n_workers)
args = parser.parse_args() # FIXED parameters noise_dim = pd.Index(range(500), name='noise_field') mask = xr.open_dataarray('/scratch/pkittiwi/fg1p/hera331_fov_mask.nc') if args.xyf_chunks is not None: chunks = {'x': args.xyf_chunks[0], 'y': args.xyf_chunks[1], 'f': args.xyf_chunks[2]} else: chunks = None # Setup and start Dask Local Cluster cluster = LocalCluster(n_workers=args.n_workers, processes=args.processes, threads_per_worker=args.threads_per_worker, scheduler_port=args.scheduler_port, diagnostics_port=args.diagnostics_port) client = Client(cluster) print('Hostname: {:s}'.format(os.environ['HOSTNAME'])) print('Dask Scheduler address: {:s}'.format(cluster.scheduler_address)) print('Dask Dashboard link: {:s}'.format(cluster.dashboard_link)) # Loop over data parameters and perform calculation for bw, fbw, t, s in itertools.product( args.bin_width, args.filter_bandwidth, args.theta, args.shift ): start_time = datetime.now() ds = xr.open_mfdataset( ['/scratch/pkittiwi/fg1p/binned_noise_map/bin{:.2f}MHz/' 'fbw{:.2f}MHz/theta{:.1f}/shift{:d}/binned_noise_map_bin{:.2f}MHz_' 'fbw{:.2f}MHz_theta{:.1f}_shift{:d}_{:03d}.nc' .format(bw, fbw, t, s, bw, fbw, t, s, i) for i in range(500)], concat_dim=noise_dim, chunks=chunks
def test_dask_dataframe(self, local_cuda_cluster: LocalCUDACluster) -> None: with Client(local_cuda_cluster) as client: run_with_dask_dataframe(dxgb.DaskDMatrix, client) run_with_dask_dataframe(dxgb.DaskDeviceQuantileDMatrix, client)
def find_neighbors(X, n_neighbors, metric, algorithm, distributed): """ Returns the indices of the k-nearest neighbors for each cell in a cell-by-feature dataframe. Parameters ---------- X : cudf.DataFrame Input cell-by-feature dataframe. n_neighbors : int Number of neighbors for kNN. metric: string Distance metric to use for kNN. Currently, only 'euclidean' is supported. algorithm: string The query algorithm to use. Currently, only 'brute' is supported. distributed: bool If True, use a multi-GPU dask cluster for kNN search. Returns ------- idx : cudf.DataFrame The indices of the kNN for each cell in X. """ print( f"Finding {n_neighbors} nearest neighbors using " f"{metric} metric and {algorithm} algorithm...", flush=True, ) if distributed: print("Running distributed kNN...") import dask_cudf from dask_cuda import LocalCUDACluster from dask.distributed import Client from cuml.dask.neighbors import NearestNeighbors cluster = LocalCUDACluster() client = Client(cluster) npartitions = len(cluster.cuda_visible_devices) X_dask = dask_cudf.from_cudf(X, npartitions=npartitions) # Use n_neighbors + 1 to account for self index model = cuml.dask.neighbors.NearestNeighbors(n_neighbors=n_neighbors + 1, client=client) model.fit(X_dask) idx = model.kneighbors(X_dask, return_distance=False).compute() client.shutdown() else: # Use n_neighbors + 1 to account for self index model = cuml.neighbors.NearestNeighbors(n_neighbors=n_neighbors + 1, metric=metric, algorithm=algorithm) model.fit(X) idx = model.kneighbors(X, return_distance=False) # Drop self index # this replace original grephano implementation that just remove the first index,assuming it is self index # which is not always the case! idx = np.vstack( list(map(lambda i: idx[i, idx[i, :] != i], range(idx.shape[0])))) return idx
def test_aamp_int_input(dask_cluster): with pytest.raises(TypeError): with Client(dask_cluster) as dask_client: aamped(dask_client, np.arange(10), 5)
from dask_jobqueue import SLURMCluster from dask.distributed import Client import time from dask.distributed import progress cluster = SLURMCluster(queue='par-multi', cores=10, memory='1000', job_mem='2000') def slow_increment(x): time.sleep(1) x = x + 1 return x cluster.scale(10) client = Client(cluster) futures = client.map(slow_increment, range(1000)) progress(futures)
import time import numpy as np import dask import sys import shutil import os PDB = "/oasis/scratch/comet/sfan19/temp_project/step5_znm.pdb" XTC = "/oasis/scratch/comet/sfan19/temp_project/step7_90ns_center.xtc" from dask.distributed import Client Scheduler_IP = sys.argv[1] print(Scheduler_IP) print(type(Scheduler_IP)) c = Client(Scheduler_IP) u = mda.Universe(PDB, XTC) g1 = u.select_atoms('name OH2') g2 = u.select_atoms('name OH2') rdfs = rdf.InterRDF(g1, g2, range=(0, 5)) t_time = [] t_io = [] t_compute = [] t_u = [] t_w = [] ns = [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72] for n in ns: rdfs.run(n_jobs=n, n_blocks=n) t_time.append([ n, rdfs.timing.total,
def test_gpu_hist(self, params, num_rounds, dataset, local_cuda_cluster): with Client(local_cuda_cluster) as client: run_gpu_hist(params, num_rounds, dataset, dxgb.DaskDMatrix, client) run_gpu_hist(params, num_rounds, dataset, dxgb.DaskDeviceQuantileDMatrix, client)
def main(args): print('\n>> CLI Parameters ...\n') print args if not os.path.isfile(args.inputImageFile): raise IOError('Input image file does not exist.') if len(args.reference_mu_lab) != 3: raise ValueError('Reference Mean LAB should be a 3 element vector.') if len(args.reference_std_lab) != 3: raise ValueError('Reference Stddev LAB should be a 3 element vector.') if len(args.analysis_roi) != 4: raise ValueError('Analysis ROI must be a vector of 4 elements.') # # Initiate Dask client # print('\n>> Creating Dask client ...\n') scheduler_address = json.loads(args.scheduler_address) if scheduler_address is None: scheduler_address = LocalCluster( n_workers=multiprocessing.cpu_count() - 1, scheduler_port=0, silence_logs=False) c = Client(scheduler_address) print c # # Read Input Image # print('\n>> Reading input image ... \n') ts = large_image.getTileSource(args.inputImageFile) ts_metadata = ts.getMetadata() print json.dumps(ts_metadata, indent=2) is_wsi = ts_metadata['magnification'] is not None # # Compute tissue/foreground mask at low-res for whole slide images # if is_wsi: print('\n>> Computing tissue/foreground mask at low-res ...\n') # get image at low-res maxSize = max(ts_metadata['sizeX'], ts_metadata['sizeY']) downsample_factor = 2**np.floor(np.log2(maxSize / 2048)) fgnd_seg_mag = ts_metadata['magnification'] / downsample_factor fgnd_seg_scale = {'magnification': fgnd_seg_mag} im_lres, _ = ts.getRegion( scale=fgnd_seg_scale, format=large_image.tilesource.TILE_FORMAT_NUMPY) im_lres = im_lres[:, :, :3] # compute foreground mask at low-res im_fgnd_mask_lres = htk_utils.simple_mask(im_lres) # # Detect nuclei in paralle using Dask # print('\n>> Detecting nuclei in parallel using Dask ...\n') it_kwargs = { 'format': large_image.tilesource.TILE_FORMAT_NUMPY, 'tile_size': { 'width': args.analysis_tile_size }, 'scale': { 'magnification': args.analysis_mag }, } if np.all(np.array(args.analysis_roi) == -1): process_whole_image = True else: process_whole_image = False if not process_whole_image: it_kwargs['region'] = { 'left': args.analysis_roi[0], 'top': args.analysis_roi[1], 'width': args.analysis_roi[2], 'height': args.analysis_roi[3], 'units': 'base_pixels' } tile_nuclei_list = [] for tile in ts.tileIterator(**it_kwargs): if is_wsi: # get current region in base_pixels rgn_hres = { 'left': tile['gx'], 'top': tile['gy'], 'right': tile['gx'] + tile['gwidth'], 'bottom': tile['gy'] + tile['gheight'], 'units': 'base_pixels' } # get foreground mask for current tile at low resolution rgn_lres = ts.convertRegionScale(rgn_hres, targetScale=fgnd_seg_scale, targetUnits='mag_pixels') top = np.int(rgn_lres['top']) bottom = np.int(rgn_lres['bottom']) left = np.int(rgn_lres['left']) right = np.int(rgn_lres['right']) im_tile_fgnd_mask_lres = im_fgnd_mask_lres[top:bottom, left:right] # skip tile if there is not enough foreground in the slide cur_fgnd_frac = im_tile_fgnd_mask_lres.mean() if np.isnan(cur_fgnd_frac) or cur_fgnd_frac <= args.min_fgnd_frac: continue # detect nuclei cur_nuclei_list = dask.delayed(detect_tile_nuclei)( args.inputImageFile, tile['tile_position']['position'], args, **it_kwargs) # append result to list tile_nuclei_list.append(cur_nuclei_list) def collect(x): return x tile_nuclei_list = dask.delayed(collect)(tile_nuclei_list).compute() nuclei_list = list(itertools.chain.from_iterable(tile_nuclei_list)) print 'Number of nuclei = ', len(nuclei_list) # # Write annotation file # print('\n>> Writing annotation file ...\n') annot_fname = os.path.splitext( os.path.basename(args.outputNucleiAnnotationFile))[0] annotation = {"name": annot_fname, "elements": nuclei_list} with open(args.outputNucleiAnnotationFile, 'w') as annotation_file: json.dump(annotation, annotation_file, indent=2, sort_keys=False)
}) inputs = targets train( inputs, targets, model=autoencoder, data=data_handler, optimizer=optimizer, regularization=regularization, epochs=epochs, convergence=convergence, lossfxn=None, device="cpu", ) latent_space = autoencoder.get_latent_space(targets, svm=True) dump(latent_space, filename="cu_training.latent") Potentials.save(autoencoder) return latent_space, energy_targets, data_handler if __name__ == "__main__": logger(filename="cu_training.log") cluster = LocalCluster() client = Client(cluster) inputs, outputs, data_handler = autoencode()
def run_dask_function(config): """Start a Dask Cluster using dask-kubernetes and run a function. Talks to kubernetes to create `n` amount of new `pods` with a dask worker inside of each forming a `dask` cluster. Then, a function specified from `config` is being imported and run with the given arguments. The tasks created by this `function` are being run on the `dask` cluster for distributed computation. The config dict must contain the following sections: * run * dask_cluster * output Args: config (dict): Config dictionary. """ output_conf = config.get('output') if output_conf: path = output_conf.get('path') if not path: raise ValueError('An output path must be provided when providing `output`.') cluster_spec = _generate_cluster_spec(config, master=False) # Importing here to avoid an aiohttp error if not used. from dask_kubernetes import KubeCluster # pylint: disable=C0415 cluster = KubeCluster.from_dict(cluster_spec) workers = config['dask_cluster'].get('workers') if not workers: cluster.adapt() elif isinstance(workers, int): cluster.scale(workers) else: cluster.adapt(**workers) client = Client(cluster) client.get_versions(check=True) client.register_worker_callbacks(_logging_setup) try: run = _import_function(config['run']) kwargs = config['run']['args'] results = run(**kwargs) finally: client.close() cluster.close() if output_conf: bucket = output_conf.get('bucket') try: if bucket: aws_key = output_conf.get('key') aws_secret = output_conf.get('secret_key') _upload_to_s3(bucket, path, results, aws_key, aws_secret) else: dirname = os.path.dirname(path) if dirname: os.makedirs(dirname, exist_ok=True) results.to_csv(path) except Exception: # pylint: disable=W0703 print('Error storing results. Falling back to console dump.') print(_dataframe_to_csv_str(results)) return None return results
# 미리 설정된 기반 스케줄러 : 워커 관리 # 워커는 스레딩함(ThreadPoolExecutor 사용) from dask.distributed import Client def square(x): return x**2 if __name__ == "__main__": client = Client() # 설정에따라 다른 컴퓨터에 있는 워커를 사용가능. # client를 이용하여, local computer의 병렬코드를 그대로 여러 머신을 사용한 분산처리에 이용가능 print( client ) #<Client: 'tcp://127.0.0.1:#' processes=4 threads=8, memory=8.44 GB> fut = client.submit(square, 2) print( fut) #<Future: pending, key: square-c94e3bc4bf0b4c9a5933f7b43187f6b0> print(fut.result()) #4 futs = client.map(square, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) print(futs[0:3]) # [<Future: pending, key: square-72f30fc48e1c42ee95576d0d3088f8db>, # <Future: pending, key: square-89e339991c492cf890f48a2ecb386f31>, # <Future: finished, type: builtins.int, key: square-c94e3bc4bf0b4c9a5933f7b43187f6b0>] # 2에대한 결과는 위로부터 캐시되어있다. futs_res = client.gather(futs)