def test_keep_results():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
assert c.get({'x': (inc, 1)}, 'x', keep_results=True) == 2
assert 'x' in a.data or 'x' in b.data
def test_error():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
assert raises(TypeError,
lambda: c.get({'x': 1, 'y': (inc, 'x', 'x')}, 'y'))
assert 'y' not in s.data
c.close()
def main():
client = Client('localhost:8786')
A = client.map(set_value, range(100))
B = client.map(square, A)
C = client.map(neg, B)
total = client.submit(sum, C)
print(progress(total))
print(total.result())
def test_get():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
dsk = {'x': 1, 'y': (add, 'x', 'x'), 'z': (inc, 'y')}
keys = ['y', 'z']
assert c.get(dsk, keys) == [2, 3]
c.close()
def test_get_with_dill():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
dsk = {'x': 1, 'y': (partial(add, 1), 'x')}
keys = 'y'
assert c.get(dsk, keys) == 2
c.close()
def test_get_workers():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
pid = os.getpid()
workers = c.get_registered_workers()
assert set(workers) == set([a.address, b.address])
assert workers[a.address]['pid'] == pid
assert workers[b.address]['pid'] == pid
s.close_workers()
assert c.get_registered_workers() == {}
def main():
# Setup logging on the main process:
_start_logging()
# Start three worker processes on the local machine:
client = Client(n_workers=3, threads_per_worker=1)
# Setup Eliot logging on each worker process:
client.run(_start_logging)
# Run the Dask computation in the worker processes:
result = main_computation()
print("Result:", result)
def main():
"""."""
host = os.getenv('DASK_SCHEDULER_HOST', default='localhost')
port = os.getenv('DASK_SCHEDULER_PORT', default=8786)
print(host, port)
client = Client('{}:{}'.format(host, port))
# client.run(init_logging)
# client.run_on_scheduler(init_logging)
# Run some mock functions and gather a result
data = client.map(print_listdir, range(10))
future = client.submit(print_values, data)
progress(future)
print('')
result = client.gather(future)
print(result)
# Run a second stage which runs some additional processing.
print('here A')
data_a = client.map(set_value, range(100))
print('here B')
data_b = client.map(square, data_a)
print('here C')
data_c = client.map(neg, data_b)
print('here D')
# Submit a function application to the scheduler
total = client.submit(sum, data_c)
print('here E')
progress(total)
print(total.result())
print('here F')
def cli(scheduler_file, jlab_port, dash_port, notebook_dir,
hostname, log_level):
logger = get_logger(log_level)
logger.info('getting client with scheduler file: %s' % scheduler_file)
client = Client(scheduler_file=scheduler_file, timeout=30)
logger.debug('Client: %s' % client)
logger.debug('Getting hostname where scheduler is running')
host = client.run_on_scheduler(socket.gethostname)
logger.info('host is %s' % host)
logger.info('Starting jupyter lab on host')
client.run_on_scheduler(start_jlab, host=host, port=jlab_port,
notebook_dir=notebook_dir)
logger.debug('Done.')
user = os.environ['USER']
print('Run the following command from your local machine:')
#print('ssh -N -L {}:{}:{} -L {}:{}:8787 {}@{}'.format(jlab_port, host, jlab_port, dash_port, host, user, hostname))
print('ssh -N -L {}:{}:{} -L {}:{}:8787 {}'.format(jlab_port, host, jlab_port, dash_port, host, hostname)) #Modification for existing ssh key
print('Then open the following URLs:')
print('\tJupyter lab: http://localhost:{}'.format(jlab_port))
print('\tDask dashboard: http://localhost:{}'.format(dash_port))
def test_register_collections():
try:
import dask.bag as db
except ImportError:
return
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
b = db.from_sequence(range(5), npartitions=2).map(inc)
assert not s.collections
c.set_collection('mybag', b)
assert 'mybag' in s.collections
d = Client(s.address_to_clients)
b2 = d.get_collection('mybag')
assert (type(b) == type(b2) and
b.npartitions == b2.npartitions)
assert list(b) == list(b2)
c.close()
d.close()
def test_multiple_clients():
with scheduler_and_workers(1) as (s, (a,)):
c = Client(s.address_to_clients)
d = Client(s.address_to_clients)
assert c.get({'x': (inc, 1)}, 'x') == d.get({'x': (inc, 1)}, 'x')
pool = ThreadPool(2)
future1 = pool.apply_async(c.get,
args=({'x': 1, 'y': (inc, 'x')}, 'y'))
future2 = pool.apply_async(d.get,
args=({'a': 1, 'b': (inc, 'a')}, 'b'))
while not (future1.ready() and future2.ready()):
sleep(1e-6)
assert future1.get() == future2.get()
c.close()
d.close()
def test_multiple_clients():
with scheduler_and_workers(1) as (s, (a,)):
c = Client(s.address_to_clients)
d = Client(s.address_to_clients)
assert c.get({'x': (inc, 1)}, 'x') == d.get({'x': (inc, 1)}, 'x')
def sleep_inc(x):
sleep(0.5)
return x + 1
pool = ThreadPool(2)
future1 = pool.apply_async(c.get,
args=({'x': 1, 'y': (sleep_inc, 'x')}, 'y'))
future2 = pool.apply_async(d.get,
args=({'a': 1, 'b': (sleep_inc, 'a')}, 'b'))
assert future1.get() == future2.get()
c.close()
d.close()
def _merge_feature_and_label(self,
feature_result_df,
label_result_df,
is_train=True):
# 获取 x_train, x_test, y_train, y_test
feature_result_df[self.default_time_col_name] = pd.to_datetime(
feature_result_df[self.default_time_col_name])
label_result_df[self.default_time_col_name] = pd.to_datetime(
label_result_df[self.default_time_col_name])
cutoff_time_column = self.default_time_col_name
label_column = self.problem_type
self.customer_entity_index = list(
set(label_result_df.columns) -
set([cutoff_time_column, label_column]))[0]
try:
train_data_df = pd.merge(
feature_result_df,
label_result_df,
on=[self.default_time_col_name, self.customer_entity_index])
except Exception as ex:
with LocalCluster(
processes=True,
threads_per_worker=1,
memory_limit='20GB',
# ip='tcp://localhost:9895',
) as cluster, Client(cluster) as client:
client.cluster.scale(int(
0.8 * mp.cpu_count())) # ask for ten 4-thread workers
# MERGE IN DASK
logging.warning(
'=============dd.merge(feature_result_df_dd, label_result_df_dd) begin'
)
feature_result_df_dd = dd.from_pandas(feature_result_df,
npartitions=10)
label_result_df_dd = dd.from_pandas(label_result_df,
npartitions=2)
train_data_df_dd = dd.merge(feature_result_df_dd,
label_result_df_dd)
train_data_df = train_data_df_dd.compute()
logging.warning(
'=============feature_result_df:{}, label_result_df:{}, train_data_df:{}'
.format(feature_result_df.shape, label_result_df.shape,
train_data_df.shape))
id_df = train_data_df[self.customer_entity_index]
cutoff_time_df = train_data_df[cutoff_time_column]
label_df = train_data_df[[
label_column, self.customer_entity_index, cutoff_time_column
]]
train_data_df.drop(columns=[label_column], inplace=True)
feature_df = train_data_df # .drop(columns=[self.customer_entity_index, cutoff_time_column])
feature_df.set_index([self.customer_entity_index, cutoff_time_column],
inplace=True)
label_df.set_index([self.customer_entity_index, cutoff_time_column],
inplace=True)
feature_df = self._deal_feature_for_data_type(feature_df)
return feature_df, label_df, id_df, cutoff_time_df
def test_dask_array(self, local_cuda_cluster):
with Client(local_cuda_cluster) as client:
run_with_dask_array(dxgb.DaskDMatrix, client)
run_with_dask_array(dxgb.DaskDeviceQuantileDMatrix, client)
shared_scratch_root.mkdir(parents=True, exist_ok=True)
assert shared_scratch_root.is_dir()
local_scratch_root = Path(config.cluster.local_scratch)
local_scratch_root.mkdir(parents=True, exist_ok=True)
assert local_scratch_root.is_dir()
# Connect
if config.cluster.run_locally:
print("Running on local machine!")
# Changes to dpg allow for a "none" dask client
dask_client = None
else:
cluster_address = f"{config.cluster.address}:{config.cluster.port}"
print("Configuring Dask, attaching to cluster")
print(f"\t- {cluster_address}")
dask_client = Client(address=cluster_address)
if config.cluster.restart:
print("\t- Restarting cluster...")
dask_client.restart()
print(f"\t- Running on {len(dask_client.nthreads())} machines.")
# Prepping all scratch dirs ###
def scratch(task_name):
"Creates a local / global scratch dir with the give name"
return file_util.prep_scratches(
local_scratch_root=local_scratch_root,
shared_scratch_root=shared_scratch_root,
task_name=task_name,
)
print("Prepping scratch directories")
# Merge
ds_out = xr.concat([da_panE, da_RegE], dim='nregions')
ds_out.name = 'Extent'
ds_out.load(
) # This prevents many errors in the dask graph (I don't know why)
# # Save regridded to netcdf file
ds_out.to_netcdf(f_out)
ds_out = None # Memory clean up
da_panE = None
da_RegE = None
ds = None
print('Saved ', f_out)
print("Finished...")
# In[4]:
if __name__ == '__main__':
# Start up Client
client = Client(n_workers=8)
print(client)
# Call function
Update_Model_Aggs()
# Close it down
client.close()
if __name__ == '__main__':
np.warnings.filterwarnings('ignore')
use_dask = False
if use_dask is False:
main()
else:
# https://docs.dask.org/en/latest/diagnostics-distributed.html
# https://docs.dask.org/en/latest/setup/single-distributed.html
from dask.distributed import Client
os.environ['NUMEXPR_MAX_THREADS'] = '8'
dask.config.set(scheduler='processes')
# dask.config.set({'array.slicing.split_large_chunks': True})
with Client(
) as client: # (n_workers=4, threads_per_worker=4, processes=True, memory_limit='15GB') as client:
status = client.scheduler_info()['services']
assert client.status == "running"
logging.info("[CMIP6_light] client {}".format(client))
logging.info(
"Dask started with status at: http://localhost:{}/status".
format(status["dashboard"]))
main()
client.close()
assert client.status == "closed"
logging.info("[CMIP6_light] Execution of downscaling completed")
def test_status():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
assert c.scheduler_status() == 'OK'
c.close()
def test_score(nrows, ncols, nclusters, n_parts, input_type, cluster):
client = None
try:
client = Client(cluster)
from cuml.dask.cluster import KMeans as cumlKMeans
from cuml.dask.datasets import make_blobs
X, y = make_blobs(n_samples=int(nrows),
n_features=ncols,
centers=nclusters,
n_parts=n_parts,
cluster_std=0.01,
shuffle=False,
random_state=10)
wait(X)
if input_type == "dataframe":
X_train = to_dask_cudf(X)
y_train = to_dask_cudf(y)
y = y_train
elif input_type == "array":
X_train, y_train = X, y
cumlModel = cumlKMeans(init="k-means||",
n_clusters=nclusters,
random_state=10)
cumlModel.fit(X_train)
actual_score = cumlModel.score(X_train)
predictions = cumlModel.predict(X_train).compute()
if input_type == "dataframe":
X = cp.array(X_train.compute().as_gpu_matrix())
predictions = cp.array(predictions)
centers = cp.array(cumlModel.cluster_centers_.as_gpu_matrix())
elif input_type == "array":
X = X_train.compute()
centers = cumlModel.cluster_centers_
expected_score = 0
for idx, label in enumerate(predictions):
x = X[idx]
y = centers[label]
dist = cp.sqrt(cp.sum((x - y)**2))
expected_score += dist**2
assert actual_score + SCORE_EPS \
>= (-1 * expected_score) \
>= actual_score - SCORE_EPS
finally:
client.close()
def test_auc(self, local_cuda_cluster: LocalCUDACluster) -> None:
with Client(local_cuda_cluster) as client:
run_auc(client, "gpu_hist")
def test_empty_dmatrix(self, local_cuda_cluster):
with Client(local_cuda_cluster) as client:
parameters = {'tree_method': 'gpu_hist', 'debug_synchronize': True}
run_empty_dmatrix_reg(client, parameters)
run_empty_dmatrix_cls(client, parameters)
def test_register_collections():
try:
import dask.bag as db
except ImportError:
return
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
b = db.from_sequence(range(5), npartitions=2).map(inc)
assert not s.collections
c.set_collection('mybag', b)
assert 'mybag' in s.collections
d = Client(s.address_to_clients)
b2 = d.get_collection('mybag')
assert (type(b) == type(b2) and b.npartitions == b2.npartitions)
assert list(b) == list(b2)
c.close()
d.close()
X = da.random.random((batch_size, 3 ** 2 * input_channels, input_size ** 2))
kernels_mean = da.random.random((total_kernels, 3 ** 2 * input_channels))
cov_list = [random_cov_DASK(3 ** 2 * input_channels) for number in range(total_kernels)]
kernels_cov = da.stack(cov_list)
# validate = True
validate = False
if validate:
numpy_validation_list = va.single_mean_single_covariance_validator(X.compute(), kernels_mean.compute(),
kernels_cov.compute(), batch_size,
total_kernels,
input_size)
times = []
with Client() as client:
for n in range(5):
client.restart() # resets cluster
# Do something using 'client'
start = time.time()
batches = []
for i in range(batch_size):
kernel_out = []
for j in range(total_kernels):
mean = da.matmul(kernels_mean[j, :], X[i, :, :])
cov = da.matmul(da.transpose(X[i, :, :]),
da.matmul(kernels_cov[j, :, :], X[i, :, :]))
z = mvn_random_DASK(mean, cov, total_samples, input_size ** 2)
g = relu(z)
mean_g = da.mean(g, axis=1)
kernel_out.append(mean_g.compute())
class dask_controller: #adapted from Charles' code
def __init__(self,n_workers=6,local=True,queue="short",\
walltime='01:30:00',cores=1,processes=1,memory='6GB',job_extra=[]):
self.local = local
self.n_workers = n_workers
self.walltime = walltime
self.queue = queue
self.processes = processes
self.memory = memory
self.cores = cores
self.job_extra = job_extra
def writedir(self, directory):
if not os.path.exists(directory):
os.makedirs(directory)
def startdask(self):
if self.local:
self.daskclient = Client()
self.daskclient.cluster.scale(self.n_workers)
else:
self.daskcluster = SLURMCluster(queue=self.queue,walltime=self.walltime,\
processes=self.processes,memory=self.memory,
cores=self.cores,job_extra=self.job_extra)
self.workers = self.daskcluster.start_workers(self.n_workers)
self.daskclient = Client(self.daskcluster)
def shutdown(self):
self.daskcluster.stop_all_jobs()
for item in os.listdir("./"):
if "worker-" in item or "slurm-" in item or ".lock" in item:
path = "./" + item
if os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
def printprogress(self):
complete = len(
[item for item in self.futures if item.status == "finished"])
print(str(complete) + "/" + str(len(self.futures)))
def mapfovs(self, function, fov_list, retries=0):
self.function = function
self.retries = retries
def mapallfovs(fov_number, function=function):
function(fov_number)
self.futures = {}
for fov in fov_list:
future = self.daskclient.submit(mapallfovs, fov, retries=retries)
self.futures[fov] = future
def retry_failed(self):
self.failed_fovs = [
fov for fov, future in self.futures.items()
if future.status != 'finished'
]
self.daskclient.restart()
time.sleep(5)
self.mapfovs(self.function, self.failed_fovs, retries=self.retries)
def retry_processing(self):
self.proc_fovs = [
fov for fov, future in self.futures.items()
if future.status == 'pending'
]
self.daskclient.restart()
time.sleep(5)
self.mapfovs(self.function, self.proc_fovs, retries=self.retries)
from dask.distributed import Client, LocalCluster, progress
import dask
import dask.dataframe as dd
import dask.array as da
if __name__ == '__main__':
start=time.time()
sink=sys.argv[1]
path_sink=sys.argv[2]
path_missing_kmers=sys.argv[3]
mem=sys.argv[4]
t=sys.argv[5]
cluster = LocalCluster(memory_limit=mem,n_workers=int(t))
client = Client(cluster)
client = Client()
print("Client was set:")
print(client)
#Load metadata from sources
metadata=pd.read_csv("metadata.csv")
#Load run accession codes for sources from k-mer matrix .fof files
colnames=pd.read_csv("kmtricks.fof", sep=" : ",\
header=None,names=["Run_accession","to_drop"],engine="python")
colnames.drop(columns="to_drop",inplace=True)
#Parse sources and classes
sources=list(colnames["Run_accession"])
def init(self):
### initializing required classes
self._execute_notebook_class = ExecuteNotebookWriter(self)
self._make_site_class = MakeSiteWriter(self)
self.executedir = self.outdir + '/executed'
self.reportdir = self.outdir + '/reports/'
self.errordir = self.outdir + "/reports/{}"
self.downloadsdir = self.outdir + "/_downloads"
self.downloadsExecutedir = self.downloadsdir + "/executed"
self.client = None
self.execution_status_code = 0
# Check default language is defined in the jupyter kernels
def_lng = self.config["jupyter_default_lang"]
if def_lng not in self.config["jupyter_kernels"]:
self.logger.warning(
"Default language defined in conf.py ({}) is not "
"defined in the jupyter_kernels in conf.py. "
"Set default language to python3".format(def_lng))
self.config["jupyter_default_lang"] = "python3"
# If the user has overridden anything on the command line, set these things which have been overridden.
instructions = []
overrides = self.config['jupyter_options']
if overrides:
instructions = overrides.split(",")
for instruction in instructions:
if instruction:
if instruction == 'code_only':
self.config["jupyter_conversion_mode"] = "code"
else:
# Fail on unrecognised command.
self.logger.warning("Unrecognise command line parameter " +
instruction + ", ignoring.")
#threads per worker for dask distributed processing
if "jupyter_threads_per_worker" in self.config:
self.threads_per_worker = self.config["jupyter_threads_per_worker"]
#number of workers for dask distributed processing
if "jupyter_number_workers" in self.config:
self.n_workers = self.config["jupyter_number_workers"]
# start a dask client to process the notebooks efficiently.
# processes = False. This is sometimes preferable if you want to avoid inter-worker communication and your computations release the GIL. This is common when primarily using NumPy or Dask Array.
if (self.config["jupyter_execute_notebooks"]):
self.client = Client(processes=False,
threads_per_worker=self.threads_per_worker,
n_workers=self.n_workers)
self.execution_vars = {
'target': 'website',
'dependency_lists': self.config["jupyter_dependency_lists"],
'executed_notebooks': [],
'delayed_notebooks': dict(),
'futures': [],
'delayed_futures': [],
'destination': self.executedir
}
if (self.config["jupyter_download_nb_execute"]):
if self.client is None:
self.client = Client(
processes=False,
threads_per_worker=self.threads_per_worker,
n_workers=self.n_workers)
self.download_execution_vars = {
'target': 'downloads',
'dependency_lists': self.config["jupyter_dependency_lists"],
'executed_notebooks': [],
'delayed_notebooks': dict(),
'futures': [],
'delayed_futures': [],
'destination': self.downloadsExecutedir
}
def test_end_to_end(nrows, ncols, nclusters, n_parts,
delayed_predict, input_type, cluster):
client = None
try:
client = Client(cluster)
from cuml.dask.cluster import KMeans as cumlKMeans
from cuml.dask.datasets import make_blobs
X, y = make_blobs(n_samples=int(nrows),
n_features=ncols,
centers=nclusters,
n_parts=n_parts,
cluster_std=0.01,
random_state=10)
wait(X)
if input_type == "dataframe":
X_train = to_dask_cudf(X)
y_train = to_dask_cudf(y)
elif input_type == "array":
X_train, y_train = X, y
cumlModel = cumlKMeans(init="k-means||",
n_clusters=nclusters,
random_state=10)
cumlModel.fit(X_train)
cumlLabels = cumlModel.predict(X_train, delayed_predict)
n_workers = len(list(client.has_what().keys()))
# Verifying we are grouping partitions. This should be changed soon.
if n_parts is not None and n_parts < n_workers:
parts_len = n_parts
else:
parts_len = n_workers
if input_type == "dataframe":
assert cumlLabels.npartitions == parts_len
cumlPred = cp.array(cumlLabels.compute().to_pandas().values)
labels = cp.squeeze(y_train.compute().to_pandas().values)
elif input_type == "array":
assert len(cumlLabels.chunks[0]) == parts_len
cumlPred = cp.array(cumlLabels.compute())
labels = cp.squeeze(y_train.compute())
assert cumlPred.shape[0] == nrows
assert cp.max(cumlPred) == nclusters - 1
assert cp.min(cumlPred) == 0
score = adjusted_rand_score(labels, cumlPred)
print(str(score))
assert 1.0 == score
finally:
client.close()
def test_with_asyncio(local_cuda_cluster):
with Client(local_cuda_cluster) as client:
address = client.scheduler.address
output = asyncio.run(run_from_dask_array_asyncio(address))
assert isinstance(output['booster'], xgboost.Booster)
assert isinstance(output['history'], dict)
class MGContext:
"""Utility Context Manager to start a multi GPU context using dask_cuda
Parameters:
-----------
number_of_devices : int
Number of devices to use, verification must be done prior to call to
ensure that there are enough devices available. If not specified, the
cluster will be initialized to use all visible devices.
rmm_managed_memory : bool
True to enable managed memory (UVM) in RMM as part of the
cluster. Default is False.
p2p : bool
Initialize UCX endpoints if True. Default is False.
"""
def __init__(self,
number_of_devices=None,
rmm_managed_memory=False,
p2p=False):
self._number_of_devices = number_of_devices
self._rmm_managed_memory = rmm_managed_memory
self._client = None
self._p2p = p2p
self._cluster = CUDACluster(
n_workers=self._number_of_devices,
rmm_managed_memory=self._rmm_managed_memory
)
@property
def client(self):
return self._client
@property
def cluster(self):
return self._cluster
def __enter__(self):
self._prepare_mg()
return self
def _prepare_mg(self):
self._prepare_client()
self._prepare_comms()
def _prepare_client(self):
self._client = Client(self._cluster)
self._client.wait_for_workers(self._number_of_devices)
def _prepare_comms(self):
Comms.initialize(p2p=self._p2p)
def _close(self):
Comms.destroy()
if self._client is not None:
self._client.close()
if self._cluster is not None:
self._cluster.close()
def __exit__(self, type, value, traceback):
self._close()
def test_status():
with scheduler_and_workers() as (s, (a, b)):
c = Client(s.address_to_clients)
assert c.scheduler_status() == 'OK'
c.close()
def _prepare_client(self):
self._client = Client(self._cluster)
self._client.wait_for_workers(self._number_of_devices)
from dask.distributed import Client
import socket
client = Client(scheduler_file='scheduler.json')
print(client)
host = client.run_on_scheduler(socket.gethostname)
def start_jlab(dask_scheduler):
import subprocess
proc = subprocess.Popen(['jupyter', 'notebook', '--ip', host])
dask_scheduler.jlab_proc = proc
client.run_on_scheduler(start_jlab)
print("HOST : %s" % host)
import dask
import dask.threaded
import dask.multiprocessing
from dask.distributed import Client
c = Client()
import xarray as xr
import numpy as np
import glob
import time
import xscale
data_dir = '/store/CT1/hmg2840/lbrodeau/eNATL60/eNATL60-BLBT02-S/'
def compute_buoy(t,s):
rau0 = 1000
grav = 9.81
buoy= -1*(grav/rau0)*sigma0(t,s)
return buoy
def sigma0(t,s):
zrau0=1000
zsr=np.sqrt(np.abs(s))
zs=s
zt=t
zr1 = ( ( ( ( 6.536332e-9*zt-1.120083e-6 )*zt+1.001685e-4)*zt - 9.095290e-3 )*zt+6.793952e-2 )*zt+999.842594
zr2= ( ( ( 5.3875e-9*zt-8.2467e-7 )*zt+7.6438e-5 ) *zt - 4.0899e-3 ) *zt+0.824493
zr3= ( -1.6546e-6*zt+1.0227e-4 ) *zt-5.72466e-3
def test_empty_dmatrix_auc(self,
local_cuda_cluster: LocalCUDACluster) -> None:
with Client(local_cuda_cluster) as client:
n_workers = len(_get_client_workers(client))
run_empty_dmatrix_auc(client, "gpu_hist", n_workers)
args = parser.parse_args()
# FIXED parameters
noise_dim = pd.Index(range(500), name='noise_field')
mask = xr.open_dataarray('/scratch/pkittiwi/fg1p/hera331_fov_mask.nc')
if args.xyf_chunks is not None:
chunks = {'x': args.xyf_chunks[0], 'y': args.xyf_chunks[1],
'f': args.xyf_chunks[2]}
else:
chunks = None
# Setup and start Dask Local Cluster
cluster = LocalCluster(n_workers=args.n_workers, processes=args.processes,
threads_per_worker=args.threads_per_worker,
scheduler_port=args.scheduler_port,
diagnostics_port=args.diagnostics_port)
client = Client(cluster)
print('Hostname: {:s}'.format(os.environ['HOSTNAME']))
print('Dask Scheduler address: {:s}'.format(cluster.scheduler_address))
print('Dask Dashboard link: {:s}'.format(cluster.dashboard_link))
# Loop over data parameters and perform calculation
for bw, fbw, t, s in itertools.product(
args.bin_width, args.filter_bandwidth, args.theta, args.shift
):
start_time = datetime.now()
ds = xr.open_mfdataset(
['/scratch/pkittiwi/fg1p/binned_noise_map/bin{:.2f}MHz/'
'fbw{:.2f}MHz/theta{:.1f}/shift{:d}/binned_noise_map_bin{:.2f}MHz_'
'fbw{:.2f}MHz_theta{:.1f}_shift{:d}_{:03d}.nc'
.format(bw, fbw, t, s, bw, fbw, t, s, i) for i in range(500)],
concat_dim=noise_dim, chunks=chunks
def test_dask_dataframe(self,
local_cuda_cluster: LocalCUDACluster) -> None:
with Client(local_cuda_cluster) as client:
run_with_dask_dataframe(dxgb.DaskDMatrix, client)
run_with_dask_dataframe(dxgb.DaskDeviceQuantileDMatrix, client)
def find_neighbors(X, n_neighbors, metric, algorithm, distributed):
"""
Returns the indices of the k-nearest neighbors for each cell
in a cell-by-feature dataframe.
Parameters
----------
X : cudf.DataFrame
Input cell-by-feature dataframe.
n_neighbors : int
Number of neighbors for kNN.
metric: string
Distance metric to use for kNN.
Currently, only 'euclidean' is supported.
algorithm: string
The query algorithm to use.
Currently, only 'brute' is supported.
distributed: bool
If True, use a multi-GPU dask cluster for kNN search.
Returns
-------
idx : cudf.DataFrame
The indices of the kNN for each cell in X.
"""
print(
f"Finding {n_neighbors} nearest neighbors using "
f"{metric} metric and {algorithm} algorithm...",
flush=True,
)
if distributed:
print("Running distributed kNN...")
import dask_cudf
from dask_cuda import LocalCUDACluster
from dask.distributed import Client
from cuml.dask.neighbors import NearestNeighbors
cluster = LocalCUDACluster()
client = Client(cluster)
npartitions = len(cluster.cuda_visible_devices)
X_dask = dask_cudf.from_cudf(X, npartitions=npartitions)
# Use n_neighbors + 1 to account for self index
model = cuml.dask.neighbors.NearestNeighbors(n_neighbors=n_neighbors +
1,
client=client)
model.fit(X_dask)
idx = model.kneighbors(X_dask, return_distance=False).compute()
client.shutdown()
else:
# Use n_neighbors + 1 to account for self index
model = cuml.neighbors.NearestNeighbors(n_neighbors=n_neighbors + 1,
metric=metric,
algorithm=algorithm)
model.fit(X)
idx = model.kneighbors(X, return_distance=False)
# Drop self index
# this replace original grephano implementation that just remove the first index,assuming it is self index
# which is not always the case!
idx = np.vstack(
list(map(lambda i: idx[i, idx[i, :] != i], range(idx.shape[0]))))
return idx
def test_aamp_int_input(dask_cluster):
with pytest.raises(TypeError):
with Client(dask_cluster) as dask_client:
aamped(dask_client, np.arange(10), 5)
from dask_jobqueue import SLURMCluster
from dask.distributed import Client
import time
from dask.distributed import progress
cluster = SLURMCluster(queue='par-multi',
cores=10,
memory='1000',
job_mem='2000')
def slow_increment(x):
time.sleep(1)
x = x + 1
return x
cluster.scale(10)
client = Client(cluster)
futures = client.map(slow_increment, range(1000))
progress(futures)
import time
import numpy as np
import dask
import sys
import shutil
import os
PDB = "/oasis/scratch/comet/sfan19/temp_project/step5_znm.pdb"
XTC = "/oasis/scratch/comet/sfan19/temp_project/step7_90ns_center.xtc"
from dask.distributed import Client
Scheduler_IP = sys.argv[1]
print(Scheduler_IP)
print(type(Scheduler_IP))
c = Client(Scheduler_IP)
u = mda.Universe(PDB, XTC)
g1 = u.select_atoms('name OH2')
g2 = u.select_atoms('name OH2')
rdfs = rdf.InterRDF(g1, g2, range=(0, 5))
t_time = []
t_io = []
t_compute = []
t_u = []
t_w = []
ns = [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72]
for n in ns:
rdfs.run(n_jobs=n, n_blocks=n)
t_time.append([
n, rdfs.timing.total,
def test_gpu_hist(self, params, num_rounds, dataset, local_cuda_cluster):
with Client(local_cuda_cluster) as client:
run_gpu_hist(params, num_rounds, dataset, dxgb.DaskDMatrix, client)
run_gpu_hist(params, num_rounds, dataset,
dxgb.DaskDeviceQuantileDMatrix, client)
def main(args):
print('\n>> CLI Parameters ...\n')
print args
if not os.path.isfile(args.inputImageFile):
raise IOError('Input image file does not exist.')
if len(args.reference_mu_lab) != 3:
raise ValueError('Reference Mean LAB should be a 3 element vector.')
if len(args.reference_std_lab) != 3:
raise ValueError('Reference Stddev LAB should be a 3 element vector.')
if len(args.analysis_roi) != 4:
raise ValueError('Analysis ROI must be a vector of 4 elements.')
#
# Initiate Dask client
#
print('\n>> Creating Dask client ...\n')
scheduler_address = json.loads(args.scheduler_address)
if scheduler_address is None:
scheduler_address = LocalCluster(
n_workers=multiprocessing.cpu_count() - 1,
scheduler_port=0,
silence_logs=False)
c = Client(scheduler_address)
print c
#
# Read Input Image
#
print('\n>> Reading input image ... \n')
ts = large_image.getTileSource(args.inputImageFile)
ts_metadata = ts.getMetadata()
print json.dumps(ts_metadata, indent=2)
is_wsi = ts_metadata['magnification'] is not None
#
# Compute tissue/foreground mask at low-res for whole slide images
#
if is_wsi:
print('\n>> Computing tissue/foreground mask at low-res ...\n')
# get image at low-res
maxSize = max(ts_metadata['sizeX'], ts_metadata['sizeY'])
downsample_factor = 2**np.floor(np.log2(maxSize / 2048))
fgnd_seg_mag = ts_metadata['magnification'] / downsample_factor
fgnd_seg_scale = {'magnification': fgnd_seg_mag}
im_lres, _ = ts.getRegion(
scale=fgnd_seg_scale,
format=large_image.tilesource.TILE_FORMAT_NUMPY)
im_lres = im_lres[:, :, :3]
# compute foreground mask at low-res
im_fgnd_mask_lres = htk_utils.simple_mask(im_lres)
#
# Detect nuclei in paralle using Dask
#
print('\n>> Detecting nuclei in parallel using Dask ...\n')
it_kwargs = {
'format': large_image.tilesource.TILE_FORMAT_NUMPY,
'tile_size': {
'width': args.analysis_tile_size
},
'scale': {
'magnification': args.analysis_mag
},
}
if np.all(np.array(args.analysis_roi) == -1):
process_whole_image = True
else:
process_whole_image = False
if not process_whole_image:
it_kwargs['region'] = {
'left': args.analysis_roi[0],
'top': args.analysis_roi[1],
'width': args.analysis_roi[2],
'height': args.analysis_roi[3],
'units': 'base_pixels'
}
tile_nuclei_list = []
for tile in ts.tileIterator(**it_kwargs):
if is_wsi:
# get current region in base_pixels
rgn_hres = {
'left': tile['gx'],
'top': tile['gy'],
'right': tile['gx'] + tile['gwidth'],
'bottom': tile['gy'] + tile['gheight'],
'units': 'base_pixels'
}
# get foreground mask for current tile at low resolution
rgn_lres = ts.convertRegionScale(rgn_hres,
targetScale=fgnd_seg_scale,
targetUnits='mag_pixels')
top = np.int(rgn_lres['top'])
bottom = np.int(rgn_lres['bottom'])
left = np.int(rgn_lres['left'])
right = np.int(rgn_lres['right'])
im_tile_fgnd_mask_lres = im_fgnd_mask_lres[top:bottom, left:right]
# skip tile if there is not enough foreground in the slide
cur_fgnd_frac = im_tile_fgnd_mask_lres.mean()
if np.isnan(cur_fgnd_frac) or cur_fgnd_frac <= args.min_fgnd_frac:
continue
# detect nuclei
cur_nuclei_list = dask.delayed(detect_tile_nuclei)(
args.inputImageFile, tile['tile_position']['position'], args,
**it_kwargs)
# append result to list
tile_nuclei_list.append(cur_nuclei_list)
def collect(x):
return x
tile_nuclei_list = dask.delayed(collect)(tile_nuclei_list).compute()
nuclei_list = list(itertools.chain.from_iterable(tile_nuclei_list))
print 'Number of nuclei = ', len(nuclei_list)
#
# Write annotation file
#
print('\n>> Writing annotation file ...\n')
annot_fname = os.path.splitext(
os.path.basename(args.outputNucleiAnnotationFile))[0]
annotation = {"name": annot_fname, "elements": nuclei_list}
with open(args.outputNucleiAnnotationFile, 'w') as annotation_file:
json.dump(annotation, annotation_file, indent=2, sort_keys=False)
})
inputs = targets
train(
inputs,
targets,
model=autoencoder,
data=data_handler,
optimizer=optimizer,
regularization=regularization,
epochs=epochs,
convergence=convergence,
lossfxn=None,
device="cpu",
)
latent_space = autoencoder.get_latent_space(targets, svm=True)
dump(latent_space, filename="cu_training.latent")
Potentials.save(autoencoder)
return latent_space, energy_targets, data_handler
if __name__ == "__main__":
logger(filename="cu_training.log")
cluster = LocalCluster()
client = Client(cluster)
inputs, outputs, data_handler = autoencode()
def run_dask_function(config):
"""Start a Dask Cluster using dask-kubernetes and run a function.
Talks to kubernetes to create `n` amount of new `pods` with a dask worker inside of each
forming a `dask` cluster. Then, a function specified from `config` is being imported and
run with the given arguments. The tasks created by this `function` are being run on the
`dask` cluster for distributed computation.
The config dict must contain the following sections:
* run
* dask_cluster
* output
Args:
config (dict):
Config dictionary.
"""
output_conf = config.get('output')
if output_conf:
path = output_conf.get('path')
if not path:
raise ValueError('An output path must be provided when providing `output`.')
cluster_spec = _generate_cluster_spec(config, master=False)
# Importing here to avoid an aiohttp error if not used.
from dask_kubernetes import KubeCluster # pylint: disable=C0415
cluster = KubeCluster.from_dict(cluster_spec)
workers = config['dask_cluster'].get('workers')
if not workers:
cluster.adapt()
elif isinstance(workers, int):
cluster.scale(workers)
else:
cluster.adapt(**workers)
client = Client(cluster)
client.get_versions(check=True)
client.register_worker_callbacks(_logging_setup)
try:
run = _import_function(config['run'])
kwargs = config['run']['args']
results = run(**kwargs)
finally:
client.close()
cluster.close()
if output_conf:
bucket = output_conf.get('bucket')
try:
if bucket:
aws_key = output_conf.get('key')
aws_secret = output_conf.get('secret_key')
_upload_to_s3(bucket, path, results, aws_key, aws_secret)
else:
dirname = os.path.dirname(path)
if dirname:
os.makedirs(dirname, exist_ok=True)
results.to_csv(path)
except Exception: # pylint: disable=W0703
print('Error storing results. Falling back to console dump.')
print(_dataframe_to_csv_str(results))
return None
return results
# 미리 설정된 기반 스케줄러 : 워커 관리
# 워커는 스레딩함(ThreadPoolExecutor 사용)
from dask.distributed import Client
def square(x):
return x**2
if __name__ == "__main__":
client = Client()
# 설정에따라 다른 컴퓨터에 있는 워커를 사용가능.
# client를 이용하여, local computer의 병렬코드를 그대로 여러 머신을 사용한 분산처리에 이용가능
print(
client
) #<Client: 'tcp://127.0.0.1:#' processes=4 threads=8, memory=8.44 GB>
fut = client.submit(square, 2)
print(
fut) #<Future: pending, key: square-c94e3bc4bf0b4c9a5933f7b43187f6b0>
print(fut.result()) #4
futs = client.map(square, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
print(futs[0:3])
# [<Future: pending, key: square-72f30fc48e1c42ee95576d0d3088f8db>,
# <Future: pending, key: square-89e339991c492cf890f48a2ecb386f31>,
# <Future: finished, type: builtins.int, key: square-c94e3bc4bf0b4c9a5933f7b43187f6b0>]
# 2에대한 결과는 위로부터 캐시되어있다.
futs_res = client.gather(futs)
