def main():
#define parallel mcmc wrapper
def parallel_mcmc(_):
return (mcmc(initial_parameters=epa_0,
proposer=normal_prop,
param2res=param2res,
costfunction=costfunction,
nsimu=5000))
#check jobs resources to initialize dask workers
num_threads = int(
environ.get('SLURM_CPUS_PER_TASK', environ.get('OMP_NUM_THREADS', 1)))
initialize(interface='ib0', nthreads=num_threads)
client = Client()
#run 10 chains
[[c_form1, j_form1], [c_form2, j_form2], [c_form3, j_form3],
[c_form4, j_form4], [c_form5, j_form5], [c_form6, j_form6],
[c_form7, j_form7], [c_form8, j_form8], [c_form9, j_form9],
[c_form10,
j_form10]] = client.gather(client.map(parallel_mcmc, range(0, 10)))
#print chain5 output as test
formal_c_path = dataPath.joinpath('chain5_pmcmc_c.csv')
formal_j_path = dataPath.joinpath('chain5_pmcmc_j.csv')
pd.DataFrame(c_form5).to_csv(formal_c_path, sep=',')
pd.DataFrame(j_form5).to_csv(formal_j_path, sep=',')
def main():
start = time.time()
initialize(interface='ib0')
client = Client()
iris = datasets.load_iris()
X = iris.data
y = iris.target
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.05)
D_test = xgb.DMatrix(X_test, label=Y_test)
params = {
'eta': 0.3,
'max_depth': 3,
'objective': 'multi:softprob',
'num_class': 3
}
bst = dxgb.train(client,
params,
da.asarray(X_train),
da.asarray(Y_train),
num_boost_round=10)
preds = bst.predict(D_test)
best_preds = np.asarray([np.argmax(line) for line in preds])
print("Precision = {}".format(
precision_score(Y_test, best_preds, average='macro')))
print("Recall = {}".format(
recall_score(Y_test, best_preds, average='macro')))
print("Accuracy = {}".format(accuracy_score(Y_test, best_preds)))
elapsed = (time.time() - start)
print(f"Elapsed time: {elapsed}")
def main(args):
# distributed setup
print("initializing...")
dask_mpi.initialize(nthreads=args.cpus_per_node)
client = Client()
print(client)
# get data
print("connecting to data...")
print(client)
container_name = "malware"
storage_options = {"account_name": "azuremlexamples"}
fs = AzureBlobFileSystem(**storage_options)
files = fs.ls(f"{container_name}/processed")
# read into dataframes
print("creating dataframes...")
print(client)
for f in files:
if "train" in f:
df_train = dd.read_parquet(f"az://{f}", storage_options=storage_options)
elif "test" in f:
df_test = dd.read_parquet(f"az://{f}", storage_options=storage_options)
# data processing
print("processing data...")
print(client)
cols = [col for col in df_train.columns if df_train.dtypes[col] != "object"]
X = df_train[cols].drop("HasDetections", axis=1).values.persist()
y = df_train["HasDetections"].persist()
# train xgboost
print("training xgboost...")
print(client)
params = {
"objective": "binary:logistic",
"learning_rate": args.learning_rate,
"gamma": args.gamma,
"max_depth": args.max_depth,
}
mlflow.log_params(params) # log to the run
dtrain = xgb.dask.DaskDMatrix(client, X, y)
model = xgb.dask.train(client, params, dtrain, num_boost_round=args.num_boost_round)
print(model)
# predict on test data
print("making predictions...")
print(client)
X_test = df_test[
[col for col in cols if "HasDetections" not in col]
].values.persist()
y_pred = xgb.dask.predict(client, model, X_test)
y_pred.to_dask_dataframe().to_csv("./outputs/predictions.csv")
# save model
print("saving model...")
print(client)
mlflow.xgboost.log_model(model["booster"], "./outputs/model")
def main(args):
dask.config.set({"distributed.comm.timeouts.connect": 60})
if args.bin_size is not None:
if args.bin_size > args.target_seq_length:
raise ValueError(
"Please provide a bin size that is <= target-seq-length")
if args.target_seq_length % args.bin_size != 0:
raise ValueError(
"Please provide a bin size that can divide the target "
"sequence length.")
if args.schedule == 'mpi':
from dask_mpi import initialize
initialize(local_directory='/tmp/dask-worker-space', nanny=False)
client = dask.distributed.Client()
else:
client = dask.distributed.Client(
n_workers=args.local_n_workers,
threads_per_worker=args.local_threads_per_worker,
)
nltk.download('punkt')
if os.path.isfile(args.vocab_file):
tokenizer = transformers.BertTokenizerFast(args.vocab_file)
else:
tokenizer = transformers.BertTokenizerFast.from_pretrained(
args.vocab_file)
tic = time.perf_counter()
pairs = _get_pairs(
wikipedia_path=args.wikipedia,
books_path=args.books,
common_crawl_path=args.common_crawl,
wikipedia_lang=args.wikipedia_lang,
target_seq_length=args.target_seq_length,
short_seq_prob=args.short_seq_prob,
blocksize=args.block_size,
num_blocks=args.num_blocks,
duplicate_factor=args.duplicate_factor,
sample_ratio=args.sample_ratio,
seed=args.seed,
tokenizer=tokenizer,
masking=args.masking,
masked_lm_ratio=args.masked_lm_ratio,
)
args.sink = expand_outdir_and_mkdir(args.sink)
_save(
pairs,
args.sink,
output_format=args.output_format,
bin_size=args.bin_size,
target_seq_length=args.target_seq_length,
masking=args.masking,
)
print('Running the dask pipeline took {} s'.format(time.perf_counter() -
tic))
def main():
# Work out from the environment how many threads to allocate
num_threads = int(
environ.get('SLURM_CPUS_PER_TASK', environ.get('OMP_NUM_THREADS', 1)))
print("num_threads: ", num_threads)
# Create the Dask workers
initialize(interface='ib0', nthreads=num_threads)
# Create the Dask object that will manage the communications
client = Client()
start = datetime.now()
results = run_test(client=client)
end = datetime.now()
print(f"Time taken: {end - start}")
print(results)
def main():
from argparse import ArgumentParser
parser = ArgumentParser()
#parser.add_argument('min_num', type=int)
#parser.add_argument('max_num', type=int)
args = parser.parse_args()
num_threads = int(
environ.get('SLURM_CPUS_PER_TASK', environ.get('OMP_NUM_THREADS', 1)))
initialize(interface='ib0', nthreads=num_threads)
client = Client()
min_num = 10
max_num = 100
start_time = datetime.now()
num_primes = sum(
client.gather(client.map(slow_is_prime, range(min_num, max_num + 1))))
end_time = datetime.now()
print(f'{num_primes} primes between {min_num} and {max_num} '
f'[{end_time - start_time}]')
def start_dask(self):
"""
Prepare dask to run under MPI. After calling this method
only a single process, MPI rank 1 will continue to exeute code
"""
# using the programmatic dask configuration system
# does not seem to work. Presumably the loggers have already
# been created by the time we modify the config. Doing it with
# env vars seems to work. If the user has already set this then
# we use that value. Otherwise we only want error logs
key = "DASK_LOGGING__DISTRIBUTED"
os.environ[key] = os.environ.get(key, "error")
try:
import dask
import dask_mpi
import dask.distributed
except ImportError: # pragma: no cover
print(
"ERROR: Using --mpi option on stages that use dask requires "
"dask[distributed] and dask_mpi to be installed."
)
raise
if self.size < 3: # pragma: no cover
raise ValueError(
"Dask requires at least three processes. One becomes a scheduler "
"process, one is a client that runs the code, and more are required "
"as worker processes."
)
# This requires my fork until/unless they merge the PR, to allow
# us to pass in these two arguments. In vanilla dask-mpi sys.exit
# is called at the end of the event loop without returning to us.
# After this point only a single process, MPI rank 1,
# should continue to exeute code. The others enter an event
# loop and return with is_client=False, which we return here
# to tell the caller that they should not run everything.
is_client = dask_mpi.initialize(comm=self.comm, exit=False)
if is_client:
# Connect this local process to remote workers.
self.dask_client = dask.distributed.Client()
# I don't yet know how to see this dashboard link at nersc
print(f"Started dask. Diagnostics at {self.dask_client.dashboard_link}")
return is_client
#!/usr/bin/env python3
import dask.array as da
from dask_mpi import initialize
from dask.distributed import Client, performance_report
initialize()
client = Client()
def example_function():
x = da.random.random((100_000, 100_000, 10), chunks=(10_000, 10_000, 5))
y = da.random.random((100_000, 100_000, 10), chunks=(10_000, 10_000, 5))
z = (da.arcsin(x) + da.arccos(y)).sum(axis=(1, 2))
with performance_report(filename="dask-report_mpi.html"):
result = z.compute()
if __name__ == "__main__":
example_function()
def __init__(self,
initialization_args: Dict = None,
client_args: Dict = None):
"""
Creating a scheduler initializes dask MPI
:param initialization_args: dictionary of keyword arguments for dask-mpi
initializer.
:param client_args: dictionary of keyword arguments for dask client.
dask-mpi arguments
------------------
interface:str
Network interface like ‘eth0’ or ‘ib0’
nthreads:int
Number of threads per worker
local_directory:str
Directory to place worker files
memory_limit:int, float, or ‘auto’
Number of bytes before spilling data to disk. This can be an integer (nbytes), float (fraction of total memory), or ‘auto’.
nanny:bool
Start workers in nanny process for management
bokeh:bool
Enable Bokeh visual diagnostics
bokeh_port:int
Bokeh port for visual diagnostics
bokeh_prefix:str
Prefix for the bokeh app
bokeh_worker_port:int
Worker’s Bokeh port for visual diagnostics
dask client arguments
---------------------
address: string, or Cluster
This can be the address of a Scheduler server like a string
'127.0.0.1:8786' or a cluster object like LocalCluster()
timeout: int
Timeout duration for initial connection to the scheduler
set_as_default: bool (True)
Claim this scheduler as the global dask scheduler
scheduler_file: string (optional)
Path to a file with scheduler information if available
security: Security or bool, optional
Optional security information. If creating a local cluster can also
pass in True, in which case temporary self-signed credentials will
be created automatically.
asynchronous: bool (False by default)
Set to True if using this client within async/await functions or within
Tornado gen.coroutines. Otherwise this should remain False for normal use.
name: string (optional)
Gives the client a name that will be included in logs generated on the
scheduler for matters relating to this client
direct_to_workers: bool (optional)
Whether or not to connect directly to the workers, or to ask the
scheduler to serve as intermediary.
heartbeat_interval: int
Time in milliseconds between heartbeats to scheduler
"""
if initialization_args is None:
initialization_args = {}
if client_args is None:
client_args = {}
from dask_mpi import initialize
initialize(**initialization_args)
# Rank 0 is initialized with scheduler
# Rank 1 will pass through and execute following code
# Rank 2+ will execute workers
self._client = Client(**client_args)
'hits based on LCA. This is a rough estimate and '
'should be revised carefully')
parser.add_argument('-o', '--out_filename', default='hit.hits', type=str,
help='name of output (filtered) file')
parser.add_argument('-f', '--outfmt', default=cols, type=str,
help='Custom format for BLAST')
parser.add_argument('-u', '--unique', default=False, action='store_true',
help='Process only unique sequences')
parser.add_argument('-v', '--version', action='version',
version='%(prog)s {}'.format(version))
print('\njustblast version:', colored(version, None, attrs=["bold", "blink"]))
print(colored('Copyright 2019', 'red', attrs=["bold"]), 'Jose Sergio Hleap\n')
args = parser.parse_args()
try:
size = os.environ["OMPI_COMM_WORLD_SIZE"]
print(f'Detecting MPI world... Initializing {size} workers')
dask_mpi.initialize()
client = Client()
except KeyError:
print(f'Creating a local cluster with {args.cpus} cpus')
client = Client(n_workers=args.cpus, threads_per_worker=1,
processes=False)
print(client)
main(db=args.db, query=args.query, evalue=args.evalue,
p_id=args.percent_id, max_targets=args.max_target_seqs,
cpus=args.cpus, qcovs=args.query_coverage, identify=args.identify,
outfile=args.out_filename, outfmt=args.outfmt, unique=args.unique)
#!/usr/bin/env python
#import psutil
#net_if_addrs = psutil.net_if_addrs()
#allowed_ifnames = list(net_if_addrs.keys())
#print('allowed interfaces are :',allowed_ifnames)
from dask_mpi import initialize
initialize(interface='ib0:0')
from distributed import Client
client = Client()
print('Currently working with '+str(len(client.scheduler_info()["workers"]))+' workers')
import os
import sys
import timeit
from dask.distributed import Client
from dask_mpi import initialize
# initialize(nthreads=1, local_directory=os.environ["TMPDIR"] + "/dask_workers", memory_limit=100e9)
initialize(nthreads=1,
local_directory="/dev/shm/dask_workers",
memory_limit=100e9)
def setup(inputs):
N = int(sys.argv[1])
block_size = sys.argv[2]
if block_size == "full":
block_size = str(N)
elif block_size == "auto":
block_size = "'auto'"
return N, block_size
if __name__ == '__main__':
client = Client()
N, block_size = setup(sys.argv[1:])
print(
timeit.timeit(
def __init__(
self, num_workers=None, threads_per_worker=1, launch=None, out=None, err=None
):
"""
Returns a Client connected to a cluster of `num_workers` workers.
"""
self._server = None
if launch is None:
launch = default_comm().size == 1
# pylint: disable=import-outside-toplevel,
if not launch:
# Then the script has been submitted in parallel with mpirun
num_workers = num_workers or default_comm().size - 2
if num_workers < 0 or default_comm().size != num_workers + 2:
raise RuntimeError(
f"""
Error: (num_workers + 2) processes required.
The script has not been submitted on enough processes.
Got {default_comm().size} processes instead of {num_workers + 2}.
"""
)
initialize(
nthreads=threads_per_worker,
local_directory=tempfile.mkdtemp(),
nanny=False,
)
super().__init__()
else:
num_workers = num_workers or (multiprocessing.cpu_count() + 1)
# Since dask-mpi produces several file we create a temporary directory
self._dir = tempfile.mkdtemp()
# The command runs in the background (_bg=True)
# and the stdout(err) is stored in self._out(err)
pwd = os.getcwd()
sh.cd(self._dir)
self._server = sh.mpirun(
"-n",
num_workers + 1,
"dask-mpi",
"--no-nanny",
"--nthreads",
threads_per_worker,
"--scheduler-file",
"scheduler.json",
_bg=True,
_out=out or sys.stdout,
_err=err or sys.stderr,
)
sh.cd(pwd)
atexit.register(self.close_server)
super().__init__(scheduler_file=self._dir + "/scheduler.json")
# Waiting for all the workers to connect
def handler(signum, frame):
if self.server is not None:
self.close_server()
raise RuntimeError(
"Couldn't connect to %d processes. Got %d workers."
% (num_workers, len(self.workers))
)
signal.signal(signal.SIGALRM, handler)
signal.alarm(5)
while len(self.workers) != num_workers:
time.sleep(0.001)
signal.alarm(0)
self.ranks = {key: val["name"] for key, val in self.workers.items()}
self._comm = self.create_comm()
#qsub mpi.qsub
def hello_world(n):
import time
time.sleep(1)
return "Hello world "+str(n)
if __name__ == '__main__':
from dask.distributed import Client, LocalCluster
from dask_mpi import initialize
import time
import numpy as np
initialize(local_directory='/scratch/users/sapatha2/', nanny=False, dashboard=False)
client = Client()
res = client.map(hello_world, np.arange(380))
start = time.time()
for r in res:
print (r.result())
client.close()
print(time.time() - start)
exit(0)
import time
import h5py
import shutil
import argparse
import itertools
from mpi4py import MPI
from copy import deepcopy
import random
import numpy as np
import dask
from dask_mpi import initialize
# with dask.config.set({"distributed.worker.resources.GPU": 1}):
initialize(nthreads=1,
memory_limit='32 GB',
interface='ipogif0',
local_directory=os.getcwd())
import dask
import dask.array
from dask.distributed import Client, wait
from svreg.archive import Archive, md5Hash
from svreg.settings import Settings
from svreg.database import SVDatabase
from svreg.regressor import SVRegressor
from svreg.evaluator import SVEvaluator
from svreg.population import Population
from svreg.functions import _function_map
from svreg.tree import MultiComponentTree as MCTree
from distributed import Client
from mpi4py.MPI import COMM_WORLD as world
from dask_mpi import initialize, send_close_signal
# Split our MPI world into two pieces, one consisting just of
# the old rank 3 process and the other with everything else
new_comm_assignment = 1 if world.rank == 3 else 0
comm = world.Split(new_comm_assignment)
if world.rank != 3:
# run tests with rest of comm
is_client = initialize(comm=comm, exit=False)
if is_client:
with Client() as c:
c.submit(lambda x: x + 1, 10).result() == 11
c.submit(lambda x: x + 1, 20).result() == 21
send_close_signal()
# check that our original comm is intact
world.Barrier()
x = 100 if world.rank == 0 else 200
x = world.bcast(x)
assert x == 100
