def main(): #define parallel mcmc wrapper def parallel_mcmc(_): return (mcmc(initial_parameters=epa_0, proposer=normal_prop, param2res=param2res, costfunction=costfunction, nsimu=5000)) #check jobs resources to initialize dask workers num_threads = int( environ.get('SLURM_CPUS_PER_TASK', environ.get('OMP_NUM_THREADS', 1))) initialize(interface='ib0', nthreads=num_threads) client = Client() #run 10 chains [[c_form1, j_form1], [c_form2, j_form2], [c_form3, j_form3], [c_form4, j_form4], [c_form5, j_form5], [c_form6, j_form6], [c_form7, j_form7], [c_form8, j_form8], [c_form9, j_form9], [c_form10, j_form10]] = client.gather(client.map(parallel_mcmc, range(0, 10))) #print chain5 output as test formal_c_path = dataPath.joinpath('chain5_pmcmc_c.csv') formal_j_path = dataPath.joinpath('chain5_pmcmc_j.csv') pd.DataFrame(c_form5).to_csv(formal_c_path, sep=',') pd.DataFrame(j_form5).to_csv(formal_j_path, sep=',')
def main(): start = time.time() initialize(interface='ib0') client = Client() iris = datasets.load_iris() X = iris.data y = iris.target X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.05) D_test = xgb.DMatrix(X_test, label=Y_test) params = { 'eta': 0.3, 'max_depth': 3, 'objective': 'multi:softprob', 'num_class': 3 } bst = dxgb.train(client, params, da.asarray(X_train), da.asarray(Y_train), num_boost_round=10) preds = bst.predict(D_test) best_preds = np.asarray([np.argmax(line) for line in preds]) print("Precision = {}".format( precision_score(Y_test, best_preds, average='macro'))) print("Recall = {}".format( recall_score(Y_test, best_preds, average='macro'))) print("Accuracy = {}".format(accuracy_score(Y_test, best_preds))) elapsed = (time.time() - start) print(f"Elapsed time: {elapsed}")
def main(args): # distributed setup print("initializing...") dask_mpi.initialize(nthreads=args.cpus_per_node) client = Client() print(client) # get data print("connecting to data...") print(client) container_name = "malware" storage_options = {"account_name": "azuremlexamples"} fs = AzureBlobFileSystem(**storage_options) files = fs.ls(f"{container_name}/processed") # read into dataframes print("creating dataframes...") print(client) for f in files: if "train" in f: df_train = dd.read_parquet(f"az://{f}", storage_options=storage_options) elif "test" in f: df_test = dd.read_parquet(f"az://{f}", storage_options=storage_options) # data processing print("processing data...") print(client) cols = [col for col in df_train.columns if df_train.dtypes[col] != "object"] X = df_train[cols].drop("HasDetections", axis=1).values.persist() y = df_train["HasDetections"].persist() # train xgboost print("training xgboost...") print(client) params = { "objective": "binary:logistic", "learning_rate": args.learning_rate, "gamma": args.gamma, "max_depth": args.max_depth, } mlflow.log_params(params) # log to the run dtrain = xgb.dask.DaskDMatrix(client, X, y) model = xgb.dask.train(client, params, dtrain, num_boost_round=args.num_boost_round) print(model) # predict on test data print("making predictions...") print(client) X_test = df_test[ [col for col in cols if "HasDetections" not in col] ].values.persist() y_pred = xgb.dask.predict(client, model, X_test) y_pred.to_dask_dataframe().to_csv("./outputs/predictions.csv") # save model print("saving model...") print(client) mlflow.xgboost.log_model(model["booster"], "./outputs/model")
def main(args): dask.config.set({"distributed.comm.timeouts.connect": 60}) if args.bin_size is not None: if args.bin_size > args.target_seq_length: raise ValueError( "Please provide a bin size that is <= target-seq-length") if args.target_seq_length % args.bin_size != 0: raise ValueError( "Please provide a bin size that can divide the target " "sequence length.") if args.schedule == 'mpi': from dask_mpi import initialize initialize(local_directory='/tmp/dask-worker-space', nanny=False) client = dask.distributed.Client() else: client = dask.distributed.Client( n_workers=args.local_n_workers, threads_per_worker=args.local_threads_per_worker, ) nltk.download('punkt') if os.path.isfile(args.vocab_file): tokenizer = transformers.BertTokenizerFast(args.vocab_file) else: tokenizer = transformers.BertTokenizerFast.from_pretrained( args.vocab_file) tic = time.perf_counter() pairs = _get_pairs( wikipedia_path=args.wikipedia, books_path=args.books, common_crawl_path=args.common_crawl, wikipedia_lang=args.wikipedia_lang, target_seq_length=args.target_seq_length, short_seq_prob=args.short_seq_prob, blocksize=args.block_size, num_blocks=args.num_blocks, duplicate_factor=args.duplicate_factor, sample_ratio=args.sample_ratio, seed=args.seed, tokenizer=tokenizer, masking=args.masking, masked_lm_ratio=args.masked_lm_ratio, ) args.sink = expand_outdir_and_mkdir(args.sink) _save( pairs, args.sink, output_format=args.output_format, bin_size=args.bin_size, target_seq_length=args.target_seq_length, masking=args.masking, ) print('Running the dask pipeline took {} s'.format(time.perf_counter() - tic))
def main(): # Work out from the environment how many threads to allocate num_threads = int( environ.get('SLURM_CPUS_PER_TASK', environ.get('OMP_NUM_THREADS', 1))) print("num_threads: ", num_threads) # Create the Dask workers initialize(interface='ib0', nthreads=num_threads) # Create the Dask object that will manage the communications client = Client() start = datetime.now() results = run_test(client=client) end = datetime.now() print(f"Time taken: {end - start}") print(results)
def main(): from argparse import ArgumentParser parser = ArgumentParser() #parser.add_argument('min_num', type=int) #parser.add_argument('max_num', type=int) args = parser.parse_args() num_threads = int( environ.get('SLURM_CPUS_PER_TASK', environ.get('OMP_NUM_THREADS', 1))) initialize(interface='ib0', nthreads=num_threads) client = Client() min_num = 10 max_num = 100 start_time = datetime.now() num_primes = sum( client.gather(client.map(slow_is_prime, range(min_num, max_num + 1)))) end_time = datetime.now() print(f'{num_primes} primes between {min_num} and {max_num} ' f'[{end_time - start_time}]')
def start_dask(self): """ Prepare dask to run under MPI. After calling this method only a single process, MPI rank 1 will continue to exeute code """ # using the programmatic dask configuration system # does not seem to work. Presumably the loggers have already # been created by the time we modify the config. Doing it with # env vars seems to work. If the user has already set this then # we use that value. Otherwise we only want error logs key = "DASK_LOGGING__DISTRIBUTED" os.environ[key] = os.environ.get(key, "error") try: import dask import dask_mpi import dask.distributed except ImportError: # pragma: no cover print( "ERROR: Using --mpi option on stages that use dask requires " "dask[distributed] and dask_mpi to be installed." ) raise if self.size < 3: # pragma: no cover raise ValueError( "Dask requires at least three processes. One becomes a scheduler " "process, one is a client that runs the code, and more are required " "as worker processes." ) # This requires my fork until/unless they merge the PR, to allow # us to pass in these two arguments. In vanilla dask-mpi sys.exit # is called at the end of the event loop without returning to us. # After this point only a single process, MPI rank 1, # should continue to exeute code. The others enter an event # loop and return with is_client=False, which we return here # to tell the caller that they should not run everything. is_client = dask_mpi.initialize(comm=self.comm, exit=False) if is_client: # Connect this local process to remote workers. self.dask_client = dask.distributed.Client() # I don't yet know how to see this dashboard link at nersc print(f"Started dask. Diagnostics at {self.dask_client.dashboard_link}") return is_client
#!/usr/bin/env python3 import dask.array as da from dask_mpi import initialize from dask.distributed import Client, performance_report initialize() client = Client() def example_function(): x = da.random.random((100_000, 100_000, 10), chunks=(10_000, 10_000, 5)) y = da.random.random((100_000, 100_000, 10), chunks=(10_000, 10_000, 5)) z = (da.arcsin(x) + da.arccos(y)).sum(axis=(1, 2)) with performance_report(filename="dask-report_mpi.html"): result = z.compute() if __name__ == "__main__": example_function()
def __init__(self, initialization_args: Dict = None, client_args: Dict = None): """ Creating a scheduler initializes dask MPI :param initialization_args: dictionary of keyword arguments for dask-mpi initializer. :param client_args: dictionary of keyword arguments for dask client. dask-mpi arguments ------------------ interface:str Network interface like ‘eth0’ or ‘ib0’ nthreads:int Number of threads per worker local_directory:str Directory to place worker files memory_limit:int, float, or ‘auto’ Number of bytes before spilling data to disk. This can be an integer (nbytes), float (fraction of total memory), or ‘auto’. nanny:bool Start workers in nanny process for management bokeh:bool Enable Bokeh visual diagnostics bokeh_port:int Bokeh port for visual diagnostics bokeh_prefix:str Prefix for the bokeh app bokeh_worker_port:int Worker’s Bokeh port for visual diagnostics dask client arguments --------------------- address: string, or Cluster This can be the address of a Scheduler server like a string '127.0.0.1:8786' or a cluster object like LocalCluster() timeout: int Timeout duration for initial connection to the scheduler set_as_default: bool (True) Claim this scheduler as the global dask scheduler scheduler_file: string (optional) Path to a file with scheduler information if available security: Security or bool, optional Optional security information. If creating a local cluster can also pass in True, in which case temporary self-signed credentials will be created automatically. asynchronous: bool (False by default) Set to True if using this client within async/await functions or within Tornado gen.coroutines. Otherwise this should remain False for normal use. name: string (optional) Gives the client a name that will be included in logs generated on the scheduler for matters relating to this client direct_to_workers: bool (optional) Whether or not to connect directly to the workers, or to ask the scheduler to serve as intermediary. heartbeat_interval: int Time in milliseconds between heartbeats to scheduler """ if initialization_args is None: initialization_args = {} if client_args is None: client_args = {} from dask_mpi import initialize initialize(**initialization_args) # Rank 0 is initialized with scheduler # Rank 1 will pass through and execute following code # Rank 2+ will execute workers self._client = Client(**client_args)
'hits based on LCA. This is a rough estimate and ' 'should be revised carefully') parser.add_argument('-o', '--out_filename', default='hit.hits', type=str, help='name of output (filtered) file') parser.add_argument('-f', '--outfmt', default=cols, type=str, help='Custom format for BLAST') parser.add_argument('-u', '--unique', default=False, action='store_true', help='Process only unique sequences') parser.add_argument('-v', '--version', action='version', version='%(prog)s {}'.format(version)) print('\njustblast version:', colored(version, None, attrs=["bold", "blink"])) print(colored('Copyright 2019', 'red', attrs=["bold"]), 'Jose Sergio Hleap\n') args = parser.parse_args() try: size = os.environ["OMPI_COMM_WORLD_SIZE"] print(f'Detecting MPI world... Initializing {size} workers') dask_mpi.initialize() client = Client() except KeyError: print(f'Creating a local cluster with {args.cpus} cpus') client = Client(n_workers=args.cpus, threads_per_worker=1, processes=False) print(client) main(db=args.db, query=args.query, evalue=args.evalue, p_id=args.percent_id, max_targets=args.max_target_seqs, cpus=args.cpus, qcovs=args.query_coverage, identify=args.identify, outfile=args.out_filename, outfmt=args.outfmt, unique=args.unique)
#!/usr/bin/env python #import psutil #net_if_addrs = psutil.net_if_addrs() #allowed_ifnames = list(net_if_addrs.keys()) #print('allowed interfaces are :',allowed_ifnames) from dask_mpi import initialize initialize(interface='ib0:0') from distributed import Client client = Client() print('Currently working with '+str(len(client.scheduler_info()["workers"]))+' workers')
import os import sys import timeit from dask.distributed import Client from dask_mpi import initialize # initialize(nthreads=1, local_directory=os.environ["TMPDIR"] + "/dask_workers", memory_limit=100e9) initialize(nthreads=1, local_directory="/dev/shm/dask_workers", memory_limit=100e9) def setup(inputs): N = int(sys.argv[1]) block_size = sys.argv[2] if block_size == "full": block_size = str(N) elif block_size == "auto": block_size = "'auto'" return N, block_size if __name__ == '__main__': client = Client() N, block_size = setup(sys.argv[1:]) print( timeit.timeit(
def __init__( self, num_workers=None, threads_per_worker=1, launch=None, out=None, err=None ): """ Returns a Client connected to a cluster of `num_workers` workers. """ self._server = None if launch is None: launch = default_comm().size == 1 # pylint: disable=import-outside-toplevel, if not launch: # Then the script has been submitted in parallel with mpirun num_workers = num_workers or default_comm().size - 2 if num_workers < 0 or default_comm().size != num_workers + 2: raise RuntimeError( f""" Error: (num_workers + 2) processes required. The script has not been submitted on enough processes. Got {default_comm().size} processes instead of {num_workers + 2}. """ ) initialize( nthreads=threads_per_worker, local_directory=tempfile.mkdtemp(), nanny=False, ) super().__init__() else: num_workers = num_workers or (multiprocessing.cpu_count() + 1) # Since dask-mpi produces several file we create a temporary directory self._dir = tempfile.mkdtemp() # The command runs in the background (_bg=True) # and the stdout(err) is stored in self._out(err) pwd = os.getcwd() sh.cd(self._dir) self._server = sh.mpirun( "-n", num_workers + 1, "dask-mpi", "--no-nanny", "--nthreads", threads_per_worker, "--scheduler-file", "scheduler.json", _bg=True, _out=out or sys.stdout, _err=err or sys.stderr, ) sh.cd(pwd) atexit.register(self.close_server) super().__init__(scheduler_file=self._dir + "/scheduler.json") # Waiting for all the workers to connect def handler(signum, frame): if self.server is not None: self.close_server() raise RuntimeError( "Couldn't connect to %d processes. Got %d workers." % (num_workers, len(self.workers)) ) signal.signal(signal.SIGALRM, handler) signal.alarm(5) while len(self.workers) != num_workers: time.sleep(0.001) signal.alarm(0) self.ranks = {key: val["name"] for key, val in self.workers.items()} self._comm = self.create_comm()
#qsub mpi.qsub def hello_world(n): import time time.sleep(1) return "Hello world "+str(n) if __name__ == '__main__': from dask.distributed import Client, LocalCluster from dask_mpi import initialize import time import numpy as np initialize(local_directory='/scratch/users/sapatha2/', nanny=False, dashboard=False) client = Client() res = client.map(hello_world, np.arange(380)) start = time.time() for r in res: print (r.result()) client.close() print(time.time() - start) exit(0)
import time import h5py import shutil import argparse import itertools from mpi4py import MPI from copy import deepcopy import random import numpy as np import dask from dask_mpi import initialize # with dask.config.set({"distributed.worker.resources.GPU": 1}): initialize(nthreads=1, memory_limit='32 GB', interface='ipogif0', local_directory=os.getcwd()) import dask import dask.array from dask.distributed import Client, wait from svreg.archive import Archive, md5Hash from svreg.settings import Settings from svreg.database import SVDatabase from svreg.regressor import SVRegressor from svreg.evaluator import SVEvaluator from svreg.population import Population from svreg.functions import _function_map from svreg.tree import MultiComponentTree as MCTree
from distributed import Client from mpi4py.MPI import COMM_WORLD as world from dask_mpi import initialize, send_close_signal # Split our MPI world into two pieces, one consisting just of # the old rank 3 process and the other with everything else new_comm_assignment = 1 if world.rank == 3 else 0 comm = world.Split(new_comm_assignment) if world.rank != 3: # run tests with rest of comm is_client = initialize(comm=comm, exit=False) if is_client: with Client() as c: c.submit(lambda x: x + 1, 10).result() == 11 c.submit(lambda x: x + 1, 20).result() == 21 send_close_signal() # check that our original comm is intact world.Barrier() x = 100 if world.rank == 0 else 200 x = world.bcast(x) assert x == 100