class MPICommEvaluator(Evaluator): """This evaluator uses the ``mpi4py`` library as backend. This evaluator consider an already existing MPI-context (with running processes), therefore it has less overhead than ``MPIPoolEvaluator`` which spawn processes dynamically. Args: run_function (callable): functions to be executed by the ``Evaluator``. num_workers (int, optional): Number of parallel Ray-workers used to compute the ``run_function``. Defaults to ``None`` which consider 1 rank as a worker (minus the master rank). callbacks (list, optional): A list of callbacks to trigger custom actions at the creation or completion of jobs. Defaults to ``None``. run_function_kwargs (dict, optional): Keyword-arguments to pass to the ``run_function``. Defaults to ``None``. comm (optional): A MPI communicator, if ``None`` it will use ``MPI.COMM_WORLD``. Defaults to ``None``. """ def __init__( self, run_function, num_workers: int = None, callbacks=None, run_function_kwargs=None, comm=None, ): super().__init__(run_function, num_workers, callbacks, run_function_kwargs) if not MPI.Is_initialized(): MPI.Init_thread() self.comm = comm if comm else MPI.COMM_WORLD self.num_workers = self.comm.Get_size() - 1 # 1 rank is the master self.sem = asyncio.Semaphore(self.num_workers) logging.info( f"Creating MPICommExecutor with {self.num_workers} max_workers...") self.executor = MPICommExecutor(comm=self.comm, root=0) self.master_executor = None logging.info("Creation of MPICommExecutor done") def __enter__(self): self.master_executor = self.executor.__enter__() if self.master_executor is not None: return self else: return None def __exit__(self, type, value, traceback): self.executor.__exit__(type, value, traceback) self.master_executor = None async def execute(self, job): async with self.sem: run_function = functools.partial(job.run_function, job.config, **self.run_function_kwargs) sol = await self.loop.run_in_executor(self.master_executor, run_function) job.result = sol return job
def test_work(marty, nruns, maxq, done, started): work = {} with MPICommExecutor(comm, root=0) as executor: if executor is None: return # worker process while done < nruns: if len(marty) < maxq and started < nruns: # our input is just a random word, could be a list of inputs or directories x = ranword() # put the work in the queue marty[x] = executor.submit(task, x, started) started = started + 1 print("added work ", done, started, x) # check our list for done work for key in list(marty): f = marty[key] if f.done(): # print result and remove it from the dictionary done = done + 1 dt = str(f.result()) dt = dt.split() if dt[0] in work: work[dt[0]] = float(dt[1]) + work[dt[0]] else: work[dt[0]] = float(dt[1]) print(f.result(), started, done, " myid ", myid) del marty[key] return (work) executor.shutdown(False)
def query(network_file, params_file="", resultsdir=""): ''' :param network_file: a .txt file containing either a single DSGRN network specification or a list of network specification strings in DSGRN format :param params_file: A json file with the key "count" = True or False (true or false in .json format) whether or not to return the number of matches (True) or just whether or not there is at least one match (False) "datetime" : optional datetime string to append to subdirectories in resultsdir, default = system time :param resultsdir: optional path to directory where results will be written, default is current directory :return: Writes a .json file containing a dictionary keyed by DSGRN network specification with a list of results. The results are DSGRN parameter count that have at least one Morse set that is a stable full cycle, or True (existence of at least one stable full cycle) or False (none exist), depending on the value of the parameter "count". The size of the DSGRN parameter graph for the network is also recorded. { networkspec : [result, DSGRN param graph size] }. ''' networks = read_networks(network_file) params = json.load(open(params_file)) datetime = None if "datetime" not in params else params["datetime"] if not networks: raise ValueError( "No networks available for analysis. Make sure network file is in the correct format." ) else: count = sanity_check(params) work_function = partial(search_over_networks, count, len(networks)) with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is not None: print("Querying networks.") output = list(executor.map(work_function, enumerate(networks))) results = dict(output) record_results(network_file, params_file, results, resultsdir, datetime)
def main(): global gpg with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is not None: bar = progressbar.ProgressBar(max_value=gpg.size()) print("Computing Morse Graphs") results = list( bar(executor.map(work, range(0, gpg.size()), chunksize=65536))) SaveDatabase(outfile, results, gpg)
def test_mandelbrot(): with MPICommExecutor() as executor: if executor is None: return # worker process tic = time.time() image = list(executor.map(mandelbrot_line, range(h), chunksize=10)) toc = time.time() print("%s Set %dx%d in %.2f seconds." % ('Mandelbrot', w, h, toc - tic)) if len(sys.argv) > 1 and sys.argv[1] == '-plot': plot(image)
def test_julia_comm(): with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is None: # worker process return print("Loaded Executor") tic = time.time() image = list(executor.map(julia_line, range(h), chunksize=10)) toc = time.time() print("%s - %s Set %dx%d in %.2f seconds." % (gethostname(), 'Julia', w, h, toc - tic))
def query(network_file, params_file, resultsdir=""): ''' Take the intersection of an arbitrary number of DSGRN fixed points in a list. :param network_file: a .txt file containing either a single DSGRN network specification or a list of network specification strings in DSGRN format :param params_file: A json file with a dictionary containing the keys "included_bounds", "excluded_bounds", and "count". The two "bounds" variables are each a list of dictionaries of variable names common to all network specifications with an associated integer range. Example: [{"X1":[2,2],"X2":[1,1],"X3":[0,1]},{"X1":[0,1],"X2":[1,1],"X3":[2,3]}] The integer ranges are the matching conditions for an FP. For example, if there are four variables X1, X2, X3, X4 in the network spec, the FP (2,1,0,*) would be a match to the first fixed point for any value of *. The "included_bounds" are those fixed points that must be present and the "excluded_bounds" are those that must be absent. Either one or both may be empty. When they are both empty, the count is just the number of parameters with at least one fixed point. "count" : True or False (true or false in .json format); whether to count all parameters with a match or shortcut at first success "datetime" : optional datetime string to append to subdirectories in resultsdir, default = system time "neighbors" : optional True or False (true or false in .json format) stating whether to query DSGRN parameters that neighbor essential DSGRN parameters, neighbor-checking is computationally expensive, default = False :param resultsdir: optional path to directory where results will be written, default is current directory :return: Writes a .json file containing a dictionary keyed by DSGRN network specification with a list of results. The results are DSGRN parameter count with successful matches to the fixed point bounds, or True (existence of at least one match) or False (no matches exist), depending on the value of the parameter "count". The size of the DSGRN parameter graph for the network is also recorded. { networkspec : [result, DSGRN param graph size] }. If "hex_constraints" are specified, then the number of DSGRN parameters that satsify those hex constraints (with or without a bounds match) are also recorded. { networkspec : [result, num params with hex constraints, DSGRN param graph size] }. ''' networks = read_networks(network_file) params = json.load(open(params_file)) datetime = None if "datetime" not in params else params["datetime"] sanity_check(params) if not networks: raise ValueError( "No networks available for analysis. Make sure network file is in the correct format." ) else: work_function = partial(search_over_networks, params, len(networks)) with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is not None: print("Querying networks.") output = list(executor.map(work_function, enumerate(networks))) results = dict(output) record_results(network_file, params_file, results, resultsdir, datetime)
def main(): init_cells = Isolation().actions() with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is None: return # worker process results = list(executor.map(test_cell_genomes, init_cells, chunksize=1)) print("Main proc done...") sys.stdout.flush() pickle.dump(results, open("./" + dir_name + "/" + "summary.pickle", "wb"))
def __init__( self, run_function, num_workers: int = None, callbacks=None, run_function_kwargs=None, comm=None, ): super().__init__(run_function, num_workers, callbacks, run_function_kwargs) if not MPI.Is_initialized(): MPI.Init_thread() self.comm = comm if comm else MPI.COMM_WORLD self.num_workers = self.comm.Get_size() - 1 # 1 rank is the master self.sem = asyncio.Semaphore(self.num_workers) logging.info( f"Creating MPICommExecutor with {self.num_workers} max_workers...") self.executor = MPICommExecutor(comm=self.comm, root=0) self.master_executor = None logging.info("Creation of MPICommExecutor done")
def getpot(self, cds, timeit=False): split_cds = np.array_split(cds, self.num_mpi) if timeit: start = time.time() with MPICommExecutor() as executor: result = list(executor.map(self.initwrapper, split_cds)) v = np.concatenate(result) if timeit: elapsed = time.time() - start return v, elapsed else: return v
def mpi_main(args): DEBUG_FLAG = args.debug ea = EA(1) ea.load_config(args.config) reseedPeriod = int(args.reseed) taskNum = int(args.task_num) np.random.seed(1) seed = np.random.randint(0, 2**32 - 1, size=(taskNum), dtype=np.uint32) seed = seed.tolist() print(seed) for i in range(int(args.generation)): if ((reseedPeriod > 0) and (i % reseedPeriod == 0)): for j in range(taskNum): seed[j] = random.randint(0, 2**32 - 1) ea_time = time.time() pop = ea.ask() ea_time = time.time() - ea_time fitnesses = [] workloads = [] num_workers = int(args.num_workers) - 1 gc.collect() prep_time = time.time() for j in range(len(pop)): workloads.append((pop[j], args.task, seed, args.debug)) prep_time = time.time() - prep_time eval_time = time.time() success = False while (success is False): try: with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is not None: results = executor.map(eval, workloads) success = True except OverflowError: success = False eval_time = time.time() - eval_time reducedResult = EvalSummary() reducedResult.reduce(results, 'pfit') ea.tell(reducedResult, args.task, seed) ea.write_history(args.output) #print(ea.fitnesses) print( 'iter: {0} fit: {1}, pfit:{7} Q: {2}, ea_time: {3}, prep_time: {4}, eval_time: {5}, max_depth:{6}' .format(i, ea.fitnesses[0], np.mean(reducedResult.get_res('Q')[0]), ea_time, prep_time, eval_time, ea.pop[0].maxDepth, np.mean(reducedResult.get_res('pfit')[0])))
def main(): num_workers = size - 1 print('Rank: %i sees main' % (rank)) for i in range(3): with MPICommExecutor(comm, root=0) as executor: start_time = time.time() tasks = list(range(num_workers)) future_list = executor.map(do_work, tasks) do_work(0) returned_ranks = [] for result in future_list: returned_ranks.append(result) used_workers = len(set(returned_ranks)) print('Map: %i used %i/%i unique workers and took %.4f s' % \ (i, used_workers, num_workers, time.time() - start_time)) if used_workers != num_workers: pprint.pprint(returned_ranks)
def main(): # Initialize the command line interface on every MPI rank. # This clobbers up the output a little bit but saves us organizing this more. parser = isle.cli.makeDefaultParser( description="Tune integrator for multiple ensembles") parser.add_argument("--overwrite", action="store_true", help="Overwrite existing output file.") clArgs = isle.initialize(parser) with MPICommExecutor() as executor: if executor is not None: # This code runs only on the master rank. # Submit jobs for worker threads with all combinations of parameters. # There is no need to iterate through tasks in order as tuneForEnsemble does not # return anything, so use unordere=True to detect exceptions as early as possible below. results = executor.starmap(tuneForEnsemble, iterParams(clArgs), unordered=True) # Iterate through results to trigger any exceptions caught by the executor. for _ in results: pass
def backend_pool(backend, n_workers=None, verbose=100, **kwargs): """Context manager to build a schwimmbad `pool` object with the `map` method. Parameters ---------- backend : str One of 'condor', 'loky', or 'mpi'. n_workers : int, optional The number of workers to use. Defaults to 1 for the 'sequential' backend, the cpu count for the 'loky' backend, and the size of the default global communicator for the 'mpi' backend. **kwargs : extra keyword arguments These are passed to ParslCondorPool. """ local_env_overrides = { "OMP_NUM_THREADS": "1", "OPENBLAS_NUM_THREADS": "1", "MKL_NUM_THREADS": "1", "VECLIB_MAXIMUM_THREADS": "1", "NUMEXPR_NUM_THREADS": "1", } if backend == "condor": with CondorExecutor(verbose=verbose, max_workers=n_workers, **kwargs) as pool: yield pool elif backend == "loky": _n_workers = get_n_workers(backend, n_workers=n_workers) yield loky.get_reusable_executor( max_workers=_n_workers, env=local_env_overrides, ) elif backend == "mpi": from mpi4py.futures import MPICommExecutor with MPICommExecutor(env=local_env_overrides) as pool: yield pool else: raise RuntimeError("backend '%s' not recognized!" % backend)
def main(): #init_cells = Isolation().actions() init_cells = [94, 43, 14, 62, 23, 56, 92, 106, 70, 57] tasks = [] for cell in init_cells: res = pickle.load( open("./" + dir_name + "/" + str(cell) + "res.pickle", "rb")) gen_results = list(map(lambda x: x[1], res)) final_genomes = list(map(lambda x: x[0], gen_results[-1])) tasks.append((cell, final_genomes)) #idx = np.random.choice(len(final_genomes), 500) #reduced_genomes = [final_genomes[i] for i in idx] #tasks.append((cell, reduced_genomes)) with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is None: return # worker process results = list(executor.map(test_cell_genomes, tasks, chunksize=1)) print("Main proc done...") sys.stdout.flush() pickle.dump(results, open("./" + dir_name + "/" + "summary.pickle", "wb"))
def futures(sim, comm=MPI.COMM_WORLD, root=0, chunksize=1): from mpi4py.futures import MPICommExecutor # @timecall def simulate(A: None, sources, params, samples=1, return_stats=True): """Simulate tweets starting from sources, return mean retweets and retweet probability.""" sample_calls = [([source], params, samples, return_stats) for source in sources] results = executor.map(worker, sample_calls, chunksize=chunksize) if return_stats: return stats_from_futures(results) return results rank = comm.Get_rank() A = None if rank == 0: A = sim.A size = comm.Get_size() assert root == 0 global global_A assert global_A is None global_A = bcast_csr_matrix(A, comm) with MPICommExecutor(comm=comm, root=root) as executor: if executor is None: yield None else: old_simulator = sim.simulator sim.simulator = simulate yield sim sim.simulator = old_simulator global_A = None
def tree_merge_analysis(base_path, **kw): """ Merge distributed analysis sets from a FileHandler. Parameters ---------- base_path : str or pathlib.Path Base path of FileHandler output Other keyword arguments passed to tree_merge_distributed_sets. Notes ----- This function is parallelized over sets, and so can be effectively parallelized up to the number of distributed sets. """ set_path = pathlib.Path(base_path) logger.info("Merging files from {}".format(base_path)) with MPICommExecutor() as executor: if executor is not None: set_paths = get_all_sets(base_path, distributed=True) tree_merge_distributed_sets(set_paths, executor, **kw)
cross_validation_path = os.path.join( work_dir, 'cross_validation_' + ADSORBATE + '_' + TARGET + '_' + str(COVERAGE)) print(cross_validation_path) CV_class = CROSS_VALIDATION(ADSORBATE=ADSORBATE,INCLUDED_BINDING_TYPES=INCLUDED_BINDING_TYPES\ ,cross_validation_path=cross_validation_path, VERBOSE=False) CV_SPLITS = 3 CV_class.generate_test_cv_indices(CV_SPLITS=CV_SPLITS, BINDING_TYPE_FOR_GCN=BINDING_TYPE_FOR_GCN\ , test_fraction=0.25, random_state=x, read_file=False, write_file=False) properties_dictionary = {'batch_size':batch_size, 'learning_rate_init':learning_rate\ , 'epsilon':epsilon,'hidden_layer_sizes':hidden_layers,'regularization':L1orL2\ ,'alpha':alpha, 'epochs_per_training_set':epochs,'training_sets':training_sets,'loss': 'wasserstein_loss'} CV_class.set_model_parameters(TARGET=TARGET, COVERAGE=COVERAGE\ , MAX_COVERAGES = MAX_COVERAGES, NN_PROPERTIES=properties_dictionary\ , NUM_TRAIN=NUM_TRAIN, NUM_VAL=10000, NUM_TEST=10000\ , MIN_GCN_PER_LABEL=12, NUM_GCN_LABELS=10, GCN_ALL = GCN_ALL, TRAINING_ERROR = TRAINING_ERROR\ , LOW_FREQUENCY=200, HIGH_FREQUENCY=HIGH_FREQUENCY, ENERGY_POINTS=ENERGY_POINTS) CV_RESULTS_FILE = ADSORBATE + '_' + TARGET + '_' + str( COVERAGE) + '_' + run_number CV_class.run_CV_multiprocess(write_file=True, CV_RESULTS_FILE=CV_RESULTS_FILE, num_procs=CV_SPLITS + 1) #CV_class.run_CV(write_file=True, CV_RESULTS_FILE = CV_RESULTS_FILE) if __name__ == "__main__": with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: jobs = range(95) if executor is not None: executor.map(fun, jobs)
# distribute to other workers (non-master MPI workers) with mpi4py's MPICommExecutor. def dispatch(): while True: channel_data, cfg, tstep, trange = dq.get() logging.info( f"\tDispatcher: read data: tstep = {tstep}, rank = {rank}") if channel_data is None: break future = executor.submit(perform_analysis, channel_data, cfg, tstep, trange) dq.task_done() # Main if __name__ == "__main__": with MPICommExecutor(comm, root=0) as executor: if executor is not None: # Only master will execute the following block # Use of "__main__" is critical # The master thread will keep reading data, while # a helper thread (dispatcher) will dispatch jobs in the queue (dq) asynchronously # and distribute jobs to other workers. # The main idea is not to slow down the master. dq = queue.Queue() dispatcher = threading.Thread(target=dispatch) dispatcher.start() # Only the master thread will open a data stream. # General reader: engine type and params can be changed with the config file if not args.debug:
counter = mp.Value('i', 0) if args.pool == 'process': logging.info(f"Using: ProcessPoolExecutor") pool = ProcessPoolExecutor(max_workers=args.nworkers, initializer=hello, initargs=(counter, )) if args.pool == 'thread': logging.info(f"Using: ThreadPoolExecutor") pool = ThreadPoolExecutor(max_workers=args.nworkers, initializer=hello, initargs=(counter, )) if args.pool == 'mpicomm': logging.info(f"Using: MPICommExecutor") pool = MPICommExecutor(comm) hello_mpi(counter) if args.pool == 'mpipool': logging.info(f"Using: MPIPoolExecutor") ## Note: max_workers will be overrite when -m mpi4py.future given in the command line ## Note: max_workers includes the master (rank 0) process too (others not.) pool = MPIPoolExecutor(max_workers=args.nworkers) hello_mpi(counter) with pool as executor: if executor is not None: # Only master will execute the following block # Use of "__main__" is critical logging.info( f"\tMaster: init. rank={rank} pid={os.getpid()} hostname={hostname} ID={pidmap[os.getpid()]}"
def query(network_file, params_file, resultsdir=""): ''' For each epsilon in a list of epsilons, a partially ordered set (poset) of maxima and minima of time series data is created or accessed from the params dictionary. This poset is matched against the domain graph for each DSGRN parameter for each network in a list of networks. The result is the count of the number of DSGRN parameters with a match OR is True if there is at least one match and False if not, depending on the choice of the parameter "count". :param network_file: a .txt file containing either a single DSGRN network specification or a list of network specification strings in DSGRN format :param params_file: A .json file containing a dictionary with the keys "domain" : True or False (true or false in .json format), whether or not to perform a path search anywhere in the domain graph "stablefc" : True or False (true or false in .json format), whether or not to perform a path search within stable full cycles Both "domain" and "stablefc" are allowed to be True. "count" : True or False (true or false in .json format), whether to count all DSGRN parameters or shortcut at first success "datetime" : optional datetime string to append to subdirectories in resultsdir, default = system time One can either specify posets directly, or extract posets from timeseries data. Include EITHER the three keys "timeseriesfname" : path to a file or list of files containing the time series data from which to make posets "tsfile_is_row_format" : True if the time series file is in row format (times are in the first row); False if in column format (times are in the first column) "epsilons" : list of floats 0 <= x <= 0.5, one poset will be made for each x Note that an epsilon of 0.10 means that the noise level is considered to be +/- 10% of the distance between global maximum and global minimum for each time series. Thus all information on curve shape is lost at epsilon = 0.5. It is recommended to stay below that level. OR the single key "posets" : a (quoted) dictionary of Python tuples of node names keying a list of tuples of epsilon with a DSGRN formatted poset: '{ ("A","B","C") : [(eps11,poset11), (eps21,poset21),...], ("A","B","D") : [(eps12,poset12), (eps22, poset22),...] }' (the quotes are to handle difficulties with the json format) This key takes specialized information and is not likely to be specified in general usage. :param resultsdir: optional path to directory where results will be written, default is current directory :return: Writes a .json file containing a dictionary keyed by DSGRN network specification with a list of results. The results are DSGRN parameter count with successful matches, or True (existence of at least one match) or False (no pattern matches exist), depending on the value of the parameter "count". The size of the DSGRN parameter graph for the network and the value of 'epsilon' for the attempted match to the time series are also recorded. { networkspec : [(eps, result, DSGRN param graph size)] }. When "stablefc" and "count" are both True, the number of stable full cycles (with or without a pattern match) is also recorded: { networkspec : [(eps, result, num stable full cycles, DSGRN param graph size)] } Separate files will be written for "domain", "stablefc", and each timeseries file name. When multiple time series are specified and "count" is True, a file with the string "all" in the name will be written with the aggregated results across time series datasets (this eliminates the double-counting of DSGRN parameters that can occur across multiple time series). The file ending in "all" records the number of DSGRN parameters with a match to at least one time series dataset. ''' networks = read_networks(network_file) params = json.load(open(params_file)) sanity_check(params) posets, networks = get_posets(networks, params) if not networks: print( "No networks available for analysis. Make sure network file is in the correct format\nand make sure that every network node name is the time series data or 'poset' value." ) return None else: work_function = partial(search_over_networks, params, posets, len(networks)) with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is not None: print("Querying networks.") output = list(executor.map(work_function, enumerate(networks))) results = dict(output) record_results(network_file, params_file, results, resultsdir, params)
def main(): # Parse command line arguments and read configuration file parser = argparse.ArgumentParser( description="Receive data and dispatch analysis tasks to a mpi queue") parser.add_argument('--config', type=str, help='Lists the configuration file', default='configs/config_null.json') parser.add_argument('--benchmark', action="store_true") args = parser.parse_args() global cfg with open(args.config, "r") as df: cfg = json.load(df) df.close() # Load logger configuration from file: # http://zetcode.com/python/logging/ with open("configs/logger.yaml", "r") as f: log_cfg = yaml.safe_load(f.read()) logging.config.dictConfig(log_cfg) comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() # Create a global executor #executor = concurrent.futures.ThreadPoolExecutor(max_workers=60) #executor = MPIPoolExecutor(max_workers=24) adios2_varname = channel_range.from_str( cfg["transport"]["channel_range"][0]) with MPICommExecutor(MPI.COMM_WORLD) as executor: if executor is not None: logger = logging.getLogger('simple') cfg["run_id"] = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) cfg["run_id"] = "ABC125" cfg["storage"]["run_id"] = cfg["run_id"] logger.info(f"Starting run {cfg['run_id']}") # Instantiate a storage backend and store the run configuration and task configuration if cfg['storage']['backend'] == "numpy": store_backend = backends.backend_numpy(cfg['storage']) elif cfg['storage']['backend'] == "mongo": store_backend = backends.backend_mongodb(cfg) elif cfg['storage']['backend'] == "null": store_backend = backends.backend_null(cfg['storage']) logger.info("Storing one") store_backend.store_one({ "run_id": cfg['run_id'], "run_config": cfg }) logger.info("Done storing. Continuing:") # Create the FFT task cfg["fft_params"]["fsample"] = cfg["ECEI_cfg"]["SampleRate"] * 1e3 my_fft = task_fft_scipy(10_000, cfg["fft_params"], normalize=True, detrend=True) fft_params = my_fft.get_fft_params() # Create ADIOS reader object reader = reader_gen(cfg["transport"]) # Create a list of individual spectral tasks #task_list = [] #for task_config in cfg["task_list"]: # #task_list.append(task_spectral(task_config, fft_params, cfg["ECEI_cfg"])) # #task_list.append(task_spectral(task_config, cfg["fft_params"], cfg["ECEI_cfg"], cfg["storage"])) # #store_backend.store_metadata(task_config, task_list[-1].get_dispatch_sequence()) dq = queue.Queue() msg = None tic_main = timeit.default_timer() workers = [] for _ in range(16): #thr = ConsumeThread(dq, executor, task_list, cfg) worker = threading.Thread(target=consume, args=(dq, executor, my_fft, task_list)) worker.start() workers.append(worker) # logger.info(f"Started thread {thr}") # reader.Open() is blocking until it opens the data file or receives the # data stream. Put this right before entering the main loop logger.info(f"{rank} Waiting for generator") reader.Open() last_step = 0 logger.info(f"Starting main loop") rx_list = [] while True: stepStatus = reader.BeginStep() logger.info(f"currentStep = {reader.CurrentStep()}") if stepStatus: # Read data stream_data = reader.Get(adios2_varname, save=False) rx_list.append(reader.CurrentStep()) # Generate message id and publish is msg = AdiosMessage(tstep_idx=reader.CurrentStep(), data=stream_data) dq.put_nowait(msg) logger.info(f"Published message {msg}") reader.EndStep() else: logger.info(f"Exiting: StepStatus={stepStatus}") break #Early stopping for debug if reader.CurrentStep() > 100: logger.info( f"Exiting: CurrentStep={reader.CurrentStep()}, StepStatus={stepStatus}" ) dq.put(AdiosMessage(tstep_idx=None, data=None)) break last_step = reader.CurrentStep() dq.join() logger.info("Queue joined") logger.info("Exiting main loop") for thr in workers: thr.join() logger.info("Workers have joined") # Shotdown the executioner executor.shutdown(wait=True) toc_main = timeit.default_timer() logger.info( f"Run {cfg['run_id']} finished in {(toc_main - tic_main):6.4f}s" ) logger.info(f"Processed time_chunks {rx_list}")
def main(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( "num_work_groups", type=int, help= "Number of work groups. The total number of ranks must be greater or equal to 2*num_work_groups+1, so that there are at least 2 ranks per group, and one queue manager", ) args = parser.parse_args() # Commands for initializing the Lammps instance # Could also read a restart file # TODO: remove hardcoding initialization = dedent("""\ atom_style atomic units metal boundary p p p lattice fcc 3.615 region box_region block 0 3 0 3 0 3 create_box 1 box_region create_atoms 1 box pair_style eam/alloy pair_coeff * * Cu_mishin1.eam.alloy Cu """) # Make a list of work for the worker groups Each entry of # the list is a tuple, and the first element must be 'True'. # The manager of each group can later tell the workers to # stop waiting for work by broadcasting (False, None). # TODO: remove hardcoding work_list = [(True, (1, i)) for i in range(2, 10)] if MPI.COMM_WORLD.size < args.num_work_groups + 1: message = ( f"number of ranks in MPI.COMM_WORLD is {MPI.COMM_WORLD.size}. Need at least {args.num_work_groups+1} " f"ranks for {args.num_work_groups} worker groups and one manager") raise ValueError(message) group = (MPI.COMM_WORLD.rank - 1) % args.num_work_groups rank_in_group = (MPI.COMM_WORLD.rank - 1) // args.num_work_groups # Setup communicators for the workers if (0 < MPI.COMM_WORLD.rank < (MPI.COMM_WORLD.size - ((MPI.COMM_WORLD.size - 1) % args.num_work_groups)) and rank_in_group != 0): MPI.COMM_WGROUP = MPI.COMM_WORLD.Split(group, rank_in_group) MPI.COMM_WGROUP.name = "comm_group_{}".format(group) else: MPI.COMM_WGROUP = MPI.COMM_WORLD.Split(MPI.UNDEFINED, MPI.COMM_WORLD.rank) if (0 < MPI.COMM_WORLD.rank < (MPI.COMM_WORLD.size - ((MPI.COMM_WORLD.size - 1) % args.num_work_groups))): MPI.COMM_GROUP = MPI.COMM_WORLD.Split(group, rank_in_group) MPI.COMM_GROUP.name = "comm_group_{}".format(group) else: MPI.COMM_GROUP = MPI.COMM_WORLD.Split(MPI.UNDEFINED, MPI.COMM_WORLD.rank) # Setup the job communicator, which will be used by MPICommExecutor to assign work # Only one rank per worker group participates if MPI.COMM_WORLD.rank == 0 or (MPI.COMM_GROUP and MPI.COMM_GROUP.rank == 0): MPI.COMM_JOB = MPI.COMM_WORLD.Split(0, MPI.COMM_WORLD.rank) else: MPI.COMM_JOB = MPI.COMM_WORLD.Split(MPI.UNDEFINED, MPI.COMM_WORLD.rank) # Create workers' Lammps instances if MPI.COMM_WORLD.rank != 0 and MPI.COMM_WGROUP: calculator = Calculator(comm=MPI.COMM_WGROUP, identifier=group) for line in initialization.splitlines(): calculator.lammps.command(line) MPI.COMM_WORLD.Barrier() if MPI.COMM_JOB != MPI.COMM_NULL: with MPICommExecutor(MPI.COMM_JOB, root=0, max_workers=args.num_work_groups) as executor: _ = executor.map(_work, work_list) if MPI.COMM_WORLD.rank == 0: pass else: MPI.COMM_GROUP.bcast((False, None), root=0) elif MPI.COMM_WGROUP != MPI.COMM_NULL: still_running = True while still_running: # synchronise the workers with the "main worker" still_running, work = MPI.COMM_GROUP.bcast(None, root=0) if work is not None: m, n = work matrix = calculator.calculate_stiffness_matrix(m, n) print(matrix)
def query(network_file, params_file, resultsdir=""): ''' For each epsilon in a list of epsilons, a partially ordered set (poset) of maxima and minima of time series data is created or accessed from the params dictionary. This poset is matched against the domain graph for each DSGRN parameter for each network in a list of networks. The result is the count of the number of DSGRN parameters with a match OR is True if there is at least one match and False if not, depending on the choice of the parameter "count". :param network_file: a .txt file containing a single DSGRN network specification specification strings in DSGRN format :param params_file: A .json file containing a dictionary with the keys "domain" : True or False (true or false in .json format), whether or not to perform a path search anywhere in the domain graph "stablefc" : True or False (true or false in .json format), whether or not to perform a path search within stable full cycles Both "domain" and "stablefc" are allowed to be True. "count" : True or False (true or false in .json format), whether to count all DSGRN parameters or shortcut at first success NOTE: Only count = True is currently implemented "datetime" : optional datetime string to append to subdirectories in resultsdir, default = system time "parameter_list" : optional sublist of the parameter graph One can either specify posets directly, or extract posets from timeseries data. Include EITHER the three keys "timeseriesfname" : path to a file or list of files containing the time series data from which to make posets "tsfile_is_row_format" : True if the time series file is in row format (times are in the first row); False if in column format (times are in the first column) "epsilons" : list of floats 0 <= x <= 0.5, one poset will be made for each x Note that an epsilon of 0.10 means that the noise level is considered to be +/- 10% of the distance between global maximum and global minimum for each time series. Thus all information on curve shape is lost at epsilon = 0.5. It is recommended to stay below that level. OR the single key "posets" : a (quoted) dictionary of Python tuples of node names keying a list of tuples of epsilon with a DSGRN formatted poset: '{ ("A","B","C") : [(eps11,poset11), (eps21,poset21),...], ("A","B","D") : [(eps12,poset12), (eps22, poset22),...] }' (the quotes are to handle difficulties with the json format) This key takes specialized information and is not likely to be specified in general usage. :param resultsdir: optional path to directory where results will be written, default is current directory :return: Writes a .json file containing a dictionary keyed by DSGRN network specification with a list of results. The results are DSGRN parameter count with successful matches, or True (existence of at least one match) or False (no pattern matches exist), depending on the value of the parameter "count". The size of the DSGRN parameter graph for the network and the value of 'epsilon' for the attempted match to the time series are also recorded. { networkspec : [(eps, result, DSGRN param graph size)] }. When "stablefc" and "count" are both True, the number of stable full cycles (with or without a pattern match) is also recorded: { networkspec : [(eps, result, num stable full cycles, DSGRN param graph size)] } Separate files will be written for "domain", "stablefc", and each timeseries file name. When multiple time series are specified and "count" is True, a file with the string "all" in the name will be written with the aggregated results across time series datasets (this eliminates the double-counting of DSGRN parameters that can occur across multiple time series). The file ending in "all" records the number of DSGRN parameters with a match to at least one time series dataset. ''' spec = read_networks(network_file) print(spec) param_dict = json.load(open(params_file)) sanity_check(param_dict) posets, networks = get_posets(spec, param_dict) print("Querying networks.\n") if not networks: print( "No networks available for analysis. Make sure network file is in the correct format\nand make sure that every network node name is the time series data or 'poset' value." ) return None else: results = {} if param_dict["count"]: with MPICommExecutor() as executor: spec = spec[0] results[spec] = {} network = DSGRN.Network(spec) param_graph = DSGRN.ParameterGraph(network) if "parameter_list" in param_dict: dsgrn_params = [(p, param_graph.parameter(p)) for p in param_dict["parameter_list"]] else: dsgrn_params = [(p, param_graph.parameter(p)) for p in range(param_graph.size())] names = tuple( sorted([network.name(k) for k in range(network.size())])) work_function = partial(PathMatch, network, posets[names], param_dict["domain"], param_dict["stablefc"]) output = dict(executor.map(work_function, dsgrn_params)) results[spec] = reformat_output(output, list(posets[names].keys()), param_dict, param_graph.size()) print("Network {} of {} complete.".format(1, 1)) sys.stdout.flush() record_results(network_file, params_file, results, resultsdir, param_dict) else: raise ValueError( "Existence of path match without counting is not yet implemented for large networks. Use CountPatternMatch.py." )
"early_config": { "pressio:metric": "composite", "composite:plugins": ["time", "size", "error_stat", "external"], "external:config_name": f"{args['compressor_id']}-{args['bound']:1.1e}", "external:command": str(Path(__file__).absolute().parent.parent / "visualize.py") }, # configure the compressor "compressor_config": args['compressor_config'] }) # run compressor to determine metrics decomp_data = input_data.copy() comp_data = compressor.encode(input_data) decomp_data = compressor.decode(comp_data, decomp_data) metrics = compressor.get_metrics() return { "compressor_id": args['compressor_id'], "bound": args['bound'], "metrics": metrics } with MPICommExecutor() as pool: for result in pool.map(run_compressor, configs, unordered=True): pprint(result)
def main(): tasks = Isolation().actions() with MPICommExecutor(MPI.COMM_WORLD, root=0) as executor: if executor is None: return # worker process _ = list(executor.map(book_process, tasks, chunksize=1)) return 1