def run_workers(self): ''' Run allworkers -- parallelized if each sim is not parallelized ''' if self.n_workers == 1: self.worker() else: sc.parallelize(self.worker, self.n_workers) return
def multi_run(orig_sim, n=4, verbose=None): ''' Ditto, for multiple runs ''' # Copy the simulations sims = [] for i in range(n): new_sim = sc.dcp(orig_sim) new_sim.pars['seed'] += i # Reset the seed, otherwise no point! new_sim.pars['n'] = int(new_sim.pars['n'] / n) # Reduce the population size accordingly sims.append(new_sim) finished_sims = sc.parallelize(single_run, iterarg=sims) output_sim = sc.dcp(finished_sims[0]) output_sim.pars['parallelized'] = n # Store how this was parallelized output_sim.pars[ 'n'] *= n # Restore this since used in later calculations -- a bit hacky, it's true for sim in finished_sims[1:]: # Skip the first one output_sim.people.update(sim.people) for key, val in sim.results.items(): if key != 't': output_sim.results[key] += sim.results[key] return output_sim
def evaluate_samples(self): ''' Actually evaluate the objective function -- copied from shell_step.py ''' if not self.parallelize: for s,sample in enumerate(self.samples): self.values[s] = self.func(sample, **self.func_args) # This is the time-consuming step!! else: valueslist = sc.parallelize(self.func, iterarg=self.samples, kwargs=self.func_args, **self.parallel_args) self.values = np.array(valueslist, dtype=float) self.allsamples = np.concatenate([self.allsamples, self.samples]) self.allvalues = np.concatenate([self.allvalues, self.values]) return
def evaluate(self): ''' Actually evaluate the objective function ''' self.results = np.zeros(self.mp.N) if not self.parallelize: for s, sample in enumerate(self.samples): self.results[s] = self.func( sample, **self.func_args) # This is the time-consuming step!! else: resultslist = sc.parallelize(self.func, iterarg=self.samples, kwargs=self.func_args, **self.parallel_args) self.results = np.array(resultslist, dtype=float) self.allresults[self.key] = sc.dcp(self.results) self.fvals[self.iteration, :] = self.results return self.results
# Run in series if run_series: sc.tic() data = [] for noiseval in noisevals: output = randgen(noiseval) data.append(output) sc.toc() # Run in parallel -- manual way if run_manual: sc.tic() multipool = mp.Pool(processes=mp.cpu_count()) data = multipool.map(randgen, noisevals) multipool.close() multipool.join() sc.toc() # Run in parallel -- easy way if run_sciris: sc.tic() data = sc.parallelize(randgen, noisevals) sc.toc() # Create 3D plot if do_plot: sc.surf3d(pl.array(data))
sim = covid_abm.Sim() sim.pars['r_contact'] = args.r sim.pars['incub'] = args.incub loglike = sim.likelihood(verbose=0) output = sc.objdict({'i': args.i, 'j': args.j, 'loglike': loglike}) return output arglist = [] results = pl.zeros((n_r, n_incub)) for i, r in enumerate(r_vec): for j, incub in enumerate(i_vec): args = sc.objdict({'i': i, 'j': j, 'r': r, 'incub': incub}) arglist.append(args) tmp_results = sc.parallelize(run_sim, iterarg=arglist) for tmp in tmp_results: results[tmp.i, tmp.j] = tmp.loglike sc.toc() #%% Plotting pl.figure(figsize=(12, 8)) delta_r = (r_vec[1] - r_vec[0]) / 2 delta_i = (i_vec[1] - i_vec[0]) / 2 plot_r_vec = pl.hstack([r_vec - delta_r, r_vec[-1] + delta_r ]) * 30 * 3 # TODO: estimate better from sim plot_i_vec = pl.hstack([i_vec - delta_i, i_vec[-1] + delta_i]) pl.pcolormesh(plot_i_vec, plot_r_vec, results, cmap=sc.parulacolormap()) # pl.imshow(results) pl.colorbar()
import sciris as sc import pylab as pl torun = ['simple', 'embarrassing', 'multiargs', 'noniterated', 'parallelcmd'] if 'doplot' not in locals(): doplot = True if __name__ == '__main__': #Example 1 -- simple usage as a shortcut to multiprocessing.map(): if 'simple' in torun: def f(x): return x * x results = sc.parallelize(f, [1, 2, 3]) print(results) #Example 2 -- simple usage for "embarrassingly parallel" processing: if 'embarrassing' in torun: def rnd(): import pylab as pl return pl.rand() results = sc.parallelize(rnd, 10) print(results) #Example 3 -- using multiple arguments: if 'multiargs' in torun:
def multi_run(sim, n_runs=4, noise=0.0, noisepar=None, iterpars=None, verbose=None, combine=False, keep_people=None, run_args=None, sim_args=None, **kwargs): ''' For running multiple runs in parallel. If the first argument is a list of sims, exactly these will be run and most other arguments will be ignored. Args: sim (Sim or list): the sim instance to be run, or a list of sims. n_runs (int): the number of parallel runs noise (float): the amount of noise to add to each run noisepar (string): the name of the parameter to add noise to iterpars (dict): any other parameters to iterate over the runs; see sc.parallelize() for syntax verbose (int): detail to print combine (bool): whether or not to combine all results into one sim, rather than return multiple sim objects keep_people (bool): whether or not to keep the people in each sim run_args (dict): arguments passed to sim.run() sim_args (dict): extra parameters to pass to the sim kwargs (dict): also passed to the sim Returns: If combine is True, a single sim object with the combined results from each sim. Otherwise, a list of sim objects (default). **Example**:: import covasim as cv sim = cv.Sim() sims = cv.multi_run(sim, n_runs=6, noise=0.2) ''' # Create the sims if sim_args is None: sim_args = {} # Handle iterpars if iterpars is None: iterpars = {} else: n_runs = None # Reset and get from length of dict instead for key, val in iterpars.items(): new_n = len(val) if n_runs is not None and new_n != n_runs: raise ValueError( f'Each entry in iterpars must have the same length, not {n_runs} and {len(val)}' ) else: n_runs = new_n # Run the sims if isinstance(sim, cvs.Sim): # Normal case: one sim iterkwargs = {'ind': np.arange(n_runs)} iterkwargs.update(iterpars) kwargs = { 'sim': sim, 'noise': noise, 'noisepar': noisepar, 'verbose': verbose, 'keep_people': keep_people, 'sim_args': sim_args, 'run_args': run_args } sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs) elif isinstance(sim, list): # List of sims iterkwargs = {'sim': sim} kwargs = { 'verbose': verbose, 'keep_people': keep_people, 'sim_args': sim_args, 'run_args': run_args } sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs) else: errormsg = f'Must be Sim object or list, not {type(sim)}' raise TypeError(errormsg) # Usual case -- return a list of sims if not combine: return sims # Or, combine them into a single sim with scaled results else: output_sim = sc.dcp(sims[0]) output_sim.parallelized = { 'parallelized': True, 'combined': True, 'n_runs': n_runs } # Store how this was parallelized output_sim['pop_size'] *= n_runs # Record the number of people for s, sim in enumerate(sims[1:]): # Skip the first one if keep_people: output_sim.people += sim.people for key in sim.result_keys(): this_res = sim.results[key] output_sim.results[key].values += this_res.values # For non-count results (scale=False), rescale them for key in output_sim.result_keys(): if not output_sim.results[key].scale: output_sim.results[key].values /= len(sims) return output_sim
def multi_run(sim, n_runs=4, noise=0.0, noisepar=None, iterpars=None, verbose=None, combine=False, run_args=None, sim_args=None, **kwargs): ''' For running multiple runs in parallel. Args: sim (Sim): the sim instance to be run n_runs (int): the number of parallel runs noise (float): the amount of noise to add to each run noisepar (string): the name of the parameter to add noise to iterpars (dict): any other parameters to iterate over the runs; see sc.parallelize() for syntax verbose (int): detail to print combine (bool): whether or not to combine all results into one sim, rather than return multiple sim objects run_args (dict): arguments passed to sim.run() sim_args (dict): extra parameters to pass to the sim kwargs (dict): also passed to the sim Returns: if combine: a single sim object with the combined results from each sim else (default): a list of sim objects Example: import covasim as cv sim = cv.Sim() sims = cv.multi_run(sim, n_runs=6, noise=0.2) ''' # Create the sims if sim_args is None: sim_args = {} # Handle iterpars if iterpars is None: iterpars = {} else: n_runs = None # Reset and get from length of dict instead for key,val in iterpars.items(): new_n = len(val) if n_runs is not None and new_n != n_runs: raise ValueError(f'Each entry in iterpars must have the same length, not {n_runs} and {len(val)}') else: n_runs = new_n # Copy the simulations iterkwargs = {'ind':np.arange(n_runs)} iterkwargs.update(iterpars) kwargs = {'sim':sim, 'noise':noise, 'noisepar':noisepar, 'verbose':verbose, 'sim_args':sim_args, 'run_args':run_args} sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs) # Usual case -- return a list of sims if not combine: return sims # Or, combine them into a single sim with scaled results else: output_sim = sc.dcp(sims[0]) output_sim.pars['parallelized'] = n_runs # Store how this was parallelized output_sim.pars['n'] *= n_runs # Restore this since used in later calculations -- a bit hacky, it's true for s,sim in enumerate(sims[1:]): # Skip the first one output_sim.people.update(sim.people) for key in sim.reskeys: this_res = sim.results[key] output_sim.results[key].values += this_res.values # For non-count results (scale=False), rescale them for key in output_sim.reskeys: if not output_sim.results[key].scale: output_sim.results[key].values /= len(sims) return output_sim
def multi_run(sim, n_runs=4, reseed=True, noise=0.0, noisepar=None, iterpars=None, verbose=None, combine=False, keep_people=None, run_args=None, sim_args=None, par_args=None, **kwargs): ''' For running multiple runs in parallel. If the first argument is a list of sims, exactly these will be run and most other arguments will be ignored. Args: sim (Sim or list): the sim instance to be run, or a list of sims. n_runs (int): the number of parallel runs reseed (bool): whether or not to generate a fresh seed for each run noise (float): the amount of noise to add to each run noisepar (string): the name of the parameter to add noise to iterpars (dict): any other parameters to iterate over the runs; see sc.parallelize() for syntax verbose (int): detail to print combine (bool): whether or not to combine all results into one sim, rather than return multiple sim objects keep_people (bool): whether or not to keep the people in each sim run_args (dict): arguments passed to sim.run() sim_args (dict): extra parameters to pass to the sim par_args (dict): arguments passed to sc.parallelize() kwargs (dict): also passed to the sim Returns: If combine is True, a single sim object with the combined results from each sim. Otherwise, a list of sim objects (default). **Example**:: import covasim as cv sim = cv.Sim() sims = cv.multi_run(sim, n_runs=6, noise=0.2) ''' # Handle inputs sim_args = sc.mergedicts(sim_args, kwargs) # Handle blank par_args = sc.mergedicts(par_args) # Handle blank # Handle iterpars if iterpars is None: iterpars = {} else: n_runs = None # Reset and get from length of dict instead for key, val in iterpars.items(): new_n = len(val) if n_runs is not None and new_n != n_runs: raise ValueError( f'Each entry in iterpars must have the same length, not {n_runs} and {len(val)}' ) else: n_runs = new_n # Run the sims if isinstance(sim, cvs.Sim): # Normal case: one sim iterkwargs = {'ind': np.arange(n_runs)} iterkwargs.update(iterpars) kwargs = dict(sim=sim, reseed=reseed, noise=noise, noisepar=noisepar, verbose=verbose, keep_people=keep_people, sim_args=sim_args, run_args=run_args) sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs, **par_args) elif isinstance(sim, list): # List of sims iterkwargs = {'sim': sim} kwargs = dict(verbose=verbose, keep_people=keep_people, sim_args=sim_args, run_args=run_args) sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs, **par_args) else: errormsg = f'Must be Sim object or list, not {type(sim)}' raise TypeError(errormsg) return sims
if __name__ == '__main__': T = sc.tic() try: sh.rmtree('./progress/', ignore_errors=True) os.makedirs('./progress/', exist_ok=True) except Exception as E: print(f'Could not make progress folder: {E}') if which == 'a': sweeps = construct1dsweeps() ntrials = len(sweeps) sc.heading(f'Beginning run for type "{which}" for {ntrials} trials...') sc.timedsleep(3) sims = sc.parallelize(run_scenario, iterarg=np.arange(ntrials), ncpus=ncpus, kwargs={'which': which}) msim = cv.MultiSim(sims) msim.save(msimfile) if doplot: msim.plot(max_sims=8, to_plot='overview', fig_args={'figsize': (38, 19)}) print('Done.') sc.toc(T)
staff_age_min=staff_age_min, staff_age_max=staff_age_max) sc.toc(T) print('Done') return if __name__ == '__main__': seeds = [0, 1, 2, 3, 4] # parallelize = True parallelize = False if parallelize: ram = psutil.virtual_memory().available / 1e9 required = 80 * len(seeds) # 8 GB per 225e3 people if required < ram: print( f'You have {ram} GB of RAM, and this is estimated to require {required} GB: you should be fine' ) else: print( f'You have {ram:0.2f} GB of RAM, but this is estimated to require {required} GB -- you are in trouble!!!!!!!!!' ) sc.parallelize(cache_populations, iterarg=seeds) # Run them in parallel else: for seed in seeds: cache_populations(seed)
def multi_run(sim, n_runs=4, noise=0.0, noisepar=None, iterpars=None, verbose=None, sim_args=None, combine=False, **kwargs): ''' For running multiple runs in parallel. Example: import covid_seattle sim = covid_seattle.Sim() sims = covid_seattle.multi_run(sim, n_runs=6, noise=0.2) ''' # Create the sims if sim_args is None: sim_args = {} # Handle iterpars if iterpars is None: iterpars = {} else: n_runs = None # Reset and get from length of dict instead for key, val in iterpars.items(): new_n = len(val) if n_runs is not None and new_n != n_runs: raise ValueError( f'Each entry in iterpars must have the same length, not {n_runs} and {len(val)}' ) else: n_runs = new_n # Copy the simulations iterkwargs = {'ind': np.arange(n_runs)} iterkwargs.update(iterpars) kwargs = { 'sim': sim, 'noise': noise, 'noisepar': noisepar, 'verbose': verbose, 'sim_args': sim_args } sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs) # Usual case -- return a list of sims if not combine: return sims # Or, combine them into a single sim with scaled results else: output_sim = sc.dcp(sims[0]) output_sim.pars[ 'parallelized'] = n_runs # Store how this was parallelized output_sim.pars[ 'n'] *= n_runs # Restore this since used in later calculations -- a bit hacky, it's true for s, sim in enumerate(sims[1:]): # Skip the first one output_sim.people.update(sim.people) for key in sim.reskeys: this_res = sim.results[key] output_sim.results[key].values += this_res.values # For non-count results (scale=False), rescale them for key in output_sim.reskeys: if not output_sim.results[key].scale: output_sim.results[key].values /= len(sims) return output_sim
count += 1 meta = sc.objdict() meta.count = count meta.n_sims = n_sims meta.inds = [i_sc, i_fst, i_fte, i_s] meta.vals = sc.objdict(scenario=scenname, future_symp_test=daily_test, future_t_eff=future_t_eff, seed=seed) ikw.append(sc.dcp(meta.vals)) ikw[-1].meta = meta # Actually run the sims kwargs = dict(calibration=False, end_day='2020-10-23') sim_configs = sc.parallelize(make_sim, iterkwargs=ikw, kwargs=kwargs) if do_save: cv.save(filename=sims_file, obj=sim_configs) # Run sims all_sims = sc.parallelize(run_sim, iterarg=sim_configs, kwargs=dict(do_load=do_load, do_save=do_save)) sims = np.empty((n_scenarios, sy_npts, tr_npts, max_seeds), dtype=object) for sim in all_sims: # Unflatten array i_sc, i_fst, i_fte, i_s = sim.meta.inds sims[i_sc, i_fst, i_fte, i_s] = sim
repeats = 10 noisevals = np.linspace(0, 1, 21) x = np.linspace(xmin, xmax, npts) def randgen(std): a = np.cos(x) b = np.random.randn(npts) * std return a + b # Start timing sc.tic() # Create object in parallel output = sc.parallelize(randgen, noisevals) # Save to files filenames = [] for n, noiseval in enumerate(noisevals): filename = 'noise%0.1f.obj' % noiseval sc.saveobj(filename, output[n]) filenames.append(filename) # Create odict from files data = sc.odict() for filename in filenames: data[filename] = sc.loadobj(filename) # Create 3D plot sc.surf3d(data[:])
def run_workers(CurrCtnyParams): # return sc.parallelize(worker, n_workers, kwargs={'CurrCtnyParams':CurrCtnyParams}, ncpus=4) return sc.parallelize(worker, n_workers, kwargs={'CurrCtnyParams': CurrCtnyParams}, ncpus=n_cpus)
def multi_run(sim, n_runs=4, reseed=True, noise=0.0, noisepar=None, iterpars=None, combine=False, keep_people=None, run_args=None, sim_args=None, par_args=None, do_run=True, parallel=True, verbose=None, **kwargs): ''' For running multiple runs in parallel. If the first argument is a list of sims, exactly these will be run and most other arguments will be ignored. Args: sim (Sim) : the sim instance to be run, or a list of sims. n_runs (int) : the number of parallel runs reseed (bool) : whether or not to generate a fresh seed for each run noise (float) : the amount of noise to add to each run noisepar (str) : the name of the parameter to add noise to iterpars (dict) : any other parameters to iterate over the runs; see sc.parallelize() for syntax combine (bool) : whether or not to combine all results into one sim, rather than return multiple sim objects keep_people(bool) : whether to keep the people after the sim run run_args (dict) : arguments passed to sim.run() sim_args (dict) : extra parameters to pass to the sim par_args (dict) : arguments passed to sc.parallelize() do_run (bool) : whether to actually run the sim (if not, just initialize it) parallel (bool) : whether to run in parallel using multiprocessing (else, just run in a loop) verbose (int) : detail to print kwargs (dict) : also passed to the sim Returns: If combine is True, a single sim object with the combined results from each sim. Otherwise, a list of sim objects (default). **Example**:: import covasim as cv sim = cv.Sim() sims = cv.multi_run(sim, n_runs=6, noise=0.2) ''' # Handle inputs sim_args = sc.mergedicts(sim_args, kwargs) # Handle blank par_args = sc.mergedicts(par_args) # Handle blank # Handle iterpars if iterpars is None: iterpars = {} else: n_runs = None # Reset and get from length of dict instead for key, val in iterpars.items(): new_n = len(val) if n_runs is not None and new_n != n_runs: raise ValueError( f'Each entry in iterpars must have the same length, not {n_runs} and {len(val)}' ) else: n_runs = new_n # Run the sims if isinstance(sim, cvs.Sim): # Normal case: one sim iterkwargs = {'ind': np.arange(n_runs)} iterkwargs.update(iterpars) kwargs = dict(sim=sim, reseed=reseed, noise=noise, noisepar=noisepar, verbose=verbose, keep_people=keep_people, sim_args=sim_args, run_args=run_args, do_run=do_run) elif isinstance(sim, list): # List of sims iterkwargs = {'sim': sim} kwargs = dict(verbose=verbose, keep_people=keep_people, sim_args=sim_args, run_args=run_args, do_run=do_run) else: errormsg = f'Must be Sim object or list, not {type(sim)}' raise TypeError(errormsg) # Actually run! if parallel: sims = sc.parallelize(single_run, iterkwargs=iterkwargs, kwargs=kwargs, **par_args) # Run in parallel else: sims = [] n_sims = len(list(iterkwargs.values( ))[0]) # Must have length >=1 and all entries must be the same length for s in range(n_sims): this_iter = {k: v[s] for k, v in iterkwargs.items() } # Pull out items specific to this iteration this_iter.update(kwargs) # Merge with the kwargs this_iter['sim'] = this_iter['sim'].copy( ) # Ensure we have a fresh sim; this happens implicitly on pickling with multiprocessing sim = single_run(**this_iter) # Run in series sims.append(sim) return sims
if __name__ == '__main__': T = sc.tic() indices = [0] ncpus = 1 do_plot = 1 scenpars = None # Settings if ncpus > 1: sims = sc.parallelize(run_sim, iterarg=indices, ncpus=ncpus, kwargs={ 'scenpars': scenpars, 'do_shrink': 0 }) else: sims = [run_sim(index, scenpars=scenpars) for index in indices] msim = cv.MultiSim(sims=sims) if do_plot: get_interv(msim.sims[0], 'contact_tracing' ).do_plot = True # Turn on plotting for this intervention msim.plot(to_plot='overview', fig_args={'figsize': (38, 19)}, plot_sims=True) sc.toc(T)
''' Pre-generate the synthpops population including school types. Takes ~102s per seed ''' import psutil import sciris as sc import covasim_schools as cvsch pop_size = 20e3 seeds = [0, 1, 2, 3, 4] parallelize = True # parallelize = False if parallelize: ram = psutil.virtual_memory().available / 1e9 required = 1 * len(seeds) * pop_size / 225e3 # 8 GB per 225e3 people if required < ram: print( f'You have {ram} GB of RAM, and this is estimated to require {required} GB: you should be fine' ) else: print( f'You have {ram:0.2f} GB of RAM, but this is estimated to require {required} GB -- you are in trouble!!!!!!!!!' ) sc.parallelize(cvsch.make_population, kwargs={'pop_size': pop_size}, iterkwargs={'rand_seed': seeds}) # Run them in parallel else: for seed in seeds: cvsch.make_population(pop_size=pop_size, rand_seed=seed)
def run_workers(): return sc.parallelize(worker, n_workers)
base_sim = cs.create_sim(params, pop_size=pop_size, folder=folder, verbose=0.1) #%% Run the sims def run_sim(scen): ''' Run a single sim ''' sim = sc.dcp(base_sim) sm = cvsch.schools_manager(scen) sim['interventions'] += [sm] sim.run(keep_people=keep_people) return sim if do_run: if parallelize: sc.heading('Running in parallel...') raw_sims = sc.parallelize(run_sim, all_scens.values()) sims = sc.odict({k:scen for k,scen in zip(all_scens.keys(), raw_sims)}) else: sc.heading('Running in serial...') sims = sc.odict() for k,scen in all_scens: sims[k] = run_sim(scen) if do_save: sc.saveobj(sims_file, sims) else: sc.heading('Loading from disk...') sims = sc.loadobj(sims_file) #%% Analysis
country = country_data['name'][c] print(f' Working on {country} ({c+1}/{len(country_data)})...') D[country].makepackage(verbose=False) meta = country_data.findrow(country, asdict=True) alloc = [] dalys = [] for spend in spendings: D[country].package().optimize(budget=spend*meta['population']) df = D[country].package().data alloc.append(sc.dcp(df['opt_spend'][:])) dalys.append(sc.dcp(df['opt_dalys_averted'][:])) meta['interv_names'] = sc.dcp(df['shortname'][:]) result = sc.odict({'meta':meta, 'alloc':pl.array(alloc), 'dalys':pl.array(dalys), 'package':D[country].package()}) return result results = sc.parallelize(optimize, iterkwargs={'c':list(range(len(country_data)))}, kwargs={'D':D, 'country_data':country_data}) for r,result in enumerate(results): R[country_data['name'][r]] = result # Saving if dosave: sc.heading('Saving...') sc.saveobj('results/rapid_data.obj', D) sc.saveobj('results/rapid_results.obj', R) sc.toc() print('Done.')
def run_workers(): ''' Run multiple workers in parallel ''' output = sc.parallelize(worker, n_workers) return output
}, ) if __name__ == '__main__': msims = sc.objdict() for which in ['actual', 'low', 'high']: scenpars = sc.objdict(sc.mergedicts(base, scens[which])) kwargs = dict(scenpars=scenpars, from_cache=from_cache, do_shrink=0) # Run the simulations if ncpus > 1: sims = sc.parallelize(cs.run_sim, iterarg=indices, ncpus=ncpus, kwargs=kwargs) else: sims = [cs.run_sim(index, **kwargs) for index in indices] # Merge into a multisim and optionally plot msim = cv.MultiSim(sims=sims) msim.reduce() msims[which] = msim if do_plot: msim.plot(to_plot='overview', fig_args={'figsize': (38, 19)}) if do_save: print('Saving...') for msim in msims.values(): for sim in msim.sims: