def run_experiments(experiments, mpi=False, **kwargs):
"""Run a list of experiments. Optionally using MPI.
:Arguments:
experiments : list of dicts
One entry for each experiment to run. Each experiment should have a dict like this:
* 'data' -> np.ndarray
* 'model_type' -> Hierarchical class or str name of class
* 'name' -> str of name of experiment (optional)
* 'kwargs' -> Keyword arguments to be supplied at model creation
mpi : bool (default=False)
Whether to run experiments in parallel using MPI. If set to True you must call
your script (which calls this function) using mpirun.
This requires mpi4py_map and mpi4py to be installed.
E.g. pip install mpi4py mpi4py_map
kwargs : dict
Additional keyword arguments to be passed to run_experiment (see run_experiment),
such as samples, burn etc.
"""
if mpi:
import mpi4py_map
results = mpi4py_map.map(run_experiment, experiments, **kwargs)
else:
results = [run_experiment(experiment, **kwargs) for experiment in experiments]
return results
def test_power_except():
result_parallel = mpi4py_map.map(power_except_on_3,
range(50),
y=2,
debug=True)
result_serial = [power(i, y=2) for i in range(50)]
result_serial[3] = None
assert result_serial == result_parallel, "Parallel result does not match direct one."
def analyze_experiments(experiments, mpi=False, plot_dic=True, **kwargs):
"""Analyze multiple experiments. Outputs will be saved to
subdirectories. Can optionally analyze in parallel.
:Arguments:
experiments : list of dicts
One entry for each experiment to analyze. Each experiment should have a dict like this:
* 'data' -> np.ndarray
* 'model_type' -> Hierarchical class or str name of class
* 'name' -> str of name of experiment (optional)
* 'kwargs' -> Keyword arguments to be supplied at model creation
mpi : bool (default=False)
Whether to run experiments in parallel using MPI. If set to True you must call
your script (which calls this function) using mpirun.
This requires mpi4py_map and mpi4py to be installed.
E.g. pip install mpi4py mpi4py_map
plot_dic : bool (default=True)
Whether to create a bar plot of DIC values for each model.
kwargs : dict
Keyword arguments to be passed to analyze_experiment. See analyze_experiment.
"""
# Load models if necessary
for experiment in experiments:
if 'model' not in experiment:
experiment['model'] = load_model(experiment)
if mpi:
import mpi4py_map
results = mpi4py_map.map(analyze_experiment, experiments, **kwargs)
else:
results = [
analyze_experiment(experiment, **kwargs)
for experiment in experiments
]
if plot_dic:
dics = [experiment['model'].mc.dic for experiment in experiments]
names = [
_parse_experiment(experiment)[-1] for experiment in experiments
]
fig = plt.figure()
x = range(len(names))
ax = plt.bar(x, dics, align='center')
plt.xticks(x, names)
plt.ylabel('DIC')
fig.autofmt_xdate()
fig.savefig('dic.png')
fig.savefig('dic.pdf')
return results
def setup_parallel_process(jobs_list, mpi):
print 'in setup_parallel_process'
#create tuples to allow multiple args for multiprocessing
jobs_tuple = []
for i in jobs_list:
jobs_tuple.append((i, ))
####################
if mpi: #run using MPI
import mpi4py_map
mpi4py_map.map(
Run_1_simulation_wrapped, jobs_tuple
) #provide parameter values for function f and store any results
##############
else: #run using python multiprocessing function
print 'Sending out :', len(
jobs_tuple), 'Jobs to', pool_size, ' CPUs using multiprocessing'
pool = multiprocessing.Pool(processes=pool_size,
initializer=start_process)
pool.map(Run_1_simulation_wrapped, jobs_tuple) #map jobs to CPUs
pool.close() # no more tasks
pool.join() # wrap up current tasks
#take note of NULL values are failed simulation
return
def setup_multiprocess(cases1, mpi):
print 'in setup_multiprocess'
cases = [] # create tuples to allow multi args for multiprocessing
for i in cases1:
cases.append((i, ))
#print cases
if mpi:
import mpi4py_map
mpi4py_map.map(
do_calculation_wrapped, cases
) #provide parameter values for function f and store any results
##############
else:
print 'Sending out :', len(
cases), 'Jobs to', pool_size, ' CPUs using multiprocessing'
pool = multiprocessing.Pool(processes=pool_size,
initializer=start_process)
pool.map(do_calculation_wrapped, cases)
pool.close() # no more tasks
pool.join() # wrap up current tasks
#take note of NULL values from failed simulation
##############
return
def analyze_experiments(experiments, mpi=False, plot_dic=True, **kwargs):
"""Analyze multiple experiments. Outputs will be saved to
subdirectories. Can optionally analyze in parallel.
:Arguments:
experiments : list of dicts
One entry for each experiment to analyze. Each experiment should have a dict like this:
* 'data' -> np.ndarray
* 'model_type' -> Hierarchical class or str name of class
* 'name' -> str of name of experiment (optional)
* 'kwargs' -> Keyword arguments to be supplied at model creation
mpi : bool (default=False)
Whether to run experiments in parallel using MPI. If set to True you must call
your script (which calls this function) using mpirun.
This requires mpi4py_map and mpi4py to be installed.
E.g. pip install mpi4py mpi4py_map
plot_dic : bool (default=True)
Whether to create a bar plot of DIC values for each model.
kwargs : dict
Keyword arguments to be passed to analyze_experiment. See analyze_experiment.
"""
# Load models if necessary
for experiment in experiments:
if 'model' not in experiment:
experiment['model'] = load_model(experiment)
if mpi:
import mpi4py_map
results = mpi4py_map.map(analyze_experiment, experiments, **kwargs)
else:
results = [analyze_experiment(experiment, **kwargs) for experiment in experiments]
if plot_dic:
dics = [experiment['model'].mc.dic for experiment in experiments]
names = [_parse_experiment(experiment)[-1] for experiment in experiments]
fig = plt.figure()
x = list(range(len(names)))
ax = plt.bar(x, dics, align='center')
plt.xticks(x, names)
plt.ylabel('DIC')
fig.autofmt_xdate()
fig.savefig('dic.png')
fig.savefig('dic.pdf')
return results
def test_power_except():
result_parallel = mpi4py_map.map(power_except_on_3, range(50), y=2, debug=True)
result_serial = [power(i, y=2) for i in range(50)]
result_serial[3] = None
assert result_serial == result_parallel, "Parallel result does not match direct one."
def test_power():
result_parallel = mpi4py_map.map(power, list(range(50)), y=2, debug=True)
result_serial = [power(i, y=2) for i in range(50)]
assert result_serial == result_parallel, "Parallel result does not match direct one."
outfile = outpath + "/diffused/" + os.path.basename(infile).strip(".nc") + "_diffused" + str(diffuse_timesteps) + ".nc"
# only copy data necessary for pism if lite
lite = True
os.system("mkdir -p " + outpath + "/diffused/")
print "initialize"
dd = DiffuseOcean.DiffuseOcean(infile, outfile, destination_grid_file, diffuse_timesteps, diffuse_variables, diffuse_missvals)
dd.getTimeIndependentInputData()
dd.prepareProjection()
dd.findAboveAndBelowSea()
dd.findAboveAndBelowSea()
# try to get the communicator object to see whether mpi is available:
try:
from mpi4py import MPI
comm = MPI.COMM_WORLD
available = False if comm.size < 2 else True
print "mpi available, with ", comm.size, " processors."
except:
available = False
if available:
result_parallel = mpi4py_map.map(dd.projAndDiffu, xrange(dd.nctimesteps), debug=True)
dd.writeNetcdf(result_parallel, lite)
else:
result_serial = map(dd.projAndDiffu, xrange(dd.nctimesteps))
dd.writeNetcdf(result_serial, lite)
