def run_pool(obj):
from pyina.launchers import Mpi
p = Mpi(2)
x = [1, 2, 3]
y = list(map(obj, x))
p.scatter = False
assert p.map(obj, x) == y
p.source = True
assert p.map(obj, x) == y
p.scatter = True
assert p.map(obj, x) == y
p.source = False
assert p.map(obj, x) == y
def test_pool(obj):
from pyina.launchers import Mpi
p = Mpi(2)
x = [1,2,3]
y = map(obj, x)
p.scatter = False
assert p.map(obj, x) == y
p.source = True
assert p.map(obj, x) == y
p.scatter = True
assert p.map(obj, x) == y
p.source = False
assert p.map(obj, x) == y
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
from pyina.schedulers import Torque
from pyina.mpi import _save, _debug
#_debug(True)
#_save(True)
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Submit an mpi job to torque in the 'productionQ' queue..."
print "Using 15 items over 5 nodes and the scatter-gather strategy"
torque = Torque('5:ppn=2',
queue='productionQ',
timelimit='20:00:00',
workdir='.')
pool = Mpi(scheduler=torque, scatter=True)
res = pool.map(host, range(15))
print pool
print '\n'.join(res)
print "hello from master"
# end of file
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
#XXX:: can fail with NameError: global name 'func' is not defined
#XXX:: can fail with RuntimeError: maximum recursion depth exceeded
#from mystic.models.poly import chebyshev8cost as func
def host(coeffs):
from mystic.models.poly import chebyshev8cost as func
return "Chebyshev%s = %s" % (coeffs, func(coeffs))
params = [(i,0,-2*i,0,4*i,0,-2*i,0,i) for i in range(10)]
pool = Mpi()
print("Evaluate the 8th order Chebyshev polynomial...")
print("Using 'dill' for 10 combinations over 4 nodes")
pool.nodes = 4
res1 = pool.map(host, params)
print(pool)
print('\n'.join(res1))
print('')
print("Using 'dill.source' for 10 combinations over 4 nodes")
pool.source = True
res2 = pool.map(host, params)
print(pool)
print('\n'.join(res2))
# end of file
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pyina/LICENSE
from pyina.launchers import Mpi
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Explicitly using the MPI launcher, we will execute..."
pool = Mpi(4)
print "10 items on 4 nodes using a worker pool:"
res1 = pool.map(host, range(10))
print pool
print '\n'.join(res1)
print ''
print "10 items on 4 nodes using scatter-gather:"
pool.scatter = True
res2 = pool.map(host, range(10))
print pool
print '\n'.join(res2)
# end of file
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print("Explicitly using the MPI launcher, we will execute...")
pool = Mpi(4)
print("10 items on 4 nodes using a worker pool:")
res1 = pool.map(host, range(10))
print(pool)
print('\n'.join(res1))
print('')
print("10 items on 4 nodes using scatter-gather:")
pool.scatter = True
res2 = pool.map(host, range(10))
print(pool)
print('\n'.join(res2))
# end of file
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pyina/LICENSE
from pyina.launchers import Mpi
from pyina.schedulers import Torque
from pyina.mpi import _save, _debug
#_debug(True)
#_save(True)
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Submit an mpi job to torque in the 'productionQ' queue..."
print "Using 15 items over 5 nodes and the scatter-gather strategy"
torque = Torque('5:ppn=2', queue='productionQ', timelimit='20:00:00', workdir='.')
pool = Mpi(scheduler=torque, scatter=True)
res = pool.map(host, range(15))
print pool
print '\n'.join(res)
print "hello from master"
# end of file
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
#XXX:: can fail with NameError: global name 'func' is not defined
#XXX:: can fail with RuntimeError: maximum recursion depth exceeded
#from mystic.models.poly import chebyshev8cost as func
def host(coeffs):
from mystic.models.poly import chebyshev8cost as func
return "Chebyshev%s = %s" % (coeffs, func(coeffs))
params = [(i,0,-2*i,0,4*i,0,-2*i,0,i) for i in range(10)]
pool = Mpi()
print("Evaluate the 8th order Chebyshev polynomial...")
print("Using 'dill' for 10 combinations over 4 nodes")
pool.nodes = 4
res1 = pool.map(host, params)
print(pool)
print('\n'.join(res1))
print('')
print("Using 'dill.source' for 10 combinations over 4 nodes")
pool.source = True
res2 = pool.map(host, params)
print(pool)
print('\n'.join(res2))
# end of file
def batch_deploy_preprocess(n_cores, subtype_output_dir, meffil, torque, run,
series, pc_qc_parameters_csv, use_cache, qc_only,
chunk_size):
"""Deploy multiple preprocessing jobs in series or parallel."""
pheno_csvs = glob.glob(os.path.join(subtype_output_dir, '*', '*.csv'))
opts = {'-n': n_cores}
try:
pc_qc_parameters = pd.read_csv(
pc_qc_parameters).drop_duplicates().set_index('subtype')
except:
pc_qc_parameters = pd.DataFrame(
[[name, -1, 0.05, 0.05, 0.05, 0.05, 5, -2]
for name in np.vectorize(lambda x: x.split('/')[-2])(pheno_csvs)],
columns=[
'subtype', 'n_pcs', 'p_beadnum_samples', 'p_detection_samples',
'p_beadnum_cpgs', 'p_detection_cpgs', 'sex_sd', 'sex_cutoff'
]).drop_duplicates().set_index('subtype')
if meffil:
opts['-m'] = ''
if use_cache:
opts['-u'] = ''
if qc_only:
opts['-qc'] = ''
commands = []
for pheno_csv in pheno_csvs:
pheno_path = os.path.abspath(pheno_csv)
subtype = pheno_path.split('/')[-2]
opts['-pc'] = int(pc_qc_parameters.loc[subtype, 'n_pcs'])
opts['-bns'] = pc_qc_parameters.loc[subtype, 'p_beadnum_samples']
opts['-pds'] = pc_qc_parameters.loc[subtype, 'p_detection_samples']
opts['-bnc'] = pc_qc_parameters.loc[subtype, 'p_beadnum_cpgs']
opts['-pdc'] = pc_qc_parameters.loc[subtype, 'p_detection_cpgs']
opts['-sc'] = pc_qc_parameters.loc[subtype, 'sex_cutoff']
opts['-sd'] = pc_qc_parameters.loc[subtype, 'sex_sd']
opts['-i'] = pheno_path[:pheno_path.rfind('/') + 1]
opts['-o'] = pheno_path[:pheno_path.rfind('/') +
1] + 'methyl_array.pkl'
command = 'pymethyl-preprocess preprocess_pipeline {}'.format(' '.join(
'{} {}'.format(k, v) for k, v in opts.items()))
commands.append(command)
if not torque:
if not series and chunk_size != -1:
#commands = np.array_split(commands,len(commands)//chunk_size)
print(commands)
with open('commands.txt', 'w') as f:
f.write('\n'.join(commands))
subprocess.call(
'cat commands.txt | xargs -L 1 -I CMD -P {} bash -c CMD'.
format(chunk_size),
shell=True) # https://www.gnu.org/software/parallel/sem.html
"""for command_list in commands:
subprocess.call('run_parallel {}'.format(' '.join(['"{}"'.format(command) for command in command_list])),shell=True)"""
else:
for command in commands:
if not series:
command = "nohup {} &".format(command)
if not run:
click.echo(command)
else:
subprocess.call(command, shell=True)
else:
run_command = lambda command: subprocess.call(
'module load cuda && module load python/3-Anaconda && source activate py36 && {}'
.format(command),
shell=True)
from pyina.schedulers import Torque
from pyina.launchers import Mpi
config = {
'nodes': '1:ppn=6',
'queue': 'default',
'timelimit': '01:00:00'
}
torque = Torque(**config)
pool = Mpi(scheduler=torque)
pool.map(run_command, commands)
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
#XXX: should not have to define "func" within mapped function
#from mystic.models import rosen as func
def host(coeffs):
from mystic.models import rosen as func
return "rosen%s = %s" % (coeffs, func(coeffs))
print "Evaluate an imported function (the rosenbrock function)..."
print "For 10 items on 4 nodes, using the default mapping strategy"
params = [(i, i, i) for i in range(10)]
pool = Mpi(4)
res = pool.map(host, params)
print pool
print '\n'.join(res)
# end of file
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pyina/LICENSE
from pyina.launchers import Mpi
#XXX: should not have to define "func" within mapped function
#from mystic.models import rosen as func
def host(coeffs):
from mystic.models import rosen as func
return "rosen%s = %s" % (coeffs, func(coeffs))
print "Evaluate an imported function (the rosenbrock function)..."
print "For 10 items on 4 nodes, using the default mapping strategy"
params = [(i,i,i) for i in range(10)]
pool = Mpi(4)
res = pool.map(host, params)
print pool
print '\n'.join(res)
# end of file