def run_pool(obj):
from pyina.launchers import Mpi
p = Mpi(2)
x = [1, 2, 3]
y = list(map(obj, x))
p.scatter = False
assert p.map(obj, x) == y
p.source = True
assert p.map(obj, x) == y
p.scatter = True
assert p.map(obj, x) == y
p.source = False
assert p.map(obj, x) == y
def test_pool(obj):
from pyina.launchers import Mpi
p = Mpi(2)
x = [1,2,3]
y = map(obj, x)
p.scatter = False
assert p.map(obj, x) == y
p.source = True
assert p.map(obj, x) == y
p.scatter = True
assert p.map(obj, x) == y
p.source = False
assert p.map(obj, x) == y
def test_launcher():
# configured launch commands for selected launchers
serial = SerialMapper()
print("non-python serial launch:", serial)
settings = {'python': '', 'program': "hostname"}
print(serial._launcher(settings), "\n")
print("serial python launch:", serial)
defaults['program'] = "tools.py"
defaults['progargs'] = "12345"
print(serial._launcher(defaults), "\n")
qsub = Torque()
serial.scheduler = qsub
print("scheduled serial launch:", serial)
settings = {'program': "tools.py", 'progargs': '12345'}
print(serial._launcher(settings), "\n")
mpi = Mpi()
print("non-scheduled parallel launch:", mpi)
print(mpi._launcher(settings), "\n")
qsub.nodes = '4:ppn=2'
mpi.nodes = mpi.njobs(qsub.nodes)
print("scheduled parallel launch:", mpi, "| Torque")
print(qsub._submit(mpi._launcher(settings)), "\n")
mpi.scheduler = qsub
print("scheduled parallel launch:", mpi)
print(mpi._launcher(settings), "\n")
_mpi = Mpi(scheduler=Torque(nodes='4:ppn=2'))
print("scheduled parallel launch:", _mpi)
print(_mpi._launcher(settings), "\n")
_mpi = TorqueMpi(nodes='4:ppn=2')
print("scheduled parallel launch:", _mpi)
print(_mpi._launcher(settings), "\n")
qsub.nodes = 1
serial = SerialMapper()
print("scheduled serial launch:", serial, "| Torque")
print(qsub._submit(serial._launcher(settings)), "\n")
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
from pyina.schedulers import Torque
from pyina.mpi import _save, _debug
#_debug(True)
#_save(True)
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Submit an mpi job to torque in the 'productionQ' queue..."
print "Using 15 items over 5 nodes and the scatter-gather strategy"
torque = Torque('5:ppn=2',
queue='productionQ',
timelimit='20:00:00',
workdir='.')
pool = Mpi(scheduler=torque, scatter=True)
res = pool.map(host, range(15))
print pool
print '\n'.join(res)
print "hello from master"
# end of file
# Copyright (c) 2016-2020 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
#XXX:: can fail with NameError: global name 'func' is not defined
#XXX:: can fail with RuntimeError: maximum recursion depth exceeded
#from mystic.models.poly import chebyshev8cost as func
def host(coeffs):
from mystic.models.poly import chebyshev8cost as func
return "Chebyshev%s = %s" % (coeffs, func(coeffs))
params = [(i,0,-2*i,0,4*i,0,-2*i,0,i) for i in range(10)]
pool = Mpi()
print("Evaluate the 8th order Chebyshev polynomial...")
print("Using 'dill' for 10 combinations over 4 nodes")
pool.nodes = 4
res1 = pool.map(host, params)
print(pool)
print('\n'.join(res1))
print('')
print("Using 'dill.source' for 10 combinations over 4 nodes")
pool.source = True
res2 = pool.map(host, params)
print(pool)
print('\n'.join(res2))
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pyina/LICENSE
from pyina.launchers import Mpi
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Explicitly using the MPI launcher, we will execute..."
pool = Mpi(4)
print "10 items on 4 nodes using a worker pool:"
res1 = pool.map(host, range(10))
print pool
print '\n'.join(res1)
print ''
print "10 items on 4 nodes using scatter-gather:"
pool.scatter = True
res2 = pool.map(host, range(10))
print pool
print '\n'.join(res2)
# end of file
import scipy.integrate
scipy.integrate.quad(lambda x: 4.0/(1+x*x), 0, 1)
"""
from numpy import arange
# default # of rectangles
n = 20000
integration_points = (arange(1, n + 1) - 0.5) / n
def f(x):
return 4.0 / (1.0 + x * x)
#from pyina.launchers import MpiScatter as Mpi
from pyina.launchers import MpiPool as Mpi
if __name__ == '__main__':
work = Mpi(2)
out = work.map(f, integration_points)
from pyina import mpi
if mpi.world.rank == 0:
print("approxmiate pi : ", sum(out) / n)
print("calculated on %d nodes " % work.nodes)
# end of file
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print("Explicitly using the MPI launcher, we will execute...")
pool = Mpi(4)
print("10 items on 4 nodes using a worker pool:")
res1 = pool.map(host, range(10))
print(pool)
print('\n'.join(res1))
print('')
print("10 items on 4 nodes using scatter-gather:")
pool.scatter = True
res2 = pool.map(host, range(10))
print(pool)
print('\n'.join(res2))
# end of file
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pyina/LICENSE
from pyina.launchers import Mpi
from pyina.schedulers import Torque
from pyina.mpi import _save, _debug
#_debug(True)
#_save(True)
def host(id):
import socket
return "Rank: %d -- %s" % (id, socket.gethostname())
print "Submit an mpi job to torque in the 'productionQ' queue..."
print "Using 15 items over 5 nodes and the scatter-gather strategy"
torque = Torque('5:ppn=2', queue='productionQ', timelimit='20:00:00', workdir='.')
pool = Mpi(scheduler=torque, scatter=True)
res = pool.map(host, range(15))
print pool
print '\n'.join(res)
print "hello from master"
# end of file
# Copyright (c) 2016-2019 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
#XXX:: can fail with NameError: global name 'func' is not defined
#XXX:: can fail with RuntimeError: maximum recursion depth exceeded
#from mystic.models.poly import chebyshev8cost as func
def host(coeffs):
from mystic.models.poly import chebyshev8cost as func
return "Chebyshev%s = %s" % (coeffs, func(coeffs))
params = [(i,0,-2*i,0,4*i,0,-2*i,0,i) for i in range(10)]
pool = Mpi()
print("Evaluate the 8th order Chebyshev polynomial...")
print("Using 'dill' for 10 combinations over 4 nodes")
pool.nodes = 4
res1 = pool.map(host, params)
print(pool)
print('\n'.join(res1))
print('')
print("Using 'dill.source' for 10 combinations over 4 nodes")
pool.source = True
res2 = pool.map(host, params)
print(pool)
print('\n'.join(res2))
def batch_deploy_preprocess(n_cores, subtype_output_dir, meffil, torque, run,
series, pc_qc_parameters_csv, use_cache, qc_only,
chunk_size):
"""Deploy multiple preprocessing jobs in series or parallel."""
pheno_csvs = glob.glob(os.path.join(subtype_output_dir, '*', '*.csv'))
opts = {'-n': n_cores}
try:
pc_qc_parameters = pd.read_csv(
pc_qc_parameters).drop_duplicates().set_index('subtype')
except:
pc_qc_parameters = pd.DataFrame(
[[name, -1, 0.05, 0.05, 0.05, 0.05, 5, -2]
for name in np.vectorize(lambda x: x.split('/')[-2])(pheno_csvs)],
columns=[
'subtype', 'n_pcs', 'p_beadnum_samples', 'p_detection_samples',
'p_beadnum_cpgs', 'p_detection_cpgs', 'sex_sd', 'sex_cutoff'
]).drop_duplicates().set_index('subtype')
if meffil:
opts['-m'] = ''
if use_cache:
opts['-u'] = ''
if qc_only:
opts['-qc'] = ''
commands = []
for pheno_csv in pheno_csvs:
pheno_path = os.path.abspath(pheno_csv)
subtype = pheno_path.split('/')[-2]
opts['-pc'] = int(pc_qc_parameters.loc[subtype, 'n_pcs'])
opts['-bns'] = pc_qc_parameters.loc[subtype, 'p_beadnum_samples']
opts['-pds'] = pc_qc_parameters.loc[subtype, 'p_detection_samples']
opts['-bnc'] = pc_qc_parameters.loc[subtype, 'p_beadnum_cpgs']
opts['-pdc'] = pc_qc_parameters.loc[subtype, 'p_detection_cpgs']
opts['-sc'] = pc_qc_parameters.loc[subtype, 'sex_cutoff']
opts['-sd'] = pc_qc_parameters.loc[subtype, 'sex_sd']
opts['-i'] = pheno_path[:pheno_path.rfind('/') + 1]
opts['-o'] = pheno_path[:pheno_path.rfind('/') +
1] + 'methyl_array.pkl'
command = 'pymethyl-preprocess preprocess_pipeline {}'.format(' '.join(
'{} {}'.format(k, v) for k, v in opts.items()))
commands.append(command)
if not torque:
if not series and chunk_size != -1:
#commands = np.array_split(commands,len(commands)//chunk_size)
print(commands)
with open('commands.txt', 'w') as f:
f.write('\n'.join(commands))
subprocess.call(
'cat commands.txt | xargs -L 1 -I CMD -P {} bash -c CMD'.
format(chunk_size),
shell=True) # https://www.gnu.org/software/parallel/sem.html
"""for command_list in commands:
subprocess.call('run_parallel {}'.format(' '.join(['"{}"'.format(command) for command in command_list])),shell=True)"""
else:
for command in commands:
if not series:
command = "nohup {} &".format(command)
if not run:
click.echo(command)
else:
subprocess.call(command, shell=True)
else:
run_command = lambda command: subprocess.call(
'module load cuda && module load python/3-Anaconda && source activate py36 && {}'
.format(command),
shell=True)
from pyina.schedulers import Torque
from pyina.launchers import Mpi
config = {
'nodes': '1:ppn=6',
'queue': 'default',
'timelimit': '01:00:00'
}
torque = Torque(**config)
pool = Mpi(scheduler=torque)
pool.map(run_command, commands)
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2018 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
from pyina.launchers import Mpi
#XXX: should not have to define "func" within mapped function
#from mystic.models import rosen as func
def host(coeffs):
from mystic.models import rosen as func
return "rosen%s = %s" % (coeffs, func(coeffs))
print "Evaluate an imported function (the rosenbrock function)..."
print "For 10 items on 4 nodes, using the default mapping strategy"
params = [(i, i, i) for i in range(10)]
pool = Mpi(4)
res = pool.map(host, params)
print pool
print '\n'.join(res)
# end of file
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pyina/LICENSE
from pyina.launchers import Mpi
#XXX: should not have to define "func" within mapped function
#from mystic.models import rosen as func
def host(coeffs):
from mystic.models import rosen as func
return "rosen%s = %s" % (coeffs, func(coeffs))
print "Evaluate an imported function (the rosenbrock function)..."
print "For 10 items on 4 nodes, using the default mapping strategy"
params = [(i,i,i) for i in range(10)]
pool = Mpi(4)
res = pool.map(host, params)
print pool
print '\n'.join(res)
# end of file
def test_launcher():
# configured launch commands for selected launchers
serial = SerialMapper()
print "non-python serial launch:", serial
settings = {'python':'', 'program':"hostname"}
print serial._launcher(settings), "\n"
print "serial python launch:", serial
defaults['program'] = "tools.py"
defaults['progargs'] = "12345"
print serial._launcher(defaults), "\n"
qsub = Torque()
serial.scheduler = qsub
print "scheduled serial launch:", serial
settings = {'program':"tools.py", 'progargs':'12345'}
print serial._launcher(settings), "\n"
mpi = Mpi()
print "non-scheduled parallel launch:", mpi
print mpi._launcher(settings), "\n"
qsub.nodes = '4:ppn=2'
mpi.nodes = mpi.njobs(qsub.nodes)
print "scheduled parallel launch:", mpi, "| Torque"
print qsub._submit(mpi._launcher(settings)), "\n"
mpi.scheduler = qsub
print "scheduled parallel launch:", mpi
print mpi._launcher(settings), "\n"
_mpi = Mpi(scheduler=Torque(nodes='4:ppn=2'))
print "scheduled parallel launch:", _mpi
print _mpi._launcher(settings), "\n"
_mpi = TorqueMpi(nodes='4:ppn=2')
print "scheduled parallel launch:", _mpi
print _mpi._launcher(settings), "\n"
qsub.nodes = 1
serial = SerialMapper()
print "scheduled serial launch:", serial, "| Torque"
print qsub._submit(serial._launcher(settings)), "\n"
