def main():
import re
from sys import path as python_path
from os import getcwd
from os.path import exists
from argparse import ArgumentParser
from pylada import jobfolder
from pylada.process.mpi import create_global_comm
from pylada.process.jobfolder import JobFolderProcess
import pylada
# below would go additional imports.
parser = ArgumentParser(prog="runasone",
description=re.sub("\\s+", " ", __doc__[1:]))
parser.add_argument("--pools", type=int, default=0, help="Number of pools")
parser.add_argument( "--ppath", dest="ppath", default=None, \
help="Directory to add to python path",
metavar="Directory" )
parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--timeout', dest="timeout", default=300, type=int,\
help="Time to wait for job-dictionary to becom available "
"before timing out (in seconds). A negative or null "
"value implies forever. Defaults to 5mn.")
parser.add_argument('pickle',
metavar='FILE',
type=str,
help='Path to a job-folder.')
try:
options = parser.parse_args()
except SystemExit:
return
# additional path to look into.
if options.ppath is not None: python_path.append(options.ppath)
if not exists(options.pickle):
print "Could not find file {0}.".format(options.pickle)
return
# Set up mpi processes.
pylada.default_comm['ppn'] = options.ppn
pylada.default_comm['n'] = options.nbprocs
create_global_comm(options.nbprocs)
timeout = None if options.timeout <= 0 else options.timeout
jobfolder = jobfolder.load(options.pickle, timeout=timeout)
process = JobFolderProcess(jobfolder,
outdir=getcwd(),
nbpools=options.pools)
process.start(pylada.default_comm)
process.wait(60)
def test(nbprocs, ppn, executable):
from os import getcwd
print 'IN DIRECTORY', getcwd()
from pylada.process.mpi import create_global_comm
from pylada.process.iterator import IteratorProcess
from functional import Functional
import pylada
print 'CREATING GLOBAL COMM'
pylada.default_comm['ppn'] = ppn
pylada.default_comm['n'] = nbprocs
create_global_comm(nbprocs)
print 'CREATING FUNCTIONALS AND PROCESSES'
lfunc = Functional(executable, range(ppn*10, ppn*10+8))
long = IteratorProcess(lfunc, outdir='long')
sfunc = Functional(executable, [10])
short0 = IteratorProcess(sfunc, outdir='short0')
short1 = IteratorProcess(sfunc, outdir='short1')
print 'CREATING COMMUNICATORS'
long_comm = pylada.default_comm.lend(3*(nbprocs//4))
assert len(long_comm.machines) == 2
short_comm0 = pylada.default_comm.lend(pylada.default_comm['n']//2)
assert len(short_comm0.machines) == 1
short_comm1 = pylada.default_comm.lend('all')
assert len(short_comm1.machines) == 1
print 'STARTING LONG PROCESS'
long.start(long_comm)
assert not long.poll()
print 'STARTING SHORT PROCESSES'
short0.start(short_comm0)
short1.start(short_comm1)
print 'TESTING PROCESS OVERLAP'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
short0.wait()
print 'FIRST SHORT PROCESS FINISHED'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
short1.wait()
print 'SECOND SHORT PROCESS FINISHED'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
long.wait()
print 'LONG PROCESS FINISHED'
assert lfunc.Extract('long').success
assert sfunc.Extract('short0').success
assert sfunc.Extract('short1').success
print 'END'
def test(nbprocs, ppn, executable):
from os import getcwd
print 'IN DIRECTORY', getcwd()
from pylada.process.mpi import create_global_comm
from pylada.process.iterator import IteratorProcess
from functional import Functional
import pylada
print 'CREATING GLOBAL COMM'
pylada.default_comm['ppn'] = ppn
pylada.default_comm['n'] = nbprocs
create_global_comm(nbprocs)
print 'CREATING FUNCTIONALS AND PROCESSES'
lfunc = Functional(executable, range(ppn*10, ppn*10+8))
long = IteratorProcess(lfunc, outdir='long')
sfunc = Functional(executable, [10])
short0 = IteratorProcess(sfunc, outdir='short0')
short1 = IteratorProcess(sfunc, outdir='short1')
print 'CREATING COMMUNICATORS'
long_comm = pylada.default_comm.lend(3*(nbprocs//4))
assert len(long_comm.machines) == 2
short_comm0 = pylada.default_comm.lend(pylada.default_comm['n']//2)
assert len(short_comm0.machines) == 1
short_comm1 = pylada.default_comm.lend('all')
assert len(short_comm1.machines) == 1
print 'STARTING LONG PROCESS'
long.start(long_comm)
assert not long.poll()
print 'STARTING SHORT PROCESSES'
short0.start(short_comm0)
short1.start(short_comm1)
print 'TESTING PROCESS OVERLAP'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
short0.wait()
print 'FIRST SHORT PROCESS FINISHED'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
short1.wait()
print 'SECOND SHORT PROCESS FINISHED'
assert not long.poll()
print 'TESTED PROCESS OVERLAP'
long.wait()
print 'LONG PROCESS FINISHED'
assert lfunc.Extract('long').success
assert sfunc.Extract('short0').success
assert sfunc.Extract('short1').success
print 'END'
def test(nbprocs, ppn):
import pylada
from pylada.process.mpi import create_global_comm
pylada.default_comm['ppn'] = ppn
pylada.default_comm['n'] = nbprocs
print 'EXPECTED N={0}, PPN={1}'.format(nbprocs, ppn)
create_global_comm(nbprocs)
print 'FOUND'
for u in pylada.default_comm.iteritems(): print u[0], u[1]
print 'MACHINES'
for u in pylada.default_comm.machines.iteritems(): print u[0], u[1]
def main():
import re
from sys import path as python_path
from os import getcwd
from os.path import exists
from argparse import ArgumentParser
from pylada import jobfolder
from pylada.process.mpi import create_global_comm
from pylada.process.jobfolder import JobFolderProcess
import pylada
# below would go additional imports.
parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:]))
parser.add_argument("--pools", type=int, default=0, help="Number of pools")
parser.add_argument("--ppath", dest="ppath", default=None,
help="Directory to add to python path",
metavar="Directory")
parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,
help="Number of processors with which to launch job.")
parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,
help="Number of processors with which to launch job.")
parser.add_argument('--timeout', dest="timeout", default=300, type=int,
help="Time to wait for job-dictionary to becom available "
"before timing out (in seconds). A negative or null "
"value implies forever. Defaults to 5mn.")
parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')
try:
options = parser.parse_args()
except SystemExit:
return
# additional path to look into.
if options.ppath is not None:
python_path.append(options.ppath)
if not exists(options.pickle):
print("Could not find file {0}.".format(options.pickle))
return
# Set up mpi processes.
pylada.default_comm['ppn'] = options.ppn
pylada.default_comm['n'] = options.nbprocs
create_global_comm(options.nbprocs)
timeout = None if options.timeout <= 0 else options.timeout
jobfolder = jobfolder.load(options.pickle, timeout=timeout)
process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools)
process.start(pylada.default_comm)
process.wait(60)
def main():
import re
from sys import path as python_path
from os.path import exists
from argparse import ArgumentParser
from pylada import jobfolder
from pylada.process.mpi import create_global_comm
import pylada
# below would go additional imports.
parser = ArgumentParser( prog="runone", description = re.sub("\\s+", " ", __doc__[1:]))
parser.add_argument( "--jobid", dest="names", nargs='+', type=str, \
help="Job name", metavar="N" )
parser.add_argument( "--ppath", dest="ppath", default=None, \
help="Directory to add to python path",
metavar="Directory" )
parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--timeout', dest="timeout", default=300, type=int,\
help="Time to wait for job-dictionary to becom available "
"before timing out (in seconds). A negative or null "
"value implies forever. Defaults to 5mn.")
parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')
try: options = parser.parse_args()
except SystemExit: return
# additional path to look into.
if options.ppath is not None: python_path.append(options.ppath)
if not exists(options.pickle):
print "Could not find file {0}.".format(options.pickle)
return
# Set up mpi processes.
pylada.default_comm['ppn'] = options.ppn
pylada.default_comm['n'] = options.nbprocs
create_global_comm(options.nbprocs)
timeout = None if options.timeout <= 0 else options.timeout
jobfolder = jobfolder.load(options.pickle, timeout=timeout)
print ' ipy/lau/scattered_script: jobfolder: ', jobfolder
print ' ipy/lau/scattered_script: options: ', options
for name in options.names:
print ' ipy/lau/scattered_script: before compute for name: ', name
jobfolder[name].compute(comm=pylada.default_comm, outdir=name)
print ' ipy/lau/scattered_script: after compute for name: ', name
def test(nbprocs, ppn):
import pylada
from pylada.process.mpi import create_global_comm
pylada.default_comm['ppn'] = ppn
pylada.default_comm['n'] = nbprocs
print 'EXPECTED N={0}, PPN={1}'.format(nbprocs, ppn)
create_global_comm(nbprocs)
print 'FOUND'
for u in pylada.default_comm.iteritems():
print u[0], u[1]
print 'MACHINES'
for u in pylada.default_comm.machines.iteritems():
print u[0], u[1]
def main():
import re
from sys import path as python_path
from os.path import exists
from argparse import ArgumentParser
from pylada import jobfolder
from pylada.process.mpi import create_global_comm
import pylada
# below would go additional imports.
parser = ArgumentParser( prog="runone", description = re.sub("\\s+", " ", __doc__[1:]))
parser.add_argument('--bugLev', dest="bugLev", default=0, type=int,\
help="Debug level.")
parser.add_argument('--testValidProgram', dest="testValidProgram",
default=None, type=str,
help="testValidProgram")
parser.add_argument( "--jobid", dest="names", nargs='+', type=str, \
help="Job name", metavar="N" )
parser.add_argument( "--ppath", dest="ppath", default=None, \
help="Directory to add to python path",
metavar="Directory" )
parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\
help="Number of processors with which to launch job.")
parser.add_argument('--timeout', dest="timeout", default=300, type=int,\
help="Time to wait for job-dictionary to becom available "
"before timing out (in seconds). A negative or null "
"value implies forever. Defaults to 5mn.")
parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')
try: options = parser.parse_args()
except SystemExit: return
from pylada.misc import setBugLev
setBugLev( options.bugLev) # set global debug level
from pylada.misc import bugLev # must import after calling setBugLev
from pylada.misc import setTestValidProgram
tstPgm = options.testValidProgram
if tstPgm.lower() == 'none': tstPgm = None
setTestValidProgram( tstPgm)
from pylada.misc import testValidProgram
# additional path to look into.
if options.ppath is not None: python_path.append(options.ppath)
if not exists(options.pickle):
print "Could not find file {0}.".format(options.pickle)
return
# Set up mpi processes.
pylada.default_comm['ppn'] = options.ppn
pylada.default_comm['n'] = options.nbprocs
if testValidProgram == None:
create_global_comm(options.nbprocs) # Sets pylada.default_comm
else:
pylada.default_comm = None # use testValidProgram
timeout = None if options.timeout <= 0 else options.timeout
jobfolder = jobfolder.load(options.pickle, timeout=timeout)
print ' ipy/lau/scattered_script: jobfolder: ', jobfolder
print ' ipy/lau/scattered_script: options: ', options
for name in options.names:
if bugLev >= 1:
print ' ipy/lau/scattered_script: testValidProgram: %s' \
% ( testValidProgram,)
print ' ipy/lau/scattered_script: name: %s' % ( name,)
print ' ipy/lau/scattered_script: jobfolder[name]: %s' \
% ( jobfolder[name],)
print ' ipy/lau/scattered_script: type(jobfolder[name]): %s' \
% ( type(jobfolder[name]),)
print ' ipy/lau/scattered_script: jobfolder[name].compute: %s' \
% ( jobfolder[name].compute,)
print ' ipy/lau/scattered_script: type(jobfolder[name].compute): %s' \
% ( type(jobfolder[name].compute),)
print ' ipy/lau/scattered_script: before compute for name: %s' \
% ( name,)
comm = pylada.default_comm
if testValidProgram != None: comm = None
jobfolder[name].compute(comm=comm, outdir=name)
if bugLev >= 1:
print ' ipy/lau/scattered_script: after compute for name: %s' \
% ( name,)
