def main(): import re from sys import path as python_path from os import getcwd from os.path import exists from argparse import ArgumentParser from pylada import jobfolder from pylada.process.mpi import create_global_comm from pylada.process.jobfolder import JobFolderProcess import pylada # below would go additional imports. parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:])) parser.add_argument("--pools", type=int, default=0, help="Number of pools") parser.add_argument( "--ppath", dest="ppath", default=None, \ help="Directory to add to python path", metavar="Directory" ) parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--timeout', dest="timeout", default=300, type=int,\ help="Time to wait for job-dictionary to becom available " "before timing out (in seconds). A negative or null " "value implies forever. Defaults to 5mn.") parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.') try: options = parser.parse_args() except SystemExit: return # additional path to look into. if options.ppath is not None: python_path.append(options.ppath) if not exists(options.pickle): print "Could not find file {0}.".format(options.pickle) return # Set up mpi processes. pylada.default_comm['ppn'] = options.ppn pylada.default_comm['n'] = options.nbprocs create_global_comm(options.nbprocs) timeout = None if options.timeout <= 0 else options.timeout jobfolder = jobfolder.load(options.pickle, timeout=timeout) process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools) process.start(pylada.default_comm) process.wait(60)
def test(nbprocs, ppn, executable): from os import getcwd print 'IN DIRECTORY', getcwd() from pylada.process.mpi import create_global_comm from pylada.process.iterator import IteratorProcess from functional import Functional import pylada print 'CREATING GLOBAL COMM' pylada.default_comm['ppn'] = ppn pylada.default_comm['n'] = nbprocs create_global_comm(nbprocs) print 'CREATING FUNCTIONALS AND PROCESSES' lfunc = Functional(executable, range(ppn*10, ppn*10+8)) long = IteratorProcess(lfunc, outdir='long') sfunc = Functional(executable, [10]) short0 = IteratorProcess(sfunc, outdir='short0') short1 = IteratorProcess(sfunc, outdir='short1') print 'CREATING COMMUNICATORS' long_comm = pylada.default_comm.lend(3*(nbprocs//4)) assert len(long_comm.machines) == 2 short_comm0 = pylada.default_comm.lend(pylada.default_comm['n']//2) assert len(short_comm0.machines) == 1 short_comm1 = pylada.default_comm.lend('all') assert len(short_comm1.machines) == 1 print 'STARTING LONG PROCESS' long.start(long_comm) assert not long.poll() print 'STARTING SHORT PROCESSES' short0.start(short_comm0) short1.start(short_comm1) print 'TESTING PROCESS OVERLAP' assert not long.poll() print 'TESTED PROCESS OVERLAP' short0.wait() print 'FIRST SHORT PROCESS FINISHED' assert not long.poll() print 'TESTED PROCESS OVERLAP' short1.wait() print 'SECOND SHORT PROCESS FINISHED' assert not long.poll() print 'TESTED PROCESS OVERLAP' long.wait() print 'LONG PROCESS FINISHED' assert lfunc.Extract('long').success assert sfunc.Extract('short0').success assert sfunc.Extract('short1').success print 'END'
def test(nbprocs, ppn): import pylada from pylada.process.mpi import create_global_comm pylada.default_comm['ppn'] = ppn pylada.default_comm['n'] = nbprocs print 'EXPECTED N={0}, PPN={1}'.format(nbprocs, ppn) create_global_comm(nbprocs) print 'FOUND' for u in pylada.default_comm.iteritems(): print u[0], u[1] print 'MACHINES' for u in pylada.default_comm.machines.iteritems(): print u[0], u[1]
def main(): import re from sys import path as python_path from os import getcwd from os.path import exists from argparse import ArgumentParser from pylada import jobfolder from pylada.process.mpi import create_global_comm from pylada.process.jobfolder import JobFolderProcess import pylada # below would go additional imports. parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:])) parser.add_argument("--pools", type=int, default=0, help="Number of pools") parser.add_argument("--ppath", dest="ppath", default=None, help="Directory to add to python path", metavar="Directory") parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int, help="Number of processors with which to launch job.") parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int, help="Number of processors with which to launch job.") parser.add_argument('--timeout', dest="timeout", default=300, type=int, help="Time to wait for job-dictionary to becom available " "before timing out (in seconds). A negative or null " "value implies forever. Defaults to 5mn.") parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.') try: options = parser.parse_args() except SystemExit: return # additional path to look into. if options.ppath is not None: python_path.append(options.ppath) if not exists(options.pickle): print("Could not find file {0}.".format(options.pickle)) return # Set up mpi processes. pylada.default_comm['ppn'] = options.ppn pylada.default_comm['n'] = options.nbprocs create_global_comm(options.nbprocs) timeout = None if options.timeout <= 0 else options.timeout jobfolder = jobfolder.load(options.pickle, timeout=timeout) process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools) process.start(pylada.default_comm) process.wait(60)
def main(): import re from sys import path as python_path from os.path import exists from argparse import ArgumentParser from pylada import jobfolder from pylada.process.mpi import create_global_comm import pylada # below would go additional imports. parser = ArgumentParser( prog="runone", description = re.sub("\\s+", " ", __doc__[1:])) parser.add_argument( "--jobid", dest="names", nargs='+', type=str, \ help="Job name", metavar="N" ) parser.add_argument( "--ppath", dest="ppath", default=None, \ help="Directory to add to python path", metavar="Directory" ) parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--timeout', dest="timeout", default=300, type=int,\ help="Time to wait for job-dictionary to becom available " "before timing out (in seconds). A negative or null " "value implies forever. Defaults to 5mn.") parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.') try: options = parser.parse_args() except SystemExit: return # additional path to look into. if options.ppath is not None: python_path.append(options.ppath) if not exists(options.pickle): print "Could not find file {0}.".format(options.pickle) return # Set up mpi processes. pylada.default_comm['ppn'] = options.ppn pylada.default_comm['n'] = options.nbprocs create_global_comm(options.nbprocs) timeout = None if options.timeout <= 0 else options.timeout jobfolder = jobfolder.load(options.pickle, timeout=timeout) print ' ipy/lau/scattered_script: jobfolder: ', jobfolder print ' ipy/lau/scattered_script: options: ', options for name in options.names: print ' ipy/lau/scattered_script: before compute for name: ', name jobfolder[name].compute(comm=pylada.default_comm, outdir=name) print ' ipy/lau/scattered_script: after compute for name: ', name
def main(): import re from sys import path as python_path from os.path import exists from argparse import ArgumentParser from pylada import jobfolder from pylada.process.mpi import create_global_comm import pylada # below would go additional imports. parser = ArgumentParser( prog="runone", description = re.sub("\\s+", " ", __doc__[1:])) parser.add_argument('--bugLev', dest="bugLev", default=0, type=int,\ help="Debug level.") parser.add_argument('--testValidProgram', dest="testValidProgram", default=None, type=str, help="testValidProgram") parser.add_argument( "--jobid", dest="names", nargs='+', type=str, \ help="Job name", metavar="N" ) parser.add_argument( "--ppath", dest="ppath", default=None, \ help="Directory to add to python path", metavar="Directory" ) parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\ help="Number of processors with which to launch job.") parser.add_argument('--timeout', dest="timeout", default=300, type=int,\ help="Time to wait for job-dictionary to becom available " "before timing out (in seconds). A negative or null " "value implies forever. Defaults to 5mn.") parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.') try: options = parser.parse_args() except SystemExit: return from pylada.misc import setBugLev setBugLev( options.bugLev) # set global debug level from pylada.misc import bugLev # must import after calling setBugLev from pylada.misc import setTestValidProgram tstPgm = options.testValidProgram if tstPgm.lower() == 'none': tstPgm = None setTestValidProgram( tstPgm) from pylada.misc import testValidProgram # additional path to look into. if options.ppath is not None: python_path.append(options.ppath) if not exists(options.pickle): print "Could not find file {0}.".format(options.pickle) return # Set up mpi processes. pylada.default_comm['ppn'] = options.ppn pylada.default_comm['n'] = options.nbprocs if testValidProgram == None: create_global_comm(options.nbprocs) # Sets pylada.default_comm else: pylada.default_comm = None # use testValidProgram timeout = None if options.timeout <= 0 else options.timeout jobfolder = jobfolder.load(options.pickle, timeout=timeout) print ' ipy/lau/scattered_script: jobfolder: ', jobfolder print ' ipy/lau/scattered_script: options: ', options for name in options.names: if bugLev >= 1: print ' ipy/lau/scattered_script: testValidProgram: %s' \ % ( testValidProgram,) print ' ipy/lau/scattered_script: name: %s' % ( name,) print ' ipy/lau/scattered_script: jobfolder[name]: %s' \ % ( jobfolder[name],) print ' ipy/lau/scattered_script: type(jobfolder[name]): %s' \ % ( type(jobfolder[name]),) print ' ipy/lau/scattered_script: jobfolder[name].compute: %s' \ % ( jobfolder[name].compute,) print ' ipy/lau/scattered_script: type(jobfolder[name].compute): %s' \ % ( type(jobfolder[name].compute),) print ' ipy/lau/scattered_script: before compute for name: %s' \ % ( name,) comm = pylada.default_comm if testValidProgram != None: comm = None jobfolder[name].compute(comm=comm, outdir=name) if bugLev >= 1: print ' ipy/lau/scattered_script: after compute for name: %s' \ % ( name,)