Example #1
0
def test(nbprocs, ppn, executable):
  from os import getcwd
  print 'IN DIRECTORY', getcwd()
  from pylada.process.mpi import create_global_comm
  from pylada.process.iterator import IteratorProcess
  from functional import Functional
  import pylada

  print 'CREATING GLOBAL COMM'
  pylada.default_comm['ppn'] = ppn
  pylada.default_comm['n'] = nbprocs
  create_global_comm(nbprocs)

  print 'CREATING FUNCTIONALS AND PROCESSES'
  lfunc = Functional(executable, range(ppn*10, ppn*10+8))
  long = IteratorProcess(lfunc, outdir='long')

  sfunc = Functional(executable, [10])
  short0 = IteratorProcess(sfunc, outdir='short0')
  short1 = IteratorProcess(sfunc, outdir='short1') 
  
  print 'CREATING COMMUNICATORS'
  long_comm = pylada.default_comm.lend(3*(nbprocs//4))
  assert len(long_comm.machines) == 2
  short_comm0 = pylada.default_comm.lend(pylada.default_comm['n']//2)
  assert len(short_comm0.machines) == 1
  short_comm1 = pylada.default_comm.lend('all')
  assert len(short_comm1.machines) == 1

  print 'STARTING LONG PROCESS'
  long.start(long_comm)
  assert not long.poll()
  print 'STARTING SHORT PROCESSES'
  short0.start(short_comm0)
  short1.start(short_comm1)
  print 'TESTING PROCESS OVERLAP'
  assert not long.poll()
  print 'TESTED PROCESS OVERLAP'
  short0.wait()
  print 'FIRST SHORT PROCESS FINISHED'
  assert not long.poll()
  print 'TESTED PROCESS OVERLAP'
  short1.wait()
  print 'SECOND SHORT PROCESS FINISHED'
  assert not long.poll()
  print 'TESTED PROCESS OVERLAP'
  long.wait()
  print 'LONG PROCESS FINISHED'

  assert lfunc.Extract('long').success
  assert sfunc.Extract('short0').success
  assert sfunc.Extract('short1').success
  print 'END'
Example #2
0
def test(nbprocs, ppn):
  import pylada
  from pylada.process.mpi import create_global_comm

  pylada.default_comm['ppn'] = ppn
  pylada.default_comm['n'] = nbprocs
  print 'EXPECTED N={0}, PPN={1}'.format(nbprocs, ppn)
  create_global_comm(nbprocs)
  print 'FOUND'
  for u in pylada.default_comm.iteritems(): print u[0], u[1]
  print 'MACHINES'
  for u in pylada.default_comm.machines.iteritems(): print u[0], u[1]
Example #3
0
def main():
    import re
    from sys import path as python_path
    from os import getcwd
    from os.path import exists
    from argparse import ArgumentParser
    from pylada import jobfolder
    from pylada.process.mpi import create_global_comm
    from pylada.process.jobfolder import JobFolderProcess
    import pylada

    # below would go additional imports.

    parser = ArgumentParser(prog="runasone", description=re.sub("\\s+", " ", __doc__[1:]))
    parser.add_argument("--pools", type=int, default=0, help="Number of pools")
    parser.add_argument("--ppath", dest="ppath", default=None,
                        help="Directory to add to python path",
                        metavar="Directory")
    parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,
                        help="Number of processors with which to launch job.")
    parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,
                        help="Number of processors with which to launch job.")
    parser.add_argument('--timeout', dest="timeout", default=300, type=int,
                        help="Time to wait for job-dictionary to becom available "
                             "before timing out (in seconds). A negative or null "
                             "value implies forever. Defaults to 5mn.")
    parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')

    try:
        options = parser.parse_args()
    except SystemExit:
        return

    # additional path to look into.
    if options.ppath is not None:
        python_path.append(options.ppath)

    if not exists(options.pickle):
        print("Could not find file {0}.".format(options.pickle))
        return

    # Set up mpi processes.
    pylada.default_comm['ppn'] = options.ppn
    pylada.default_comm['n'] = options.nbprocs
    create_global_comm(options.nbprocs)

    timeout = None if options.timeout <= 0 else options.timeout

    jobfolder = jobfolder.load(options.pickle, timeout=timeout)
    process = JobFolderProcess(jobfolder, outdir=getcwd(), nbpools=options.pools)
    process.start(pylada.default_comm)
    process.wait(60)
Example #4
0
def main():
  import re 
  from sys import path as python_path
  from os.path import exists
  from argparse import ArgumentParser
  from pylada import jobfolder
  from pylada.process.mpi import create_global_comm
  import pylada

  # below would go additional imports.

  parser = ArgumentParser( prog="runone", description = re.sub("\\s+", " ", __doc__[1:]))
  parser.add_argument( "--jobid", dest="names", nargs='+', type=str, \
                       help="Job name", metavar="N" )
  parser.add_argument( "--ppath", dest="ppath", default=None, \
                       help="Directory to add to python path",
                       metavar="Directory" )
  parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\
                      help="Number of processors with which to launch job.")
  parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\
                      help="Number of processors with which to launch job.")
  parser.add_argument('--timeout', dest="timeout", default=300, type=int,\
                      help="Time to wait for job-dictionary to becom available "
                           "before timing out (in seconds). A negative or null "
                           "value implies forever. Defaults to 5mn.")
  parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')

  try: options = parser.parse_args()
  except SystemExit: return

  # additional path to look into.
  if options.ppath is not None: python_path.append(options.ppath)

  if not exists(options.pickle): 
    print "Could not find file {0}.".format(options.pickle)
    return

  # Set up mpi processes.
  pylada.default_comm['ppn'] = options.ppn
  pylada.default_comm['n'] = options.nbprocs
  create_global_comm(options.nbprocs)

  timeout = None if options.timeout <= 0 else options.timeout
  
  jobfolder = jobfolder.load(options.pickle, timeout=timeout)
  print '  ipy/lau/scattered_script: jobfolder: ', jobfolder
  print '  ipy/lau/scattered_script: options: ', options
  for name in options.names:
    print '  ipy/lau/scattered_script: before compute for name: ', name
    jobfolder[name].compute(comm=pylada.default_comm, outdir=name)
    print '  ipy/lau/scattered_script: after compute for name: ', name
Example #5
0
def main():
  import re 
  from sys import path as python_path
  from os.path import exists
  from argparse import ArgumentParser
  from pylada import jobfolder
  from pylada.process.mpi import create_global_comm
  import pylada

  # below would go additional imports.

  parser = ArgumentParser( prog="runone", description = re.sub("\\s+", " ", __doc__[1:]))

  parser.add_argument('--bugLev', dest="bugLev", default=0, type=int,\
                      help="Debug level.")
  parser.add_argument('--testValidProgram', dest="testValidProgram",
                      default=None, type=str,
                      help="testValidProgram")
  parser.add_argument( "--jobid", dest="names", nargs='+', type=str, \
                       help="Job name", metavar="N" )
  parser.add_argument( "--ppath", dest="ppath", default=None, \
                       help="Directory to add to python path",
                       metavar="Directory" )
  parser.add_argument('--nbprocs', dest="nbprocs", default=pylada.default_comm['n'], type=int,\
                      help="Number of processors with which to launch job.")
  parser.add_argument('--ppn', dest="ppn", default=pylada.default_comm['ppn'], type=int,\
                      help="Number of processors with which to launch job.")
  parser.add_argument('--timeout', dest="timeout", default=300, type=int,\
                      help="Time to wait for job-dictionary to becom available "
                           "before timing out (in seconds). A negative or null "
                           "value implies forever. Defaults to 5mn.")
  parser.add_argument('pickle', metavar='FILE', type=str, help='Path to a job-folder.')

  try: options = parser.parse_args()
  except SystemExit: return

  from pylada.misc import setBugLev
  setBugLev( options.bugLev)         # set global debug level
  from pylada.misc import bugLev     # must import after calling setBugLev

  from pylada.misc import setTestValidProgram
  tstPgm = options.testValidProgram
  if tstPgm.lower() == 'none': tstPgm = None
  setTestValidProgram( tstPgm)
  from pylada.misc import testValidProgram

  # additional path to look into.
  if options.ppath is not None: python_path.append(options.ppath)

  if not exists(options.pickle): 
    print "Could not find file {0}.".format(options.pickle)
    return

  # Set up mpi processes.
  pylada.default_comm['ppn'] = options.ppn
  pylada.default_comm['n'] = options.nbprocs
  if testValidProgram == None:
    create_global_comm(options.nbprocs)   # Sets pylada.default_comm
  else:
    pylada.default_comm = None            # use testValidProgram

  timeout = None if options.timeout <= 0 else options.timeout
  
  jobfolder = jobfolder.load(options.pickle, timeout=timeout)
  print '  ipy/lau/scattered_script: jobfolder: ', jobfolder
  print '  ipy/lau/scattered_script: options: ', options
  for name in options.names:
    if bugLev >= 1:
      print '  ipy/lau/scattered_script: testValidProgram: %s' \
        % ( testValidProgram,)
      print '  ipy/lau/scattered_script: name: %s' % ( name,)
      print '  ipy/lau/scattered_script: jobfolder[name]: %s' \
        % ( jobfolder[name],)
      print '  ipy/lau/scattered_script: type(jobfolder[name]): %s' \
        % ( type(jobfolder[name]),)
      print '  ipy/lau/scattered_script: jobfolder[name].compute: %s' \
        % ( jobfolder[name].compute,)
      print '  ipy/lau/scattered_script: type(jobfolder[name].compute): %s' \
        % ( type(jobfolder[name].compute),)
      print '  ipy/lau/scattered_script: before compute for name: %s' \
        % ( name,)

    comm = pylada.default_comm
    if testValidProgram != None: comm = None
    jobfolder[name].compute(comm=comm, outdir=name)
    if bugLev >= 1:
      print '  ipy/lau/scattered_script: after compute for name: %s' \
        % ( name,)