示例#1
0
def test_failures():
  """ Tests whether scheduling jobs works on known failure cases. """
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.pool import PoolProcess
  from pylada import default_comm
  from functional import Functional
  root = JobFolder()
  for n in xrange(8):
    job = root / "a{0}".format(n)
    job.functional = Functional("whatever", [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  d = {'a1': 1, 'a0': 3, 'a3': 3, 'a2': 3, 'a5': 3, 'a4': 2, 'a7': 2, 'a6': 1}
  def processalloc_test1(job):  return d[job.name[1:-1]]

  program = PoolProcess(root, processalloc=processalloc_test1,
          outdir="whatever")
  program._comm = comm
  for i in xrange(10000):
    jobs = program._getjobs()
    assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
           (jobs, [program._alloc[u] for u in jobs])
示例#2
0
def test_getjobs(nprocs=8, njobs=20):
  """ Test scheduling. """
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.pool import PoolProcess
  from pylada import default_comm
  from functional import Functional

  root = JobFolder()
  for n in xrange(njobs):
    job = root / "a{0}".format(n)
    job.functional = Functional("whatever", [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = nprocs
  def processalloc(job):
    """ returns a random number between 1 and 4 included. """
    from random import randint
    return randint(1, comm['n'])

  for j in xrange(100):
    program = PoolProcess(root, processalloc=processalloc, outdir="whatever")
    program._comm = comm
    for i in xrange(1000):
      jobs = program._getjobs()
      assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
             (jobs, [program._alloc[u] for u in jobs])
示例#3
0
def test_getjobs(nprocs=8, njobs=20):
    """ Test scheduling. """
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.pool import PoolProcess
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(njobs):
        job = root / "a{0}".format(n)
        job.functional = Functional("whatever", [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = nprocs

    def processalloc(job):
        """ returns a random number between 1 and 4 included. """
        from random import randint
        return randint(1, comm['n'])

    for j in xrange(100):
        program = PoolProcess(root,
                              processalloc=processalloc,
                              outdir="whatever")
        program._comm = comm
        for i in xrange(1000):
            jobs = program._getjobs()
            assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
                   (jobs, [program._alloc[u] for u in jobs])
示例#4
0
def test_failures(d):
    """ Tests whether scheduling jobs works on known failure cases. """
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.pool import PoolProcess
    from pylada import default_comm
    from functional import Functional
    root = JobFolder()
    for n in xrange(8):
        job = root / "a{0}".format(n)
        job.functional = Functional("whatever", [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    def processalloc_test1(job):
        return d[job.name[1:-1]]

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir="whatever")
    program._comm = comm
    for i in xrange(10000):
        jobs = program._getjobs()
        assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
               (jobs, [program._alloc[u] for u in jobs])
示例#5
0
def test_update_with_fail():
  """ Tests JobFolderProcess with update. """
  from tempfile import mkdtemp
  from os.path import join, abspath, dirname
  from shutil import rmtree
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.jobfolder import save
  from pylada.process.pool import PoolProcess
  from pylada.process import Fail
  from pylada import default_comm
  from pylada.process.tests.functional import Functional
  from pylada.process.tests.pifunctional import __file__ as executable
  executable = join(dirname(executable), "pifunctional.py")

  root = JobFolder()
  for n in xrange(3):
    job = root / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  root['1'].functional.order = 50
  root['1'].functional.fail = 'midway'
  root['1'].sleep = None
  supp = JobFolder()
  for n in xrange(3, 6):
    job = supp / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  supp['5'].sleep = 0
  supp['5'].functional.order = 34
  supp['5'].functional.fail = 'end'

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  save(root, join(dir, 'dict.dict'), overwrite=True)
  try: 
    program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
    assert program.nbjobsleft > 0
    # compute current jobs.
    program.start(comm)
    try: program.wait()
    except Fail: pass
    else:
      print program.errors
      raise Exception()
    assert hasattr(program, '_comm')

    # compute second set of updated jobs
    program.update(supp)
    try: program.wait()
    except Fail: pass
    else: raise Exception()

  finally:
    try: rmtree(dir)
    except: pass
示例#6
0
def test_large():
  """ Test speed of job-scheduling for largers, more numerous jobs. """
  from random import random
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.pool import PoolProcess
  from pylada.process.dummy import DummyFunctional
  from pylada import default_comm
  root = JobFolder()
  for n in xrange(100):
    job = root / "a{0}".format(n)
    job.functional = DummyFunctional(chance=random()*0.5+0.15)

  comm = default_comm.copy()
  comm['n'] = 256

  def processalloc_test1(job): 
    """ returns a random number between 1 and 4 included. """
    from random import randint
    return randint(1, 64)

  program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
  program.start(comm)
  while not program.poll():
    # print 256 - program._comm['n'], len(program.process)
    continue

  def processalloc_test1(job): 
    """ returns a random number between 1 and 4 included. """
    from random import choice
    return choice([31, 37, 43])
  program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
  program.start(comm)
  while not program.poll():
    # print 256 - program._comm['n'], len(program.process)
    continue
示例#7
0
def test_getjobs(comm, tmpdir, executable, nprocs, njobs):
    """ Test scheduling. """
    root = jobfolders(executable, 0, 8)

    def processalloc(job):
        """ returns a random number between 1 and 4 included. """
        from random import randint
        return randint(1, comm['n'])

    for j in range(100):
        program = PoolProcess(root, processalloc=processalloc, outdir=str(tmpdir))
        program._comm = comm
        for i in range(1000):
            jobs = program._getjobs()
            assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
                (jobs, [program._alloc[u] for u in jobs])
示例#8
0
def test_update(executable):
    """ Tests JobFolderProcess with update. """
    from tempfile import mkdtemp
    from os.path import join
    from shutil import rmtree
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.jobfolder import save
    from pylada.process.pool import PoolProcess
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(3):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1
    supp = JobFolder()
    for n in xrange(3, 6):
        job = supp / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    dir = mkdtemp()
    save(root, join(dir, 'dict.dict'), overwrite=True)
    try:
        program = PoolProcess(root,
                              processalloc=processalloc,
                              outdir=dir,
                              keepalive=True)
        assert program.nbjobsleft > 0
        # compute current jobs.
        program.start(comm)
        program.wait()
        assert hasattr(program, '_comm')

        # compute second set of updated jobs
        program.update(supp)
        program.wait()

    finally:
        try:
            rmtree(dir)
        except:
            pass
示例#9
0
def test_failures(tmpdir, executable, comm):
    """Tests whether scheduling jobs works on known failure cases."""
    from pylada import default_comm
    root = jobfolders(executable, 0, 8)

    def processalloc_test1(job):
        d = {'1': 1, '0': 3, '3': 3, '2': 3, '5': 3, '4': 2, '7': 2, '6': 1}
        return d[job.name[1:-1]]

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir=str(tmpdir))
    program._comm = comm
    for i in range(10000):
        jobs = program._getjobs()
        assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
            (jobs, [program._alloc[u] for u in jobs])
示例#10
0
def test_failures(tmpdir, executable, comm):
    """ Tests whether scheduling jobs works on known failure cases. """
    from pylada import default_comm
    from pylada.process.tests.functional import Functional
    root = jobfolders(executable, 0, 8)

    def processalloc_test1(job):
        d = {'1': 1, '0': 3, '3': 3, '2': 3, '5': 3, '4': 2, '7': 2, '6': 1}
        return d[job.name[1:-1]]

    program = PoolProcess(root, processalloc=processalloc_test1,
                          outdir=str(tmpdir))
    program._comm = comm
    for i in range(10000):
        jobs = program._getjobs()
        assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
            (jobs, [program._alloc[u] for u in jobs])
示例#11
0
def test_getjobs(comm, tmpdir, executable, nprocs, njobs):
    """Test scheduling."""
    root = jobfolders(executable, 0, 8)

    def processalloc(job):
        """returns a random number between 1 and 4 included."""
        from random import randint
        return randint(1, comm['n'])

    for j in range(100):
        program = PoolProcess(root,
                              processalloc=processalloc,
                              outdir=str(tmpdir))
        program._comm = comm
        for i in range(1000):
            jobs = program._getjobs()
            assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
                (jobs, [program._alloc[u] for u in jobs])
示例#12
0
def test_large():
    """ Test speed of job-scheduling for largers, more numerous jobs. """
    from random import random
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.pool import PoolProcess
    from pylada.process.dummy import DummyFunctional
    from pylada import default_comm
    root = JobFolder()
    for n in xrange(100):
        job = root / "a{0}".format(n)
        job.functional = DummyFunctional(chance=random() * 0.5 + 0.15)

    comm = default_comm.copy()
    comm['n'] = 256

    def processalloc_test1(job):
        """ returns a random number between 1 and 4 included. """
        from random import randint
        return randint(1, 64)

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir="whatever")
    program.start(comm)
    while not program.poll():
        # print 256 - program._comm['n'], len(program.process)
        continue

    def processalloc_test1(job):
        """ returns a random number between 1 and 4 included. """
        from random import choice
        return choice([31, 37, 43])

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir="whatever")
    program.start(comm)
    while not program.poll():
        # print 256 - program._comm['n'], len(program.process)
        continue
示例#13
0
def test_update(executable):
  """ Tests JobFolderProcess with update. """
  from tempfile import mkdtemp
  from os.path import join
  from shutil import rmtree
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.jobfolder import save
  from pylada.process.pool import PoolProcess
  from pylada import default_comm
  from functional import Functional

  root = JobFolder()
  for n in xrange(3):
    job = root / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  supp = JobFolder()
  for n in xrange(3, 6):
    job = supp / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  save(root, join(dir, 'dict.dict'), overwrite=True)
  try: 
    program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
    assert program.nbjobsleft > 0
    # compute current jobs.
    program.start(comm)
    program.wait()
    assert hasattr(program, '_comm')

    # compute second set of updated jobs
    program.update(supp)
    program.wait()

  finally:
    try: rmtree(dir)
    except: pass
示例#14
0
def test_fakeexec():
  """ Tests Pool. Includes failure modes.  """
  from tempfile import mkdtemp
  from os.path import join, abspath, dirname
  from shutil import rmtree
  from numpy import all, arange, abs, array
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.jobfolder.massextract import MassExtract
  from pylada.jobfolder import save
  from pylada.process.pool import PoolProcess
  from pylada.process import Fail, NotStarted
  from pylada import default_comm
  from pylada.process.tests.functional import Functional
  from pylada.process.tests.pifunctional import __file__ as executable
  executable = join(dirname(executable), "pifunctional.py")

  root = JobFolder()
  for n in xrange(8):
    job = root / "a{0}".format(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  save(root, join(dir, 'dict.dict'), overwrite=True)
  try: 
    program = PoolProcess(root, processalloc=processalloc, outdir=dir)
    assert program.nbjobsleft > 0
    # program not started. should fail.
    try: program.poll()
    except NotStarted: pass
    else: raise Exception()
    try: program.wait()
    except NotStarted: pass
    else: raise Exception()
    
    # now starting for real.
    program.start(comm)
    program.wait()
    assert program.nbjobsleft == 0
    extract = MassExtract(join(dir, 'dict.dict'))
    assert all(extract.success.itervalues())
    order = array(extract.order.values()).flatten()
    assert all(arange(8) - order == 0)
    pi = array(extract.pi.values()).flatten()
    expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
            3.143293]
    assert all(abs(pi - array(expected)) < 1e-5)
    error = array(extract.error.values()).flatten()
    expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
            0.00333321, 0.002314774, 0.001700664]
    assert all(abs(error - array(expected)) < 1e-5)
    assert all(n['n'] == comm['n'] for n in extract.comm)
    # restart
    assert program.poll()
    assert len(program.process) == 0
    program.start(comm)
    assert len(program.process) == 0
    assert program.poll()
    
    try: 
      job = root / str(666)
      job.functional = Functional(executable, [50], fail='end')
      program = PoolProcess(root, nbpools=2,
              outdir=dir, processalloc=processalloc)
      assert program.nbjobsleft > 0
      program.start(comm)
      program.wait()
      assert False # Should not reach this point
    except Fail: 
      assert len(program.errors.keys()) == 1
      assert '666' in program.errors
    else: raise Exception()

    job.functional.order = [50]
    program = PoolProcess(root, nbpools=2,
            outdir=dir, processalloc=processalloc)
    assert program.nbjobsleft > 0
    program.start(comm)
    program.wait()
    assert program.nbjobsleft == 0
  finally:
    try: rmtree(dir)
    except: pass
示例#15
0
def test(executable):
    """ Tests JobFolderProcess. Includes failure modes.  """
    from tempfile import mkdtemp
    from os.path import join
    from shutil import rmtree
    from numpy import all, arange, abs, array
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.jobfolder.massextract import MassExtract
    from pylada.jobfolder import save
    from pylada.process.pool import PoolProcess
    from pylada.process import Fail, NotStarted
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(8):
        job = root / "a{0}".format(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    dir = mkdtemp()
    save(root, join(dir, 'dict.dict'), overwrite=True)
    try:
        program = PoolProcess(root, processalloc=processalloc, outdir=dir)
        assert program.nbjobsleft > 0
        # program not started. should fail.
        try:
            program.poll()
        except NotStarted:
            pass
        else:
            raise Exception()
        try:
            program.wait()
        except NotStarted:
            pass
        else:
            raise Exception()

        # now starting for real.
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
        extract = MassExtract(join(dir, 'dict.dict'))
        assert all(extract.success.itervalues())
        order = array(extract.order.values()).flatten()
        assert all(arange(8) - order == 0)
        pi = array(extract.pi.values()).flatten()
        assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
                                   3.146801, 3.144926, 3.143907, 3.143293]))\
                    < 1e-5 )
        error = array(extract.error.values()).flatten()
        assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
                                      0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
                    < 1e-5 )
        assert all(n['n'] == comm['n'] for n in extract.comm)
        # restart
        assert program.poll()
        assert len(program.process) == 0
        program.start(comm)
        assert len(program.process) == 0
        assert program.poll()
    finally:
        try:
            rmtree(dir)
        except:
            pass
    return

    try:
        job = root / str(666)
        job.functional = Functional(executable, [666])
        program = PoolProcess(root,
                              nbpools=2,
                              outdir=dir,
                              processalloc=processalloc)
        assert program.nbjobsleft > 0
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
    except Fail:
        assert len(program.errors.keys()) == 1
        assert '666' in program.errors
    else:
        raise Exception
    finally:
        try:
            rmtree(dir)
        except:
            pass
    try:
        job.functional.order = [667]
        program = PoolProcess(root,
                              nbpools=2,
                              outdir=dir,
                              processalloc=processalloc)
        assert program.nbjobsleft > 0
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
    finally:
        try:
            rmtree(dir)
        except:
            pass
示例#16
0
def pool_process(tmpdir, root, pal=processalloc, **kwargs):
    from pylada.process.pool import PoolProcess
    return PoolProcess(root, processalloc=pal, outdir=str(tmpdir), **kwargs)