Beispiel #1
0
def test_getjobs(nprocs=8, njobs=20):
  """ Test scheduling. """
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.pool import PoolProcess
  from pylada import default_comm
  from functional import Functional

  root = JobFolder()
  for n in xrange(njobs):
    job = root / "a{0}".format(n)
    job.functional = Functional("whatever", [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = nprocs
  def processalloc(job):
    """ returns a random number between 1 and 4 included. """
    from random import randint
    return randint(1, comm['n'])

  for j in xrange(100):
    program = PoolProcess(root, processalloc=processalloc, outdir="whatever")
    program._comm = comm
    for i in xrange(1000):
      jobs = program._getjobs()
      assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
             (jobs, [program._alloc[u] for u in jobs])
Beispiel #2
0
def test_update(executable):
  """ Tests JobFolderProcess with update. """
  from tempfile import mkdtemp
  from shutil import rmtree
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.jobfolder import JobFolderProcess
  from pylada import default_comm
  from functional import Functional

  root = JobFolder()
  for n in xrange(3):
    job = root / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  supp = JobFolder()
  for n in xrange(3, 6):
    job = supp / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  try: 
    program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
    assert program.keepalive 

    # compute current jobs.
    program.start(comm)
    program.wait()
    assert hasattr(program, '_comm')

    # compute second set of updated jobs
    program.update(supp)
    program.wait()

  finally:
    try: rmtree(dir)
    except: pass

  # check with deleteold=True
  dir = mkdtemp()
  try: 
    program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
    assert program.keepalive 

    # compute current jobs.
    program.start(comm)
    program.wait()
    assert hasattr(program, '_comm')

    # compute second set of updated jobs
    program.update(supp, deleteold=True)
    assert hasattr(program, '_comm')
    program.wait()

  finally:
    try: rmtree(dir)
    except: pass
Beispiel #3
0
def test_getjobs(nprocs=8, njobs=20):
    """ Test scheduling. """
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.pool import PoolProcess
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(njobs):
        job = root / "a{0}".format(n)
        job.functional = Functional("whatever", [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = nprocs

    def processalloc(job):
        """ returns a random number between 1 and 4 included. """
        from random import randint
        return randint(1, comm['n'])

    for j in xrange(100):
        program = PoolProcess(root,
                              processalloc=processalloc,
                              outdir="whatever")
        program._comm = comm
        for i in xrange(1000):
            jobs = program._getjobs()
            assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
                   (jobs, [program._alloc[u] for u in jobs])
Beispiel #4
0
def test_failures(d):
    """ Tests whether scheduling jobs works on known failure cases. """
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.pool import PoolProcess
    from pylada import default_comm
    from functional import Functional
    root = JobFolder()
    for n in xrange(8):
        job = root / "a{0}".format(n)
        job.functional = Functional("whatever", [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    def processalloc_test1(job):
        return d[job.name[1:-1]]

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir="whatever")
    program._comm = comm
    for i in xrange(10000):
        jobs = program._getjobs()
        assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
               (jobs, [program._alloc[u] for u in jobs])
Beispiel #5
0
def test_large():
  """ Test speed of job-scheduling for largers, more numerous jobs. """
  from random import random
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.pool import PoolProcess
  from pylada.process.dummy import DummyFunctional
  from pylada import default_comm
  root = JobFolder()
  for n in xrange(100):
    job = root / "a{0}".format(n)
    job.functional = DummyFunctional(chance=random()*0.5+0.15)

  comm = default_comm.copy()
  comm['n'] = 256

  def processalloc_test1(job): 
    """ returns a random number between 1 and 4 included. """
    from random import randint
    return randint(1, 64)

  program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
  program.start(comm)
  while not program.poll():
    # print 256 - program._comm['n'], len(program.process)
    continue

  def processalloc_test1(job): 
    """ returns a random number between 1 and 4 included. """
    from random import choice
    return choice([31, 37, 43])
  program = PoolProcess(root, processalloc=processalloc_test1, outdir="whatever")
  program.start(comm)
  while not program.poll():
    # print 256 - program._comm['n'], len(program.process)
    continue
Beispiel #6
0
def test_failures():
  """ Tests whether scheduling jobs works on known failure cases. """
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.pool import PoolProcess
  from pylada import default_comm
  from functional import Functional
  root = JobFolder()
  for n in xrange(8):
    job = root / "a{0}".format(n)
    job.functional = Functional("whatever", [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  d = {'a1': 1, 'a0': 3, 'a3': 3, 'a2': 3, 'a5': 3, 'a4': 2, 'a7': 2, 'a6': 1}
  def processalloc_test1(job):  return d[job.name[1:-1]]

  program = PoolProcess(root, processalloc=processalloc_test1,
          outdir="whatever")
  program._comm = comm
  for i in xrange(10000):
    jobs = program._getjobs()
    assert sum(program._alloc[u] for u in jobs) <= program._comm['n'],\
           (jobs, [program._alloc[u] for u in jobs])
Beispiel #7
0
def test_update_with_fail(executable):
    """ Tests JobFolderProcess with update. """
    from tempfile import mkdtemp
    from shutil import rmtree
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.jobfolder import JobFolderProcess
    from pylada.process import Fail
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(3):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params["sleep"] = 1
    root["1"].functional.order = 666
    root["1"].sleep = None
    supp = JobFolder()
    for n in xrange(3, 6):
        job = supp / str(n)
        job.functional = Functional(executable, [n])
        job.params["sleep"] = 1
    supp["5"].sleep = 0
    supp["5"].functional.order = 666

    comm = default_comm.copy()
    comm["n"] = 4

    dir = mkdtemp()
    try:
        program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)

        # compute current jobs.
        program.start(comm)
        try:
            program.wait()
        except Fail:
            pass
        else:
            raise Exception()
        assert len(program.errors) == 1

        # compute second set of updated jobs
        program.update(supp)
        try:
            program.wait()
        except Fail:
            pass
        else:
            raise Exception()
        assert len(program.errors) == 2
        program.errors.clear()

    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #8
0
def test_update_with_fail(executable):
    """ Tests JobFolderProcess with update. """
    from tempfile import mkdtemp
    from shutil import rmtree
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.jobfolder import JobFolderProcess
    from pylada.process import Fail
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(3):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1
    root['1'].functional.order = 666
    root['1'].sleep = None
    supp = JobFolder()
    for n in xrange(3, 6):
        job = supp / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1
    supp['5'].sleep = 0
    supp['5'].functional.order = 666

    comm = default_comm.copy()
    comm['n'] = 4

    dir = mkdtemp()
    try:
        program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)

        # compute current jobs.
        program.start(comm)
        try:
            program.wait()
        except Fail:
            pass
        else:
            raise Exception()
        assert len(program.errors) == 1

        # compute second set of updated jobs
        program.update(supp)
        try:
            program.wait()
        except Fail:
            pass
        else:
            raise Exception()
        assert len(program.errors) == 2
        program.errors.clear()

    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #9
0
def test_update_with_fail():
  """ Tests JobFolderProcess with update. """
  from tempfile import mkdtemp
  from os.path import join, abspath, dirname
  from shutil import rmtree
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.jobfolder import save
  from pylada.process.pool import PoolProcess
  from pylada.process import Fail
  from pylada import default_comm
  from pylada.process.tests.functional import Functional
  from pylada.process.tests.pifunctional import __file__ as executable
  executable = join(dirname(executable), "pifunctional.py")

  root = JobFolder()
  for n in xrange(3):
    job = root / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  root['1'].functional.order = 50
  root['1'].functional.fail = 'midway'
  root['1'].sleep = None
  supp = JobFolder()
  for n in xrange(3, 6):
    job = supp / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  supp['5'].sleep = 0
  supp['5'].functional.order = 34
  supp['5'].functional.fail = 'end'

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  save(root, join(dir, 'dict.dict'), overwrite=True)
  try: 
    program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
    assert program.nbjobsleft > 0
    # compute current jobs.
    program.start(comm)
    try: program.wait()
    except Fail: pass
    else:
      print program.errors
      raise Exception()
    assert hasattr(program, '_comm')

    # compute second set of updated jobs
    program.update(supp)
    try: program.wait()
    except Fail: pass
    else: raise Exception()

  finally:
    try: rmtree(dir)
    except: pass
Beispiel #10
0
def test_update_with_fail():
  """ Tests JobFolderProcess with update and failure. """
  from os.path import join, abspath, dirname
  from tempfile import mkdtemp
  from shutil import rmtree
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.process.jobfolder import JobFolderProcess
  from pylada.process import Fail
  from pylada import default_comm
  from pylada.process.tests.functional import Functional
  from pylada.process.tests.pifunctional import __file__ as executable
  executable = join(dirname(executable), "pifunctional.py")

  root = JobFolder()
  for n in xrange(3):
    job = root / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  root['1'].functional.order = 68
  root['1'].functional.fail = 'end'
  root['1'].sleep = None
  supp = JobFolder()
  for n in xrange(3, 6):
    job = supp / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  supp['5'].sleep = 0
  supp['5'].functional.order = 78
  supp['5'].functional.fail = 'midway'

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  try: 
    program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)

    # compute current jobs.
    program.start(comm)
    try: program.wait()
    except Fail: pass
    else: raise Exception()
    assert len(program.errors) == 1

    # compute second set of updated jobs
    program.update(supp)
    try: program.wait()
    except Fail: pass
    else: raise Exception()
    assert len(program.errors) == 2
    program.errors.clear()


  finally:
    try: rmtree(dir)
    except: pass
Beispiel #11
0
    def __init__(self, *args, **kwargs):
        from pylada import default_comm
        keywords = default_comm.copy()
        keywords['n'] = 0
        keywords.update(kwargs)
        super(Communicator, self).__init__(*args)
        super(Communicator, self).update(keywords)
        self._nodefile = None
        """ Path to temporary node file.

            None if no temporary node file yet.
        """
        self.machines = {}
        """ Holds map of machines this communicator can access. """
        self.parent = None
        """ Reference to parent communicator from which machines are acquired. 
Beispiel #12
0
def test_update(executable):
    """ Tests JobFolderProcess with update. """
    from tempfile import mkdtemp
    from os.path import join
    from shutil import rmtree
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.jobfolder import save
    from pylada.process.pool import PoolProcess
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(3):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1
    supp = JobFolder()
    for n in xrange(3, 6):
        job = supp / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    dir = mkdtemp()
    save(root, join(dir, 'dict.dict'), overwrite=True)
    try:
        program = PoolProcess(root,
                              processalloc=processalloc,
                              outdir=dir,
                              keepalive=True)
        assert program.nbjobsleft > 0
        # compute current jobs.
        program.start(comm)
        program.wait()
        assert hasattr(program, '_comm')

        # compute second set of updated jobs
        program.update(supp)
        program.wait()

    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #13
0
def test_update(executable):
  """ Tests JobFolderProcess with update. """
  from tempfile import mkdtemp
  from os.path import join
  from shutil import rmtree
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.jobfolder import save
  from pylada.process.pool import PoolProcess
  from pylada import default_comm
  from functional import Functional

  root = JobFolder()
  for n in xrange(3):
    job = root / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1
  supp = JobFolder()
  for n in xrange(3, 6):
    job = supp / str(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  save(root, join(dir, 'dict.dict'), overwrite=True)
  try: 
    program = PoolProcess(root, processalloc=processalloc, outdir=dir, keepalive=True)
    assert program.nbjobsleft > 0
    # compute current jobs.
    program.start(comm)
    program.wait()
    assert hasattr(program, '_comm')

    # compute second set of updated jobs
    program.update(supp)
    program.wait()

  finally:
    try: rmtree(dir)
    except: pass
Beispiel #14
0
def test_large():
    """ Test speed of job-scheduling for largers, more numerous jobs. """
    from random import random
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.pool import PoolProcess
    from pylada.process.dummy import DummyFunctional
    from pylada import default_comm
    root = JobFolder()
    for n in xrange(100):
        job = root / "a{0}".format(n)
        job.functional = DummyFunctional(chance=random() * 0.5 + 0.15)

    comm = default_comm.copy()
    comm['n'] = 256

    def processalloc_test1(job):
        """ returns a random number between 1 and 4 included. """
        from random import randint
        return randint(1, 64)

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir="whatever")
    program.start(comm)
    while not program.poll():
        # print 256 - program._comm['n'], len(program.process)
        continue

    def processalloc_test1(job):
        """ returns a random number between 1 and 4 included. """
        from random import choice
        return choice([31, 37, 43])

    program = PoolProcess(root,
                          processalloc=processalloc_test1,
                          outdir="whatever")
    program.start(comm)
    while not program.poll():
        # print 256 - program._comm['n'], len(program.process)
        continue
Beispiel #15
0
def test(executable):
    """ Tests JobFolderProcess. Includes failure modes.  """
    from tempfile import mkdtemp
    from os.path import join
    from shutil import rmtree
    from numpy import all, arange, abs, array
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.jobfolder.massextract import MassExtract
    from pylada.jobfolder import save
    from pylada.process.jobfolder import JobFolderProcess
    from pylada.process import Fail, AlreadyStarted, NotStarted
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(8):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    dir = mkdtemp()
    try:
        program = JobFolderProcess(root, nbpools=2, outdir=dir)
        assert program.nbjobsleft > 0
        # program not started. should fail.
        try:
            program.poll()
        except NotStarted:
            pass
        else:
            raise Exception()
        try:
            program.wait()
        except NotStarted:
            pass
        else:
            raise Exception()

        # now starting for real.
        program.start(comm)
        assert len(program.process) == 2
        # Should not be possible to start twice.
        try:
            program.start(comm)
        except AlreadyStarted:
            pass
        else:
            raise Exception()
        while not program.poll():
            continue
        assert program.nbjobsleft == 0
        save(root, join(dir, 'dict.dict'), overwrite=True)
        extract = MassExtract(join(dir, 'dict.dict'))
        assert all(extract.success.itervalues())
        order = array(extract.order.values()).flatten()
        assert all(arange(8) - order == 0)
        pi = array(extract.pi.values()).flatten()
        assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849,
                                   3.146801, 3.144926, 3.143907, 3.143293]))\
                    < 1e-5 )
        error = array(extract.error.values()).flatten()
        assert all(abs(error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
                                      0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
                    < 1e-5 )
        assert all(n['n'] == comm['n'] for n in extract.comm)
        # restart
        assert program.poll()
        assert len(program.process) == 0
        program.start(comm)
        assert len(program.process) == 0
        assert program.poll()
    finally:
        try:
            rmtree(dir)
        except:
            pass

    try:
        job = root / str(666)
        job.functional = Functional(executable, [666])
        program = JobFolderProcess(root, nbpools=2, outdir=dir)
        assert program.nbjobsleft > 0
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
    except Fail as r:
        assert len(program.errors.keys()) == 1
        assert '666' in program.errors
        assert len(program._finished) == 8
    else:
        raise Exception
    finally:
        try:
            rmtree(dir)
        except:
            pass
    try:
        job.functional.order = [667]
        program = JobFolderProcess(root, nbpools=2, outdir=dir)
        assert program.nbjobsleft > 0
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #16
0
def test_update(executable):
    """ Tests JobFolderProcess with update. """
    from tempfile import mkdtemp
    from shutil import rmtree
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.process.jobfolder import JobFolderProcess
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(3):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1
    supp = JobFolder()
    for n in xrange(3, 6):
        job = supp / str(n)
        job.functional = Functional(executable, [n])
        job.params['sleep'] = 1

    comm = default_comm.copy()
    comm['n'] = 4

    dir = mkdtemp()
    try:
        program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
        assert program.keepalive

        # compute current jobs.
        program.start(comm)
        program.wait()
        assert hasattr(program, '_comm')

        # compute second set of updated jobs
        program.update(supp)
        program.wait()

    finally:
        try:
            rmtree(dir)
        except:
            pass

    # check with deleteold=True
    dir = mkdtemp()
    try:
        program = JobFolderProcess(root, nbpools=2, outdir=dir, keepalive=True)
        assert program.keepalive

        # compute current jobs.
        program.start(comm)
        program.wait()
        assert hasattr(program, '_comm')

        # compute second set of updated jobs
        program.update(supp, deleteold=True)
        assert hasattr(program, '_comm')
        program.wait()

    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #17
0
def test(executable):
    """ Tests IteratorProcess. Includes failure modes.  """
    from tempfile import mkdtemp
    from shutil import rmtree
    from numpy import all, arange, abs, array
    from pylada.process.iterator import IteratorProcess
    from pylada.process import Fail, NotStarted
    from pylada import default_comm
    from functional import Functional
    comm = default_comm.copy()
    dir = mkdtemp()
    try:
        functional = Functional(executable, range(8))
        program = IteratorProcess(functional, outdir=dir)
        # program not started. should fail.
        try:
            program.poll()
        except NotStarted:
            pass
        else:
            raise Exception()
        try:
            program.wait()
        except NotStarted:
            pass
        else:
            raise Exception()

        # now starting for real.
        program.start(comm)
        while not program.poll():
            continue
        extract = functional.Extract(dir)
        assert extract.success
        assert all(arange(8) - extract.order == 0)
        assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
                                           3.146801, 3.144926, 3.143907, 3.143293]))\
                    < 1e-5 )
        assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
                                              0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
                    < 1e-5 )
        assert all(n['n'] == comm['n'] for n in extract.comm)
        # restart
        assert program.poll()
        program.start(comm)
        assert program.process is None
        assert program.poll()
        # true restart
        program = IteratorProcess(functional, outdir=dir)
        program.start(comm)
        assert program.process is None
        assert program.poll()
        extract = functional.Extract(dir)
        assert extract.success
        assert all(arange(8) - extract.order == 0)
        assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
                                           3.146801, 3.144926, 3.143907, 3.143293]))\
                    < 1e-5 )
        assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
                                              0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
                    < 1e-5 )
    finally:
        try:
            rmtree(dir)
        except:
            pass

    try:
        functional = Functional(executable, [666])
        program = IteratorProcess(functional, outdir=dir)
        program.start(comm)
        program.wait()
    except Fail:
        pass
    else:
        raise Exception
    finally:
        try:
            rmtree(dir)
        except:
            pass
    try:
        functional = Functional(executable, [667])
        program = IteratorProcess(functional, outdir=dir)
        program.start(comm)
        program.wait()
    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #18
0
def comm():
    from pylada import default_comm
    result = default_comm.copy()
    result['n'] = 4
    return result
Beispiel #19
0
def test(executable):
    """ Tests JobFolderProcess. Includes failure modes.  """
    from tempfile import mkdtemp
    from os.path import join
    from shutil import rmtree
    from numpy import all, arange, abs, array
    from pylada.jobfolder.jobfolder import JobFolder
    from pylada.jobfolder.massextract import MassExtract
    from pylada.jobfolder import save
    from pylada.process.jobfolder import JobFolderProcess
    from pylada.process import Fail, AlreadyStarted, NotStarted
    from pylada import default_comm
    from functional import Functional

    root = JobFolder()
    for n in xrange(8):
        job = root / str(n)
        job.functional = Functional(executable, [n])
        job.params["sleep"] = 1

    comm = default_comm.copy()
    comm["n"] = 4

    dir = mkdtemp()
    try:
        program = JobFolderProcess(root, nbpools=2, outdir=dir)
        assert program.nbjobsleft > 0
        # program not started. should fail.
        try:
            program.poll()
        except NotStarted:
            pass
        else:
            raise Exception()
        try:
            program.wait()
        except NotStarted:
            pass
        else:
            raise Exception()

        # now starting for real.
        program.start(comm)
        assert len(program.process) == 2
        # Should not be possible to start twice.
        try:
            program.start(comm)
        except AlreadyStarted:
            pass
        else:
            raise Exception()
        while not program.poll():
            continue
        assert program.nbjobsleft == 0
        save(root, join(dir, "dict.dict"), overwrite=True)
        extract = MassExtract(join(dir, "dict.dict"))
        assert all(extract.success.itervalues())
        order = array(extract.order.values()).flatten()
        assert all(arange(8) - order == 0)
        pi = array(extract.pi.values()).flatten()
        assert all(abs(pi - array([0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907, 3.143293])) < 1e-5)
        error = array(extract.error.values()).flatten()
        assert all(
            abs(
                error
                - array(
                    [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865, 0.00333321, 0.002314774, 0.001700664]
                )
            )
            < 1e-5
        )
        assert all(n["n"] == comm["n"] for n in extract.comm)
        # restart
        assert program.poll()
        assert len(program.process) == 0
        program.start(comm)
        assert len(program.process) == 0
        assert program.poll()
    finally:
        try:
            rmtree(dir)
        except:
            pass

    try:
        job = root / str(666)
        job.functional = Functional(executable, [666])
        program = JobFolderProcess(root, nbpools=2, outdir=dir)
        assert program.nbjobsleft > 0
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
    except Fail as r:
        assert len(program.errors.keys()) == 1
        assert "666" in program.errors
        assert len(program._finished) == 8
    else:
        raise Exception
    finally:
        try:
            rmtree(dir)
        except:
            pass
    try:
        job.functional.order = [667]
        program = JobFolderProcess(root, nbpools=2, outdir=dir)
        assert program.nbjobsleft > 0
        program.start(comm)
        program.wait()
        assert program.nbjobsleft == 0
    finally:
        try:
            rmtree(dir)
        except:
            pass
Beispiel #20
0
def test(executable):
  """ Tests IteratorProcess. Includes failure modes.  """
  from tempfile import mkdtemp
  from shutil import rmtree
  from numpy import all, arange, abs, array
  from pylada.process.iterator import IteratorProcess
  from pylada.process import Fail, NotStarted
  from pylada import default_comm
  from functional import Functional
  comm = default_comm.copy()
  dir = mkdtemp()
  try: 
    functional = Functional(executable, range(8))
    program = IteratorProcess(functional, outdir=dir)
    # program not started. should fail.
    try: program.poll()
    except NotStarted: pass
    else: raise Exception()
    try: program.wait()
    except NotStarted: pass
    else: raise Exception()

    # now starting for real.
    program.start(comm)
    while not program.poll():  continue
    extract = functional.Extract(dir)
    assert extract.success
    assert all(arange(8) - extract.order == 0)
    assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
                                       3.146801, 3.144926, 3.143907, 3.143293]))\
                < 1e-5 )
    assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
                                          0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
                < 1e-5 )
    assert all(n['n'] == comm['n'] for n in extract.comm)
    # restart
    assert program.poll()
    program.start(comm)
    assert program.process is None
    assert program.poll()
    # true restart
    program = IteratorProcess(functional, outdir=dir)
    program.start(comm)
    assert program.process is None
    assert program.poll()
    extract = functional.Extract(dir)
    assert extract.success
    assert all(arange(8) - extract.order == 0)
    assert all(abs(extract.pi - array([0.0, 3.2, 3.162353, 3.150849,
                                       3.146801, 3.144926, 3.143907, 3.143293]))\
                < 1e-5 )
    assert all(abs(extract.error - array([3.141593, 0.05840735, 0.02076029, 0.009256556,
                                          0.005207865, 0.00333321, 0.002314774, 0.001700664]))\
                < 1e-5 )
  finally:
    try: rmtree(dir)
    except: pass

  try: 
    functional = Functional(executable, [666])
    program = IteratorProcess(functional, outdir=dir)
    program.start(comm)
    program.wait()
  except Fail: pass
  else: raise Exception
  finally:
    try: rmtree(dir)
    except: pass
  try: 
    functional = Functional(executable, [667])
    program = IteratorProcess(functional, outdir=dir)
    program.start(comm)
    program.wait()
  finally:
    try: rmtree(dir)
    except: pass
Beispiel #21
0
def test_call():
  """ Tests CallProcess. Includes failure modes.  """
  from tempfile import mkdtemp
  from os.path import join, abspath, dirname
  from shutil import rmtree
  from numpy import all, arange, abs, array
  from pylada.process.call import CallProcess
  from pylada.process import Fail, NotStarted
  from pylada import default_comm
  from pylada.process.tests.functional import Functional
  from pylada.process.tests.pifunctional import __file__ as executable
  executable = join(dirname(executable), "pifunctional.py")

  comm = default_comm.copy()
  dir = mkdtemp()
  try: 
    functional = Functional(executable, range(8))
    program = CallProcess(functional, outdir=dir, dompi=False )
    # program not started. should fail.
    try: program.poll()
    except NotStarted: pass
    else: raise Exception()
    try: program.wait()
    except NotStarted: pass
    else: raise Exception()

    # now starting for real.
    assert program.start(comm) == False
    assert program.process is not None
    while not program.poll():  continue
    extract = functional.Extract(dir)
    assert extract.success
    assert all(arange(8) - extract.order == 0)
    expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
            3.143293]
    assert all(abs(extract.pi - array(expected)) < 1e-5)
    expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
            0.00333321, 0.002314774, 0.001700664]
    assert all(abs(extract.error -array(expected)) < 1e-5)
    assert all(n['n'] == comm['n'] for n in extract.comm)
    # restart
    assert program.start(comm)
    assert program.process is None
    # true restart
    program = CallProcess(functional, outdir=dir, dompi=False )
    assert program.start(comm)
    assert program.process is None
    assert program.poll()
    extract = functional.Extract(dir)
    assert extract.success
    assert all(arange(8) - extract.order == 0)
    expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
            3.143293]
    assert all(abs(extract.pi - array(expected)) < 1e-5)
    expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
            0.00333321, 0.002314774, 0.001700664]
    assert all(abs(extract.error - array(expected)) < 1e-5)
  finally:
    try: rmtree(dir)
    except: pass

  try: 
    functional = Functional(executable, [50], fail='midway')
    program = CallProcess(functional, outdir=dir, stderr=join(dir, 'error'),
            dompi=False)
    program.start(comm)
    program.wait()
  except Fail: pass
  else: raise Exception
  finally:
    try: rmtree(dir)
    except: pass
  try: 
    functional = Functional(executable, [50])
    program = CallProcess(functional, outdir=dir, dompi=False )
    program.start(comm)
    program.wait()
  finally:
    try: rmtree(dir)
    except: pass
Beispiel #22
0
def test_fakeexec():
  """ Tests Pool. Includes failure modes.  """
  from tempfile import mkdtemp
  from os.path import join, abspath, dirname
  from shutil import rmtree
  from numpy import all, arange, abs, array
  from pylada.jobfolder.jobfolder import JobFolder
  from pylada.jobfolder.massextract import MassExtract
  from pylada.jobfolder import save
  from pylada.process.pool import PoolProcess
  from pylada.process import Fail, NotStarted
  from pylada import default_comm
  from pylada.process.tests.functional import Functional
  from pylada.process.tests.pifunctional import __file__ as executable
  executable = join(dirname(executable), "pifunctional.py")

  root = JobFolder()
  for n in xrange(8):
    job = root / "a{0}".format(n)
    job.functional = Functional(executable, [n])
    job.params['sleep'] = 1

  comm = default_comm.copy()
  comm['n'] = 4

  dir = mkdtemp()
  save(root, join(dir, 'dict.dict'), overwrite=True)
  try: 
    program = PoolProcess(root, processalloc=processalloc, outdir=dir)
    assert program.nbjobsleft > 0
    # program not started. should fail.
    try: program.poll()
    except NotStarted: pass
    else: raise Exception()
    try: program.wait()
    except NotStarted: pass
    else: raise Exception()
    
    # now starting for real.
    program.start(comm)
    program.wait()
    assert program.nbjobsleft == 0
    extract = MassExtract(join(dir, 'dict.dict'))
    assert all(extract.success.itervalues())
    order = array(extract.order.values()).flatten()
    assert all(arange(8) - order == 0)
    pi = array(extract.pi.values()).flatten()
    expected = [0.0, 3.2, 3.162353, 3.150849, 3.146801, 3.144926, 3.143907,
            3.143293]
    assert all(abs(pi - array(expected)) < 1e-5)
    error = array(extract.error.values()).flatten()
    expected = [3.141593, 0.05840735, 0.02076029, 0.009256556, 0.005207865,
            0.00333321, 0.002314774, 0.001700664]
    assert all(abs(error - array(expected)) < 1e-5)
    assert all(n['n'] == comm['n'] for n in extract.comm)
    # restart
    assert program.poll()
    assert len(program.process) == 0
    program.start(comm)
    assert len(program.process) == 0
    assert program.poll()
    
    try: 
      job = root / str(666)
      job.functional = Functional(executable, [50], fail='end')
      program = PoolProcess(root, nbpools=2,
              outdir=dir, processalloc=processalloc)
      assert program.nbjobsleft > 0
      program.start(comm)
      program.wait()
      assert False # Should not reach this point
    except Fail: 
      assert len(program.errors.keys()) == 1
      assert '666' in program.errors
    else: raise Exception()

    job.functional.order = [50]
    program = PoolProcess(root, nbpools=2,
            outdir=dir, processalloc=processalloc)
    assert program.nbjobsleft > 0
    program.start(comm)
    program.wait()
    assert program.nbjobsleft == 0
  finally:
    try: rmtree(dir)
    except: pass
Beispiel #23
0
def comm():
    from pylada import default_comm
    result = default_comm.copy()
    result['n'] = 4
    return result