def test_recyclingWithQueueOverload(self): """ Test that we get the correct number of different results when we overload the pool of calls. """ MAX = 5 MIN = 1 RECYCLE_AFTER = 10 CALLS = 60 pp = pool.ProcessPool(ampChild=PidChild, min=MIN, max=MAX, recycleAfter=RECYCLE_AFTER) self.addCleanup(pp.stop) def _check(results): s = set() for succeed, response in results: s.add(response['pid']) # For the first C{MAX} calls, each is basically guaranteed to go to # a different child. After that, though, there are no guarantees. # All the rest might go to a single child, since the child to # perform a job is selected arbitrarily from the "ready" set. Fair # distribution of jobs needs to be implemented; right now it's "set # ordering" distribution of jobs. self.assertTrue(len(s) > MAX) def _work(_): l = [pp.doWork(Pid) for x in xrange(CALLS)] d = defer.DeferredList(l) return d.addCallback(_check) d = pp.start() d.addCallback(_work) return d
def test_adjustPoolSize(self): """ Test that calls to pool.adjustPoolSize are correctly handled. """ pp = pool.ProcessPool(min=10) self.assertEquals(pp.started, False) self.assertEquals(pp.finished, False) self.assertEquals(pp.processes, set()) self.assertEquals(pp._finishCallbacks, {}) def _resize1(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) return pp.adjustPoolSize(min=2, max=3) def _resize2(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(pp.max, 3) self.assertEquals(pp.min, 2) self.assertEquals(len(pp.processes), pp.max) self.assertEquals(len(pp._finishCallbacks), pp.max) def _resize3(_): self.assertRaises(AssertionError, pp.adjustPoolSize, min=-1, max=5) self.assertRaises(AssertionError, pp.adjustPoolSize, min=5, max=1) return pp.stop() return pp.start( ).addCallback(_resize1 ).addCallback(_resize2 ).addCallback(_resize3)
def test_disableProcessRecycling(self): """ Test that by setting 0 to recycleAfter we actually disable process recycling. """ MAX = 1 MIN = 1 RECYCLE_AFTER = 0 pp = pool.ProcessPool(ampChild=PidChild, min=MIN, max=MAX, recycleAfter=RECYCLE_AFTER) def _checks(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) return pp.doWork(Pid ).addCallback(lambda response: response['pid']) def _checks2(pid): return pp.doWork(Pid ).addCallback(lambda response: response['pid'] ).addCallback(self.assertEquals, pid ).addCallback(lambda _: pid) def finish(reason): return pp.stop().addCallback(lambda _: reason) return pp.start( ).addCallback(_checks ).addCallback(_checks2 ).addCallback(_checks2 ).addCallback(finish)
def test_startAndStop(self): """ Test that a process pool's start and stop method create the expected number of workers and keep state consistent in the process pool. """ pp = pool.ProcessPool() self.assertEquals(pp.started, False) self.assertEquals(pp.finished, False) self.assertEquals(pp.processes, set()) self.assertEquals(pp._finishCallbacks, {}) def _checks(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) return pp.stop() def _closingUp(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, True) self.assertEquals(len(pp.processes), 0) self.assertEquals(pp._finishCallbacks, {}) return pp.start().addCallback(_checks).addCallback(_closingUp)
def test_startStopWorker(self): """ Test that starting and stopping a worker keeps the state of the process pool consistent. """ pp = pool.ProcessPool() self.assertEquals(pp.started, False) self.assertEquals(pp.finished, False) self.assertEquals(pp.processes, set()) self.assertEquals(pp._finishCallbacks, {}) def _checks(): self.assertEquals(pp.started, False) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), 1) self.assertEquals(len(pp._finishCallbacks), 1) return pp.stopAWorker() def _closingUp(_): self.assertEquals(pp.started, False) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), 0) self.assertEquals(pp._finishCallbacks, {}) pp.startAWorker() return _checks().addCallback(_closingUp).addCallback(lambda _: pp.stop())
def test_growingToMax(self): """ Test that the pool grows over time until it reaches max processes. """ MAX = 5 pp = pool.ProcessPool(ampChild=WaitingChild, min=1, max=MAX) def _checks(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) D = "DATA" d = [pp.doWork(First, data=D) for x in xrange(MAX)] self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.max) self.assertEquals(len(pp._finishCallbacks), pp.max) [child.callRemote(Second) for child in pp.processes] return defer.DeferredList(d) return pp.start( ).addCallback(_checks ).addCallback(lambda _: pp.stop())
def test_recycling(self): """ Test that after a given number of calls subprocesses are recycled. """ MAX = 1 MIN = 1 RECYCLE_AFTER = 1 pp = pool.ProcessPool(ampChild=PidChild, min=MIN, max=MAX, recycleAfter=RECYCLE_AFTER) self.addCleanup(pp.stop) def _checks(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) return pp.doWork(Pid ).addCallback(lambda response: response['pid']) def _checks2(pid): return pp.doWork(Pid ).addCallback(lambda response: response['pid'] ).addCallback(self.assertNotEquals, pid) d = pp.start() d.addCallback(_checks) d.addCallback(_checks2) return d
def checkPool(_): pp = pool.ProcessPool( starter=main.ProcessStarter( childReactor=SECOND, packages=("twisted",)), ampChild=ReactorChild, min=MIN, max=MAX) pp.start() return pp.doWork(Reactor ).addCallback(self.assertEquals, {'classname': "PollReactor"} ).addCallback(lambda _: pp.stop())
def start_plugin_services(server): """ This will be called by the Evennia Server when starting up. server - the main Evennia server application """ if not PROCPOOL_ENABLED: return # terminal output print ' amp (Process Pool): %s' % PROCPOOL_PORT from evennia.contrib.procpools.ampoule import main as ampoule_main from evennia.contrib.procpools.ampoule import service as ampoule_service from evennia.contrib.procpools.ampoule import pool as ampoule_pool from evennia.contrib.procpools.ampoule.main import BOOTSTRAP as _BOOTSTRAP from evennia.contrib.procpools.python_procpool import PythonProcPoolChild # for some reason absolute paths don't work here, only relative ones. apackages = ("twisted", "evennia", "settings") aenv = { "DJANGO_SETTINGS_MODULE": "settings", "DATABASE_NAME": settings.DATABASES.get("default", {}).get("NAME") or settings.DATABASE_NAME } if PROCPOOL_DEBUG: _BOOTSTRAP = _BOOTSTRAP % "log.startLogging(sys.stderr)" else: _BOOTSTRAP = _BOOTSTRAP % "" procpool_starter = ampoule_main.ProcessStarter( packages=apackages, env=aenv, path=PROCPOOL_DIRECTORY, uid=PROCPOOL_UID, gid=PROCPOOL_GID, bootstrap=_BOOTSTRAP, childReactor=sys.platform == 'linux2' and "epoll" or "default") procpool = ampoule_pool.ProcessPool(name=SERVICE_NAME, min=PROCPOOL_MIN_NPROC, max=PROCPOOL_MAX_NPROC, recycleAfter=500, timeout=PROCPOOL_TIMEOUT, maxIdle=PROCPOOL_IDLETIME, ampChild=PythonProcPoolChild, starter=procpool_starter) procpool_service = ampoule_service.AMPouleService(procpool, PythonProcPoolChild, PROCPOOL_PORT, PROCPOOL_INTERFACE) procpool_service.setName(SERVICE_NAME) # add the new services to the server server.services.addService(procpool_service)
def test_processBeforeDeadline(self): pp = pool.ProcessPool(PidChild, min=1, max=1) def _work(_): d = pp.callRemote(Pid, _deadline=reactor.seconds() + 10) d.addCallback(lambda result: self.assertNotEqual(result['pid'], 0)) return d return pp.start( ).addCallback(_work ).addCallback(lambda _: pp.stop())
def processTimeoutTest(self, timeout): pp = pool.ProcessPool(WaitingChild, min=1, max=1) def _work(_): d = pp.callRemote(First, data="ciao", _timeout=timeout) self.assertFailure(d, error.ProcessTerminated) return d return pp.start( ).addCallback(_work ).addCallback(lambda _: pp.stop())
def test_processDeadline(self): pp = pool.ProcessPool(WaitingChild, min=1, max=1) def _work(_): d = pp.callRemote(First, data="ciao", _deadline=reactor.seconds()) self.assertFailure(d, error.ProcessTerminated) return d return pp.start( ).addCallback(_work ).addCallback(lambda _: pp.stop())
def test_SupplyChildArgs(self): """Ensure that arguments for the child constructor are passed in.""" pp = pool.ProcessPool(Writer, ampChildArgs=['body'], min=0) def _check(result): return pp.doWork(Write).addCallback( self.assertEquals, {'response': 'body'}) return pp.start( ).addCallback(_check ).addCallback(lambda _: pp.stop())
def test_processGlobalTimeout(self): """ Test that a call that doesn't finish within the given global timeout time is correctly handled. """ pp = pool.ProcessPool(WaitingChild, min=1, max=1, timeout=1) def _work(_): d = pp.callRemote(First, data="ciao") self.assertFailure(d, error.ProcessTerminated) return d return pp.start( ).addCallback(_work ).addCallback(lambda _: pp.stop())
def test_growingToMaxAndShrinking(self): """ Test that the pool grows but after 'idle' time the number of processes goes back to the minimum. """ MAX = 5 MIN = 1 IDLE = 1 pp = pool.ProcessPool(ampChild=WaitingChild, min=MIN, max=MAX, maxIdle=IDLE) def _checks(_): self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) D = "DATA" d = [pp.doWork(First, data=D) for x in xrange(MAX)] self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.max) self.assertEquals(len(pp._finishCallbacks), pp.max) [child.callRemote(Second) for child in pp.processes] return defer.DeferredList(d).addCallback(_realChecks) def _realChecks(_): from twisted.internet import reactor d = defer.Deferred() def _cb(): def __(_): try: self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) d.callback(None) except Exception, e: d.errback(e) return pp._pruneProcesses().addCallback(__) # just to be shure we are called after the pruner pp.looping.stop() # stop the looping, we don't want it to # this right here reactor.callLater(IDLE, _cb) return d
def test_commandsWithoutResponse(self): """ Test that if we send a command without a required answer we actually don't have any problems. """ DATA = "hello" pp = pool.ProcessPool(ampChild=NoResponseChild, min=1, max=1) def _check(_): return pp.doWork(GetResponse ).addCallback(self.assertEquals, {"response": DATA}) def _work(_): return pp.doWork(NoResponse, arg=DATA) return pp.start( ).addCallback(_work ).addCallback(_check ).addCallback(lambda _: pp.stop())
def test_childRestart(self): """ Test that a failing child process is immediately restarted. """ pp = pool.ProcessPool(ampChild=BadChild, min=1) STRING = "DATA" def _checks(_): d = pp._finishCallbacks.values()[0] pp.doWork(Die).addErrback(lambda _: None) return d.addBoth(_checksAgain) def _checksAgain(_): return pp.doWork(commands.Echo, data=STRING ).addCallback(lambda result: self.assertEquals(result['response'], STRING)) return pp.start( ).addCallback(_checks ).addCallback(lambda _: pp.stop())
def test_processTimeoutSignal(self): """ Test that a call that doesn't finish within the given timeout time is correctly handled. """ pp = pool.ProcessPool(WaitingChild, min=1, max=1, timeout_signal=SIGHUP) def _work(_): d = pp.callRemote(First, data="ciao", _timeout=1) d.addCallback(lambda d: self.fail()) text = 'signal %d' % SIGHUP d.addErrback( lambda f: self.assertTrue(text in f.value[0], '"%s" not in "%s"' % (text, f.value[0]))) return d return pp.start( ).addCallback(_work ).addCallback(lambda _: pp.stop())
def test_parentProtocolChange(self): """ Test that the father can use an AMP protocol too. """ DATA = "CIAO" APPEND = "123" class Parent(amp.AMP): def pong(self, data): return {'response': DATA+APPEND} Pong.responder(pong) pp = pool.ProcessPool(ampChild=Child, ampParent=Parent) def _checks(_): return pp.doWork(Ping, data=DATA ).addCallback(lambda response: self.assertEquals(response['response'], DATA+APPEND) ) return pp.start().addCallback(_checks).addCallback(lambda _: pp.stop())
def setUp(self): """ Setup the proxy service and the client connection to the proxy service in order to run call through them. Inspiration comes from twisted.test.test_amp """ self.pp = pool.ProcessPool() self.svc = service.AMPouleService(self.pp, child.AMPChild, 0, "") self.svc.startService() self.proxy_port = self.svc.server.getHost().port self.clientFactory = ClientFactory() self.clientFactory.protocol = ClientAMP d = self.clientFactory.onMade = defer.Deferred() self.clientConn = reactor.connectTCP("127.0.0.1", self.proxy_port, self.clientFactory) self.addCleanup(self.clientConn.disconnect) self.addCleanup(self.svc.stopService) def setClient(_): self.client = self.clientFactory.theProto return d.addCallback(setClient)
def test_checkStateInPool(self): """ Test that busy and ready lists are correctly maintained. """ pp = pool.ProcessPool(ampChild=WaitingChild) DATA = "foobar" def _checks(_): d = pp.callRemote(First, data=DATA) self.assertEquals(pp.started, True) self.assertEquals(pp.finished, False) self.assertEquals(len(pp.processes), pp.min) self.assertEquals(len(pp._finishCallbacks), pp.min) self.assertEquals(len(pp.ready), pp.min-1) self.assertEquals(len(pp.busy), 1) child = pp.busy.pop() pp.busy.add(child) child.callRemote(Second) return d return pp.start( ).addCallback(_checks ).addCallback(lambda _: pp.stop())