def __init__(self, ampChild=None, ampParent=None, min=5, max=20, name=None, maxIdle=20, recycleAfter=500, starter=None, timeout=None, timeout_signal=DIE, ampChildArgs=()): self.starter = starter self.ampChildArgs = tuple(ampChildArgs) if starter is None: self.starter = main.ProcessStarter(packages=("twisted", "ampoule")) self.ampParent = ampParent self.ampChild = ampChild if ampChild is None: from ampoule.child import AMPChild self.ampChild = AMPChild self.min = min self.max = max self.name = name self.maxIdle = maxIdle self.recycleAfter = recycleAfter self.timeout = timeout self.timeout_signal = timeout_signal self._queue = [] self.processes = set() self.ready = set() self.busy = set() self._finishCallbacks = {} self._lastUsage = {} self._calls = {} self.looping = task.LoopingCall(self._pruneProcesses) self.looping.start(maxIdle, now=False)
def checkPool(_): pp = pool.ProcessPool(starter=main.ProcessStarter( childReactor=SECOND, packages=("twisted", "ampoule")), ampChild=ReactorChild, min=MIN, max=MAX) pp.start() return (pp.doWork(Reactor).addCallback(self.assertEquals, { 'classname': b"PollReactor" }).addCallback(lambda _: pp.stop()))
def test_roundtripError(self): """ Test that invoking a child using an unreachable class raises a L{RunTimeError} . """ class Child(child.AMPChild): pass starter = main.ProcessStarter(packages=("twisted", "ampoule")) self.assertRaises(RuntimeError, starter.startAMPProcess, ampChild=Child)
def test_BootstrapContext(self): starter = main.ProcessStarter(packages=('twisted', 'ampoule')) c, finished = starter.startAMPProcess(TempDirChild) cwd = [] def checkBootstrap(response): cwd.append(response['cwd']) self.assertNotEquals(cwd, os.getcwd()) d = c.callRemote(GetCWD) d.addCallback(checkBootstrap) d.addCallback(lambda _: c.callRemote(commands.Shutdown)) finished.addCallback(lambda _: self.assertFalse(os.path.exists(cwd[0]))) return finished
def test_startAMPProcess(self): """ Test that you can start an AMP subprocess and that it correctly accepts commands and correctly answers them. """ STRING = b"ciao" starter = main.ProcessStarter(packages=("twisted", "ampoule")) c, finished = starter.startAMPProcess(child.AMPChild) c.callRemote(commands.Echo, data=STRING).addCallback( lambda response: self.assertEquals(response['response'], STRING) ).addCallback(lambda _: c.callRemote(commands.Shutdown)) return finished
def __init__(self, job_source, dbuser, logger=None, error_utility=None): env = {'PATH': os.environ['PATH']} if 'LPCONFIG' in os.environ: env['LPCONFIG'] = os.environ['LPCONFIG'] env['PYTHONPATH'] = os.pathsep.join(sys.path) starter = main.ProcessStarter(env=env) super(TwistedJobRunner, self).__init__(logger, error_utility) self.job_source = job_source self.import_name = '%s.%s' % ( removeSecurityProxy(job_source).__module__, job_source.__name__) self.pool = pool.ProcessPool( JobRunnerProcess, ampChildArgs=[self.import_name, str(dbuser)], starter=starter, min=0, timeout_signal=SIGHUP)
def test_env_setting(self): """ Test that and environment variable passed to the process starter is correctly passed to the child process. """ s = sio() a = FakeAMP(s) STRING = b"ciao" BOOT = """\ import sys, io, os def main(): with io.open(4, 'w' + ('b' if bytes is str else '')) as f: f.write(os.environ['FOOBAR']) main() """ starter = main.ProcessStarter(bootstrap=BOOT, packages=("twisted", "ampoule"), env={"FOOBAR": STRING}) amp, finished = starter.startPythonProcess(main.AMPConnector(a), "I'll be ignored") return finished.addCallback(lambda _: self.assertEquals(s.getvalue(), STRING))
def test_failing_deferToProcess(self): """ Test failing subprocesses and the way they terminate and preserve failing information. """ s = sio() a = FakeAMP(s) STRING = b"ciao" BOOT = """\ import sys def main(arg): raise Exception(arg) main(sys.argv[1]) """ starter = main.ProcessStarter(bootstrap=BOOT, args=(STRING,), packages=("twisted", "ampoule")) ready, finished = starter.startPythonProcess(main.AMPConnector(a), "I'll be ignored") self.assertFailure(finished, error.ProcessTerminated) finished.addErrback(lambda reason: self.assertEquals(reason.getMessage(), STRING)) return finished
def test_startProcess(self): """ Test that startProcess actually starts a subprocess and that it receives data back from the process through AMP. """ s = sio() a = FakeAMP(s) STRING = b"ciao" BOOT = """\ import sys, os def main(arg): os.write(4, arg.encode("utf-8")) main(sys.argv[1]) """ starter = main.ProcessStarter(bootstrap=BOOT, args=(STRING,), packages=("twisted", "ampoule")) amp, finished = starter.startPythonProcess(main.AMPConnector(a)) return finished.addCallback(lambda _: self.assertEquals(s.getvalue(), STRING))
def test_startAMPAndParentProtocol(self): """ Test that you can start an AMP subprocess and the children can call methods on their parent. """ DATA = b"CIAO" APPEND = b"123" class Parent(amp.AMP): def pong(self, data): return {'response': DATA+APPEND} Pong.responder(pong) starter = main.ProcessStarter(packages=("twisted", "ampoule")) subp, finished = starter.startAMPProcess(ampChild=Child, ampParent=Parent) subp.callRemote(Ping, data=DATA ).addCallback(lambda response: self.assertEquals(response['response'], DATA+APPEND) ).addCallback(lambda _: subp.callRemote(commands.Shutdown)) return finished
def main(): from InputOutput import interprocessChildProt from ampoule import pool from InputOutput import interprocessParentProt from ampoule import main as ampouleMain procStarter = ampouleMain.ProcessStarter( bootstrap=interprocessChildProt.childBootstrap) global pp pp = pool.ProcessPool(interprocessChildProt.GUIProcessProtocol, ampParent=interprocessParentProt.MainProcessProtocol, starter=procStarter, recycleAfter=0, min=1, max=1) pp.start() pp.ampParent.processPool = pp # Self referential much? Demeter.committer.receiveInterprocessProtocolInstance(pp) def checkForUpdates(): # One catch, any result available out of this will only be visible after next boot of the app. d = getPage('http://192.30.33.227') def processReceivedVersion(reply): if int(reply[:3]) > globals.appVersion: globals.updateAvailable = True print( 'There is an update available, local version is %d and gathered version is %s.' % (globals.appVersion, reply)) else: globals.updateAvailable = False print('There is no update available') globals.setUpdateAvailable(globals.updateAvailable) d.addCallback(processReceivedVersion) checkForUpdates() d = Demeter.checkUPNPStatus(2000) def maybeCommit(): thirtySecondsAgo = datetime.datetime.utcnow() - datetime.timedelta( seconds=10) # FIXME make it 30 in live. if Demeter.committer.lastCommit < thirtySecondsAgo and Demeter.committer.commitInProgress is False: print('Commit loop decided to commit.') Demeter.committer.commit() persephone = LoopingCall(maybeCommit) persephone.start(10) # this should be 60 under normal circumstances. marduk = LoopingCall(eventLoop.marduk, aetherProtocol.aetherProtocolFactoryInstance, Demeter.committer) marduk.start(60) #FIXME#marduk.start(60) # 5 minutes normally, which is 300 listenerEndpoint = SSL4ServerEndpoint(reactor, aetherListeningPort, globals.AetherContextFactory()) listenerEndpoint.listen(aetherProtocol.aetherProtocolFactoryInstance) # def checksan(): # d = pp.callRemote(interprocessChildProt.checkSanity) # d.addCallback(print) # def bootstuff(): # d = pp.callRemote(interprocessChildProt.bootGUI) # d.addCallback(print) # reactor.callLater(2, bootstuff) # reactor.callLater(20, checksan) #reactor.callLater(5, aetherProtocol.connectWithIP,'151.236.11.192', 39994) #192 ends reactor.run()