def __init__(self):
self.root = None
self.web = None
self.mode = MULTI_SERVER_MODE
self.node = None
self.script = "appmain.py"
self.process = ProcessMonitor()
self.service = service.MultiService()
self.start_time = 0
def setUp(self):
"""
Create an L{ProcessMonitor} wrapped around a fake reactor.
"""
self.reactor = DummyProcessReactor()
self.pm = ProcessMonitor(reactor=self.reactor)
self.pm.minRestartDelay = 2
self.pm.maxRestartDelay = 10
self.pm.threshold = 10
def run_zeo(db):
"""Spawns a zeo daemon and restart it if it crashes"""
runzeo = 'bin/runzeo'
# XXX: compat mode for buildout-less runs
if not os.path.exists('bin/runzeo'):
runzeo = 'runzeo'
pm = ProcessMonitor()
pm.addProcess('zeo', ['/bin/sh', '-c', '%s -f %s/data.fs -a %s/socket >%s/zeo.log 2>&1' % (runzeo, db, db, db)], env=os.environ)
pm.startService()
def makeService():
s = ProcessMonitor()
s.threshold = 1
s.killTime = 5
s.minRestartDelay = 1
s.maxRestartDelay = 5
return s
class DeprecationTests(unittest.SynchronousTestCase):
"""
Tests that check functionality that should be deprecated is deprecated.
"""
def setUp(self):
"""
Create reactor and process monitor.
"""
self.reactor = DummyProcessReactor()
self.pm = ProcessMonitor(reactor=self.reactor)
def test_toTuple(self):
"""
_Process.toTuple is deprecated.
When getting the deprecated processes property, the actual
data (kept in the class _Process) is converted to a tuple --
which produces a DeprecationWarning per process so converted.
"""
self.pm.addProcess("foo", ["foo"])
myprocesses = self.pm.processes
self.assertEquals(len(myprocesses), 1)
warnings = self.flushWarnings()
foundToTuple = False
for warning in warnings:
self.assertIs(warning["category"], DeprecationWarning)
if "toTuple" in warning["message"]:
foundToTuple = True
self.assertTrue(foundToTuple, f"no tuple deprecation found:{repr(warnings)}")
def test_processes(self):
"""
Accessing L{ProcessMonitor.processes} results in deprecation warning
Even when there are no processes, and thus no process is converted
to a tuple, accessing the L{ProcessMonitor.processes} property
should generate its own DeprecationWarning.
"""
myProcesses = self.pm.processes
self.assertEquals(myProcesses, {})
warnings = self.flushWarnings()
first = warnings.pop(0)
self.assertIs(first["category"], DeprecationWarning)
self.assertEquals(warnings, [])
def test_getstate(self):
"""
Pickling an L{ProcessMonitor} results in deprecation warnings
"""
pickle.dumps(self.pm)
warnings = self.flushWarnings()
for warning in warnings:
self.assertIs(warning["category"], DeprecationWarning)
def __init__(self, log=None, reactor=None):
if reactor:
BaseMonitor.__init__(self, reactor=reactor)
else:
BaseMonitor.__init__(self)
EventEmitter.__init__(self)
self.log = log or self.log
self.settings = dict()
self.states = dict()
def test_addProcessEnv(self):
"""
L{ProcessMonitor.addProcess} takes an C{env} parameter that is passed
to C{spawnProcess}.
"""
spawnedProcesses = []
def fakeSpawnProcess(*args, **kwargs):
spawnedProcesses.append((args, kwargs))
self.patch(reactor, "spawnProcess", fakeSpawnProcess)
pm = ProcessMonitor()
pm.active = True
fakeEnv = {"KEY": "value"}
pm.addProcess("foo", ["foo"], uid=1, gid=2, env=fakeEnv)
self.assertEquals(
spawnedProcesses,
[((pm.protocols["foo"], "foo", ["foo"]),
{"uid": 1, "gid": 2, "env": fakeEnv})])
def makeService(config):
s = ProcessMonitor()
s.threshold = config["threshold"]
s.killTime = config["killtime"]
s.minRestartDelay = config["minrestartdelay"]
s.maxRestartDelay = config["maxrestartdelay"]
s.addProcess(" ".join(config["args"]), config["args"])
return s
def test_addProcess(self):
"""
L{ProcessMonitor.addProcess} starts the named program and tracks the
resulting process, a protocol for collecting its stdout, and the time
it was started.
"""
spawnedProcesses = []
def fakeSpawnProcess(*args, **kwargs):
spawnedProcesses.append((args, kwargs))
self.patch(reactor, "spawnProcess", fakeSpawnProcess)
pm = ProcessMonitor()
pm.active = True
pm.addProcess("foo", ["arg1", "arg2"], uid=1, gid=2)
self.assertEquals(pm.processes, {"foo": (["arg1", "arg2"], 1, 2, {})})
self.assertEquals(pm.protocols.keys(), ["foo"])
lp = pm.protocols["foo"]
self.assertEquals(
spawnedProcesses,
[((lp, "arg1", ["arg1", "arg2"]),
{"uid": 1, "gid": 2, "env": {}})])
def test_addProcessEnv(self):
"""
L{ProcessMonitor.addProcess} takes an C{env} parameter that is passed
to C{spawnProcess}.
"""
spawnedProcesses = []
def fakeSpawnProcess(*args, **kwargs):
spawnedProcesses.append((args, kwargs))
self.patch(reactor, "spawnProcess", fakeSpawnProcess)
pm = ProcessMonitor()
pm.active = True
fakeEnv = {"KEY": "value"}
pm.addProcess("foo", ["foo"], uid=1, gid=2, env=fakeEnv)
self.assertEquals(spawnedProcesses,
[((pm.protocols["foo"], "foo", ["foo"]), {
"uid": 1,
"gid": 2,
"env": fakeEnv
})])
def makeService(config):
s = ProcessMonitor()
s.threshold = config["threshold"]
s.killTime = config["killtime"]
s.minRestartDelay = config["minrestartdelay"]
s.maxRestartDelay = config["maxrestartdelay"]
s.addProcess(" ".join(config["args"]), config["args"])
return s
def test_addProcess(self):
"""
L{ProcessMonitor.addProcess} starts the named program and tracks the
resulting process, a protocol for collecting its stdout, and the time
it was started.
"""
spawnedProcesses = []
def fakeSpawnProcess(*args, **kwargs):
spawnedProcesses.append((args, kwargs))
self.patch(reactor, "spawnProcess", fakeSpawnProcess)
pm = ProcessMonitor()
pm.active = True
pm.addProcess("foo", ["arg1", "arg2"], uid=1, gid=2)
self.assertEquals(pm.processes, {"foo": (["arg1", "arg2"], 1, 2, {})})
self.assertEquals(pm.protocols.keys(), ["foo"])
lp = pm.protocols["foo"]
self.assertEquals(spawnedProcesses, [((lp, "arg1", ["arg1", "arg2"]), {
"uid": 1,
"gid": 2,
"env": {}
})])
def run_zeo(db):
"""Spawns a zeo daemon and restart it if it crashes"""
runzeo = 'bin/runzeo'
# XXX: compat mode for buildout-less runs
if not os.path.exists('bin/runzeo'):
runzeo = 'runzeo'
pm = ProcessMonitor()
pm.addProcess('zeo', [
'/bin/sh', '-c',
'%s -f %s/data.fs -a %s/socket >%s/zeo.log 2>&1' % (runzeo, db, db, db)
],
env=os.environ)
pm.startService()
from twisted.runner.procmon import ProcessMonitor
import time
def makeService():
s = ProcessMonitor()
s.threshold = 1
s.killTime = 5
s.minRestartDelay = 1
s.maxRestartDelay = 5
return s
ss = ProcessMonitor()
#ss.startService()
ss.startService()
ss.addProcess("calc.exe *32", ['c:\windows\system32\calc.exe', ])
while True:
#ss.startProcess('calc.exe')
time.sleep(5)
break
#break
ss.stopService()
ss.startService()
while True:
class ProcmonTests(unittest.TestCase):
"""
Tests for L{ProcessMonitor}.
"""
def setUp(self):
"""
Create an L{ProcessMonitor} wrapped around a fake reactor.
"""
self.reactor = DummyProcessReactor()
self.pm = ProcessMonitor(reactor=self.reactor)
self.pm.minRestartDelay = 2
self.pm.maxRestartDelay = 10
self.pm.threshold = 10
def test_getStateIncludesProcesses(self):
"""
The list of monitored processes must be included in the pickle state.
"""
self.pm.addProcess("foo", ["arg1", "arg2"],
uid=1, gid=2, env={})
self.assertEqual(self.pm.__getstate__()['processes'],
{'foo': (['arg1', 'arg2'], 1, 2, {})})
def test_getStateExcludesReactor(self):
"""
The private L{ProcessMonitor._reactor} instance variable should not be
included in the pickle state.
"""
self.assertNotIn('_reactor', self.pm.__getstate__())
def test_addProcess(self):
"""
L{ProcessMonitor.addProcess} only starts the named program if
L{ProcessMonitor.startService} has been called.
"""
self.pm.addProcess("foo", ["arg1", "arg2"],
uid=1, gid=2, env={})
self.assertEqual(self.pm.protocols, {})
self.assertEqual(self.pm.processes,
{"foo": (["arg1", "arg2"], 1, 2, {})})
self.pm.startService()
self.reactor.advance(0)
self.assertEqual(self.pm.protocols.keys(), ["foo"])
def test_addProcessDuplicateKeyError(self):
"""
L{ProcessMonitor.addProcess} raises a C{KeyError} if a process with the
given name already exists.
"""
self.pm.addProcess("foo", ["arg1", "arg2"],
uid=1, gid=2, env={})
self.assertRaises(KeyError, self.pm.addProcess,
"foo", ["arg1", "arg2"], uid=1, gid=2, env={})
def test_addProcessEnv(self):
"""
L{ProcessMonitor.addProcess} takes an C{env} parameter that is passed to
L{IReactorProcess.spawnProcess}.
"""
fakeEnv = {"KEY": "value"}
self.pm.startService()
self.pm.addProcess("foo", ["foo"], uid=1, gid=2, env=fakeEnv)
self.reactor.advance(0)
self.assertEqual(
self.reactor.spawnedProcesses[0]._environment, fakeEnv)
def test_removeProcess(self):
"""
L{ProcessMonitor.removeProcess} removes the process from the public
processes list.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertEqual(len(self.pm.processes), 1)
self.pm.removeProcess("foo")
self.assertEqual(len(self.pm.processes), 0)
def test_removeProcessUnknownKeyError(self):
"""
L{ProcessMonitor.removeProcess} raises a C{KeyError} if the given
process name isn't recognised.
"""
self.pm.startService()
self.assertRaises(KeyError, self.pm.removeProcess, "foo")
def test_startProcess(self):
"""
When a process has been started, an instance of L{LoggingProtocol} will
be added to the L{ProcessMonitor.protocols} dict and the start time of
the process will be recorded in the L{ProcessMonitor.timeStarted}
dictionary.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.startProcess("foo")
self.assertIsInstance(self.pm.protocols["foo"], LoggingProtocol)
self.assertIn("foo", self.pm.timeStarted.keys())
def test_startProcessAlreadyStarted(self):
"""
L{ProcessMonitor.startProcess} silently returns if the named process is
already started.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.startProcess("foo")
self.assertIsNone(self.pm.startProcess("foo"))
def test_startProcessUnknownKeyError(self):
"""
L{ProcessMonitor.startProcess} raises a C{KeyError} if the given
process name isn't recognised.
"""
self.assertRaises(KeyError, self.pm.startProcess, "foo")
def test_stopProcessNaturalTermination(self):
"""
L{ProcessMonitor.stopProcess} immediately sends a TERM signal to the
named process.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertIn("foo", self.pm.protocols)
# Configure fake process to die 1 second after receiving term signal
timeToDie = self.pm.protocols["foo"].transport._terminationDelay = 1
# Advance the reactor to just before the short lived process threshold
# and leave enough time for the process to die
self.reactor.advance(self.pm.threshold)
# Then signal the process to stop
self.pm.stopProcess("foo")
# Advance the reactor just enough to give the process time to die and
# verify that the process restarts
self.reactor.advance(timeToDie)
# We expect it to be restarted immediately
self.assertEqual(self.reactor.seconds(),
self.pm.timeStarted["foo"])
def test_stopProcessForcedKill(self):
"""
L{ProcessMonitor.stopProcess} kills a process which fails to terminate
naturally within L{ProcessMonitor.killTime} seconds.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertIn("foo", self.pm.protocols)
self.reactor.advance(self.pm.threshold)
proc = self.pm.protocols["foo"].transport
# Arrange for the fake process to live longer than the killTime
proc._terminationDelay = self.pm.killTime + 1
self.pm.stopProcess("foo")
# If process doesn't die before the killTime, procmon should
# terminate it
self.reactor.advance(self.pm.killTime - 1)
self.assertEqual(0.0, self.pm.timeStarted["foo"])
self.reactor.advance(1)
# We expect it to be immediately restarted
self.assertEqual(self.reactor.seconds(), self.pm.timeStarted["foo"])
def test_stopProcessUnknownKeyError(self):
"""
L{ProcessMonitor.stopProcess} raises a C{KeyError} if the given process
name isn't recognised.
"""
self.assertRaises(KeyError, self.pm.stopProcess, "foo")
def test_stopProcessAlreadyStopped(self):
"""
L{ProcessMonitor.stopProcess} silently returns if the named process
is already stopped. eg Process has crashed and a restart has been
rescheduled, but in the meantime, the service is stopped.
"""
self.pm.addProcess("foo", ["foo"])
self.assertIsNone(self.pm.stopProcess("foo"))
def test_connectionLostLongLivedProcess(self):
"""
L{ProcessMonitor.connectionLost} should immediately restart a process
if it has been running longer than L{ProcessMonitor.threshold} seconds.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
# Long time passes
self.reactor.advance(self.pm.threshold)
# Process dies after threshold
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertNotIn("foo", self.pm.protocols)
# Process should be restarted immediately
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
def test_connectionLostMurderCancel(self):
"""
L{ProcessMonitor.connectionLost} cancels a scheduled process killer and
deletes the DelayedCall from the L{ProcessMonitor.murder} list.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# Advance 1s to start the process then ask ProcMon to stop it
self.reactor.advance(1)
self.pm.stopProcess("foo")
# A process killer has been scheduled, delayedCall is active
self.assertIn("foo", self.pm.murder)
delayedCall = self.pm.murder["foo"]
self.assertTrue(delayedCall.active())
# Advance to the point at which the dummy process exits
self.reactor.advance(
self.pm.protocols["foo"].transport._terminationDelay)
# Now the delayedCall has been cancelled and deleted
self.assertFalse(delayedCall.active())
self.assertNotIn("foo", self.pm.murder)
def test_connectionLostProtocolDeletion(self):
"""
L{ProcessMonitor.connectionLost} removes the corresponding
ProcessProtocol instance from the L{ProcessMonitor.protocols} list.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertIn("foo", self.pm.protocols)
self.pm.protocols["foo"].transport.signalProcess("KILL")
self.reactor.advance(
self.pm.protocols["foo"].transport._terminationDelay)
self.assertNotIn("foo", self.pm.protocols)
def test_connectionLostMinMaxRestartDelay(self):
"""
L{ProcessMonitor.connectionLost} will wait at least minRestartDelay s
and at most maxRestartDelay s
"""
self.pm.minRestartDelay = 2
self.pm.maxRestartDelay = 3
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertEqual(self.pm.delay["foo"], self.pm.minRestartDelay)
self.reactor.advance(self.pm.threshold - 1)
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertEqual(self.pm.delay["foo"], self.pm.maxRestartDelay)
def test_connectionLostBackoffDelayDoubles(self):
"""
L{ProcessMonitor.connectionLost} doubles the restart delay each time
the process dies too quickly.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.reactor.advance(self.pm.threshold - 1) #9s
self.assertIn("foo", self.pm.protocols)
self.assertEqual(self.pm.delay["foo"], self.pm.minRestartDelay)
# process dies within the threshold and should not restart immediately
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertEqual(self.pm.delay["foo"], self.pm.minRestartDelay * 2)
def test_startService(self):
"""
L{ProcessMonitor.startService} starts all monitored processes.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
def test_stopService(self):
"""
L{ProcessMonitor.stopService} should stop all monitored processes.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.addProcess("bar", ["bar"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the processes
self.reactor.advance(self.pm.threshold)
self.assertIn("foo", self.pm.protocols)
self.assertIn("bar", self.pm.protocols)
self.reactor.advance(1)
self.pm.stopService()
# Advance to beyond the killTime - all monitored processes
# should have exited
self.reactor.advance(self.pm.killTime + 1)
# The processes shouldn't be restarted
self.assertEqual({}, self.pm.protocols)
def test_stopServiceCancelRestarts(self):
"""
L{ProcessMonitor.stopService} should cancel any scheduled process
restarts.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the processes
self.reactor.advance(self.pm.threshold)
self.assertIn("foo", self.pm.protocols)
self.reactor.advance(1)
# Kill the process early
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertTrue(self.pm.restart['foo'].active())
self.pm.stopService()
# Scheduled restart should have been cancelled
self.assertFalse(self.pm.restart['foo'].active())
def test_stopServiceCleanupScheduledRestarts(self):
"""
L{ProcessMonitor.stopService} should cancel all scheduled process
restarts.
"""
self.pm.threshold = 5
self.pm.minRestartDelay = 5
# Start service and add a process (started immediately)
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
# Stop the process after 1s
self.reactor.advance(1)
self.pm.stopProcess("foo")
# Wait 1s for it to exit it will be scheduled to restart 5s later
self.reactor.advance(1)
# Meanwhile stop the service
self.pm.stopService()
# Advance to beyond the process restart time
self.reactor.advance(6)
# The process shouldn't have restarted because stopService has cancelled
# all pending process restarts.
self.assertEqual(self.pm.protocols, {})
class Master:
def __init__(self):
self.root = None
self.web = None
self.mode = MULTI_SERVER_MODE
self.node = None
self.script = "appmain.py"
self.process = ProcessMonitor()
self.service = service.MultiService()
self.start_time = 0
def set_mode(self, mode):
self.mode = mode
def set_node(self, node):
self.node = node
def set_script(self, script):
self.script = script
def create_master(self):
"""
创建Master服务
:return:
"""
config = Config().config
GlobalObject().json_config = config
mastercnf = config.get('master')
rootport = mastercnf.get('rootport')
webport = mastercnf.get('webport')
masterlog = mastercnf.get('log')
self.root = PBRoot()
rootservice = services.Service("rootservice")
self.root.addServiceChannel(rootservice)
self.web = vhost.NameVirtualHost()
self.web.addHost('0.0.0.0', './')
GlobalObject().root = self.root
GlobalObject().webroot = self.web
import webapp
import rootapp
internet.TCPServer(webport,
DelaySite(self.web)).setServiceParent(self.service)
internet.TCPServer(rootport, BilateralFactory(
self.root)).setServiceParent(self.service)
self.process.setServiceParent(self.service)
def create_node(self, name):
"""
创建节点服务
:param name:
:return:
"""
args = ["python", self.script, name]
self.process.addProcess(name, args, env=os.environ)
def start(self, app):
"""
启动APP
:param app:
:return:
"""
self.start_time = reactor.seconds()
if self.mode == MULTI_SERVER_MODE:
self.create_master()
servers = Config().servers
for name in servers.keys():
self.create_node(name)
elif self.mode == SINGLE_SERVER_MODE:
self.create_node(self.node)
else:
self.create_master()
reactor.addSystemEventTrigger('after', 'startup', self.startAfter)
reactor.addSystemEventTrigger('before', 'shutdown', self.stopBefore)
if "-y" in sys.argv and "-n" not in sys.argv:
app.setComponent(
log.ILogObserver,
log.FileLogObserver(DailyLogFile("logs/master.log", "")).emit)
self.service.setServiceParent(app)
GlobalObject().server = self
def startAfter(self):
"""
启动之后
:return:
"""
log.msg("*** The master in the %s launch ***" % time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(self.start_time)))
# 在根目录写入启动进程信息
process = [(os.getpid(), 'master')]
for name, proc in self.process.protocols.items():
process.append((proc.transport.pid, name))
with open("status.json", "w+") as f:
f.write(Jsonify(process))
f.close()
def stopBefore(self):
"""
关闭之前
:return:
"""
log.msg("*** Wait for the child process to exit ***")
wait = dict([(proc.transport.pid, name)
for name, proc in self.process.protocols.items()])
while True:
try:
pid = os.wait()[0]
log.msg("[%s] child node has quit, pid: %s" %
(wait.get(pid), pid))
except:
break
signal.alarm(1)
def stopService(self):
self.connectionLost = self.stoppedConnectionLost
# Can't use super here as ProcessMonitor is an old-style object.
return ProcessMonitor.stopService(self)
def makeService(self, options):
if not os.path.isfile(options['config']):
raise ConfigNotFoundException()
config = yaml.load(open(options['config']))
use_ssl = config['usessl']
from apiserver import settings
settings.BASE_URL = '%s://%s:%s' % (use_ssl and 'https' or 'http', config['external_ip'], config['port'])
settings.METADATA_FILE = config['metadata_database']
settings.DB_FILE = config['database']
if 'datapath' in config:
if os.path.isdir(config['datapath']):
settings.DATA_PATH = config['datapath']
else:
log.err("Datapath '%s' does not exist" % config['datapath'])
from apiserver.interfaces import IBackend, ISection, IService
import apiserver.backends
backend_types = dict((p.name, p) for p in getPlugins(IBackend, apiserver.backends))
import apiserver.sections
section_types = dict((p.name, p) for p in getPlugins(ISection, apiserver.sections))
import apiserver.services
settings.SERVICES = list(getPlugins(IService, apiserver.services))
log.err('Found services: %s' % ', '.join(s.name for s in settings.SERVICES))
known_nodes = {}
for name, node in config['nodes'].iteritems():
if node['backend'] not in backend_types:
log.err("Unknown backend %s for node %s" % (node['backend'], name))
continue
node['type'] = backend_types[node['backend']]
node['id'] = name
known_nodes[name] = node
for name, cfg in config['sections'].items():
if 'nodes' not in cfg:
log.err("No nodes found for section %s" % name)
continue
if 'type' not in cfg:
log.err("Type missing for section %s" % name)
continue
if cfg['type'] not in section_types:
log.err("Unknown section type %s for node %s" % (cfg['type'], name))
continue
section_type = section_types[cfg['type']]
section = settings.SECTIONS[name] = section_type(name, cfg)
for node_name, section_cfg in cfg['nodes'].items():
if node_name not in known_nodes:
log.err("Unknown node %s for setion %s" % (node_name, name))
continue
node_cfg = known_nodes[node_name]
node_type = node_cfg['type']
node = node_type(section, node_cfg, section_cfg)
section.nodes.append(node)
if 'plugins' in config:
settings.PLUGINS.update(config['plugins'])
from apiserver.resource import TidalStreamHTTPAuthSessionWrapper
from apiserver.server import RootResource
from apiserver.tokenauth import InternalTokenAuthFactory, TokenPasswordChecker
settings.PASSWORD_CHECKER = password_checker = FilePasswordDB(config['userfile'])
settings.TOKEN_PASSWORD_CHECKER = token_password_checker = TokenPasswordChecker()
checkers = [password_checker, token_password_checker]
wrapper = TidalStreamHTTPAuthSessionWrapper(
Portal(TidalStreamRealm(RootResource()),
checkers), [guard.BasicCredentialFactory('TidalStream'), InternalTokenAuthFactory()])
site = server.Site(wrapper)
ms = MultiService()
if config.get('phantomjs', None):
phantom_exe = config['phantomjs']
if os.path.isfile(phantom_exe):
pm = ProcessMonitor()
pm.addProcess('phantomjs', [phantom_exe, '--webdriver=127.0.0.1:%s' % settings.PHANTOMJS_PORT])
ms.addService(pm)
if use_ssl:
ms.addService(internet.SSLServer(config['port'], site,
ServerContextFactory(config['certfile'])))
else:
ms.addService(internet.TCPServer(config['port'], site))
return ms
def setUp(self):
"""
Create reactor and process monitor.
"""
self.reactor = DummyProcessReactor()
self.pm = ProcessMonitor(reactor=self.reactor)
class ProcmonTests(unittest.TestCase):
"""
Tests for L{ProcessMonitor}.
"""
def setUp(self):
"""
Create an L{ProcessMonitor} wrapped around a fake reactor.
"""
self.reactor = DummyProcessReactor()
self.pm = ProcessMonitor(reactor=self.reactor)
self.pm.minRestartDelay = 2
self.pm.maxRestartDelay = 10
self.pm.threshold = 10
def test_reprLooksGood(self):
"""
Repr includes all details
"""
self.pm.addProcess("foo", ["arg1", "arg2"], uid=1, gid=2, env={})
representation = repr(self.pm)
self.assertIn("foo", representation)
self.assertIn("1", representation)
self.assertIn("2", representation)
def test_simpleReprLooksGood(self):
"""
Repr does not include unneeded details.
Values of attributes that just mean "inherit from launching
process" do not appear in the repr of a process.
"""
self.pm.addProcess("foo", ["arg1", "arg2"], env={})
representation = repr(self.pm)
self.assertNotIn("(", representation)
self.assertNotIn(")", representation)
def test_getStateIncludesProcesses(self):
"""
The list of monitored processes must be included in the pickle state.
"""
self.pm.addProcess("foo", ["arg1", "arg2"], uid=1, gid=2, env={})
self.assertEqual(self.pm.__getstate__()["processes"],
{"foo": (["arg1", "arg2"], 1, 2, {})})
def test_getStateExcludesReactor(self):
"""
The private L{ProcessMonitor._reactor} instance variable should not be
included in the pickle state.
"""
self.assertNotIn("_reactor", self.pm.__getstate__())
def test_addProcess(self):
"""
L{ProcessMonitor.addProcess} only starts the named program if
L{ProcessMonitor.startService} has been called.
"""
self.pm.addProcess("foo", ["arg1", "arg2"], uid=1, gid=2, env={})
self.assertEqual(self.pm.protocols, {})
self.assertEqual(self.pm.processes,
{"foo": (["arg1", "arg2"], 1, 2, {})})
self.pm.startService()
self.reactor.advance(0)
self.assertEqual(list(self.pm.protocols.keys()), ["foo"])
def test_addProcessDuplicateKeyError(self):
"""
L{ProcessMonitor.addProcess} raises a C{KeyError} if a process with the
given name already exists.
"""
self.pm.addProcess("foo", ["arg1", "arg2"], uid=1, gid=2, env={})
self.assertRaises(KeyError,
self.pm.addProcess,
"foo", ["arg1", "arg2"],
uid=1,
gid=2,
env={})
def test_addProcessEnv(self):
"""
L{ProcessMonitor.addProcess} takes an C{env} parameter that is passed to
L{IReactorProcess.spawnProcess}.
"""
fakeEnv = {"KEY": "value"}
self.pm.startService()
self.pm.addProcess("foo", ["foo"], uid=1, gid=2, env=fakeEnv)
self.reactor.advance(0)
self.assertEqual(self.reactor.spawnedProcesses[0]._environment,
fakeEnv)
def test_addProcessCwd(self):
"""
L{ProcessMonitor.addProcess} takes an C{cwd} parameter that is passed
to L{IReactorProcess.spawnProcess}.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"], cwd="/mnt/lala")
self.reactor.advance(0)
self.assertEqual(self.reactor.spawnedProcesses[0]._path, "/mnt/lala")
def test_removeProcess(self):
"""
L{ProcessMonitor.removeProcess} removes the process from the public
processes list.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertEqual(len(self.pm.processes), 1)
self.pm.removeProcess("foo")
self.assertEqual(len(self.pm.processes), 0)
def test_removeProcessUnknownKeyError(self):
"""
L{ProcessMonitor.removeProcess} raises a C{KeyError} if the given
process name isn't recognised.
"""
self.pm.startService()
self.assertRaises(KeyError, self.pm.removeProcess, "foo")
def test_startProcess(self):
"""
When a process has been started, an instance of L{LoggingProtocol} will
be added to the L{ProcessMonitor.protocols} dict and the start time of
the process will be recorded in the L{ProcessMonitor.timeStarted}
dictionary.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.startProcess("foo")
self.assertIsInstance(self.pm.protocols["foo"], LoggingProtocol)
self.assertIn("foo", self.pm.timeStarted.keys())
def test_startProcessAlreadyStarted(self):
"""
L{ProcessMonitor.startProcess} silently returns if the named process is
already started.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.startProcess("foo")
self.assertIsNone(self.pm.startProcess("foo"))
def test_startProcessUnknownKeyError(self):
"""
L{ProcessMonitor.startProcess} raises a C{KeyError} if the given
process name isn't recognised.
"""
self.assertRaises(KeyError, self.pm.startProcess, "foo")
def test_stopProcessNaturalTermination(self):
"""
L{ProcessMonitor.stopProcess} immediately sends a TERM signal to the
named process.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertIn("foo", self.pm.protocols)
# Configure fake process to die 1 second after receiving term signal
timeToDie = self.pm.protocols["foo"].transport._terminationDelay = 1
# Advance the reactor to just before the short lived process threshold
# and leave enough time for the process to die
self.reactor.advance(self.pm.threshold)
# Then signal the process to stop
self.pm.stopProcess("foo")
# Advance the reactor just enough to give the process time to die and
# verify that the process restarts
self.reactor.advance(timeToDie)
# We expect it to be restarted immediately
self.assertEqual(self.reactor.seconds(), self.pm.timeStarted["foo"])
def test_stopProcessForcedKill(self):
"""
L{ProcessMonitor.stopProcess} kills a process which fails to terminate
naturally within L{ProcessMonitor.killTime} seconds.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertIn("foo", self.pm.protocols)
self.reactor.advance(self.pm.threshold)
proc = self.pm.protocols["foo"].transport
# Arrange for the fake process to live longer than the killTime
proc._terminationDelay = self.pm.killTime + 1
self.pm.stopProcess("foo")
# If process doesn't die before the killTime, procmon should
# terminate it
self.reactor.advance(self.pm.killTime - 1)
self.assertEqual(0.0, self.pm.timeStarted["foo"])
self.reactor.advance(1)
# We expect it to be immediately restarted
self.assertEqual(self.reactor.seconds(), self.pm.timeStarted["foo"])
def test_stopProcessUnknownKeyError(self):
"""
L{ProcessMonitor.stopProcess} raises a C{KeyError} if the given process
name isn't recognised.
"""
self.assertRaises(KeyError, self.pm.stopProcess, "foo")
def test_stopProcessAlreadyStopped(self):
"""
L{ProcessMonitor.stopProcess} silently returns if the named process
is already stopped. eg Process has crashed and a restart has been
rescheduled, but in the meantime, the service is stopped.
"""
self.pm.addProcess("foo", ["foo"])
self.assertIsNone(self.pm.stopProcess("foo"))
def test_outputReceivedCompleteLine(self):
"""
Getting a complete output line on stdout generates a log message.
"""
events = []
self.addCleanup(globalLogPublisher.removeObserver, events.append)
globalLogPublisher.addObserver(events.append)
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# Advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
# Long time passes
self.reactor.advance(self.pm.threshold)
# Process greets
self.pm.protocols["foo"].outReceived(b"hello world!\n")
self.assertEquals(len(events), 1)
namespace = events[0]["log_namespace"]
stream = events[0]["stream"]
tag = events[0]["tag"]
line = events[0]["line"]
self.assertEquals(namespace, "twisted.runner.procmon.ProcessMonitor")
self.assertEquals(stream, "stdout")
self.assertEquals(tag, "foo")
self.assertEquals(line, "hello world!")
def test_ouputReceivedCompleteErrLine(self):
"""
Getting a complete output line on stderr generates a log message.
"""
events = []
self.addCleanup(globalLogPublisher.removeObserver, events.append)
globalLogPublisher.addObserver(events.append)
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# Advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
# Long time passes
self.reactor.advance(self.pm.threshold)
# Process greets
self.pm.protocols["foo"].errReceived(b"hello world!\n")
self.assertEquals(len(events), 1)
namespace = events[0]["log_namespace"]
stream = events[0]["stream"]
tag = events[0]["tag"]
line = events[0]["line"]
self.assertEquals(namespace, "twisted.runner.procmon.ProcessMonitor")
self.assertEquals(stream, "stderr")
self.assertEquals(tag, "foo")
self.assertEquals(line, "hello world!")
def test_outputReceivedCompleteLineInvalidUTF8(self):
"""
Getting invalid UTF-8 results in the repr of the raw message
"""
events = []
self.addCleanup(globalLogPublisher.removeObserver, events.append)
globalLogPublisher.addObserver(events.append)
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# Advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
# Long time passes
self.reactor.advance(self.pm.threshold)
# Process greets
self.pm.protocols["foo"].outReceived(b"\xffhello world!\n")
self.assertEquals(len(events), 1)
message = events[0]
namespace = message["log_namespace"]
stream = message["stream"]
tag = message["tag"]
output = message["line"]
self.assertEquals(namespace, "twisted.runner.procmon.ProcessMonitor")
self.assertEquals(stream, "stdout")
self.assertEquals(tag, "foo")
self.assertEquals(output, repr(b"\xffhello world!"))
def test_outputReceivedPartialLine(self):
"""
Getting partial line results in no events until process end
"""
events = []
self.addCleanup(globalLogPublisher.removeObserver, events.append)
globalLogPublisher.addObserver(events.append)
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# Advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
# Long time passes
self.reactor.advance(self.pm.threshold)
# Process greets
self.pm.protocols["foo"].outReceived(b"hello world!")
self.assertEquals(len(events), 0)
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertEquals(len(events), 1)
namespace = events[0]["log_namespace"]
stream = events[0]["stream"]
tag = events[0]["tag"]
line = events[0]["line"]
self.assertEquals(namespace, "twisted.runner.procmon.ProcessMonitor")
self.assertEquals(stream, "stdout")
self.assertEquals(tag, "foo")
self.assertEquals(line, "hello world!")
def test_connectionLostLongLivedProcess(self):
"""
L{ProcessMonitor.connectionLost} should immediately restart a process
if it has been running longer than L{ProcessMonitor.threshold} seconds.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
# Long time passes
self.reactor.advance(self.pm.threshold)
# Process dies after threshold
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertNotIn("foo", self.pm.protocols)
# Process should be restarted immediately
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
def test_connectionLostMurderCancel(self):
"""
L{ProcessMonitor.connectionLost} cancels a scheduled process killer and
deletes the DelayedCall from the L{ProcessMonitor.murder} list.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# Advance 1s to start the process then ask ProcMon to stop it
self.reactor.advance(1)
self.pm.stopProcess("foo")
# A process killer has been scheduled, delayedCall is active
self.assertIn("foo", self.pm.murder)
delayedCall = self.pm.murder["foo"]
self.assertTrue(delayedCall.active())
# Advance to the point at which the dummy process exits
self.reactor.advance(
self.pm.protocols["foo"].transport._terminationDelay)
# Now the delayedCall has been cancelled and deleted
self.assertFalse(delayedCall.active())
self.assertNotIn("foo", self.pm.murder)
def test_connectionLostProtocolDeletion(self):
"""
L{ProcessMonitor.connectionLost} removes the corresponding
ProcessProtocol instance from the L{ProcessMonitor.protocols} list.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertIn("foo", self.pm.protocols)
self.pm.protocols["foo"].transport.signalProcess("KILL")
self.reactor.advance(
self.pm.protocols["foo"].transport._terminationDelay)
self.assertNotIn("foo", self.pm.protocols)
def test_connectionLostMinMaxRestartDelay(self):
"""
L{ProcessMonitor.connectionLost} will wait at least minRestartDelay s
and at most maxRestartDelay s
"""
self.pm.minRestartDelay = 2
self.pm.maxRestartDelay = 3
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.assertEqual(self.pm.delay["foo"], self.pm.minRestartDelay)
self.reactor.advance(self.pm.threshold - 1)
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertEqual(self.pm.delay["foo"], self.pm.maxRestartDelay)
def test_connectionLostBackoffDelayDoubles(self):
"""
L{ProcessMonitor.connectionLost} doubles the restart delay each time
the process dies too quickly.
"""
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
self.reactor.advance(self.pm.threshold - 1) # 9s
self.assertIn("foo", self.pm.protocols)
self.assertEqual(self.pm.delay["foo"], self.pm.minRestartDelay)
# process dies within the threshold and should not restart immediately
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertEqual(self.pm.delay["foo"], self.pm.minRestartDelay * 2)
def test_startService(self):
"""
L{ProcessMonitor.startService} starts all monitored processes.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the process
self.reactor.advance(0)
self.assertIn("foo", self.pm.protocols)
def test_stopService(self):
"""
L{ProcessMonitor.stopService} should stop all monitored processes.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.addProcess("bar", ["bar"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the processes
self.reactor.advance(self.pm.threshold)
self.assertIn("foo", self.pm.protocols)
self.assertIn("bar", self.pm.protocols)
self.reactor.advance(1)
self.pm.stopService()
# Advance to beyond the killTime - all monitored processes
# should have exited
self.reactor.advance(self.pm.killTime + 1)
# The processes shouldn't be restarted
self.assertEqual({}, self.pm.protocols)
def test_restartAllRestartsOneProcess(self):
"""
L{ProcessMonitor.restartAll} succeeds when there is one process.
"""
self.pm.addProcess("foo", ["foo"])
self.pm.startService()
self.reactor.advance(1)
self.pm.restartAll()
# Just enough time for the process to die,
# not enough time to start a new one.
self.reactor.advance(1)
processes = list(self.reactor.spawnedProcesses)
myProcess = processes.pop()
self.assertEquals(processes, [])
self.assertIsNone(myProcess.pid)
def test_stopServiceCancelRestarts(self):
"""
L{ProcessMonitor.stopService} should cancel any scheduled process
restarts.
"""
self.pm.addProcess("foo", ["foo"])
# Schedule the process to start
self.pm.startService()
# advance the reactor to start the processes
self.reactor.advance(self.pm.threshold)
self.assertIn("foo", self.pm.protocols)
self.reactor.advance(1)
# Kill the process early
self.pm.protocols["foo"].processEnded(Failure(ProcessDone(0)))
self.assertTrue(self.pm.restart["foo"].active())
self.pm.stopService()
# Scheduled restart should have been cancelled
self.assertFalse(self.pm.restart["foo"].active())
def test_stopServiceCleanupScheduledRestarts(self):
"""
L{ProcessMonitor.stopService} should cancel all scheduled process
restarts.
"""
self.pm.threshold = 5
self.pm.minRestartDelay = 5
# Start service and add a process (started immediately)
self.pm.startService()
self.pm.addProcess("foo", ["foo"])
# Stop the process after 1s
self.reactor.advance(1)
self.pm.stopProcess("foo")
# Wait 1s for it to exit it will be scheduled to restart 5s later
self.reactor.advance(1)
# Meanwhile stop the service
self.pm.stopService()
# Advance to beyond the process restart time
self.reactor.advance(6)
# The process shouldn't have restarted because stopService has cancelled
# all pending process restarts.
self.assertEqual(self.pm.protocols, {})