def testInvalidConfigValue(self):
"""throw when we get a bad config value"""
try:
config.get_value("testing")
self.fail("Not throwing expection on missing configuration option!")
except KeyError:
pass
def _heartbeat(identity, g):
"""Given an identity and its corresponding g, start a heartbeat loop.
Maintain loop until either g dies or connection does not return a
BPPing message in a timely manner. If message is not returned, kill
corresponding g.
"""
while not g.ready():
e = event.add(identity, "BPPing")
queue.add(identity, ["s", "BPPing"])
try:
e.get(block=True, timeout=config.get_value("ping_max"))
except gevent.Timeout:
logging.info("Identity %s died via heartbeat", identity)
g.kill()
return
except greenlet.BPGreenletExit:
logging.debug("Heartbeat for %s exiting...", identity)
return
if g.ready():
logging.debug("Heartbeat for %s exiting...", identity)
return
try:
gevent.sleep(config.get_value("ping_rate"))
except greenlet.BPGreenletExit:
logging.debug("Heartbeat for %s exiting...", identity)
return
def setUp(self):
reload(config)
reload(plugin)
reload(server)
reload(utils)
self.tmpdir = tempfile.mkdtemp(prefix="fetest-")
with mock.patch('sys.argv', ['b******g', '--config_dir', self.tmpdir]):
config.init()
self.plugin_dest = os.path.join(config.get_dir("plugin"), "test-plugin")
self.server_loop_trigger = gevent.event.Event()
self.copyPlugin()
plugin.scan_for_plugins()
def server_loop():
server.init()
utils.spawn_gevent_func("main socket loop", "main", server.msg_loop)
gevent.sleep()
self.server_loop_trigger.wait()
server.shutdown()
self.server_greenlet = gevent.spawn(server_loop)
# Start a new socket
self.test_socket = ClaimFlowTests.ClaimFlowTestSocket(config.get_value("server_address"))
# Attach to router
self.test_socket_greenlet = gevent.spawn(self.test_socket.run)
def setUp(self):
reload(config)
reload(server)
reload(utils)
self.tmpdir = tempfile.mkdtemp(prefix="fetest-")
with mock.patch('sys.argv', ['b******g', '--config_dir', self.tmpdir]):
config.init()
# Set ping rates REALLY low so these will move quickly
config.set_value("ping_rate", .01)
config.set_value("ping_max", .05)
self.trigger = gevent.event.Event()
self.server_loop_trigger = gevent.event.Event()
def server_loop():
server.init()
utils.spawn_gevent_func("main socket loop", "main", server.msg_loop)
gevent.sleep()
self.server_loop_trigger.wait()
server.shutdown()
self.server_greenlet = gevent.spawn(server_loop)
# Start a new socket
self.test_socket = HeartbeatTests.HeartbeatTestSocket(config.get_value("server_address"), self.trigger)
# Attach to router
self.test_socket_greenlet = gevent.spawn(self.test_socket.run)
def testConfigFileLoad(self):
"""load config files correctly"""
config.set_value("server_address", "ipc://wat")
reload(config)
with mock.patch('sys.argv', ['b******g', '--config_dir', self.tmpdir]):
config.init()
self.failIf(config.get_value("server_address") != "ipc://wat")
def _run_count_plugin(plugin):
"""Runs the count process for a plugin. Constantly polls for list of devices,
keeping an internal reference of the devices available from the plugin.
"""
count_identity = greenlet.random_ident()
e = event.add(count_identity, "BPPluginRegisterCount")
count_process_cmd = [plugin.executable_path, "--server_port=%s" %
config.get_value("server_address"), "--count"]
count_process = _start_process(count_process_cmd, count_identity)
if not count_process:
logging.warning("%s count process unable to start. removing.",
plugin.name)
return
try:
e.get(block=True, timeout=1)
except gevent.Timeout:
logging.info("%s count process never registered, removing.",
plugin.name)
return
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
return
logging.info("Count process for %s up on identity %s", plugin.name,
count_identity)
greenlet.add_identity_greenlet(count_identity, gevent.getcurrent())
hb = heartbeat.spawn_heartbeat(count_identity, gevent.getcurrent())
_plugins[plugin.name] = plugin
while True:
queue.add(count_identity, ["s", "BPPluginDeviceList"])
e = event.add(count_identity, "BPPluginDeviceList")
try:
(i, msg) = e.get(block=True, timeout=1)
except gevent.Timeout:
logging.info("%s count process timed out, removing.", plugin.name)
break
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
break
_devices[plugin.name] = msg[2]
# TODO: Make this a configuration value
try:
gevent.sleep(1)
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
break
# Heartbeat may already be dead if we're shutting down, so check first
if not hb.ready():
hb.kill(exception=greenlet.BPGreenletExit, block=True, timeout=1)
# Remove ourselves, but don't kill since we're already shutting down
greenlet.remove_identity_greenlet(count_identity, kill_greenlet=False)
# TODO: If a count process goes down, does every associated device go with
# it?
del _plugins[plugin.name]
queue.add(count_identity, ["s", "BPClose"])
logging.debug("Count process %s for %s exiting...", count_identity,
plugin.name)
def init():
"""Initialize structures and sockets needed to run router.
"""
_zmq["context"] = zmq.Context()
queue.init(_zmq["context"])
_zmq["router"] = _zmq["context"].socket(zmq.ROUTER)
_zmq["router"].bind(config.get_value("server_address"))
_zmq["router"].bind("inproc://ws-queue")
_zmq["router"].setsockopt(zmq.LINGER, 100)
_zmq["queue"] = _zmq["context"].socket(zmq.PULL)
_zmq["queue"].connect(queue.QUEUE_ADDRESS)
_zmq["poller"] = zmq.Poller()
_zmq["poller"].register(_zmq["router"], zmq.POLLIN)
_zmq["poller"].register(_zmq["queue"], zmq.POLLIN)
if config.get_value("websocket_address") is not "":
wsclient.init_ws(_zmq["context"])
def init():
"""Initialize structures and sockets needed to run router.
"""
_zmq["context"] = zmq.Context()
queue.init(_zmq["context"])
_zmq["router"] = _zmq["context"].socket(zmq.ROUTER)
_zmq["router"].bind(config.get_value("server_address"))
_zmq["router"].bind("inproc://ws-queue")
_zmq["router"].setsockopt(zmq.LINGER, 100)
_zmq["queue"] = _zmq["context"].socket(zmq.PULL)
_zmq["queue"].connect(queue.QUEUE_ADDRESS)
_zmq["poller"] = zmq.Poller()
_zmq["poller"].register(_zmq["router"], zmq.POLLIN)
_zmq["poller"].register(_zmq["queue"], zmq.POLLIN)
if config.get_value("websocket_address") is not "":
wsclient.init_ws(_zmq["context"])
def init_ws(context):
"""Initialize websocket connection handler.
"""
addr = config.get_value("websocket_address")
logging.info("Opening websocket server on %s", addr)
# TODO: Believe it or not this is not a valid way to check an address
_ws_server = WSGIServer(addr,
WebSocketClientFactory(context),
handler_class=WebSocketHandler)
_ws_server.start()
def run_device_plugin(identity, msg):
"""Execute the plugin claim protocol. This happens whenever a client requests
to claim a resource advertised by a plugin.
"""
# Figure out the plugin that owns the device we want
p = None
dev_id = msg[2]
for (plugin_name, device_list) in _devices.items():
if dev_id in device_list:
p = _plugins[plugin_name]
if p is None:
logging.warning("Cannot find device %s, failing claim", dev_id)
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
# See whether we already have a claim on the device
if dev_id in _dtc.keys():
logging.warning("Device %s already claimed, failing claim", dev_id)
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
# Client to system: bring up device process.
#
# Just name the new plugin process socket identity after the device id,
# because why not.
device_process = _start_process([p.executable_path, "--server_port=%s" %
config.get_value("server_address")],
dev_id)
if not device_process:
logging.warning("%s device process unable to start. removing.",
p.name)
return
# Device process to system: Register with known identity
e = event.add(dev_id, "BPPluginRegisterClaim")
try:
# TODO: Make device open timeout a config option
(i, m) = e.get(timeout=5)
except greenlet.BPGreenletExit:
# If we shut down now, just drop
return
except gevent.Timeout:
# If we timeout, fail the claim
logging.info("Device %s failed to start...", dev_id)
queue.add(dev_id, ["s", "BPClose"])
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
greenlet.add_identity_greenlet(dev_id, gevent.getcurrent())
# Add a heartbeat now that the process is up
hb = heartbeat.spawn_heartbeat(dev_id, gevent.getcurrent())
# System to device process: Open device
queue.add(dev_id, ["s", "BPPluginOpenDevice", dev_id])
e = event.add(dev_id, "BPPluginOpenDevice")
try:
(i, m) = e.get()
except greenlet.BPGreenletExit:
queue.add(dev_id, ["s", "BPClose"])
return
# Device process to system: Open or fail
if m[3] is False:
logging.info("Device %s failed to open...", dev_id)
queue.add(dev_id, ["s", "BPClose"])
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
# System to client: confirm device claim
queue.add(identity, ["s", "BPClaimDevice", dev_id, True])
if identity not in _ctd.keys():
_ctd[identity] = []
_ctd[identity].append(dev_id)
if dev_id not in _dtc.keys():
_dtc[dev_id] = []
_dtc[dev_id].append(identity)
while True:
try:
gevent.sleep(1)
except greenlet.BPGreenletExit:
break
if not hb.ready():
hb.kill(exception=greenlet.BPGreenletExit, block=True, timeout=1)
_ctd[identity].remove(dev_id)
del _dtc[dev_id]
# Remove ourselves, but don't kill since we're already shutting down
greenlet.remove_identity_greenlet(dev_id, kill_greenlet=False)
queue.add(dev_id, ["s", "BPClose"])
logging.debug("Device keeper %s exiting...", dev_id)
def testValidConfigValue(self):
"""get a matching config value"""
v = config.get_value("server_address")
self.failIf(v != config._config["server_address"])
def run_device_plugin(identity, msg):
"""Execute the plugin claim protocol. This happens whenever a client requests
to claim a resource advertised by a plugin.
"""
# Figure out the plugin that owns the device we want
p = None
dev_id = msg[2]
for (plugin_name, device_list) in _devices.items():
if dev_id in device_list:
p = _plugins[plugin_name]
if p is None:
logging.warning("Cannot find device %s, failing claim", dev_id)
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
# See whether we already have a claim on the device
if dev_id in _dtc.keys():
logging.warning("Device %s already claimed, failing claim", dev_id)
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
# Client to system: bring up device process.
#
# Just name the new plugin process socket identity after the device id,
# because why not.
device_process = _start_process([
p.executable_path,
"--server_port=%s" % config.get_value("server_address")
], dev_id)
if not device_process:
logging.warning("%s device process unable to start. removing.", p.name)
return
# Device process to system: Register with known identity
e = event.add(dev_id, "BPPluginRegisterClaim")
try:
# TODO: Make device open timeout a config option
(i, m) = e.get(timeout=5)
except greenlet.BPGreenletExit:
# If we shut down now, just drop
return
except gevent.Timeout:
# If we timeout, fail the claim
logging.info("Device %s failed to start...", dev_id)
queue.add(dev_id, ["s", "BPClose"])
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
greenlet.add_identity_greenlet(dev_id, gevent.getcurrent())
# Add a heartbeat now that the process is up
hb = heartbeat.spawn_heartbeat(dev_id, gevent.getcurrent())
# System to device process: Open device
queue.add(dev_id, ["s", "BPPluginOpenDevice", dev_id])
e = event.add(dev_id, "BPPluginOpenDevice")
try:
(i, m) = e.get()
except greenlet.BPGreenletExit:
queue.add(dev_id, ["s", "BPClose"])
return
# Device process to system: Open or fail
if m[3] is False:
logging.info("Device %s failed to open...", dev_id)
queue.add(dev_id, ["s", "BPClose"])
queue.add(identity, ["s", "BPClaimDevice", dev_id, False])
return
# System to client: confirm device claim
queue.add(identity, ["s", "BPClaimDevice", dev_id, True])
if identity not in _ctd.keys():
_ctd[identity] = []
_ctd[identity].append(dev_id)
if dev_id not in _dtc.keys():
_dtc[dev_id] = []
_dtc[dev_id].append(identity)
while True:
try:
gevent.sleep(1)
except greenlet.BPGreenletExit:
break
if not hb.ready():
hb.kill(exception=greenlet.BPGreenletExit, block=True, timeout=1)
_ctd[identity].remove(dev_id)
del _dtc[dev_id]
# Remove ourselves, but don't kill since we're already shutting down
greenlet.remove_identity_greenlet(dev_id, kill_greenlet=False)
queue.add(dev_id, ["s", "BPClose"])
logging.debug("Device keeper %s exiting...", dev_id)
def _run_count_plugin(plugin):
"""Runs the count process for a plugin. Constantly polls for list of devices,
keeping an internal reference of the devices available from the plugin.
"""
count_identity = greenlet.random_ident()
e = event.add(count_identity, "BPPluginRegisterCount")
count_process_cmd = [
plugin.executable_path,
"--server_port=%s" % config.get_value("server_address"), "--count"
]
count_process = _start_process(count_process_cmd, count_identity)
if not count_process:
logging.warning("%s count process unable to start. removing.",
plugin.name)
return
try:
e.get(block=True, timeout=1)
except gevent.Timeout:
logging.info("%s count process never registered, removing.",
plugin.name)
return
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
return
logging.info("Count process for %s up on identity %s", plugin.name,
count_identity)
greenlet.add_identity_greenlet(count_identity, gevent.getcurrent())
hb = heartbeat.spawn_heartbeat(count_identity, gevent.getcurrent())
_plugins[plugin.name] = plugin
while True:
queue.add(count_identity, ["s", "BPPluginDeviceList"])
e = event.add(count_identity, "BPPluginDeviceList")
try:
(i, msg) = e.get(block=True, timeout=1)
except gevent.Timeout:
logging.info("%s count process timed out, removing.", plugin.name)
break
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
break
_devices[plugin.name] = msg[2]
# TODO: Make this a configuration value
try:
gevent.sleep(1)
except greenlet.BPGreenletExit:
logging.debug("Shutting down count process for %s", plugin.name)
break
# Heartbeat may already be dead if we're shutting down, so check first
if not hb.ready():
hb.kill(exception=greenlet.BPGreenletExit, block=True, timeout=1)
# Remove ourselves, but don't kill since we're already shutting down
greenlet.remove_identity_greenlet(count_identity, kill_greenlet=False)
# TODO: If a count process goes down, does every associated device go with
# it?
del _plugins[plugin.name]
queue.add(count_identity, ["s", "BPClose"])
logging.debug("Count process %s for %s exiting...", count_identity,
plugin.name)
