def __init__(self):
self.config = BootstrapFile()
self.routers = []
signal(2, self.__stopSequence)
self.__stopping = False
self.module_manager = ModuleManager()
def show(self):
'''Maps to the CLI command and shows the docstring of the Wishbone module.
'''
module_manager = ModuleManager()
module_manager.validateModuleName(self.module)
module_manager.exists(self.module)
print((self.generateHeader()))
try:
(category, group, self.module) = self.module.split('.')
except ValueError:
(category, sub, group, self.module) = self.module.split('.')
category = "%s.%s" % (category, sub)
try:
title = module_manager.getModuleTitle(category, group, self.module)
version = module_manager.getModuleVersion(category, group,
self.module)
header = "%s.%s.%s" % (category, group, self.module)
print("")
print(("=" * len(header)))
print(header)
print(("=" * len(header)))
print("")
print(("Version: %s" % (version)))
print("")
print(title)
print(("-" * len(title)))
print((module_manager.getModuleDoc(category, group, self.module)))
except Exception as err:
print(("Failed to load module %s.%s.%s. Reason: %s" %
(category, group, self.module, err)))
def show(self, command, module):
'''
Shows the help message of a module.
'''
module_manager = ModuleManager()
module_manager.validateModuleName(module)
module_manager.exists(module)
print((self.generateHeader()))
try:
(category, group, module) = module.split('.')
except ValueError:
(category, sub, group, module) = module.split('.')
category = "%s.%s" % (category, sub)
try:
title = module_manager.getModuleTitle(category, group, module)
version = module_manager.getModuleVersion(category, group, module)
header = "%s.%s.%s" % (category, group, module)
print("")
print("="*len(header))
print(header)
print("="*len(header))
print("")
print("Version: %s" % (version))
print("")
print(title)
print("-"*len(title))
print(module_manager.getModuleDoc(category, group, module))
except Exception as err:
print("Failed to load module %s.%s.%s. Reason: %s" % (category, group, module, err))
def show(self):
'''Maps to the CLI command and shows the docstring of the Wishbone module.
'''
module_manager = ModuleManager()
module_manager.validateModuleName(self.module)
module_manager.exists(self.module)
print((self.generateHeader()))
try:
(category, group, self.module) = self.module.split('.')
except ValueError:
(category, sub, group, self.module) = self.module.split('.')
category = "%s.%s" % (category, sub)
try:
title = module_manager.getModuleTitle(category, group, self.module)
version = module_manager.getModuleVersion(category, group, self.module)
header = "%s.%s.%s" % (category, group, self.module)
print("")
print(("="*len(header)))
print(header)
print(("="*len(header)))
print("")
print(("Version: %s" % (version)))
print("")
print(title)
print(("-"*len(title)))
print((module_manager.getModuleDoc(category, group, self.module)))
except Exception as err:
print(("Failed to load module %s.%s.%s. Reason: %s" % (category, group, self.module, err)))
def list(self):
'''Maps to the CLI command and lists all Wishbone entrypoint modules it can find.
'''
categories = ["wishbone", "wishbone_contrib"]
groups = ["flow", "encode", "decode", "function", "input", "output"]
print((self.generateHeader()))
print("Available event modules:")
print((ModuleManager(categories=categories,
groups=groups).getModuleTable()))
print("\n")
print("Available lookup function modules:")
print((ModuleManager(categories=categories,
groups=["lookup"]).getModuleTable()))
def __init__(self,
config=None,
size=100,
frequency=1,
identification="wishbone",
graph=False,
graph_include_sys=False):
self.module_manager = ModuleManager()
self.config = config
self.size = size
self.frequency = frequency
self.identification = identification
self.graph = graph
self.graph_include_sys = graph_include_sys
self.module_pool = ModulePool()
self.__block = event.Event()
self.__block.clear()
def __init__(self, config=None, size=100, frequency=1, identification="wishbone", graph=False, graph_include_sys=False):
self.module_manager = ModuleManager()
self.config = config
self.size = size
self.frequency = frequency
self.identification = identification
self.graph = graph
self.graph_include_sys = graph_include_sys
self.module_pool = ModulePool()
self.__block = event.Event()
self.__block.clear()
class Dispatch():
'''
Handles the Wishbone instance commands.
'''
def __init__(self):
self.config = BootstrapFile()
self.routers = []
signal(2, self.__stopSequence)
self.__stopping = False
self.module_manager = ModuleManager()
def generateHeader(self):
'''
Prints a header.
'''
return """ __ __ __
.--.--.--|__.-----| |--| |--.-----.-----.-----.
| | | | |__ --| | _ | _ | | -__|
|________|__|_____|__|__|_____|_____|__|__|_____|
version %s
Build event pipeline servers with minimal effort.
""" % (get_distribution('wishbone').version)
def debug(self, command, config, instances, queue_size, frequency, identification):
'''
Handles the Wishbone debug command.
'''
config = self.config.load(config)
if instances == 1:
self.routers.append(RouterBootstrap(config, debug=True, queue_size=queue_size, frequency=frequency, identification=identification))
self.routers[-1].start()
else:
for x in xrange(instances):
self.routers.append(RouterBootstrapProcess(config, debug=True, queue_size=queue_size, frequency=frequency, identification=identification))
self.routers[-1].start()
while multiprocessing.active_children():
sleep(1)
sys.exit(0)
def list(self, command, group, category=None, include_groups=[]):
print self.generateHeader()
print "Available modules:"
print self.module_manager.getModuleTable(category, group, include_groups)
def show(self, command, module):
'''
Shows the help message of a module.
'''
print self.generateHeader()
try:
(category, group, module) = module.split('.')
except ValueError:
(category, sub, group, module) = module.split('.')
category = "%s.%s" % (category, sub)
try:
title = self.module_manager.getModuleTitle(category, group, module)
version = self.module_manager.getModuleVersion(category, group, module)
header = "%s.%s.%s" % (category, group, module)
print
print "="*len(header)
print header
print "="*len(header)
print
print "Version: %s" % (version)
print
print title
print "-"*len(title)
print self.module_manager.getModuleDoc(category, group, module)
except Exception:
print "Failed to load module %s.%s.%s." % (category, group, module)
def start(self, command, config, instances, pid, queue_size, frequency, identification):
'''
Handles the Wishbone start command.
'''
config = self.config.load(config)
self.pid = PIDFile(pid)
if instances == 1:
print ("Starting 1 instance to background with pid %s." % (os.getpid()))
try:
with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, files_preserve=self.__getCurrentFD(), detach_process=True):
self.pid.create([os.getpid()])
instance = RouterBootstrap(config, debug=False, queue_size=queue_size, frequency=frequency, identification=identification)
instance.start()
except Exception as err:
sys.stdout.write("Failed to start instance. Reason: %s\n" % (err))
else:
try:
print "Starting %s instances in background." % (instances)
with DaemonContext(stdout=sys.stdout, stderr=sys.stderr, files_preserve=self.__getCurrentFD(), detach_process=True):
pids = []
processes = []
for counter in xrange(instances):
processes.append(RouterBootstrapProcess(config, debug=False, queue_size=queue_size, frequency=frequency, identification=identification))
processes[-1].start()
pids.append(processes[-1].pid)
self.pid.create(pids)
for process in processes:
process.join()
except Exception as err:
sys.stdout.write("Failed to start instance. Reason: %s\n" % (err))
def stop(self, command, pid):
'''
Handles the Wishbone stop command.
'''
try:
pid = PIDFile(pid)
sys.stdout.write("Stopping instance with PID ")
sys.stdout.flush()
for entry in pid.read():
sys.stdout.write(" %s " % (entry))
sys.stdout.flush()
pid.sendSigint(entry)
pid.cleanup()
print("")
except Exception as err:
print ("")
print ("Failed to stop instances. Reason: %s" % (err))
def __stopSequence(self):
'''
Calls the stop() function of each instance.
'''
if not self.__stopping:
# TODO: Weird hack, otherwise when trapping signal(2) this function is
# executed many times.
self.__stopping = True
for instance in self.routers:
if hasattr(instance, "stop"):
instance.stop()
def __getCurrentFD(self):
'''
returns a list with filedescriptors in use.
'''
try:
return [int(x) for x in os.listdir("/proc/self/fd")]
except Exception as err:
print ("Failed to get active filedescriptors. Reason: %s." % (err))
sys.exit(1)
def __alive(self, pid):
try:
os.kill(pid, 0)
return True
except:
False
class Default(object):
'''The default Wishbone router.
A Wishbone router is responsible for shoveling the messages from one
module to the other.
Args:
config (obj): The router setup configuration.
size (int): The size of all queues.
frequency (int)(1): The frequency at which metrics are produced.
identification (wishbone): A string identifying this instance in logging.
'''
def __init__(self, config=None, size=100, frequency=1, identification="wishbone", graph=False, graph_include_sys=False):
self.module_manager = ModuleManager()
self.config = config
self.size = size
self.frequency = frequency
self.identification = identification
self.graph = graph
self.graph_include_sys = graph_include_sys
self.module_pool = ModulePool()
self.__block = event.Event()
self.__block.clear()
def block(self):
'''Blocks until stop() is called and the shutdown process ended.'''
self.__block.wait()
def connectQueue(self, source, destination):
'''Connects one queue to the other.
For convenience, the syntax of the queues is <modulename>.<queuename>
For example:
stdout.inbox
Args:
source (str): The source queue in <module.queue_name> syntax
destination (str): The destination queue in <module.queue_name> syntax
'''
(source_module, source_queue) = source.split('.')
(destination_module, destination_queue) = destination.split('.')
source = self.module_pool.getModule(source_module)
destination = self.module_pool.getModule(destination_module)
source.connect(source_queue, destination, destination_queue)
def getChildren(self, module):
'''Returns all the connected child modules
Args:
module (str): The name of the module.
Returns:
list: A list of module names.
'''
children = []
def lookupChildren(module, children):
for module in self.module_pool.getModule(module).getChildren():
name = module.split(".")[0]
if name not in children:
children.append(name)
lookupChildren(name, children)
try:
lookupChildren(module, children)
except NoSuchModule:
return []
else:
return children
def registerModule(self, module, actor_config, arguments={}):
'''Initializes the wishbone module module.
Args:
module (Actor): A Wishbone module object (not initialized)
actor_config (ActorConfig): The module's actor configuration
arguments (dict): The parameters to initialize the module.
'''
try:
setattr(self.module_pool.module, actor_config.name, module(actor_config, **arguments))
except Exception as err:
raise ModuleInitFailure("Problem loading module %s. Reason: %s" % (actor_config.name, err))
def stop(self):
'''Stops all running modules.'''
for module in self.module_pool.list():
if module.name not in self.getChildren("_logs") + ["_logs"] and not module.stopped:
module.stop()
while not self.__logsEmpty():
sleep(0.1)
self.__running = False
self.__block.set()
def start(self):
'''Starts all registered modules.'''
if self.config is not None:
self.__initConfig()
if self.graph:
self.graph = GraphWebserver(self.config, self.module_pool, self.__block, self.graph_include_sys)
self.graph.start()
for module in self.module_pool.list():
module.start()
def __initConfig(self):
'''Setup all modules and routes.'''
lookup_modules = {}
for name, instance in list(self.config.lookups.items()):
lookup_modules[name] = self.__registerLookupModule(instance.module, **instance.arguments)
for name, instance in list(self.config.modules.items()):
pmodule = self.module_manager.getModuleByName(instance.module)
if instance.description == "":
instance.description = pmodule.__doc__.split("\n")[0].replace('*', '')
actor_config = ActorConfig(name, self.size, self.frequency, lookup_modules, instance.description)
self.registerModule(pmodule, actor_config, instance.arguments)
self.__setupConnections()
def __logsEmpty(self):
'''Checks each module whether any logs have stayed behind.'''
for module in self.module_pool.list():
if not module.pool.queue.logs.size() == 0:
return False
else:
return True
def __registerLookupModule(self, module, **kwargs):
'''Registers a lookupmodule
Args:
module (Looup): The lookup module (not initialized)
**kwargs (dict): The parameters used to initiolize the lookup module.
Raises:
FunctionInitFailure: An error occurred loading and initializing the module.
'''
for group in ["wishbone.lookup", "wishbone_contrib.lookup"]:
for entry_point in iter_entry_points(group=group, name=None):
if "%s.%s" % (group, entry_point.name) == module:
l = entry_point.load()(**kwargs)
if hasattr(l, "lookup"):
return l.lookup
else:
raise FunctionInitFailure("Lookup module '%s' does not seem to have a 'lookup' method" % (l.module_name))
raise FunctionInitFailure("Lookup module '%s' does not exist." % (module))
def __setupConnections(self):
'''Setup all connections as defined by configuration_manager'''
for route in self.config.routingtable:
self.connectQueue("%s.%s" % (route.source_module, route.source_queue), "%s.%s" % (route.destination_module, route.destination_queue))
class Default(object):
'''The default Wishbone router.
A Wishbone router is responsible for shoveling the messages from one
module to the other.
Args:
config (obj): The router setup configuration.
size (int): The size of all queues.
frequency (int)(1): The frequency at which metrics are produced.
identification (wishbone): A string identifying this instance in logging.
'''
def __init__(self,
config=None,
size=100,
frequency=1,
identification="wishbone",
graph=False,
graph_include_sys=False):
self.module_manager = ModuleManager()
self.config = config
self.size = size
self.frequency = frequency
self.identification = identification
self.graph = graph
self.graph_include_sys = graph_include_sys
self.module_pool = ModulePool()
self.__block = event.Event()
self.__block.clear()
def block(self):
'''Blocks until stop() is called and the shutdown process ended.'''
self.__block.wait()
def connectQueue(self, source, destination):
'''Connects one queue to the other.
For convenience, the syntax of the queues is <modulename>.<queuename>
For example:
stdout.inbox
Args:
source (str): The source queue in <module.queue_name> syntax
destination (str): The destination queue in <module.queue_name> syntax
'''
(source_module, source_queue) = source.split('.')
(destination_module, destination_queue) = destination.split('.')
source = self.module_pool.getModule(source_module)
destination = self.module_pool.getModule(destination_module)
source.connect(source_queue, destination, destination_queue)
def getChildren(self, module):
'''Returns all the connected child modules
Args:
module (str): The name of the module.
Returns:
list: A list of module names.
'''
children = []
def lookupChildren(module, children):
for module in self.module_pool.getModule(module).getChildren():
name = module.split(".")[0]
if name not in children:
children.append(name)
lookupChildren(name, children)
try:
lookupChildren(module, children)
except NoSuchModule:
return []
else:
return children
def registerModule(self, module, actor_config, arguments={}):
'''Initializes the wishbone module module.
Args:
module (Actor): A Wishbone module object (not initialized)
actor_config (ActorConfig): The module's actor configuration
arguments (dict): The parameters to initialize the module.
'''
try:
setattr(self.module_pool.module, actor_config.name,
module(actor_config, **arguments))
except Exception as err:
raise ModuleInitFailure("Problem loading module %s. Reason: %s" %
(actor_config.name, err))
def stop(self):
'''Stops all running modules.'''
for module in self.module_pool.list():
if module.name not in self.getChildren("_logs") + [
"_logs"
] and not module.stopped:
module.stop()
while not self.__logsEmpty():
sleep(0.1)
self.__running = False
self.__block.set()
def start(self):
'''Starts all registered modules.'''
if self.config is not None:
self.__initConfig()
if self.graph:
self.graph = GraphWebserver(self.config, self.module_pool,
self.__block, self.graph_include_sys)
self.graph.start()
for module in self.module_pool.list():
module.start()
def __initConfig(self):
'''Setup all modules and routes.'''
lookup_modules = {}
for name, instance in list(self.config.lookups.items()):
lookup_modules[name] = self.__registerLookupModule(
instance.module, **instance.arguments)
for name, instance in list(self.config.modules.items()):
pmodule = self.module_manager.getModuleByName(instance.module)
if instance.description == "":
instance.description = pmodule.__doc__.split("\n")[0].replace(
'*', '')
actor_config = ActorConfig(name, self.size, self.frequency,
lookup_modules, instance.description)
self.registerModule(pmodule, actor_config, instance.arguments)
self.__setupConnections()
def __logsEmpty(self):
'''Checks each module whether any logs have stayed behind.'''
for module in self.module_pool.list():
if not module.pool.queue.logs.size() == 0:
return False
else:
return True
def __registerLookupModule(self, module, **kwargs):
'''Registers a lookupmodule
Args:
module (Looup): The lookup module (not initialized)
**kwargs (dict): The parameters used to initiolize the lookup module.
Raises:
FunctionInitFailure: An error occurred loading and initializing the module.
'''
for group in ["wishbone.lookup", "wishbone_contrib.lookup"]:
for entry_point in iter_entry_points(group=group, name=None):
if "%s.%s" % (group, entry_point.name) == module:
l = entry_point.load()(**kwargs)
if hasattr(l, "lookup"):
return l.lookup
else:
raise FunctionInitFailure(
"Lookup module '%s' does not seem to have a 'lookup' method"
% (l.module_name))
raise FunctionInitFailure("Lookup module '%s' does not exist." %
(module))
def __setupConnections(self):
'''Setup all connections as defined by configuration_manager'''
for route in self.config.routingtable:
self.connectQueue(
"%s.%s" % (route.source_module, route.source_queue),
"%s.%s" % (route.destination_module, route.destination_queue))
