class PluginToCauldron(Thread,Countable):
def __init__(self,name,cauldronAddr,plugin,log,
outFilter,outValueFilter=None):
Thread.__init__(self,name="%s (out)" % (name))
Countable.__init__(self)
self.daemon=True
self.cauldronAddr=cauldronAddr
self.plugin=plugin
self.allowCache={}
self.buffer=[]
self.outFilterExpr=outFilter
self.outFilter=re.compile(outFilter)
self.log=log
if outValueFilter!=None:
self.outValueFilter=re.compile(outValueFilter)
else:
self.outValueFilter=None
def run(self):
self.running=True
self.log("running")
self.cauldron=CauldronSender(self.cauldronAddr,max_shuttle_size=512,max_shuttle_age=.2)
for recordBytes in self.plugin:
if not self.running:
return
if len(recordBytes)>1:
try:
record=str(recordBytes,'UTF-8')
(label,value)=record.rstrip().split('=')
try:
allow=self.allowCache[label]
except KeyError:
self.count('new_labels_per_second',1)
allow=(self.outFilter.search(label)!=None)
self.allowCache[label]=allow
self.labels_allowed+=1
if allow:
self.cauldron.put(label,value)
self.count('records_per_second',1)
self.count('bytes_per_second',len(recordBytes))
else:
self.log("Label [%s] does not match filter (%s)" %
(label,self.outFilterExpr),
syslog.LOG_WARNING)
except ValueError:
self.log("Can't Parse [%s] [%s]" % (
str(recordBytes,'utf-8').rstrip(),
''.join(['%02x' % (thebyte) for thebyte in recordBytes])))
else:
self.count('shuttles_per_second',1)
def stop(self):
self.running=False
class Coven(object):
def __init__(self,
path,
prefix,
cauldronAddr,
ignore=[
r'^\.', r'\.x?swp$', r'~', r'^__', r'__$', r'\.jar$',
r'\.db$'
]):
syslog.openlog(ident="X3Coven", facility=syslog.LOG_DAEMON)
self.path = path
self.prefix = prefix
self.cauldronAddr = cauldronAddr
self.plugins = {}
self.plugins_lock = Lock()
self.confRE = re.compile('^#\\s*x3\.([a-z0-9.]+)\\s*=\\s*(.*)\\s*$')
self.dirManager = DirManager(self.path, self, ignore=ignore)
self.cauldron = CauldronSender(self.cauldronAddr)
def getOSPath(self, name):
return path.join(self.path, name)
def isShebangExecutable(self, path):
try:
with open(path, 'rb') as plugin_file:
magic_bytes = plugin_file.read(2)
if magic_bytes == b'#!':
return True
except:
pass
return False
def getConfig(self, name):
pluginPath = self.getOSPath(name)
if not self.isShebangExecutable(pluginPath):
return None
conf = {}
try:
with open(pluginPath, 'r') as plugin_file:
for line in plugin_file:
config_match = self.confRE.match(line)
if config_match:
conf[config_match.group(1)] = config_match.group(2)
except:
return None
return conf
def put(self, label, value):
self.cauldron.put('%s.coven.%s' % (self.prefix, label), value)
def log(self, message, component=None, priority=syslog.LOG_DEBUG):
syslog.syslog(
priority,
message if component == None else '[%s] %s' % (component, message))
def practice(self):
try:
self.dirManager.start()
while True:
totals = {}
with self.plugins_lock:
self.put('plugins.running', len(self.plugins))
for plugin in self.plugins.values():
for (values_type,
values) in plugin.get_counters().items():
try:
total_values = totals[values_type]
except KeyError:
total_values = {}
totals[values_type] = total_values
for (label, value) in values.items():
try:
total_values[label] += value
except KeyError:
total_values[label] = value
self.put(
'plugin.%s.%s.%s' %
(plugin.name, values_type, label), value)
plugin.reset_counters()
for (values_type, values) in totals.items():
for (label, value) in values.items():
self.put('plugins.total.%s.%s' % (values_type, label),
value)
time.sleep(1.0)
except KeyboardInterrupt:
self.log("Stopping..")
self.dirManager.stop()
with self.plugins_lock:
for partingPlugin in self.plugins.values():
partingPlugin.stop()
for partingPlugin in self.plugins.values():
partingPlugin.join()
self.log("Stopped..", priority=syslog.LOG_INFO)
# ----------------------------------------------------------------------
def startPlugin(self, name):
self.log("Starting %s" % (name))
config = self.getConfig(name)
if config:
try:
with self.plugins_lock:
self.plugins[name] = Plugin(name, self.getOSPath(name),
self.prefix, config,
self.cauldronAddr, self.log)
self.plugins[name].start()
self.log("Started %s" % (name), priority=syslog.LOG_INFO)
except:
self.log("Failed To Start %s" % (name),
priority=syslog.LOG_ERR)
else:
self.log("Not an X3 Plugin: %s" % (name),
priority=syslog.LOG_WARNING)
def stopPlugin(self, name):
self.log("Stopping %s" % (name))
try:
with self.plugins_lock:
self.plugins[name].stop()
del self.plugins[name]
self.log("Stopped %s" % (name), priority=syslog.LOG_INFO)
except:
self.log("Failed To Stop %s" % (name), priority=syslog.LOG_ERR)
# ----------------------------------------------------------------------
def process_FileCreated(self, name):
self.startPlugin(name)
def process_FileDeleted(self, name):
self.stopPlugin(name)
def process_FileChanged(self, name):
self.log("Restarting %s" % (name), priority=syslog.LOG_INFO)
self.stopPlugin(name)
self.startPlugin(name)
class PluginToCauldron(Thread, Countable):
def __init__(self,
name,
cauldronAddr,
plugin,
log,
outFilter,
outValueFilter=None):
Thread.__init__(self, name="%s (out)" % (name))
Countable.__init__(self)
self.daemon = True
self.cauldronAddr = cauldronAddr
self.plugin = plugin
self.allowCache = {}
self.buffer = []
self.outFilterExpr = outFilter
self.outFilter = re.compile(outFilter)
self.log = log
if outValueFilter != None:
self.outValueFilter = re.compile(outValueFilter)
else:
self.outValueFilter = None
def run(self):
self.running = True
self.log("running")
self.cauldron = CauldronSender(self.cauldronAddr,
max_shuttle_size=512,
max_shuttle_age=.2)
for recordBytes in self.plugin:
if not self.running:
return
if len(recordBytes) > 1:
try:
record = str(recordBytes, 'UTF-8')
(label, value) = record.rstrip().split('=')
try:
allow = self.allowCache[label]
except KeyError:
self.count('new_labels_per_second', 1)
allow = (self.outFilter.search(label) != None)
self.allowCache[label] = allow
self.labels_allowed += 1
if allow:
self.cauldron.put(label, value)
self.count('records_per_second', 1)
self.count('bytes_per_second', len(recordBytes))
else:
self.log(
"Label [%s] does not match filter (%s)" %
(label, self.outFilterExpr), syslog.LOG_WARNING)
except ValueError:
self.log(
"Can't Parse [%s] [%s]" %
(str(recordBytes, 'utf-8').rstrip(), ''.join(
['%02x' % (thebyte) for thebyte in recordBytes])))
else:
self.count('shuttles_per_second', 1)
def stop(self):
self.running = False
class Coven(object):
def __init__( self,path,prefix,cauldronAddr,
ignore=[r'^\.',r'\.x?swp$',r'~',r'^__',r'__$',r'\.jar$',r'\.db$']):
syslog.openlog(ident="X3Coven",facility=syslog.LOG_DAEMON)
self.path=path
self.prefix=prefix
self.cauldronAddr=cauldronAddr
self.plugins={}
self.confRE=re.compile('^#\\s*x3\.([a-z0-9.]+)\\s*=\\s*(.*)\\s*$')
self.dirManager=DirManager(self.path,self,ignore=ignore)
self.cauldron=CauldronSender(self.cauldronAddr)
def getOSPath(self,name):
return path.join(self.path,name)
def isShebangExecutable(self,path):
try:
with open(path,'rb') as plugin_file:
magic_bytes=plugin_file.read(2)
if magic_bytes==b'#!':
return True
except:
pass
return False
def getConfig(self,name):
pluginPath=self.getOSPath(name)
if not self.isShebangExecutable(pluginPath):
return None
conf={}
try:
with open(pluginPath,'r') as plugin_file:
for line in plugin_file:
config_match=self.confRE.match(line)
if config_match:
conf[config_match.group(1)]=config_match.group(2)
except:
return None
return conf
def put(self,label,value):
self.cauldron.put('%s.coven.%s' % (self.prefix,label),value)
def log(self,message,component=None,priority=syslog.LOG_DEBUG):
syslog.syslog(priority,message if component==None
else '[%s] %s' % (component,message))
def practice(self):
try:
self.dirManager.start()
while True:
totals={}
self.put('plugins.running',len(self.plugins))
for plugin in self.plugins.values():
for (values_type,values) in plugin.get_counters().items():
try:
total_values=totals[values_type]
except KeyError:
total_values={}
totals[values_type]=total_values
for (label,value) in values.items():
try:
total_values[label]+=value
except KeyError:
total_values[label]=value
self.put('plugin.%s.%s.%s' %
(plugin.name,values_type,label),value)
plugin.reset_counters()
for (values_type,values) in totals.items():
for (label,value) in values.items():
self.put('plugins.total.%s.%s' % (values_type,label),value)
time.sleep(1.0)
except KeyboardInterrupt:
self.log("Stopping..")
self.dirManager.stop()
for partingPlugin in self.plugins.values():
partingPlugin.stop()
for partingPlugin in self.plugins.values():
partingPlugin.join()
self.log("Stopped..",priority=syslog.LOG_INFO)
# ------------------------------------------------------------------------
def startPlugin(self,name):
self.log("Starting %s" % (name))
config=self.getConfig(name)
if config:
try:
self.plugins[name]=Plugin(name, self.getOSPath(name),
self.prefix, config, self.cauldronAddr,
self.log)
self.plugins[name].start()
self.log("Started %s" % (name),priority=syslog.LOG_INFO)
except:
self.log("Failed To Start %s" % (name),priority=syslog.LOG_ERR)
else:
self.log("Not an X3 Plugin: %s" % (name),
priority=syslog.LOG_WARNING)
def stopPlugin(self,name):
self.log("Stopping %s" % (name))
try:
self.plugins[name].stop()
del self.plugins[name]
self.log("Stopped %s" % (name),priority=syslog.LOG_INFO)
except:
self.log("Failed To Stop %s" % (name),priority=syslog.LOG_ERR)
# ------------------------------------------------------------------------
def process_FileCreated(self,name):
self.startPlugin(name)
def process_FileDeleted(self,name):
self.stopPlugin(name)
def process_FileChanged(self,name):
self.log("Restarting %s" % (name),priority=syslog.LOG_INFO)
self.stopPlugin(name)
self.startPlugin(name)