Ejemplo n.º 1
0
 def run(self):
     self.running = True
     self.log("running")
     self.cauldron = CauldronSender(self.cauldronAddr,
                                    max_shuttle_size=512,
                                    max_shuttle_age=.2)
     for recordBytes in self.plugin:
         if not self.running:
             return
         if len(recordBytes) > 1:
             try:
                 record = str(recordBytes, 'UTF-8')
                 (label, value) = record.rstrip().split('=')
                 try:
                     allow = self.allowCache[label]
                 except KeyError:
                     self.count('new_labels_per_second', 1)
                     allow = (self.outFilter.search(label) != None)
                     self.allowCache[label] = allow
                     self.labels_allowed += 1
                 if allow:
                     self.cauldron.put(label, value)
                     self.count('records_per_second', 1)
                     self.count('bytes_per_second', len(recordBytes))
                 else:
                     self.log(
                         "Label [%s] does not match filter (%s)" %
                         (label, self.outFilterExpr), syslog.LOG_WARNING)
             except ValueError:
                 self.log(
                     "Can't Parse [%s] [%s]" %
                     (str(recordBytes, 'utf-8').rstrip(), ''.join(
                         ['%02x' % (thebyte) for thebyte in recordBytes])))
         else:
             self.count('shuttles_per_second', 1)
Ejemplo n.º 2
0
class PluginToCauldron(Thread,Countable):
  def __init__(self,name,cauldronAddr,plugin,log,
               outFilter,outValueFilter=None):

    Thread.__init__(self,name="%s (out)" % (name))
    Countable.__init__(self)
    self.daemon=True
    self.cauldronAddr=cauldronAddr
    self.plugin=plugin
    self.allowCache={}
    self.buffer=[]
    self.outFilterExpr=outFilter
    self.outFilter=re.compile(outFilter)
    self.log=log

    if outValueFilter!=None:
      self.outValueFilter=re.compile(outValueFilter)
    else:
      self.outValueFilter=None

  def run(self):
    self.running=True
    self.log("running")
    self.cauldron=CauldronSender(self.cauldronAddr,max_shuttle_size=512,max_shuttle_age=.2)
    for recordBytes in self.plugin:
      if not self.running:
        return
      if len(recordBytes)>1:
        try:
          record=str(recordBytes,'UTF-8')
          (label,value)=record.rstrip().split('=')
          try:
            allow=self.allowCache[label]
          except KeyError:
            self.count('new_labels_per_second',1)
            allow=(self.outFilter.search(label)!=None)
            self.allowCache[label]=allow
            self.labels_allowed+=1
          if allow:
            self.cauldron.put(label,value)
            self.count('records_per_second',1)
            self.count('bytes_per_second',len(recordBytes))
          else:
            self.log("Label [%s] does not match filter (%s)" %
                     (label,self.outFilterExpr),
                      syslog.LOG_WARNING)
        except ValueError:
          self.log("Can't Parse [%s] [%s]" % (
              str(recordBytes,'utf-8').rstrip(),
              ''.join(['%02x' % (thebyte) for thebyte in recordBytes])))
      else:
        self.count('shuttles_per_second',1)

  def stop(self):
    self.running=False
Ejemplo n.º 3
0
 def run(self):
   self.running=True
   self.log("running")
   self.cauldron=CauldronSender(self.cauldronAddr,max_shuttle_size=512,max_shuttle_age=.2)
   for recordBytes in self.plugin:
     if not self.running:
       return
     if len(recordBytes)>1:
       try:
         record=str(recordBytes,'UTF-8')
         (label,value)=record.rstrip().split('=')
         try:
           allow=self.allowCache[label]
         except KeyError:
           self.count('new_labels_per_second',1)
           allow=(self.outFilter.search(label)!=None)
           self.allowCache[label]=allow
           self.labels_allowed+=1
         if allow:
           self.cauldron.put(label,value)
           self.count('records_per_second',1)
           self.count('bytes_per_second',len(recordBytes))
         else:
           self.log("Label [%s] does not match filter (%s)" %
                    (label,self.outFilterExpr),
                     syslog.LOG_WARNING)
       except ValueError:
         self.log("Can't Parse [%s] [%s]" % (
             str(recordBytes,'utf-8').rstrip(),
             ''.join(['%02x' % (thebyte) for thebyte in recordBytes])))
     else:
       self.count('shuttles_per_second',1)
Ejemplo n.º 4
0
 def __init__(self,
              path,
              prefix,
              cauldronAddr,
              ignore=[
                  r'^\.', r'\.x?swp$', r'~', r'^__', r'__$', r'\.jar$',
                  r'\.db$'
              ]):
     syslog.openlog(ident="X3Coven", facility=syslog.LOG_DAEMON)
     self.path = path
     self.prefix = prefix
     self.cauldronAddr = cauldronAddr
     self.plugins = {}
     self.plugins_lock = Lock()
     self.confRE = re.compile('^#\\s*x3\.([a-z0-9.]+)\\s*=\\s*(.*)\\s*$')
     self.dirManager = DirManager(self.path, self, ignore=ignore)
     self.cauldron = CauldronSender(self.cauldronAddr)
Ejemplo n.º 5
0
 def __init__( self,path,prefix,cauldronAddr,
     ignore=[r'^\.',r'\.x?swp$',r'~',r'^__',r'__$',r'\.jar$',r'\.db$']):
   syslog.openlog(ident="X3Coven",facility=syslog.LOG_DAEMON)
   self.path=path
   self.prefix=prefix
   self.cauldronAddr=cauldronAddr
   self.plugins={}
   self.confRE=re.compile('^#\\s*x3\.([a-z0-9.]+)\\s*=\\s*(.*)\\s*$')
   self.dirManager=DirManager(self.path,self,ignore=ignore)
   self.cauldron=CauldronSender(self.cauldronAddr)
Ejemplo n.º 6
0
class Coven(object):
    def __init__(self,
                 path,
                 prefix,
                 cauldronAddr,
                 ignore=[
                     r'^\.', r'\.x?swp$', r'~', r'^__', r'__$', r'\.jar$',
                     r'\.db$'
                 ]):
        syslog.openlog(ident="X3Coven", facility=syslog.LOG_DAEMON)
        self.path = path
        self.prefix = prefix
        self.cauldronAddr = cauldronAddr
        self.plugins = {}
        self.plugins_lock = Lock()
        self.confRE = re.compile('^#\\s*x3\.([a-z0-9.]+)\\s*=\\s*(.*)\\s*$')
        self.dirManager = DirManager(self.path, self, ignore=ignore)
        self.cauldron = CauldronSender(self.cauldronAddr)

    def getOSPath(self, name):
        return path.join(self.path, name)

    def isShebangExecutable(self, path):
        try:
            with open(path, 'rb') as plugin_file:
                magic_bytes = plugin_file.read(2)
                if magic_bytes == b'#!':
                    return True
        except:
            pass
        return False

    def getConfig(self, name):
        pluginPath = self.getOSPath(name)
        if not self.isShebangExecutable(pluginPath):
            return None
        conf = {}
        try:
            with open(pluginPath, 'r') as plugin_file:
                for line in plugin_file:
                    config_match = self.confRE.match(line)
                    if config_match:
                        conf[config_match.group(1)] = config_match.group(2)
        except:
            return None
        return conf

    def put(self, label, value):
        self.cauldron.put('%s.coven.%s' % (self.prefix, label), value)

    def log(self, message, component=None, priority=syslog.LOG_DEBUG):
        syslog.syslog(
            priority,
            message if component == None else '[%s] %s' % (component, message))

    def practice(self):
        try:
            self.dirManager.start()
            while True:
                totals = {}
                with self.plugins_lock:
                    self.put('plugins.running', len(self.plugins))
                    for plugin in self.plugins.values():
                        for (values_type,
                             values) in plugin.get_counters().items():
                            try:
                                total_values = totals[values_type]
                            except KeyError:
                                total_values = {}
                                totals[values_type] = total_values
                            for (label, value) in values.items():
                                try:
                                    total_values[label] += value
                                except KeyError:
                                    total_values[label] = value
                                self.put(
                                    'plugin.%s.%s.%s' %
                                    (plugin.name, values_type, label), value)
                        plugin.reset_counters()

                for (values_type, values) in totals.items():
                    for (label, value) in values.items():
                        self.put('plugins.total.%s.%s' % (values_type, label),
                                 value)
                time.sleep(1.0)
        except KeyboardInterrupt:
            self.log("Stopping..")
            self.dirManager.stop()
            with self.plugins_lock:
                for partingPlugin in self.plugins.values():
                    partingPlugin.stop()
                for partingPlugin in self.plugins.values():
                    partingPlugin.join()
            self.log("Stopped..", priority=syslog.LOG_INFO)

# ----------------------------------------------------------------------

    def startPlugin(self, name):
        self.log("Starting %s" % (name))
        config = self.getConfig(name)
        if config:
            try:
                with self.plugins_lock:
                    self.plugins[name] = Plugin(name, self.getOSPath(name),
                                                self.prefix, config,
                                                self.cauldronAddr, self.log)
                    self.plugins[name].start()
                    self.log("Started %s" % (name), priority=syslog.LOG_INFO)
            except:
                self.log("Failed To Start %s" % (name),
                         priority=syslog.LOG_ERR)
        else:
            self.log("Not an X3 Plugin: %s" % (name),
                     priority=syslog.LOG_WARNING)

    def stopPlugin(self, name):
        self.log("Stopping %s" % (name))
        try:
            with self.plugins_lock:
                self.plugins[name].stop()
                del self.plugins[name]
            self.log("Stopped %s" % (name), priority=syslog.LOG_INFO)
        except:
            self.log("Failed To Stop %s" % (name), priority=syslog.LOG_ERR)

# ----------------------------------------------------------------------

    def process_FileCreated(self, name):
        self.startPlugin(name)

    def process_FileDeleted(self, name):
        self.stopPlugin(name)

    def process_FileChanged(self, name):
        self.log("Restarting %s" % (name), priority=syslog.LOG_INFO)
        self.stopPlugin(name)
        self.startPlugin(name)
Ejemplo n.º 7
0
class PluginToCauldron(Thread, Countable):
    def __init__(self,
                 name,
                 cauldronAddr,
                 plugin,
                 log,
                 outFilter,
                 outValueFilter=None):

        Thread.__init__(self, name="%s (out)" % (name))
        Countable.__init__(self)
        self.daemon = True
        self.cauldronAddr = cauldronAddr
        self.plugin = plugin
        self.allowCache = {}
        self.buffer = []
        self.outFilterExpr = outFilter
        self.outFilter = re.compile(outFilter)
        self.log = log

        if outValueFilter != None:
            self.outValueFilter = re.compile(outValueFilter)
        else:
            self.outValueFilter = None

    def run(self):
        self.running = True
        self.log("running")
        self.cauldron = CauldronSender(self.cauldronAddr,
                                       max_shuttle_size=512,
                                       max_shuttle_age=.2)
        for recordBytes in self.plugin:
            if not self.running:
                return
            if len(recordBytes) > 1:
                try:
                    record = str(recordBytes, 'UTF-8')
                    (label, value) = record.rstrip().split('=')
                    try:
                        allow = self.allowCache[label]
                    except KeyError:
                        self.count('new_labels_per_second', 1)
                        allow = (self.outFilter.search(label) != None)
                        self.allowCache[label] = allow
                        self.labels_allowed += 1
                    if allow:
                        self.cauldron.put(label, value)
                        self.count('records_per_second', 1)
                        self.count('bytes_per_second', len(recordBytes))
                    else:
                        self.log(
                            "Label [%s] does not match filter (%s)" %
                            (label, self.outFilterExpr), syslog.LOG_WARNING)
                except ValueError:
                    self.log(
                        "Can't Parse [%s] [%s]" %
                        (str(recordBytes, 'utf-8').rstrip(), ''.join(
                            ['%02x' % (thebyte) for thebyte in recordBytes])))
            else:
                self.count('shuttles_per_second', 1)

    def stop(self):
        self.running = False
Ejemplo n.º 8
0
class Coven(object):
  def __init__( self,path,prefix,cauldronAddr,
      ignore=[r'^\.',r'\.x?swp$',r'~',r'^__',r'__$',r'\.jar$',r'\.db$']):
    syslog.openlog(ident="X3Coven",facility=syslog.LOG_DAEMON)
    self.path=path
    self.prefix=prefix
    self.cauldronAddr=cauldronAddr
    self.plugins={}
    self.confRE=re.compile('^#\\s*x3\.([a-z0-9.]+)\\s*=\\s*(.*)\\s*$')
    self.dirManager=DirManager(self.path,self,ignore=ignore)
    self.cauldron=CauldronSender(self.cauldronAddr)

  def getOSPath(self,name):
    return path.join(self.path,name)

  def isShebangExecutable(self,path):
    try:
      with open(path,'rb') as plugin_file:
        magic_bytes=plugin_file.read(2)
        if magic_bytes==b'#!':
          return True
    except:
      pass
    return False

  def getConfig(self,name):
    pluginPath=self.getOSPath(name)    
    if not self.isShebangExecutable(pluginPath):
      return None
    conf={}
    try:
      with open(pluginPath,'r') as plugin_file:
        for line in plugin_file:
          config_match=self.confRE.match(line)
          if config_match:
            conf[config_match.group(1)]=config_match.group(2)
    except:
      return None
    return conf

  def put(self,label,value):
    self.cauldron.put('%s.coven.%s' % (self.prefix,label),value)
  
  def log(self,message,component=None,priority=syslog.LOG_DEBUG):
    syslog.syslog(priority,message if component==None 
                                else '[%s] %s' % (component,message))
  def practice(self):
    try:
      self.dirManager.start()
      while True:
        totals={}
        self.put('plugins.running',len(self.plugins))
        for plugin in self.plugins.values():
          for (values_type,values) in plugin.get_counters().items():
            try:
              total_values=totals[values_type]
            except KeyError:
              total_values={}
              totals[values_type]=total_values
            for (label,value) in values.items():
              try:
                total_values[label]+=value
              except KeyError:
                total_values[label]=value
              self.put('plugin.%s.%s.%s' %
                                (plugin.name,values_type,label),value)
          plugin.reset_counters()

        for (values_type,values) in totals.items():
          for (label,value) in values.items():
            self.put('plugins.total.%s.%s' % (values_type,label),value)
        time.sleep(1.0)
    except KeyboardInterrupt:
      self.log("Stopping..")
      self.dirManager.stop()
      for partingPlugin in self.plugins.values():
        partingPlugin.stop()
      for partingPlugin in self.plugins.values():
        partingPlugin.join()
      self.log("Stopped..",priority=syslog.LOG_INFO)

# ------------------------------------------------------------------------

  def startPlugin(self,name):
    self.log("Starting %s" % (name)) 
    config=self.getConfig(name)
    if config:
      try:
        self.plugins[name]=Plugin(name, self.getOSPath(name),
                                 self.prefix, config, self.cauldronAddr,
                                 self.log)
        self.plugins[name].start()
        self.log("Started %s" % (name),priority=syslog.LOG_INFO)
      except:
        self.log("Failed To Start %s" % (name),priority=syslog.LOG_ERR)
    else:
      self.log("Not an X3 Plugin: %s" % (name),
               priority=syslog.LOG_WARNING)

  def stopPlugin(self,name):
    self.log("Stopping %s" % (name))
    try:
      self.plugins[name].stop()
      del self.plugins[name]
      self.log("Stopped %s" % (name),priority=syslog.LOG_INFO) 
    except:
      self.log("Failed To Stop %s" % (name),priority=syslog.LOG_ERR) 

# ------------------------------------------------------------------------

  def process_FileCreated(self,name):
    self.startPlugin(name)

  def process_FileDeleted(self,name):
    self.stopPlugin(name)

  def process_FileChanged(self,name):
    self.log("Restarting %s" % (name),priority=syslog.LOG_INFO) 
    self.stopPlugin(name)
    self.startPlugin(name)