Exemplo n.º 1
0
class LogConsumerThread(threading.Thread):
    def __init__(self, host, *args, **kwargs):
        super(LogConsumerThread, self).__init__(*args, **kwargs)
        self.l = logging.getLogger('collector')
        self._host = host

    def callback(self, channel, method, properties, body):
        if not s_queue.full():
            #print "adding to queue"
            s_queue.put(body)
    
    def run(self):
        
        self.rabbitcom = Comm('10.10.0.134','logs','logs','/','logs','logs','collector')
        self.connect = self.rabbitcom.ampq_connect()
        #credentials = pika.PlainCredentials("logs", "logs")

        #connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._host,credentials=credentials))

        #channel = connection.channel()

        #channel.queue_bind(exchange='logs',queue='logs')

        self.rabbitcom.channel.basic_consume(self.callback,
                              "logs",
                              no_ack=True)
        
        self.rabbitcom.channel.start_consuming()
Exemplo n.º 2
0
class LogConsumerThread(threading.Thread):
    def __init__(self, host, *args, **kwargs):
        super(LogConsumerThread, self).__init__(*args, **kwargs)
        self.l = logging.getLogger("collector")
        self._host = host

    def callback(self, channel, method, properties, body):
        if not s_queue.full():
            # print "adding to queue"
            s_queue.put(body)

    def run(self):

        self.rabbitcom = Comm("10.10.0.134", "logs", "logs", "/", "logs", "logs", "collector")
        self.connect = self.rabbitcom.ampq_connect()
        # credentials = pika.PlainCredentials("logs", "logs")

        # connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._host,credentials=credentials))

        # channel = connection.channel()

        # channel.queue_bind(exchange='logs',queue='logs')

        self.rabbitcom.channel.basic_consume(self.callback, "logs", no_ack=True)

        self.rabbitcom.channel.start_consuming()
Exemplo n.º 3
0
    def run(self):
        
        self.rabbitcom = Comm('10.10.0.134','logs','logs','/','logs','logs','collector')
        self.connect = self.rabbitcom.ampq_connect()
        #credentials = pika.PlainCredentials("logs", "logs")

        #connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._host,credentials=credentials))

        #channel = connection.channel()

        #channel.queue_bind(exchange='logs',queue='logs')

        self.rabbitcom.channel.basic_consume(self.callback,
                              "logs",
                              no_ack=True)
        
        self.rabbitcom.channel.start_consuming()
Exemplo n.º 4
0
    def initialize(self):

        self.rabbitcom = Comm('10.10.0.134', 'logs', 'logs', '/', 'jobs', '',
                              'jobmon')

        # Server main loop
        while True:
            self.getJobs()
            time.sleep(4)
Exemplo n.º 5
0
    def initialize(self):

        self.rabbitcom = Comm('10.10.0.134', 'logs', 'logs', '/', 'logs', '',
                              'sysmon')
        self.connect = self.rabbitcom.ampq_connect()

        # Pre-Compile our regex for the minimal performance gain
        self.dev_re = re.compile(
            '([A-Za-z0-9\.]+):\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)'
        )
        self.meminfo_re = re.compile('([\w]+):\s(\d+)\s')
        self.proc_stats_re = re.compile('([A-Za-z0-9]+)\s+(\d+)')
        self.diskstatus_re = re.compile(
            '\s+(\d+)\s+(\d+)\s(\w+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)'
        )

        # Server main loop
        while True:
            self.getProcesses()
            time.sleep(1)
Exemplo n.º 6
0
    def initialize(self):
        
        self.rabbitcom = Comm('10.10.0.134','logs','logs','/','logs','','sysmon')
        self.connect = self.rabbitcom.ampq_connect()
            
        # Pre-Compile our regex for the minimal performance gain
        self.dev_re = re.compile('([A-Za-z0-9\.]+):\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)')
        self.meminfo_re = re.compile('([\w]+):\s(\d+)\s')
        self.proc_stats_re = re.compile('([A-Za-z0-9]+)\s+(\d+)')
        self.diskstatus_re = re.compile('\s+(\d+)\s+(\d+)\s(\w+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)')
 
        # Server main loop
        while True:
            self.getProcesses()
            time.sleep(1)        
Exemplo n.º 7
0
    def run(self):

        self.rabbitcom = Comm("10.10.0.134", "logs", "logs", "/", "logs", "logs", "collector")
        self.connect = self.rabbitcom.ampq_connect()
        # credentials = pika.PlainCredentials("logs", "logs")

        # connection = pika.BlockingConnection(pika.ConnectionParameters(host=self._host,credentials=credentials))

        # channel = connection.channel()

        # channel.queue_bind(exchange='logs',queue='logs')

        self.rabbitcom.channel.basic_consume(self.callback, "logs", no_ack=True)

        self.rabbitcom.channel.start_consuming()
Exemplo n.º 8
0
class SysmonDaemon(Daemon):

    def run(self):
        
        self.l = logging.getLogger('sysmon')
        ch = logging.StreamHandler()
        ch.setLevel(logging.ERROR)
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.l.addHandler(ch)
        
        self.initialize()
         
    def initialize(self):
        
        self.rabbitcom = Comm('10.10.0.134','logs','logs','/','logs','','sysmon')
        self.connect = self.rabbitcom.ampq_connect()
            
        # Pre-Compile our regex for the minimal performance gain
        self.dev_re = re.compile('([A-Za-z0-9\.]+):\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)')
        self.meminfo_re = re.compile('([\w]+):\s(\d+)\s')
        self.proc_stats_re = re.compile('([A-Za-z0-9]+)\s+(\d+)')
        self.diskstatus_re = re.compile('\s+(\d+)\s+(\d+)\s(\w+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)')
 
        # Server main loop
        while True:
            self.getProcesses()
            time.sleep(1)        
 
    def S(self,key ):
        return key.replace('.','_')
    
    
    def getUUID(self):
        return re.sub('_|-|=','0',base64.urlsafe_b64encode(uuid.uuid4().bytes))
        
    # Get all the processes that are not system
    # processes
    def getProcesses(self, ):
        """ Get all system processes """
        # walk the /proc
        data = {}
        stats = {}
        
        data['uuid'] = self.getUUID()
        d = datetime.datetime.utcnow()
        data['hostname'] = socket.gethostname()
        data['timestamp_year'] = d.strftime('%y')
        data['timestamp_month'] = d.strftime('%m')
        data['timestamp_day'] = d.strftime('%d')
        data['timestamp_hour'] = d.strftime('%H')
        data['timestamp_minute'] = d.strftime('%M')
        data['timestamp_second'] = d.strftime('%S')
        
        for item in os.listdir('/proc'):
            if re.match('[\d]+',item):
                # Start collecting statistics
                # We only want statistics for processes that
                # are not system processes, i.e. user jobs
                stat_info = None
                try:
                    stat_info = os.stat('/proc/' + item + '/status')
                except Exception, err:
                    self.l.error(err)
                
                if stat_info is not None:   
                    # It is assumed that the ownership of the pid directory
                    # represents the owner of the process and as such we can
                    # use this to determine if this is a "non-system" or "non-root"
                    # process
                    if stat_info.st_uid >= 500:
                        self.getProcStatistics(item,stats)

        
        # Get memory information           
        self.getMemInfo(stats)
        
        # Get network statistics
        self.getNetworkStats(stats)
        
        # Get processor statistics
        self.getProcessorStats(stats)
        
        # Get Disk statistics
        self.getDiskStats(stats)
        
        # Get Infiniband statistics
        self.getInfinibandStats(stats)
        
        data['data'] = stats
        
        self.l.info("Sending data")
        self.rabbitcom.send_message(data)
Exemplo n.º 9
0
class SysmonDaemon(Daemon):
    def run(self):

        self.l = logging.getLogger('sysmon')
        ch = logging.StreamHandler()
        ch.setLevel(logging.ERROR)
        formatter = logging.Formatter(
            '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        ch.setFormatter(formatter)
        self.l.addHandler(ch)

        self.initialize()

    def initialize(self):

        self.rabbitcom = Comm('10.10.0.134', 'logs', 'logs', '/', 'logs', '',
                              'sysmon')
        self.connect = self.rabbitcom.ampq_connect()

        # Pre-Compile our regex for the minimal performance gain
        self.dev_re = re.compile(
            '([A-Za-z0-9\.]+):\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)'
        )
        self.meminfo_re = re.compile('([\w]+):\s(\d+)\s')
        self.proc_stats_re = re.compile('([A-Za-z0-9]+)\s+(\d+)')
        self.diskstatus_re = re.compile(
            '\s+(\d+)\s+(\d+)\s(\w+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)'
        )

        # Server main loop
        while True:
            self.getProcesses()
            time.sleep(1)

    def S(self, key):
        return key.replace('.', '_')

    def getUUID(self):
        return re.sub('_|-|=', '0',
                      base64.urlsafe_b64encode(uuid.uuid4().bytes))

    # Get all the processes that are not system
    # processes
    def getProcesses(self, ):
        """ Get all system processes """
        # walk the /proc
        data = {}
        stats = {}

        data['uuid'] = self.getUUID()
        d = datetime.datetime.utcnow()
        data['hostname'] = socket.gethostname()
        data['timestamp_year'] = d.strftime('%y')
        data['timestamp_month'] = d.strftime('%m')
        data['timestamp_day'] = d.strftime('%d')
        data['timestamp_hour'] = d.strftime('%H')
        data['timestamp_minute'] = d.strftime('%M')
        data['timestamp_second'] = d.strftime('%S')

        for item in os.listdir('/proc'):
            if re.match('[\d]+', item):
                # Start collecting statistics
                # We only want statistics for processes that
                # are not system processes, i.e. user jobs
                stat_info = None
                try:
                    stat_info = os.stat('/proc/' + item + '/status')
                except Exception, err:
                    self.l.error(err)

                if stat_info is not None:
                    # It is assumed that the ownership of the pid directory
                    # represents the owner of the process and as such we can
                    # use this to determine if this is a "non-system" or "non-root"
                    # process
                    if stat_info.st_uid >= 500:
                        self.getProcStatistics(item, stats)

        # Get memory information
        self.getMemInfo(stats)

        # Get network statistics
        self.getNetworkStats(stats)

        # Get processor statistics
        self.getProcessorStats(stats)

        # Get Disk statistics
        self.getDiskStats(stats)

        # Get Infiniband statistics
        self.getInfinibandStats(stats)

        data['data'] = stats

        self.l.info("Sending data")
        self.rabbitcom.send_message(data)