class ZenTestSyslogRules(CmdBase): def __init__(self): CmdBase.__init__(self) self.processor = SyslogProcessor(self.sendEvent, self.options.minpriority, False, "localhost", 2) self.keptEvent = False self.totalSent = 0 def sendEvent(self, evt, *args, **kwargs): self.keptEvent = True self.totalSent += 1 def getSyslogMsg(self, line): # Throw out captured meta-data return line[ORID_SYSLOG_METADATA_END_POS:-1] def run(self): # Apply the regex rules against all captured syslog messages totalEvents = 0 counts = {} for line in open(self.options.infile): line = line.strip() if not line or line.startswith("#"): continue totalEvents += 1 self.keptEvent = False self.processor.process(line, "127.0.0.1", "localhost", None) if self.keptEvent: key = self.getSyslogMsg(line) counts.setdefault(key, 0) counts[key] += 1 # Print the output sorted by the number of times each message # has occurred in the original input for key, count in sorted(counts.items(), key=lambda x: x[1]): print "%d %s" % (count, key) droppedCount = totalEvents - self.totalSent droppedPct = 0 if totalEvents: droppedPct = droppedCount * 100.0 / totalEvents self.log.info( "Test event stats: dropped=%d sent=%d total=%d dropped_pct=%.1f", droppedCount, self.totalSent, totalEvents, droppedPct) def buildOptions(self): CmdBase.buildOptions(self) self.parser.add_option('--infile', dest='infile', default=defaultInfile, help="File containing captured syslog events.") self.parser.add_option( '--minpriority', dest='minpriority', default=6, type='int', help='Minimum priority message that zensyslog will accept')
class ZenTestSyslogRules(CmdBase): def __init__(self): CmdBase.__init__(self) self.processor = SyslogProcessor(self.sendEvent, self.options.minpriority, False, "localhost", 2) self.keptEvent = False self.totalSent = 0 def sendEvent(self, evt, *args, **kwargs): self.keptEvent = True self.totalSent += 1 def getSyslogMsg(self, line): # Throw out captured meta-data return line[ORID_SYSLOG_METADATA_END_POS:-1] def run(self): # Apply the regex rules against all captured syslog messages totalEvents = 0 counts= {} for line in open(self.options.infile): line = line.strip() if not line or line.startswith("#"): continue totalEvents += 1 self.keptEvent = False self.processor.process(line, "127.0.0.1", "localhost", None) if self.keptEvent: key = self.getSyslogMsg(line) counts.setdefault(key, 0) counts[key] += 1 # Print the output sorted by the number of times each message # has occurred in the original input for key, count in sorted(counts.items(), key=lambda x: x[1]): print "%d %s" % (count, key) droppedCount = totalEvents - self.totalSent droppedPct = 0 if totalEvents: droppedPct = droppedCount * 100.0 / totalEvents self.log.info("Test event stats: dropped=%d sent=%d total=%d dropped_pct=%.1f", droppedCount, self.totalSent, totalEvents, droppedPct) def buildOptions(self): CmdBase.buildOptions(self) self.parser.add_option('--infile', dest='infile', default=defaultInfile, help="File containing captured syslog events.") self.parser.add_option('--minpriority', dest='minpriority', default=6, type='int', help='Minimum priority message that zensyslog will accept')
def testProcess(self): long_text_message = "long text message " * 20 msg = "2016-08-08T11:07:33.660820-04:00 devname=localhost log_id=98765434 type=component {}".format(long_text_message) ipaddr = "127.0.0.1" host = "8080" rtime = "1416111" s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3) s.process(msg, ipaddr, host, rtime) evt = self.sent self.assertEquals(evt.get('device'), host) self.assertEquals(evt.get('ipAddress'), ipaddr) self.assertEquals(evt.get('firstTime'), rtime) self.assertEquals(evt.get('lastTime'), rtime) self.assertEquals(evt.get('eventGroup'), 'syslog') self.assertEquals(evt.get('message'), unicode(msg)) self.assertEquals(evt.get('summary'), unicode(msg))
class SyslogTask(BaseTask, DatagramProtocol): """ Listen for syslog messages and turn them into events Connects to the TrapService service in zenhub. """ zope.interface.implements(IScheduledTask) SYSLOG_DATE_FORMAT = '%b %d %H:%M:%S' SAMPLE_DATE = 'Apr 10 15:19:22' def __init__(self, taskName, configId, scheduleIntervalSeconds=3600, taskConfig=None): BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds, taskConfig) self.log = log # Needed for interface self.name = taskName self.configId = configId self.state = TaskStates.STATE_IDLE self.interval = scheduleIntervalSeconds self._preferences = taskConfig self._daemon = zope.component.getUtility(ICollector) self._eventService = zope.component.queryUtility(IEventService) self._statService = zope.component.queryUtility(IStatisticsService) self._preferences = self._daemon self.options = self._daemon.options self.stats = Stats() if not self.options.useFileDescriptor\ and self.options.syslogport < 1024: self._daemon.openPrivilegedPort( '--listen', '--proto=udp', '--port=%s:%d' % (self.options.listenip, self.options.syslogport)) self._daemon.changeUser() self.minpriority = self.options.minpriority self.processor = None if self.options.logorig: self.olog = logging.getLogger('origsyslog') self.olog.setLevel(20) self.olog.propagate = False lname = zenPath('log/origsyslog.log') hdlr = logging.FileHandler(lname) hdlr.setFormatter(logging.Formatter('%(message)s')) self.olog.addHandler(hdlr) if self.options.useFileDescriptor is not None: self.useUdpFileDescriptor(int(self.options.useFileDescriptor)) else: reactor.listenUDP(self.options.syslogport, self, interface=self.options.listenip) # yield self.model().callRemote('getDefaultPriority') self.processor = SyslogProcessor(self._eventService.sendEvent, self.options.minpriority, self.options.parsehost, self.options.monitor, self._daemon.defaultPriority) def doTask(self): """ This is a wait-around task since we really are called asynchronously. """ return defer.succeed("Waiting for syslog messages...") def useUdpFileDescriptor(self, fd): s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM) os.close(fd) port = s.getsockname()[1] transport = udp.Port(port, self) s.setblocking(0) transport.socket = s transport.fileno = s.fileno transport.connected = 1 transport._realPortNumber = port self.transport = transport # hack around startListening not being called self.numPorts = 1 transport.startReading() def expand(self, msg, client_address): """ Expands a syslog message into a string format suitable for writing to the filesystem such that it appears the same as it would had the message been logged by the syslog daemon. @param msg: syslog message @type msg: string @param client_address: IP info of the remote device (ipaddr, port) @type client_address: tuple of (string, number) @return: message @rtype: string """ # pri := facility * severity stop = msg.find('>') # check for a datestamp. default to right now if date not present start = stop + 1 stop = start + len(SyslogTask.SAMPLE_DATE) dateField = msg[start:stop] try: date = time.strptime(dateField, SyslogTask.SYSLOG_DATE_FORMAT) year = time.localtime()[0] date = (year, ) + date[1:] start = stop + 1 except ValueError: # date not present, so use today's date date = time.localtime() # check for a hostname. default to localhost if not present stop = msg.find(' ', start) if msg[stop - 1] == ':': hostname = client_address[0] else: hostname = msg[start:stop] start = stop + 1 # the message content body = msg[start:] # assemble the message prettyTime = time.strftime(SyslogTask.SYSLOG_DATE_FORMAT, date) message = '%s %s %s' % (prettyTime, hostname, body) return message def datagramReceived(self, msg, client_address): """ Consume the network packet @param msg: syslog message @type msg: string @param client_address: IP info of the remote device (ipaddr, port) @type client_address: tuple of (string, number) """ if msg == "": self.log.debug("Received empty datagram. Discarding.") return (ipaddr, port) = client_address if self.options.logorig: if self.options.logformat == 'human': message = self.expand(msg, client_address) else: message = msg self.olog.info(message) if self.options.noreverseLookup: d = defer.succeed(ipaddr) else: d = asyncNameLookup(ipaddr) d.addBoth(self.gotHostname, (msg, ipaddr, time.time())) def gotHostname(self, response, data): """ Send the resolved address, if possible, and the event via the thread @param response: Twisted response @type response: Twisted response @param data: (msg, ipaddr, rtime) @type data: tuple of (string, string, datetime object) """ (msg, ipaddr, rtime) = data if isinstance(response, failure.Failure): host = ipaddr else: host = response if self.processor: self.processor.process(msg, ipaddr, host, rtime) totalTime, totalEvents, maxTime = self.stats.report() stat = self._statService.getStatistic("events") stat.value = totalEvents def displayStatistics(self): totalTime, totalEvents, maxTime = self.stats.report() display = "%d events processed in %.2f seconds" % (totalEvents, totalTime) if totalEvents > 0: display += """ %.5f average seconds per event Maximum processing time for one event was %.5f""" % ( (totalTime / totalEvents), maxTime) return display def cleanup(self): status = self.displayStatistics() self.log.info(status)
class SyslogTask(BaseTask, DatagramProtocol): """ Listen for syslog messages and turn them into events Connects to the TrapService service in zenhub. """ zope.interface.implements(IScheduledTask) SYSLOG_DATE_FORMAT = '%b %d %H:%M:%S' SAMPLE_DATE = 'Apr 10 15:19:22' def __init__(self, taskName, configId, scheduleIntervalSeconds=3600, taskConfig=None): BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds, taskConfig) self.log = log # Needed for interface self.name = taskName self.configId = configId self.state = TaskStates.STATE_IDLE self.interval = scheduleIntervalSeconds self._preferences = taskConfig self._daemon = zope.component.getUtility(ICollector) self._eventService = zope.component.queryUtility(IEventService) self._statService = zope.component.queryUtility(IStatisticsService) self._preferences = self._daemon self.options = self._daemon.options self.stats = Stats() if not self.options.useFileDescriptor\ and self.options.syslogport < 1024: self._daemon.openPrivilegedPort('--listen', '--proto=udp', '--port=%s:%d' % (self.options.listenip, self.options.syslogport)) self._daemon.changeUser() self.minpriority = self.options.minpriority self.processor = None if self.options.logorig: self.olog = logging.getLogger('origsyslog') self.olog.setLevel(20) self.olog.propagate = False lname = zenPath('log/origsyslog.log') hdlr = logging.FileHandler(lname) hdlr.setFormatter(logging.Formatter('%(message)s')) self.olog.addHandler(hdlr) if self.options.useFileDescriptor is not None: self.useUdpFileDescriptor(int(self.options.useFileDescriptor)) else: reactor.listenUDP(self.options.syslogport, self, interface=self.options.listenip) # yield self.model().callRemote('getDefaultPriority') self.processor = SyslogProcessor(self._eventService.sendEvent, self.options.minpriority, self.options.parsehost, self.options.monitor, self._daemon.defaultPriority) def doTask(self): """ This is a wait-around task since we really are called asynchronously. """ return defer.succeed("Waiting for syslog messages...") def useUdpFileDescriptor(self, fd): s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM) os.close(fd) port = s.getsockname()[1] transport = udp.Port(port, self) s.setblocking(0) transport.socket = s transport.fileno = s.fileno transport.connected = 1 transport._realPortNumber = port self.transport = transport # hack around startListening not being called self.numPorts = 1 transport.startReading() def expand(self, msg, client_address): """ Expands a syslog message into a string format suitable for writing to the filesystem such that it appears the same as it would had the message been logged by the syslog daemon. @param msg: syslog message @type msg: string @param client_address: IP info of the remote device (ipaddr, port) @type client_address: tuple of (string, number) @return: message @rtype: string """ # pri := facility * severity stop = msg.find('>') # check for a datestamp. default to right now if date not present start = stop + 1 stop = start + len(SyslogTask.SAMPLE_DATE) dateField = msg[start:stop] try: date = time.strptime(dateField, SyslogTask.SYSLOG_DATE_FORMAT) year = time.localtime()[0] date = (year, ) + date[1:] start = stop + 1 except ValueError: # date not present, so use today's date date = time.localtime() # check for a hostname. default to localhost if not present stop = msg.find(' ', start) if msg[stop - 1] == ':': hostname = client_address[0] else: hostname = msg[start:stop] start = stop + 1 # the message content body = msg[start:] # assemble the message prettyTime = time.strftime(SyslogTask.SYSLOG_DATE_FORMAT, date) message = '%s %s %s' % (prettyTime, hostname, body) return message def datagramReceived(self, msg, client_address): """ Consume the network packet @param msg: syslog message @type msg: string @param client_address: IP info of the remote device (ipaddr, port) @type client_address: tuple of (string, number) """ (ipaddr, port) = client_address if self.options.logorig: if self.options.logformat == 'human': message = self.expand(msg, client_address) else: message = msg self.olog.info(message) if self.options.noreverseLookup: d = defer.succeed(ipaddr) else: d = asyncNameLookup(ipaddr) d.addBoth(self.gotHostname, (msg, ipaddr, time.time())) def gotHostname(self, response, data): """ Send the resolved address, if possible, and the event via the thread @param response: Twisted response @type response: Twisted response @param data: (msg, ipaddr, rtime) @type data: tuple of (string, string, datetime object) """ (msg, ipaddr, rtime) = data if isinstance(response, failure.Failure): host = ipaddr else: host = response if self.processor: self.processor.process(msg, ipaddr, host, rtime) totalTime, totalEvents, maxTime = self.stats.report() stat = self._statService.getStatistic("events") stat.value = totalEvents def displayStatistics(self): totalTime, totalEvents, maxTime = self.stats.report() display = "%d events processed in %.2f seconds" % ( totalEvents, totalTime) if totalEvents > 0: display += """ %.5f average seconds per event Maximum processing time for one event was %.5f""" % ( (totalTime / totalEvents), maxTime) return display def cleanup(self): status = self.displayStatistics() self.log.info(status)