class ZenTestSyslogRules(CmdBase):
def __init__(self):
CmdBase.__init__(self)
self.processor = SyslogProcessor(self.sendEvent,
self.options.minpriority, False,
"localhost", 2)
self.keptEvent = False
self.totalSent = 0
def sendEvent(self, evt, *args, **kwargs):
self.keptEvent = True
self.totalSent += 1
def getSyslogMsg(self, line):
# Throw out captured meta-data
return line[ORID_SYSLOG_METADATA_END_POS:-1]
def run(self):
# Apply the regex rules against all captured syslog messages
totalEvents = 0
counts = {}
for line in open(self.options.infile):
line = line.strip()
if not line or line.startswith("#"):
continue
totalEvents += 1
self.keptEvent = False
self.processor.process(line, "127.0.0.1", "localhost", None)
if self.keptEvent:
key = self.getSyslogMsg(line)
counts.setdefault(key, 0)
counts[key] += 1
# Print the output sorted by the number of times each message
# has occurred in the original input
for key, count in sorted(counts.items(), key=lambda x: x[1]):
print "%d %s" % (count, key)
droppedCount = totalEvents - self.totalSent
droppedPct = 0
if totalEvents:
droppedPct = droppedCount * 100.0 / totalEvents
self.log.info(
"Test event stats: dropped=%d sent=%d total=%d dropped_pct=%.1f",
droppedCount, self.totalSent, totalEvents, droppedPct)
def buildOptions(self):
CmdBase.buildOptions(self)
self.parser.add_option('--infile',
dest='infile',
default=defaultInfile,
help="File containing captured syslog events.")
self.parser.add_option(
'--minpriority',
dest='minpriority',
default=6,
type='int',
help='Minimum priority message that zensyslog will accept')
class ZenTestSyslogRules(CmdBase):
def __init__(self):
CmdBase.__init__(self)
self.processor = SyslogProcessor(self.sendEvent,
self.options.minpriority, False,
"localhost", 2)
self.keptEvent = False
self.totalSent = 0
def sendEvent(self, evt, *args, **kwargs):
self.keptEvent = True
self.totalSent += 1
def getSyslogMsg(self, line):
# Throw out captured meta-data
return line[ORID_SYSLOG_METADATA_END_POS:-1]
def run(self):
# Apply the regex rules against all captured syslog messages
totalEvents = 0
counts= {}
for line in open(self.options.infile):
line = line.strip()
if not line or line.startswith("#"):
continue
totalEvents += 1
self.keptEvent = False
self.processor.process(line, "127.0.0.1", "localhost", None)
if self.keptEvent:
key = self.getSyslogMsg(line)
counts.setdefault(key, 0)
counts[key] += 1
# Print the output sorted by the number of times each message
# has occurred in the original input
for key, count in sorted(counts.items(), key=lambda x: x[1]):
print "%d %s" % (count, key)
droppedCount = totalEvents - self.totalSent
droppedPct = 0
if totalEvents:
droppedPct = droppedCount * 100.0 / totalEvents
self.log.info("Test event stats: dropped=%d sent=%d total=%d dropped_pct=%.1f",
droppedCount, self.totalSent, totalEvents, droppedPct)
def buildOptions(self):
CmdBase.buildOptions(self)
self.parser.add_option('--infile',
dest='infile', default=defaultInfile,
help="File containing captured syslog events.")
self.parser.add_option('--minpriority', dest='minpriority',
default=6, type='int',
help='Minimum priority message that zensyslog will accept')
def testProcess(self):
long_text_message = "long text message " * 20
msg = "2016-08-08T11:07:33.660820-04:00 devname=localhost log_id=98765434 type=component {}".format(long_text_message)
ipaddr = "127.0.0.1"
host = "8080"
rtime = "1416111"
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
s.process(msg, ipaddr, host, rtime)
evt = self.sent
self.assertEquals(evt.get('device'), host)
self.assertEquals(evt.get('ipAddress'), ipaddr)
self.assertEquals(evt.get('firstTime'), rtime)
self.assertEquals(evt.get('lastTime'), rtime)
self.assertEquals(evt.get('eventGroup'), 'syslog')
self.assertEquals(evt.get('message'), unicode(msg))
self.assertEquals(evt.get('summary'), unicode(msg))
class SyslogTask(BaseTask, DatagramProtocol):
"""
Listen for syslog messages and turn them into events
Connects to the TrapService service in zenhub.
"""
zope.interface.implements(IScheduledTask)
SYSLOG_DATE_FORMAT = '%b %d %H:%M:%S'
SAMPLE_DATE = 'Apr 10 15:19:22'
def __init__(self,
taskName,
configId,
scheduleIntervalSeconds=3600,
taskConfig=None):
BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds,
taskConfig)
self.log = log
# Needed for interface
self.name = taskName
self.configId = configId
self.state = TaskStates.STATE_IDLE
self.interval = scheduleIntervalSeconds
self._preferences = taskConfig
self._daemon = zope.component.getUtility(ICollector)
self._eventService = zope.component.queryUtility(IEventService)
self._statService = zope.component.queryUtility(IStatisticsService)
self._preferences = self._daemon
self.options = self._daemon.options
self.stats = Stats()
if not self.options.useFileDescriptor\
and self.options.syslogport < 1024:
self._daemon.openPrivilegedPort(
'--listen', '--proto=udp', '--port=%s:%d' %
(self.options.listenip, self.options.syslogport))
self._daemon.changeUser()
self.minpriority = self.options.minpriority
self.processor = None
if self.options.logorig:
self.olog = logging.getLogger('origsyslog')
self.olog.setLevel(20)
self.olog.propagate = False
lname = zenPath('log/origsyslog.log')
hdlr = logging.FileHandler(lname)
hdlr.setFormatter(logging.Formatter('%(message)s'))
self.olog.addHandler(hdlr)
if self.options.useFileDescriptor is not None:
self.useUdpFileDescriptor(int(self.options.useFileDescriptor))
else:
reactor.listenUDP(self.options.syslogport,
self,
interface=self.options.listenip)
# yield self.model().callRemote('getDefaultPriority')
self.processor = SyslogProcessor(self._eventService.sendEvent,
self.options.minpriority,
self.options.parsehost,
self.options.monitor,
self._daemon.defaultPriority)
def doTask(self):
"""
This is a wait-around task since we really are called
asynchronously.
"""
return defer.succeed("Waiting for syslog messages...")
def useUdpFileDescriptor(self, fd):
s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM)
os.close(fd)
port = s.getsockname()[1]
transport = udp.Port(port, self)
s.setblocking(0)
transport.socket = s
transport.fileno = s.fileno
transport.connected = 1
transport._realPortNumber = port
self.transport = transport
# hack around startListening not being called
self.numPorts = 1
transport.startReading()
def expand(self, msg, client_address):
"""
Expands a syslog message into a string format suitable for writing
to the filesystem such that it appears the same as it would
had the message been logged by the syslog daemon.
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
@return: message
@rtype: string
"""
# pri := facility * severity
stop = msg.find('>')
# check for a datestamp. default to right now if date not present
start = stop + 1
stop = start + len(SyslogTask.SAMPLE_DATE)
dateField = msg[start:stop]
try:
date = time.strptime(dateField, SyslogTask.SYSLOG_DATE_FORMAT)
year = time.localtime()[0]
date = (year, ) + date[1:]
start = stop + 1
except ValueError:
# date not present, so use today's date
date = time.localtime()
# check for a hostname. default to localhost if not present
stop = msg.find(' ', start)
if msg[stop - 1] == ':':
hostname = client_address[0]
else:
hostname = msg[start:stop]
start = stop + 1
# the message content
body = msg[start:]
# assemble the message
prettyTime = time.strftime(SyslogTask.SYSLOG_DATE_FORMAT, date)
message = '%s %s %s' % (prettyTime, hostname, body)
return message
def datagramReceived(self, msg, client_address):
"""
Consume the network packet
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
"""
if msg == "":
self.log.debug("Received empty datagram. Discarding.")
return
(ipaddr, port) = client_address
if self.options.logorig:
if self.options.logformat == 'human':
message = self.expand(msg, client_address)
else:
message = msg
self.olog.info(message)
if self.options.noreverseLookup:
d = defer.succeed(ipaddr)
else:
d = asyncNameLookup(ipaddr)
d.addBoth(self.gotHostname, (msg, ipaddr, time.time()))
def gotHostname(self, response, data):
"""
Send the resolved address, if possible, and the event via the thread
@param response: Twisted response
@type response: Twisted response
@param data: (msg, ipaddr, rtime)
@type data: tuple of (string, string, datetime object)
"""
(msg, ipaddr, rtime) = data
if isinstance(response, failure.Failure):
host = ipaddr
else:
host = response
if self.processor:
self.processor.process(msg, ipaddr, host, rtime)
totalTime, totalEvents, maxTime = self.stats.report()
stat = self._statService.getStatistic("events")
stat.value = totalEvents
def displayStatistics(self):
totalTime, totalEvents, maxTime = self.stats.report()
display = "%d events processed in %.2f seconds" % (totalEvents,
totalTime)
if totalEvents > 0:
display += """
%.5f average seconds per event
Maximum processing time for one event was %.5f""" % (
(totalTime / totalEvents), maxTime)
return display
def cleanup(self):
status = self.displayStatistics()
self.log.info(status)
class SyslogTask(BaseTask, DatagramProtocol):
"""
Listen for syslog messages and turn them into events
Connects to the TrapService service in zenhub.
"""
zope.interface.implements(IScheduledTask)
SYSLOG_DATE_FORMAT = '%b %d %H:%M:%S'
SAMPLE_DATE = 'Apr 10 15:19:22'
def __init__(self, taskName, configId,
scheduleIntervalSeconds=3600, taskConfig=None):
BaseTask.__init__(self, taskName, configId,
scheduleIntervalSeconds, taskConfig)
self.log = log
# Needed for interface
self.name = taskName
self.configId = configId
self.state = TaskStates.STATE_IDLE
self.interval = scheduleIntervalSeconds
self._preferences = taskConfig
self._daemon = zope.component.getUtility(ICollector)
self._eventService = zope.component.queryUtility(IEventService)
self._statService = zope.component.queryUtility(IStatisticsService)
self._preferences = self._daemon
self.options = self._daemon.options
self.stats = Stats()
if not self.options.useFileDescriptor\
and self.options.syslogport < 1024:
self._daemon.openPrivilegedPort('--listen', '--proto=udp',
'--port=%s:%d'
% (self.options.listenip,
self.options.syslogport))
self._daemon.changeUser()
self.minpriority = self.options.minpriority
self.processor = None
if self.options.logorig:
self.olog = logging.getLogger('origsyslog')
self.olog.setLevel(20)
self.olog.propagate = False
lname = zenPath('log/origsyslog.log')
hdlr = logging.FileHandler(lname)
hdlr.setFormatter(logging.Formatter('%(message)s'))
self.olog.addHandler(hdlr)
if self.options.useFileDescriptor is not None:
self.useUdpFileDescriptor(int(self.options.useFileDescriptor))
else:
reactor.listenUDP(self.options.syslogport, self,
interface=self.options.listenip)
# yield self.model().callRemote('getDefaultPriority')
self.processor = SyslogProcessor(self._eventService.sendEvent,
self.options.minpriority, self.options.parsehost,
self.options.monitor, self._daemon.defaultPriority)
def doTask(self):
"""
This is a wait-around task since we really are called
asynchronously.
"""
return defer.succeed("Waiting for syslog messages...")
def useUdpFileDescriptor(self, fd):
s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM)
os.close(fd)
port = s.getsockname()[1]
transport = udp.Port(port, self)
s.setblocking(0)
transport.socket = s
transport.fileno = s.fileno
transport.connected = 1
transport._realPortNumber = port
self.transport = transport
# hack around startListening not being called
self.numPorts = 1
transport.startReading()
def expand(self, msg, client_address):
"""
Expands a syslog message into a string format suitable for writing
to the filesystem such that it appears the same as it would
had the message been logged by the syslog daemon.
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
@return: message
@rtype: string
"""
# pri := facility * severity
stop = msg.find('>')
# check for a datestamp. default to right now if date not present
start = stop + 1
stop = start + len(SyslogTask.SAMPLE_DATE)
dateField = msg[start:stop]
try:
date = time.strptime(dateField,
SyslogTask.SYSLOG_DATE_FORMAT)
year = time.localtime()[0]
date = (year, ) + date[1:]
start = stop + 1
except ValueError:
# date not present, so use today's date
date = time.localtime()
# check for a hostname. default to localhost if not present
stop = msg.find(' ', start)
if msg[stop - 1] == ':':
hostname = client_address[0]
else:
hostname = msg[start:stop]
start = stop + 1
# the message content
body = msg[start:]
# assemble the message
prettyTime = time.strftime(SyslogTask.SYSLOG_DATE_FORMAT, date)
message = '%s %s %s' % (prettyTime, hostname, body)
return message
def datagramReceived(self, msg, client_address):
"""
Consume the network packet
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
"""
(ipaddr, port) = client_address
if self.options.logorig:
if self.options.logformat == 'human':
message = self.expand(msg, client_address)
else:
message = msg
self.olog.info(message)
if self.options.noreverseLookup:
d = defer.succeed(ipaddr)
else:
d = asyncNameLookup(ipaddr)
d.addBoth(self.gotHostname, (msg, ipaddr, time.time()))
def gotHostname(self, response, data):
"""
Send the resolved address, if possible, and the event via the thread
@param response: Twisted response
@type response: Twisted response
@param data: (msg, ipaddr, rtime)
@type data: tuple of (string, string, datetime object)
"""
(msg, ipaddr, rtime) = data
if isinstance(response, failure.Failure):
host = ipaddr
else:
host = response
if self.processor:
self.processor.process(msg, ipaddr, host, rtime)
totalTime, totalEvents, maxTime = self.stats.report()
stat = self._statService.getStatistic("events")
stat.value = totalEvents
def displayStatistics(self):
totalTime, totalEvents, maxTime = self.stats.report()
display = "%d events processed in %.2f seconds" % (
totalEvents,
totalTime)
if totalEvents > 0:
display += """
%.5f average seconds per event
Maximum processing time for one event was %.5f""" % (
(totalTime / totalEvents), maxTime)
return display
def cleanup(self):
status = self.displayStatistics()
self.log.info(status)
