def __init__(self):
CmdBase.__init__(self)
self.processor = SyslogProcessor(self.sendEvent,
self.options.minpriority, False,
"localhost", 2)
self.keptEvent = False
self.totalSent = 0
class ZenTestSyslogRules(CmdBase):
def __init__(self):
CmdBase.__init__(self)
self.processor = SyslogProcessor(self.sendEvent,
self.options.minpriority, False,
"localhost", 2)
self.keptEvent = False
self.totalSent = 0
def sendEvent(self, evt, *args, **kwargs):
self.keptEvent = True
self.totalSent += 1
def getSyslogMsg(self, line):
# Throw out captured meta-data
return line[ORID_SYSLOG_METADATA_END_POS:-1]
def run(self):
# Apply the regex rules against all captured syslog messages
totalEvents = 0
counts = {}
for line in open(self.options.infile):
line = line.strip()
if not line or line.startswith("#"):
continue
totalEvents += 1
self.keptEvent = False
self.processor.process(line, "127.0.0.1", "localhost", None)
if self.keptEvent:
key = self.getSyslogMsg(line)
counts.setdefault(key, 0)
counts[key] += 1
# Print the output sorted by the number of times each message
# has occurred in the original input
for key, count in sorted(counts.items(), key=lambda x: x[1]):
print "%d %s" % (count, key)
droppedCount = totalEvents - self.totalSent
droppedPct = 0
if totalEvents:
droppedPct = droppedCount * 100.0 / totalEvents
self.log.info(
"Test event stats: dropped=%d sent=%d total=%d dropped_pct=%.1f",
droppedCount, self.totalSent, totalEvents, droppedPct)
def buildOptions(self):
CmdBase.buildOptions(self)
self.parser.add_option('--infile',
dest='infile',
default=defaultInfile,
help="File containing captured syslog events.")
self.parser.add_option(
'--minpriority',
dest='minpriority',
default=6,
type='int',
help='Minimum priority message that zensyslog will accept')
def testNetAppSyslogParser(self):
"""
Test NetApp syslog parser.
"""
msg = '[deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])'
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag({}, msg)
self.assertEquals(evt.get('component'), '10/100/1000/e1a')
self.assertEquals(evt.get('summary'), 'Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])')
def testNetAppSyslogParser(self):
"""
Test NetApp syslog parser.
"""
msg = '[deviceName: 10/100/1000/e1a:warning]: Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])'
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag({}, msg)
self.assertEquals(evt.get('component'), '10/100/1000/e1a')
self.assertEquals(evt.get('summary'), 'Client 10.0.0.101 (xid 4251521131) is trying to access an unexported mount (fileid 64, snapid 0, generation 6111516 and flags 0x0 on volume 0xc97d89a [No volume name available])')
def testCiscoStandardMessageSeverity(self):
"""
Test that the event severity is correctly extracted from the
Cisco standard message body
"""
msg = '2014 Jan 31 19:45:51 R2-N6K1-2010-P1 %ETH_PORT_CHANNEL-5-CREATED: port-channel1 created'
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('overwriteSeverity'), '5' )
def testCiscoStandardMessageSeverity(self):
"""
Test that the event severity is correctly extracted from the
Cisco standard message body
"""
msg = '2014 Jan 31 19:45:51 R2-N6K1-2010-P1 %ETH_PORT_CHANNEL-5-CREATED: port-channel1 created'
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('overwriteSeverity'), '5' )
def testCiscoVpnConcentrator(self):
"""
Test of Cisco VPN Concentrator syslog message parsing
"""
msg = "54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept."
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), 'HTTP/42' )
self.assertEquals( evt.get('summary'), 'Error on socket accept.' )
def testCiscoVpnConcentrator(self):
"""
Test of Cisco VPN Concentrator syslog message parsing
"""
msg = "54884 05/25/2009 13:41:14.060 SEV=3 HTTP/42 RPT=4623 Error on socket accept."
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), 'HTTP/42' )
self.assertEquals( evt.get('summary'), 'Error on socket accept.' )
def __init__(self,
taskName,
configId,
scheduleIntervalSeconds=3600,
taskConfig=None):
BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds,
taskConfig)
self.log = log
# Needed for interface
self.name = taskName
self.configId = configId
self.state = TaskStates.STATE_IDLE
self.interval = scheduleIntervalSeconds
self._preferences = taskConfig
self._daemon = zope.component.getUtility(ICollector)
self._eventService = zope.component.queryUtility(IEventService)
self._statService = zope.component.queryUtility(IStatisticsService)
self._preferences = self._daemon
self.options = self._daemon.options
self.stats = Stats()
if not self.options.useFileDescriptor\
and self.options.syslogport < 1024:
self._daemon.openPrivilegedPort(
'--listen', '--proto=udp', '--port=%s:%d' %
(self.options.listenip, self.options.syslogport))
self._daemon.changeUser()
self.minpriority = self.options.minpriority
self.processor = None
if self.options.logorig:
self.olog = logging.getLogger('origsyslog')
self.olog.setLevel(20)
self.olog.propagate = False
lname = zenPath('log/origsyslog.log')
hdlr = logging.FileHandler(lname)
hdlr.setFormatter(logging.Formatter('%(message)s'))
self.olog.addHandler(hdlr)
if self.options.useFileDescriptor is not None:
self.useUdpFileDescriptor(int(self.options.useFileDescriptor))
else:
reactor.listenUDP(self.options.syslogport,
self,
interface=self.options.listenip)
# yield self.model().callRemote('getDefaultPriority')
self.processor = SyslogProcessor(self._eventService.sendEvent,
self.options.minpriority,
self.options.parsehost,
self.options.monitor,
self._daemon.defaultPriority)
class ZenTestSyslogRules(CmdBase):
def __init__(self):
CmdBase.__init__(self)
self.processor = SyslogProcessor(self.sendEvent,
self.options.minpriority, False,
"localhost", 2)
self.keptEvent = False
self.totalSent = 0
def sendEvent(self, evt, *args, **kwargs):
self.keptEvent = True
self.totalSent += 1
def getSyslogMsg(self, line):
# Throw out captured meta-data
return line[ORID_SYSLOG_METADATA_END_POS:-1]
def run(self):
# Apply the regex rules against all captured syslog messages
totalEvents = 0
counts= {}
for line in open(self.options.infile):
line = line.strip()
if not line or line.startswith("#"):
continue
totalEvents += 1
self.keptEvent = False
self.processor.process(line, "127.0.0.1", "localhost", None)
if self.keptEvent:
key = self.getSyslogMsg(line)
counts.setdefault(key, 0)
counts[key] += 1
# Print the output sorted by the number of times each message
# has occurred in the original input
for key, count in sorted(counts.items(), key=lambda x: x[1]):
print "%d %s" % (count, key)
droppedCount = totalEvents - self.totalSent
droppedPct = 0
if totalEvents:
droppedPct = droppedCount * 100.0 / totalEvents
self.log.info("Test event stats: dropped=%d sent=%d total=%d dropped_pct=%.1f",
droppedCount, self.totalSent, totalEvents, droppedPct)
def buildOptions(self):
CmdBase.buildOptions(self)
self.parser.add_option('--infile',
dest='infile', default=defaultInfile,
help="File containing captured syslog events.")
self.parser.add_option('--minpriority', dest='minpriority',
default=6, type='int',
help='Minimum priority message that zensyslog will accept')
def testDellSyslog2(self):
"""
Test dell stuf
"""
msg = ("2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.")
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), '43.3.2' )
self.assertEquals( evt.get('summary'),
"Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.")
def testCheckFortigate(self):
"""
Test of Fortigate syslog message parsing
"""
msg = "date=xxxx devname=blue log_id=987654321 type=myComponent blah blah blah"
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), '987654321' )
self.assertEquals( evt.get('component'), 'myComponent' )
self.assertEquals( evt.get('summary'), 'devname=blue log_id=987654321 type=myComponent blah blah blah' )
def testDellSyslog(self):
"""
Test dell stuf
"""
msg = ("1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.")
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), '8.2.5' )
self.assertEquals( evt.get('summary'),
"Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.")
def testCheckFortigate(self):
"""
Test of Fortigate syslog message parsing
"""
msg = "date=xxxx devname=blue log_id=987654321 type=myComponent blah blah blah"
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), '987654321' )
self.assertEquals( evt.get('component'), 'myComponent' )
self.assertEquals( evt.get('summary'), 'devname=blue log_id=987654321 type=myComponent blah blah blah' )
def testDellSyslog2(self):
"""
Test dell stuf
"""
msg = ("2626:48:VolExec:27-Aug-2009 13:15:58.072049:VE_VolSetWorker.hh:75:WARNING:43.3.2:Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.")
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), '43.3.2' )
self.assertEquals( evt.get('summary'),
"Volume volumeName has reached 96 percent of its reported size and is currently using 492690MB.")
def testDellSyslog(self):
"""
Test dell stuf
"""
msg = ("1-Oct-2009 23:00:00.383809:snapshotDelete.cc:290:INFO:8.2.5:Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.")
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('eventClassKey'), '8.2.5' )
self.assertEquals( evt.get('summary'),
"Successfully deleted snapshot 'UNVSQLCLUSTERTEMPDB-2009-09-30-23:00:14.11563'.")
def testCheckFortigate(self):
"""
Test of Fortigate syslog message parsing
"""
msg = "date=xxxx devname=blue log_id=987654321 type=myComponent blah blah blah"
s = SyslogProcessor(self.sendEvent, 6, False, "localhost", 3)
evt = s.parseTag({}, msg)
self.assertEquals(evt.get("eventClassKey"), "987654321")
self.assertEquals(evt.get("component"), "myComponent")
self.assertEquals(evt.get("summary"), "devname=blue log_id=987654321 type=myComponent blah blah blah")
def testCheckCiscoPortStatus(self):
"""
Test of Cisco port status syslog message parsing
"""
msg = "Process 10532, Nbr 192.168.10.13 on GigabitEthernet2/15 from LOADING to FULL, Loading Done"
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('device'), '192.168.10.13' )
self.assertEquals( evt.get('process_id'), '10532' )
self.assertEquals( evt.get('interface'), 'GigabitEthernet2/15' )
self.assertEquals( evt.get('start_state'), 'LOADING' )
self.assertEquals( evt.get('end_state'), 'FULL' )
self.assertEquals( evt.get('summary'), 'Loading Done')
def testCheckCiscoPortStatus(self):
"""
Test of Cisco port status syslog message parsing
"""
msg = "Process 10532, Nbr 192.168.10.13 on GigabitEthernet2/15 from LOADING to FULL, Loading Done"
s = SyslogProcessor(self.sendEvent, 6, False, "localhost", 3)
evt = s.parseTag({}, msg)
self.assertEquals(evt.get("device"), "192.168.10.13")
self.assertEquals(evt.get("process_id"), "10532")
self.assertEquals(evt.get("interface"), "GigabitEthernet2/15")
self.assertEquals(evt.get("start_state"), "LOADING")
self.assertEquals(evt.get("end_state"), "FULL")
self.assertEquals(evt.get("summary"), "Loading Done")
def testCheckCiscoPortStatus(self):
"""
Test of Cisco port status syslog message parsing
"""
msg = "Process 10532, Nbr 192.168.10.13 on GigabitEthernet2/15 from LOADING to FULL, Loading Done"
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
evt = s.parseTag( {}, msg )
self.assertEquals( evt.get('device'), '192.168.10.13' )
self.assertEquals( evt.get('process_id'), '10532' )
self.assertEquals( evt.get('interface'), 'GigabitEthernet2/15' )
self.assertEquals( evt.get('start_state'), 'LOADING' )
self.assertEquals( evt.get('end_state'), 'FULL' )
self.assertEquals( evt.get('summary'), 'Loading Done')
def testProcess(self):
long_text_message = "long text message " * 20
msg = "2016-08-08T11:07:33.660820-04:00 devname=localhost log_id=98765434 type=component {}".format(long_text_message)
ipaddr = "127.0.0.1"
host = "8080"
rtime = "1416111"
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
s.process(msg, ipaddr, host, rtime)
evt = self.sent
self.assertEquals(evt.get('device'), host)
self.assertEquals(evt.get('ipAddress'), ipaddr)
self.assertEquals(evt.get('firstTime'), rtime)
self.assertEquals(evt.get('lastTime'), rtime)
self.assertEquals(evt.get('eventGroup'), 'syslog')
self.assertEquals(evt.get('message'), unicode(msg))
self.assertEquals(evt.get('summary'), unicode(msg))
def __init__(self):
CmdBase.__init__(self)
self.processor = SyslogProcessor(self.sendEvent,
self.options.minpriority, False,
"localhost", 2)
self.keptEvent = False
self.totalSent = 0
def __init__(self, taskName, configId,
scheduleIntervalSeconds=3600, taskConfig=None):
BaseTask.__init__(self, taskName, configId,
scheduleIntervalSeconds, taskConfig)
self.log = log
# Needed for interface
self.name = taskName
self.configId = configId
self.state = TaskStates.STATE_IDLE
self.interval = scheduleIntervalSeconds
self._preferences = taskConfig
self._daemon = zope.component.getUtility(ICollector)
self._eventService = zope.component.queryUtility(IEventService)
self._statService = zope.component.queryUtility(IStatisticsService)
self._preferences = self._daemon
self.options = self._daemon.options
self.stats = Stats()
if not self.options.useFileDescriptor\
and self.options.syslogport < 1024:
self._daemon.openPrivilegedPort('--listen', '--proto=udp',
'--port=%s:%d'
% (self.options.listenip,
self.options.syslogport))
self._daemon.changeUser()
self.minpriority = self.options.minpriority
self.processor = None
if self.options.logorig:
self.olog = logging.getLogger('origsyslog')
self.olog.setLevel(20)
self.olog.propagate = False
lname = zenPath('log/origsyslog.log')
hdlr = logging.FileHandler(lname)
hdlr.setFormatter(logging.Formatter('%(message)s'))
self.olog.addHandler(hdlr)
if self.options.useFileDescriptor is not None:
self.useUdpFileDescriptor(int(self.options.useFileDescriptor))
else:
reactor.listenUDP(self.options.syslogport, self,
interface=self.options.listenip)
# yield self.model().callRemote('getDefaultPriority')
self.processor = SyslogProcessor(self._eventService.sendEvent,
self.options.minpriority, self.options.parsehost,
self.options.monitor, self._daemon.defaultPriority)
def testBuildEventClassKey(self):
"Simple, brain-dead testing of SyslogProcessor"
base = dict(device='localhost', component='component', severity=3)
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
self.assert_(s.buildEventClassKey({}) == {})
evt = dict(eventClassKey='akey', **base)
self.assert_(s.buildEventClassKey(evt.copy()) == evt)
evt = dict(eventClassKey='akey', ntevid='1234', **base)
self.assert_(s.buildEventClassKey(evt.copy()) == evt)
evt = dict(ntevid='1234', **base)
self.assert_(s.buildEventClassKey(evt)['eventClassKey'] ==
'component_1234')
evt = dict(**base)
self.assert_(s.buildEventClassKey(evt)['eventClassKey'] == 'component')
def testBuildEventClassKey(self):
"Simple, brain-dead testing of SyslogProcessor"
base = dict(device="localhost", component="component", severity=3)
s = SyslogProcessor(self.sendEvent, 6, False, "localhost", 3)
self.assert_(s.buildEventClassKey({}) == {})
evt = dict(eventClassKey="akey", **base)
self.assert_(s.buildEventClassKey(evt.copy()) == evt)
evt = dict(eventClassKey="akey", ntevid="1234", **base)
self.assert_(s.buildEventClassKey(evt.copy()) == evt)
evt = dict(ntevid="1234", **base)
self.assert_(s.buildEventClassKey(evt)["eventClassKey"] == "component_1234")
evt = dict(**base)
self.assert_(s.buildEventClassKey(evt)["eventClassKey"] == "component")
def testBuildEventClassKey(self):
"Simple, brain-dead testing of SyslogProcessor"
base = dict(device='localhost', component='component', severity=3)
s = SyslogProcessor(self.sendEvent, 6, False, 'localhost', 3)
self.assert_(s.buildEventClassKey({}) == {})
evt = dict(eventClassKey='akey', **base)
self.assert_(s.buildEventClassKey(evt.copy()) == evt)
evt = dict(eventClassKey='akey', ntevid='1234', **base)
self.assert_(s.buildEventClassKey(evt.copy()) == evt)
evt = dict(ntevid='1234', **base)
self.assert_(s.buildEventClassKey(evt)['eventClassKey'] ==
'component_1234')
evt = dict(**base)
self.assert_(s.buildEventClassKey(evt)['eventClassKey'] == 'component')
class SyslogTask(BaseTask, DatagramProtocol):
"""
Listen for syslog messages and turn them into events
Connects to the TrapService service in zenhub.
"""
zope.interface.implements(IScheduledTask)
SYSLOG_DATE_FORMAT = '%b %d %H:%M:%S'
SAMPLE_DATE = 'Apr 10 15:19:22'
def __init__(self,
taskName,
configId,
scheduleIntervalSeconds=3600,
taskConfig=None):
BaseTask.__init__(self, taskName, configId, scheduleIntervalSeconds,
taskConfig)
self.log = log
# Needed for interface
self.name = taskName
self.configId = configId
self.state = TaskStates.STATE_IDLE
self.interval = scheduleIntervalSeconds
self._preferences = taskConfig
self._daemon = zope.component.getUtility(ICollector)
self._eventService = zope.component.queryUtility(IEventService)
self._statService = zope.component.queryUtility(IStatisticsService)
self._preferences = self._daemon
self.options = self._daemon.options
self.stats = Stats()
if not self.options.useFileDescriptor\
and self.options.syslogport < 1024:
self._daemon.openPrivilegedPort(
'--listen', '--proto=udp', '--port=%s:%d' %
(self.options.listenip, self.options.syslogport))
self._daemon.changeUser()
self.minpriority = self.options.minpriority
self.processor = None
if self.options.logorig:
self.olog = logging.getLogger('origsyslog')
self.olog.setLevel(20)
self.olog.propagate = False
lname = zenPath('log/origsyslog.log')
hdlr = logging.FileHandler(lname)
hdlr.setFormatter(logging.Formatter('%(message)s'))
self.olog.addHandler(hdlr)
if self.options.useFileDescriptor is not None:
self.useUdpFileDescriptor(int(self.options.useFileDescriptor))
else:
reactor.listenUDP(self.options.syslogport,
self,
interface=self.options.listenip)
# yield self.model().callRemote('getDefaultPriority')
self.processor = SyslogProcessor(self._eventService.sendEvent,
self.options.minpriority,
self.options.parsehost,
self.options.monitor,
self._daemon.defaultPriority)
def doTask(self):
"""
This is a wait-around task since we really are called
asynchronously.
"""
return defer.succeed("Waiting for syslog messages...")
def useUdpFileDescriptor(self, fd):
s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM)
os.close(fd)
port = s.getsockname()[1]
transport = udp.Port(port, self)
s.setblocking(0)
transport.socket = s
transport.fileno = s.fileno
transport.connected = 1
transport._realPortNumber = port
self.transport = transport
# hack around startListening not being called
self.numPorts = 1
transport.startReading()
def expand(self, msg, client_address):
"""
Expands a syslog message into a string format suitable for writing
to the filesystem such that it appears the same as it would
had the message been logged by the syslog daemon.
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
@return: message
@rtype: string
"""
# pri := facility * severity
stop = msg.find('>')
# check for a datestamp. default to right now if date not present
start = stop + 1
stop = start + len(SyslogTask.SAMPLE_DATE)
dateField = msg[start:stop]
try:
date = time.strptime(dateField, SyslogTask.SYSLOG_DATE_FORMAT)
year = time.localtime()[0]
date = (year, ) + date[1:]
start = stop + 1
except ValueError:
# date not present, so use today's date
date = time.localtime()
# check for a hostname. default to localhost if not present
stop = msg.find(' ', start)
if msg[stop - 1] == ':':
hostname = client_address[0]
else:
hostname = msg[start:stop]
start = stop + 1
# the message content
body = msg[start:]
# assemble the message
prettyTime = time.strftime(SyslogTask.SYSLOG_DATE_FORMAT, date)
message = '%s %s %s' % (prettyTime, hostname, body)
return message
def datagramReceived(self, msg, client_address):
"""
Consume the network packet
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
"""
if msg == "":
self.log.debug("Received empty datagram. Discarding.")
return
(ipaddr, port) = client_address
if self.options.logorig:
if self.options.logformat == 'human':
message = self.expand(msg, client_address)
else:
message = msg
self.olog.info(message)
if self.options.noreverseLookup:
d = defer.succeed(ipaddr)
else:
d = asyncNameLookup(ipaddr)
d.addBoth(self.gotHostname, (msg, ipaddr, time.time()))
def gotHostname(self, response, data):
"""
Send the resolved address, if possible, and the event via the thread
@param response: Twisted response
@type response: Twisted response
@param data: (msg, ipaddr, rtime)
@type data: tuple of (string, string, datetime object)
"""
(msg, ipaddr, rtime) = data
if isinstance(response, failure.Failure):
host = ipaddr
else:
host = response
if self.processor:
self.processor.process(msg, ipaddr, host, rtime)
totalTime, totalEvents, maxTime = self.stats.report()
stat = self._statService.getStatistic("events")
stat.value = totalEvents
def displayStatistics(self):
totalTime, totalEvents, maxTime = self.stats.report()
display = "%d events processed in %.2f seconds" % (totalEvents,
totalTime)
if totalEvents > 0:
display += """
%.5f average seconds per event
Maximum processing time for one event was %.5f""" % (
(totalTime / totalEvents), maxTime)
return display
def cleanup(self):
status = self.displayStatistics()
self.log.info(status)
class SyslogTask(BaseTask, DatagramProtocol):
"""
Listen for syslog messages and turn them into events
Connects to the TrapService service in zenhub.
"""
zope.interface.implements(IScheduledTask)
SYSLOG_DATE_FORMAT = '%b %d %H:%M:%S'
SAMPLE_DATE = 'Apr 10 15:19:22'
def __init__(self, taskName, configId,
scheduleIntervalSeconds=3600, taskConfig=None):
BaseTask.__init__(self, taskName, configId,
scheduleIntervalSeconds, taskConfig)
self.log = log
# Needed for interface
self.name = taskName
self.configId = configId
self.state = TaskStates.STATE_IDLE
self.interval = scheduleIntervalSeconds
self._preferences = taskConfig
self._daemon = zope.component.getUtility(ICollector)
self._eventService = zope.component.queryUtility(IEventService)
self._statService = zope.component.queryUtility(IStatisticsService)
self._preferences = self._daemon
self.options = self._daemon.options
self.stats = Stats()
if not self.options.useFileDescriptor\
and self.options.syslogport < 1024:
self._daemon.openPrivilegedPort('--listen', '--proto=udp',
'--port=%s:%d'
% (self.options.listenip,
self.options.syslogport))
self._daemon.changeUser()
self.minpriority = self.options.minpriority
self.processor = None
if self.options.logorig:
self.olog = logging.getLogger('origsyslog')
self.olog.setLevel(20)
self.olog.propagate = False
lname = zenPath('log/origsyslog.log')
hdlr = logging.FileHandler(lname)
hdlr.setFormatter(logging.Formatter('%(message)s'))
self.olog.addHandler(hdlr)
if self.options.useFileDescriptor is not None:
self.useUdpFileDescriptor(int(self.options.useFileDescriptor))
else:
reactor.listenUDP(self.options.syslogport, self,
interface=self.options.listenip)
# yield self.model().callRemote('getDefaultPriority')
self.processor = SyslogProcessor(self._eventService.sendEvent,
self.options.minpriority, self.options.parsehost,
self.options.monitor, self._daemon.defaultPriority)
def doTask(self):
"""
This is a wait-around task since we really are called
asynchronously.
"""
return defer.succeed("Waiting for syslog messages...")
def useUdpFileDescriptor(self, fd):
s = socket.fromfd(fd, socket.AF_INET, socket.SOCK_DGRAM)
os.close(fd)
port = s.getsockname()[1]
transport = udp.Port(port, self)
s.setblocking(0)
transport.socket = s
transport.fileno = s.fileno
transport.connected = 1
transport._realPortNumber = port
self.transport = transport
# hack around startListening not being called
self.numPorts = 1
transport.startReading()
def expand(self, msg, client_address):
"""
Expands a syslog message into a string format suitable for writing
to the filesystem such that it appears the same as it would
had the message been logged by the syslog daemon.
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
@return: message
@rtype: string
"""
# pri := facility * severity
stop = msg.find('>')
# check for a datestamp. default to right now if date not present
start = stop + 1
stop = start + len(SyslogTask.SAMPLE_DATE)
dateField = msg[start:stop]
try:
date = time.strptime(dateField,
SyslogTask.SYSLOG_DATE_FORMAT)
year = time.localtime()[0]
date = (year, ) + date[1:]
start = stop + 1
except ValueError:
# date not present, so use today's date
date = time.localtime()
# check for a hostname. default to localhost if not present
stop = msg.find(' ', start)
if msg[stop - 1] == ':':
hostname = client_address[0]
else:
hostname = msg[start:stop]
start = stop + 1
# the message content
body = msg[start:]
# assemble the message
prettyTime = time.strftime(SyslogTask.SYSLOG_DATE_FORMAT, date)
message = '%s %s %s' % (prettyTime, hostname, body)
return message
def datagramReceived(self, msg, client_address):
"""
Consume the network packet
@param msg: syslog message
@type msg: string
@param client_address: IP info of the remote device (ipaddr, port)
@type client_address: tuple of (string, number)
"""
(ipaddr, port) = client_address
if self.options.logorig:
if self.options.logformat == 'human':
message = self.expand(msg, client_address)
else:
message = msg
self.olog.info(message)
if self.options.noreverseLookup:
d = defer.succeed(ipaddr)
else:
d = asyncNameLookup(ipaddr)
d.addBoth(self.gotHostname, (msg, ipaddr, time.time()))
def gotHostname(self, response, data):
"""
Send the resolved address, if possible, and the event via the thread
@param response: Twisted response
@type response: Twisted response
@param data: (msg, ipaddr, rtime)
@type data: tuple of (string, string, datetime object)
"""
(msg, ipaddr, rtime) = data
if isinstance(response, failure.Failure):
host = ipaddr
else:
host = response
if self.processor:
self.processor.process(msg, ipaddr, host, rtime)
totalTime, totalEvents, maxTime = self.stats.report()
stat = self._statService.getStatistic("events")
stat.value = totalEvents
def displayStatistics(self):
totalTime, totalEvents, maxTime = self.stats.report()
display = "%d events processed in %.2f seconds" % (
totalEvents,
totalTime)
if totalEvents > 0:
display += """
%.5f average seconds per event
Maximum processing time for one event was %.5f""" % (
(totalTime / totalEvents), maxTime)
return display
def cleanup(self):
status = self.displayStatistics()
self.log.info(status)
