def generateDicts(sock):
severityMap = {
"0": "emerg",
"1": "alert",
"2": "crit",
"3": "err",
"4": "warning",
"5": "notice",
"6": "info",
"7": "debug"
}
facilityMap = {
"0": "kernel",
"1": "user",
"2": "mail",
"3": "system",
"4": "auth",
"5": "syslog",
"6": "lpd",
"7": "news",
"8": "uucp",
"9": "time",
"10": "auth",
"11": "ftp",
"12": "ntp",
"13": "logaudit",
"14": "logalert",
"15": "clock",
"16": "local0",
"17": "local1",
"18": "local2",
"19": "local3",
"20": "local4",
"21": "local5",
"22": "local6",
"23": "local7"
}
skip = 0
skipcount = 0
ssec = datetime.utcnow().strftime("%S")
yieldcount = 0
# Compile regex patterns for iteration on each component of the message
pats = {}
pristrings = [r'^<(?P<pri>\d{1,3})>(\d*:?)?']
pats['pri'] = []
for i in pristrings:
pats['pri'].append(re.compile(i + r'(?P<space>\s?)\S+'))
# Date/time
datestrings = [
r'(?P<date>[A-Za-z]+ [ \d]?\d \d\d:\d\d:\d\d( [A-Z]{3}:)?)',
r'(?P<date>\d{4} [A-Za-z]+ [ \d]?\d \d\d:\d\d:\d\d( [A-Z]{3}:)?)',
r'(?P<date>\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z)'
]
pats['date'] = []
for i in datestrings:
pats['date'].append(re.compile(i + r'(?P<space>\s+)\S+'))
# Host/IP
hoststrings = [
r'(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})',
r'(?P<host>[a-z0-9_-]+(\.[a-z0-9_-]+)*(\.[a-z]+[0-9]?))',
r'(?P<host>[a-z0-9_-]+)'
]
pats['host'] = []
for i in hoststrings:
pats['host'].append(re.compile(i + r'(?P<space>\s+)\S+',
re.IGNORECASE))
# Rest of line
pats['message'] = [re.compile(r'(?P<message>.*)(?P<space>\s*)$')]
currentDict = {}
a10 = Device.A10('logs')
arista = Device.Arista('logs')
brocade = Device.Brocade('logs')
f5 = Device.F5('logs')
force10 = Device.Force10('logs')
juniper = Device.Juniper('logs')
linux = Device.Linux('logs')
pktbuf = []
pktbuf_peak = 0
packets = 0
processed = 0
while True:
try:
data, (src_ip, port) = sock.recvfrom(8192)
pktbuf.append((data, src_ip, port))
packets += 1
except socket.error:
if int(datetime.utcnow().strftime("%S")) != ssec:
ssec = int(datetime.utcnow().strftime("%S"))
eprint(
"%28s Messages in buffer (peak): %d read: %d processed: %d yielded: %d skipped: %d"
% (str(datetime.utcnow()), pktbuf_peak, packets, processed,
yieldcount, skipcount))
pktbuf_peak = 0
if len(pktbuf) == 0:
continue
else:
buf_len = len(pktbuf)
if buf_len > pktbuf_peak:
pktbuf_peak = buf_len
while pktbuf:
line, src_ip, port = pktbuf.pop(0)
processed += 1
# We can safely init the dict here because multiline messages are
# still contained within single datagrams
currentDict = {}
# Pristine copy of what we received
currentDict['raw_message'] = line
# Strip any leading junk
if line[0] == '\0':
line = line.lstrip('\r\n\0 ')
for pname in ['pri', 'date', 'host', 'message']:
for p in pats[pname]:
matched = p.match(line)
if matched:
if pname == 'pri':
currentDict['severity_int'] = str(
int(matched.group('pri')) & 7)
currentDict['facility_int'] = str(
int(matched.group('pri')) >> 3 & 23)
currentDict['severity_label'] = severityMap[
currentDict['severity_int']]
currentDict['facility_label'] = facilityMap[
currentDict['facility_int']]
currentDict[pname] = matched.group(pname)
# Trim the line up to the ending space from the last match
line = line[matched.end('space'):]
# We matched this element so no need to keep looping on it
break
# None of the patterns matched for this field
if pname not in currentDict:
eprint("Did not match for %s: %s" % (pname, line))
# Chop off any remaining crap
line = line.rstrip()
# Finished parsing but did not consume the whole line (should never happen)
if len(line) > 0:
eprint("still some line left: [%s]" % line)
# Did not match anything at all?
if currentDict == {}:
eprint("matched nothing: [%s]" % line)
skipcount += 1
continue
else:
skip = 0
vendor = None
#currentDict['fromhost'] = resolveHostname(src_ip)
currentDict['fromhost'] = src_ip
currentDict['fromhost-ip'] = src_ip
if 'host' not in currentDict:
currentDict['host'] = currentDict['fromhost'].lower()
else:
currentDict['host'] = currentDict['host'].lower()
try:
if currentDict['host'].find('v-') >= 0 and currentDict[
'host'].find('-net') >= 7:
vendor = linux
elif currentDict['host'].find(
'bar') == 0 or currentDict['host'].find(
'bcr') == 0 or currentDict['host'].find(
'scr'
) == 0 or currentDict['host'].find(
'sff'
) == 0 or currentDict['host'].find(
'mfw'
) == 0 or currentDict['host'].find(
're') == 0 or currentDict['host'].find(
'bmr'
) == 0 or currentDict['host'].find(
'fw'
) == 0 or currentDict['host'].find(
'r1') == 0 or currentDict[
'host'].find('r2') == 0:
vendor = juniper
elif currentDict['host'].find(
'ma') == 0 or currentDict['host'].find(
'trr') == 0 or currentDict['host'].find(
'spr'
) == 0 or currentDict['host'].find(
'ssr') == 0 or currentDict[
'host'].find('ser') == 0:
vendor = arista
elif currentDict['host'].find(
'slb') == 0 or currentDict['host'].find(
'mlb') == 0 or currentDict['host'].find(
'glb') == 0 or currentDict[
'host'].find('vpr') == 0:
vendor = a10
elif currentDict['host'].find('lb') == 0:
vendor = f5
elif currentDict['host'].find('sw') == 0:
vendor = brocade
elif currentDict['host'].find('10.1') == 0:
vendor = force10
if vendor:
currentDict['vendor'] = vendor.vendor
if not vendor.matchLogPattern(currentDict):
eprint(
"Did not match %s message for host %s: %s"
% (vendor.vendor, currentDict['host'],
currentDict['message']))
# Flag as unmatched message
currentDict['state'] = 5
else:
eprint(
"Did not match host pattern for host: %s message: %s"
%
(currentDict['host'], currentDict['message']))
except KeyError:
eprint("Field not found:", currentDict)
skip = 1
skipcount += 1
if skip == 0:
yield (currentDict)
yieldcount += 1
