def generateDicts(sock): severityMap = { "0": "emerg", "1": "alert", "2": "crit", "3": "err", "4": "warning", "5": "notice", "6": "info", "7": "debug" } facilityMap = { "0": "kernel", "1": "user", "2": "mail", "3": "system", "4": "auth", "5": "syslog", "6": "lpd", "7": "news", "8": "uucp", "9": "time", "10": "auth", "11": "ftp", "12": "ntp", "13": "logaudit", "14": "logalert", "15": "clock", "16": "local0", "17": "local1", "18": "local2", "19": "local3", "20": "local4", "21": "local5", "22": "local6", "23": "local7" } skip = 0 skipcount = 0 ssec = datetime.utcnow().strftime("%S") yieldcount = 0 # Compile regex patterns for iteration on each component of the message pats = {} pristrings = [r'^<(?P<pri>\d{1,3})>(\d*:?)?'] pats['pri'] = [] for i in pristrings: pats['pri'].append(re.compile(i + r'(?P<space>\s?)\S+')) # Date/time datestrings = [ r'(?P<date>[A-Za-z]+ [ \d]?\d \d\d:\d\d:\d\d( [A-Z]{3}:)?)', r'(?P<date>\d{4} [A-Za-z]+ [ \d]?\d \d\d:\d\d:\d\d( [A-Z]{3}:)?)', r'(?P<date>\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z)' ] pats['date'] = [] for i in datestrings: pats['date'].append(re.compile(i + r'(?P<space>\s+)\S+')) # Host/IP hoststrings = [ r'(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})', r'(?P<host>[a-z0-9_-]+(\.[a-z0-9_-]+)*(\.[a-z]+[0-9]?))', r'(?P<host>[a-z0-9_-]+)' ] pats['host'] = [] for i in hoststrings: pats['host'].append(re.compile(i + r'(?P<space>\s+)\S+', re.IGNORECASE)) # Rest of line pats['message'] = [re.compile(r'(?P<message>.*)(?P<space>\s*)$')] currentDict = {} a10 = Device.A10('logs') arista = Device.Arista('logs') brocade = Device.Brocade('logs') f5 = Device.F5('logs') force10 = Device.Force10('logs') juniper = Device.Juniper('logs') linux = Device.Linux('logs') pktbuf = [] pktbuf_peak = 0 packets = 0 processed = 0 while True: try: data, (src_ip, port) = sock.recvfrom(8192) pktbuf.append((data, src_ip, port)) packets += 1 except socket.error: if int(datetime.utcnow().strftime("%S")) != ssec: ssec = int(datetime.utcnow().strftime("%S")) eprint( "%28s Messages in buffer (peak): %d read: %d processed: %d yielded: %d skipped: %d" % (str(datetime.utcnow()), pktbuf_peak, packets, processed, yieldcount, skipcount)) pktbuf_peak = 0 if len(pktbuf) == 0: continue else: buf_len = len(pktbuf) if buf_len > pktbuf_peak: pktbuf_peak = buf_len while pktbuf: line, src_ip, port = pktbuf.pop(0) processed += 1 # We can safely init the dict here because multiline messages are # still contained within single datagrams currentDict = {} # Pristine copy of what we received currentDict['raw_message'] = line # Strip any leading junk if line[0] == '\0': line = line.lstrip('\r\n\0 ') for pname in ['pri', 'date', 'host', 'message']: for p in pats[pname]: matched = p.match(line) if matched: if pname == 'pri': currentDict['severity_int'] = str( int(matched.group('pri')) & 7) currentDict['facility_int'] = str( int(matched.group('pri')) >> 3 & 23) currentDict['severity_label'] = severityMap[ currentDict['severity_int']] currentDict['facility_label'] = facilityMap[ currentDict['facility_int']] currentDict[pname] = matched.group(pname) # Trim the line up to the ending space from the last match line = line[matched.end('space'):] # We matched this element so no need to keep looping on it break # None of the patterns matched for this field if pname not in currentDict: eprint("Did not match for %s: %s" % (pname, line)) # Chop off any remaining crap line = line.rstrip() # Finished parsing but did not consume the whole line (should never happen) if len(line) > 0: eprint("still some line left: [%s]" % line) # Did not match anything at all? if currentDict == {}: eprint("matched nothing: [%s]" % line) skipcount += 1 continue else: skip = 0 vendor = None #currentDict['fromhost'] = resolveHostname(src_ip) currentDict['fromhost'] = src_ip currentDict['fromhost-ip'] = src_ip if 'host' not in currentDict: currentDict['host'] = currentDict['fromhost'].lower() else: currentDict['host'] = currentDict['host'].lower() try: if currentDict['host'].find('v-') >= 0 and currentDict[ 'host'].find('-net') >= 7: vendor = linux elif currentDict['host'].find( 'bar') == 0 or currentDict['host'].find( 'bcr') == 0 or currentDict['host'].find( 'scr' ) == 0 or currentDict['host'].find( 'sff' ) == 0 or currentDict['host'].find( 'mfw' ) == 0 or currentDict['host'].find( 're') == 0 or currentDict['host'].find( 'bmr' ) == 0 or currentDict['host'].find( 'fw' ) == 0 or currentDict['host'].find( 'r1') == 0 or currentDict[ 'host'].find('r2') == 0: vendor = juniper elif currentDict['host'].find( 'ma') == 0 or currentDict['host'].find( 'trr') == 0 or currentDict['host'].find( 'spr' ) == 0 or currentDict['host'].find( 'ssr') == 0 or currentDict[ 'host'].find('ser') == 0: vendor = arista elif currentDict['host'].find( 'slb') == 0 or currentDict['host'].find( 'mlb') == 0 or currentDict['host'].find( 'glb') == 0 or currentDict[ 'host'].find('vpr') == 0: vendor = a10 elif currentDict['host'].find('lb') == 0: vendor = f5 elif currentDict['host'].find('sw') == 0: vendor = brocade elif currentDict['host'].find('10.1') == 0: vendor = force10 if vendor: currentDict['vendor'] = vendor.vendor if not vendor.matchLogPattern(currentDict): eprint( "Did not match %s message for host %s: %s" % (vendor.vendor, currentDict['host'], currentDict['message'])) # Flag as unmatched message currentDict['state'] = 5 else: eprint( "Did not match host pattern for host: %s message: %s" % (currentDict['host'], currentDict['message'])) except KeyError: eprint("Field not found:", currentDict) skip = 1 skipcount += 1 if skip == 0: yield (currentDict) yieldcount += 1