def test_parse_leap_day_in_non_leap_year(self, current_year): data = b'<165>Feb 29 19:56:43 localhost foobar' fake_date = '{:d}-01-01'.format(current_year) with self.assertRaises(MessageFormatError): with freeze_time(fake_date): parse(data)
def parse_syslog(data): parsed_data = syslogmp.parse(data) try: full_message = parsed_data.message.decode() except UnicodeDecodeError: return None if not full_message.startswith('kernel: ['): # Not a kernel message return None close_bracket = full_message.find(']') if close_bracket == -1: return None iptables_message = full_message[close_bracket + 1:].strip() parts = iptables_message.split(' ') # parts[0] is the log-prefix # parts[1..] is either KEY=VALUE or just KEY if not parts[1].startswith('IN='): # not an iptables message return None fields = { k: v for k, _, v in (field.partition('=') for field in parts[1:]) } fields['hostname'] = parsed_data.hostname fields['message'] = iptables_message return fields
def parse_rfc_3164_format( log_message: bytes) -> Optional[SyslogMessageExtract]: """ Receives a log message which is in RFC 3164 format. Parses it into SyslogMessageExtract data class object Args: log_message (bytes): Syslog message. Returns: (Optional[SyslogMessageExtract]): Extraction data class """ try: syslog_message: syslogmp.Message = syslogmp.parse(log_message) except syslogmp.parser.MessageFormatError: return None return SyslogMessageExtract( app_name=None, facility=syslog_message.facility.name, host_name=syslog_message.hostname, msg=syslog_message.message.decode('utf-8'), msg_id=None, process_id=None, sd={}, severity=syslog_message.severity.name, timestamp=syslog_message.timestamp.isoformat(), version=None, # Because RF-3164 doesn't return localized date, can't determine the localized time it occurred. occurred=None)
def parseLog(log, log_format): format_found = False log_format_regex = "" for format_ in log_formats: if (log_format in format_): log_format_regex = format_[log_format] format_found = True if (log_format == "RFC 3164"): log_byte = str.encode(log) message = parse(log_byte) splited_msg = message.message.decode('utf-8').split("]:", 1) datetime = message.timestamp log_json = {} log_json["logged_date"] = datetime.strftime('%Y-%m-%d') log_json["logged_time"] = datetime.strftime('%Y-%m-%d %H:%m:%S') log_json["host"] = message.hostname log_json["process"] = splited_msg[0].strip() + ']' if (message.severity.name == "informational"): log_json["severity"] = "INFO" else: log_json["severity"] = message.severity.name.upper() log_json["message"] = splited_msg[1].strip() print(log_json) return log_json # <30>Jun 5 02:36:06 stefan-pc systemd: Started Syslog service for accepting logs from journald. if (format_found): log_format_raw = log_format_regex.encode('unicode_escape').decode() pattern = re.compile(log_format_regex) match = pattern.match(log) else: print("Bad or no logging format provided.") return None if (match == None): print("Bad configuration(log format regex) - No match") print(log) return None else: log_json = {} # time = match.group("time").split(",")[0] if ("date" in pattern.groupindex and "time" in pattern.groupindex): datetime = match.group("date") + " " + match.group("time") date = parser.parse(datetime) datetime_formated = date.strftime('%d-%m-%Y %H:%M:%S') date_formated = date.strftime('%d-%m-%Y') log_json["logged_date"] = date_formated log_json["logged_time"] = datetime_formated if ("host" in pattern.groupindex): log_json["host"] = match.group("host") if ("process" in pattern.groupindex): log_json["process"] = match.group("process") if ("severity" in pattern.groupindex): log_json["severity"] = match.group("severity") if ("message" in pattern.groupindex): log_json["message"] = match.group("message") return log_json
def test_parse_leap_day_in_leap_year(self, current_year): data = b'<165>Feb 29 19:56:43 localhost foobar' fake_date = '{:d}-01-01'.format(current_year) expected_timestamp = datetime(current_year, 2, 29, 19, 56, 43) with freeze_time(fake_date): actual = parse(data) self.assertEqual(actual.timestamp, expected_timestamp)
def orchestrate(self): if config.CONFIG.isWindows(): return True s = self.getLine() while (s != None): message = syslogmp.parse(s) if (message): msg = message.message.decode("utf-8") log.L.debug("syslog: " + repr(msg)) # 127.0.0.1:45940 [04/Sep/2018:18:40:06.448] ssltunnel~ b-preproxy/s-proxy 1/0/0/12/+12 200 +37 - - ---- 2/2/1/1/0 0/0 {1ACCCCC|TO_PROXY_CONNECT|} "CONNECT www.idnes.cz:443 HTTP/1.1" p = re.search( "(^\d*\.\d*\.\d*\.\d*)|(\[(.*)\]):(\d*) " # Host and port + "\[(.*)\] " # Date + "(\w*)~? " # Frontend + "(b-\w*)/(<?s-\w*>?) " # Backend/Server + "(.\d*/.\d*/.\d*/.\d*/.\d*) " # Times + "(\d*) " # State + ".*" # Not needed + "{(.*)\|(.*)\|(.*)} " # authid, reason, overlimit + '"(.*) (.*) (.*)"', msg) if (p): ip = p.group(1) port = p.group(2) dte = p.group(3) frontend = p.group(4) backend = p.group(5) server = p.group(6) code = p.group(8) authid = p.group(9) reason = p.group(10) overlimit = p.group(11) method = p.group(12) uri = p.group(13) proto = p.group(14) if (server == "s-proxy"): sessions.SESSIONS.add(authid, ip, port, method=method, uri=uri, proto=proto) else: log.L.debug("Ignoring haproxy log") else: p = re.search( "(^\d*\.\d*\.\d*\.\d*):(\d*) \[(.*)\] ssltunnel", msg) if (p): log.L.info("Ignoring haproxy log: " + repr(msg)) s = self.getLine()
def handle(self): try: data = self.request[0] message = syslogmp.parse(data) except ValueError: log('Invalid message received from {}:{:d}.', *self.client_address) return port = self.server.get_port() log('Received message from {0[0]}:{0[1]:d} on port {1:d} -> {2}', self.client_address, port, format_message_for_log(message)) syslog_message_received.send(port, source_address=self.client_address, message=message)
def parse(self, rawdata): """ Parse any of the two types of Syslog Formats and stores the result in this object """ # RFC 5424 try: msg = RFC5452SyslogMessage.parse(rawdata.decode()).as_dict() self.id = msg.get('msgid') self.message = msg.get('msg', '').rstrip(' \t\r\n\0') self.facility = self.facilities[msg['facility']] self.facility_name = msg['facility'] self.severity = self.severities[msg['severity']] self.severity_name = msg['severity'] self.identifier = msg.get('appname') self.pid = msg.get('procid') self.hostname = msg.get('hostname') return True except (ParseError, ValueError): # log err ? pass # RFC 3164 try: if len(rawdata) > 1024: self.logger.warning('truncating message ({} > 1024)'.format( len(rawdata))) rawdata = rawdata[:1024] # truncate to 1024 msg = syslogmp.parse(rawdata) self.id = None (self.identifier, self.pid, self.message) = self.parse_3164_msg(msg.message.decode().strip()) self.message = self.message.rstrip(' \t\r\n\0') self.facility = int(msg.facility.value) self.facility_name = str(msg.facility).lower().split('.')[-1] self.severity = int(msg.severity.value) self.severity_name = str(msg.severity).lower().split('.')[-1] self.hostname = msg.hostname return True except (syslogmp.parser.MessageFormatError, ValueError): # log err ? pass return False
def test_parse( self, data, expected_facility, expected_facility_description, expected_severity, expected_timestamp, expected_hostname, expected_message): """Test parsing of a syslog message.""" actual = parse(data) self.assertEqual(actual.facility, expected_facility) self.assertEqual(actual.facility.description, expected_facility_description) self.assertEqual(actual.severity, expected_severity) self.assertEqual(actual.timestamp, expected_timestamp) self.assertEqual(actual.hostname, expected_hostname) self.assertEqual(actual.message, expected_message)
def parse(self, line): """Turns the line into a LogEntry""" # syslogmp expects a byte string line_bytes = str.encode(line) syslog_msg = parse(line_bytes) if b'filterlog:' in syslog_msg.message: return FirewallLogEntry(syslog_msg.message.decode('utf-8'), self.resolver) elif b'dhcpd:' in syslog_msg.message: e = DHCPDLogEntry(line) # We only care about DHCPACK and DHCPRequest. All other lease logs # are not interesting. if 'DHCPACK' in str(e) or 'DHCPREQUEST' in str(e): return e else: return None else: return LogEntry(f'Cannot process line {line}')
def test_parse_erroneous_message(self, data): with self.assertRaises(MessageFormatError): parse(data)
def test_parse_message_just_not_too_long(self): data = create_long_data(1024) self.assertEqual(len(data), 1024) parse(data)