def test_parse_leap_day_in_non_leap_year(self, current_year):
data = b'<165>Feb 29 19:56:43 localhost foobar'
fake_date = '{:d}-01-01'.format(current_year)
with self.assertRaises(MessageFormatError):
with freeze_time(fake_date):
parse(data)
def parse_syslog(data):
parsed_data = syslogmp.parse(data)
try:
full_message = parsed_data.message.decode()
except UnicodeDecodeError:
return None
if not full_message.startswith('kernel: ['):
# Not a kernel message
return None
close_bracket = full_message.find(']')
if close_bracket == -1:
return None
iptables_message = full_message[close_bracket + 1:].strip()
parts = iptables_message.split(' ')
# parts[0] is the log-prefix
# parts[1..] is either KEY=VALUE or just KEY
if not parts[1].startswith('IN='):
# not an iptables message
return None
fields = {
k: v
for k, _, v in (field.partition('=') for field in parts[1:])
}
fields['hostname'] = parsed_data.hostname
fields['message'] = iptables_message
return fields
def parse_rfc_3164_format(
log_message: bytes) -> Optional[SyslogMessageExtract]:
"""
Receives a log message which is in RFC 3164 format. Parses it into SyslogMessageExtract data class object
Args:
log_message (bytes): Syslog message.
Returns:
(Optional[SyslogMessageExtract]): Extraction data class
"""
try:
syslog_message: syslogmp.Message = syslogmp.parse(log_message)
except syslogmp.parser.MessageFormatError:
return None
return SyslogMessageExtract(
app_name=None,
facility=syslog_message.facility.name,
host_name=syslog_message.hostname,
msg=syslog_message.message.decode('utf-8'),
msg_id=None,
process_id=None,
sd={},
severity=syslog_message.severity.name,
timestamp=syslog_message.timestamp.isoformat(),
version=None,
# Because RF-3164 doesn't return localized date, can't determine the localized time it occurred.
occurred=None)
def parseLog(log, log_format):
format_found = False
log_format_regex = ""
for format_ in log_formats:
if (log_format in format_):
log_format_regex = format_[log_format]
format_found = True
if (log_format == "RFC 3164"):
log_byte = str.encode(log)
message = parse(log_byte)
splited_msg = message.message.decode('utf-8').split("]:", 1)
datetime = message.timestamp
log_json = {}
log_json["logged_date"] = datetime.strftime('%Y-%m-%d')
log_json["logged_time"] = datetime.strftime('%Y-%m-%d %H:%m:%S')
log_json["host"] = message.hostname
log_json["process"] = splited_msg[0].strip() + ']'
if (message.severity.name == "informational"):
log_json["severity"] = "INFO"
else:
log_json["severity"] = message.severity.name.upper()
log_json["message"] = splited_msg[1].strip()
print(log_json)
return log_json
# <30>Jun 5 02:36:06 stefan-pc systemd: Started Syslog service for accepting logs from journald.
if (format_found):
log_format_raw = log_format_regex.encode('unicode_escape').decode()
pattern = re.compile(log_format_regex)
match = pattern.match(log)
else:
print("Bad or no logging format provided.")
return None
if (match == None):
print("Bad configuration(log format regex) - No match")
print(log)
return None
else:
log_json = {}
# time = match.group("time").split(",")[0]
if ("date" in pattern.groupindex and "time" in pattern.groupindex):
datetime = match.group("date") + " " + match.group("time")
date = parser.parse(datetime)
datetime_formated = date.strftime('%d-%m-%Y %H:%M:%S')
date_formated = date.strftime('%d-%m-%Y')
log_json["logged_date"] = date_formated
log_json["logged_time"] = datetime_formated
if ("host" in pattern.groupindex):
log_json["host"] = match.group("host")
if ("process" in pattern.groupindex):
log_json["process"] = match.group("process")
if ("severity" in pattern.groupindex):
log_json["severity"] = match.group("severity")
if ("message" in pattern.groupindex):
log_json["message"] = match.group("message")
return log_json
def test_parse_leap_day_in_leap_year(self, current_year):
data = b'<165>Feb 29 19:56:43 localhost foobar'
fake_date = '{:d}-01-01'.format(current_year)
expected_timestamp = datetime(current_year, 2, 29, 19, 56, 43)
with freeze_time(fake_date):
actual = parse(data)
self.assertEqual(actual.timestamp, expected_timestamp)
def orchestrate(self):
if config.CONFIG.isWindows():
return True
s = self.getLine()
while (s != None):
message = syslogmp.parse(s)
if (message):
msg = message.message.decode("utf-8")
log.L.debug("syslog: " + repr(msg))
# 127.0.0.1:45940 [04/Sep/2018:18:40:06.448] ssltunnel~ b-preproxy/s-proxy 1/0/0/12/+12 200 +37 - - ---- 2/2/1/1/0 0/0 {1ACCCCC|TO_PROXY_CONNECT|} "CONNECT www.idnes.cz:443 HTTP/1.1"
p = re.search(
"(^\d*\.\d*\.\d*\.\d*)|(\[(.*)\]):(\d*) " # Host and port
+ "\[(.*)\] " # Date
+ "(\w*)~? " # Frontend
+ "(b-\w*)/(<?s-\w*>?) " # Backend/Server
+ "(.\d*/.\d*/.\d*/.\d*/.\d*) " # Times
+ "(\d*) " # State
+ ".*" # Not needed
+ "{(.*)\|(.*)\|(.*)} " # authid, reason, overlimit
+ '"(.*) (.*) (.*)"',
msg)
if (p):
ip = p.group(1)
port = p.group(2)
dte = p.group(3)
frontend = p.group(4)
backend = p.group(5)
server = p.group(6)
code = p.group(8)
authid = p.group(9)
reason = p.group(10)
overlimit = p.group(11)
method = p.group(12)
uri = p.group(13)
proto = p.group(14)
if (server == "s-proxy"):
sessions.SESSIONS.add(authid,
ip,
port,
method=method,
uri=uri,
proto=proto)
else:
log.L.debug("Ignoring haproxy log")
else:
p = re.search(
"(^\d*\.\d*\.\d*\.\d*):(\d*) \[(.*)\] ssltunnel", msg)
if (p):
log.L.info("Ignoring haproxy log: " + repr(msg))
s = self.getLine()
def handle(self):
try:
data = self.request[0]
message = syslogmp.parse(data)
except ValueError:
log('Invalid message received from {}:{:d}.',
*self.client_address)
return
port = self.server.get_port()
log('Received message from {0[0]}:{0[1]:d} on port {1:d} -> {2}',
self.client_address, port, format_message_for_log(message))
syslog_message_received.send(port,
source_address=self.client_address,
message=message)
def parse(self, rawdata):
"""
Parse any of the two types of Syslog Formats
and stores the result in this object
"""
# RFC 5424
try:
msg = RFC5452SyslogMessage.parse(rawdata.decode()).as_dict()
self.id = msg.get('msgid')
self.message = msg.get('msg', '').rstrip(' \t\r\n\0')
self.facility = self.facilities[msg['facility']]
self.facility_name = msg['facility']
self.severity = self.severities[msg['severity']]
self.severity_name = msg['severity']
self.identifier = msg.get('appname')
self.pid = msg.get('procid')
self.hostname = msg.get('hostname')
return True
except (ParseError, ValueError):
# log err ?
pass
# RFC 3164
try:
if len(rawdata) > 1024:
self.logger.warning('truncating message ({} > 1024)'.format(
len(rawdata)))
rawdata = rawdata[:1024]
# truncate to 1024
msg = syslogmp.parse(rawdata)
self.id = None
(self.identifier, self.pid,
self.message) = self.parse_3164_msg(msg.message.decode().strip())
self.message = self.message.rstrip(' \t\r\n\0')
self.facility = int(msg.facility.value)
self.facility_name = str(msg.facility).lower().split('.')[-1]
self.severity = int(msg.severity.value)
self.severity_name = str(msg.severity).lower().split('.')[-1]
self.hostname = msg.hostname
return True
except (syslogmp.parser.MessageFormatError, ValueError):
# log err ?
pass
return False
def test_parse(
self,
data,
expected_facility,
expected_facility_description,
expected_severity,
expected_timestamp,
expected_hostname,
expected_message):
"""Test parsing of a syslog message."""
actual = parse(data)
self.assertEqual(actual.facility, expected_facility)
self.assertEqual(actual.facility.description, expected_facility_description)
self.assertEqual(actual.severity, expected_severity)
self.assertEqual(actual.timestamp, expected_timestamp)
self.assertEqual(actual.hostname, expected_hostname)
self.assertEqual(actual.message, expected_message)
def parse(self, line):
"""Turns the line into a LogEntry"""
# syslogmp expects a byte string
line_bytes = str.encode(line)
syslog_msg = parse(line_bytes)
if b'filterlog:' in syslog_msg.message:
return FirewallLogEntry(syslog_msg.message.decode('utf-8'),
self.resolver)
elif b'dhcpd:' in syslog_msg.message:
e = DHCPDLogEntry(line)
# We only care about DHCPACK and DHCPRequest. All other lease logs
# are not interesting.
if 'DHCPACK' in str(e) or 'DHCPREQUEST' in str(e):
return e
else:
return None
else:
return LogEntry(f'Cannot process line {line}')
def test_parse_erroneous_message(self, data):
with self.assertRaises(MessageFormatError):
parse(data)
def test_parse_message_just_not_too_long(self):
data = create_long_data(1024)
self.assertEqual(len(data), 1024)
parse(data)
