def test_integration(*args):
try:
working_dir = tempfile.mkdtemp()
s_path = os.path.join(working_dir, 'sock')
s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
s.bind(s_path)
r = logging.makeLogRecord({
'name': 'root',
'msg': 'A Message',
'levelname': 'DEBUG'
})
h = logging.handlers.SysLogHandler(
s_path, facility=logging.handlers.SysLogHandler.LOG_USER)
h.setFormatter(RFC5424Formatter())
h.handle(r)
s.settimeout(1)
msg_body = s.recv(1024)
assert msg_body == b'<15>1 1970-01-01T00:00:00Z the_host root 1 - - A Message\x00'
fields = SyslogMessage.parse(msg_body.decode('utf-8'))
assert fields.severity == SyslogSeverity.debug
assert fields.facility == SyslogFacility.user
assert fields.version == 1
assert fields.hostname == 'the_host'
assert fields.appname == 'root'
assert fields.msg == 'A Message\x00'
assert fields.procid == 1
finally:
shutil.rmtree(working_dir)
def datagramReceived(self, data, addr):
data = bytes.decode(data.strip())
try:
message = SyslogMessage.parse(data).as_dict()
message['hostname'] = reactor.singleton.LOG_PREFIX
msg = message.get('msg', None)
if msg not in ['', ' ', None]:
reactor.singleton.COUNT.RECEIVED += 1
appName = message.get('appname', 'default')
level = message.get('severity', 'info')
if appName not in reactor.singleton.APP_LOGS:
openLog(appName)
log_msg = f"{message.get('timestamp', datetime.now())} - {level} - {msg}"
reactor.singleton.APP_LOGS[appName].log(
reactor.singleton.LOG_LEVELS.get(level), log_msg)
reactor.singleton.COUNT.PROCESSED += 1
if reactor.singleton.ES.HOST is not None:
reactor.singleton.ES.QUEUE.put(
dict(
index=
f'log_{reactor.singleton.LOG_PREFIX}-{datetime.now():%Y.%m.%d}',
doc_type='log',
body=message))
except ParseError as e:
print(f"Error {e.__class__.__name__} - {getattr(e, 'message', e)}",
flush=True)
def parse_rfc_5424_format(
log_message: bytes) -> Optional[SyslogMessageExtract]:
"""
Receives a log message which is in RFC 5424 format. Parses it into SyslogMessageExtract data class object
Args:
log_message (bytes): Syslog message.
Returns:
(Optional[SyslogMessageExtract]): Extraction data class
"""
try:
syslog_message: SyslogMessage = SyslogMessage.parse(
log_message.decode('utf-8'))
except ParseError:
return None
return SyslogMessageExtract(app_name=syslog_message.appname,
facility=syslog_message.facility.name,
host_name=syslog_message.hostname,
msg=syslog_message.msg,
msg_id=syslog_message.msgid,
process_id=syslog_message.procid,
sd=syslog_message.sd,
severity=syslog_message.severity.name,
timestamp=syslog_message.timestamp,
version=syslog_message.version,
occurred=syslog_message.timestamp)
def test_as_dict(input_line, expected):
m = SyslogMessage.parse(input_line)
dictified = m.as_dict()
expected_dict = expected._asdict()
expected_dict['severity'] = expected_dict['severity'].name
expected_dict['facility'] = expected_dict['facility'].name
assert dictified == expected_dict
def syslog_verify(self, data, facility, severity, appname, msgid, sd=None,
msg=None):
"""
Verify syslog message
:param data: syslog message
:param facility: facility value
:param severity: severity level
:param appname: application name that originate message
:param msgid: message identifier
:param sd: structured data (optional)
:param msg: free-form message (optional)
"""
message = data.decode('utf-8')
if sd is None:
sd = {}
try:
message = SyslogMessage.parse(message)
except ParseError as e:
self.logger.error(e)
raise
else:
self.assertEqual(message.facility, facility)
self.assertEqual(message.severity, severity)
self.assertEqual(message.appname, appname)
self.assertEqual(message.msgid, msgid)
self.assertEqual(message.msg, msg)
self.assertEqual(message.sd, sd)
self.assertEqual(message.version, 1)
self.assertEqual(message.hostname, self.pg0.local_ip4)
def test_as_dict(input_line, expected):
m = SyslogMessage.parse(input_line)
dictified = m.as_dict()
expected_dict = expected._asdict()
expected_dict['severity'] = expected_dict['severity'].name
expected_dict['facility'] = expected_dict['facility'].name
assert dictified == expected_dict
def syslog_verify(
self, data, facility, severity, appname, msgid, sd=None, msg=None
):
"""
Verify syslog message
:param data: syslog message
:param facility: facility value
:param severity: severity level
:param appname: application name that originate message
:param msgid: message identifier
:param sd: structured data (optional)
:param msg: free-form message (optional)
"""
message = data.decode("utf-8")
if sd is None:
sd = {}
try:
message = SyslogMessage.parse(message)
except ParseError as e:
self.logger.error(e)
raise
else:
self.assertEqual(message.facility, facility)
self.assertEqual(message.severity, severity)
self.assertEqual(message.appname, appname)
self.assertEqual(message.msgid, msgid)
self.assertEqual(message.msg, msg)
self.assertEqual(message.sd, sd)
self.assertEqual(message.version, 1)
self.assertEqual(message.hostname, self.pg0.local_ip4)
async def json(self) -> List[dict]:
if not hasattr(self, "_json"):
self._json = []
async for line in self.read_lines():
print("line=", line)
line = line.decode("utf-8")
msg = SyslogMessage.parse(line)
self._json.append(msg.as_dict())
return self._json
def test_vector(input_line, expected):
parsed = SyslogMessage.parse(input_line)
assert parsed.severity == expected.severity
assert parsed.facility == expected.facility
assert parsed.version == expected.version
assert parsed.timestamp == expected.timestamp
assert parsed.hostname == expected.hostname
assert parsed.appname == expected.appname
assert parsed.procid == expected.procid
assert parsed.msgid == expected.msgid
assert parsed.msg == expected.msg
assert parsed.sd == expected.sd
def test_vector(input_line, expected):
parsed = SyslogMessage.parse(input_line)
assert parsed.severity == expected.severity
assert parsed.facility == expected.facility
assert parsed.version == expected.version
assert parsed.timestamp == expected.timestamp
assert parsed.hostname == expected.hostname
assert parsed.appname == expected.appname
assert parsed.procid == expected.procid
assert parsed.msgid == expected.msgid
assert parsed.msg == expected.msg
assert parsed.sd == expected.sd
def parse(self, rawdata):
"""
Parse any of the two types of Syslog Formats
and stores the result in this object
"""
# RFC 5424
try:
msg = RFC5452SyslogMessage.parse(rawdata.decode()).as_dict()
self.id = msg.get('msgid')
self.message = msg.get('msg', '').rstrip(' \t\r\n\0')
self.facility = self.facilities[msg['facility']]
self.facility_name = msg['facility']
self.severity = self.severities[msg['severity']]
self.severity_name = msg['severity']
self.identifier = msg.get('appname')
self.pid = msg.get('procid')
self.hostname = msg.get('hostname')
return True
except (ParseError, ValueError):
# log err ?
pass
# RFC 3164
try:
if len(rawdata) > 1024:
self.logger.warning('truncating message ({} > 1024)'.format(
len(rawdata)))
rawdata = rawdata[:1024]
# truncate to 1024
msg = syslogmp.parse(rawdata)
self.id = None
(self.identifier, self.pid,
self.message) = self.parse_3164_msg(msg.message.decode().strip())
self.message = self.message.rstrip(' \t\r\n\0')
self.facility = int(msg.facility.value)
self.facility_name = str(msg.facility).lower().split('.')[-1]
self.severity = int(msg.severity.value)
self.severity_name = str(msg.severity).lower().split('.')[-1]
self.hostname = msg.hostname
return True
except (syslogmp.parser.MessageFormatError, ValueError):
# log err ?
pass
return False
def main():
host = '0.0.0.0'
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--protocol',
required=True,
help='Protocol: udp/tcp')
parser.add_argument('--port', required=True, help='Port number')
args = parser.parse_args()
protocol = args.protocol.lower()
if protocol == 'udp':
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
elif protocol == 'tcp':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
else:
print("error, protocol not UDP or TCP")
sys.exit(1)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
port = int(args.port)
print("Listening on %s %s:%s" % (protocol, host, port))
s.bind((host, port))
if protocol == 'udp':
connection = s
else:
s.listen(1)
# Wait for a connection
print('waiting for a connection')
connection, client_address = s.accept()
print('accepted connection from: %s %s' % (client_address))
while True:
message = connection.recv(4096)
# Technically, messages are only UTF-8 if they have a BOM; otherwise they're binary. However, I'm not
# aware of any Syslog servers that handle that. *shrug*
message = message.decode('utf-8')
try:
message = SyslogMessage.parse(message)
print(json.dumps(message.as_dict()))
except ParseError as e:
print(e, file=sys.stderr)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-B', '--bind-path', required=True,
help='Path at which to bind a Datagram-mode UNIX domain socket')
args = parser.parse_args()
s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
temp_name = args.bind_path + '.' + str(os.getpid())
s.bind(temp_name)
os.rename(temp_name, args.bind_path)
while True:
message = s.recv(4096)
# Technically, messages are only UTF-8 if they have a BOM; otherwise they're binary. However, I'm not
# aware of any Syslog servers that handle that. *shrug*
message = message.decode('utf-8')
try:
message = SyslogMessage.parse(message)
print(json.dumps(message.as_dict()))
except ParseError as e:
print(e, file=sys.stderr)
def verify_syslog_apmadd(self, data, isaddr, isport, xsaddr, xsport,
sv6enc, proto):
message = data.decode("utf-8")
try:
message = SyslogMessage.parse(message)
except ParseError as e:
self.logger.error(e)
else:
self.assertEqual(message.severity, SyslogSeverity.info)
self.assertEqual(message.appname, "NAT")
self.assertEqual(message.msgid, "APMADD")
sd_params = message.sd.get("napmap")
self.assertTrue(sd_params is not None)
self.assertEqual(sd_params.get("IATYP"), "IPv4")
self.assertEqual(sd_params.get("ISADDR"), isaddr)
self.assertEqual(sd_params.get("ISPORT"), "%d" % isport)
self.assertEqual(sd_params.get("XATYP"), "IPv4")
self.assertEqual(sd_params.get("XSADDR"), xsaddr)
self.assertEqual(sd_params.get("XSPORT"), "%d" % xsport)
self.assertEqual(sd_params.get("PROTO"), "%d" % proto)
self.assertTrue(sd_params.get("SSUBIX") is not None)
self.assertEqual(sd_params.get("SV6ENC"), sv6enc)
def verify_syslog_apmadd(self, data, isaddr, isport, xsaddr, xsport,
sv6enc, proto):
message = data.decode('utf-8')
try:
message = SyslogMessage.parse(message)
except ParseError as e:
self.logger.error(e)
else:
self.assertEqual(message.severity, SyslogSeverity.info)
self.assertEqual(message.appname, 'NAT')
self.assertEqual(message.msgid, 'APMADD')
sd_params = message.sd.get('napmap')
self.assertTrue(sd_params is not None)
self.assertEqual(sd_params.get('IATYP'), 'IPv4')
self.assertEqual(sd_params.get('ISADDR'), isaddr)
self.assertEqual(sd_params.get('ISPORT'), "%d" % isport)
self.assertEqual(sd_params.get('XATYP'), 'IPv4')
self.assertEqual(sd_params.get('XSADDR'), xsaddr)
self.assertEqual(sd_params.get('XSPORT'), "%d" % xsport)
self.assertEqual(sd_params.get('PROTO'), "%d" % proto)
self.assertTrue(sd_params.get('SSUBIX') is not None)
self.assertEqual(sd_params.get('SV6ENC'), sv6enc)
def test_integration(*args):
try:
working_dir = tempfile.mkdtemp()
s_path = os.path.join(working_dir, 'sock')
s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
s.bind(s_path)
r = logging.makeLogRecord({'name': 'root', 'msg': 'A Message', 'levelname': 'DEBUG'})
h = logging.handlers.SysLogHandler(s_path, facility=logging.handlers.SysLogHandler.LOG_USER)
h.setFormatter(RFC5424Formatter())
h.handle(r)
s.settimeout(1)
msg_body = s.recv(1024)
assert msg_body == b'<15>1 1970-01-01T00:00:00Z the_host root 1 - - A Message\x00'
fields = SyslogMessage.parse(msg_body.decode('utf-8'))
assert fields.severity == SyslogSeverity.debug
assert fields.facility == SyslogFacility.user
assert fields.version == 1
assert fields.hostname == 'the_host'
assert fields.appname == 'root'
assert fields.msg == 'A Message\x00'
assert fields.procid == 1
finally:
shutil.rmtree(working_dir)
def parse(self, line, parseSocket):
if parseSocket:
try:
payload = SyslogMessage.parse(line)
except:
raise ParseError
else:
parsed = self.__pattern.parseString(line)
payload = {}
payload["timestamp"] = int(
time.mktime(
time.strptime(
str(time.localtime().tm_year) + " " +
' '.join(parsed[:3]), "%Y %b %d %H:%M:%S")))
payload["hostname"] = parsed[3]
payload["appname"] = parsed[4]
if len(parsed) == 7:
payload["pid"] = parsed[5]
payload["msg"] = parsed[-1]
return payload
def test_emitter_round_trip(input_line):
m = SyslogMessage.parse(input_line)
assert str(m) == input_line
def test_dumping_with_bad_pri_fails():
m = SyslogMessage(facility=SyslogFacility.unknown,
severity=SyslogSeverity.emerg)
with pytest.raises(ValueError):
str(m)
config.print_env_vars()
# setup shutdown handlers
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]:
signal.signal(sig, shutdown_handler)
UDPSock = socket(AF_INET, SOCK_DGRAM)
UDPSock.bind((config.syslog_agent_host, config.syslog_agent_port))
# Receive messages - processing this could be a bottleneck - place into thread if so
while True:
# if a log message gets truncated that's ok so long as the core syslog info is presented
data, addr = UDPSock.recvfrom(4096)
message = None
try:
message = SyslogMessage.parse(data.decode('utf-8'))
except ParseError as e:
print(e)
message = message.as_dict()
print(message)
if config.active_sentinel:
# appname in msg is the short container id
series_name = create_container_series(
base_url=config.base_url,
username=config.username,
user_apikey=config.syslog_agent_user_apikey,
space_name=config.space,
series_name=message['appname'])
send_msg(json.dumps(message),
space=config.topic_name,
def test_emitter():
m = SyslogMessage(facility=SyslogFacility.cron,
severity=SyslogSeverity.info)
assert '<78>1 - - - - - -' == str(m)
def test_emitter_with_unix_timestamp():
m = SyslogMessage(facility=SyslogFacility.kern,
severity=SyslogSeverity.emerg,
timestamp=0)
assert '<0>1 1970-01-01T00:00:00Z - - - - -' == str(m)
def test_repr_does_not_raise():
m = SyslogMessage(facility=SyslogFacility.cron,
severity=SyslogSeverity.info)
repr(m)
def test_unparseable():
with pytest.raises(ParseError):
SyslogMessage.parse('garbage')
def test_unparseable():
with pytest.raises(ParseError):
SyslogMessage.parse('garbage')
def test_emitter_round_trip(input_line):
m = SyslogMessage.parse(input_line)
assert str(m) == input_line
