def datagramReceived(self, data, addr): data = bytes.decode(data.strip()) try: message = SyslogMessage.parse(data).as_dict() message['hostname'] = reactor.singleton.LOG_PREFIX msg = message.get('msg', None) if msg not in ['', ' ', None]: reactor.singleton.COUNT.RECEIVED += 1 appName = message.get('appname', 'default') level = message.get('severity', 'info') if appName not in reactor.singleton.APP_LOGS: openLog(appName) log_msg = f"{message.get('timestamp', datetime.now())} - {level} - {msg}" reactor.singleton.APP_LOGS[appName].log( reactor.singleton.LOG_LEVELS.get(level), log_msg) reactor.singleton.COUNT.PROCESSED += 1 if reactor.singleton.ES.HOST is not None: reactor.singleton.ES.QUEUE.put( dict( index= f'log_{reactor.singleton.LOG_PREFIX}-{datetime.now():%Y.%m.%d}', doc_type='log', body=message)) except ParseError as e: print(f"Error {e.__class__.__name__} - {getattr(e, 'message', e)}", flush=True)
def syslog_verify( self, data, facility, severity, appname, msgid, sd=None, msg=None ): """ Verify syslog message :param data: syslog message :param facility: facility value :param severity: severity level :param appname: application name that originate message :param msgid: message identifier :param sd: structured data (optional) :param msg: free-form message (optional) """ message = data.decode("utf-8") if sd is None: sd = {} try: message = SyslogMessage.parse(message) except ParseError as e: self.logger.error(e) raise else: self.assertEqual(message.facility, facility) self.assertEqual(message.severity, severity) self.assertEqual(message.appname, appname) self.assertEqual(message.msgid, msgid) self.assertEqual(message.msg, msg) self.assertEqual(message.sd, sd) self.assertEqual(message.version, 1) self.assertEqual(message.hostname, self.pg0.local_ip4)
def test_as_dict(input_line, expected): m = SyslogMessage.parse(input_line) dictified = m.as_dict() expected_dict = expected._asdict() expected_dict['severity'] = expected_dict['severity'].name expected_dict['facility'] = expected_dict['facility'].name assert dictified == expected_dict
def syslog_verify(self, data, facility, severity, appname, msgid, sd=None, msg=None): """ Verify syslog message :param data: syslog message :param facility: facility value :param severity: severity level :param appname: application name that originate message :param msgid: message identifier :param sd: structured data (optional) :param msg: free-form message (optional) """ message = data.decode('utf-8') if sd is None: sd = {} try: message = SyslogMessage.parse(message) except ParseError as e: self.logger.error(e) raise else: self.assertEqual(message.facility, facility) self.assertEqual(message.severity, severity) self.assertEqual(message.appname, appname) self.assertEqual(message.msgid, msgid) self.assertEqual(message.msg, msg) self.assertEqual(message.sd, sd) self.assertEqual(message.version, 1) self.assertEqual(message.hostname, self.pg0.local_ip4)
def parse_rfc_5424_format( log_message: bytes) -> Optional[SyslogMessageExtract]: """ Receives a log message which is in RFC 5424 format. Parses it into SyslogMessageExtract data class object Args: log_message (bytes): Syslog message. Returns: (Optional[SyslogMessageExtract]): Extraction data class """ try: syslog_message: SyslogMessage = SyslogMessage.parse( log_message.decode('utf-8')) except ParseError: return None return SyslogMessageExtract(app_name=syslog_message.appname, facility=syslog_message.facility.name, host_name=syslog_message.hostname, msg=syslog_message.msg, msg_id=syslog_message.msgid, process_id=syslog_message.procid, sd=syslog_message.sd, severity=syslog_message.severity.name, timestamp=syslog_message.timestamp, version=syslog_message.version, occurred=syslog_message.timestamp)
def test_integration(*args): try: working_dir = tempfile.mkdtemp() s_path = os.path.join(working_dir, 'sock') s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) s.bind(s_path) r = logging.makeLogRecord({ 'name': 'root', 'msg': 'A Message', 'levelname': 'DEBUG' }) h = logging.handlers.SysLogHandler( s_path, facility=logging.handlers.SysLogHandler.LOG_USER) h.setFormatter(RFC5424Formatter()) h.handle(r) s.settimeout(1) msg_body = s.recv(1024) assert msg_body == b'<15>1 1970-01-01T00:00:00Z the_host root 1 - - A Message\x00' fields = SyslogMessage.parse(msg_body.decode('utf-8')) assert fields.severity == SyslogSeverity.debug assert fields.facility == SyslogFacility.user assert fields.version == 1 assert fields.hostname == 'the_host' assert fields.appname == 'root' assert fields.msg == 'A Message\x00' assert fields.procid == 1 finally: shutil.rmtree(working_dir)
async def json(self) -> List[dict]: if not hasattr(self, "_json"): self._json = [] async for line in self.read_lines(): print("line=", line) line = line.decode("utf-8") msg = SyslogMessage.parse(line) self._json.append(msg.as_dict()) return self._json
def test_vector(input_line, expected): parsed = SyslogMessage.parse(input_line) assert parsed.severity == expected.severity assert parsed.facility == expected.facility assert parsed.version == expected.version assert parsed.timestamp == expected.timestamp assert parsed.hostname == expected.hostname assert parsed.appname == expected.appname assert parsed.procid == expected.procid assert parsed.msgid == expected.msgid assert parsed.msg == expected.msg assert parsed.sd == expected.sd
def parse(self, rawdata): """ Parse any of the two types of Syslog Formats and stores the result in this object """ # RFC 5424 try: msg = RFC5452SyslogMessage.parse(rawdata.decode()).as_dict() self.id = msg.get('msgid') self.message = msg.get('msg', '').rstrip(' \t\r\n\0') self.facility = self.facilities[msg['facility']] self.facility_name = msg['facility'] self.severity = self.severities[msg['severity']] self.severity_name = msg['severity'] self.identifier = msg.get('appname') self.pid = msg.get('procid') self.hostname = msg.get('hostname') return True except (ParseError, ValueError): # log err ? pass # RFC 3164 try: if len(rawdata) > 1024: self.logger.warning('truncating message ({} > 1024)'.format( len(rawdata))) rawdata = rawdata[:1024] # truncate to 1024 msg = syslogmp.parse(rawdata) self.id = None (self.identifier, self.pid, self.message) = self.parse_3164_msg(msg.message.decode().strip()) self.message = self.message.rstrip(' \t\r\n\0') self.facility = int(msg.facility.value) self.facility_name = str(msg.facility).lower().split('.')[-1] self.severity = int(msg.severity.value) self.severity_name = str(msg.severity).lower().split('.')[-1] self.hostname = msg.hostname return True except (syslogmp.parser.MessageFormatError, ValueError): # log err ? pass return False
def main(): host = '0.0.0.0' parser = argparse.ArgumentParser() parser.add_argument('-p', '--protocol', required=True, help='Protocol: udp/tcp') parser.add_argument('--port', required=True, help='Port number') args = parser.parse_args() protocol = args.protocol.lower() if protocol == 'udp': s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) elif protocol == 'tcp': s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) else: print("error, protocol not UDP or TCP") sys.exit(1) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) port = int(args.port) print("Listening on %s %s:%s" % (protocol, host, port)) s.bind((host, port)) if protocol == 'udp': connection = s else: s.listen(1) # Wait for a connection print('waiting for a connection') connection, client_address = s.accept() print('accepted connection from: %s %s' % (client_address)) while True: message = connection.recv(4096) # Technically, messages are only UTF-8 if they have a BOM; otherwise they're binary. However, I'm not # aware of any Syslog servers that handle that. *shrug* message = message.decode('utf-8') try: message = SyslogMessage.parse(message) print(json.dumps(message.as_dict())) except ParseError as e: print(e, file=sys.stderr)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-B', '--bind-path', required=True, help='Path at which to bind a Datagram-mode UNIX domain socket') args = parser.parse_args() s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) temp_name = args.bind_path + '.' + str(os.getpid()) s.bind(temp_name) os.rename(temp_name, args.bind_path) while True: message = s.recv(4096) # Technically, messages are only UTF-8 if they have a BOM; otherwise they're binary. However, I'm not # aware of any Syslog servers that handle that. *shrug* message = message.decode('utf-8') try: message = SyslogMessage.parse(message) print(json.dumps(message.as_dict())) except ParseError as e: print(e, file=sys.stderr)
def verify_syslog_apmadd(self, data, isaddr, isport, xsaddr, xsport, sv6enc, proto): message = data.decode("utf-8") try: message = SyslogMessage.parse(message) except ParseError as e: self.logger.error(e) else: self.assertEqual(message.severity, SyslogSeverity.info) self.assertEqual(message.appname, "NAT") self.assertEqual(message.msgid, "APMADD") sd_params = message.sd.get("napmap") self.assertTrue(sd_params is not None) self.assertEqual(sd_params.get("IATYP"), "IPv4") self.assertEqual(sd_params.get("ISADDR"), isaddr) self.assertEqual(sd_params.get("ISPORT"), "%d" % isport) self.assertEqual(sd_params.get("XATYP"), "IPv4") self.assertEqual(sd_params.get("XSADDR"), xsaddr) self.assertEqual(sd_params.get("XSPORT"), "%d" % xsport) self.assertEqual(sd_params.get("PROTO"), "%d" % proto) self.assertTrue(sd_params.get("SSUBIX") is not None) self.assertEqual(sd_params.get("SV6ENC"), sv6enc)
def verify_syslog_apmadd(self, data, isaddr, isport, xsaddr, xsport, sv6enc, proto): message = data.decode('utf-8') try: message = SyslogMessage.parse(message) except ParseError as e: self.logger.error(e) else: self.assertEqual(message.severity, SyslogSeverity.info) self.assertEqual(message.appname, 'NAT') self.assertEqual(message.msgid, 'APMADD') sd_params = message.sd.get('napmap') self.assertTrue(sd_params is not None) self.assertEqual(sd_params.get('IATYP'), 'IPv4') self.assertEqual(sd_params.get('ISADDR'), isaddr) self.assertEqual(sd_params.get('ISPORT'), "%d" % isport) self.assertEqual(sd_params.get('XATYP'), 'IPv4') self.assertEqual(sd_params.get('XSADDR'), xsaddr) self.assertEqual(sd_params.get('XSPORT'), "%d" % xsport) self.assertEqual(sd_params.get('PROTO'), "%d" % proto) self.assertTrue(sd_params.get('SSUBIX') is not None) self.assertEqual(sd_params.get('SV6ENC'), sv6enc)
def test_integration(*args): try: working_dir = tempfile.mkdtemp() s_path = os.path.join(working_dir, 'sock') s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) s.bind(s_path) r = logging.makeLogRecord({'name': 'root', 'msg': 'A Message', 'levelname': 'DEBUG'}) h = logging.handlers.SysLogHandler(s_path, facility=logging.handlers.SysLogHandler.LOG_USER) h.setFormatter(RFC5424Formatter()) h.handle(r) s.settimeout(1) msg_body = s.recv(1024) assert msg_body == b'<15>1 1970-01-01T00:00:00Z the_host root 1 - - A Message\x00' fields = SyslogMessage.parse(msg_body.decode('utf-8')) assert fields.severity == SyslogSeverity.debug assert fields.facility == SyslogFacility.user assert fields.version == 1 assert fields.hostname == 'the_host' assert fields.appname == 'root' assert fields.msg == 'A Message\x00' assert fields.procid == 1 finally: shutil.rmtree(working_dir)
def parse(self, line, parseSocket): if parseSocket: try: payload = SyslogMessage.parse(line) except: raise ParseError else: parsed = self.__pattern.parseString(line) payload = {} payload["timestamp"] = int( time.mktime( time.strptime( str(time.localtime().tm_year) + " " + ' '.join(parsed[:3]), "%Y %b %d %H:%M:%S"))) payload["hostname"] = parsed[3] payload["appname"] = parsed[4] if len(parsed) == 7: payload["pid"] = parsed[5] payload["msg"] = parsed[-1] return payload
config.print_env_vars() # setup shutdown handlers for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGQUIT]: signal.signal(sig, shutdown_handler) UDPSock = socket(AF_INET, SOCK_DGRAM) UDPSock.bind((config.syslog_agent_host, config.syslog_agent_port)) # Receive messages - processing this could be a bottleneck - place into thread if so while True: # if a log message gets truncated that's ok so long as the core syslog info is presented data, addr = UDPSock.recvfrom(4096) message = None try: message = SyslogMessage.parse(data.decode('utf-8')) except ParseError as e: print(e) message = message.as_dict() print(message) if config.active_sentinel: # appname in msg is the short container id series_name = create_container_series( base_url=config.base_url, username=config.username, user_apikey=config.syslog_agent_user_apikey, space_name=config.space, series_name=message['appname']) send_msg(json.dumps(message), space=config.topic_name,
def test_emitter_round_trip(input_line): m = SyslogMessage.parse(input_line) assert str(m) == input_line
def test_unparseable(): with pytest.raises(ParseError): SyslogMessage.parse('garbage')