def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None # self.mysql_logger = None self.json_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean("sqlite", "enabled"): self.sqlite_logger = SQLiteLogger() # if config.getboolean('mysql', 'enabled'): # host = config.get('mysql', 'host') # port = config.getint('mysql', 'port') # db = config.get('mysql', 'db') # username = config.get('mysql', 'username') # passphrase = config.get('mysql', 'passphrase') # logdevice = config.get('mysql', 'device') # logsocket = config.get('mysql', 'socket') # sensorid = config.get('common', 'sensorid') # self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid) if config.getboolean("json", "enabled"): filename = config.get("json", "filename") sensorid = config.get("common", "sensorid") self.json_logger = JsonLogger(filename, sensorid, public_ip) if config.getboolean("hpfriends", "enabled"): host = config.get("hpfriends", "host") port = config.getint("hpfriends", "port") ident = config.get("hpfriends", "ident") secret = config.get("hpfriends", "secret") channels = eval(config.get("hpfriends", "channels")) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e) self.friends_feeder = None if config.getboolean("syslog", "enabled"): host = config.get("syslog", "host") port = config.getint("syslog", "port") facility = config.get("syslog", "facility") logdevice = config.get("syslog", "device") logsocket = config.get("syslog", "socket") self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean("taxii", "enabled"): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True
def test_hpfriends(self): """ Objective: Test if data can be published to hpfriends without errors. """ host = 'hpfriends.honeycloud.net' port = 20000 ident = 'HBmU08rR' secret = 'XDNNuMGYUuWFaWyi' channels = ["test.test", ] hpf = HPFriendsLogger(host, port, ident, secret, channels) error_message = hpf.log('some some test data') self.assertIsNone(error_message, 'Unexpected error message: {0}'.format(error_message))
def test_hpfriends(self): """ Objective: Test if data can be published to hpfriends without errors. """ host = 'hpfriends.honeycloud.net' port = 20000 ident = 'HBmU08rR' secret = 'XDNNuMGYUuWFaWyi' channels = [ "test.test", ] hpf = HPFriendsLogger(host, port, ident, secret, channels) error_message = hpf.log('some some test data') self.assertIsNone( error_message, 'Unexpected error message: {0}'.format(error_message))
def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None self.mysql_logger = None self.json_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() if config.getboolean('mysql', 'enabled'): host = config.get('mysql', 'host') port = config.getint('mysql', 'port') db = config.get('mysql', 'db') username = config.get('mysql', 'username') passphrase = config.get('mysql', 'passphrase') logdevice = config.get('mysql', 'device') logsocket = config.get('mysql', 'socket') sensorid = config.get('common', 'sensorid') self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid) if config.getboolean('json', 'enabled'): filename = config.get('json', 'filename') sensorid = config.get('common', 'sensorid') self.json_logger = JsonLogger(filename, sensorid, public_ip) if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True
def test_hpfriends(self): """ Objective: Test if data can be published to hpfriends without errors. """ host = "hpfriends.honeycloud.net" port = 20000 ident = "HBmU08rR" secret = "XDNNuMGYUuWFaWyi" channels = [ "test.test", ] hpf = HPFriendsLogger(host, port, ident, secret, channels) gevent.sleep(0.5) error_message = hpf.log("some some test data") gevent.sleep(0.5) self.assertIsNone( error_message, "Unexpected error message: {0}".format(error_message))
def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True
class LogWorker(object): def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None self.mysql_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() if config.getboolean('mysql', 'enabled'): host = config.get('mysql', 'host') port = config.getint('mysql', 'port') db = config.get('mysql', 'db') username = config.get('mysql', 'username') passphrase = config.get('mysql', 'passphrase') logdevice = config.get('mysql', 'device') logsocket = config.get('mysql', 'socket') sensorid = config.get('common', 'sensorid') self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid) if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True def _json_default(self, obj): if isinstance(obj, datetime): return obj.isoformat() elif isinstance(obj, uuid.UUID): return str(obj) else: return None def _process_sessions(self): sessions = self.session_manager._sessions try: session_timeout = self.config.get("session", "timeout") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): session_timeout = 5 for session in sessions: if len(session.data) > 0: sec_last_event = max(session.data) / 1000 else: sec_last_event = 0 sec_session_start = time.mktime(session.timestamp.timetuple()) sec_now = time.mktime(datetime.utcnow().timetuple()) if (sec_now - (sec_session_start + sec_last_event)) >= session_timeout: # TODO: We need to close sockets in this case logger.info('Session timed out: %s', session.id) session.set_ended() sessions.remove(session) def start(self): self.enabled = True while self.enabled: try: event = self.log_queue.get(timeout=2) except Empty: self._process_sessions() else: if self.public_ip: event["public_ip"] = self.public_ip if self.friends_feeder: self.friends_feeder.log(json.dumps( event, ensure_ascii=False, default=self._json_default)) if self.sqlite_logger: self.sqlite_logger.log(event) if self.mysql_logger: self.mysql_logger.log(event) if self.syslog_client: self.syslog_client.log(event) if self.taxii_logger: self.taxii_logger.log(event) def stop(self): self.enabled = False
class LogWorker(object): def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None # self.mysql_logger = None self.json_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() # if config.getboolean('mysql', 'enabled'): # host = config.get('mysql', 'host') # port = config.getint('mysql', 'port') # db = config.get('mysql', 'db') # username = config.get('mysql', 'username') # passphrase = config.get('mysql', 'passphrase') # logdevice = config.get('mysql', 'device') # logsocket = config.get('mysql', 'socket') # sensorid = config.get('common', 'sensorid') # self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid) if config.getboolean('json', 'enabled'): todaydate = datetime.now() todaystr = todaydate.strftime('%Y-%m-%d') filename = config.get('json', 'filename') + '.' + todaystr sensorid = config.get('common', 'sensorid') self.json_logger = JsonLogger(filename, sensorid, public_ip) if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True def _process_sessions(self): sessions = self.session_manager._sessions try: session_timeout = self.config.get("session", "timeout") except (configparser.NoSectionError, configparser.NoOptionError): session_timeout = 5 for session in sessions: if len(session.data) > 0: sec_last_event = max(session.data) / 1000 else: sec_last_event = 0 sec_session_start = time.mktime(session.timestamp.timetuple()) sec_now = time.mktime(datetime.utcnow().timetuple()) if (sec_now - (sec_session_start + sec_last_event)) >= float(session_timeout): # TODO: We need to close sockets in this case logger.info('Session timed out: %s', session.id) session.set_ended() sessions.remove(session) def start(self): self.enabled = True while self.enabled: try: event = self.log_queue.get(timeout=2) except Empty: self._process_sessions() else: if self.public_ip: event["public_ip"] = self.public_ip if self.friends_feeder: self.friends_feeder.log( json.dumps(event, default=json_default)) if self.sqlite_logger: self.sqlite_logger.log(event) # if self.mysql_logger: # self.mysql_logger.log(event) if self.syslog_client: self.syslog_client.log(event) if self.taxii_logger: self.taxii_logger.log(event) if self.json_logger: self.json_logger.log(event) def stop(self): self.enabled = False
class LogWorker(object): def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None # self.mysql_logger = None self.json_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None self.logstash_logger = None self.mongodb_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() # if config.getboolean('mysql', 'enabled'): # host = config.get('mysql', 'host') # port = config.getint('mysql', 'port') # db = config.get('mysql', 'db') # username = config.get('mysql', 'username') # passphrase = config.get('mysql', 'passphrase') # logdevice = config.get('mysql', 'device') # logsocket = config.get('mysql', 'socket') # sensorid = config.get('common', 'sensorid') # self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid) if config.getboolean('json', 'enabled'): filename = config.get('json', 'filename') sensorid = config.get('common', 'sensorid') self.json_logger = JsonLogger(filename, sensorid, public_ip) if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) if config.getboolean('logstash', 'enabled'): host = config.get('logstash', 'host') port = int(config.get('logstash', 'port')) self.logstash_logger = Logstash(host, port) if config.getboolean('mongodb', 'enabled'): host = config.get('mongodb', 'host') port = config.get('mongodb', 'port') db = config.get('mongodb', 'db') collection = config.get('mongodb', 'collection') sensorid = config.get('common', 'sensorid') self.mongodb_logger = MongodbLogger(host, port, db, collection, sensorid) self.enabled = True def _process_sessions(self): sessions = self.session_manager._sessions try: session_timeout = self.config.get("session", "timeout") except (configparser.NoSectionError, configparser.NoOptionError): session_timeout = 5 for session in sessions: if len(session.data) > 0: sec_last_event = max(session.data) / 1000 else: sec_last_event = 0 sec_session_start = time.mktime(session.timestamp.timetuple()) sec_now = time.mktime(datetime.utcnow().timetuple()) if (sec_now - (sec_session_start + sec_last_event)) >= float(session_timeout): # TODO: We need to close sockets in this case logger.info('Session timed out: %s', session.id) session.set_ended() sessions.remove(session) def start(self): self.enabled = True while self.enabled: # Changed from blocking to non-blocking to prevent endless blocking caused by rapid succession of events # Sleep added to approximately emulate previous behavior # This problem did not exist previously and needs further investigation if self.log_queue.empty(): time.sleep(2) try: event = self.log_queue.get_nowait() except Empty: self._process_sessions() else: if self.public_ip: event["public_ip"] = self.public_ip if self.friends_feeder: self.friends_feeder.log( json.dumps(event, default=json_default)) if self.sqlite_logger: self.sqlite_logger.log(event) # if self.mysql_logger: # self.mysql_logger.log(event) if self.syslog_client: self.syslog_client.log(event) if self.taxii_logger: self.taxii_logger.log(event) if self.json_logger: self.json_logger.log(event) if self.logstash_logger: self.logstash_logger.log(event) if self.mongodb_logger: self.mongodb_logger.log(event) def stop(self): self.enabled = False
class LogWorker(object): def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True def _json_default(self, obj): if isinstance(obj, datetime): return obj.isoformat() elif isinstance(obj, uuid.UUID): return str(obj) else: return None def _process_sessions(self): sessions = self.session_manager._sessions try: session_timeout = self.config.get("session", "timeout") except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): session_timeout = 5 for session in sessions: if len(session.data) > 0: sec_last_event = max(session.data) / 1000 else: sec_last_event = 0 sec_session_start = time.mktime(session.timestamp.timetuple()) sec_now = time.mktime(datetime.utcnow().timetuple()) if (sec_now - (sec_session_start + sec_last_event)) >= session_timeout: logger.info("Session timed out: {0}".format(session.id)) sessions.remove(session) def start(self): self.enabled = True while self.enabled: try: event = self.log_queue.get(timeout=2) except Empty: self._process_sessions() else: if self.public_ip: event["public_ip"] = self.public_ip if self.friends_feeder: self.friends_feeder.log( json.dumps(event, default=self._json_default)) if self.sqlite_logger: self.sqlite_logger.log(event) if self.syslog_client: self.syslog_client.log(event) if self.taxii_logger: self.taxii_logger.log(event) def stop(self): self.enabled = False
def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None # self.mysql_logger = None self.json_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean('sqlite', 'enabled'): self.sqlite_logger = SQLiteLogger() if config.getboolean('json', 'enabled'): filename = config.get('json', 'filename') sensorid = config.get('common', 'sensorid') self.json_logger = JsonLogger(filename, sensorid, public_ip) if config.getboolean('hpfriends', 'enabled'): host = config.get('hpfriends', 'host') port = config.getint('hpfriends', 'port') ident = config.get('hpfriends', 'ident') secret = config.get('hpfriends', 'secret') channels = eval(config.get('hpfriends', 'channels')) if config.get('hpfriends', 'reported_ip'): self.reported_ip = config.get('hpfriends', 'reported_ip') if self.reported_ip == 'UNSET_REPORTED_IP': self.reported_ip = None try: self.tags = [ tag.strip() for tag in config.get('hpfriends', 'tags').split(',') ] except Exception as e: logger.exception(e.message) self.tags = [] try: self.friends_feeder = HPFriendsLogger(host, port, ident, secret, channels) except Exception as e: logger.exception(e.message) self.friends_feeder = None if config.getboolean('syslog', 'enabled'): host = config.get('syslog', 'host') port = config.getint('syslog', 'port') facility = config.get('syslog', 'facility') logdevice = config.get('syslog', 'device') logsocket = config.get('syslog', 'socket') self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean('taxii', 'enabled'): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True
class LogWorker(object): def __init__(self, config, dom, session_manager, public_ip): self.config = config self.log_queue = session_manager.log_queue self.session_manager = session_manager self.sqlite_logger = None self.json_logger = None self.friends_feeder = None self.syslog_client = None self.public_ip = public_ip self.taxii_logger = None if config.getboolean("sqlite", "enabled"): self.sqlite_logger = SQLiteLogger() if config.getboolean("json", "enabled"): filename = config.get("json", "filename") sensorid = config.get("common", "sensorid") self.json_logger = JsonLogger(filename, sensorid, public_ip) if config.getboolean("hpfriends", "enabled"): host = config.get("hpfriends", "host") port = config.getint("hpfriends", "port") ident = config.get("hpfriends", "ident") secret = config.get("hpfriends", "secret") channels = eval(config.get("hpfriends", "channels")) try: self.friends_feeder = HPFriendsLogger( host, port, ident, secret, channels ) except Exception as e: logger.exception(e) self.friends_feeder = None if config.getboolean("syslog", "enabled"): host = config.get("syslog", "host") port = config.getint("syslog", "port") facility = config.get("syslog", "facility") logdevice = config.get("syslog", "device") logsocket = config.get("syslog", "socket") self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket) if config.getboolean("taxii", "enabled"): # TODO: support for certificates self.taxii_logger = TaxiiLogger(config, dom) self.enabled = True def _process_sessions(self): sessions = self.session_manager._sessions try: session_timeout = self.config.get("session", "timeout") except (configparser.NoSectionError, configparser.NoOptionError): session_timeout = 5 for session in sessions: if len(session.data) > 0: sec_last_event = max(session.data) / 1000 else: sec_last_event = 0 sec_session_start = time.mktime(session.timestamp.timetuple()) sec_now = time.mktime(datetime.utcnow().timetuple()) if (sec_now - (sec_session_start + sec_last_event)) >= float( session_timeout ): # TODO: We need to close sockets in this case logger.info("Session timed out: %s", session.id) session.set_ended() sessions.remove(session) def start(self): self.enabled = True while self.enabled: try: event = self.log_queue.get(timeout=2) except Empty: self._process_sessions() else: if self.public_ip: event["public_ip"] = self.public_ip if self.friends_feeder: self.friends_feeder.log(json.dumps(event, default=json_default)) if self.sqlite_logger: self.sqlite_logger.log(event) if self.syslog_client: self.syslog_client.log(event) if self.taxii_logger: self.taxii_logger.log(event) if self.json_logger: self.json_logger.log(event) def stop(self): self.enabled = False