class LogWorker(object):
def __init__(self, config, dom, session_manager, public_ip):
self.config = config
self.log_queue = session_manager.log_queue
self.session_manager = session_manager
self.sqlite_logger = None
# self.mysql_logger = None
self.json_logger = None
self.friends_feeder = None
self.syslog_client = None
self.public_ip = public_ip
self.taxii_logger = None
if config.getboolean('sqlite', 'enabled'):
self.sqlite_logger = SQLiteLogger()
# if config.getboolean('mysql', 'enabled'):
# host = config.get('mysql', 'host')
# port = config.getint('mysql', 'port')
# db = config.get('mysql', 'db')
# username = config.get('mysql', 'username')
# passphrase = config.get('mysql', 'passphrase')
# logdevice = config.get('mysql', 'device')
# logsocket = config.get('mysql', 'socket')
# sensorid = config.get('common', 'sensorid')
# self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid)
if config.getboolean('json', 'enabled'):
todaydate = datetime.now()
todaystr = todaydate.strftime('%Y-%m-%d')
filename = config.get('json', 'filename') + '.' + todaystr
sensorid = config.get('common', 'sensorid')
self.json_logger = JsonLogger(filename, sensorid, public_ip)
if config.getboolean('hpfriends', 'enabled'):
host = config.get('hpfriends', 'host')
port = config.getint('hpfriends', 'port')
ident = config.get('hpfriends', 'ident')
secret = config.get('hpfriends', 'secret')
channels = eval(config.get('hpfriends', 'channels'))
try:
self.friends_feeder = HPFriendsLogger(host, port, ident,
secret, channels)
except Exception as e:
logger.exception(e.message)
self.friends_feeder = None
if config.getboolean('syslog', 'enabled'):
host = config.get('syslog', 'host')
port = config.getint('syslog', 'port')
facility = config.get('syslog', 'facility')
logdevice = config.get('syslog', 'device')
logsocket = config.get('syslog', 'socket')
self.syslog_client = SysLogger(host, port, facility, logdevice,
logsocket)
if config.getboolean('taxii', 'enabled'):
# TODO: support for certificates
self.taxii_logger = TaxiiLogger(config, dom)
self.enabled = True
def _process_sessions(self):
sessions = self.session_manager._sessions
try:
session_timeout = self.config.get("session", "timeout")
except (configparser.NoSectionError, configparser.NoOptionError):
session_timeout = 5
for session in sessions:
if len(session.data) > 0:
sec_last_event = max(session.data) / 1000
else:
sec_last_event = 0
sec_session_start = time.mktime(session.timestamp.timetuple())
sec_now = time.mktime(datetime.utcnow().timetuple())
if (sec_now - (sec_session_start +
sec_last_event)) >= float(session_timeout):
# TODO: We need to close sockets in this case
logger.info('Session timed out: %s', session.id)
session.set_ended()
sessions.remove(session)
def start(self):
self.enabled = True
while self.enabled:
try:
event = self.log_queue.get(timeout=2)
except Empty:
self._process_sessions()
else:
if self.public_ip:
event["public_ip"] = self.public_ip
if self.friends_feeder:
self.friends_feeder.log(
json.dumps(event, default=json_default))
if self.sqlite_logger:
self.sqlite_logger.log(event)
# if self.mysql_logger:
# self.mysql_logger.log(event)
if self.syslog_client:
self.syslog_client.log(event)
if self.taxii_logger:
self.taxii_logger.log(event)
if self.json_logger:
self.json_logger.log(event)
def stop(self):
self.enabled = False
class LogWorker(object):
def __init__(self, config, dom, session_manager, public_ip):
self.config = config
self.log_queue = session_manager.log_queue
self.session_manager = session_manager
self.sqlite_logger = None
# self.mysql_logger = None
self.json_logger = None
self.friends_feeder = None
self.syslog_client = None
self.public_ip = public_ip
self.taxii_logger = None
self.logstash_logger = None
self.mongodb_logger = None
if config.getboolean('sqlite', 'enabled'):
self.sqlite_logger = SQLiteLogger()
# if config.getboolean('mysql', 'enabled'):
# host = config.get('mysql', 'host')
# port = config.getint('mysql', 'port')
# db = config.get('mysql', 'db')
# username = config.get('mysql', 'username')
# passphrase = config.get('mysql', 'passphrase')
# logdevice = config.get('mysql', 'device')
# logsocket = config.get('mysql', 'socket')
# sensorid = config.get('common', 'sensorid')
# self.mysql_logger = MySQLlogger(host, port, db, username, passphrase, logdevice, logsocket, sensorid)
if config.getboolean('json', 'enabled'):
filename = config.get('json', 'filename')
sensorid = config.get('common', 'sensorid')
self.json_logger = JsonLogger(filename, sensorid, public_ip)
if config.getboolean('hpfriends', 'enabled'):
host = config.get('hpfriends', 'host')
port = config.getint('hpfriends', 'port')
ident = config.get('hpfriends', 'ident')
secret = config.get('hpfriends', 'secret')
channels = eval(config.get('hpfriends', 'channels'))
try:
self.friends_feeder = HPFriendsLogger(host, port, ident,
secret, channels)
except Exception as e:
logger.exception(e.message)
self.friends_feeder = None
if config.getboolean('syslog', 'enabled'):
host = config.get('syslog', 'host')
port = config.getint('syslog', 'port')
facility = config.get('syslog', 'facility')
logdevice = config.get('syslog', 'device')
logsocket = config.get('syslog', 'socket')
self.syslog_client = SysLogger(host, port, facility, logdevice,
logsocket)
if config.getboolean('taxii', 'enabled'):
# TODO: support for certificates
self.taxii_logger = TaxiiLogger(config, dom)
if config.getboolean('logstash', 'enabled'):
host = config.get('logstash', 'host')
port = int(config.get('logstash', 'port'))
self.logstash_logger = Logstash(host, port)
if config.getboolean('mongodb', 'enabled'):
host = config.get('mongodb', 'host')
port = config.get('mongodb', 'port')
db = config.get('mongodb', 'db')
collection = config.get('mongodb', 'collection')
sensorid = config.get('common', 'sensorid')
self.mongodb_logger = MongodbLogger(host, port, db, collection,
sensorid)
self.enabled = True
def _process_sessions(self):
sessions = self.session_manager._sessions
try:
session_timeout = self.config.get("session", "timeout")
except (configparser.NoSectionError, configparser.NoOptionError):
session_timeout = 5
for session in sessions:
if len(session.data) > 0:
sec_last_event = max(session.data) / 1000
else:
sec_last_event = 0
sec_session_start = time.mktime(session.timestamp.timetuple())
sec_now = time.mktime(datetime.utcnow().timetuple())
if (sec_now - (sec_session_start +
sec_last_event)) >= float(session_timeout):
# TODO: We need to close sockets in this case
logger.info('Session timed out: %s', session.id)
session.set_ended()
sessions.remove(session)
def start(self):
self.enabled = True
while self.enabled:
# Changed from blocking to non-blocking to prevent endless blocking caused by rapid succession of events
# Sleep added to approximately emulate previous behavior
# This problem did not exist previously and needs further investigation
if self.log_queue.empty():
time.sleep(2)
try:
event = self.log_queue.get_nowait()
except Empty:
self._process_sessions()
else:
if self.public_ip:
event["public_ip"] = self.public_ip
if self.friends_feeder:
self.friends_feeder.log(
json.dumps(event, default=json_default))
if self.sqlite_logger:
self.sqlite_logger.log(event)
# if self.mysql_logger:
# self.mysql_logger.log(event)
if self.syslog_client:
self.syslog_client.log(event)
if self.taxii_logger:
self.taxii_logger.log(event)
if self.json_logger:
self.json_logger.log(event)
if self.logstash_logger:
self.logstash_logger.log(event)
if self.mongodb_logger:
self.mongodb_logger.log(event)
def stop(self):
self.enabled = False
class LogWorker(object):
def __init__(self, config, dom, session_manager, public_ip):
self.config = config
self.log_queue = session_manager.log_queue
self.session_manager = session_manager
self.sqlite_logger = None
self.syslog_client = None
self.public_ip = public_ip
self.shodan_logger = ShodanLogger()
if config.getboolean('sqlite', 'enabled'):
self.sqlite_logger = SQLiteLogger()
if config.getboolean('syslog', 'enabled'):
host = config.get('syslog', 'host')
port = config.getint('syslog', 'port')
facility = config.get('syslog', 'facility')
logdevice = config.get('syslog', 'device')
logsocket = config.get('syslog', 'socket')
self.syslog_client = SysLogger(host, port, facility, logdevice,
logsocket)
self.enabled = True
def _json_default(self, obj):
if isinstance(obj, datetime):
return obj.isoformat()
elif isinstance(obj, uuid.UUID):
return str(obj)
else:
return None
def _process_sessions(self):
sessions = self.session_manager._sessions
try:
session_timeout = self.config.get("session", "timeout")
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
session_timeout = 5
for session in sessions:
if len(session.data) > 0:
sec_last_event = max(session.data) / 1000
else:
sec_last_event = 0
sec_session_start = time.mktime(session.timestamp.timetuple())
sec_now = time.mktime(datetime.utcnow().timetuple())
if (sec_now -
(sec_session_start + sec_last_event)) >= session_timeout:
# TODO: We need to close sockets in this case
logger.info("Session timed out: {0}".format(session.id))
session.set_ended()
sessions.remove(session)
def start(self):
self.enabled = True
while self.enabled:
try:
event = self.log_queue.get(timeout=2)
except Empty:
self._process_sessions()
else:
if self.public_ip:
event["public_ip"] = self.public_ip
# if self.friends_feeder:
# self.friends_feeder.log(json.dumps(event, default=self._json_default))
if self.sqlite_logger:
self.sqlite_logger.log(event)
if self.syslog_client:
self.syslog_client.log(event)
if self.shodan_logger:
self.shodan_logger.log(event)
def stop(self):
self.enabled = False
class LogWorker(object):
def __init__(self, config, dom, session_manager, public_ip):
self.config = config
self.log_queue = session_manager.log_queue
self.session_manager = session_manager
self.sqlite_logger = None
self.json_logger = None
self.friends_feeder = None
self.syslog_client = None
self.public_ip = public_ip
self.taxii_logger = None
if config.getboolean("sqlite", "enabled"):
self.sqlite_logger = SQLiteLogger()
if config.getboolean("json", "enabled"):
filename = config.get("json", "filename")
sensorid = config.get("common", "sensorid")
self.json_logger = JsonLogger(filename, sensorid, public_ip)
if config.getboolean("hpfriends", "enabled"):
host = config.get("hpfriends", "host")
port = config.getint("hpfriends", "port")
ident = config.get("hpfriends", "ident")
secret = config.get("hpfriends", "secret")
channels = eval(config.get("hpfriends", "channels"))
try:
self.friends_feeder = HPFriendsLogger(
host, port, ident, secret, channels
)
except Exception as e:
logger.exception(e)
self.friends_feeder = None
if config.getboolean("syslog", "enabled"):
host = config.get("syslog", "host")
port = config.getint("syslog", "port")
facility = config.get("syslog", "facility")
logdevice = config.get("syslog", "device")
logsocket = config.get("syslog", "socket")
self.syslog_client = SysLogger(host, port, facility, logdevice, logsocket)
if config.getboolean("taxii", "enabled"):
# TODO: support for certificates
self.taxii_logger = TaxiiLogger(config, dom)
self.enabled = True
def _process_sessions(self):
sessions = self.session_manager._sessions
try:
session_timeout = self.config.get("session", "timeout")
except (configparser.NoSectionError, configparser.NoOptionError):
session_timeout = 5
for session in sessions:
if len(session.data) > 0:
sec_last_event = max(session.data) / 1000
else:
sec_last_event = 0
sec_session_start = time.mktime(session.timestamp.timetuple())
sec_now = time.mktime(datetime.utcnow().timetuple())
if (sec_now - (sec_session_start + sec_last_event)) >= float(
session_timeout
):
# TODO: We need to close sockets in this case
logger.info("Session timed out: %s", session.id)
session.set_ended()
sessions.remove(session)
def start(self):
self.enabled = True
while self.enabled:
try:
event = self.log_queue.get(timeout=2)
except Empty:
self._process_sessions()
else:
if self.public_ip:
event["public_ip"] = self.public_ip
if self.friends_feeder:
self.friends_feeder.log(json.dumps(event, default=json_default))
if self.sqlite_logger:
self.sqlite_logger.log(event)
if self.syslog_client:
self.syslog_client.log(event)
if self.taxii_logger:
self.taxii_logger.log(event)
if self.json_logger:
self.json_logger.log(event)
def stop(self):
self.enabled = False