def log_event(event_tuple, packet=None, skip_write=False, skip_condensing=False):
try:
sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info, reference = event_tuple
if ignore_event(event_tuple):
return
if not (any(check_whitelisted(_) for _ in (src_ip, dst_ip)) and trail_type != TRAIL.DNS): # DNS requests/responses can't be whitelisted based on src_ip/dst_ip
if not skip_write:
localtime = "%s.%06d" % (time.strftime(TIME_FORMAT, time.localtime(int(sec))), usec)
if not skip_condensing:
if any(_ in info for _ in CONDENSE_ON_INFO_KEYWORDS):
with _condensing_lock:
key = (src_ip, trail)
if key not in _condensed_events:
_condensed_events[key] = []
_condensed_events[key].append(event_tuple)
return
current_bucket = sec / config.PROCESS_COUNT
if getattr(_thread_data, "log_bucket", None) != current_bucket: # log throttling
_thread_data.log_bucket = current_bucket
_thread_data.log_trails = set()
else:
if any(_ in _thread_data.log_trails for _ in ((src_ip, trail), (dst_ip, trail))):
return
else:
_thread_data.log_trails.add((src_ip, trail))
_thread_data.log_trails.add((dst_ip, trail))
event = "%s %s %s\n" % (safe_value(localtime), safe_value(config.SENSOR_NAME), " ".join(safe_value(_) for _ in event_tuple[2:]))
if not config.DISABLE_LOCAL_LOG_STORAGE:
handle = get_event_log_handle(sec)
os.write(handle, event)
if config.LOG_SERVER:
remote_host, remote_port = config.LOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto("%s %s" % (sec, event), (remote_host, int(remote_port)))
if config.SYSLOG_SERVER:
extension = "src=%s spt=%s dst=%s dpt=%s trail=%s ref=%s" % (src_ip, src_port, dst_ip, dst_port, trail, reference)
_ = CEF_FORMAT.format(syslog_time=time.strftime("%b %d %H:%M:%S", time.localtime(int(sec))), host=HOSTNAME, device_vendor=NAME, device_product="sensor", device_version=VERSION, signature_id=time.strftime("%Y-%m-%d", time.localtime(os.path.getctime(TRAILS_FILE))), name=info, severity=0, extension=extension)
remote_host, remote_port = config.SYSLOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto(_, (remote_host, int(remote_port)))
if config.DISABLE_LOCAL_LOG_STORAGE and not any(config.LOG_SERVER, config.SYSLOG_SERVER) or config.console:
sys.stderr.write(event)
sys.stderr.flush()
if config.plugin_functions:
for _ in config.plugin_functions:
_(event_tuple, packet)
except (OSError, IOError):
if config.SHOW_DEBUG:
traceback.print_exc()
def log_event(event_tuple,
packet=None,
skip_write=False,
skip_condensing=False):
global _condensing_thread
if _condensing_thread is None:
_condensing_thread = threading.Thread(target=flush_condensed_events)
_condensing_thread.daemon = True
_condensing_thread.start()
try:
sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info, reference = event_tuple
if ignore_event(event_tuple):
return
if not (
any(check_whitelisted(_)
for _ in (src_ip, dst_ip)) and trail_type != TRAIL.DNS
): # DNS requests/responses can't be whitelisted based on src_ip/dst_ip
if not skip_write:
localtime = "%s.%06d" % (time.strftime(
TIME_FORMAT, time.localtime(int(sec))), usec)
if not skip_condensing:
if any(_ in info for _ in CONDENSE_ON_INFO_KEYWORDS):
with _condensing_lock:
key = (src_ip, trail)
if key not in _condensed_events:
_condensed_events[key] = []
_condensed_events[key].append(event_tuple)
return
current_bucket = sec // config.PROCESS_COUNT
if getattr(_thread_data, "log_bucket",
None) != current_bucket: # log throttling
_thread_data.log_bucket = current_bucket
_thread_data.log_trails = set()
else:
if any(_ in _thread_data.log_trails
for _ in ((src_ip, trail), (dst_ip, trail))):
return
else:
_thread_data.log_trails.add((src_ip, trail))
_thread_data.log_trails.add((dst_ip, trail))
event = "%s %s %s\n" % (
safe_value(localtime), safe_value(config.SENSOR_NAME),
" ".join(safe_value(_) for _ in event_tuple[2:]))
if not config.DISABLE_LOCAL_LOG_STORAGE:
handle = get_event_log_handle(sec)
os.write(handle, event.encode(UNICODE_ENCODING))
if config.LOG_SERVER:
if config.LOG_SERVER.count(':') > 1:
remote_host, remote_port = config.LOG_SERVER.replace(
'[', '').replace(']', '').rsplit(':', 1)
# Reference: https://github.com/squeaky-pl/zenchmarks/blob/master/vendor/twisted/internet/tcp.py
_AI_NUMERICSERV = getattr(socket, "AI_NUMERICSERV", 0)
_NUMERIC_ONLY = socket.AI_NUMERICHOST | _AI_NUMERICSERV
_address = socket.getaddrinfo(
remote_host,
int(remote_port)
if str(remote_port or "").isdigit() else 0, 0, 0,
0, _NUMERIC_ONLY)[0][4]
else:
remote_host, remote_port = config.LOG_SERVER.split(':')
_address = (remote_host, int(remote_port))
s = socket.socket(
socket.AF_INET if len(_address) == 2 else
socket.AF_INET6, socket.SOCK_DGRAM)
s.sendto(("%s %s" % (sec, event)).encode(UNICODE_ENCODING),
_address)
if config.SYSLOG_SERVER:
extension = "src=%s spt=%s dst=%s dpt=%s trail=%s ref=%s" % (
src_ip, src_port, dst_ip, dst_port, trail, reference)
_ = CEF_FORMAT.format(
syslog_time=time.strftime("%b %d %H:%M:%S",
time.localtime(int(sec))),
host=HOSTNAME,
device_vendor=NAME,
device_product="sensor",
device_version=VERSION,
signature_id=time.strftime(
"%Y-%m-%d",
time.localtime(os.path.getctime(
config.TRAILS_FILE))),
name=info,
severity=0,
extension=extension)
remote_host, remote_port = config.SYSLOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto(_.encode(UNICODE_ENCODING),
(remote_host, int(remote_port)))
if (config.DISABLE_LOCAL_LOG_STORAGE and not any(
(config.LOG_SERVER, config.SYSLOG_SERVER))
) or config.console:
sys.stderr.write(event)
sys.stderr.flush()
if config.plugin_functions:
for _ in config.plugin_functions:
_(event_tuple, packet)
except (OSError, IOError):
if config.SHOW_DEBUG:
traceback.print_exc()
def log_event(event_tuple,
packet=None,
skip_write=False,
skip_condensing=False):
try:
sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info, reference = event_tuple
if not any(check_whitelisted(_) for _ in (src_ip, dst_ip)):
if not skip_write:
localtime = "%s.%06d" % (time.strftime(
TIME_FORMAT, time.localtime(int(sec))), usec)
if not skip_condensing:
if (sec -
getattr(_thread_data, "condensed_events_flush_sec",
0)) > CONDENSED_EVENTS_FLUSH_PERIOD:
_thread_data.condensed_events_flush_sec = sec
for key in getattr(_thread_data, "condensed_events",
[]):
condensed = False
events = _thread_data.condensed_events[key]
first_event = events[0]
condensed_event = [_ for _ in first_event]
for i in xrange(1, len(events)):
current_event = events[i]
for j in xrange(
3, 7
): # src_port, dst_ip, dst_port, proto
if current_event[j] != condensed_event[j]:
condensed = True
if not isinstance(
condensed_event[j], set):
condensed_event[j] = set(
(condensed_event[j], ))
condensed_event[j].add(
current_event[j])
if condensed:
for i in xrange(len(condensed_event)):
if isinstance(condensed_event[i], set):
condensed_event[i] = ','.join(
str(_) for _ in sorted(
condensed_event[i]))
log_event(condensed_event, skip_condensing=True)
_thread_data.condensed_events = {}
if any(_ in info for _ in CONDENSE_ON_INFO_KEYWORDS):
if not hasattr(_thread_data, "condensed_events"):
_thread_data.condensed_events = {}
key = (src_ip, trail)
if key not in _thread_data.condensed_events:
_thread_data.condensed_events[key] = []
_thread_data.condensed_events[key].append(event_tuple)
return
event = "%s %s %s\n" % (
safe_value(localtime), safe_value(config.SENSOR_NAME),
" ".join(safe_value(_) for _ in event_tuple[2:]))
if not config.DISABLE_LOCAL_LOG_STORAGE:
handle = get_event_log_handle(sec)
os.write(handle, event)
if config.LOG_SERVER:
remote_host, remote_port = config.LOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto("%s %s" % (sec, event),
(remote_host, int(remote_port)))
if config.SYSLOG_SERVER:
extension = "src=%s spt=%s dst=%s dpt=%s trail=%s ref=%s" % (
src_ip, src_port, dst_ip, dst_port, trail, reference)
_ = CEF_FORMAT.format(
syslog_time=time.strftime("%b %d %H:%M:%S",
time.gmtime(int(sec))),
host=HOSTNAME,
device_vendor=NAME,
device_product="sensor",
device_version=VERSION,
signature_id=time.strftime(
"%Y-%m-%d",
time.gmtime(os.path.getctime(TRAILS_FILE))),
name=info,
severity=0,
extension=extension)
remote_host, remote_port = config.SYSLOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto(_, (remote_host, int(remote_port)))
if config.DISABLE_LOCAL_LOG_STORAGE and not any(
config.LOG_SERVER,
config.SYSLOG_SERVER) or config.console:
sys.stderr.write(event)
sys.stderr.flush()
if config.plugin_functions:
for _ in config.plugin_functions:
_(event_tuple, packet)
except (OSError, IOError):
if config.SHOW_DEBUG:
traceback.print_exc()
def log_event(event_tuple, packet=None, skip_write=False, skip_condensing=False):
try:
sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info, reference = event_tuple
if not any(check_whitelisted(_) for _ in (src_ip, dst_ip)):
if not skip_write:
localtime = "%s.%06d" % (time.strftime(TIME_FORMAT, time.localtime(int(sec))), usec)
if not skip_condensing:
if (sec - getattr(_thread_data, "condensed_events_flush_sec", 0)) > CONDENSED_EVENTS_FLUSH_PERIOD:
_thread_data.condensed_events_flush_sec = sec
for key in getattr(_thread_data, "condensed_events", []):
condensed = False
events = _thread_data.condensed_events[key]
first_event = events[0]
condensed_event = [_ for _ in first_event]
for i in xrange(1, len(events)):
current_event = events[i]
for j in xrange(3, 7): # src_port, dst_ip, dst_port, proto
if current_event[j] != condensed_event[j]:
condensed = True
if not isinstance(condensed_event[j], set):
condensed_event[j] = set((condensed_event[j],))
condensed_event[j].add(current_event[j])
if condensed:
for i in xrange(len(condensed_event)):
if isinstance(condensed_event[i], set):
condensed_event[i] = ','.join(str(_) for _ in sorted(condensed_event[i]))
log_event(condensed_event, skip_condensing=True)
_thread_data.condensed_events = {}
if any(_ in info for _ in CONDENSE_ON_INFO_KEYWORDS):
if not hasattr(_thread_data, "condensed_events"):
_thread_data.condensed_events = {}
key = (src_ip, trail)
if key not in _thread_data.condensed_events:
_thread_data.condensed_events[key] = []
_thread_data.condensed_events[key].append(event_tuple)
return
event = "%s %s %s\n" % (safe_value(localtime), safe_value(config.SENSOR_NAME), " ".join(safe_value(_) for _ in event_tuple[2:]))
if not config.DISABLE_LOCAL_LOG_STORAGE:
handle = get_event_log_handle(sec)
os.write(handle, event)
if config.LOG_SERVER:
remote_host, remote_port = config.LOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto("%s %s" % (sec, event), (remote_host, int(remote_port)))
if config.SYSLOG_SERVER:
extension = "src=%s spt=%s dst=%s dpt=%s trail=%s ref=%s" % (src_ip, src_port, dst_ip, dst_port, trail, reference)
_ = CEF_FORMAT.format(syslog_time=time.strftime("%b %d %H:%M:%S", time.gmtime(int(sec))), host=HOSTNAME, device_vendor=NAME, device_product="sensor", device_version=VERSION, signature_id=time.strftime("%Y-%m-%d", time.gmtime(os.path.getctime(TRAILS_FILE))), name=info, severity=0, extension=extension)
remote_host, remote_port = config.SYSLOG_SERVER.split(':')
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.sendto(_, (remote_host, int(remote_port)))
if config.DISABLE_LOCAL_LOG_STORAGE and not any(config.LOG_SERVER, config.SYSLOG_SERVER) or config.console:
sys.stderr.write(event)
sys.stderr.flush()
if config.plugin_functions:
for _ in config.plugin_functions:
_(event_tuple, packet)
except (OSError, IOError):
if config.SHOW_DEBUG:
traceback.print_exc()