def handle(self, paths, profile, format, progress=False, *args, **options):
assert profile_loader.get_profile(
profile), "Invalid profile: %s" % profile
t0 = time.time()
ruleset = RuleSet()
ruleset.load()
self.print("Ruleset load in %.2fms" % ((time.time() - t0) * 1000))
reader = getattr(self, "read_%s" % format, None)
assert reader, "Invalid format %s" % format
self.managed_object = ManagedObject(id=1,
name="test",
address="127.0.0.1",
profile_name=profile)
t0 = time.time()
stats = defaultdict(int)
total = 0
for p in paths:
if not os.path.isfile(p):
continue
for f in iter_open(p):
for event in reader(f):
e_vars = event.raw_vars.copy()
if event.source == "SNMP Trap":
e_vars.update(MIB.resolve_vars(event.raw_vars))
rule, r_vars = ruleset.find_rule(event, e_vars)
stats[rule.event_class.name] += 1
total += 1
if progress and total % 1000 == 0:
self.print("%d records processed" % total)
dt = time.time() - t0
self.print("%d events processed in %.2fms (%.fevents/sec)" %
(total, dt * 1000, float(total) / dt))
if stats:
# Prepare statistics
s_data = sorted([(k, stats[k]) for k in stats],
key=operator.itemgetter(1),
reverse=True)
s_total = sum(stats[k] for k in stats if not self.is_ignored(k))
data = [["Events", "%", "Event class"]]
for ecls, qty in s_data:
data += [[
str(qty),
"%3.2f%%" % (float(stats[ecls] * 100) / float(total)), ecls
]]
# Calculate classification quality
data += [[
"",
"%3.2f%%" % (float(s_total * 100) / total),
"Classification Quality"
]]
# Ruleset hit rate
rs_rate = float(metrics["rules_checked"].value) / float(total)
data += [["", "%.2f" % rs_rate, "Rule checks per event"]]
# Dump table
self.print("Event classes summary:")
self.print(format_table([4, 6, 10], data))
def test_event(ruleset, event):
e, expected_class, expected_vars = event
e_vars = e.raw_vars.copy()
if e.source == "SNMP Trap":
e_vars.update(MIB.resolve_vars(e.raw_vars))
rule, r_vars = ruleset.find_rule(e, e_vars)
assert rule is not None, "Cannot find matching rule"
assert rule.event_class == expected_class, "Mismatched event class %s vs %s" % (
rule.event_class.name, expected_class.name)
ruleset.eval_vars(event, rule.event_class, e_vars)
assert e_vars == expected_vars, "Mismatched vars"
def api_test(self, request):
q = self.deserialize(request.raw_post_data)
errors = []
patterns = []
result = False
# Get data
data = {}
vars = {}
required_vars = set()
r_patterns = []
event_class = None
subject = None
body = None
if "data" in q:
if is_objectid(q["data"]):
event = get_event(q["data"])
if event:
data = event.raw_vars.copy()
data["profile"] = event.managed_object.profile.name
data["source"] = event.source
else:
errors += ["Event not found: %s" % q["data"]]
else:
# Decode json
try:
e = self.deserialize(q["data"])
except Exception:
errors += ["Cannot decode JSON"]
e = None
if isinstance(e, list):
e = e[0]
if not isinstance(e, dict) or "raw_vars" not in e:
errors += ["Invalid JSON data"]
else:
data = e["raw_vars"]
if "profile" in e:
data["profile"] = e["profile"]
if "source" in e:
data["source"] = e["source"]
if data.get("source") == "SNMP Trap":
# Resolve MIBs
data.update(MIB.resolve_vars(data))
# Check event class
if "event_class" in q:
event_class = self.get_object_or_404(EventClass,
id=q["event_class"])
for v in event_class.vars:
if v.required:
required_vars.add(v.name)
vars[v.name] = "MISSED!"
# Check patterns
if "patterns" in q:
for p in q["patterns"]:
if "key_re" in p and "value_re" in p:
k = None
v = None
try:
k = re.compile(p["key_re"])
except re.error as why:
errors += [
"Invalid key regular expression <<<%s>>>: %s" %
(p["key_re"], why)
]
try:
v = re.compile(p["value_re"])
except re.error as why:
errors += [
"Invalid value regular expression <<<%s>>>: %s" %
(p["value_re"], why)
]
if k and v:
patterns += [(k, v)]
# Try to match rule
if patterns and not errors:
s_patterns = []
i_patterns = []
for pkey, pvalue in patterns:
matched = False
for k in data:
k_match = pkey.search(k)
if k_match:
v_match = pvalue.search(data[k])
if v_match:
# Line match
# Update vars
v = {}
v.update(k_match.groupdict())
v.update(v_match.groupdict())
vars.update(v)
# Save patterns
s_patterns += [{
"status":
True,
"key":
k,
"value":
data[k],
"key_re":
pkey.pattern,
"value_re":
pvalue.pattern,
"vars": [{
"key": k,
"value": v[k]
} for k in v]
}]
else:
i_patterns += [{
"status": False,
"key": k,
"value": data[k],
"key_re": pkey.pattern,
"value_re": pvalue.pattern,
"vars": {}
}]
matched = True
break
if not matched:
i_patterns += [{
"status": False,
"key": None,
"value": None,
"key_re": pkey.pattern,
"value_re": pvalue.pattern,
"vars": {}
}]
if s_patterns and not i_patterns:
result = True
r_patterns = s_patterns + i_patterns
# Calculate rule variables
if "vars" in q and q["vars"]:
for v in q["vars"]:
if v["value"].startswith("="):
# Evaluate
try:
vars[v["name"]] = eval(v["value"][1:], {}, vars)
except Exception as why:
errors += [
"Error when evaluating '%s': %s" % (v["name"], why)
]
else:
vars[v["name"]] = v["value"]
# Check required variables
for rvars in required_vars:
if rvars not in vars:
errors += ["Missed required variable: %s" % rvars]
# Fill event class template
if event_class:
# lang = "en"
ctx = Context(vars)
subject = Template(event_class.subject_template).render(ctx)
body = Template(event_class.body_template).render(ctx)
# Check expression
r = {"result": result}
if errors:
r["errors"] = errors
if vars:
r["vars"] = [{"key": k, "value": vars[k]} for k in vars]
if r_patterns:
r["patterns"] = r_patterns
if subject:
r["subject"] = subject
if body:
r["body"] = body
return r
async def classify_event(self, event, data):
"""
Perform event classification.
Classification steps are:
1. Format SNMP values accordind to MIB definitions (for SNMP events only)
2. Find matching classification rule
3. Calculate rule variables
:param event: Event to classify
:type event: NewEvent
:returns: Classification status (CR_*)
"""
metrics[E_SRC_METRICS.get(event.source, E_SRC_OTHER)] += 1
is_unknown = False
#
pre_event = data.pop("$event", None)
# Resolve MIB variables for SNMP Traps
resolved_vars = {"profile": event.managed_object.profile.name}
# Store event variables
event.raw_vars = data
if event.source == E_SRC_SNMP_TRAP:
resolved_vars.update(MIB.resolve_vars(event.raw_vars))
event.resolved_vars = resolved_vars
# Get matched event class
if pre_event:
# Event is preprocessed, get class and variables
event_class_name = pre_event.get("class")
event_class = EventClass.get_by_name(event_class_name)
if not event_class:
self.logger.error(
"[%s|%s|%s] Failed to process event: Invalid event class '%s'",
event.id,
event.managed_object.name,
event.managed_object,
event_class_name,
)
metrics[CR_FAILED] += 1
return # Drop malformed message
event.event_class = event_class
event.vars = pre_event.get("vars", {})
else:
# Prevent unclassified events flood
if self.check_unclassified_syslog_flood(event):
return
# Find matched event class
c_vars = event.raw_vars.copy()
c_vars.update({
k: smart_text(fm_unescape(resolved_vars[k]))
for k in resolved_vars
})
rule, vars = self.ruleset.find_rule(event, c_vars)
if rule is None:
# Something goes wrong.
# No default rule found. Exit immediately
self.logger.error("No default rule found. Exiting")
os._exit(1)
if rule.to_drop:
# Silently drop event if declared by action
self.logger.info(
"[%s|%s|%s] Dropped by action",
event.id,
event.managed_object.name,
event.managed_object.address,
)
metrics[CR_DELETED] += 1
return
if rule.is_unknown_syslog:
# Append to codebook
msg = event.raw_vars.get("message", "")
cb = self.get_msg_codebook(msg)
o_id = event.managed_object.id
if o_id not in self.unclassified_codebook:
self.unclassified_codebook[o_id] = []
cbs = [cb] + self.unclassified_codebook[o_id]
cbs = cbs[:self.unclassified_codebook_depth]
self.unclassified_codebook[o_id] = cbs
self.logger.debug(
"[%s|%s|%s] Matching rule: %s",
event.id,
event.managed_object.name,
event.managed_object.address,
rule.name,
)
event.event_class = rule.event_class
# Calculate rule variables
event.vars = self.ruleset.eval_vars(event, event.event_class, vars)
message = "Classified as '%s' by rule '%s'" % (
event.event_class.name, rule.name)
event.log += [
EventLog(
timestamp=datetime.datetime.now(),
from_status="N",
to_status="A",
message=message,
)
]
is_unknown = rule.is_unknown
# Event class found, process according to rules
self.logger.info(
"[%s|%s|%s] Event class: %s (%s)",
event.id,
event.managed_object.name,
event.managed_object.address,
event.event_class.name,
event.vars,
)
# Deduplication
if self.deduplicate_event(event):
return
# Suppress repeats
if self.suppress_repeats(event):
return
# Activate event
event.expires = event.timestamp + datetime.timedelta(
seconds=event.event_class.ttl)
event.save()
# Fill deduplication filter
self.dedup_filter.register(event)
# Fill suppress filter
self.suppress_filter.register(event)
# Call handlers
if self.call_event_handlers(event):
return
# Additionally check link events
if await self.check_link_event(event):
return
# Call triggers
if self.call_event_triggers(event):
return
# Finally dispose event to further processing by correlator
if event.to_dispose:
await self.dispose_event(event)
if is_unknown:
metrics[CR_UNKNOWN] += 1
elif pre_event:
metrics[CR_PREPROCESSED] += 1
else:
metrics[CR_CLASSIFIED] += 1
def api_test(self, request):
q = self.deserialize(request.raw_post_data)
errors = []
patterns = []
result = False
# Get data
data = {}
vars = {}
required_vars = set()
r_patterns = []
event_class = None
subject = None
body = None
if "data" in q:
if is_objectid(q["data"]):
event = get_event(q["data"])
if event:
data = event.raw_vars.copy()
data["profile"] = event.managed_object.profile_name
else:
errors += ["Event not found: %s" % q["data"]]
else:
# Decode json
try:
e = self.deserialize(q["data"])
except:
errors += ["Cannot decode JSON"]
e = None
if isinstance(e, list):
e = e[0]
if not isinstance(e, dict) or "raw_vars" not in e:
errors += ["Invalid JSON data"]
else:
data = e["raw_vars"]
if "profile" in e:
data["profile"] = e["profile"]
if data.get("source") == "SNMP Trap":
# Resolve MIBs
data.update(MIB.resolve_vars(data))
# Check event class
if "event_class" in q:
event_class = self.get_object_or_404(EventClass,
id=q["event_class"])
for v in event_class.vars:
if v.required:
required_vars.add(v.name)
vars[v.name] = "MISSED!"
# Check patterns
if "patterns" in q:
for p in q["patterns"]:
if "key_re" in p and "value_re" in p:
k = None
v = None
try:
k = re.compile(p["key_re"])
except re.error, why:
errors += [
"Invalid key regular expression <<<%s>>>: %s" %
(p["key_re"], why)
]
try:
v = re.compile(p["value_re"])
except re.error, why:
errors += [
"Invalid value regular expression <<<%s>>>: %s" %
(p["value_re"], why)
]
if k and v:
patterns += [(k, v)]
