def handle(self, paths, profile, format, progress=False, *args, **options): assert profile_loader.get_profile( profile), "Invalid profile: %s" % profile t0 = time.time() ruleset = RuleSet() ruleset.load() self.print("Ruleset load in %.2fms" % ((time.time() - t0) * 1000)) reader = getattr(self, "read_%s" % format, None) assert reader, "Invalid format %s" % format self.managed_object = ManagedObject(id=1, name="test", address="127.0.0.1", profile_name=profile) t0 = time.time() stats = defaultdict(int) total = 0 for p in paths: if not os.path.isfile(p): continue for f in iter_open(p): for event in reader(f): e_vars = event.raw_vars.copy() if event.source == "SNMP Trap": e_vars.update(MIB.resolve_vars(event.raw_vars)) rule, r_vars = ruleset.find_rule(event, e_vars) stats[rule.event_class.name] += 1 total += 1 if progress and total % 1000 == 0: self.print("%d records processed" % total) dt = time.time() - t0 self.print("%d events processed in %.2fms (%.fevents/sec)" % (total, dt * 1000, float(total) / dt)) if stats: # Prepare statistics s_data = sorted([(k, stats[k]) for k in stats], key=operator.itemgetter(1), reverse=True) s_total = sum(stats[k] for k in stats if not self.is_ignored(k)) data = [["Events", "%", "Event class"]] for ecls, qty in s_data: data += [[ str(qty), "%3.2f%%" % (float(stats[ecls] * 100) / float(total)), ecls ]] # Calculate classification quality data += [[ "", "%3.2f%%" % (float(s_total * 100) / total), "Classification Quality" ]] # Ruleset hit rate rs_rate = float(metrics["rules_checked"].value) / float(total) data += [["", "%.2f" % rs_rate, "Rule checks per event"]] # Dump table self.print("Event classes summary:") self.print(format_table([4, 6, 10], data))
def test_event(ruleset, event): e, expected_class, expected_vars = event e_vars = e.raw_vars.copy() if e.source == "SNMP Trap": e_vars.update(MIB.resolve_vars(e.raw_vars)) rule, r_vars = ruleset.find_rule(e, e_vars) assert rule is not None, "Cannot find matching rule" assert rule.event_class == expected_class, "Mismatched event class %s vs %s" % ( rule.event_class.name, expected_class.name) ruleset.eval_vars(event, rule.event_class, e_vars) assert e_vars == expected_vars, "Mismatched vars"
def api_test(self, request): q = self.deserialize(request.raw_post_data) errors = [] patterns = [] result = False # Get data data = {} vars = {} required_vars = set() r_patterns = [] event_class = None subject = None body = None if "data" in q: if is_objectid(q["data"]): event = get_event(q["data"]) if event: data = event.raw_vars.copy() data["profile"] = event.managed_object.profile.name data["source"] = event.source else: errors += ["Event not found: %s" % q["data"]] else: # Decode json try: e = self.deserialize(q["data"]) except Exception: errors += ["Cannot decode JSON"] e = None if isinstance(e, list): e = e[0] if not isinstance(e, dict) or "raw_vars" not in e: errors += ["Invalid JSON data"] else: data = e["raw_vars"] if "profile" in e: data["profile"] = e["profile"] if "source" in e: data["source"] = e["source"] if data.get("source") == "SNMP Trap": # Resolve MIBs data.update(MIB.resolve_vars(data)) # Check event class if "event_class" in q: event_class = self.get_object_or_404(EventClass, id=q["event_class"]) for v in event_class.vars: if v.required: required_vars.add(v.name) vars[v.name] = "MISSED!" # Check patterns if "patterns" in q: for p in q["patterns"]: if "key_re" in p and "value_re" in p: k = None v = None try: k = re.compile(p["key_re"]) except re.error as why: errors += [ "Invalid key regular expression <<<%s>>>: %s" % (p["key_re"], why) ] try: v = re.compile(p["value_re"]) except re.error as why: errors += [ "Invalid value regular expression <<<%s>>>: %s" % (p["value_re"], why) ] if k and v: patterns += [(k, v)] # Try to match rule if patterns and not errors: s_patterns = [] i_patterns = [] for pkey, pvalue in patterns: matched = False for k in data: k_match = pkey.search(k) if k_match: v_match = pvalue.search(data[k]) if v_match: # Line match # Update vars v = {} v.update(k_match.groupdict()) v.update(v_match.groupdict()) vars.update(v) # Save patterns s_patterns += [{ "status": True, "key": k, "value": data[k], "key_re": pkey.pattern, "value_re": pvalue.pattern, "vars": [{ "key": k, "value": v[k] } for k in v] }] else: i_patterns += [{ "status": False, "key": k, "value": data[k], "key_re": pkey.pattern, "value_re": pvalue.pattern, "vars": {} }] matched = True break if not matched: i_patterns += [{ "status": False, "key": None, "value": None, "key_re": pkey.pattern, "value_re": pvalue.pattern, "vars": {} }] if s_patterns and not i_patterns: result = True r_patterns = s_patterns + i_patterns # Calculate rule variables if "vars" in q and q["vars"]: for v in q["vars"]: if v["value"].startswith("="): # Evaluate try: vars[v["name"]] = eval(v["value"][1:], {}, vars) except Exception as why: errors += [ "Error when evaluating '%s': %s" % (v["name"], why) ] else: vars[v["name"]] = v["value"] # Check required variables for rvars in required_vars: if rvars not in vars: errors += ["Missed required variable: %s" % rvars] # Fill event class template if event_class: # lang = "en" ctx = Context(vars) subject = Template(event_class.subject_template).render(ctx) body = Template(event_class.body_template).render(ctx) # Check expression r = {"result": result} if errors: r["errors"] = errors if vars: r["vars"] = [{"key": k, "value": vars[k]} for k in vars] if r_patterns: r["patterns"] = r_patterns if subject: r["subject"] = subject if body: r["body"] = body return r
async def classify_event(self, event, data): """ Perform event classification. Classification steps are: 1. Format SNMP values accordind to MIB definitions (for SNMP events only) 2. Find matching classification rule 3. Calculate rule variables :param event: Event to classify :type event: NewEvent :returns: Classification status (CR_*) """ metrics[E_SRC_METRICS.get(event.source, E_SRC_OTHER)] += 1 is_unknown = False # pre_event = data.pop("$event", None) # Resolve MIB variables for SNMP Traps resolved_vars = {"profile": event.managed_object.profile.name} # Store event variables event.raw_vars = data if event.source == E_SRC_SNMP_TRAP: resolved_vars.update(MIB.resolve_vars(event.raw_vars)) event.resolved_vars = resolved_vars # Get matched event class if pre_event: # Event is preprocessed, get class and variables event_class_name = pre_event.get("class") event_class = EventClass.get_by_name(event_class_name) if not event_class: self.logger.error( "[%s|%s|%s] Failed to process event: Invalid event class '%s'", event.id, event.managed_object.name, event.managed_object, event_class_name, ) metrics[CR_FAILED] += 1 return # Drop malformed message event.event_class = event_class event.vars = pre_event.get("vars", {}) else: # Prevent unclassified events flood if self.check_unclassified_syslog_flood(event): return # Find matched event class c_vars = event.raw_vars.copy() c_vars.update({ k: smart_text(fm_unescape(resolved_vars[k])) for k in resolved_vars }) rule, vars = self.ruleset.find_rule(event, c_vars) if rule is None: # Something goes wrong. # No default rule found. Exit immediately self.logger.error("No default rule found. Exiting") os._exit(1) if rule.to_drop: # Silently drop event if declared by action self.logger.info( "[%s|%s|%s] Dropped by action", event.id, event.managed_object.name, event.managed_object.address, ) metrics[CR_DELETED] += 1 return if rule.is_unknown_syslog: # Append to codebook msg = event.raw_vars.get("message", "") cb = self.get_msg_codebook(msg) o_id = event.managed_object.id if o_id not in self.unclassified_codebook: self.unclassified_codebook[o_id] = [] cbs = [cb] + self.unclassified_codebook[o_id] cbs = cbs[:self.unclassified_codebook_depth] self.unclassified_codebook[o_id] = cbs self.logger.debug( "[%s|%s|%s] Matching rule: %s", event.id, event.managed_object.name, event.managed_object.address, rule.name, ) event.event_class = rule.event_class # Calculate rule variables event.vars = self.ruleset.eval_vars(event, event.event_class, vars) message = "Classified as '%s' by rule '%s'" % ( event.event_class.name, rule.name) event.log += [ EventLog( timestamp=datetime.datetime.now(), from_status="N", to_status="A", message=message, ) ] is_unknown = rule.is_unknown # Event class found, process according to rules self.logger.info( "[%s|%s|%s] Event class: %s (%s)", event.id, event.managed_object.name, event.managed_object.address, event.event_class.name, event.vars, ) # Deduplication if self.deduplicate_event(event): return # Suppress repeats if self.suppress_repeats(event): return # Activate event event.expires = event.timestamp + datetime.timedelta( seconds=event.event_class.ttl) event.save() # Fill deduplication filter self.dedup_filter.register(event) # Fill suppress filter self.suppress_filter.register(event) # Call handlers if self.call_event_handlers(event): return # Additionally check link events if await self.check_link_event(event): return # Call triggers if self.call_event_triggers(event): return # Finally dispose event to further processing by correlator if event.to_dispose: await self.dispose_event(event) if is_unknown: metrics[CR_UNKNOWN] += 1 elif pre_event: metrics[CR_PREPROCESSED] += 1 else: metrics[CR_CLASSIFIED] += 1
def api_test(self, request): q = self.deserialize(request.raw_post_data) errors = [] patterns = [] result = False # Get data data = {} vars = {} required_vars = set() r_patterns = [] event_class = None subject = None body = None if "data" in q: if is_objectid(q["data"]): event = get_event(q["data"]) if event: data = event.raw_vars.copy() data["profile"] = event.managed_object.profile_name else: errors += ["Event not found: %s" % q["data"]] else: # Decode json try: e = self.deserialize(q["data"]) except: errors += ["Cannot decode JSON"] e = None if isinstance(e, list): e = e[0] if not isinstance(e, dict) or "raw_vars" not in e: errors += ["Invalid JSON data"] else: data = e["raw_vars"] if "profile" in e: data["profile"] = e["profile"] if data.get("source") == "SNMP Trap": # Resolve MIBs data.update(MIB.resolve_vars(data)) # Check event class if "event_class" in q: event_class = self.get_object_or_404(EventClass, id=q["event_class"]) for v in event_class.vars: if v.required: required_vars.add(v.name) vars[v.name] = "MISSED!" # Check patterns if "patterns" in q: for p in q["patterns"]: if "key_re" in p and "value_re" in p: k = None v = None try: k = re.compile(p["key_re"]) except re.error, why: errors += [ "Invalid key regular expression <<<%s>>>: %s" % (p["key_re"], why) ] try: v = re.compile(p["value_re"]) except re.error, why: errors += [ "Invalid value regular expression <<<%s>>>: %s" % (p["value_re"], why) ] if k and v: patterns += [(k, v)]