def get_data(self, **kwargs): c = EventClass.objects.filter(name="Unknown | SNMP Trap").first() pipeline = [{ "$match": { "event_class": c.id } }, { "$project": { "vars": 1 } }, { "$group": { "_id": "$vars.trap_oid", "count": { "$sum": 1 } } }] oids = ActiveEvent._get_collection().aggregate(pipeline) data = [(e["_id"], MIB.get_name(e["_id"]), e["count"]) for e in oids] data = sorted(data, key=lambda x: -x[2]) return self.from_dataset(title=self.title, columns=[ "OID", "Name", TableColumn("Count", format="integer", align="right", total="sum") ], data=data)
def event(request): path, cfg = request.param coll = cfg.get("$collection", COLLECTION_NAME) assert coll == COLLECTION_NAME, "Invalid collection %s" % coll ec = EventClass.get_by_name( cfg.get("eventclass__name", DEFAULT_EVENT_CLASS) ) mo = ManagedObject( id=MO_ID, name=MO_NAME, address=MO_ADDRESS, profile=Profile.get_by_name(cfg.get("profile__name", DEFAULT_PROFILE)) ) now = datetime.datetime.now() data = cfg.get("data", {}) source = data.pop("source", "other") event = ActiveEvent( timestamp=now, start_timestamp=now, managed_object=mo, source=source, raw_vars=data, repeats=1 ) request.fixturename = "events-%s" % cfg.get("uuid") # request.fspath = path return event, ec, cfg.get("vars", {})
def get_data(self, **kwargs): c = EventClass.objects.filter(name="Unknown | SNMP Trap").first() # Переделать на agregate Функция считает число OID'ов в переменных аварий # и проверяет их на опознанность pipeline = [{ "$match": { "event_class": c.id } }, { "$project": { "vars": 1 } }, { "$group": { "_id": "$vars.trap_oid", "count": { "$sum": 1 } } }] oids = ActiveEvent._get_collection().aggregate(pipeline) d = [(e["_id"], MIB.get_name(e["_id"]), e["count"]) for e in oids] print d data = [(o, n, c) for o, n, c in d if self.rx_unclassified.search(n)] return self.from_dataset(title=self.title, columns=[ "OID", "Name", TableColumn("Count", format="integer", align="right", total="sum") ], data=data)
async def raise_abduct_event(self, event: ActiveEvent) -> None: """ Create Cable Abduct Event and dispose it to correlator :param event: :return: """ if not self.cable_abduct_ecls: self.cable_abduct_ecls = EventClass.get_by_name(CABLE_ABDUCT) abd_event = ActiveEvent( timestamp=event.timestamp, start_timestamp=event.timestamp, managed_object=event.managed_object, source=event.source, repeats=1, event_class=self.cable_abduct_ecls, ) abd_event.save() await self.dispose_event(abd_event)
def read_syslog(self, f): now = datetime.datetime.now() for line in f: yield ActiveEvent( timestamp=now, start_timestamp=now, managed_object=self.managed_object, source="syslog", raw_vars={"collector": "default", "message": line[:-1]}, repeats=1, )
def get_event_from_hint(self, hint): """ Get ActiveEvent from json hint :param hint: :return: """ metrics["event_hints"] += 1 e = ActiveEvent.from_json(hint) # Prevent TypeError: can't compare offset-naive and offset-aware datetimes # when calculating alarm timestamp e.timestamp = e.timestamp.replace(tzinfo=None) return e
def suppress_repeats(self, event: ActiveEvent) -> bool: """ Suppress repeated events :param event: :param vars: :return: """ se_id = self.suppress_filter.find(event) if not se_id: return False self.logger.info( "[%s|%s|%s] Suppressed by event %s", event.id, event.managed_object.name, event.managed_object.address, se_id, ) # Update suppressing event ActiveEvent.log_suppression(se_id, event.timestamp) # Delete suppressed event metrics[CR_SUPPRESSED] += 1 return True
def handle_clean(self, options, events): before = options.get("before") if before: datetime.datetime.strptime(before, "%Y-%m-%d") else: self.print("Before is not set, use default") before = datetime.datetime.now() - DEFAULT_CLEAN force = options.get("force") aa = ActiveAlarm._get_collection() ah = ArchivedAlarm._get_collection() ae = ActiveEvent._get_collection() event_ts = ae.find_one({"timestamp": {"$lte": before}}, limit=1, sort=[("timestamp", 1)]) event_ts = event_ts["timestamp"] print("[%s] Cleaned before %s ... \n" % ("events", before), end="") bulk = [] window = CLEAN_WINDOW while event_ts < before: refer_event_ids = [] for e in [aa, ah]: for ee in e.find( {"timestamp": {"$gte": event_ts, "$lte": event_ts + CLEAN_WINDOW}}, {"opening_event": 1, "closing_event": 1}, ): if "opening_event" in ee: refer_event_ids += [ee["opening_event"]] if "closing_event" in ee: refer_event_ids += [ee["closing_event"]] try: clear_qs = { "timestamp": {"$gte": event_ts, "$lte": event_ts + CLEAN_WINDOW}, "_id": {"$nin": refer_event_ids}, } self.print( "Interval: %s, %s; Count: %d" % (event_ts, event_ts + CLEAN_WINDOW, ae.count(clear_qs)) ) bulk += [DeleteMany(clear_qs)] event_ts += window if window != CLEAN_WINDOW: window = CLEAN_WINDOW except DocumentTooLarge: window = window // 2 if window < datetime.timedelta(hours=1): self.die("Too many events for delete in interval %s" % window) event_ts -= window if force: self.print("All data before %s from active events will be Remove..\n" % before) for i in reversed(range(1, 10)): self.print("%d\n" % i) time.sleep(1) ae.bulk_write(bulk)
def lookup_event(self, event_id): """ Lookup event by id. Uses cache heating effect from classifier :param event_id: :return: ActiveEvent instance or None """ self.logger.info("[%s] Lookup event", event_id) e = ActiveEvent.get_by_id(event_id) if not e: self.logger.info("[%s] Event not found, skipping", event_id) metrics["event_lookup_failed"] += 1 metrics["event_lookups"] += 1 return e
async def on_event(self, msg: Message): # Decode message event = orjson.loads(msg.value) object = event.get("object") data = event.get("data") # Process event event_ts = datetime.datetime.fromtimestamp(event.get("ts")) # Generate or reuse existing object id event_id = ObjectId(event.get("id")) # Calculate message processing delay lag = (time.time() - float(msg.timestamp) / NS) * 1000 metrics["lag_us"] = int(lag * 1000) self.logger.debug("[%s] Receiving new event: %s (Lag: %.2fms)", event_id, data, lag) metrics[CR_PROCESSED] += 1 # Resolve managed object mo = ManagedObject.get_by_id(object) if not mo: self.logger.info("[%s] Unknown managed object id %s. Skipping", event_id, object) metrics[CR_UOBJECT] += 1 return self.logger.info("[%s|%s|%s] Managed object found", event_id, mo.name, mo.address) # Process event source = data.pop("source", "other") event = ActiveEvent( id=event_id, timestamp=event_ts, start_timestamp=event_ts, managed_object=mo, source=source, repeats=1, ) # raw_vars will be filled by classify_event() # Ignore event if self.patternset.find_ignore_rule(event, data): self.logger.debug("Ignored event %s vars %s", event, data) metrics[CR_IGNORED] += 1 return # Classify event try: await self.classify_event(event, data) except Exception as e: self.logger.error("[%s|%s|%s] Failed to process event: %s", event.id, mo.name, mo.address, e) metrics[CR_FAILED] += 1 return self.logger.info("[%s|%s|%s] Event processed successfully", event.id, mo.name, mo.address)
def on_event(self, message, ts=None, object=None, data=None, id=None, *args, **kwargs): event_ts = datetime.datetime.fromtimestamp(ts) # Generate or reuse existing object id event_id = ObjectId(id) # Calculate messate processing delay lag = (time.time() - ts) * 1000 metrics["lag_us"] = int(lag * 1000) self.logger.debug("[%s] Receiving new event: %s (Lag: %.2fms)", event_id, data, lag) metrics[CR_PROCESSED] += 1 # Resolve managed object mo = ManagedObject.get_by_id(object) if not mo: self.logger.info("[%s] Unknown managed object id %s. Skipping", event_id, object) metrics[CR_UOBJECT] += 1 return True self.logger.info("[%s|%s|%s] Managed object found", event_id, mo.name, mo.address) # Process event source = data.pop("source", "other") event = ActiveEvent( id=event_id, timestamp=event_ts, start_timestamp=event_ts, managed_object=mo, source=source, repeats=1, ) # raw_vars will be filled by classify_event() # Classify event try: self.classify_event(event, data) except Exception as e: self.logger.error("[%s|%s|%s] Failed to process event: %s", event.id, mo.name, mo.address, e) metrics[CR_FAILED] += 1 return False self.logger.info("[%s|%s|%s] Event processed successfully", event.id, mo.name, mo.address) return True
async def on_dispose_event(self, msg: Message) -> None: """ Called on new dispose message """ data = orjson.loads(msg.value) event_id = data["event_id"] hint = data["event"] self.logger.info("[%s] Receiving message", event_id) metrics["alarm_dispose"] += 1 try: event = ActiveEvent.from_json(hint) event.timestamp = event.timestamp.replace(tzinfo=None) await self.dispose_event(event) except Exception: metrics["alarm_dispose_error"] += 1 error_report() finally: if self.topo_rca_lock: # Release pending RCA Lock await self.topo_rca_lock.release() self.topo_rca_lock = None
from noc.fm.models.activeevent import ActiveEvent from noc.core.mongo.connection import connect from datetime import * connect() import re events = ActiveEvent.objects() re_events = re.compile(".+[LOGIN|LOGOUT|login|logout].+217.76.35.203") events = ActiveEvent.objects() for event in events: if 'message' in event.raw_vars: if re_events.match(event.raw_vars['message']): event.delete() re2_events = re.compile(".+217.76.35.203.+") for event in events: if '1.3.6.1.2.1.16.9.1.1.2.178' in event.raw_vars: if re2_events.match(event.raw_vars['1.3.6.1.2.1.16.9.1.1.2.178']): event.delete()