def extract(self): value = (get_db()["noc.gridvcs.config.files"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate([ { "$match": { "object": { "$in": self.sync_ids } } }, { "$group": { "_id": "$object", "last_ts": { "$max": "$ts" } } }, { "$sort": { "_id": 1 } }, ])) for v in value: if not v["_id"]: continue yield v["_id"], v["last_ts"]
def _3_isp(self, index): # Profiles detect problems match = { "$or": [ { "problems.suggest_snmp.": { "$regex": "Failed to guess SNMP community", "$options": "i", } }, { "problems.profile.": "Cannot fetch snmp data, check device for SNMP access" }, { "problems.profile.": "Cannot detect profile" }, ] } c = set( int(r["_id"].rsplit("-")[-1]) for r in get_db()["noc.joblog"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).find( match)) if index == "0": return self.common_filter - c return c
def extract(self): match = {"data.management.managed_object": {"$exists": True}} if self.sync_ids: match = {"data.management.managed_object": {"$in": self.sync_ids}} value = ( get_db()["noc.objects"] .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) .aggregate( [ {"$match": match}, {"$sort": {"data.management.managed_object": 1}}, { "$lookup": { "from": "noc.objects", "localField": "container", "foreignField": "_id", "as": "cont", } }, {"$project": {"data": 1, "cont.data": 1}}, ] ) ) for v in value: r = {} if "asset" in v["data"]: # r[v["data"]["management"]["managed_object"]].update(v["data"]["asset"]) r.update(v["data"]["asset"]) if v["cont"]: if "data" in v["cont"][0]: # r[v["data"]["management"]["managed_object"]].update(v["cont"][0]["data"].get("address", {})) r.update(v["cont"][0]["data"].get("address", {})) yield v["data"]["management"]["managed_object"], r
def get_last_extract(self, name): coll = get_db()["noc.bi_timestamps"] d = coll.find_one({"_id": name}) if d: return d["last_extract"] else: return None
def archive(self): """ Move data to archive collection :return: """ def spool(collection_name): coll = db[collection_name] try: r = coll.insert_many(data[collection_name]) except BulkWriteError as e: print(e.details) return None return r self.require_db_connect() db = get_db() # Compile name template tpl = self.get_archived_template() # collection name -> data data = defaultdict(list) # Collect data and spool full batches for d in self.iter_archived_items(): cname = str(tpl.render({"doc": d})) data[cname] += [d] if len(data[cname]) >= self.archive_batch_limit: result = spool(cname) if result: data[cname] = [] else: break # Spool remaining incomplete batches for cname in data: if data[cname]: spool(cname)
def extract(self): i_type = "physical" match = {"type": i_type} if len(self.sync_ids) < 20000: # @todo Very large list slowest encode, need research match = {"type": i_type, "managed_object": {"$in": self.sync_ids}} value = (get_db()["noc.interfaces"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate( [ { "$match": match }, { "$group": { "_id": "$managed_object", "count": { "$sum": 1 } } }, { "$sort": { "_id": 1 } }, ], allowDiskUse=True, )) for v in value: if not v["_id"]: continue yield v["_id"], v["count"]
def _2_isp(self, index): # CLI Problem match = { "$and": [ { "problems.profile.": { "$ne": "Cannot fetch snmp data, check device for SNMP access" } }, {"problems.profile.": {"$ne": "Cannot detect profile"}}, { "$or": [ {"problems.version.": {"$regex": r"Remote error code \d+"}}, { "problems.suggest_cli.": { "$regex": r"Failed to guess CLI credentials", "$options": "i", } }, ] }, ] } c = set( int(r["_id"].rsplit("-")[-1]) for r in get_db()["noc.joblog"] .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) .find(match) ) if index == "0": return self.common_filter - c return c
def handle(self, host=None, port=None, *args, **options): connect() db = get_db() collections = set(db.list_collection_names()) for model_id in iter_model_id(): model = get_model(model_id) if not model: self.die("Invalid model: %s" % model_id) if not is_document(model): continue # Rename collections when necessary legacy_collections = model._meta.get("legacy_collections", []) for old_name in legacy_collections: if old_name in collections: new_name = model._meta["collection"] self.print("[%s] Renaming %s to %s" % (model_id, old_name, new_name)) db[old_name].rename(new_name) break # Ensure only documents with auto_create_index == False if model._meta.get("auto_create_index", True): continue # Index model self.index_model(model_id, model) # Index datastreams self.index_datastreams() # Index GridVCS self.index_gridvcs() # Index mongo cache self.index_cache() # Index datasource cache self.index_datasource_cache() # @todo: Detect changes self.print("OK")
def has_asset_members(cls): """ Returns True if cache contains asset.members :return: """ db = get_db() collection = db.noc.whois.asset.members return bool(collection.count_documents({}))
def __iter__(self): for p in self.mos_pools: r = (get_db()[self.coll_name % p.name].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate( self.pipelines.get(p.name, self.pipeline()))) for x in r: # @todo Append info for MO yield x
def get_collection(cls) -> pymongo.collection: """ Get pymongo collection instance :return: """ if not cls._coll: cls._coll = get_db()[cls.COLL_NAME] return cls._coll
def has_origin_routes(cls): """ Returns True if cache contains origin.routes :return: """ db = get_db() collection = db.noc.whois.origin.route return bool(collection.count_documents({}))
def fix(): # Initialize with distinct values coll = get_db()["noc.platforms"] bulk = [] for d in coll.find({"full_name": {"$exists": False}}, {"_id": 1}): bulk += [UpdateOne({"_id": d["_id"]}, {"$set": {"full_name": str(d["_id"])}})] if bulk: coll.bulk_write(bulk) fix_full_name()
def get_collection(self): """ Returns mongo collection instance """ if not self.collection: self.logger.debug("Open collection %s", self.collection_name) self.collection = get_db()[self.collection_name] self.bulk = [] return self.collection
def extract(self): match = {"data.address.text": {"$exists": True}} # if self.sync_ids: # containers = dict(ManagedObject.objects.filter(id__in=self.sync_ids).values_list("id", "container")) # match = {"_id": {"$in": list(containers)}} # if self.sync_ids: # match = {"data.management.managed_object": {"$in": self.sync_ids}} value = ( get_db()["noc.objects"] .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) .aggregate( [ {"$match": match}, # {"$sort": {"_id": 1}}, { "$project": { "parent_address": "$data.address.text", "parent_name": 1, "_id": 1, } }, { "$lookup": { "from": "noc.objects", "localField": "_id", "foreignField": "container", "as": "child_cont", } }, { "$project": { "parent_address": 1, "parent_name": 1, "_id": 1, "child_cont._id": 1, "child_cont.name": 1, } }, {"$unwind": {"path": "$child_cont", "preserveNullAndEmptyArrays": True}}, ] ) ) r = {} for v in value: cid = str(v["_id"]) if "child_cont" in v: # ccid = str(v["child_cont"]["_id"]) r[str(v["child_cont"]["_id"])] = v["parent_address"].strip() if cid not in r: r[cid] = v["parent_address"].strip() for mo_id, container in ( ManagedObject.objects.filter(id__in=self.sync_ids) .values_list("id", "container") .order_by("id") ): if container in r: yield mo_id, r[container]
def extract(self): # @todo Make reports field """ { "_id" : { "managed_object" : 6757 }, "count_in_speed" : 3 } { "_id" : { "oper_status" : true, "in_speed" : 10000, "managed_object" : 6757 }, "count_in_speed" : 2 } { "_id" : { "oper_status" : true, "in_speed" : 1000000, "managed_object" : 6757 }, "count_in_speed" : 11 } { "_id" : { "oper_status" : false, "in_speed" : 1000000, "managed_object" : 6757 }, "count_in_speed" : 62 } { "_id" : { "oper_status" : true, "in_speed" : 10000000, "managed_object" : 6757 }, "count_in_speed" : 5 } { "_id" : { "oper_status" : false, "in_speed" : 100000, "managed_object" : 6757 }, "count_in_speed" : 1 } :return: """ def humanize_speed(speed): if not speed: return "-" for t, n in [(1000000, "G"), (1000, "M"), (1, "k")]: if speed >= t: if speed // t * t == speed: return "%d%s" % (speed // t, n) else: return "%.2f%s" % (float(speed) / t, n) return str(speed) oper = True group = {"in_speed": "$in_speed", "managed_object": "$managed_object"} if oper: group["oper_status"] = "$oper_status" match = {"type": "physical"} if self.sync_ids: match = {"type": "physical", "managed_object": {"$in": self.sync_ids}} value = ( get_db()["noc.interfaces"] .with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) .aggregate( [{"$match": match}, {"$group": {"_id": group, "count": {"$sum": 1}}}], allowDiskUse=True, ) ) r = defaultdict(lambda: [""] * len(self.ATTRS)) # @todo Fix Down for v in value: c = ( {True: "Up", False: "Down", None: "-"}[v["_id"].get("oper_status", None)] if oper else "" ) if v["_id"].get("in_speed", None): c += "/" + humanize_speed(v["_id"]["in_speed"]) else: c += "/-" # r[v["_id"]["managed_object"]].append((c, v["count"])) if c in self.ATTRS: r[v["_id"]["managed_object"]][self.ATTRS.index(c)] = v["count"] for val in r: yield val, r[val]
def get_collection(cls): """ Get pymongo Collection object :return: """ coll = getattr(cls, "_collection", None) if not coll: coll = get_db()[cls.get_collection_name()] cls._collection = coll return coll
def _resolve_as_set_prefixes(cls, as_set): db = get_db() collection = db.noc.whois.origin.route # Resolve prefixes = set() for a in cls.resolve_as_set(as_set): o = collection.find_one({"_id": a}, {"routes": 1}) if o: prefixes.update(o["routes"]) return prefixes
def _4_isp(self, index): # Undefined profiles match = {"problems.profile.": {"$regex": "Not find profile for OID"}} c = set( int(r["_id"].rsplit("-")[-1]) for r in get_db()["noc.joblog"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).find( match)) if index == "0": return self.common_filter - c return c
def api_job_log(self, request, id, job): o = self.get_object_or_404(ManagedObject, id=id) if not o.has_access(request.user): return self.response_forbidden("Access denied") # fs = gridfs.GridFS(get_db(), "noc.joblog") key = "discovery-%s-%s" % (job, o.id) d = get_db()["noc.joblog"].find_one({"_id": key}) if d and d["log"]: return self.render_plain_text(zlib.decompress(str(d["log"]))) else: return self.render_plain_text("No data")
def extract(self): # load(self, mo_ids, attributes): # Namedtuple caps, for save Caps = namedtuple("Caps", list(self.ATTRS)) Caps.__new__.__defaults__ = ("", ) * len(Caps._fields) mo_ids = self.sync_ids[:] while mo_ids: chunk, mo_ids = mo_ids[:self.CHUNK_SIZE], mo_ids[self.CHUNK_SIZE:] match = {"_id": {"$in": chunk}} value = (get_db()["noc.sa.objectcapabilities"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate([ { "$match": match }, { "$unwind": "$caps" }, { "$match": { "caps.source": { "$in": ["caps", "manual"] } } }, { "$project": { "key": "$caps.capability", "value": "$caps.value" } }, { "$group": { "_id": "$_id", "cap": { "$push": { "item": "$key", "val": "$value" } }, } }, { "$sort": { "_id": 1 } }, ])) for v in value: r = { f'c_{ll["item"]}': ll["val"] for ll in v["cap"] if f'c_{ll["item"]}' in self.ATTRS } yield v["_id"], Caps(**r)
def extract(self): db = get_db()["noc.objectfacts"] mos_filter = {"label": "system"} if self.sync_ids: mos_filter["object"] = {"$in": list(self.sync_ids)} for val in ( db.with_options(read_preference=ReadPreference.SECONDARY_PREFERRED) .find(mos_filter, {"_id": 0, "object": 1, "attrs.hostname": 1}) .sort("object") ): yield val["object"], val["attrs"].get("hostname", self.unknown_value)
def has_asset(cls, as_set): """ Returns true if as-set has members in cache :param as_set: :return: """ if is_asn(as_set[2:]): return True db = get_db() collection = db.noc.whois.asset.members return bool(collection.find_one({"_id": as_set}, {"_id": 1}))
def load(mo_ids): # match = {"links.mo": {"$in": mo_ids}} match = {"int.managed_object": {"$in": mo_ids}} group = { "_id": "$_id", "links": { "$push": { "iface_n": "$int.name", # "iface_id": "$int._id", # "iface_descr": "$int.description", # "iface_speed": "$int.in_speed", # "dis_method": "$discovery_method", # "last_seen": "$last_seen", "mo": "$int.managed_object", "linked_obj": "$linked_objects", } }, } value = (get_db()["noc.links"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate( [ { "$unwind": "$interfaces" }, { "$lookup": { "from": "noc.interfaces", "localField": "interfaces", "foreignField": "_id", "as": "int", } }, { "$match": match }, { "$group": group }, ], allowDiskUse=True, )) res = defaultdict(dict) for v in value: if v["_id"]: for vv in v["links"]: if len(vv["linked_obj"]) == 2: mo = vv["mo"][0] iface = vv["iface_n"] for i in vv["linked_obj"]: if mo != i: res[mo][i] = iface[0] return res
def _0_isp(self, index): # Common Problem match = {"problems": {"$exists": True}} c = set( int(r["_id"].rsplit("-")[-1]) for r in get_db()["noc.joblog"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).find( match)) if index == "0": return self.common_filter - c else: return c
def is_p(self, num, value, inverse=False): """ Problem match :param discovery: Discovery name :param problem: Problem text :return: """ match = {"problems": {"$exists": True}} if num == "1": # SNMP Problem match = { "problems": { "$exists": True }, "problems.suggest_snmp.": { "$regex": "Failed to guess SNMP community", "$options": "i", }, } if num == "2": # CLI Problem match = { "$and": [ { "problems.profile.": { "$ne": "Cannot fetch snmp data, check device for SNMP access" } }, { "problems.profile.": { "$ne": "Cannot detect profile" } }, { "problems.version.": { "$regex": "/Remote error code \d+/" } }, ] } c = set( int(r["_id"].rsplit("-")[-1]) for r in get_db()["noc.joblog"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).find( match)) if inverse: return set(ManagedObject.objects.filter().values_list( "id", flat=True)) - c else: return c
def get_collection(cls, fmt: Optional[str] = None ) -> pymongo.collection.Collection: """ Get pymongo Collection object :return: """ c_name = cls.get_collection_name(fmt) coll = cls._collections.get(c_name) if not coll: coll = get_db()[c_name] cls._collections[c_name] = coll return coll
def get_format_role(cls, fmt: str) -> Optional[str]: """ Returns format role, if any :param fmt: :return: """ doc = get_db()["datastreamconfigs"].find_one({"name": cls.name}) if not doc: return None for f in doc.get("formats", []): if f.get("name") == fmt: return f.get("role") or None return None
def api_job_log(self, request, id): o = self.get_object_or_404(ManagedObject, id=id) if not o.has_access(request.user): return self.response_forbidden("Access denied") r = [] for job in self.job_map: key = "discovery-%s-%s" % (job, o.id) d = get_db()["noc.joblog"].find_one({"_id": key}) if d and d["log"]: r += [b"\n", smart_bytes(job), b"\n"] r += [zlib.decompress(smart_bytes((d["log"])))] if r: return self.render_plain_text(b"".join(r)) else: return self.render_plain_text("No data")
def _refresh_labels(self): """ Recalculate labels on model :return: """ from django.db import connection for scope in self.iter_scopes(): labels = [f"noc::rxfilter::{self.name}::{scope}::="] + (self.labels or []) model, field = scope.split("_") if model == "managedobject": # Cleanup current labels logger.info("[%s] Cleanup ManagedObject effective labels: %s", self.name, labels) Label.reset_model_labels("sa.ManagedObject", labels) # Apply new rule logger.info("[%s] Apply new regex '%s' labels", self.name, self.regexp) sql = f""" UPDATE sa_managedobject SET effective_labels=ARRAY ( SELECT DISTINCT e FROM unnest(effective_labels || %s::varchar[]) AS a(e) ) WHERE {field} ~ %s """ cursor = connection.cursor() cursor.execute(sql, [labels, self.regexp]) elif model == "interface": # Cleanup current labels logger.info("[%s] Cleanup Interface effective labels: %s", self.name, labels) Label.reset_model_labels("inv.Interface", labels) # Apply new rule coll = get_db()["noc.interfaces"] coll.bulk_write([ UpdateMany[{ field: { "$re": self.regexp } }, { "$addToSet": { "effective_labels": { "$each": labels } } }, ] ])