def migrate(self): mdb = self.mongo_db # Get default prefix profile coll = mdb["prefixprofiles"] d = coll.find_one({"name": "default"}) default_prefix_profile = d["_id"] # coll = mdb["vpnprofiles"] default_id = bson.ObjectId() wf = bson.ObjectId("5a01d980b6f529000100d37a") profiles = [ { "_id": default_id, "name": "default VRF", "type": "vrf", "description": "Default VRF profile", "workflow": wf, "default_prefix_profile": default_prefix_profile, "bi_id": bson.int64.Int64(bi_hash(default_id)), } ] # Convert styles style_profiles = {None: default_id} for (style_id,) in self.db.execute("SELECT DISTINCT style_id FROM ip_vrf"): if not style_id: continue p_id = bson.ObjectId() p = { "_id": p_id, "name": "VRF Style %s" % style_id, "type": "vrf", "description": "Auto-converted for VRF style %s" % style_id, "workflow": wf, "style": style_id, "default_prefix_profile": default_prefix_profile, "bi_id": bson.int64.Int64(bi_hash(p_id)), } style_profiles[style_id] = p_id profiles += [p] # Insert profiles to database coll.insert_many(profiles) # Create Prefix.profile field self.db.add_column( "ip_vrf", "profile", DocumentReferenceField("vc.VPNProfile", null=True, blank=True) ) # Migrate profile styles for style_id in style_profiles: if style_id: cond = "style_id = %s" % style_id else: cond = "style_id IS NULL" self.db.execute( "UPDATE ip_vrf SET profile = %%s WHERE %s" % cond, [str(style_profiles[style_id])] ) # Make Prefix.profile not nullable self.db.execute("ALTER TABLE ip_vrf ALTER profile SET NOT NULL") # Drop Prefix.style self.db.delete_column("ip_vrf", "style_id")
def migrate(self): coll = self.mongo_db["addressprofiles"] default_id = bson.ObjectId() wf = bson.ObjectId("5a01d980b6f529000100d37a") profiles = [ { "_id": default_id, "name": "default", "description": "Default address profile", "workflow": wf, "bi_id": bson.int64.Int64(bi_hash(default_id)), } ] # Convert styles style_profiles = {} for (style_id,) in self.db.execute("SELECT DISTINCT style_id FROM ip_address"): if not style_id: style_profiles[None] = default_id continue p_id = bson.ObjectId() p = { "_id": p_id, "name": "Style %s" % style_id, "description": "Auto-converted for style %s" % style_id, "workflow": wf, "style": style_id, "bi_id": bson.int64.Int64(bi_hash(p_id)), } style_profiles[style_id] = p_id profiles += [p] # Insert profiles to database coll.insert_many(profiles) # Create Prefix.profile field self.db.add_column( "ip_address", "profile", DocumentReferenceField("ip.AddressProfile", null=True, blank=True), ) # Migrate profile styles for style_id in style_profiles: if style_id: cond = "style_id = %s" % style_id else: cond = "style_id IS NULL" self.db.execute( "UPDATE ip_address SET profile = %%s WHERE %s" % cond, [str(style_profiles[style_id])], ) # Make Prefix.profile not nullable self.db.execute("ALTER TABLE ip_address ALTER profile SET NOT NULL") # Drop Prefix.style self.db.delete_column("ip_address", "style_id")
def migrate(self): # Get existing termination groups tg_data = self.db.execute( "SELECT id, name, description, remote_system, remote_id, tags FROM sa_terminationgroup" ) if not tg_data: return # Nothing to convert bulk = [] # Create root node for migrated termination groups root_id = bson.ObjectId() bulk += [ InsertOne({ "_id": root_id, "name": "Converted T.G.", "parent": None, "description": "Created from former termination groups", "technology": bson.ObjectId("5b6d6819d706360001a0b716"), # Group "bi_id": bson.Int64(bi_hash(root_id)), }) ] # Attach termination groups for id, name, description, remote_system, remote_id, tags in tg_data: new_id = bson.ObjectId() bulk += [ InsertOne({ "_id": new_id, "name": name, "parent": root_id, "description": description, # May be changed by phone migration "technology": bson.ObjectId("5b6d6be1d706360001f5c04e" ), # Network | IPoE Termination "remote_system": bson.ObjectId(remote_system) if remote_system else None, "remote_id": remote_id, "bi_id": bson.Int64(bi_hash(new_id)), "_legacy_id": id, # To be removed in future migrations }) ] # Apply groups self.mongo_db.resourcegroups.bulk_write(bulk)
def migrate(self): MODELS = [ "sa_administrativedomain", "sa_authprofile", "sa_managedobject", "sa_managedobjectprofile", "sa_terminationgroup", ] # Update postgresql tables for table in MODELS: rows = self.db.execute("SELECT id FROM %s WHERE bi_id IS NULL" % table) values = ["(%d, %d)" % (r[0], bi_hash(r[0])) for r in rows] while values: chunk, values = values[:PG_CHUNK], values[PG_CHUNK:] self.db.execute(""" UPDATE %s AS t SET bi_id = c.bi_id FROM ( VALUES %s ) AS c(id, bi_id) WHERE c.id = t.id """ % (table, ",\n".join(chunk))) # Update mongodb collections mdb = self.mongo_db for coll_name in [ "noc.profiles", "noc.services", "noc.serviceprofiles" ]: coll = mdb[coll_name] updates = [] for d in coll.find({"bi_id": {"$exists": False}}, {"_id": 1}): updates += [ UpdateOne( {"_id": d["_id"]}, {"$set": { "bi_id": bson.Int64(bi_hash(d["_id"])) }}) ] if len(updates) >= MONGO_CHUNK: coll.bulk_write(updates) updates = [] if updates: coll.bulk_write(updates) # Alter bi_id fields and create indexes for table in MODELS: self.db.execute("ALTER TABLE %s ALTER bi_id SET NOT NULL" % table) self.db.create_index(table, ["bi_id"], unique=True)
def forwards(self): # Update mongodb collections mdb = get_db() for coll_name in ["noc.firmwares", "noc.interface_profiles", "noc.networksegments", "noc.networksegmentprofiles", "noc.objects", "noc.platforms", "noc.vendors"]: coll = mdb[coll_name] updates = [] for d in coll.find({"bi_id": {"$exists": False}}, {"_id": 1}): updates += [ UpdateOne({ "_id": d["_id"] }, { "$set": { "bi_id": bson.Int64(bi_hash(d["_id"])) } }) ] if len(updates) >= MONGO_CHUNK: coll.bulk_write(updates) updates = [] if updates: coll.bulk_write(updates)
def extract(self): o = Object._get_collection().with_options( read_preference=ReadPreference.SECONDARY_PREFERRED) for obj in o.find( {}, { "_id": 1, "bi_id": 1, "name": 1, "container": 1, "data.address.text": 1 }, no_cursor_timeout=True, ): address = [ a for a in obj["data"] if a and a["interface"] == "address" and a["attr"] == "text" ] yield ( obj["bi_id"], obj["_id"], obj.get("name", ""), bi_hash(obj["container"]) if obj.get("container") else "", address[0] if address else "", )
def forwards(self): coll = get_db()["prefixprofiles"] default_id = bson.ObjectId() wf = bson.ObjectId("5a01d980b6f529000100d37a") profiles = [{ "_id": default_id, "name": "default", "description": "Default prefix profile", "workflow": wf, "bi_id": bson.int64.Int64(bi_hash(default_id)) }] # Convert styles style_profiles = {} for style_id, in db.execute("SELECT DISTINCT style_id FROM ip_prefix"): if not style_id: style_profiles[None] = default_id continue p_id = bson.ObjectId() p = { "_id": p_id, "name": "Style %s" % style_id, "description": "Auto-converted for style %s" % style_id, "workflow": wf, "style": style_id, "bi_id": bson.int64.Int64(bi_hash(p_id)) } style_profiles[style_id] = p_id profiles += [p] # Insert profiles to database coll.insert(profiles) # Create Prefix.profile field db.add_column( "ip_prefix", "profile", DocumentReferenceField("ip.PrefixProfile", null=True, blank=True)) # Migrate profile styles for style_id in style_profiles: if style_id: cond = "style_id = %s" % style_id else: cond = "style_id IS NULL" db.execute("UPDATE ip_prefix SET profile = %%s WHERE %s" % cond, [str(style_profiles[style_id])]) # Make Prefix.profile not nullable db.execute("ALTER TABLE ip_prefix ALTER profile SET NOT NULL") # Drop Prefix.style db.drop_column("ip_prefix", "style_id")
def get_data(self): # type: () -> Dict[str, Any] r = { "object": self.object, "paths": [], "link_sets": 0, "if_hash": {}, "error": None, "ajax_query_key": None, } # type: Dict[str, Any] mo = self.object.managed_object target_level = (mo.object_profile.level // 10 + 1) * 10 try: finder = KSPFinder( mo, ManagedObjectLevelGoal(target_level), constraint=self.get_constraint(), n_shortest=self.N_PATHS, ) for path in finder.iter_shortest_paths(): items = [] # type: List[Dict[str, Any]] ingress_links = [[self.object]] # type: List[List[Interface]] for pi in path: item = { "object": pi.start, "ingress": ingress_links, "egress": [], } # type: Dict[str, Any] ingress_links = [] for link in pi.links: egress, ingress = self.split_interfaces(pi.start, link.interfaces) ingress_links += [ingress] item["egress"] += [egress] if item["ingress"] == item["egress"]: item["ingress"] = [] r["link_sets"] = max(r["link_sets"], len(item["egress"])) items += [item] items += [{"object": pi.end, "ingress": ingress_links, "egress": []}] r["paths"] += [items] except ValueError as e: r["error"] = str(e) return r # Build interface hashes to_collect = set() # type: Set[Tuple[int, int, str]] for path in r["paths"]: for item in path: for direction in ("ingress", "egress"): for link in item[direction]: for iface in link: ifname = iface.name if ifname not in r["if_hash"]: r["if_hash"][ifname] = bi_hash(ifname) to_collect.add( (iface.managed_object.id, iface.managed_object.bi_id, ifname) ) # @todo: Encrypt r["ajax_query_key"] = self.encode_query(to_collect) return r
def fix_document(model): coll = model._get_collection() bulk = [] for d in coll.find({"bi_id": {"$exists": False}}, {"_id": 1}): bi_id = bi_hash(d["_id"]) bulk += [UpdateOne({"_id": d["_id"]}, {"$set": {"bi_id": bson.Int64(bi_id)}})] if bulk: print(" Update %d items" % len(bulk)) coll.bulk_write(bulk)
def migrate(self): # Update mongodb collections mdb = self.mongo_db for coll_name in ["noc.alarmclasses"]: coll = mdb[coll_name] updates = [] for d in coll.find({"bi_id": {"$exists": False}}, {"_id": 1}): updates += [ UpdateOne({"_id": d["_id"]}, {"$set": {"bi_id": bson.Int64(bi_hash(d["_id"]))}}) ] if len(updates) >= MONGO_CHUNK: coll.bulk_write(updates) updates = [] if updates: coll.bulk_write(updates)
def extract(self): o = Object._get_collection().with_options( read_preference=ReadPreference.SECONDARY_PREFERRED) for obj in o.find({}, { "_id": 1, "bi_id": 1, "name": 1, "container": 1, "data.address.text": 1 }, no_cursor_timeout=True): data = obj.get("data", {}) yield (obj["bi_id"], obj["_id"], obj.get("name", ""), bi_hash(obj["container"]) if obj.get("container") else "", data["address"].get("text", "") if data and "address" in data else "")
def forwards(self): # Update mongodb collections mdb = get_db() for coll_name in ["noc.metrictypes"]: coll = mdb[coll_name] updates = [] for d in coll.find({"bi_id": {"$exists": False}}, {"_id": 1}): updates += [ UpdateOne( {"_id": d["_id"]}, {"$set": { "bi_id": bson.Int64(bi_hash(d["_id"])) }}) ] if len(updates) >= MONGO_CHUNK: coll.bulk_write(updates) updates = [] if updates: coll.bulk_write(updates)
def migrate(self): table = "project_project" rows = self.db.execute("SELECT id FROM %s WHERE bi_id IS NULL" % table) values = ["(%d, %d)" % (r[0], bi_hash(r[0])) for r in rows] while values: chunk, values = values[:PG_CHUNK], values[PG_CHUNK:] self.db.execute( """ UPDATE %s AS t SET bi_id = c.bi_id FROM ( VALUES %s ) AS c(id, bi_id) WHERE c.id = t.id """ % (table, ",\n".join(chunk)) ) self.db.execute("ALTER TABLE %s ALTER bi_id SET NOT NULL" % table)
def get_ajax_data(self, **kwargs): # Parse query params query = self.decode_query( self.handler.get_argument("key") ) # type: List[Tuple[int, int, str]] # Get metrics from_ts = datetime.datetime.now() - datetime.timedelta(seconds=1800) from_ts = from_ts.replace(microsecond=0) interface_sql = """ SELECT managed_object, path[4] AS iface, argMax(load_in, ts) AS load_in, argMax(load_out, ts) AS load_out, argMax(packets_in, ts) AS packets_in, argMax(packets_out, ts) AS packets_out, argMax(errors_in, ts) AS errors_in, argMax(errors_out, ts) AS errors_out FROM interface WHERE date >= toDate('%s') AND ts >= toDateTime('%s') AND (%s) GROUP BY managed_object, iface """ % ( from_ts.date().isoformat(), from_ts.isoformat(sep=" "), " OR ".join( "(managed_object=%d AND path[4]='%s')" % (q[1], q[2].replace("'", "''")) for q in query ), ) # Get data metrics = [] # type: List[Tuple[int, str, str, str, str, str]] ch = ch_connection() try: for ( mo, iface, load_in, load_out, packets_in, packets_out, errors_in, errors_out, ) in ch.execute(post=interface_sql): if_hash = str(bi_hash(iface)) metrics += [ # (mo, if_hash, "speed", self.humanize_metric(speed)), (mo, if_hash, "load_in", self.humanize_metric(load_in)), (mo, if_hash, "load_out", self.humanize_metric(load_out)), (mo, if_hash, "packets_in", self.humanize_metric(packets_in)), (mo, if_hash, "packets_out", self.humanize_metric(packets_out)), (mo, if_hash, "errors_in", self.humanize_metric(errors_in)), (mo, if_hash, "errors_out", self.humanize_metric(errors_out)), ] except ClickhouseError: pass # Set defaults m_index = set() # type: Set[Tuple[int, str]] for mo_bi_id, iface, _, _ in metrics: m_index.add((int(mo_bi_id), iface)) interface_metrics = { "speed", "load_in", "load_out", "packets_in", "packets_out", "errors_in", "errors_out", } for _, mo_bi_id, iface in query: if (int(mo_bi_id), str(bi_hash(iface))) not in m_index: for metric in interface_metrics: metrics += [(str(mo_bi_id), str(bi_hash(iface)), metric, "-")] # managed object id -> bi id mo_map = {q[0]: q[1] for q in query} # type: Dict[int, int] # Get interface statuses for doc in Interface._get_collection().find( {"$or": [{"managed_object": q[0], "name": q[2]} for q in query]}, { "_id": 0, "managed_object": 1, "name": 1, "admin_status": 1, "oper_status": 1, "in_speed": 1, "out_speed": 1, "full_duplex": 1, }, ): mo = str(mo_map[doc["managed_object"]]) if_hash = str(bi_hash(doc["name"])) status = 0 if "admin_status" in doc and doc["admin_status"]: status = 2 if doc["oper_status"] else 1 duplex = "-" if "full_duplex" in doc: duplex = "Full" if doc["full_duplex"] else "Half" speed = "-" if "in_speed" in doc: speed = self.humanize_metric(doc["in_speed"] * 1000) metrics += [ (mo, if_hash, "speed", speed), (mo, if_hash, "duplex", duplex), (mo, if_hash, "status", status), ] # Get current object statuses obj_statuses = ObjectStatus.get_statuses(list(mo_map)) statuses = {str(mo_map[mo_id]): obj_statuses.get(mo_id, True) for mo_id in obj_statuses} return {"metrics": metrics, "statuses": list(statuses.items())}