class MapApplication(ExtApplication): """ inv.net application """ title = _("Network Map") menu = _("Network Map") glyph = "globe" implied_permissions = {"launch": ["inv:networksegment:lookup"]} # Object statuses ST_UNKNOWN = 0 # Object state is unknown ST_OK = 1 # Object is OK ST_ALARM = 2 # Object is reachable, Active alarms ST_UNREACH = 3 # Object is unreachable due to other's object failure ST_DOWN = 4 # Object is down ST_MAINTENANCE = 32 # Maintenance bit # Maximum object to be shown MAX_OBJECTS = 300 @view("^(?P<id>[0-9a-f]{24})/data/$", method=["GET"], access="read", api=True) def api_data(self, request, id): def q_mo(d): x = d.copy() if x["type"] == "managedobject": del x["mo"] x["external"] = x["id"] not in mos if is_view else x.get( "role") != "segment" # x["id"] = str(x["id"]) elif d["type"] == "cloud": del x["link"] x["external"] = False return x # Find segment segment = self.get_object_or_404(NetworkSegment, id=id) if segment.managed_objects.count() > self.MAX_OBJECTS: # Too many objects return { "id": str(segment.id), "name": segment.name, "error": _("Too many objects") } # if we set selector in segment is_view = segment.selector if is_view: mos = segment.selector.managed_objects.values_list("id", flat=True) # Load settings settings = MapSettings.objects.filter(segment=id).first() node_hints = {} link_hints = {} if settings: self.logger.info("Using stored positions") for n in settings.nodes: node_hints[n.id] = { "type": n.type, "id": n.id, "x": n.x, "y": n.y } for l in settings.links: link_hints[l.id] = { "connector": l.connector if len(l.vertices) else "normal", "vertices": [{ "x": v.x, "y": v.y } for v in l.vertices], } else: self.logger.info("Generating positions") # Generate topology topology = SegmentTopology( segment, node_hints, link_hints, force_spring=request.GET.get("force") == "spring") topology.layout() # Build output r = { "id": str(segment.id), "max_links": int(segment.max_shown_downlinks), "name": segment.name, "caps": list(topology.caps), "nodes": [q_mo(x) for x in six.itervalues(topology.G.node)], "links": [topology.G[u][v] for u, v in topology.G.edges()], } # Parent info if segment.parent: r["parent"] = { "id": str(segment.parent.id), "name": segment.parent.name } # Save settings if not settings: self.logger.debug("Saving first-time layout") MapSettings.load_json({ "id": str(segment.id), "nodes": [{ "type": n["type"], "id": n["id"], "x": n["x"], "y": n["y"] } for n in r["nodes"] if n.get("x") is not None and n.get("y") is not None], "links": [{ "type": n["type"], "id": n["id"], "vertices": n.get("vertices", []), "connector": n.get("connector", "normal"), } for n in r["links"]], }) return r @view("^(?P<id>[0-9a-f]{24})/data/$", method=["POST"], access="write", api=True) def api_save(self, request, id): self.get_object_or_404(NetworkSegment, id=id) data = self.deserialize(request.body) data["id"] = id MapSettings.load_json(data, request.user.username) return {"status": True} @view(url="^(?P<id>[0-9a-f]{24})/info/segment/$", method=["GET"], access="read", api=True) def api_info_segment(self, request, id): segment = self.get_object_or_404(NetworkSegment, id=id) r = { "name": segment.name, "description": segment.description, "objects": segment.managed_objects.count(), } return r @view( url="^(?P<id>[0-9a-f]{24})/info/managedobject/(?P<mo_id>\d+)/$", method=["GET"], access="read", api=True, ) def api_info_managedobject(self, request, id, mo_id): segment = self.get_object_or_404(NetworkSegment, id=id) object = self.get_object_or_404(ManagedObject, id=int(mo_id)) s = {1: "telnet", 2: "ssh", 3: "http", 4: "https"}[object.scheme] r = { "id": object.id, "name": object.name, "description": object.description, "address": object.address, "platform": object.platform.full_name if object.platform else "", "profile": object.profile.name, "external": object.segment.id != segment.id, "external_segment": { "id": str(object.segment.id), "name": object.segment.name }, "caps": object.get_caps(), "console_url": "%s://%s/" % (s, object.address), } return r @view( url="^(?P<id>[0-9a-f]{24})/info/link/(?P<link_id>[0-9a-f]{24})/$", method=["GET"], access="read", api=True, ) def api_info_link(self, request, id, link_id): def q(s): if isinstance(s, unicode): s = s.encode("utf-8") return s self.get_object_or_404(NetworkSegment, id=id) link = self.get_object_or_404(Link, id=link_id) r = { "id": str(link.id), "name": link.name or None, "description": link.description or None, "objects": [], "method": link.discovery_method, } o = defaultdict(list) for i in link.interfaces: o[i.managed_object] += [i] for mo in sorted(o, key=lambda x: x.name): r["objects"] += [{ "id": mo.id, "name": mo.name, "interfaces": [{ "name": i.name, "description": i.description or None, "status": i.status } for i in sorted(o[mo], key=lambda x: split_alnum(x.name))], }] # Get link bandwidth mo_in = defaultdict(float) mo_out = defaultdict(float) mos = [ManagedObject.get_by_id(mo["id"]) for mo in r["objects"]] metric_map, last_ts = get_interface_metrics(list(six.iterkeys(o))) for mo in o: if mo not in metric_map: continue for i in o[mo]: if i.name not in metric_map[mo]: continue mo_in[mo] += metric_map[mo][i.name]["Interface | Load | In"] mo_out[mo] += metric_map[mo][i.name]["Interface | Load | Out"] if len(mos) == 2: mo1, mo2 = mos r["utilisation"] = [ int(max(mo_in[mo1], mo_out[mo2])), int(max(mo_in[mo2], mo_out[mo1])), ] else: mv = list(six.itervalues(mo_in)) + list(six.itervalues(mo_out)) if mv: r["utilisation"] = [int(max(mv))] else: r["utilisation"] = 0 return r @view( url="^(?P<id>[0-9a-f]{24})/info/cloud/(?P<link_id>[0-9a-f]{24})/$", method=["GET"], access="read", api=True, ) def api_info_cloud(self, request, id, link_id): def q(s): if isinstance(s, unicode): s = s.encode("utf-8") return s self.get_object_or_404(NetworkSegment, id=id) link = self.get_object_or_404(Link, id=link_id) r = { "id": str(link.id), "name": link.name or None, "description": link.description or None, "objects": [], "method": link.discovery_method, } o = defaultdict(list) for i in link.interfaces: o[i.managed_object] += [i] for mo in sorted(o, key=lambda x: x.name): r["objects"] += [{ "id": mo.id, "name": mo.name, "interfaces": [{ "name": i.name, "description": i.description or None, "status": i.status } for i in sorted(o[mo], key=lambda x: split_alnum(x.name))], }] return r @view( url="^objects_statuses/$", method=["POST"], access="read", api=True, validate={"objects": ListOfParameter(IntParameter())}, ) def api_objects_statuses(self, request, objects): def get_alarms(objects): """ Returns a set of objects with alarms """ r = set() c = ActiveAlarm._get_collection() while objects: chunk, objects = objects[:500], objects[500:] a = c.aggregate([ { "$match": { "managed_object": { "$in": chunk } } }, { "$group": { "_id": "$managed_object", "count": { "$sum": 1 } } }, ]) r.update([d["_id"] for d in a]) return r def get_maintenance(objects): """ Returns a set of objects currently in maintenance :param objects: :return: """ now = datetime.datetime.now() so = set(objects) r = set() for m in Maintenance._get_collection().find( { "is_completed": False, "start": { "$lte": now } }, { "_id": 0, "affected_objects": 1 }): mo = set(r["object"] for r in m["affected_objects"]) r |= so & mo return r # Mark all as unknown r = dict((o, self.ST_UNKNOWN) for o in objects) sr = ObjectStatus.get_statuses(objects) sa = get_alarms(objects) mo = get_maintenance(objects) for o in sr: if sr[o]: # Check for alarms if o in sa: r[o] = self.ST_ALARM else: r[o] = self.ST_OK else: r[o] = self.ST_DOWN if o in mo: r[o] |= self.ST_MAINTENANCE return r @classmethod @cachedmethod(key="managedobject-name-to-id-%s", lock=lambda _: tags_lock) def managedobject_name_to_id(cls, name): r = ManagedObject.objects.filter(name=name).values_list("id") if r: return r[0][0] else: return None @classmethod @cachedmethod(key="interface-tags-to-id-%s-%s", lock=lambda _: tags_lock) def interface_tags_to_id(cls, object_name, interface_name): mo = cls.managedobject_name_to_id(object_name) i = Interface._get_collection().find_one({ "managed_object": mo, "name": interface_name }) if i: return i["_id"] else: return None @view( url="^metrics/$", method=["POST"], access="read", api=True, validate={ "metrics": DictListParameter( attrs={ "id": StringParameter(), "metric": StringParameter(), "tags": DictParameter(), }) }, ) def api_metrics(self, request, metrics): def q(s): if isinstance(s, unicode): s = s.encode("utf-8") return s def qt(t): return "|".join(["%s=%s" % (v, t[v]) for v in sorted(t)]) # Build query tag_id = {} # object, interface -> id if_ids = {} # id -> port id mlst = [] # (metric, object, interface) for m in metrics: if "object" in m["tags"] and "interface" in m["tags"]: if not m["tags"]["object"]: continue try: if_ids[self.interface_tags_to_id( m["tags"]["object"], m["tags"]["interface"])] = m["id"] object = ManagedObject.objects.get( name=m["tags"]["object"]) tag_id[object, m["tags"]["interface"]] = m["id"] mlst += [(m["metric"], object, m["tags"]["interface"])] except KeyError: pass # @todo: Get last values from cache if not mlst: return {} r = {} # Apply interface statuses for d in Interface._get_collection().find( {"_id": { "$in": list(if_ids) }}, { "_id": 1, "admin_status": 1, "oper_status": 1 }): r[if_ids[d["_id"]]] = { "admin_status": d.get("admin_status", True), "oper_status": d.get("oper_status", True), } metric_map, last_ts = get_interface_metrics([m[1] for m in mlst]) # Apply metrics for rq_mo, rq_iface in tag_id: pid = tag_id.get((rq_mo, rq_iface)) if not pid: continue if pid not in r: r[pid] = {} if rq_mo not in metric_map: continue if rq_iface not in metric_map[rq_mo]: continue r[pid]["Interface | Load | In"] = metric_map[rq_mo][rq_iface][ "Interface | Load | In"] r[pid]["Interface | Load | Out"] = metric_map[rq_mo][rq_iface][ "Interface | Load | Out"] return r @view("^(?P<id>[0-9a-f]{24})/data/$", method=["DELETE"], access="write", api=True) def api_reset(self, request, id): self.get_object_or_404(NetworkSegment, id=id) MapSettings.objects.filter(segment=id).delete() return {"status": True} @view( url="^stp/status/$", method=["POST"], access="read", api=True, validate={"objects": ListOfParameter(IntParameter())}, ) def api_objects_stp_status(self, request, objects): def get_stp_status(object_id): roots = set() blocked = set() object = ManagedObject.get_by_id(object_id) sr = object.scripts.get_spanning_tree() for instance in sr["instances"]: ro = DiscoveryID.find_object(instance["root_id"]) if ro: roots.add(ro) for i in instance["interfaces"]: if i["state"] == "discarding" and i["role"] == "alternate": iface = object.get_interface(i["interface"]) if iface: link = iface.link if link: blocked.add(str(link.id)) return object_id, roots, blocked r = {"roots": [], "blocked": []} futures = [] with ThreadPoolExecutor(max_workers=10) as executor: for o in objects: futures += [executor.submit(get_stp_status, o)] for future in as_completed(futures): try: obj, roots, blocked = future.result() for ro in roots: if ro.id not in r["roots"]: r["roots"] += [ro.id] r["blocked"] += blocked except Exception as e: self.logger.error("[stp] Exception: %s", e) return r
"remote_system": StringParameter(), "remote_id": StringParameter() }) Pointer = PointerId | PointerRemote # from: section ObjectPointer = DictParameter( attrs={ "object": Pointer, "interface": DictParameter(attrs={"name": StringParameter()}, required=False), }) InterfacePointer = DictParameter( attrs={"interface": DictParameter(attrs={"id": ObjectIdParameter()})}) LevelPointer = DictParameter(attrs={"level": IntParameter()}) ServicePointer = DictParameter(attrs={"service": Pointer}) RequestFrom = ObjectPointer | InterfacePointer | ServicePointer # to: section RequestTo = ObjectPointer | LevelPointer | InterfacePointer | ServicePointer # config: section RequestConfig = DictParameter( attrs={ "max_depth": IntParameter(default=MAX_DEPTH_DEFAULT), "n_shortest": IntParameter(default=N_SHORTEST_DEFAULT), }, required=False, ) # constraints: section RequestVLANConstraint = DictParameter( attrs={
class VCApplication(ExtModelApplication): """ VC application """ title = _("VC") menu = _("Virtual Circuits") model = VC query_fields = ["name", "description"] query_condition = "icontains" int_query_fields = ["l1", "l2"] implied_permissions = {"read": ["vc:vcdomain:lookup", "main:style:lookup"]} def get_vc_domain_objects(self, vc_domain): return vc_domain.managedobject_set.all() def lookup_vcfilter(self, q, name, value): """ Resolve __vcflter lookups :param q: :param name: :param value: :return: """ value = ModelParameter(VCFilter).clean(value) x = value.to_sql(name) try: q[None] += [x] except KeyError: q[None] = [x] @cachedmethod(key="vc-interface-count-%s") def get_vc_interfaces_count(self, vc_id): vc = VC.get_by_id(vc_id) if not vc: return 0 objects = vc.vc_domain.managedobject_set.values_list("id", flat=True) l1 = vc.l1 n = SubInterface.objects.filter( Q(managed_object__in=objects) & (Q(untagged_vlan=l1, enabled_afi=["BRIDGE"]) | Q(tagged_vlans=l1, enabled_afi=["BRIDGE"]) | Q(vlan_ids=l1))).count() return n @cachedmethod(key="vc-prefixes-%s") def get_vc_prefixes(self, vc_id): vc = VC.get_by_id(vc_id) if not vc: return [] objects = vc.vc_domain.managedobject_set.values_list("id", flat=True) ipv4 = set() ipv6 = set() # @todo: Exact match on vlan_ids for si in SubInterface.objects.filter( Q(managed_object__in=objects) & Q(vlan_ids=vc.l1) & (Q(enabled_afi=["IPv4"]) | Q(enabled_afi=["IPv6"]))).only( "enabled_afi", "ipv4_addresses", "ipv6_addresses"): if "IPv4" in si.enabled_afi: ipv4.update([IP.prefix(ip).first for ip in si.ipv4_addresses]) if "IPv6" in si.enabled_afi: ipv6.update([IP.prefix(ip).first for ip in si.ipv6_addresses]) p = [str(x.first) for x in sorted(ipv4)] p += [str(x.first) for x in sorted(ipv6)] return p def field_interfaces_count(self, obj): return self.get_vc_interfaces_count(obj.id) def field_prefixes(self, obj): p = self.get_vc_prefixes(obj.id) if p: return ", ".join(p) else: return "-" def field_row_class(self, o): return o.style.css_class_name if o.style else "" @view( url="^find_free/$", method=["GET"], access="read", api=True, validate={ "vc_domain": ModelParameter(VCDomain), "vc_filter": ModelParameter(VCFilter) }, ) def api_find_free(self, request, vc_domain, vc_filter, **kwargs): return vc_domain.get_free_label(vc_filter) @view( url="^bulk/import/", method=["POST"], access="import", api=True, validate={ "vc_domain": ModelParameter(VCDomain), "items": ListOfParameter(element=DictParameter( attrs={ "l1": IntParameter(), "l2": IntParameter(), "name": StringParameter(), "description": StringParameter(default=""), })), }, ) def api_bulk_import(self, request, vc_domain, items): n = 0 for i in items: if not VC.objects.filter( vc_domain=vc_domain, l1=i["l1"], l2=i["l2"]).exists(): # Add only not-existing VC( vc_domain=vc_domain, l1=i["l1"], l2=i["l2"], name=i["name"], description=i["description"], ).save() n += 1 return {"status": True, "imported": n} @view(url=r"^(?P<vc_id>\d+)/interfaces/$", method=["GET"], access="read", api=True) def api_interfaces(self, request, vc_id): """ Returns a dict of {untagged: ..., tagged: ...., l3: ...} :param request: :param vc_id: :return: """ vc = self.get_object_or_404(VC, id=int(vc_id)) l1 = vc.l1 # Managed objects in VC domain objects = set( vc.vc_domain.managedobject_set.values_list("id", flat=True)) # Find untagged interfaces si_objects = defaultdict(list) for si in SubInterface.objects.filter(managed_object__in=objects, untagged_vlan=l1, enabled_afi="BRIDGE"): si_objects[si.managed_object] += [{"name": si.name}] untagged = [{ "managed_object_id": o.id, "managed_object_name": o.name, "interfaces": sorted(si_objects[o], key=lambda x: x["name"]), } for o in si_objects] # Find tagged interfaces si_objects = defaultdict(list) for si in SubInterface.objects.filter(managed_object__in=objects, tagged_vlans=l1, enabled_afi="BRIDGE"): si_objects[si.managed_object] += [{"name": si.name}] tagged = [{ "managed_object_id": o.id, "managed_object_name": o.name, "interfaces": sorted(si_objects[o], key=lambda x: x["name"]), } for o in si_objects] # Find l3 interfaces si_objects = defaultdict(list) for si in SubInterface.objects.filter(managed_object__in=objects, vlan_ids=l1): si_objects[si.managed_object] += [{ "name": si.name, "ipv4_addresses": si.ipv4_addresses, "ipv6_addresses": si.ipv6_addresses, }] l3 = [{ "managed_object_id": o.id, "managed_object_name": o.name, "interfaces": sorted(si_objects[o], key=lambda x: x["name"]), } for o in si_objects] # Update caches ic = sum(len(x["interfaces"]) for x in untagged) ic += sum(len(x["interfaces"]) for x in tagged) ic += sum(len(x["interfaces"]) for x in l3) # return { "untagged": sorted(untagged, key=lambda x: x["managed_object_name"]), "tagged": sorted(tagged, key=lambda x: x["managed_object_name"]), "l3": sorted(l3, key=lambda x: x["managed_object_name"]), }
from .error import ModelDataError from noc.lib.utils import deep_copy from noc.lib.escape import json_escape as q from noc.sa.interfaces.base import ( StringParameter, BooleanParameter, FloatParameter, IntParameter, StringListParameter, ) id_lock = Lock() T_MAP = { "str": StringParameter(), "int": IntParameter(), "float": FloatParameter(), "bool": BooleanParameter(), "strlist": StringListParameter(), } A_TYPE = ["str", "int", "float", "bool", "objectid", "ref", "strlist"] @six.python_2_unicode_compatible class ModelInterfaceAttr(EmbeddedDocument): meta = {"strict": False, "auto_create_index": False} name = StringField() type = StringField(choices=[(t, t) for t in A_TYPE]) description = StringField() required = BooleanField(default=False)