def get_data(self, request, pool=None, obj_profile=None, **kwargs):
data = []
if pool:
pool = Pool.get_by_id(pool)
else:
pool = Pool.get_by_name("default")
# Get all managed objects
mos = ManagedObject.objects.filter(is_managed=True, pool=pool)
if not request.user.is_superuser:
mos = ManagedObject.objects.filter(
is_managed=True,
pool=pool,
administrative_domain__in=UserAccess.get_domains(request.user),
)
if obj_profile:
mos = mos.filter(object_profile=obj_profile)
columns = (_("Managed Object"), _("Address"), _("Object"),
_("Capabilities"))
for mo in mos:
mo.get_caps()
data += [(mo.name, mo.address, _("Main"), ";".join(mo.get_caps()))]
for i in Interface.objects.filter(managed_object=mo):
if i.type == "SVI":
continue
data += [(mo.name, mo.address, i.name,
";".join(i.enabled_protocols))]
return self.from_dataset(title=self.title, columns=columns, data=data)
def get_data(self, request, repo="config", days=1, **kwargs):
data = []
baseline = datetime.datetime.now() - datetime.timedelta(days=days)
if repo == "config":
mos = ManagedObject.objects.filter()
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
mos = dict(mos.values_list("id", "name"))
config_db = get_db()["noc.gridvcs.config.files"].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED)
pipeline = [{"$match": {"ts": {"$gte": baseline}}},
{"$group": {"_id": "$object", "last_ts": {"$max": "$ts"}}},
{"$sort": {"_id": 1}}]
for value in config_db.aggregate(pipeline):
if value["_id"] not in mos:
continue
data += [(mos[value["_id"]], value["last_ts"])]
else:
oc = Object.get_object_class(repo)
data = [(o, o.last_modified) for o in
oc.objects.filter(last_modified__gte=baseline).order_by("-last_modified")]
return self.from_dataset(
title="%s: %s in %d days" % (self.title, repo, days),
columns=[
"Object",
TableColumn(_("Last Changed"), format="datetime")],
data=data,
enumerate=True)
def get_report_object(self,
user=None,
is_managed=None,
adm=None,
selector=None,
pool=None,
segment=None,
ids=None):
mos = ManagedObject.objects.filter()
if user.is_superuser and not adm and not selector and not segment:
mos = ManagedObject.objects.filter()
if ids:
mos = ManagedObject.objects.filter(id__in=[ids])
if is_managed is not None:
mos = ManagedObject.objects.filter(is_managed=is_managed)
if pool:
p = Pool.get_by_name(pool or "default")
mos = mos.filter(pool=p)
if not user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(user))
if adm:
ads = AdministrativeDomain.get_nested_ids(int(adm))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if segment:
segment = NetworkSegment.objects.filter(id=segment).first()
if segment:
mos = mos.filter(segment__in=segment.get_nested_ids())
return mos
def transform_query(cls, query, user):
"""
Transform query, possibly applying access restrictions
:param query:
:param user:
:return: query dict or None if access denied
"""
if not user or user.is_superuser:
return query # No restrictions
if query["datasource"].startswith("dictionaries."):
return query # To dictionaries
# Get user domains
domains = UserAccess.get_domains(user)
# Resolve domains against dict
mos_bi = [
int(bi_id) for bi_id in ManagedObject.objects.filter(
administrative_domain__in=domains).values_list("bi_id",
flat=True)
]
filter = query.get("filter", {})
dl = len(mos_bi)
if not dl:
return None
elif dl == 1:
q = {"$eq": [{"$field": "managed_object"}, mos_bi]}
else:
q = {"$in": [{"$field": "managed_object"}, mos_bi]}
if filter:
query["filter"] = {"$and": [query["filter"], q]}
else:
query["filter"] = q
return query
def queryset(self, request, query=None):
"""
Filter records for lookup
"""
qs = super().queryset(request)
if not request.user.is_superuser:
user_ads = UserAccess.get_domains(request.user)
qs = qs.filter(Q(administrative_domain=[]) | Q(administrative_domain__in=user_ads))
if query and self.query_fields:
q = qs.filter(self.get_Q(request, query))
if q:
return q
sq = ManagedObject.get_search_Q(query)
if sq:
obj = ManagedObject.objects.filter(sq)
else:
obj = ManagedObject.objects.filter(name__contains=query)
if obj:
mos = obj.values_list("id", flat=True)
ao = AffectedObjects.objects.filter(affected_objects__object__in=mos).values_list(
"maintenance"
)
return ao
return qs.filter(type=None)
else:
return qs
def get_data(self,
request,
interval,
from_date=None,
to_date=None,
**kwargs):
interval = int(interval)
if not from_date:
interval = 1
if interval:
ts = datetime.datetime.now() - datetime.timedelta(days=interval)
q = {"timestamp": {"$gte": ts}}
else:
t0 = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date:
t1 = datetime.datetime.now()
else:
t1 = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
q = {"timestamp": {"$gte": t0, "$lte": t1}}
q["escalation_tt"] = {"$exists": True}
if not request.user.is_superuser:
q["adm_path"] = {"$in": UserAccess.get_domains(request.user)}
data = []
for ac in (ActiveAlarm, ArchivedAlarm):
for d in ac._get_collection().find(q):
mo = ManagedObject.get_by_id(d["managed_object"])
if not mo:
continue
data += [(
d["timestamp"].strftime("%Y-%m-%d %H:%M:%S"),
d["escalation_ts"].strftime("%Y-%m-%d %H:%M:%S"),
mo.name.split("#", 1)[0],
mo.address,
mo.platform,
mo.segment.name,
d["escalation_tt"],
sum(ss["summary"] for ss in d["total_objects"]),
sum(ss["summary"] for ss in d["total_subscribers"]),
)]
data = sorted(data, key=operator.itemgetter(0))
return self.from_dataset(
title=self.title,
columns=[
_("Timestamp"),
_("Escalation Timestamp"),
_("Managed Object"),
_("Address"),
_("Platform"),
_("Segment"),
_("TT"),
_("Objects"),
_("Subscribers"),
],
data=data,
enumerate=True,
)
def get_data(self, request, pool=None, selector=None, report_type=None, **kwargs):
data = []
columns, columns_desr = [], []
r_map = [
(_("Not Available"), "2is1.3isp1.3is1"),
(_("Failed to guess CLI credentials"), "2is1.3isp0.2isp1"),
(_("Failed to guess SNMP community"), "2is1.3isp1.3is2.1isp1")
]
for x, y in r_map:
columns += [y]
columns_desr += [x]
mos = ManagedObject.objects.filter()
if pool:
mos = mos.filter(pool=pool)
data += [SectionRow(name=pool.name)]
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
mos = list(mos.values_list("id", flat=True).order_by("id"))
mos_s = set(mos)
report = ReportModelFilter()
result = report.proccessed(",".join(columns))
mo_hostname = ReportObjectsHostname1(sync_ids=mos)
mo_hostname = mo_hostname.get_dictionary()
d_result = ReportDiscoveryResult(sync_ids=mos)
d_result = d_result.get_dictionary()
for col in columns:
for mo_id in result[col.strip()].intersection(mos_s):
mo = ManagedObject.get_by_id(mo_id)
problem = self.decode_problem(d_result.get(mo_id))
if not problem and mo_id not in d_result:
problem = "Discovery disabled"
data += [(
mo.name,
mo.address,
mo.administrative_domain.name,
mo.profile.name,
mo_hostname.get(mo.id, ""),
mo.auth_profile if mo.auth_profile else "",
mo.auth_profile.user if mo.auth_profile else mo.user,
mo.auth_profile.snmp_ro if mo.auth_profile else mo.snmp_ro,
_("No") if not mo.get_status() else _("Yes"),
columns_desr[columns.index(col)],
problem
)]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"), _("Address"), _("Administrative Domain"), _("Profile"), _("Hostname"),
_("Auth Profile"), _("Username"), _("SNMP Community"),
_("Avail"), _("Error"), _("Error Detail")
],
data=data)
def get_data(self, request, sel=None):
qs = ManagedObject.objects
if not request.user.is_superuser:
qs = ManagedObject.objects.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
# Get all managed objects by selector
mos_list = qs.filter(sel.Q)
columns = [
_("Managed Objects"),
_("Address"),
_("Vendor"),
_("Platform"),
_("HW Revision"),
_("SW Version"),
_("Serial"),
]
data = []
for mo in mos_list:
q = Object._get_collection().count_documents(
{"data.management.managed_object": {
"$in": [mo.id]
}})
if q == 0:
data += [[
mo.name,
mo.address,
mo.vendor or None,
mo.platform.full_name if mo.platform else None,
mo.get_attr("HW version") or None,
mo.version.version if mo.version else None,
mo.get_attr("Serial Number") or None,
None,
]]
else:
for x in Object._get_collection().find(
{"data.management.managed_object": {
"$in": [mo.id]
}}):
data += [[
x["name"],
mo.address,
mo.vendor or None,
mo.platform.full_name if mo.platform else None,
mo.get_attr("HW version") or None,
mo.version.version if mo.version else None,
x["data"]["asset"]["serial"],
]]
return self.from_dataset(title=self.title,
columns=columns,
data=data,
enumerate=True)
def has_repo_config_access(self, user, obj):
"""
Check user has access to object
:param user: User instance
:param obj: ManagedObject instance
:return: True if user has access, False otherwise
"""
if user.is_superuser:
return True
return ManagedObject.objects.filter(id=obj.id).filter(UserAccess.Q(user)).exists()
def get_data(self,
request,
pool=None,
int_profile=None,
mop=None,
avail_status=None,
**kwargs):
data = []
mos = ManagedObject.objects.filter(is_managed=True)
# % fixme remove.
if not pool and request.user.is_superuser:
pool = Pool.get_by_name("STAGEMO")
if pool:
mos = mos.filter(pool=pool)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if mop:
mos = mos.filter(object_profile=mop)
mos_ids = mos.values_list("id", flat=True)
iface = Interface.objects.filter(managed_object__in=mos,
profile=int_profile,
type="physical").values_list(
"managed_object", "name")
res = []
n = 0
# Interface._get_collection()
while mos_ids[(0 + n):(10000 + n)]:
mos_ids_f = mos_ids[(0 + n):(10000 + n)]
s_iface = {"%d.%s" % (mo.id, name) for mo, name in iface}
of = ObjectFact.objects.filter(
object__in=mos_ids_f,
cls="subinterface",
attrs__traffic_control_broadcast=False)
a_f = {".".join((str(o.object.id), o.attrs["name"])) for o in of}
res.extend(a_f.intersection(s_iface))
n += 10000
for s in res:
mo, iface = s.split(".")
mo = ManagedObject.get_by_id(mo)
data += [(mo.name, mo.address, mo.profile.name, iface)]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"),
_("Address"),
_("SA Profile"),
_("Interface")
],
data=data,
)
def queryset(self, request, query=None):
"""
Filter records for lookup
"""
status = request.GET.get("status", "A")
if status not in self.model_map:
raise Exception("Invalid status")
model = self.model_map[status]
if request.user.is_superuser:
return model.objects.filter().read_preference(ReadPreference.SECONDARY_PREFERRED).all()
else:
return model.objects.filter(
adm_path__in=UserAccess.get_domains(request.user),
).read_preference(ReadPreference.SECONDARY_PREFERRED)
def search(cls, handler, query):
q = Q(name__icontains=query)
sq = ManagedObject.get_search_Q(query)
if sq:
q |= sq
if not handler.current_user.is_superuser:
q &= UserAccess.Q(handler.current_user)
r = []
for mo in ManagedObject.objects.filter(q):
r += [{
"scope": "managedobject",
"id": mo.id,
"label": "%s (%s) [%s]" % (mo.name, mo.address, mo.platform),
}]
return r
def queryset(self, request, query=None):
"""
Filter records for lookup
"""
self.logger.info("Queryset %s" % query)
if self.managed_filter:
q = d_Q(is_managed=True)
else:
q = d_Q()
if not request.user.is_superuser:
q &= UserAccess.Q(request.user)
if query:
sq = ManagedObject.get_search_Q(query)
if sq:
q &= sq
else:
q &= d_Q(name__contains=query)
return self.model.objects.filter(q)
def transform_query(cls, query, user):
if not user or user.is_superuser:
return query # No restrictions
# Get user domains
domains = UserAccess.get_domains(user)
# Resolve domains against dict
domain_ids = [x.bi_id for x in AdministrativeDomainM.objects.filter(id__in=domains)]
filter = query.get("filter", {})
dl = len(domain_ids)
if not dl:
return None
elif dl == 1:
q = {"$eq": [{"$field": "administrative_domain"}, domain_ids[0]]}
else:
q = {"$in": [{"$field": "administrative_domain"}, domain_ids]}
if filter:
query["filter"] = {"$and": [query["filter"], q]}
else:
query["filter"] = q
return query
def get_data(self,
request,
pool=None,
obj_profile=None,
selector=None,
avail_status=None,
profile_check_only=None,
failed_scripts_only=None,
filter_pending_links=None,
filter_none_objects=None,
filter_view_other=None,
**kwargs):
data = []
match = None
code_map = {
"1": "Unknown error",
"10000": "Unspecified CLI error",
"10005": "Connection refused",
"10001": "Authentication failed",
"10002": "No super command defined",
"10003": "No super privileges",
"10004": "SSH Protocol error"
}
if not pool:
pool = Pool.objects.filter()[0]
data += [SectionRow(name="Report by %s" % pool.name)]
if selector:
mos = ManagedObject.objects.filter(selector.Q)
else:
mos = ManagedObject.objects.filter(pool=pool, is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if obj_profile:
mos = mos.filter(object_profile=obj_profile)
if filter_view_other:
mnp_in = list(
ManagedObjectProfile.objects.filter(enable_ping=False))
mos = mos.filter(profile=Profile.objects.get(
name=GENERIC_PROFILE)).exclude(object_profile__in=mnp_in)
discovery = "noc.services.discovery.jobs.box.job.BoxDiscoveryJob"
mos_id = list(mos.values_list("id", flat=True))
if avail_status:
avail = ObjectStatus.get_statuses(mos_id)
if profile_check_only:
match = {
"$or": [{
"job.problems.suggest_cli": {
"$exists": True
}
}, {
"job.problems.suggest_snmp": {
"$exists": True
}
}, {
"job.problems.profile.": {
"$regex": "Cannot detect profile"
}
}, {
"job.problems.version.": {
"$regex": "Remote error code 1000[1234]"
}
}]
}
elif failed_scripts_only:
match = {
"$and": [{
"job.problems": {
"$exists": "true",
"$ne": {}
}
}, {
"job.problems.suggest_snmp": {
"$exists": False
}
}, {
"job.problems.suggest_cli": {
"$exists": False
}
}]
}
elif filter_view_other:
match = {"job.problems.suggest_snmp": {"$exists": False}}
rdp = ReportDiscoveryProblem(mos, avail_only=avail_status, match=match)
exclude_method = []
if filter_pending_links:
exclude_method += ["lldp", "lacp", "cdp", "huawei_ndp"]
for discovery in rdp:
mo = ManagedObject.get_by_id(discovery["key"])
for method in ifilterfalse(lambda x: x in exclude_method,
discovery["job"][0]["problems"]):
problem = discovery["job"][0]["problems"][method]
if filter_none_objects and not problem:
continue
if isinstance(problem, dict) and "" in problem:
problem = problem.get("", "")
if "Remote error code" in problem:
problem = code_map.get(problem.split(" ")[-1], problem)
if isinstance(problem, six.string_types):
problem = problem.replace("\n", " ").replace("\r", " ")
data += [(mo.name, mo.address,
mo.profile.name, mo.administrative_domain.name,
_("Yes") if mo.get_status() else _("No"),
discovery["st"].strftime("%d.%m.%Y %H:%M")
if "st" in discovery else "", method, problem)]
return self.from_dataset(title=self.title,
columns=[
_("Managed Object"),
_("Address"),
_("Profile"),
_("Administrative Domain"),
_("Avail"),
_("Last successful discovery"),
_("Discovery"),
_("Error")
],
data=data)
def get_data(self, request, report_type=None, **kwargs):
wr = ("", "")
# wr_and = ("", "",)
wr_and2 = ("", "")
platform = {
str(p["_id"]): p["name"]
for p in Platform.objects.all().as_pymongo().scalar("id", "name")
}
version = {
str(p["_id"]): p["version"]
for p in Firmware.objects.all().as_pymongo().scalar(
"id", "version")
}
profile = {
str(p["_id"]): p["name"]
for p in Profile.objects.all().as_pymongo().scalar("id", "name")
}
if not request.user.is_superuser:
ad = tuple(UserAccess.get_domains(request.user))
wr = ("WHERE administrative_domain_id in ", ad)
# wr_and = ("AND sam.administrative_domain_id in ", ad)
wr_and2 = ("AND administrative_domain_id in ", ad)
if len(ad) == 1:
wr = ("WHERE administrative_domain_id in (%s)" % ad, "")
# wr_and = ("AND sam.administrative_domain_id in (%s)" % ad, "")
wr_and2 = ("AND administrative_domain_id in (%s)" % ad, "")
# By Profile
if report_type == "profile":
columns = [_("Profile")]
query = ("""SELECT profile,COUNT(*) FROM sa_managedobject
%s%s GROUP BY 1 ORDER BY 2 DESC""" % wr)
# By Administrative Domain
elif report_type == "domain":
columns = [_("Administrative Domain")]
query = ("""SELECT a.name,COUNT(*)
FROM sa_managedobject o JOIN sa_administrativedomain a ON (o.administrative_domain_id=a.id)
%s%s
GROUP BY 1
ORDER BY 2 DESC""" % wr)
# By Profile and Administrative Domains
elif report_type == "domain-profile":
columns = [_("Administrative Domain"), _("Profile")]
query = ("""SELECT d.name,profile,COUNT(*)
FROM sa_managedobject o JOIN sa_administrativedomain d ON (o.administrative_domain_id=d.id)
%s%s
GROUP BY 1,2
""" % wr)
# By tags
elif report_type == "tag":
columns = [_("Tag")]
query = ("""
SELECT t.tag, COUNT(*)
FROM (
SELECT unnest(tags) AS tag
FROM sa_managedobject
WHERE
tags IS NOT NULL
%s%s
AND array_length(tags, 1) > 0
) t
GROUP BY 1
ORDER BY 2 DESC;
""" % wr_and2)
elif report_type == "platform":
columns = [_("Platform"), _("Profile")]
query = ("""select sam.profile, sam.platform, COUNT(platform)
from sa_managedobject sam %s%s group by 1,2 order by count(platform) desc;"""
% wr)
elif report_type == "version":
columns = [_("Profile"), _("Version")]
query = ("""select sam.profile, sam.version, COUNT(version)
from sa_managedobject sam %s%s group by 1,2 order by count(version) desc;"""
% wr)
else:
raise Exception("Invalid report type: %s" % report_type)
for r, t in report_types:
if r == report_type:
title = self.title + ": " + t
break
columns += [
TableColumn(_("Quantity"),
align="right",
total="sum",
format="integer")
]
cursor = self.cursor()
cursor.execute(query, ())
data = []
for c in cursor.fetchall():
if report_type == "profile":
data += [(profile.get(c[0]), c[1])]
elif report_type == "domain-profile":
data += [(c[0], profile.get(c[1]), c[2])]
elif report_type == "platform":
data += [(profile.get(c[0]), platform.get(c[1]), c[2])]
elif report_type == "version":
data += [(profile.get(c[0]), version.get(c[1]), c[2])]
else:
data += [c]
return self.from_dataset(title=title,
columns=columns,
data=data,
enumerate=True)
def get_user_domains(self):
return UserAccess.get_domains(self.current_user)
def api_report(
self,
request,
o_format,
is_managed=None,
administrative_domain=None,
selector=None,
pool=None,
segment=None,
avail_status=False,
columns=None,
ids=None,
enable_autowidth=False,
):
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, str):
return smart_text(v)
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
type_columns = ["Up/10G", "Up/1G", "Up/100M", "Down/-", "-"]
cols = [
"object1_admin_domain",
# "id",
"object1_name",
"object1_address",
"object1_platform",
"object1_segment",
"object1_tags",
"object1_iface",
"object1_descr",
"object1_speed",
"object2_admin_domain",
"object2_name",
"object2_address",
"object2_platform",
"object2_segment",
"object2_tags",
"object2_iface",
"object2_descr",
"object2_speed",
"link_proto",
"last_seen",
]
header_row = [
"OBJECT1_ADMIN_DOMAIN",
"OBJECT1_NAME",
"OBJECT1_ADDRESS",
"OBJECT1_PLATFORM",
"OBJECT1_SEGMENT",
"OBJECT1_TAGS",
"OBJECT1_IFACE",
"OBJECT1_DESCR",
"OBJECT1_SPEED",
"OBJECT2_ADMIN_DOMAIN",
"OBJECT2_NAME",
"OBJECT2_ADDRESS",
"OBJECT2_PLATFORM",
"OBJECT2_SEGMENT",
"OBJECT2_TAGS",
"OBJECT2_IFACE",
"OBJECT2_DESCR",
"OBJECT2_SPEED",
"LINK_PROTO",
"LAST_SEEN",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
if "interface_type_count" in columns.split(","):
r[-1].extend(type_columns)
# self.logger.info(r)
# self.logger.info("---------------------------------")
# print("-----------%s------------%s" % (administrative_domain, columns))
p = Pool.get_by_name(pool or "default")
mos = ManagedObject.objects.filter()
if request.user.is_superuser and not administrative_domain and not selector and not segment:
mos = ManagedObject.objects.filter(pool=p)
if ids:
mos = ManagedObject.objects.filter(id__in=[ids])
if is_managed is not None:
mos = ManagedObject.objects.filter(is_managed=is_managed)
if pool:
mos = mos.filter(pool=p)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if administrative_domain:
ads = AdministrativeDomain.get_nested_ids(
int(administrative_domain))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if segment:
segment = NetworkSegment.objects.filter(id=segment).first()
if segment:
mos = mos.filter(segment__in=segment.get_nested_ids())
mos_id = list(mos.values_list("id", flat=True))
rld = ReportLinksDetail(mos_id)
mo_resolv = {
mo[0]: mo[1:]
for mo in ManagedObject.objects.filter().values_list(
"id",
"administrative_domain__name",
"name",
"address",
"segment",
"platform",
"labels",
)
}
for link in rld.out:
if len(rld.out[link]) != 2:
# Multilink or bad link
continue
s1, s2 = rld.out[link]
seg1, seg2 = None, None
if "object1_segment" in columns.split(
",") or "object2_segment" in columns.split(","):
seg1, seg2 = mo_resolv[s1["mo"][0]][3], mo_resolv[s2["mo"]
[0]][3]
plat1, plat2 = None, None
if "object1_platform" in columns.split(
",") or "object2_platform" in columns.split(","):
plat1, plat2 = mo_resolv[s1["mo"][0]][4], mo_resolv[s2["mo"]
[0]][4]
r += [
translate_row(
row([
mo_resolv[s1["mo"][0]][0],
mo_resolv[s1["mo"][0]][1],
mo_resolv[s1["mo"][0]][2],
"" if not plat1 else Platform.get_by_id(plat1),
"" if not seg1 else NetworkSegment.get_by_id(seg1),
";".join(mo_resolv[s1["mo"][0]][5] or []),
s1["iface_n"][0],
s1.get("iface_descr")[0]
if s1.get("iface_descr") else "",
s1.get("iface_speed")[0]
if s1.get("iface_speed") else 0,
mo_resolv[s2["mo"][0]][0],
mo_resolv[s2["mo"][0]][1],
mo_resolv[s2["mo"][0]][2],
"" if not plat2 else Platform.get_by_id(plat2),
"" if not seg2 else NetworkSegment.get_by_id(seg2),
";".join(mo_resolv[s2["mo"][0]][5] or []),
s2["iface_n"][0],
s2.get("iface_descr")[0]
if s2.get("iface_descr") else "",
s2.get("iface_speed")[0]
if s2.get("iface_speed") else 0,
s2.get("dis_method", ""),
s2.get("last_seen", ""),
]),
cmap,
)
]
filename = "links_detail_report_%s" % datetime.datetime.now().strftime(
"%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response,
dialect="excel",
delimiter=",",
quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr("%s.csv" % filename, f.read())
zf.filename = "%s.csv.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Objects")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
# response = HttpResponse(
# content_type="application/x-ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def api_report(
self,
request,
from_date,
to_date,
o_format,
administrative_domain=None,
columns=None,
source="both",
alarm_class=None,
enable_autowidth=False,
):
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, str):
return smart_text(v)
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return smart_text(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
cols = [
"id",
"alarm_class",
"alarm_from_ts",
"alarm_to_ts",
"alarm_tt",
"object_name",
"object_address",
"object_admdomain",
"log_timestamp",
"log_source",
"log_message",
# "tt",
# "escalation_ts",
]
header_row = [
"ID",
_("ALARM_CLASS"),
_("ALARM_FROM_TS"),
_("ALARM_TO_TS"),
_("ALARM_TT"),
_("OBJECT_NAME"),
_("OBJECT_ADDRESS"),
_("OBJECT_ADMDOMAIN"),
_("LOG_TIMESTAMP"),
_("LOG_SOURCE"),
_("LOG_MESSAGE"),
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
fd = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
match = {
"timestamp": {
"$gte": datetime.datetime.strptime(from_date, "%d.%m.%Y"),
"$lte": fd
}
}
mos = ManagedObject.objects.filter()
ads = []
if administrative_domain:
if administrative_domain.isdigit():
administrative_domain = [int(administrative_domain)]
ads = AdministrativeDomain.get_nested_ids(
administrative_domain[0])
if not request.user.is_superuser:
user_ads = UserAccess.get_domains(request.user)
if administrative_domain and ads:
if administrative_domain[0] not in user_ads:
ads = list(set(ads) & set(user_ads))
if not ads:
return HttpResponse(
"<html><body>Permission denied: Invalid Administrative Domain</html></body>"
)
else:
ads = user_ads
if ads:
mos = mos.filter(administrative_domain__in=ads)
# Working if Administrative domain set
if ads:
try:
match["adm_path"] = {"$in": ads}
# @todo More 2 level hierarhy
except bson.errors.InvalidId:
pass
addr_map = {
mo[0]: (mo[1], mo[2])
for mo in mos.values_list("id", "name", "address")
}
# Active Alarms
coll = ActiveAlarm._get_collection()
for aa in coll.aggregate([
{
"$match": match
},
{
"$unwind": "$log"
},
{
"$match": {
"log.source": {
"$exists": True,
"$ne": None
}
}
},
{
"$project": {
"timestamp": 1,
"managed_object": 1,
"alarm_class": 1,
"escalation_tt": 1,
"adm_path": 1,
"log": 1,
}
},
{
"$sort": {
"_id": 1,
"log.timestamp": 1
}
},
]):
r += [
translate_row(
row([
smart_text(aa["_id"]),
AlarmClass.get_by_id(aa["alarm_class"]).name,
aa["timestamp"],
"",
aa.get("escalation_tt", ""),
addr_map[aa["managed_object"]][0],
addr_map[aa["managed_object"]][1],
AdministrativeDomain.get_by_id(
aa["adm_path"][-1]).name,
aa["log"]["timestamp"],
aa["log"]["source"],
aa["log"]["message"],
]),
cmap,
)
]
# Active Alarms
coll = ArchivedAlarm._get_collection()
for aa in coll.aggregate([
{
"$match": match
},
{
"$unwind": "$log"
},
{
"$match": {
"log.source": {
"$exists": True
}
}
},
{
"$project": {
"timestamp": 1,
"clear_timestamp": 1,
"managed_object": 1,
"alarm_class": 1,
"escalation_tt": 1,
"adm_path": 1,
"log": 1,
}
},
{
"$sort": {
"_id": 1,
"log.timestamp": 1
}
},
]):
r += [
translate_row(
row([
smart_text(aa["_id"]),
AlarmClass.get_by_id(aa["alarm_class"]).name,
aa["timestamp"],
aa["clear_timestamp"],
aa.get("escalation_tt", ""),
addr_map[aa["managed_object"]][0],
addr_map[aa["managed_object"]][1],
AdministrativeDomain.get_by_id(
aa["adm_path"][-1]).name,
aa["log"]["timestamp"],
aa["log"]["source"],
aa["log"]["message"],
]),
cmap,
)
]
filename = "alarm_comments.csv"
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s"' % filename
writer = csv.writer(response)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerow(columns)
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr(filename, f.read())
zf.filename = "%s.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="alarm_comments.xlsx"'
response.close()
return response
def get_data(self,
request,
duration,
from_date=None,
to_date=None,
**kwargs):
now = datetime.datetime.now()
if not from_date:
duration = 1
if int(duration):
self.logger.info("Use duration\n")
d = datetime.timedelta(seconds=int(duration))
b = now - d
q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
else:
b = datetime.datetime.strptime(from_date, "%d.%m.%Y")
q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
if to_date:
if from_date == to_date:
t1 = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(1)
else:
t1 = datetime.datetime.strptime(to_date, "%d.%m.%Y")
else:
t1 = now
q &= Q(start__lte=t1) | Q(stop__lte=t1)
d = datetime.timedelta(seconds=int((t1 - b).total_seconds()))
outages = defaultdict(list)
otime = defaultdict(int)
for o in Outage.objects.filter(q):
start = max(o.start, b)
stop = o.stop if o.stop else now
outages[o.object] += [o]
otime[o.object] += (stop - start).total_seconds()
td = d.total_seconds()
if not request.user.is_superuser:
for mo in ManagedObject.objects.exclude(
administrative_domain__in=UserAccess.get_domains(
request.user)):
if mo.id in otime:
otime.pop(mo.id)
# Load managed objects
mos = list(otime)
chunk = 500
mo = {}
while mos:
for o in ManagedObject.objects.filter(id__in=mos[:chunk]):
mo[o.id] = o
mos = mos[chunk:]
r = []
for o in sorted(otime, key=lambda x: -otime[x]):
m = mo.get(o)
if not m:
continue # Hanging Outage
dt = min(td, otime[o])
downtime = "%02d:%02d:%02d" % ((dt // 3600) % 24,
(dt // 60) % 60, dt % 60)
if dt >= 86400:
downtime = "%dd %s" % (dt // 86400, downtime)
if td:
avail = float(td - dt) * 100 / td
else:
avail = 0
r += [(
m.name,
m.address,
m.profile.name,
m.platform.name if m.platform else "",
_("Yes") if m.is_managed else _("No"),
_("Yes") if m.get_status() else _("No"),
downtime,
avail,
len(outages[o]),
)]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"),
_("Address"),
_("Profile"),
_("Platform"),
TableColumn(_("Managed"), align="right"),
TableColumn(_("Status"), align="right"),
TableColumn(_("Downtime"), align="right"),
TableColumn(_("Availability"), align="right",
format="percent"),
TableColumn(_("Downs"), align="right", format="integer"),
],
data=r,
enumerate=True,
)
def get_ajax_data(self, **kwargs):
def update_dict(d, s):
for k in s:
if k in d:
d[k] += s[k]
else:
d[k] = s[k]
zoom = int(self.handler.get_argument("z"))
west = float(self.handler.get_argument("w"))
east = float(self.handler.get_argument("e"))
north = float(self.handler.get_argument("n"))
south = float(self.handler.get_argument("s"))
ms = int(self.handler.get_argument("maintenance"))
active_layers = [
l_r for l_r in self.get_pop_layers()
if l_r.min_zoom <= zoom <= l_r.max_zoom
]
alarms = []
res = {}
services = {}
subscribers = {}
t_data = defaultdict(list)
mos = ManagedObject.objects.filter(is_managed=True).values(
"id", "name", "x", "y")
if not self.current_user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(
self.current_user))
for mo in mos:
res[mo.pop("id")] = mo
mos_id = list(res.keys())
if ms == 0:
mos_id = list(
set(list(res.keys())) - set(Maintenance.currently_affected()))
for a in ActiveAlarm._get_collection().find(
{"managed_object": {
"$in": mos_id,
"$exists": True
}},
{
"_id": 1,
"managed_object": 1,
"direct_subscribers": 1,
"direct_services": 1
},
):
s_sub, s_service = {}, {}
if a.get("direct_subscribers"):
s_sub = {
dsub["profile"]: dsub["summary"]
for dsub in a["direct_subscribers"]
}
if a.get("direct_services"):
s_service = {
dserv["profile"]: dserv["summary"]
for dserv in a["direct_services"]
}
mo = res.get(a["managed_object"])
if not mo:
continue
if mo["x"] and mo["y"]:
w = ServiceSummary.get_weight({
"subscriber": s_sub,
"service": s_service
})
# @todo: Should we add the object's weight to summary?
# @todo: Check west/south hemisphere
if active_layers and west <= mo["x"] <= east and south <= mo[
"y"] <= north:
t_data[mo["x"], mo["y"]] += [(mo, w)]
else:
w = 0
alarms += [{
"alarm_id": str(a.get("_id")),
"managed_object": mo["name"],
"x": mo["x"],
"y": mo["y"],
"w": max(w, 1),
}]
if s_service:
update_dict(services, s_service)
if s_sub:
update_dict(subscribers, s_sub)
links = None
o_seen = set()
points = None
o_data = {}
if t_data and active_layers:
# Create lines
bbox = get_bbox(west, east, north, south)
lines = []
for d in ObjectConnection._get_collection().find(
{
"type": "pop_link",
"layer": {
"$in": [a_l.id for a_l in active_layers]
},
"line": {
"$geoIntersects": {
"$geometry": bbox
}
},
},
{
"_id": 0,
"connection": 1,
"line": 1
},
):
for c in d["line"]["coordinates"]:
if tuple(c) in t_data:
for c in d["line"]["coordinates"]:
tc = tuple(c)
o_data[tc] = t_data.get(tc, [])
o_seen.add(tuple(c))
lines += [d["line"]]
break
if lines:
links = geojson.FeatureCollection(features=lines)
# Create points
points = []
for x, y in o_data:
data = {}
for mo, w in o_data[x, y]:
if mo not in data:
data[mo] = w
data = sorted(data, key=lambda z: data[z],
reverse=True)[:self.TOOLTIP_LIMIT]
points += [
geojson.Feature(
geometry=geojson.Point(coordinates=[x, y]),
properties={
"alarms":
len(t_data[x, y]),
"objects": [{
"id": mo.id,
"name": mo.name,
"address": mo.address
} for mo in mos],
},
)
]
points = geojson.FeatureCollection(features=points)
return {
"alarms":
alarms,
"summary":
self.f_glyph_summary({
"service": services,
"subscriber": subscribers
}),
"links":
links,
"pops":
points,
}
def get_data(self,
request,
interval=1,
from_date=None,
to_date=None,
skip_avail=False,
skip_zero_avail=False,
filter_zero_access=False,
**kwargs):
"""
a1 = self.get_availability(1)
a7 = self.get_availability(7)
a30 = self.get_availability(30)
"""
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(
days=interval)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
a = self.get_availability(start_date=from_date,
stop_date=to_date,
skip_zero_avail=skip_zero_avail)
rb = self.get_reboots(start_date=from_date, stop_date=to_date)
r = [SectionRow("Report from %s to %s" % (from_date, to_date))]
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if skip_avail:
mos = mos.filter(id__in=list(a))
mos_id = list(mos.order_by("id").values_list("id", flat=True))
if filter_zero_access:
iface_p = InterfaceProfile.objects.get(name="Клиентский порт")
match = {"profile": iface_p.id, "managed_object": {"$in": mos_id}}
pipeline = [
{
"$match": match
},
{
"$group": {
"_id": "$managed_object",
"count": {
"$sum": 1
},
"m": {
"$max": "$oper_status"
},
}
},
{
"$match": {
"m": False
}
},
{
"$project": {
"_id": True
}
},
]
# data = Interface.objects._get_collection().aggregate(pipeline,
data = (get_db()["noc.interfaces"].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate(
pipeline))
data = [d["_id"] for d in data]
mos = mos.exclude(id__in=data)
mo_hostname = ReportObjectsHostname1(sync_ids=mos_id)
mo_hostname = mo_hostname.get_dictionary()
for o in mos:
s = [
o.administrative_domain.name,
o.name,
mo_hostname.get(o.id, ""),
o.address,
o.profile.name,
round(a.get(o.id, (100.0, 0, 0))[0], 2),
]
s.extend(a.get(o.id, (100.0, 0, 0))[1:])
s.append(rb[o.id] if o.id in rb else 0)
r += [s]
"""
a1.get(o.id, 100),
a7.get(o.id, 100),
a30.get(o.id, 100)
"""
# print r
return self.from_dataset(
title=self.title,
columns=[
_("Adm. Domain"),
_("Managed Object"),
_("Hostname"),
_("Address"),
_("Profile"),
# TableColumn(_("Avail"), align="right", format="percent"),
# TableColumn(_("Total avail (sec)"), align="right", format="numeric"),
_("Avail"),
_("Total unavail (sec)"),
_("Count outages"),
_("Reboots"),
],
data=r,
enumerate=True,
)
def post(self, *args, **kwargs):
"""
Request is the list of
{
id: <managed object id>,
script: <script name>,
args: <arguments>
}
:param args:
:param kwargs:
:return:
"""
metrics["mrt_requests"] += 1
# Parse request
req = ujson.loads(self.request.body)
# Disable nginx proxy buffering
self.set_header("X-Accel-Buffering", "no")
# Object ids
ids = set(int(d["id"]) for d in req if "id" in d and "script" in d)
logger.info(
"Run task on parralels: %d (Max concurrent %d), for User: %s",
len(req),
self.CONCURRENCY,
self.current_user,
)
# Check access
qs = ManagedObject.objects.filter(id__in=list(ids))
if not self.current_user.is_superuser:
adm_domains = UserAccess.get_domains(self.current_user)
qs = qs.filter(administrative_domain__in=adm_domains)
ids = dict(qs.values_list("id", "bi_id"))
with Span(
sample=int(config.mrt.enable_command_logging),
server="MRT",
service="post",
client=self.current_user,
in_label=req,
) as span:
if self.service.use_telemetry:
logger.info("[%s] Enable telemetry for task, user: %s",
span.span_id, self.current_user)
futures = []
for d in req:
if "id" not in d or "script" not in d:
continue
oid = int(d["id"])
if oid not in ids:
yield self.write_chunk({
"id": str(d["id"]),
"error": "Access denied"
})
metrics["mrt_access_denied"] += 1
if len(futures) >= config.mrt.max_concurrency:
wi = tornado.gen.WaitIterator(*futures)
r = yield next(wi)
del futures[wi.current_index]
yield self.write_chunk(r)
futures += [
self.run_script(oid,
d["script"],
d.get("args"),
span_id=span.span_id,
bi_id=ids.get(oid))
]
# Wait for rest
wi = tornado.gen.WaitIterator(*futures)
while not wi.done():
r = yield next(wi)
yield self.write_chunk(r)
logger.info("Done")
def get_data(self,
request,
pool=None,
obj_profile=None,
filter_ignore_iface=True,
**kwargs):
rn = re.compile(
r"'remote_chassis_id': u'(?P<rem_ch_id>\S+)'.+'remote_system_name': u'(?P<rem_s_name>\S+)'",
re.IGNORECASE,
)
problem = {
"Not found iface on remote": "->",
"Not found local iface on remote": "<-",
"Remote object is not found": "X",
}
data = []
# MAC, hostname, count
not_found = defaultdict(int)
# Name, IP, count
local_on_remote = defaultdict(int)
# Get all managed objects
mos = ManagedObject.objects.filter(is_managed=True, pool=pool)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if obj_profile:
mos = mos.filter(object_profile=obj_profile)
mos_id = dict((mo.id, mo) for mo in mos)
report = ReportPendingLinks(
list(six.iterkeys(mos_id)),
ignore_profiles=list(
InterfaceProfile.objects.filter(discovery_policy="I")),
)
problems = report.out
for mo_id in problems:
mo = mos_id.get(mo_id, ManagedObject.get_by_id(mo_id))
for iface in problems[mo_id]:
data += [(
mo.name,
mo.address,
mo.profile.name,
mo.administrative_domain.name,
iface,
problem[problems[mo_id][iface]["problem"]],
problems[mo_id][iface]["remote_id"],
)]
if problems[mo_id][iface][
"problem"] == "Remote object is not found":
match = rn.findall(problems[mo_id][iface]["remote_id"])
if match:
not_found[match[0]] += 1
elif problems[mo_id][iface][
"problem"] == "Not found iface on remote":
local_on_remote[(mo.name, mo.address)] += 1
data += [SectionRow(name="Summary information on u_object")]
for c in not_found:
if not_found[c] > 4:
data += [c]
data += [SectionRow(name="Summary information on agg")]
for c in local_on_remote:
if local_on_remote[c] > 4:
data += [c]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"),
_("Address"),
_("Profile"),
_("Administrative domain"),
_("Interface"),
_("Direction"),
_("Remote Object")
# _("Discovery"), _("Error")
],
data=data,
)
def get_data(self, request, pool, obj_profile=None, **kwargs):
problems = {} # id -> problem
if not obj_profile:
# Get all managed objects
mos = dict(
(mo.id, mo)
for mo in ManagedObject.objects.filter(is_managed=True, pool=pool)
)
if not request.user.is_superuser:
mos = dict(
(mo.id, mo)
for mo in ManagedObject.objects.filter(is_managed=True, pool=pool,
administrative_domain__in=UserAccess.get_domains(request.user))
)
else:
# Get all managed objects
mos = dict(
(mo.id, mo)
for mo in ManagedObject.objects.filter(is_managed=True, pool=pool, object_profile=obj_profile)
)
if not request.user.is_superuser:
mos = dict(
(mo.id, mo)
for mo in ManagedObject.objects.filter(is_managed=True, pool=pool, object_profile=obj_profile,
administrative_domain__in=UserAccess.get_domains(request.user))
)
mos_set = set(mos)
# Get all managed objects with generic profile
for mo in mos:
if mos[mo].profile.name == GENERIC_PROFILE:
problems[mo] = _("Profile check failed")
# Get all managed objects without interfaces
if_mo = dict(
(x["_id"], x.get("managed_object"))
for x in Interface._get_collection().find(
{},
{"_id": 1, "managed_object": 1}
)
)
for mo in (mos_set - set(problems) - set(if_mo.itervalues())):
problems[mo] = _("No interfaces")
# Get all managed objects without links
linked_mos = set()
for d in Link._get_collection().find({}):
for i in d["interfaces"]:
linked_mos.add(if_mo.get(i))
for mo in (mos_set - set(problems) - linked_mos):
problems[mo] = _("No links")
# Get all managed objects without uplinks
uplinks = {}
for d in ObjectData._get_collection().find():
nu = len(d.get("uplinks", []))
if nu:
uplinks[d["_id"]] = nu
for mo in (mos_set - set(problems) - set(uplinks)):
problems[mo] = _("No uplinks")
#
data = []
for mo_id in problems:
if mo_id not in mos:
continue
mo = mos[mo_id]
data += [[
mo.name,
mo.address,
mo.profile.name,
mo.platform.name if mo.platform else "",
mo.segment.name if mo.segment else "",
problems[mo_id]
]]
data = sorted(data)
return self.from_dataset(
title=self.title,
columns=[
"Name",
"Address",
"Profile",
"Platform",
"Segment",
"Problem"
],
data=data,
enumerate=True
)
def api_report(
self,
request,
reporttype=None,
from_date=None,
to_date=None,
object_profile=None,
filter_default=None,
exclude_zero=True,
interface_profile=None,
selector=None,
administrative_domain=None,
columns=None,
description=None,
o_format=None,
enable_autowidth=False,
**kwargs,
):
def load(mo_ids):
# match = {"links.mo": {"$in": mo_ids}}
match = {"int.managed_object": {"$in": mo_ids}}
group = {
"_id": "$_id",
"links": {
"$push": {
"iface_n": "$int.name",
# "iface_id": "$int._id",
# "iface_descr": "$int.description",
# "iface_speed": "$int.in_speed",
# "dis_method": "$discovery_method",
# "last_seen": "$last_seen",
"mo": "$int.managed_object",
"linked_obj": "$linked_objects",
}
},
}
value = (get_db()["noc.links"].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate(
[
{
"$unwind": "$interfaces"
},
{
"$lookup": {
"from": "noc.interfaces",
"localField": "interfaces",
"foreignField": "_id",
"as": "int",
}
},
{
"$match": match
},
{
"$group": group
},
],
allowDiskUse=True,
))
res = defaultdict(dict)
for v in value:
if v["_id"]:
for vv in v["links"]:
if len(vv["linked_obj"]) == 2:
mo = vv["mo"][0]
iface = vv["iface_n"]
for i in vv["linked_obj"]:
if mo != i:
res[mo][i] = iface[0]
return res
def translate_row(row, cmap):
return [row[i] for i in cmap]
def str_to_float(str):
return float("{0:.3f}".format(float(str)))
cols = [
"object_id",
"object_name",
"object_address",
"object_platform",
"object_adm_domain",
"object_segment",
"object_container",
# "object_hostname",
# "object_status",
# "profile_name",
# "object_profile",
# "object_vendor",
"iface_name",
"iface_description",
"iface_speed",
"max_load_in",
"max_load_in_time",
"max_load_out",
"max_load_out_time",
"avg_load_in",
"avg_load_out",
"total_in",
"total_out",
"uplink_iface_name",
"uplink_iface_description",
"uplink_iface_speed",
"uplink_max_load_in",
"uplink_max_load_in_time",
"uplink_max_load_out",
"uplink_max_load_out_time",
"uplink_avg_load_in",
"uplink_avg_load_out",
"uplink_total_in",
"uplink_total_out",
]
header_row = [
"ID",
_("OBJECT_NAME"),
_("OBJECT_ADDRESS"),
_("OBJECT_PLATFORM"),
_("OBJECT_ADMDOMAIN"),
_("OBJECT_SEGMENT"),
_("CONTAINER_ADDRESS"),
_("IFACE_NAME"),
_("IFACE_DESCRIPTION"),
_("IFACE_SPEED"),
_("MAX_LOAD_IN, Mbps"),
_("MAX_LOAD_IN_TIME"),
_("MAX_LOAD_OUT, Mbps"),
_("MAX_LOAD_OUT_TIME"),
_("AVG_LOAD_IN, Mbps"),
_("AVG_LOAD_OUT, Mbps"),
_("TOTAL_IN, Mbyte"),
_("TOTAL_OUT, Mbyte"),
_("UPLINK_IFACE_NAME"),
_("UPLINK_IFACE_DESCRIPTION"),
_("UPLINK_IFACE_SPEED"),
_("UPLINK_MAX_LOAD_IN, Mbps"),
_("UPLINK_MAX_TIME_IN"),
_("UPLINK_MAX_LOAD_OUT, Mbps"),
_("UPLINK_MAX_TIME_OUT"),
_("UPLINK_AVG_LOAD_IN, Mbps"),
_("UPLINK_AVG_LOAD_OUT, Mbps"),
_("UPLINK_TOTAL_IN, Mbyte"),
_("UPLINK_TOTAL_OUT, Mbyte"),
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
columns_order = columns.split(",")
columns_filter = set(columns_order)
r = [translate_row(header_row, cmap)]
# Date Time Block
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(days=1)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
diff = to_date - from_date
# Load managed objects
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if selector:
mos = mos.filter(
ManagedObjectSelector.objects.get(id=int(selector)).Q)
if administrative_domain:
mos = mos.filter(
administrative_domain__in=AdministrativeDomain.get_nested_ids(
int(administrative_domain)))
if object_profile:
mos = mos.filter(object_profile=object_profile)
if interface_profile:
interface_profile = InterfaceProfile.objects.filter(
id=interface_profile).first()
mo_attrs = namedtuple("MOATTRs",
[c for c in cols if c.startswith("object")])
containers_address = {}
if "object_container" in columns_filter:
containers_address = ReportContainerData(
set(mos.values_list("id", flat=True)))
containers_address = dict(list(containers_address.extract()))
moss = {}
for row in mos.values_list("bi_id", "name", "address", "platform",
"administrative_domain__name", "segment",
"id"):
moss[row[0]] = mo_attrs(*[
row[6],
row[1],
row[2],
smart_text(Platform.get_by_id(row[3]) if row[3] else ""),
row[4],
smart_text(NetworkSegment.get_by_id(row[5])) if row[5] else "",
containers_address.
get(row[6], "") if containers_address and row[6] else "",
])
report_metric = ReportInterfaceMetrics(tuple(sorted(moss)),
from_date,
to_date,
columns=None)
report_metric.SELECT_QUERY_MAP = {
(0, "managed_object", "id"): "managed_object",
(1, "path", "iface_name"): "arrayStringConcat(path)",
(
2,
"",
"iface_description",
):
"dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))",
(
3,
"",
"profile",
):
"dictGetString('interfaceattributes', 'profile', (managed_object, arrayStringConcat(path)))",
(
4,
"speed",
"iface_speed",
):
"dictGetUInt64('interfaceattributes', 'in_speed', (managed_object, arrayStringConcat(path)))",
(5, "load_in_max", "load_in_max"): "divide(max(load_in),1048576)",
(6, "load_out_max", "load_out_max"):
"divide(max(load_out),1048576)",
(7, "max_load_in_time", "max_load_in_time"): "argMax(ts,load_in)",
(8, "max_load_out_time", "max_load_out_time"):
"argMax(ts,load_out)",
(9, "avg_load_in", "avg_load_in"): "divide(avg(load_in),1048576)",
(10, "avg_load_out", "avg_load_out"):
"divide(avg(load_out),1048576)",
}
ifaces_metrics = defaultdict(dict)
for row in report_metric.do_query():
avg_in = str_to_float(row[9])
avg_out = str_to_float(row[10])
total_in = avg_in * diff.total_seconds() / 8
total_out = avg_out * diff.total_seconds() / 8
ifaces_metrics[row[0]][row[1]] = {
"description": row[2],
"profile": row[3],
"bandwidth": row[4],
"max_load_in": str_to_float(row[5]),
"max_load_out": str_to_float(row[6]),
"max_load_in_time": row[7],
"max_load_out_time": row[8],
"avg_load_in": avg_in,
"avg_load_out": avg_out,
"total_in": float("{0:.1f}".format(total_in)),
"total_out": float("{0:.1f}".format(total_out)),
}
# find uplinks
links = {}
if cmap[-1] > 17:
mos_id = list(mos.values_list("id", flat=True))
uplinks = {obj: [] for obj in mos_id}
for d in ObjectData._get_collection().find(
{"_id": {
"$in": mos_id
}}, {
"_id": 1,
"uplinks": 1
}):
uplinks[d["_id"]] = d.get("uplinks", [])
rld = load(mos_id)
for mo in uplinks:
for uplink in uplinks[mo]:
if rld[mo]:
if mo in links:
links[mo] += [rld[mo][uplink]]
else:
links[mo] = [rld[mo][uplink]]
for mo_bi in ifaces_metrics:
mo_id = moss[int(mo_bi)]
mo_ids = getattr(mo_id, "object_id")
for i in ifaces_metrics[mo_bi]:
if not exclude_zero:
if (ifaces_metrics[mo_bi][i]["max_load_in"] == 0
and ifaces_metrics[mo_bi][i]["max_load_out"] == 0):
continue
if description:
if description not in ifaces_metrics[mo_bi][i][
"description"]:
continue
if interface_profile:
if interface_profile.name not in ifaces_metrics[mo_bi][i][
"profile"]:
continue
row2 = [
mo_ids,
getattr(mo_id, "object_name"),
getattr(mo_id, "object_address"),
getattr(mo_id, "object_platform"),
getattr(mo_id, "object_adm_domain"),
getattr(mo_id, "object_segment"),
getattr(mo_id, "object_container"),
i,
ifaces_metrics[mo_bi][i]["description"],
ifaces_metrics[mo_bi][i]["bandwidth"],
ifaces_metrics[mo_bi][i]["max_load_in"],
ifaces_metrics[mo_bi][i]["max_load_in_time"],
ifaces_metrics[mo_bi][i]["max_load_out"],
ifaces_metrics[mo_bi][i]["max_load_out_time"],
ifaces_metrics[mo_bi][i]["avg_load_in"],
ifaces_metrics[mo_bi][i]["avg_load_out"],
ifaces_metrics[mo_bi][i]["total_in"],
ifaces_metrics[mo_bi][i]["total_out"],
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
]
ss = True
if mo_ids in links:
for ifname_uplink in links[mo_ids]:
if ifname_uplink in ifaces_metrics[mo_bi]:
row2[18] = ifname_uplink
row2[19] = ifaces_metrics[mo_bi][ifname_uplink][
"description"]
row2[20] = ifaces_metrics[mo_bi][ifname_uplink][
"bandwidth"]
row2[21] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_in"]
row2[22] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_in_time"]
row2[23] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_out"]
row2[24] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_out_time"]
row2[25] = ifaces_metrics[mo_bi][ifname_uplink][
"avg_load_in"]
row2[26] = ifaces_metrics[mo_bi][ifname_uplink][
"avg_load_out"]
row2[27] = ifaces_metrics[mo_bi][ifname_uplink][
"total_in"]
row2[28] = ifaces_metrics[mo_bi][ifname_uplink][
"total_out"]
r += [translate_row(row2, cmap)]
ss = False
if ss:
r += [translate_row(row2, cmap)]
filename = "metrics_detail_report_%s" % datetime.datetime.now(
).strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response,
dialect="excel",
delimiter=",",
quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr("%s.csv" % filename, f.read())
zf.filename = "%s.csv.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Metrics")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def get_data(self,
request,
interval,
from_date=None,
to_date=None,
**kwargs):
interval = int(interval)
if not from_date:
interval = 1
if interval:
ts = datetime.datetime.now() - datetime.timedelta(days=interval)
match = {"ts": {"$gte": ts}}
else:
t0 = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date:
t1 = datetime.datetime.now()
else:
t1 = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
match = {"ts": {"$gte": t0, "$lte": t1}}
pipeline = [
{
"$match": match
},
{
"$group": {
"_id": "$object",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
]
data = list(Reboot._get_collection().aggregate(pipeline))
# Get managed objects
if not request.user.is_superuser:
id_perm = [
mo.id for mo in ManagedObject.objects.filter(
administrative_domain__in=UserAccess.get_domains(
request.user))
]
ids = [x["_id"] for x in data if x["_id"] in id_perm]
else:
ids = [x["_id"] for x in data]
mo_names = {} # mo id -> mo name
cursor = connection.cursor()
while ids:
chunk = [str(x) for x in ids[:500]]
ids = ids[500:]
cursor.execute("""
SELECT id, address, name
FROM sa_managedobject
WHERE id IN (%s)""" % ", ".join(chunk))
mo_names.update(dict((c[0], c[1:3]) for c in cursor))
#
if not request.user.is_superuser:
data = [(mo_names.get(x["_id"], "---")[1],
mo_names.get(x["_id"], "---")[0], x["count"])
for x in data if x["_id"] in id_perm]
else:
data = [(mo_names.get(x["_id"], "---")[1],
mo_names.get(x["_id"], "---")[0], x["count"])
for x in data]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"),
_("Address"),
TableColumn(_("Reboots"),
align="right",
format="numeric",
total="sum"),
],
data=data,
enumerate=True,
)
def api_report(
self,
request,
o_format,
administrative_domain=None,
selector=None,
interface_profile=None,
zero=None,
def_profile=None,
columns=None,
enable_autowidth=False,
):
def humanize_speed(speed):
if not speed:
return "-"
for t, n in [(1000000, "G"), (1000, "M"), (1, "k")]:
if speed >= t:
if speed // t * t == speed:
return "%d%s" % (speed // t, n)
else:
return "%.2f%s" % (float(speed) / t, n)
return str(speed)
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, unicode):
return v.encode("utf-8")
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
cols = [
"object_name",
"object_address",
"object_model",
"object_software",
"object_port_name",
"object_port_profile_name",
"object_port_status",
"object_link_status",
"object_port_speed",
"object_port_duplex",
"object_port_untagged_vlan",
"object_port_tagged_vlans",
]
header_row = [
"MANAGED_OBJECT",
"OBJECT_ADDRESS",
"OBJECT_MODEL",
"OBJECT_SOFTWARE",
"PORT_NAME",
"PORT_PROFILE_NAME",
"PORT_STATUS",
"LINK_STATUS",
"PORT_SPEED",
"PORT_DUPLEX",
"PORT_UNTAGGED_VLAN",
"PORT_TAGGED_VLANS",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
mo = {}
if_p = {}
DUPLEX = {True: "Full", False: "Half"}
for ifp in InterfaceProfile.objects.filter():
if_p[ifp.id] = {"name": ifp.name}
mos = ManagedObject.objects.filter(is_managed=True)
if (request.user.is_superuser and not administrative_domain
and not selector and not interface_profile):
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if administrative_domain:
ads = AdministrativeDomain.get_nested_ids(
int(administrative_domain))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
for o in mos:
mo[o.id] = {
"type": "managedobject",
"id": str(o.id),
"name": o.name,
"status": o.is_managed,
"address": o.address,
"vendor": o.vendor,
"version": o.version,
"platform": o.platform,
}
mos_id = list(mos.values_list("id", flat=True))
rld = ReportInterfaceStatus(mos_id, zero, def_profile,
interface_profile)
for i in rld.out:
untag, tagged = "", ""
if i["subs"]:
untag = i["subs"][0].get("untagged_vlan", "")
tagged = list_to_ranges(i["subs"][0].get("tagged_vlans", []))
r += [
translate_row(
row([
mo[i["managed_object"]]["name"],
mo[i["managed_object"]]["address"],
"%s %s" % (
str(mo[i["managed_object"]]["vendor"]),
str(mo[i["managed_object"]]["platform"]),
),
str(mo[i["managed_object"]]["version"]),
i["name"],
if_p[i["profile"]]["name"],
"UP" if i["admin_status"] is True else "Down",
"UP" if "oper_status" in i and i["oper_status"] is True
else "Down",
humanize_speed(i["in_speed"])
if "in_speed" in i else "-",
DUPLEX.get(i["full_duplex"])
if "full_duplex" in i and "in_speed" in i else "-",
untag,
tagged,
]),
cmap,
)
]
filename = "interface_status_report_%s" % datetime.datetime.now(
).strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response, dialect="excel", delimiter=";")
writer.writerows(r)
return response
elif o_format == "xlsx":
response = StringIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Objects")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
# response = HttpResponse(
# content_type="application/x-ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def queryset(self, request, query=None):
qs = super(ManagedObjectApplication, self).queryset(request, query)
if not request.user.is_superuser:
qs = qs.filter(UserAccess.Q(request.user))
qs = qs.exclude(name__startswith="wiping-")
return qs
def api_report(
self,
request,
reporttype=None,
from_date=None,
to_date=None,
object_profile=None,
filter_default=None,
exclude_zero=None,
interface_profile=None,
selector=None,
administrative_domain=None,
columns=None,
o_format=None,
enable_autowidth=False,
**kwargs
):
def translate_row(row, cmap):
return [row[i] for i in cmap]
map_table = {
"load_interfaces": "/Interface\s\|\sLoad\s\|\s[In|Out]/",
"load_cpu": "/[CPU|Memory]\s\|\sUsage/",
"errors": "/Interface\s\|\s[Errors|Discards]\s\|\s[In|Out]/",
"ping": "/Ping\s\|\sRTT/",
}
cols = [
"id",
"object_name",
"object_address",
"object_platform",
"object_adm_domain",
"object_segment",
# "object_hostname",
# "object_status",
# "profile_name",
# "object_profile",
# "object_vendor",
"iface_name",
"iface_description",
"iface_speed",
"load_in",
"load_in_p",
"load_out",
"load_out_p",
"errors_in",
"errors_out",
"slot",
"cpu_usage",
"memory_usage",
"ping_rtt",
"ping_attempts",
"interface_flap",
"interface_load_url",
]
header_row = [
"ID",
"OBJECT_NAME",
"OBJECT_ADDRESS",
"OBJECT_PLATFORM",
"OBJECT_ADM_DOMAIN",
"OBJECT_SEGMENT",
"IFACE_NAME",
"IFACE_DESCRIPTION",
"IFACE_SPEED",
"LOAD_IN",
"LOAD_IN_P",
"LOAD_OUT",
"LOAD_OUT_P",
"ERRORS_IN",
"ERRORS_OUT",
"CPU_USAGE",
"MEMORY_USAGE",
"PING_RTT",
"PING_ATTEMPTS",
"INTERFACE_FLAP",
"INTERFACE_LOAD_URL",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
columns_order = columns.split(",")
columns_filter = set(columns_order)
r = [translate_row(header_row, cmap)]
object_columns = [c for c in columns_order if c.startswith("object")]
# Date Time Block
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(days=1)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
# interval = (to_date - from_date).days
ts_from_date = time.mktime(from_date.timetuple())
ts_to_date = time.mktime(to_date.timetuple())
# Load managed objects
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
if selector:
mos = mos.filter(ManagedObjectSelector.objects.get(id=int(selector)).Q)
if administrative_domain:
mos = mos.filter(
administrative_domain__in=AdministrativeDomain.get_nested_ids(
int(administrative_domain)
)
)
if object_profile:
mos = mos.filter(object_profile=object_profile)
# iface_dict = {}
d_url = {
"path": "/ui/grafana/dashboard/script/report.js",
"rname": map_table[reporttype],
"from": str(int(ts_from_date * 1000)),
"to": str(int(ts_to_date * 1000)),
# o.name.replace("#", "%23")
"biid": "",
"oname": "",
"iname": "",
}
report_map = {
"load_interfaces": {
"url": "%(path)s?title=interface&biid=%(biid)s"
"&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s",
"q_group": ["interface"],
"q_select": {
(0, "managed_object", "id"): "managed_object",
(1, "path", "iface_name"): "arrayStringConcat(path)",
},
},
"errors": {
"url": """%(path)s?title=errors&biid=%(biid)s&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s""",
"q_group": ["interface"],
},
"load_cpu": {
"url": """%(path)s?title=cpu&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
"q_select": {
(0, "managed_object", "id"): "managed_object",
(1, "path", "slot"): "arrayStringConcat(path)",
},
},
"ping": {
"url": """%(path)s?title=ping&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
"q_select": {(0, "managed_object", "id"): "managed_object"},
},
}
query_map = {
# "iface_description": ('', 'iface_description', "''"),
"iface_description": (
"",
"iface_description",
"dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))",
),
"iface_speed": (
"speed",
"iface_speed",
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed))",
),
"load_in": ("load_in", "l_in", "round(quantile(0.90)(load_in), 0)"),
"load_in_p": (
"load_in",
"l_in_p",
"replaceOne(toString(round(quantile(0.90)(load_in) / "
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
),
"load_out": ("load_out", "l_out", "round(quantile(0.90)(load_out), 0)"),
"load_out_p": (
"load_out",
"l_out_p",
"replaceOne(toString(round(quantile(0.90)(load_out) / "
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
),
"errors_in": ("errors_in", "err_in", "quantile(0.90)(errors_in)"),
"errors_out": ("errors_out", "err_out", "quantile(0.90)(errors_out)"),
"cpu_usage": ("usage", "cpu_usage", "quantile(0.90)(usage)"),
"ping_rtt": ("rtt", "ping_rtt", "round(quantile(0.90)(rtt) / 1000, 2)"),
"ping_attempts": ("attempts", "ping_attempts", "avg(attempts)"),
}
query_fields = []
for c in report_map[reporttype]["q_select"]:
query_fields += [c[2]]
field_shift = len(query_fields) # deny replacing field
for c in columns.split(","):
if c not in query_map:
continue
field, alias, func = query_map[c]
report_map[reporttype]["q_select"][
(columns_order.index(c) + field_shift, field, alias)
] = func
query_fields += [c]
metrics_attrs = namedtuple("METRICSATTRs", query_fields)
mo_attrs = namedtuple("MOATTRs", [c for c in cols if c.startswith("object")])
moss = {}
for row in mos.values_list(
"bi_id", "name", "address", "platform", "administrative_domain__name", "segment"
):
moss[row[0]] = mo_attrs(
*[
row[1],
row[2],
str(Platform.get_by_id(row[3]) if row[3] else ""),
row[4],
str(NetworkSegment.get_by_id(row[5])) if row[5] else "",
]
)
url = report_map[reporttype].get("url", "")
report_metric = self.metric_source[reporttype](
tuple(sorted(moss)), from_date, to_date, columns=None
)
report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"]
if exclude_zero and reporttype == "load_interfaces":
report_metric.CUSTOM_FILTER["having"] += ["max(load_in) != 0 AND max(load_out) != 0"]
if interface_profile:
interface_profile = InterfaceProfile.objects.filter(id=interface_profile).first()
report_metric.CUSTOM_FILTER["having"] += [
"dictGetString('interfaceattributes', 'profile', "
"(managed_object, arrayStringConcat(path))) = '%s'" % interface_profile.name
]
# OBJECT_PLATFORM, ADMIN_DOMAIN, SEGMENT, OBJECT_HOSTNAME
for row in report_metric.do_query():
mm = metrics_attrs(*row)
mo = moss[int(mm.id)]
res = []
for y in columns_order:
if y in object_columns:
res += [getattr(mo, y)]
else:
res += [getattr(mm, y)]
if "interface_load_url" in columns_filter:
d_url["biid"] = mm.id
d_url["oname"] = mo[2].replace("#", "%23")
# res += [url % d_url, interval]
res.insert(columns_order.index("interface_load_url"), url % d_url)
r += [res]
filename = "metrics_detail_report_%s" % datetime.datetime.now().strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response, dialect="excel", delimiter=",", quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "xlsx":
response = StringIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (
r[0][cn] not in max_column_data_length
or len(str(c)) > max_column_data_length[r[0][cn]]
):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(), content_type="application/vnd.ms-excel")
response["Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
