class ReportEscalationsApplication(SimpleReport):
title = _("Escalations")
form = ReportForm
predefined_reports = {
"1d": PredefinedReport(_("Escalations (1 day)"), {"interval": 1}),
"7d": PredefinedReport(_("Escalations (7 days)"), {"interval": 7}),
"30d": PredefinedReport(_("Escalations (30 day)"), {"interval": 30}),
}
def get_data(self,
request,
interval,
from_date=None,
to_date=None,
**kwargs):
interval = int(interval)
if not from_date:
interval = 1
if interval:
ts = datetime.datetime.now() - datetime.timedelta(days=interval)
q = {"timestamp": {"$gte": ts}}
else:
t0 = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date:
t1 = datetime.datetime.now()
else:
t1 = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
q = {"timestamp": {"$gte": t0, "$lte": t1}}
q["escalation_tt"] = {"$exists": True}
if not request.user.is_superuser:
q["adm_path"] = {"$in": UserAccess.get_domains(request.user)}
data = []
for ac in (ActiveAlarm, ArchivedAlarm):
for d in ac._get_collection().find(q):
mo = ManagedObject.get_by_id(d["managed_object"])
if not mo:
continue
data += [(
d["timestamp"].strftime("%Y-%m-%d %H:%M:%S"),
d["escalation_ts"].strftime("%Y-%m-%d %H:%M:%S"),
mo.name.split("#", 1)[0],
mo.address,
mo.platform,
mo.segment.name,
d["escalation_tt"],
sum(ss["summary"] for ss in d["total_objects"]),
sum(ss["summary"] for ss in d["total_subscribers"]),
)]
data = sorted(data, key=operator.itemgetter(0))
return self.from_dataset(
title=self.title,
columns=[
_("Timestamp"),
_("Escalation Timestamp"),
_("Managed Object"),
_("Address"),
_("Platform"),
_("Segment"),
_("TT"),
_("Objects"),
_("Subscribers"),
],
data=data,
enumerate=True,
)
class ReportRebootsApplication(SimpleReport):
title = _("Reboots")
form = ReportForm
predefined_reports = {
"1d": PredefinedReport(_("Reboots (1 day)"), {"interval": 1}),
"7d": PredefinedReport(_("Reboots (7 days)"), {"interval": 7}),
"30d": PredefinedReport(_("Reboot (30 day)"), {"interval": 30}),
}
def get_data(self,
request,
interval,
from_date=None,
to_date=None,
**kwargs):
interval = int(interval)
if not from_date:
interval = 1
if interval:
ts = datetime.datetime.now() - datetime.timedelta(days=interval)
match = {"ts": {"$gte": ts}}
else:
t0 = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date:
t1 = datetime.datetime.now()
else:
t1 = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
match = {"ts": {"$gte": t0, "$lte": t1}}
pipeline = [
{
"$match": match
},
{
"$group": {
"_id": "$object",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
]
data = list(Reboot._get_collection().aggregate(pipeline))
# Get managed objects
if not request.user.is_superuser:
id_perm = [
mo.id for mo in ManagedObject.objects.filter(
administrative_domain__in=UserAccess.get_domains(
request.user))
]
ids = [x["_id"] for x in data if x["_id"] in id_perm]
else:
ids = [x["_id"] for x in data]
mo_names = {} # mo id -> mo name
cursor = connection.cursor()
while ids:
chunk = [str(x) for x in ids[:500]]
ids = ids[500:]
cursor.execute("""
SELECT id, address, name
FROM sa_managedobject
WHERE id IN (%s)""" % ", ".join(chunk))
mo_names.update(dict((c[0], c[1:3]) for c in cursor))
#
if not request.user.is_superuser:
data = [(mo_names.get(x["_id"], "---")[1],
mo_names.get(x["_id"], "---")[0], x["count"])
for x in data if x["_id"] in id_perm]
else:
data = [(mo_names.get(x["_id"], "---")[1],
mo_names.get(x["_id"], "---")[0], x["count"])
for x in data]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"),
_("Address"),
TableColumn(_("Reboots"),
align="right",
format="numeric",
total="sum"),
],
data=data,
enumerate=True,
)
class ReportFilterApplication(SimpleReport):
title = _("Discovery Problem")
form = ReportForm
try:
default_selector = ManagedObjectSelector.resolve_expression(
"@Problem Discovery Report")
except ManagedObjectSelector.DoesNotExist:
default_selector = None
predefined_reports = {
"default":
PredefinedReport(_("Problem Discovery 2(default)"),
{"selector": default_selector})
}
def get_data(self,
request,
pool=None,
obj_profile=None,
selector=None,
avail_status=None,
profile_check_only=None,
failed_scripts_only=None,
filter_pending_links=None,
filter_none_objects=None,
filter_view_other=None,
**kwargs):
data = []
match = None
code_map = {
"1": "Unknown error",
"10000": "Unspecified CLI error",
"10005": "Connection refused",
"10001": "Authentication failed",
"10002": "No super command defined",
"10003": "No super privileges",
"10004": "SSH Protocol error"
}
if not pool:
pool = Pool.objects.filter()[0]
data += [SectionRow(name="Report by %s" % pool.name)]
if selector:
mos = ManagedObject.objects.filter(selector.Q)
else:
mos = ManagedObject.objects.filter(pool=pool, is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if obj_profile:
mos = mos.filter(object_profile=obj_profile)
if filter_view_other:
mnp_in = list(
ManagedObjectProfile.objects.filter(enable_ping=False))
mos = mos.filter(profile=Profile.objects.get(
name=GENERIC_PROFILE)).exclude(object_profile__in=mnp_in)
discovery = "noc.services.discovery.jobs.box.job.BoxDiscoveryJob"
mos_id = list(mos.values_list("id", flat=True))
if avail_status:
avail = ObjectStatus.get_statuses(mos_id)
if profile_check_only:
match = {
"$or": [{
"job.problems.suggest_cli": {
"$exists": True
}
}, {
"job.problems.suggest_snmp": {
"$exists": True
}
}, {
"job.problems.profile.": {
"$regex": "Cannot detect profile"
}
}, {
"job.problems.version.": {
"$regex": "Remote error code 1000[1234]"
}
}]
}
elif failed_scripts_only:
match = {
"$and": [{
"job.problems": {
"$exists": "true",
"$ne": {}
}
}, {
"job.problems.suggest_snmp": {
"$exists": False
}
}, {
"job.problems.suggest_cli": {
"$exists": False
}
}]
}
elif filter_view_other:
match = {"job.problems.suggest_snmp": {"$exists": False}}
rdp = ReportDiscoveryProblem(mos, avail_only=avail_status, match=match)
exclude_method = []
if filter_pending_links:
exclude_method += ["lldp", "lacp", "cdp", "huawei_ndp"]
for discovery in rdp:
mo = ManagedObject.get_by_id(discovery["key"])
for method in ifilterfalse(lambda x: x in exclude_method,
discovery["job"][0]["problems"]):
problem = discovery["job"][0]["problems"][method]
if filter_none_objects and not problem:
continue
if isinstance(problem, dict) and "" in problem:
problem = problem.get("", "")
if "Remote error code" in problem:
problem = code_map.get(problem.split(" ")[-1], problem)
if isinstance(problem, six.string_types):
problem = problem.replace("\n", " ").replace("\r", " ")
data += [(mo.name, mo.address,
mo.profile.name, mo.administrative_domain.name,
_("Yes") if mo.get_status() else _("No"),
discovery["st"].strftime("%d.%m.%Y %H:%M")
if "st" in discovery else "", method, problem)]
return self.from_dataset(title=self.title,
columns=[
_("Managed Object"),
_("Address"),
_("Profile"),
_("Administrative Domain"),
_("Avail"),
_("Last successful discovery"),
_("Discovery"),
_("Error")
],
data=data)
class ReportFilterApplication(SimpleReport):
title = _("Failed Discovery")
form = ReportForm
try:
default_pool = Pool.objects.get(name="default")
except Exception:
default_pool = Pool.objects.all()[0]
predefined_reports = {
"default": PredefinedReport(
_("Failed Discovery (default)"), {
"pool": default_pool
}
)
}
@staticmethod
def decode_problem(problems):
"""
:param problems:
:type problems: collections.namedtuple
:return:
"""
decode_map = {
"Cannot detect profile": "SNMP Timeout",
"Remote error code 10000": "CLI Problem: Unspecified CLI error",
"Remote error code 10001": "CLI Problem: Authentication failed",
"Remote error code 10002": "CLI Problem: No super command defined",
"Remote error code 10003": "CLI Problem: No super privileges",
"Remote error code 10004": "CLI Problem: SSH Protocol error",
"Remote error code 10005": "CLI Problem: Connection refused",
"Remote error code 10200": "SNMP Problem",
"Remote error code 10201": "SNMP Timeout",
"Remote error code 599": "HTTP Error: Connection Timeout",
"Remote error code 1": "Adapter failed"}
decode, message = None, ""
if not problems:
return message
for index, message in enumerate(problems):
if not message:
continue
decode = decode_map.get(str(message))
break
if decode is None:
decode = message
return decode
def get_data(self, request, pool=None, selector=None, report_type=None, **kwargs):
data = []
columns, columns_desr = [], []
r_map = [
(_("Not Available"), "2is1.3isp1.3is1"),
(_("Failed to guess CLI credentials"), "2is1.3isp0.2isp1"),
(_("Failed to guess SNMP community"), "2is1.3isp1.3is2.1isp1")
]
for x, y in r_map:
columns += [y]
columns_desr += [x]
mos = ManagedObject.objects.filter()
if pool:
mos = mos.filter(pool=pool)
data += [SectionRow(name=pool.name)]
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
mos = list(mos.values_list("id", flat=True).order_by("id"))
mos_s = set(mos)
report = ReportModelFilter()
result = report.proccessed(",".join(columns))
mo_hostname = ReportObjectsHostname1(sync_ids=mos)
mo_hostname = mo_hostname.get_dictionary()
d_result = ReportDiscoveryResult(sync_ids=mos)
d_result = d_result.get_dictionary()
for col in columns:
for mo_id in result[col.strip()].intersection(mos_s):
mo = ManagedObject.get_by_id(mo_id)
problem = self.decode_problem(d_result.get(mo_id))
if not problem and mo_id not in d_result:
problem = "Discovery disabled"
data += [(
mo.name,
mo.address,
mo.administrative_domain.name,
mo.profile.name,
mo_hostname.get(mo.id, ""),
mo.auth_profile if mo.auth_profile else "",
mo.auth_profile.user if mo.auth_profile else mo.user,
mo.auth_profile.snmp_ro if mo.auth_profile else mo.snmp_ro,
_("No") if not mo.get_status() else _("Yes"),
columns_desr[columns.index(col)],
problem
)]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"), _("Address"), _("Administrative Domain"), _("Profile"), _("Hostname"),
_("Auth Profile"), _("Username"), _("SNMP Community"),
_("Avail"), _("Error"), _("Error Detail")
],
data=data)
class ReportObjectsSummary(SimpleReport):
title = _("Managed Objects Summary")
form = ReportForm
predefined_reports = {
"profile":
PredefinedReport(_("Managed Objects Summary (profile)"),
{"report_type": "profile"}),
"domain":
PredefinedReport(_("Managed Objects Summary (domain)"),
{"report_type": "domain"}),
"domain-profile":
PredefinedReport(_("Managed Objects Summary (domain-profile)"),
{"report_type": "domain-profile"}),
"tag":
PredefinedReport(_("Managed Objects Summary (tag)"),
{"report_type": "tag"}),
"platform":
PredefinedReport(_("Managed Objects Summary (platform)"),
{"report_type": "platform"}),
"version":
PredefinedReport(_("Managed Objects Summary (version)"),
{"report_type": "version"}),
}
def get_data(self, request, report_type=None, **kwargs):
wr = ("", "")
# wr_and = ("", "",)
wr_and2 = ("", "")
platform = {
str(p["_id"]): p["name"]
for p in Platform.objects.all().as_pymongo().scalar("id", "name")
}
version = {
str(p["_id"]): p["version"]
for p in Firmware.objects.all().as_pymongo().scalar(
"id", "version")
}
profile = {
str(p["_id"]): p["name"]
for p in Profile.objects.all().as_pymongo().scalar("id", "name")
}
if not request.user.is_superuser:
ad = tuple(UserAccess.get_domains(request.user))
wr = ("WHERE administrative_domain_id in ", ad)
# wr_and = ("AND sam.administrative_domain_id in ", ad)
wr_and2 = ("AND administrative_domain_id in ", ad)
if len(ad) == 1:
wr = ("WHERE administrative_domain_id in (%s)" % ad, "")
# wr_and = ("AND sam.administrative_domain_id in (%s)" % ad, "")
wr_and2 = ("AND administrative_domain_id in (%s)" % ad, "")
# By Profile
if report_type == "profile":
columns = [_("Profile")]
query = ("""SELECT profile,COUNT(*) FROM sa_managedobject
%s%s GROUP BY 1 ORDER BY 2 DESC""" % wr)
# By Administrative Domain
elif report_type == "domain":
columns = [_("Administrative Domain")]
query = ("""SELECT a.name,COUNT(*)
FROM sa_managedobject o JOIN sa_administrativedomain a ON (o.administrative_domain_id=a.id)
%s%s
GROUP BY 1
ORDER BY 2 DESC""" % wr)
# By Profile and Administrative Domains
elif report_type == "domain-profile":
columns = [_("Administrative Domain"), _("Profile")]
query = ("""SELECT d.name,profile,COUNT(*)
FROM sa_managedobject o JOIN sa_administrativedomain d ON (o.administrative_domain_id=d.id)
%s%s
GROUP BY 1,2
""" % wr)
# By tags
elif report_type == "tag":
columns = [_("Tag")]
query = ("""
SELECT t.tag, COUNT(*)
FROM (
SELECT unnest(tags) AS tag
FROM sa_managedobject
WHERE
tags IS NOT NULL
%s%s
AND array_length(tags, 1) > 0
) t
GROUP BY 1
ORDER BY 2 DESC;
""" % wr_and2)
elif report_type == "platform":
columns = [_("Platform"), _("Profile")]
query = ("""select sam.profile, sam.platform, COUNT(platform)
from sa_managedobject sam %s%s group by 1,2 order by count(platform) desc;"""
% wr)
elif report_type == "version":
columns = [_("Profile"), _("Version")]
query = ("""select sam.profile, sam.version, COUNT(version)
from sa_managedobject sam %s%s group by 1,2 order by count(version) desc;"""
% wr)
else:
raise Exception("Invalid report type: %s" % report_type)
for r, t in report_types:
if r == report_type:
title = self.title + ": " + t
break
columns += [
TableColumn(_("Quantity"),
align="right",
total="sum",
format="integer")
]
cursor = self.cursor()
cursor.execute(query, ())
data = []
for c in cursor.fetchall():
if report_type == "profile":
data += [(profile.get(c[0]), c[1])]
elif report_type == "domain-profile":
data += [(c[0], profile.get(c[1]), c[2])]
elif report_type == "platform":
data += [(profile.get(c[0]), platform.get(c[1]), c[2])]
elif report_type == "version":
data += [(profile.get(c[0]), version.get(c[1]), c[2])]
else:
data += [c]
return self.from_dataset(title=title,
columns=columns,
data=data,
enumerate=True)
class ReportOutagesApplication(SimpleReport):
title = _("Outages")
form = ReportForm
predefined_reports = {
"1d": PredefinedReport(_("Outages (1 day)"), {"duration": 86400}),
"7d": PredefinedReport(_("Outages (7 days)"), {"duration": 7 * 86400}),
"30d": PredefinedReport(_("Outages (30 day)"),
{"duration": 30 * 86400}),
}
def get_data(self,
request,
duration,
from_date=None,
to_date=None,
**kwargs):
now = datetime.datetime.now()
if not from_date:
duration = 1
if int(duration):
self.logger.info("Use duration\n")
d = datetime.timedelta(seconds=int(duration))
b = now - d
q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
else:
b = datetime.datetime.strptime(from_date, "%d.%m.%Y")
q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
if to_date:
if from_date == to_date:
t1 = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(1)
else:
t1 = datetime.datetime.strptime(to_date, "%d.%m.%Y")
else:
t1 = now
q &= Q(start__lte=t1) | Q(stop__lte=t1)
d = datetime.timedelta(seconds=int((t1 - b).total_seconds()))
outages = defaultdict(list)
otime = defaultdict(int)
for o in Outage.objects.filter(q):
start = max(o.start, b)
stop = o.stop if o.stop else now
outages[o.object] += [o]
otime[o.object] += (stop - start).total_seconds()
td = d.total_seconds()
if not request.user.is_superuser:
for mo in ManagedObject.objects.exclude(
administrative_domain__in=UserAccess.get_domains(
request.user)):
if mo.id in otime:
otime.pop(mo.id)
# Load managed objects
mos = list(otime)
chunk = 500
mo = {}
while mos:
for o in ManagedObject.objects.filter(id__in=mos[:chunk]):
mo[o.id] = o
mos = mos[chunk:]
r = []
for o in sorted(otime, key=lambda x: -otime[x]):
m = mo.get(o)
if not m:
continue # Hanging Outage
dt = min(td, otime[o])
downtime = "%02d:%02d:%02d" % ((dt // 3600) % 24,
(dt // 60) % 60, dt % 60)
if dt >= 86400:
downtime = "%dd %s" % (dt // 86400, downtime)
if td:
avail = float(td - dt) * 100 / td
else:
avail = 0
r += [(
m.name,
m.address,
m.profile.name,
m.platform.name if m.platform else "",
_("Yes") if m.is_managed else _("No"),
_("Yes") if m.get_status() else _("No"),
downtime,
avail,
len(outages[o]),
)]
return self.from_dataset(
title=self.title,
columns=[
_("Managed Object"),
_("Address"),
_("Profile"),
_("Platform"),
TableColumn(_("Managed"), align="right"),
TableColumn(_("Status"), align="right"),
TableColumn(_("Downtime"), align="right"),
TableColumn(_("Availability"), align="right",
format="percent"),
TableColumn(_("Downs"), align="right", format="integer"),
],
data=r,
enumerate=True,
)
class ReportFilterApplication(SimpleReport):
title = _("Managed Object Profile Check Summary")
predefined_reports = {
"default": PredefinedReport(_("Managed Object Profile Check Summary"),
{})
}
save_perc = None
def calc_percent(self, column, val):
if column == _("Is Managed, object type defined"):
if val == 0:
return "%.2f %%" % 100
else:
r = "%.2f %%" % ((val / float(self.save_perc)) * 100)
self.save_perc = None
return r
elif column == _("Is Managed"):
self.save_perc = val
return ""
def get_data(self, request, report_type=None, **kwargs):
columns, columns_desr = [], []
sc_code = [
"1.1",
"1.2",
"1.2.1",
"1.2.1.1",
"1.2.2",
"1.2.2.1",
"1.2.2.2",
"1.2.2.2.1",
"1.2.2.2.2",
"1.2.2.2.2.1",
"1.2.3",
]
r_map = [
(_("Not Managed"), "1is1"),
(_("Is Managed"), "2is1"),
# (_("Is Managed not monitoring"), "2is2"),
# (_("Is monitoring"), "2is1"),
(_("Is Managed, object type defined"), "2is1.6is1.3isp0"),
(_("Is Managed, object type defined bad CLI Credential"),
"2is1.6is1.3isp0.2isp1"),
(_("Is Managed, object type undefined"), "2is1.6is1.3isp1"),
(_("Is Managed, object type undefined not ping response"),
"2is1.6is1.3isp1.3is1"),
(_("Is Managed, object type undefined has ping response"),
"2is1.6is1.3isp1.3is2"),
(
_("Is Managed, object type undefined bad SNMP Credential"),
"2is1.6is1.3isp1.3is2.1isp1",
),
(
_("Is Managed, object type undefined for various reasons"),
"2is1.6is1.3isp1.3is2.1isp0",
),
(_("Is Managed, object type Profile is not know"),
"2is1.6is1.9a1.3is2.4isp1"),
(_("Is monitoring, object type undefined, only availablity check"),
"2is1.6is2"),
]
for x, y in r_map:
columns += [y]
columns_desr += [x]
report = ReportModelFilter()
result = report.proccessed(",".join(columns))
# result = stat.proccessed(",".join(columns))
# result = stat.api_result(",".join(columns))
summary = defaultdict(int)
data = []
# url = "/sa/reportstat/repstat_download/?report=%s"
url = "/sa/reportobjectdetail/download/?" + "&".join([
"o_format=xlsx",
"columns=object_name,object_address,object_profile,object_status,profile_name,admin_domain,segment",
"detail_stat=%s&pool=%s",
])
for p in Pool.objects.filter().order_by("name"):
m = []
moss = set(
ManagedObject.objects.filter(pool=p).values_list("id",
flat=True))
for col in columns:
m += [len(result[col.strip()].intersection(moss))]
summary[col] += m[-1]
data += [SectionRow(name=p.name)]
data += [(sc, x, y, self.calc_percent(x, y),
url % (columns[columns_desr.index(x)], p.name))
for sc, x, y in zip(sc_code, columns_desr, m)]
data += [("1.2.2.2.2.2",
_("Is Managed, objects not processed yet"), 0, "")]
data += [SectionRow(name="Summary")]
summary = [summary[k] for k in columns]
data += [(sc, x, y, self.calc_percent(x, y),
url % (columns[columns_desr.index(x)], ""))
for sc, x, y in zip(sc_code, columns_desr, summary)]
data += [("1.2.2.2.2.2", _("Is Managed, objects not processed yet"), 0,
"")]
# columns = ["ID", "Value", "Percent", TableColumn(_("Detail"), format="url")]
columns = [
_("PP"),
_("Status"),
_("Quantity"),
_("Percent"),
TableColumn(_("Detail"), format="url"),
]
return self.from_dataset(title=self.title, columns=columns, data=data)
class ReportFilterApplication(SimpleReport):
title = _("Discovery Links Summary")
predefined_reports = {
"default": PredefinedReport(_("Discovery Links Summary"), {})
}
def get_data(self, request, **kwargs):
data = []
value = get_db()["noc.links"].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate([{
"$unwind":
"$interfaces"
}, {
"$lookup": {
"from": "noc.interfaces",
"localField": "interfaces",
"foreignField": "_id",
"as": "int"
}
}, {
"$group": {
"_id": "$int.managed_object",
"count": {
"$sum": 1
}
}
}])
count = {0: set([]), 1: set([]), 2: set([]), 3: set([])}
ap = AuthProfile.objects.filter(name__startswith="TG")
for v in value:
if v["count"] > 2:
count[3].add(v["_id"][0])
continue
if not v["_id"]:
self.logger.warning("No IDS in response query")
continue
count[v["count"]].add(v["_id"][0])
for p in Pool.objects.order_by("name"):
if p.name == "P0001":
continue
data += [SectionRow(name=p.name)]
smos = set(
ManagedObject.objects.filter(pool=p, is_managed=True).exclude(
profile=Profile.get_by_name(GENERIC_PROFILE)).exclude(
auth_profile__in=ap).values_list('id', flat=True))
all_p = 100.0 / len(smos) if len(smos) else 1.0
data += [("All polling", len(smos))]
for c in count:
if c == 3:
data += [
("More 3", len(count[c].intersection(smos)),
"%.2f %%" %
round(len(count[c].intersection(smos)) * all_p, 2))
]
continue
data += [(c, len(count[c].intersection(smos)), "%.2f %%" %
round(len(count[c].intersection(smos)) * all_p), 2)]
# 0 links - All discovering- summary with links
s0 = len(smos) - sum([d[1] for d in data[-3:]])
data.pop(-4)
data.insert(-3, (0, s0, "%.2f %%" % round(s0 * all_p, 2)))
return self.from_dataset(
title=self.title,
columns=[_("Links count"),
_("MO Count"),
_("Percent at All")],
data=data)
class ReportTTSystemStatApplication(SimpleReport):
title = _("TT system statistics")
form = ReportForm
predefined_reports = {
"1d":
PredefinedReport(_("TTSystem Stat (1 day)"), {"interval": 1}),
"7d":
PredefinedReport(_("TTSystem Stat (7 days)"), {"interval": 7}),
"30d":
PredefinedReport(_("TTSystem Stat (30 day)"), {"interval": 30}),
"1d_errors":
PredefinedReport(_("TTSystem Errors (1 day)"), {
"interval": 1,
"repo_format": "1"
})
}
def get_data(self,
request,
interval=1,
repo_format=0,
from_date=None,
to_date=None,
**kwargs):
# Date Time Block
if from_date:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
elif interval:
from_date = datetime.datetime.now() - datetime.timedelta(
days=int(interval))
else:
from_date = datetime.datetime.now() - datetime.timedelta(days=1)
if to_date:
to_date = datetime.datetime.strptime(to_date, "%d.%m.%Y")
if from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
elif interval:
to_date = from_date + datetime.timedelta(days=int(interval))
else:
to_date = from_date + datetime.timedelta(days=1)
columns = [
_("Server"),
_("Service"),
_("Request count"),
_("Success request count"),
_("Failed request count"),
_("Success request (%)"),
_("Q1 (ms)"),
_("Q2 (ms)"),
_("Q3 (ms)"),
_("p95 (ms)"),
_("max (ms)")
]
if repo_format == "1":
columns = [
_("Timestamp"),
_("Server"),
_("Service"),
_("Managed object"),
_("TT ID"),
_("Error code"),
_("Error text")
]
ts_from_date = time.mktime(from_date.timetuple())
ts_to_date = time.mktime(to_date.timetuple())
tt_systems = TTSystem.objects.filter().scalar("name")
# Manged Object block
q1 = """select server, service, count(), round(quantile(0.25)(duration), 0)/1000 as q1,
round(quantile(0.5)(duration), 0)/1000 as q2,
round(quantile(0.75)(duration), 0)/1000 as q3,
round(quantile(0.95)(duration),0)/1000 as p95,
round(max(duration),0)/1000 as max from span where %s
group by server, service"""
q2 = """select server, service, error_code, count(), avg(duration)
from span where %s group by server, service, error_code"""
q3 = """select ts, server, service, in_label, in_label, error_code, error_text from span
where service IN ('create_massive_damage_outer', 'change_massive_damage_outer_close') and
error_code <> 0 and %s"""
q_where = ["server IN ('%s')" % "', '".join(tt_systems)]
# q_where = ["managed_object IN (%s)" % ", ".join(mo_bi_dict.keys())]
q_where += [
"(date >= toDate(%d)) AND (ts >= toDateTime(%d) AND ts <= toDateTime(%d))"
% (ts_from_date, ts_from_date, ts_to_date)
]
r = []
ch = connection()
if repo_format == "1":
aa = {
aa.escalation_tt.split(":")[-1]: aa
for aa in ArchivedAlarm.objects.filter(
clear_timestamp__gte=from_date,
clear_timestamp__lte=to_date,
escalation_tt__exists=True)
}
query = q3 % " and ".join(q_where)
for row in ch.execute(query):
if row[2] in ["create_massive_damage_outer"]:
row[2] = u"Создание ТТ"
try:
row[3] = ManagedObject.objects.get(
tt_system_id=int(row[3]))
row[4] = ""
except ManagedObject.DoesNotExist:
pass
except ManagedObject.MultipleObjectsReturned:
row[3] = ManagedObject.objects.get(tt_system_id=int(
row[3]),
is_managed=True)
row[4] = ""
elif row[2] in ["change_massive_damage_outer_close"]:
row[2] = u"Закрытие ТТ"
row[4] = row[3]
row[3] = aa[
row[3]].managed_object if row[3] in aa else row[3]
else:
continue
r += [row]
else:
query = q1 % " and ".join(q_where)
# (server, service)
tt_s = {}
for row in ch.execute(query):
tt_s[(row[0], row[1])] = [row[2]] + [0, 0, 0] + row[3:]
query = q2 % " and ".join(q_where)
for row in ch.execute(query):
if row[2] == "0":
tt_s[(row[0], row[1])][1] = row[3]
else:
tt_s[(row[0], row[1])][2] += int(row[3])
r += [
SectionRow(name="Report from %s to %s" %
(from_date.strftime("%d.%m.%Y %H:%M"),
to_date.strftime("%d.%m.%Y %H:%M")))
]
for line in sorted(tt_s, key=lambda x: x[0]):
data = list(line)
data += tt_s[line]
data[5] = round((float(data[3]) / float(data[2])) * 100.0, 2)
r += [data]
return self.from_dataset(title=self.title,
columns=columns,
data=r,
enumerate=True)
class ReportAvailabilityApplication(SimpleReport):
title = _("Availability")
form = ReportForm
predefined_reports = {
"1d": PredefinedReport(_("Availability (last day)"), {"interval": 1}),
"7d": PredefinedReport(_("Availability (last week)"), {"interval": 7}),
"30d": PredefinedReport(_("Availability (last month)"),
{"interval": 30}),
}
@staticmethod
def get_availability_interval(days):
now = datetime.datetime.now()
d = datetime.timedelta(days=days)
b = now - d
outages = defaultdict(int)
q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
for o in Outage.objects.filter(
q, read_preference=ReadPreference.SECONDARY_PREFERRED):
start = max(o.start, b)
stop = o.stop if o.stop else now
outages[o.object] += (stop - start).total_seconds()
td = d.total_seconds() # Normalize to percents
return dict((o, (td - outages[o]) * 100.0 / td) for o in outages)
@staticmethod
def get_availability(start_date, stop_date, skip_zero_avail=False):
# now = datetime.datetime.now()
b = start_date
d = stop_date
outages = defaultdict(list)
td = (d - b).total_seconds()
# q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)
q = (Q(start__gte=b) | Q(stop__gte=b)
| Q(stop__exists=False)) & Q(start__lt=d)
for o in Outage.objects.filter(q):
start = max(o.start, b)
stop = o.stop if (o.stop and o.stop < d) else d
if (stop - start).total_seconds() == td and skip_zero_avail:
continue
outages[o.object] += [(stop - start).total_seconds()]
# Normalize to percents
return dict((o, ((td - sum(outages[o])) * 100.0 / td,
int(sum(outages[o])), len(outages[o])))
for o in outages)
@staticmethod
def get_reboots(start_date=None, stop_date=None):
match = {"ts": {"$gte": start_date, "$lte": stop_date}}
pipeline = [
{
"$match": match
},
{
"$group": {
"_id": "$object",
"count": {
"$sum": 1
}
}
},
{
"$sort": {
"count": -1
}
},
]
data = (Reboot._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate(
pipeline))
# data = data["result"]
return dict((rb["_id"], rb["count"]) for rb in data)
def get_data(self,
request,
interval=1,
from_date=None,
to_date=None,
skip_avail=False,
skip_zero_avail=False,
filter_zero_access=False,
**kwargs):
"""
a1 = self.get_availability(1)
a7 = self.get_availability(7)
a30 = self.get_availability(30)
"""
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(
days=interval)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
a = self.get_availability(start_date=from_date,
stop_date=to_date,
skip_zero_avail=skip_zero_avail)
rb = self.get_reboots(start_date=from_date, stop_date=to_date)
r = [SectionRow("Report from %s to %s" % (from_date, to_date))]
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if skip_avail:
mos = mos.filter(id__in=list(a))
mos_id = list(mos.order_by("id").values_list("id", flat=True))
if filter_zero_access:
iface_p = InterfaceProfile.objects.get(name="Клиентский порт")
match = {"profile": iface_p.id, "managed_object": {"$in": mos_id}}
pipeline = [
{
"$match": match
},
{
"$group": {
"_id": "$managed_object",
"count": {
"$sum": 1
},
"m": {
"$max": "$oper_status"
},
}
},
{
"$match": {
"m": False
}
},
{
"$project": {
"_id": True
}
},
]
# data = Interface.objects._get_collection().aggregate(pipeline,
data = (get_db()["noc.interfaces"].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate(
pipeline))
data = [d["_id"] for d in data]
mos = mos.exclude(id__in=data)
mo_hostname = ReportObjectsHostname1(sync_ids=mos_id)
mo_hostname = mo_hostname.get_dictionary()
for o in mos:
s = [
o.administrative_domain.name,
o.name,
mo_hostname.get(o.id, ""),
o.address,
o.profile.name,
round(a.get(o.id, (100.0, 0, 0))[0], 2),
]
s.extend(a.get(o.id, (100.0, 0, 0))[1:])
s.append(rb[o.id] if o.id in rb else 0)
r += [s]
"""
a1.get(o.id, 100),
a7.get(o.id, 100),
a30.get(o.id, 100)
"""
# print r
return self.from_dataset(
title=self.title,
columns=[
_("Adm. Domain"),
_("Managed Object"),
_("Hostname"),
_("Address"),
_("Profile"),
# TableColumn(_("Avail"), align="right", format="percent"),
# TableColumn(_("Total avail (sec)"), align="right", format="numeric"),
_("Avail"),
_("Total unavail (sec)"),
_("Count outages"),
_("Reboots"),
],
data=r,
enumerate=True,
)
class ReportFilterApplication(SimpleReport):
title = _("Discovery Links Summary")
predefined_reports = {
"default": PredefinedReport(_("Discovery Links Summary"), {})
}
save_perc = None
def calc_percent(self, column, val):
if column != _("All polling"):
if val == 0:
return "%.2f %%" % 100
else:
r = "%.2f %%" % ((val / float(self.save_perc)) * 100)
# self.save_perc = None
return r
elif column == _("All polling"):
self.save_perc = val
return ""
def get_data(self, request, **kwargs):
columns, columns_desr = [], []
r_map = [
(_("All polling"),
"2is1.6is1.7a2"), # "Is Managed, object type defined"
(_("0"), "2is1.6is1.7a2.3hs0"), # "Has 0 Links w type defined"
(_("1"), "2is1.6is1.3hs2"), # "Has 1 links"
(_("2"), "2is1.6is1.3hs3"), # "Has 2 links"
(_("More 3"), "2is1.6is1.3hs4"), # "Has more 3 links"
]
for x, y in r_map:
columns += [y]
columns_desr += [x]
report = ReportModelFilter()
result = report.proccessed(",".join(columns))
summary = defaultdict(int)
data = []
# url = "/sa/reportstat/repstat_download/?report=%s"
url = "/sa/reportobjectdetail/download/?" + "&".join([
"o_format=xlsx",
"columns=object_name,object_address,object_profile,object_status,profile_name,admin_domain,segment",
"detail_stat=%s&pool=%s",
])
for p in Pool.objects.filter().order_by("name"):
m = []
moss = set(
ManagedObject.objects.filter(pool=p).values_list("id",
flat=True))
for col in columns:
m += [len(result[col.strip()].intersection(moss))]
summary[col] += m[-1]
data += [SectionRow(name=p.name)]
data += [(x, y, self.calc_percent(x, y),
url % (columns[columns_desr.index(x)], p.name))
for x, y in zip(columns_desr, m)]
return self.from_dataset(
title=self.title,
columns=[
_("Links count"),
_("MO Count"),
_("Percent at All"),
TableColumn(_("Detail"), format="url"),
],
data=data,
)
class ReportFilterApplication(SimpleReport):
title = _("Check Interface Facts")
try:
default_pool = Pool.objects.get(name="default")
except Pool.DoesNotExist:
default_pool = Pool.objects.all()[0]
predefined_reports = {
"default": PredefinedReport(_("Check Interface Facts (default)"), {"pool": default_pool})
}
def get_form(self):
class ReportForm(forms.Form):
pool = forms.ChoiceField(
label=_("Managed Objects Pool"),
required=False,
choices=list(Pool.objects.order_by("name").scalar("id", "name"))
+ [(None, "-" * 9)],
)
int_profile = forms.ChoiceField(
label=_("Interface Profile"),
required=True,
choices=list(InterfaceProfile.objects.order_by("name").scalar("id", "name"))
+ [(None, "-" * 9)],
)
mop = forms.ModelChoiceField(
label=_("Managed Object Profile"),
required=False,
queryset=ManagedObjectProfile.objects.order_by("name"),
)
# int_fact = forms.ModelChoiceField(
# label=_("Check Facts"),
# required=False,
# queryset=ManagedObjectProfile.objects.order_by("name"))
return ReportForm
def get_data(self, request, pool=None, int_profile=None, mop=None, avail_status=None, **kwargs):
data = []
mos = ManagedObject.objects.filter(is_managed=True)
# % fixme remove.
if not pool and request.user.is_superuser:
pool = Pool.get_by_name("STAGEMO")
if pool:
mos = mos.filter(pool=pool)
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
if mop:
mos = mos.filter(object_profile=mop)
mos_ids = mos.values_list("id", flat=True)
iface = Interface.objects.filter(
managed_object__in=mos, profile=int_profile, type="physical"
).values_list("managed_object", "name")
res = []
n = 0
# Interface._get_collection()
while mos_ids[(0 + n) : (10000 + n)]:
mos_ids_f = mos_ids[(0 + n) : (10000 + n)]
s_iface = {"%d.%s" % (mo.id, name) for mo, name in iface}
of = ObjectFact.objects.filter(
object__in=mos_ids_f, cls="subinterface", attrs__traffic_control_broadcast=False
)
a_f = {".".join((str(o.object.id), o.attrs["name"])) for o in of}
res.extend(a_f.intersection(s_iface))
n += 10000
for s in res:
mo, iface = s.split(".")
mo = ManagedObject.get_by_id(mo)
data += [(mo.name, mo.address, mo.profile.name, iface)]
return self.from_dataset(
title=self.title,
columns=[_("Managed Object"), _("Address"), _("SA Profile"), _("Interface")],
data=data,
)
