class ReportEscalationsApplication(SimpleReport): title = _("Escalations") form = ReportForm predefined_reports = { "1d": PredefinedReport(_("Escalations (1 day)"), {"interval": 1}), "7d": PredefinedReport(_("Escalations (7 days)"), {"interval": 7}), "30d": PredefinedReport(_("Escalations (30 day)"), {"interval": 30}), } def get_data(self, request, interval, from_date=None, to_date=None, **kwargs): interval = int(interval) if not from_date: interval = 1 if interval: ts = datetime.datetime.now() - datetime.timedelta(days=interval) q = {"timestamp": {"$gte": ts}} else: t0 = datetime.datetime.strptime(from_date, "%d.%m.%Y") if not to_date: t1 = datetime.datetime.now() else: t1 = datetime.datetime.strptime( to_date, "%d.%m.%Y") + datetime.timedelta(days=1) q = {"timestamp": {"$gte": t0, "$lte": t1}} q["escalation_tt"] = {"$exists": True} if not request.user.is_superuser: q["adm_path"] = {"$in": UserAccess.get_domains(request.user)} data = [] for ac in (ActiveAlarm, ArchivedAlarm): for d in ac._get_collection().find(q): mo = ManagedObject.get_by_id(d["managed_object"]) if not mo: continue data += [( d["timestamp"].strftime("%Y-%m-%d %H:%M:%S"), d["escalation_ts"].strftime("%Y-%m-%d %H:%M:%S"), mo.name.split("#", 1)[0], mo.address, mo.platform, mo.segment.name, d["escalation_tt"], sum(ss["summary"] for ss in d["total_objects"]), sum(ss["summary"] for ss in d["total_subscribers"]), )] data = sorted(data, key=operator.itemgetter(0)) return self.from_dataset( title=self.title, columns=[ _("Timestamp"), _("Escalation Timestamp"), _("Managed Object"), _("Address"), _("Platform"), _("Segment"), _("TT"), _("Objects"), _("Subscribers"), ], data=data, enumerate=True, )
class ReportRebootsApplication(SimpleReport): title = _("Reboots") form = ReportForm predefined_reports = { "1d": PredefinedReport(_("Reboots (1 day)"), {"interval": 1}), "7d": PredefinedReport(_("Reboots (7 days)"), {"interval": 7}), "30d": PredefinedReport(_("Reboot (30 day)"), {"interval": 30}), } def get_data(self, request, interval, from_date=None, to_date=None, **kwargs): interval = int(interval) if not from_date: interval = 1 if interval: ts = datetime.datetime.now() - datetime.timedelta(days=interval) match = {"ts": {"$gte": ts}} else: t0 = datetime.datetime.strptime(from_date, "%d.%m.%Y") if not to_date: t1 = datetime.datetime.now() else: t1 = datetime.datetime.strptime( to_date, "%d.%m.%Y") + datetime.timedelta(days=1) match = {"ts": {"$gte": t0, "$lte": t1}} pipeline = [ { "$match": match }, { "$group": { "_id": "$object", "count": { "$sum": 1 } } }, { "$sort": { "count": -1 } }, ] data = list(Reboot._get_collection().aggregate(pipeline)) # Get managed objects if not request.user.is_superuser: id_perm = [ mo.id for mo in ManagedObject.objects.filter( administrative_domain__in=UserAccess.get_domains( request.user)) ] ids = [x["_id"] for x in data if x["_id"] in id_perm] else: ids = [x["_id"] for x in data] mo_names = {} # mo id -> mo name cursor = connection.cursor() while ids: chunk = [str(x) for x in ids[:500]] ids = ids[500:] cursor.execute(""" SELECT id, address, name FROM sa_managedobject WHERE id IN (%s)""" % ", ".join(chunk)) mo_names.update(dict((c[0], c[1:3]) for c in cursor)) # if not request.user.is_superuser: data = [(mo_names.get(x["_id"], "---")[1], mo_names.get(x["_id"], "---")[0], x["count"]) for x in data if x["_id"] in id_perm] else: data = [(mo_names.get(x["_id"], "---")[1], mo_names.get(x["_id"], "---")[0], x["count"]) for x in data] return self.from_dataset( title=self.title, columns=[ _("Managed Object"), _("Address"), TableColumn(_("Reboots"), align="right", format="numeric", total="sum"), ], data=data, enumerate=True, )
class ReportFilterApplication(SimpleReport): title = _("Discovery Problem") form = ReportForm try: default_selector = ManagedObjectSelector.resolve_expression( "@Problem Discovery Report") except ManagedObjectSelector.DoesNotExist: default_selector = None predefined_reports = { "default": PredefinedReport(_("Problem Discovery 2(default)"), {"selector": default_selector}) } def get_data(self, request, pool=None, obj_profile=None, selector=None, avail_status=None, profile_check_only=None, failed_scripts_only=None, filter_pending_links=None, filter_none_objects=None, filter_view_other=None, **kwargs): data = [] match = None code_map = { "1": "Unknown error", "10000": "Unspecified CLI error", "10005": "Connection refused", "10001": "Authentication failed", "10002": "No super command defined", "10003": "No super privileges", "10004": "SSH Protocol error" } if not pool: pool = Pool.objects.filter()[0] data += [SectionRow(name="Report by %s" % pool.name)] if selector: mos = ManagedObject.objects.filter(selector.Q) else: mos = ManagedObject.objects.filter(pool=pool, is_managed=True) if not request.user.is_superuser: mos = mos.filter( administrative_domain__in=UserAccess.get_domains(request.user)) if obj_profile: mos = mos.filter(object_profile=obj_profile) if filter_view_other: mnp_in = list( ManagedObjectProfile.objects.filter(enable_ping=False)) mos = mos.filter(profile=Profile.objects.get( name=GENERIC_PROFILE)).exclude(object_profile__in=mnp_in) discovery = "noc.services.discovery.jobs.box.job.BoxDiscoveryJob" mos_id = list(mos.values_list("id", flat=True)) if avail_status: avail = ObjectStatus.get_statuses(mos_id) if profile_check_only: match = { "$or": [{ "job.problems.suggest_cli": { "$exists": True } }, { "job.problems.suggest_snmp": { "$exists": True } }, { "job.problems.profile.": { "$regex": "Cannot detect profile" } }, { "job.problems.version.": { "$regex": "Remote error code 1000[1234]" } }] } elif failed_scripts_only: match = { "$and": [{ "job.problems": { "$exists": "true", "$ne": {} } }, { "job.problems.suggest_snmp": { "$exists": False } }, { "job.problems.suggest_cli": { "$exists": False } }] } elif filter_view_other: match = {"job.problems.suggest_snmp": {"$exists": False}} rdp = ReportDiscoveryProblem(mos, avail_only=avail_status, match=match) exclude_method = [] if filter_pending_links: exclude_method += ["lldp", "lacp", "cdp", "huawei_ndp"] for discovery in rdp: mo = ManagedObject.get_by_id(discovery["key"]) for method in ifilterfalse(lambda x: x in exclude_method, discovery["job"][0]["problems"]): problem = discovery["job"][0]["problems"][method] if filter_none_objects and not problem: continue if isinstance(problem, dict) and "" in problem: problem = problem.get("", "") if "Remote error code" in problem: problem = code_map.get(problem.split(" ")[-1], problem) if isinstance(problem, six.string_types): problem = problem.replace("\n", " ").replace("\r", " ") data += [(mo.name, mo.address, mo.profile.name, mo.administrative_domain.name, _("Yes") if mo.get_status() else _("No"), discovery["st"].strftime("%d.%m.%Y %H:%M") if "st" in discovery else "", method, problem)] return self.from_dataset(title=self.title, columns=[ _("Managed Object"), _("Address"), _("Profile"), _("Administrative Domain"), _("Avail"), _("Last successful discovery"), _("Discovery"), _("Error") ], data=data)
class ReportFilterApplication(SimpleReport): title = _("Failed Discovery") form = ReportForm try: default_pool = Pool.objects.get(name="default") except Exception: default_pool = Pool.objects.all()[0] predefined_reports = { "default": PredefinedReport( _("Failed Discovery (default)"), { "pool": default_pool } ) } @staticmethod def decode_problem(problems): """ :param problems: :type problems: collections.namedtuple :return: """ decode_map = { "Cannot detect profile": "SNMP Timeout", "Remote error code 10000": "CLI Problem: Unspecified CLI error", "Remote error code 10001": "CLI Problem: Authentication failed", "Remote error code 10002": "CLI Problem: No super command defined", "Remote error code 10003": "CLI Problem: No super privileges", "Remote error code 10004": "CLI Problem: SSH Protocol error", "Remote error code 10005": "CLI Problem: Connection refused", "Remote error code 10200": "SNMP Problem", "Remote error code 10201": "SNMP Timeout", "Remote error code 599": "HTTP Error: Connection Timeout", "Remote error code 1": "Adapter failed"} decode, message = None, "" if not problems: return message for index, message in enumerate(problems): if not message: continue decode = decode_map.get(str(message)) break if decode is None: decode = message return decode def get_data(self, request, pool=None, selector=None, report_type=None, **kwargs): data = [] columns, columns_desr = [], [] r_map = [ (_("Not Available"), "2is1.3isp1.3is1"), (_("Failed to guess CLI credentials"), "2is1.3isp0.2isp1"), (_("Failed to guess SNMP community"), "2is1.3isp1.3is2.1isp1") ] for x, y in r_map: columns += [y] columns_desr += [x] mos = ManagedObject.objects.filter() if pool: mos = mos.filter(pool=pool) data += [SectionRow(name=pool.name)] if not request.user.is_superuser: mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user)) mos = list(mos.values_list("id", flat=True).order_by("id")) mos_s = set(mos) report = ReportModelFilter() result = report.proccessed(",".join(columns)) mo_hostname = ReportObjectsHostname1(sync_ids=mos) mo_hostname = mo_hostname.get_dictionary() d_result = ReportDiscoveryResult(sync_ids=mos) d_result = d_result.get_dictionary() for col in columns: for mo_id in result[col.strip()].intersection(mos_s): mo = ManagedObject.get_by_id(mo_id) problem = self.decode_problem(d_result.get(mo_id)) if not problem and mo_id not in d_result: problem = "Discovery disabled" data += [( mo.name, mo.address, mo.administrative_domain.name, mo.profile.name, mo_hostname.get(mo.id, ""), mo.auth_profile if mo.auth_profile else "", mo.auth_profile.user if mo.auth_profile else mo.user, mo.auth_profile.snmp_ro if mo.auth_profile else mo.snmp_ro, _("No") if not mo.get_status() else _("Yes"), columns_desr[columns.index(col)], problem )] return self.from_dataset( title=self.title, columns=[ _("Managed Object"), _("Address"), _("Administrative Domain"), _("Profile"), _("Hostname"), _("Auth Profile"), _("Username"), _("SNMP Community"), _("Avail"), _("Error"), _("Error Detail") ], data=data)
class ReportObjectsSummary(SimpleReport): title = _("Managed Objects Summary") form = ReportForm predefined_reports = { "profile": PredefinedReport(_("Managed Objects Summary (profile)"), {"report_type": "profile"}), "domain": PredefinedReport(_("Managed Objects Summary (domain)"), {"report_type": "domain"}), "domain-profile": PredefinedReport(_("Managed Objects Summary (domain-profile)"), {"report_type": "domain-profile"}), "tag": PredefinedReport(_("Managed Objects Summary (tag)"), {"report_type": "tag"}), "platform": PredefinedReport(_("Managed Objects Summary (platform)"), {"report_type": "platform"}), "version": PredefinedReport(_("Managed Objects Summary (version)"), {"report_type": "version"}), } def get_data(self, request, report_type=None, **kwargs): wr = ("", "") # wr_and = ("", "",) wr_and2 = ("", "") platform = { str(p["_id"]): p["name"] for p in Platform.objects.all().as_pymongo().scalar("id", "name") } version = { str(p["_id"]): p["version"] for p in Firmware.objects.all().as_pymongo().scalar( "id", "version") } profile = { str(p["_id"]): p["name"] for p in Profile.objects.all().as_pymongo().scalar("id", "name") } if not request.user.is_superuser: ad = tuple(UserAccess.get_domains(request.user)) wr = ("WHERE administrative_domain_id in ", ad) # wr_and = ("AND sam.administrative_domain_id in ", ad) wr_and2 = ("AND administrative_domain_id in ", ad) if len(ad) == 1: wr = ("WHERE administrative_domain_id in (%s)" % ad, "") # wr_and = ("AND sam.administrative_domain_id in (%s)" % ad, "") wr_and2 = ("AND administrative_domain_id in (%s)" % ad, "") # By Profile if report_type == "profile": columns = [_("Profile")] query = ("""SELECT profile,COUNT(*) FROM sa_managedobject %s%s GROUP BY 1 ORDER BY 2 DESC""" % wr) # By Administrative Domain elif report_type == "domain": columns = [_("Administrative Domain")] query = ("""SELECT a.name,COUNT(*) FROM sa_managedobject o JOIN sa_administrativedomain a ON (o.administrative_domain_id=a.id) %s%s GROUP BY 1 ORDER BY 2 DESC""" % wr) # By Profile and Administrative Domains elif report_type == "domain-profile": columns = [_("Administrative Domain"), _("Profile")] query = ("""SELECT d.name,profile,COUNT(*) FROM sa_managedobject o JOIN sa_administrativedomain d ON (o.administrative_domain_id=d.id) %s%s GROUP BY 1,2 """ % wr) # By tags elif report_type == "tag": columns = [_("Tag")] query = (""" SELECT t.tag, COUNT(*) FROM ( SELECT unnest(tags) AS tag FROM sa_managedobject WHERE tags IS NOT NULL %s%s AND array_length(tags, 1) > 0 ) t GROUP BY 1 ORDER BY 2 DESC; """ % wr_and2) elif report_type == "platform": columns = [_("Platform"), _("Profile")] query = ("""select sam.profile, sam.platform, COUNT(platform) from sa_managedobject sam %s%s group by 1,2 order by count(platform) desc;""" % wr) elif report_type == "version": columns = [_("Profile"), _("Version")] query = ("""select sam.profile, sam.version, COUNT(version) from sa_managedobject sam %s%s group by 1,2 order by count(version) desc;""" % wr) else: raise Exception("Invalid report type: %s" % report_type) for r, t in report_types: if r == report_type: title = self.title + ": " + t break columns += [ TableColumn(_("Quantity"), align="right", total="sum", format="integer") ] cursor = self.cursor() cursor.execute(query, ()) data = [] for c in cursor.fetchall(): if report_type == "profile": data += [(profile.get(c[0]), c[1])] elif report_type == "domain-profile": data += [(c[0], profile.get(c[1]), c[2])] elif report_type == "platform": data += [(profile.get(c[0]), platform.get(c[1]), c[2])] elif report_type == "version": data += [(profile.get(c[0]), version.get(c[1]), c[2])] else: data += [c] return self.from_dataset(title=title, columns=columns, data=data, enumerate=True)
class ReportOutagesApplication(SimpleReport): title = _("Outages") form = ReportForm predefined_reports = { "1d": PredefinedReport(_("Outages (1 day)"), {"duration": 86400}), "7d": PredefinedReport(_("Outages (7 days)"), {"duration": 7 * 86400}), "30d": PredefinedReport(_("Outages (30 day)"), {"duration": 30 * 86400}), } def get_data(self, request, duration, from_date=None, to_date=None, **kwargs): now = datetime.datetime.now() if not from_date: duration = 1 if int(duration): self.logger.info("Use duration\n") d = datetime.timedelta(seconds=int(duration)) b = now - d q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False) else: b = datetime.datetime.strptime(from_date, "%d.%m.%Y") q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False) if to_date: if from_date == to_date: t1 = datetime.datetime.strptime( to_date, "%d.%m.%Y") + datetime.timedelta(1) else: t1 = datetime.datetime.strptime(to_date, "%d.%m.%Y") else: t1 = now q &= Q(start__lte=t1) | Q(stop__lte=t1) d = datetime.timedelta(seconds=int((t1 - b).total_seconds())) outages = defaultdict(list) otime = defaultdict(int) for o in Outage.objects.filter(q): start = max(o.start, b) stop = o.stop if o.stop else now outages[o.object] += [o] otime[o.object] += (stop - start).total_seconds() td = d.total_seconds() if not request.user.is_superuser: for mo in ManagedObject.objects.exclude( administrative_domain__in=UserAccess.get_domains( request.user)): if mo.id in otime: otime.pop(mo.id) # Load managed objects mos = list(otime) chunk = 500 mo = {} while mos: for o in ManagedObject.objects.filter(id__in=mos[:chunk]): mo[o.id] = o mos = mos[chunk:] r = [] for o in sorted(otime, key=lambda x: -otime[x]): m = mo.get(o) if not m: continue # Hanging Outage dt = min(td, otime[o]) downtime = "%02d:%02d:%02d" % ((dt // 3600) % 24, (dt // 60) % 60, dt % 60) if dt >= 86400: downtime = "%dd %s" % (dt // 86400, downtime) if td: avail = float(td - dt) * 100 / td else: avail = 0 r += [( m.name, m.address, m.profile.name, m.platform.name if m.platform else "", _("Yes") if m.is_managed else _("No"), _("Yes") if m.get_status() else _("No"), downtime, avail, len(outages[o]), )] return self.from_dataset( title=self.title, columns=[ _("Managed Object"), _("Address"), _("Profile"), _("Platform"), TableColumn(_("Managed"), align="right"), TableColumn(_("Status"), align="right"), TableColumn(_("Downtime"), align="right"), TableColumn(_("Availability"), align="right", format="percent"), TableColumn(_("Downs"), align="right", format="integer"), ], data=r, enumerate=True, )
class ReportFilterApplication(SimpleReport): title = _("Managed Object Profile Check Summary") predefined_reports = { "default": PredefinedReport(_("Managed Object Profile Check Summary"), {}) } save_perc = None def calc_percent(self, column, val): if column == _("Is Managed, object type defined"): if val == 0: return "%.2f %%" % 100 else: r = "%.2f %%" % ((val / float(self.save_perc)) * 100) self.save_perc = None return r elif column == _("Is Managed"): self.save_perc = val return "" def get_data(self, request, report_type=None, **kwargs): columns, columns_desr = [], [] sc_code = [ "1.1", "1.2", "1.2.1", "1.2.1.1", "1.2.2", "1.2.2.1", "1.2.2.2", "1.2.2.2.1", "1.2.2.2.2", "1.2.2.2.2.1", "1.2.3", ] r_map = [ (_("Not Managed"), "1is1"), (_("Is Managed"), "2is1"), # (_("Is Managed not monitoring"), "2is2"), # (_("Is monitoring"), "2is1"), (_("Is Managed, object type defined"), "2is1.6is1.3isp0"), (_("Is Managed, object type defined bad CLI Credential"), "2is1.6is1.3isp0.2isp1"), (_("Is Managed, object type undefined"), "2is1.6is1.3isp1"), (_("Is Managed, object type undefined not ping response"), "2is1.6is1.3isp1.3is1"), (_("Is Managed, object type undefined has ping response"), "2is1.6is1.3isp1.3is2"), ( _("Is Managed, object type undefined bad SNMP Credential"), "2is1.6is1.3isp1.3is2.1isp1", ), ( _("Is Managed, object type undefined for various reasons"), "2is1.6is1.3isp1.3is2.1isp0", ), (_("Is Managed, object type Profile is not know"), "2is1.6is1.9a1.3is2.4isp1"), (_("Is monitoring, object type undefined, only availablity check"), "2is1.6is2"), ] for x, y in r_map: columns += [y] columns_desr += [x] report = ReportModelFilter() result = report.proccessed(",".join(columns)) # result = stat.proccessed(",".join(columns)) # result = stat.api_result(",".join(columns)) summary = defaultdict(int) data = [] # url = "/sa/reportstat/repstat_download/?report=%s" url = "/sa/reportobjectdetail/download/?" + "&".join([ "o_format=xlsx", "columns=object_name,object_address,object_profile,object_status,profile_name,admin_domain,segment", "detail_stat=%s&pool=%s", ]) for p in Pool.objects.filter().order_by("name"): m = [] moss = set( ManagedObject.objects.filter(pool=p).values_list("id", flat=True)) for col in columns: m += [len(result[col.strip()].intersection(moss))] summary[col] += m[-1] data += [SectionRow(name=p.name)] data += [(sc, x, y, self.calc_percent(x, y), url % (columns[columns_desr.index(x)], p.name)) for sc, x, y in zip(sc_code, columns_desr, m)] data += [("1.2.2.2.2.2", _("Is Managed, objects not processed yet"), 0, "")] data += [SectionRow(name="Summary")] summary = [summary[k] for k in columns] data += [(sc, x, y, self.calc_percent(x, y), url % (columns[columns_desr.index(x)], "")) for sc, x, y in zip(sc_code, columns_desr, summary)] data += [("1.2.2.2.2.2", _("Is Managed, objects not processed yet"), 0, "")] # columns = ["ID", "Value", "Percent", TableColumn(_("Detail"), format="url")] columns = [ _("PP"), _("Status"), _("Quantity"), _("Percent"), TableColumn(_("Detail"), format="url"), ] return self.from_dataset(title=self.title, columns=columns, data=data)
class ReportFilterApplication(SimpleReport): title = _("Discovery Links Summary") predefined_reports = { "default": PredefinedReport(_("Discovery Links Summary"), {}) } def get_data(self, request, **kwargs): data = [] value = get_db()["noc.links"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate([{ "$unwind": "$interfaces" }, { "$lookup": { "from": "noc.interfaces", "localField": "interfaces", "foreignField": "_id", "as": "int" } }, { "$group": { "_id": "$int.managed_object", "count": { "$sum": 1 } } }]) count = {0: set([]), 1: set([]), 2: set([]), 3: set([])} ap = AuthProfile.objects.filter(name__startswith="TG") for v in value: if v["count"] > 2: count[3].add(v["_id"][0]) continue if not v["_id"]: self.logger.warning("No IDS in response query") continue count[v["count"]].add(v["_id"][0]) for p in Pool.objects.order_by("name"): if p.name == "P0001": continue data += [SectionRow(name=p.name)] smos = set( ManagedObject.objects.filter(pool=p, is_managed=True).exclude( profile=Profile.get_by_name(GENERIC_PROFILE)).exclude( auth_profile__in=ap).values_list('id', flat=True)) all_p = 100.0 / len(smos) if len(smos) else 1.0 data += [("All polling", len(smos))] for c in count: if c == 3: data += [ ("More 3", len(count[c].intersection(smos)), "%.2f %%" % round(len(count[c].intersection(smos)) * all_p, 2)) ] continue data += [(c, len(count[c].intersection(smos)), "%.2f %%" % round(len(count[c].intersection(smos)) * all_p), 2)] # 0 links - All discovering- summary with links s0 = len(smos) - sum([d[1] for d in data[-3:]]) data.pop(-4) data.insert(-3, (0, s0, "%.2f %%" % round(s0 * all_p, 2))) return self.from_dataset( title=self.title, columns=[_("Links count"), _("MO Count"), _("Percent at All")], data=data)
class ReportTTSystemStatApplication(SimpleReport): title = _("TT system statistics") form = ReportForm predefined_reports = { "1d": PredefinedReport(_("TTSystem Stat (1 day)"), {"interval": 1}), "7d": PredefinedReport(_("TTSystem Stat (7 days)"), {"interval": 7}), "30d": PredefinedReport(_("TTSystem Stat (30 day)"), {"interval": 30}), "1d_errors": PredefinedReport(_("TTSystem Errors (1 day)"), { "interval": 1, "repo_format": "1" }) } def get_data(self, request, interval=1, repo_format=0, from_date=None, to_date=None, **kwargs): # Date Time Block if from_date: from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y") elif interval: from_date = datetime.datetime.now() - datetime.timedelta( days=int(interval)) else: from_date = datetime.datetime.now() - datetime.timedelta(days=1) if to_date: to_date = datetime.datetime.strptime(to_date, "%d.%m.%Y") if from_date == to_date: to_date = from_date + datetime.timedelta(days=1) elif interval: to_date = from_date + datetime.timedelta(days=int(interval)) else: to_date = from_date + datetime.timedelta(days=1) columns = [ _("Server"), _("Service"), _("Request count"), _("Success request count"), _("Failed request count"), _("Success request (%)"), _("Q1 (ms)"), _("Q2 (ms)"), _("Q3 (ms)"), _("p95 (ms)"), _("max (ms)") ] if repo_format == "1": columns = [ _("Timestamp"), _("Server"), _("Service"), _("Managed object"), _("TT ID"), _("Error code"), _("Error text") ] ts_from_date = time.mktime(from_date.timetuple()) ts_to_date = time.mktime(to_date.timetuple()) tt_systems = TTSystem.objects.filter().scalar("name") # Manged Object block q1 = """select server, service, count(), round(quantile(0.25)(duration), 0)/1000 as q1, round(quantile(0.5)(duration), 0)/1000 as q2, round(quantile(0.75)(duration), 0)/1000 as q3, round(quantile(0.95)(duration),0)/1000 as p95, round(max(duration),0)/1000 as max from span where %s group by server, service""" q2 = """select server, service, error_code, count(), avg(duration) from span where %s group by server, service, error_code""" q3 = """select ts, server, service, in_label, in_label, error_code, error_text from span where service IN ('create_massive_damage_outer', 'change_massive_damage_outer_close') and error_code <> 0 and %s""" q_where = ["server IN ('%s')" % "', '".join(tt_systems)] # q_where = ["managed_object IN (%s)" % ", ".join(mo_bi_dict.keys())] q_where += [ "(date >= toDate(%d)) AND (ts >= toDateTime(%d) AND ts <= toDateTime(%d))" % (ts_from_date, ts_from_date, ts_to_date) ] r = [] ch = connection() if repo_format == "1": aa = { aa.escalation_tt.split(":")[-1]: aa for aa in ArchivedAlarm.objects.filter( clear_timestamp__gte=from_date, clear_timestamp__lte=to_date, escalation_tt__exists=True) } query = q3 % " and ".join(q_where) for row in ch.execute(query): if row[2] in ["create_massive_damage_outer"]: row[2] = u"Создание ТТ" try: row[3] = ManagedObject.objects.get( tt_system_id=int(row[3])) row[4] = "" except ManagedObject.DoesNotExist: pass except ManagedObject.MultipleObjectsReturned: row[3] = ManagedObject.objects.get(tt_system_id=int( row[3]), is_managed=True) row[4] = "" elif row[2] in ["change_massive_damage_outer_close"]: row[2] = u"Закрытие ТТ" row[4] = row[3] row[3] = aa[ row[3]].managed_object if row[3] in aa else row[3] else: continue r += [row] else: query = q1 % " and ".join(q_where) # (server, service) tt_s = {} for row in ch.execute(query): tt_s[(row[0], row[1])] = [row[2]] + [0, 0, 0] + row[3:] query = q2 % " and ".join(q_where) for row in ch.execute(query): if row[2] == "0": tt_s[(row[0], row[1])][1] = row[3] else: tt_s[(row[0], row[1])][2] += int(row[3]) r += [ SectionRow(name="Report from %s to %s" % (from_date.strftime("%d.%m.%Y %H:%M"), to_date.strftime("%d.%m.%Y %H:%M"))) ] for line in sorted(tt_s, key=lambda x: x[0]): data = list(line) data += tt_s[line] data[5] = round((float(data[3]) / float(data[2])) * 100.0, 2) r += [data] return self.from_dataset(title=self.title, columns=columns, data=r, enumerate=True)
class ReportAvailabilityApplication(SimpleReport): title = _("Availability") form = ReportForm predefined_reports = { "1d": PredefinedReport(_("Availability (last day)"), {"interval": 1}), "7d": PredefinedReport(_("Availability (last week)"), {"interval": 7}), "30d": PredefinedReport(_("Availability (last month)"), {"interval": 30}), } @staticmethod def get_availability_interval(days): now = datetime.datetime.now() d = datetime.timedelta(days=days) b = now - d outages = defaultdict(int) q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False) for o in Outage.objects.filter( q, read_preference=ReadPreference.SECONDARY_PREFERRED): start = max(o.start, b) stop = o.stop if o.stop else now outages[o.object] += (stop - start).total_seconds() td = d.total_seconds() # Normalize to percents return dict((o, (td - outages[o]) * 100.0 / td) for o in outages) @staticmethod def get_availability(start_date, stop_date, skip_zero_avail=False): # now = datetime.datetime.now() b = start_date d = stop_date outages = defaultdict(list) td = (d - b).total_seconds() # q = Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False) q = (Q(start__gte=b) | Q(stop__gte=b) | Q(stop__exists=False)) & Q(start__lt=d) for o in Outage.objects.filter(q): start = max(o.start, b) stop = o.stop if (o.stop and o.stop < d) else d if (stop - start).total_seconds() == td and skip_zero_avail: continue outages[o.object] += [(stop - start).total_seconds()] # Normalize to percents return dict((o, ((td - sum(outages[o])) * 100.0 / td, int(sum(outages[o])), len(outages[o]))) for o in outages) @staticmethod def get_reboots(start_date=None, stop_date=None): match = {"ts": {"$gte": start_date, "$lte": stop_date}} pipeline = [ { "$match": match }, { "$group": { "_id": "$object", "count": { "$sum": 1 } } }, { "$sort": { "count": -1 } }, ] data = (Reboot._get_collection().with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate( pipeline)) # data = data["result"] return dict((rb["_id"], rb["count"]) for rb in data) def get_data(self, request, interval=1, from_date=None, to_date=None, skip_avail=False, skip_zero_avail=False, filter_zero_access=False, **kwargs): """ a1 = self.get_availability(1) a7 = self.get_availability(7) a30 = self.get_availability(30) """ if not from_date: from_date = datetime.datetime.now() - datetime.timedelta( days=interval) else: from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y") if not to_date or from_date == to_date: to_date = from_date + datetime.timedelta(days=1) else: to_date = datetime.datetime.strptime( to_date, "%d.%m.%Y") + datetime.timedelta(days=1) a = self.get_availability(start_date=from_date, stop_date=to_date, skip_zero_avail=skip_zero_avail) rb = self.get_reboots(start_date=from_date, stop_date=to_date) r = [SectionRow("Report from %s to %s" % (from_date, to_date))] mos = ManagedObject.objects.filter(is_managed=True) if not request.user.is_superuser: mos = mos.filter( administrative_domain__in=UserAccess.get_domains(request.user)) if skip_avail: mos = mos.filter(id__in=list(a)) mos_id = list(mos.order_by("id").values_list("id", flat=True)) if filter_zero_access: iface_p = InterfaceProfile.objects.get(name="Клиентский порт") match = {"profile": iface_p.id, "managed_object": {"$in": mos_id}} pipeline = [ { "$match": match }, { "$group": { "_id": "$managed_object", "count": { "$sum": 1 }, "m": { "$max": "$oper_status" }, } }, { "$match": { "m": False } }, { "$project": { "_id": True } }, ] # data = Interface.objects._get_collection().aggregate(pipeline, data = (get_db()["noc.interfaces"].with_options( read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate( pipeline)) data = [d["_id"] for d in data] mos = mos.exclude(id__in=data) mo_hostname = ReportObjectsHostname1(sync_ids=mos_id) mo_hostname = mo_hostname.get_dictionary() for o in mos: s = [ o.administrative_domain.name, o.name, mo_hostname.get(o.id, ""), o.address, o.profile.name, round(a.get(o.id, (100.0, 0, 0))[0], 2), ] s.extend(a.get(o.id, (100.0, 0, 0))[1:]) s.append(rb[o.id] if o.id in rb else 0) r += [s] """ a1.get(o.id, 100), a7.get(o.id, 100), a30.get(o.id, 100) """ # print r return self.from_dataset( title=self.title, columns=[ _("Adm. Domain"), _("Managed Object"), _("Hostname"), _("Address"), _("Profile"), # TableColumn(_("Avail"), align="right", format="percent"), # TableColumn(_("Total avail (sec)"), align="right", format="numeric"), _("Avail"), _("Total unavail (sec)"), _("Count outages"), _("Reboots"), ], data=r, enumerate=True, )
class ReportFilterApplication(SimpleReport): title = _("Discovery Links Summary") predefined_reports = { "default": PredefinedReport(_("Discovery Links Summary"), {}) } save_perc = None def calc_percent(self, column, val): if column != _("All polling"): if val == 0: return "%.2f %%" % 100 else: r = "%.2f %%" % ((val / float(self.save_perc)) * 100) # self.save_perc = None return r elif column == _("All polling"): self.save_perc = val return "" def get_data(self, request, **kwargs): columns, columns_desr = [], [] r_map = [ (_("All polling"), "2is1.6is1.7a2"), # "Is Managed, object type defined" (_("0"), "2is1.6is1.7a2.3hs0"), # "Has 0 Links w type defined" (_("1"), "2is1.6is1.3hs2"), # "Has 1 links" (_("2"), "2is1.6is1.3hs3"), # "Has 2 links" (_("More 3"), "2is1.6is1.3hs4"), # "Has more 3 links" ] for x, y in r_map: columns += [y] columns_desr += [x] report = ReportModelFilter() result = report.proccessed(",".join(columns)) summary = defaultdict(int) data = [] # url = "/sa/reportstat/repstat_download/?report=%s" url = "/sa/reportobjectdetail/download/?" + "&".join([ "o_format=xlsx", "columns=object_name,object_address,object_profile,object_status,profile_name,admin_domain,segment", "detail_stat=%s&pool=%s", ]) for p in Pool.objects.filter().order_by("name"): m = [] moss = set( ManagedObject.objects.filter(pool=p).values_list("id", flat=True)) for col in columns: m += [len(result[col.strip()].intersection(moss))] summary[col] += m[-1] data += [SectionRow(name=p.name)] data += [(x, y, self.calc_percent(x, y), url % (columns[columns_desr.index(x)], p.name)) for x, y in zip(columns_desr, m)] return self.from_dataset( title=self.title, columns=[ _("Links count"), _("MO Count"), _("Percent at All"), TableColumn(_("Detail"), format="url"), ], data=data, )
class ReportFilterApplication(SimpleReport): title = _("Check Interface Facts") try: default_pool = Pool.objects.get(name="default") except Pool.DoesNotExist: default_pool = Pool.objects.all()[0] predefined_reports = { "default": PredefinedReport(_("Check Interface Facts (default)"), {"pool": default_pool}) } def get_form(self): class ReportForm(forms.Form): pool = forms.ChoiceField( label=_("Managed Objects Pool"), required=False, choices=list(Pool.objects.order_by("name").scalar("id", "name")) + [(None, "-" * 9)], ) int_profile = forms.ChoiceField( label=_("Interface Profile"), required=True, choices=list(InterfaceProfile.objects.order_by("name").scalar("id", "name")) + [(None, "-" * 9)], ) mop = forms.ModelChoiceField( label=_("Managed Object Profile"), required=False, queryset=ManagedObjectProfile.objects.order_by("name"), ) # int_fact = forms.ModelChoiceField( # label=_("Check Facts"), # required=False, # queryset=ManagedObjectProfile.objects.order_by("name")) return ReportForm def get_data(self, request, pool=None, int_profile=None, mop=None, avail_status=None, **kwargs): data = [] mos = ManagedObject.objects.filter(is_managed=True) # % fixme remove. if not pool and request.user.is_superuser: pool = Pool.get_by_name("STAGEMO") if pool: mos = mos.filter(pool=pool) if not request.user.is_superuser: mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user)) if mop: mos = mos.filter(object_profile=mop) mos_ids = mos.values_list("id", flat=True) iface = Interface.objects.filter( managed_object__in=mos, profile=int_profile, type="physical" ).values_list("managed_object", "name") res = [] n = 0 # Interface._get_collection() while mos_ids[(0 + n) : (10000 + n)]: mos_ids_f = mos_ids[(0 + n) : (10000 + n)] s_iface = {"%d.%s" % (mo.id, name) for mo, name in iface} of = ObjectFact.objects.filter( object__in=mos_ids_f, cls="subinterface", attrs__traffic_control_broadcast=False ) a_f = {".".join((str(o.object.id), o.attrs["name"])) for o in of} res.extend(a_f.intersection(s_iface)) n += 10000 for s in res: mo, iface = s.split(".") mo = ManagedObject.get_by_id(mo) data += [(mo.name, mo.address, mo.profile.name, iface)] return self.from_dataset( title=self.title, columns=[_("Managed Object"), _("Address"), _("SA Profile"), _("Interface")], data=data, )