def update_access(self):
from noc.sa.models.administrativedomain import AdministrativeDomain
# Get all own administrative domains
adm_domains = set(
d["administrative_domain"]
for d in self.managed_objects.values("administrative_domain")
.annotate(count=Count("id"))
.order_by("count")
)
p = set()
for a in adm_domains:
a = AdministrativeDomain.get_by_id(a)
p |= set(a.get_path())
adm_domains |= p
# Merge with children's administrative domains
for s in NetworkSegment.objects.filter(parent=self.id).only("adm_domains"):
adm_domains |= set(s.adm_domains or [])
# Check for changes
if set(self.adm_domains) != adm_domains:
self.adm_domains = sorted(adm_domains)
self.save()
# Propagate to parents
if self.parent:
self.parent.update_access()
def get_report_object(self,
user=None,
is_managed=None,
adm=None,
selector=None,
pool=None,
segment=None,
ids=None):
mos = ManagedObject.objects.filter()
if user.is_superuser and not adm and not selector and not segment:
mos = ManagedObject.objects.filter()
if ids:
mos = ManagedObject.objects.filter(id__in=[ids])
if is_managed is not None:
mos = ManagedObject.objects.filter(is_managed=is_managed)
if pool:
p = Pool.get_by_name(pool or "default")
mos = mos.filter(pool=p)
if not user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(user))
if adm:
ads = AdministrativeDomain.get_nested_ids(int(adm))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if segment:
segment = NetworkSegment.objects.filter(id=segment).first()
if segment:
mos = mos.filter(segment__in=segment.get_nested_ids())
return mos
def selectors_for_object(cls, object):
from noc.sa.models.administrativedomain import AdministrativeDomain
from noc.sa.models.managedobject import ManagedObject
from django.db import connection
selectors = cls.get_active_selectors()
if not selectors:
return set()
cls.ensure_models()
sql = []
params = []
for s in selectors:
if s.filter_id and object.id != s.filter_id:
continue
if s.filter_managed is not None and object.is_managed != s.filter_managed:
continue
if s.filter_profile and object.profile.id != s.filter_profile.id:
continue
if s.filter_pool and object.pool.id != s.filter_pool.id:
continue
if (s.filter_administrative_domain
and object.administrative_domain.id
not in AdministrativeDomain.get_nested_ids(
s.filter_administrative_domain.id)):
continue
if s.filter_name:
try:
if not re.search(s.filter_name, object.name):
continue
except re.error:
pass
q = (ManagedObject.objects.filter(
s.Q, id=object.id).extra(select={
"selector": s.id
}).values_list("selector"))
t, p = q.query.sql_with_params()
sql += [t.rsplit(" ORDER BY ", 1)[0]]
params += p
if not sql:
# Fully optimized, fully negative
return set()
sql = " UNION ALL ".join(sql)
sql = """
WITH sa_managedobject_item AS (
SELECT *
FROM sa_managedobject
WHERE id = %d
)
""" % object.id + sql.replace('"sa_managedobject"',
'"sa_managedobject_item"')
cursor = connection.cursor()
cursor.execute(sql, params)
return set(row[0] for row in cursor)
def cleaned_query(self, q):
if "administrative_domain" in q:
ad = AdministrativeDomain.get_nested_ids(int(q["administrative_domain"]))
if ad:
del q["administrative_domain"]
else:
ad = None
if "selector" in q:
s = self.get_object_or_404(ManagedObjectSelector, id=int(q["selector"]))
del q["selector"]
else:
s = None
r = super(ManagedObjectApplication, self).cleaned_query(q)
if s:
r["id__in"] = ManagedObject.objects.filter(s.Q)
if ad:
r["administrative_domain__in"] = ad
return r
def cleaned_query(self, q):
geoaddr = q.pop("__geoaddress", None)
if "administrative_domain" in q:
ad = AdministrativeDomain.get_nested_ids(
int(q["administrative_domain"]))
if ad:
del q["administrative_domain"]
else:
ad = None
if "selector" in q:
s = self.get_object_or_404(ManagedObjectSelector,
id=int(q["selector"]))
del q["selector"]
else:
s = None
r = super().cleaned_query(q)
if s:
r["id__in"] = ManagedObject.objects.filter(s.Q)
if ad:
r["administrative_domain__in"] = ad
if geoaddr:
scope, query = geoaddr.split(":", 1)
geocoder = geocoder_loader.get_class(scope)()
addr_ids = {r.id for r in geocoder.iter_recursive_query(query)}
addr_mo = set()
for o in Object.iter_by_address_id(list(addr_ids), scope):
addr_mo |= set(o.iter_managed_object_id())
# If ManagedObject has container refer to Object
addr_mo |= set(
ManagedObject.objects.filter(
container__in=o.get_nested_ids()).values_list(
"id", flat=True))
# Intersect with selector expression
if "id__in" in r:
addr_mo &= set(r["id__in"])
r["id__in"] = list(addr_mo)
return r
def api_report(
self,
request,
o_format,
administrative_domain=None,
selector=None,
interface_profile=None,
zero=None,
def_profile=None,
columns=None,
enable_autowidth=False,
):
def humanize_speed(speed):
if not speed:
return "-"
for t, n in [(1000000, "G"), (1000, "M"), (1, "k")]:
if speed >= t:
if speed // t * t == speed:
return "%d%s" % (speed // t, n)
else:
return "%.2f%s" % (float(speed) / t, n)
return str(speed)
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, unicode):
return v.encode("utf-8")
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
cols = [
"object_name",
"object_address",
"object_model",
"object_software",
"object_port_name",
"object_port_profile_name",
"object_port_status",
"object_link_status",
"object_port_speed",
"object_port_duplex",
"object_port_untagged_vlan",
"object_port_tagged_vlans",
]
header_row = [
"MANAGED_OBJECT",
"OBJECT_ADDRESS",
"OBJECT_MODEL",
"OBJECT_SOFTWARE",
"PORT_NAME",
"PORT_PROFILE_NAME",
"PORT_STATUS",
"LINK_STATUS",
"PORT_SPEED",
"PORT_DUPLEX",
"PORT_UNTAGGED_VLAN",
"PORT_TAGGED_VLANS",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
mo = {}
if_p = {}
DUPLEX = {True: "Full", False: "Half"}
for ifp in InterfaceProfile.objects.filter():
if_p[ifp.id] = {"name": ifp.name}
mos = ManagedObject.objects.filter(is_managed=True)
if (request.user.is_superuser and not administrative_domain
and not selector and not interface_profile):
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if administrative_domain:
ads = AdministrativeDomain.get_nested_ids(
int(administrative_domain))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
for o in mos:
mo[o.id] = {
"type": "managedobject",
"id": str(o.id),
"name": o.name,
"status": o.is_managed,
"address": o.address,
"vendor": o.vendor,
"version": o.version,
"platform": o.platform,
}
mos_id = list(mos.values_list("id", flat=True))
rld = ReportInterfaceStatus(mos_id, zero, def_profile,
interface_profile)
for i in rld.out:
untag, tagged = "", ""
if i["subs"]:
untag = i["subs"][0].get("untagged_vlan", "")
tagged = list_to_ranges(i["subs"][0].get("tagged_vlans", []))
r += [
translate_row(
row([
mo[i["managed_object"]]["name"],
mo[i["managed_object"]]["address"],
"%s %s" % (
str(mo[i["managed_object"]]["vendor"]),
str(mo[i["managed_object"]]["platform"]),
),
str(mo[i["managed_object"]]["version"]),
i["name"],
if_p[i["profile"]]["name"],
"UP" if i["admin_status"] is True else "Down",
"UP" if "oper_status" in i and i["oper_status"] is True
else "Down",
humanize_speed(i["in_speed"])
if "in_speed" in i else "-",
DUPLEX.get(i["full_duplex"])
if "full_duplex" in i and "in_speed" in i else "-",
untag,
tagged,
]),
cmap,
)
]
filename = "interface_status_report_%s" % datetime.datetime.now(
).strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response, dialect="excel", delimiter=";")
writer.writerows(r)
return response
elif o_format == "xlsx":
response = StringIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Objects")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
# response = HttpResponse(
# content_type="application/x-ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def cleaned_query(self, q):
nq = {}
for k in q:
if not k.startswith("_") and "__" not in k:
nq[k] = q[k]
ids = set()
self.logger.debug("Cleaned query: %s" % nq)
if "ids" in nq:
ids = {int(nid) for nid in nq["ids"]}
del nq["ids"]
if "administrative_domain" in nq:
ad = AdministrativeDomain.get_nested_ids(
int(nq["administrative_domain"]))
if ad:
del nq["administrative_domain"]
nq["administrative_domain__in"] = ad
if "selector" in nq:
s = self.get_object_or_404(ManagedObjectSelector,
id=int(q["selector"]))
if s:
if ids:
# nq["id__in"] = set(ManagedObject.objects.filter(s.Q).values_list("id", flat=True))
ids = ids.intersection(
set(
ManagedObject.objects.filter(s.Q).values_list(
"id", flat=True)))
else:
ids = set(
ManagedObject.objects.filter(s.Q).values_list(
"id", flat=True))
del nq["selector"]
mq = None
c_in = []
c_nin = []
for cc in [part for part in nq if part.startswith("caps")]:
"""
Caps: caps0=CapsID,caps1=CapsID:true....
cq - caps query
mq - main_query
caps0=CapsID - caps is exists
caps0=!CapsID - caps is not exists
caps0=CapsID:true - caps value equal True
caps0=CapsID:2~50 - caps value many then 2 and less then 50
c_ids = set(ObjectCapabilities.objects(cq).distinct('object'))
"""
# @todo Убирать дубликаты (повторно не добавлять)
c = nq.pop(cc)
if not c:
continue
if not mq:
mq = m_Q()
self.logger.info("Caps: %s" % c)
if "!" in c:
# @todo Добавить исключение (только этот) !ID
c_id = c[1:]
c_query = "nexists"
elif ":" not in c:
c_id = c
c_query = "exists"
else:
c_id, c_query = c.split(":", 1)
try:
c_id = bson.ObjectId(c_id)
except bson.errors.InvalidId as e:
self.logger.warning(e)
continue
if "~" in c_query:
l, r = c_query.split("~")
if not l:
cond = {"$lte": int(r)}
elif not r:
cond = {"$gte": int(l)}
else:
cond = {"$lte": int(r), "$gte": int(l)}
cq = m_Q(__raw__={
"caps": {
"$elemMatch": {
"capability": c_id,
"value": cond
}
}
})
elif c_query in ("false", "true"):
cq = m_Q(caps__match={
"capability": c_id,
"value": c_query == "true"
})
elif c_query == "exists":
c_in += [c_id]
continue
elif c_query == "nexists":
c_nin += [c_id]
continue
else:
try:
c_query = int(c_query)
cq = m_Q(
__raw__={
"caps": {
"$elemMatch": {
"capability": c_id,
"value": int(c_query)
}
}
})
except ValueError:
cq = m_Q(
__raw__={
"caps": {
"$elemMatch": {
"capability": c_id,
"value": {
"$regex": c_query
}
}
}
})
mq &= cq
if c_in:
mq &= m_Q(caps__capability__in=c_in)
if c_nin:
mq &= m_Q(caps__capability__nin=c_nin)
if mq:
c_ids = set(el["_id"] for el in ObjectCapabilities.objects(
mq).values_list("object").as_pymongo())
self.logger.info("Caps objects count: %d" % len(c_ids))
ids = ids.intersection(c_ids) if ids else c_ids
if "addresses" in nq:
if isinstance(nq["addresses"], list):
nq["address__in"] = nq["addresses"]
else:
nq["address__in"] = [nq["addresses"]]
del nq["addresses"]
if ids:
nq["id__in"] = list(ids)
xf = list((set(nq.keys())) -
set(f.name for f in self.model._meta.get_fields()))
# @todo move validation fields
for x in xf:
if x in ["address__in", "id__in", "administrative_domain__in"]:
continue
self.logger.warning("Remove element not in model: %s" % x)
del nq[x]
return nq
def api_report(
self,
request,
from_date,
to_date,
o_format,
administrative_domain=None,
columns=None,
source="both",
alarm_class=None,
enable_autowidth=False,
):
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, str):
return smart_text(v)
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return smart_text(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
cols = [
"id",
"alarm_class",
"alarm_from_ts",
"alarm_to_ts",
"alarm_tt",
"object_name",
"object_address",
"object_admdomain",
"log_timestamp",
"log_source",
"log_message",
# "tt",
# "escalation_ts",
]
header_row = [
"ID",
_("ALARM_CLASS"),
_("ALARM_FROM_TS"),
_("ALARM_TO_TS"),
_("ALARM_TT"),
_("OBJECT_NAME"),
_("OBJECT_ADDRESS"),
_("OBJECT_ADMDOMAIN"),
_("LOG_TIMESTAMP"),
_("LOG_SOURCE"),
_("LOG_MESSAGE"),
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
fd = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
match = {
"timestamp": {
"$gte": datetime.datetime.strptime(from_date, "%d.%m.%Y"),
"$lte": fd
}
}
mos = ManagedObject.objects.filter()
ads = []
if administrative_domain:
if administrative_domain.isdigit():
administrative_domain = [int(administrative_domain)]
ads = AdministrativeDomain.get_nested_ids(
administrative_domain[0])
if not request.user.is_superuser:
user_ads = UserAccess.get_domains(request.user)
if administrative_domain and ads:
if administrative_domain[0] not in user_ads:
ads = list(set(ads) & set(user_ads))
if not ads:
return HttpResponse(
"<html><body>Permission denied: Invalid Administrative Domain</html></body>"
)
else:
ads = user_ads
if ads:
mos = mos.filter(administrative_domain__in=ads)
# Working if Administrative domain set
if ads:
try:
match["adm_path"] = {"$in": ads}
# @todo More 2 level hierarhy
except bson.errors.InvalidId:
pass
addr_map = {
mo[0]: (mo[1], mo[2])
for mo in mos.values_list("id", "name", "address")
}
# Active Alarms
coll = ActiveAlarm._get_collection()
for aa in coll.aggregate([
{
"$match": match
},
{
"$unwind": "$log"
},
{
"$match": {
"log.source": {
"$exists": True,
"$ne": None
}
}
},
{
"$project": {
"timestamp": 1,
"managed_object": 1,
"alarm_class": 1,
"escalation_tt": 1,
"adm_path": 1,
"log": 1,
}
},
{
"$sort": {
"_id": 1,
"log.timestamp": 1
}
},
]):
r += [
translate_row(
row([
smart_text(aa["_id"]),
AlarmClass.get_by_id(aa["alarm_class"]).name,
aa["timestamp"],
"",
aa.get("escalation_tt", ""),
addr_map[aa["managed_object"]][0],
addr_map[aa["managed_object"]][1],
AdministrativeDomain.get_by_id(
aa["adm_path"][-1]).name,
aa["log"]["timestamp"],
aa["log"]["source"],
aa["log"]["message"],
]),
cmap,
)
]
# Active Alarms
coll = ArchivedAlarm._get_collection()
for aa in coll.aggregate([
{
"$match": match
},
{
"$unwind": "$log"
},
{
"$match": {
"log.source": {
"$exists": True
}
}
},
{
"$project": {
"timestamp": 1,
"clear_timestamp": 1,
"managed_object": 1,
"alarm_class": 1,
"escalation_tt": 1,
"adm_path": 1,
"log": 1,
}
},
{
"$sort": {
"_id": 1,
"log.timestamp": 1
}
},
]):
r += [
translate_row(
row([
smart_text(aa["_id"]),
AlarmClass.get_by_id(aa["alarm_class"]).name,
aa["timestamp"],
aa["clear_timestamp"],
aa.get("escalation_tt", ""),
addr_map[aa["managed_object"]][0],
addr_map[aa["managed_object"]][1],
AdministrativeDomain.get_by_id(
aa["adm_path"][-1]).name,
aa["log"]["timestamp"],
aa["log"]["source"],
aa["log"]["message"],
]),
cmap,
)
]
filename = "alarm_comments.csv"
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s"' % filename
writer = csv.writer(response)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerow(columns)
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr(filename, f.read())
zf.filename = "%s.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="alarm_comments.xlsx"'
response.close()
return response
def api_report(
self,
request,
from_date,
to_date,
o_format,
min_duration=0,
max_duration=0,
min_objects=0,
min_subscribers=0,
segment=None,
administrative_domain=None,
selector=None,
ex_selector=None,
columns=None,
source="both",
alarm_class=None,
subscribers=None,
enable_autowidth=False,
):
def row(row, container_path, segment_path):
def qe(v):
if v is None:
return ""
if isinstance(v, str):
return smart_text(v)
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return smart_text(v)
else:
return v
r = [qe(x) for x in row]
if len(container_path) < self.CONTAINER_PATH_DEPTH:
container_path += [""] * (self.CONTAINER_PATH_DEPTH -
len(container_path))
else:
container_path = container_path[:self.CONTAINER_PATH_DEPTH]
if len(segment_path) < self.SEGMENT_PATH_DEPTH:
segment_path += [""] * (self.SEGMENT_PATH_DEPTH -
len(segment_path))
else:
segment_path = segment_path[:self.SEGMENT_PATH_DEPTH]
return r + container_path + segment_path
def translate_row(row, cmap):
return [row[i] for i in cmap]
cols = ([
"id",
"root_id",
"from_ts",
"to_ts",
"duration_sec",
"object_name",
"object_address",
"object_hostname",
"object_profile",
"object_admdomain",
"object_platform",
"object_version",
"alarm_class",
"alarm_subject",
"maintenance",
"objects",
"subscribers",
"tt",
"escalation_ts",
"location",
"container_address",
] + ["container_%d" % i for i in range(self.CONTAINER_PATH_DEPTH)] +
["segment_%d" % i for i in range(self.SEGMENT_PATH_DEPTH)])
header_row = (
[
"ID",
_("ROOT_ID"),
_("FROM_TS"),
_("TO_TS"),
_("DURATION_SEC"),
_("OBJECT_NAME"),
_("OBJECT_ADDRESS"),
_("OBJECT_HOSTNAME"),
_("OBJECT_PROFILE"),
_("OBJECT_ADMDOMAIN"),
_("OBJECT_PLATFORM"),
_("OBJECT_VERSION"),
_("ALARM_CLASS"),
_("ALARM_SUBJECT"),
_("MAINTENANCE"),
_("OBJECTS"),
_("SUBSCRIBERS"),
_("TT"),
_("ESCALATION_TS"),
_("LOCATION"),
_("CONTAINER_ADDRESS"),
] + ["CONTAINER_%d" % i
for i in range(self.CONTAINER_PATH_DEPTH)] +
["SEGMENT_%d" % i for i in range(self.SEGMENT_PATH_DEPTH)])
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
subscribers_profile = self.default_subscribers_profile
if subscribers:
subscribers_profile = set(
SubscriberProfile.objects.filter(
id__in=subscribers.split(",")).scalar("id"))
r = [translate_row(header_row, cmap)]
fd = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
match = {
"timestamp": {
"$gte": datetime.datetime.strptime(from_date, "%d.%m.%Y"),
"$lte": fd
}
}
match_duration = {"duration": {"$gte": min_duration}}
if max_duration:
match_duration = {
"duration": {
"$gte": min_duration,
"$lte": max_duration
}
}
mos = ManagedObject.objects.filter(is_managed=True)
if segment:
try:
match["segment_path"] = bson.ObjectId(segment)
except bson.errors.InvalidId:
pass
ads = []
if administrative_domain:
if administrative_domain.isdigit():
administrative_domain = [int(administrative_domain)]
ads = AdministrativeDomain.get_nested_ids(
administrative_domain[0])
if not request.user.is_superuser:
user_ads = UserAccess.get_domains(request.user)
if administrative_domain and ads:
if administrative_domain[0] not in user_ads:
ads = list(set(ads) & set(user_ads))
if not ads:
return HttpResponse(
"<html><body>Permission denied: Invalid Administrative Domain</html></body>"
)
else:
ads = user_ads
if ads:
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if ex_selector:
ex_selector = ManagedObjectSelector.get_by_id(int(ex_selector))
mos = mos.exclude(ex_selector.Q)
# Working if Administrative domain set
if ads:
try:
match["adm_path"] = {"$in": ads}
# @todo More 2 level hierarhy
except bson.errors.InvalidId:
pass
mos_id = list(mos.order_by("id").values_list("id", flat=True))
mo_hostname = {}
maintenance = []
if mos_id and (selector or ex_selector):
match["managed_object"] = {"$in": mos_id}
if "maintenance" in columns.split(","):
maintenance = Maintenance.currently_affected()
if "object_hostname" in columns.split(","):
mo_hostname = ReportObjectsHostname1(sync_ids=mos_id)
mo_hostname = mo_hostname.get_dictionary()
moss = ReportAlarmObjects(mos_id).get_all()
# container_lookup = ReportContainer(mos_id)
container_lookup = None
subject = "alarm_subject" in columns
loc = AlarmApplication([])
if source in ["archive", "both"]:
# Archived Alarms
for a in (ArchivedAlarm._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED
).aggregate([
{
"$match": match
},
{
"$addFields": {
"duration": {
"$divide": [
{
"$subtract":
["$clear_timestamp", "$timestamp"]
},
1000,
]
}
}
},
{
"$match": match_duration
},
# {"$sort": {"timestamp": 1}}
])):
if int(a["managed_object"]) not in moss:
continue
dt = a["clear_timestamp"] - a["timestamp"]
duration = int(dt.total_seconds())
total_objects = sum(ss["summary"] for ss in a["total_objects"])
if min_objects and total_objects < min_objects:
continue
total_subscribers = sum(
ss["summary"] for ss in a["total_subscribers"]
if subscribers_profile
and ss["profile"] in subscribers_profile)
if min_subscribers and total_subscribers < min_subscribers:
continue
if "segment_" in columns.split(
",") or "container_" in columns.split(","):
path = ObjectPath.get_path(a["managed_object"])
if path:
segment_path = [
NetworkSegment.get_by_id(s).name
for s in path.segment_path
if NetworkSegment.get_by_id(s)
]
container_path = [
Object.get_by_id(s).name
for s in path.container_path if Object.get_by_id(s)
]
else:
segment_path = []
container_path = []
else:
segment_path = []
container_path = []
r += [
translate_row(
row(
[
smart_text(a["_id"]),
smart_text(a["root"]) if a.get("root") else "",
a["timestamp"],
a["clear_timestamp"],
smart_text(duration),
moss[a["managed_object"]][0],
moss[a["managed_object"]][1],
smart_text(
mo_hostname.get(a["managed_object"], "")),
Profile.get_by_id(
moss[a["managed_object"]][3]).name
if moss[a["managed_object"]][5] else "",
moss[a["managed_object"]][6],
Platform.get_by_id(
moss[a["managed_object"]][9])
if moss[a["managed_object"]][9] else "",
smart_text(
Firmware.get_by_id(
moss[a["managed_object"]][10]).version)
if moss[a["managed_object"]][10] else "",
AlarmClass.get_by_id(a["alarm_class"]).name,
ArchivedAlarm.objects.get(
id=a["_id"]).subject if subject else "",
"",
total_objects,
total_subscribers,
a.get("escalation_tt"),
a.get("escalation_ts"),
", ".join(ll for ll in (
loc.location(moss[a["managed_object"]][5]
) if moss[a["managed_object"]]
[5] is not None else "") if ll),
container_lookup[a["managed_object"]].get(
"text", "") if container_lookup else "",
],
container_path,
segment_path,
),
cmap,
)
]
# Active Alarms
if source in ["active", "both"]:
datenow = datetime.datetime.now()
for a in (ActiveAlarm._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).
aggregate([
{
"$match": match
},
{
"$addFields": {
"duration": {
"$divide": [{
"$subtract": [fd, "$timestamp"]
}, 1000]
}
}
},
{
"$match": match_duration
},
# {"$sort": {"timestamp": 1}}
])):
dt = datenow - a["timestamp"]
duration = int(dt.total_seconds())
total_objects = sum(ss["summary"] for ss in a["total_objects"])
if min_objects and total_objects < min_objects:
continue
total_subscribers = sum(
ss["summary"] for ss in a["total_subscribers"]
if subscribers_profile
and ss["profile"] in subscribers_profile)
if min_subscribers and total_subscribers < min_subscribers:
continue
if "segment_" in columns.split(
",") or "container_" in columns.split(","):
path = ObjectPath.get_path(a["managed_object"])
if path:
segment_path = [
NetworkSegment.get_by_id(s).name
for s in path.segment_path
if NetworkSegment.get_by_id(s)
]
container_path = [
Object.get_by_id(s).name
for s in path.container_path if Object.get_by_id(s)
]
else:
segment_path = []
container_path = []
else:
segment_path = []
container_path = []
r += [
translate_row(
row(
[
smart_text(a["_id"]),
smart_text(a["root"]) if a.get("root") else "",
a["timestamp"],
# a["clear_timestamp"],
"",
smart_text(duration),
moss[a["managed_object"]][0],
moss[a["managed_object"]][1],
smart_text(
mo_hostname.get(a["managed_object"], "")),
Profile.get_by_id(moss[a["managed_object"]][3])
if moss[a["managed_object"]][5] else "",
moss[a["managed_object"]][6],
Platform.get_by_id(
moss[a["managed_object"]][9])
if moss[a["managed_object"]][9] else "",
smart_text(
Firmware.get_by_id(
moss[a["managed_object"]][10]).version)
if moss[a["managed_object"]][10] else "",
AlarmClass.get_by_id(a["alarm_class"]).name,
ActiveAlarm.objects.get(
id=a["_id"]).subject if subject else None,
"Yes" if a["managed_object"] in maintenance
else "No",
total_objects,
total_subscribers,
a.get("escalation_tt"),
a.get("escalation_ts"),
", ".join(ll for ll in (
loc.location(moss[a["managed_object"]][5]
) if moss[a["managed_object"]]
[5] is not None else "") if ll),
container_lookup[a["managed_object"]].get(
"text", "") if container_lookup else "",
],
container_path,
segment_path,
),
cmap,
)
]
if source in ["long_archive"]:
o_format = "csv_zip"
columns = [
"ALARM_ID",
"MO_ID",
"OBJECT_PROFILE",
"VENDOR",
"PLATFORM",
"VERSION",
"OPEN_TIMESTAMP",
"CLOSE_TIMESTAMP",
"LOCATION",
"",
"POOL",
"ADM_DOMAIN",
"MO_NAME",
"IP",
"ESCALATION_TT",
"DURATION",
"SEVERITY",
"REBOOTS",
]
from noc.core.clickhouse.connect import connection
ch = connection()
fd = datetime.datetime.strptime(from_date, "%d.%m.%Y")
td = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
if td - fd > datetime.timedelta(days=390):
return HttpResponseBadRequest(
_("Report more than 1 year not allowed. If nedeed - request it from Administrator"
))
ac = AlarmClass.objects.get(
name="NOC | Managed Object | Ping Failed")
subs = ", ".join(
"subscribers.summary[indexOf(subscribers.profile, '%s')] as `%s`"
% (sp.bi_id, sp.name)
for sp in SubscriberProfile.objects.filter().order_by("name"))
if subs:
columns += [
sp.name for sp in
SubscriberProfile.objects.filter().order_by("name")
]
r = ch.execute(LONG_ARCHIVE_QUERY % (
", %s" % subs if subs else "",
fd.date().isoformat(),
td.date().isoformat(),
ac.bi_id,
))
filename = "alarms.csv"
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s"' % filename
writer = csv.writer(response)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerow(columns)
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr(filename, f.read())
zf.filename = "%s.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="alarms.xlsx"'
response.close()
return response
def api_report(
self,
request,
reporttype=None,
from_date=None,
to_date=None,
object_profile=None,
filter_default=None,
exclude_zero=True,
interface_profile=None,
selector=None,
administrative_domain=None,
columns=None,
description=None,
o_format=None,
enable_autowidth=False,
**kwargs,
):
def load(mo_ids):
# match = {"links.mo": {"$in": mo_ids}}
match = {"int.managed_object": {"$in": mo_ids}}
group = {
"_id": "$_id",
"links": {
"$push": {
"iface_n": "$int.name",
# "iface_id": "$int._id",
# "iface_descr": "$int.description",
# "iface_speed": "$int.in_speed",
# "dis_method": "$discovery_method",
# "last_seen": "$last_seen",
"mo": "$int.managed_object",
"linked_obj": "$linked_objects",
}
},
}
value = (get_db()["noc.links"].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).aggregate(
[
{
"$unwind": "$interfaces"
},
{
"$lookup": {
"from": "noc.interfaces",
"localField": "interfaces",
"foreignField": "_id",
"as": "int",
}
},
{
"$match": match
},
{
"$group": group
},
],
allowDiskUse=True,
))
res = defaultdict(dict)
for v in value:
if v["_id"]:
for vv in v["links"]:
if len(vv["linked_obj"]) == 2:
mo = vv["mo"][0]
iface = vv["iface_n"]
for i in vv["linked_obj"]:
if mo != i:
res[mo][i] = iface[0]
return res
def translate_row(row, cmap):
return [row[i] for i in cmap]
def str_to_float(str):
return float("{0:.3f}".format(float(str)))
cols = [
"object_id",
"object_name",
"object_address",
"object_platform",
"object_adm_domain",
"object_segment",
"object_container",
# "object_hostname",
# "object_status",
# "profile_name",
# "object_profile",
# "object_vendor",
"iface_name",
"iface_description",
"iface_speed",
"max_load_in",
"max_load_in_time",
"max_load_out",
"max_load_out_time",
"avg_load_in",
"avg_load_out",
"total_in",
"total_out",
"uplink_iface_name",
"uplink_iface_description",
"uplink_iface_speed",
"uplink_max_load_in",
"uplink_max_load_in_time",
"uplink_max_load_out",
"uplink_max_load_out_time",
"uplink_avg_load_in",
"uplink_avg_load_out",
"uplink_total_in",
"uplink_total_out",
]
header_row = [
"ID",
_("OBJECT_NAME"),
_("OBJECT_ADDRESS"),
_("OBJECT_PLATFORM"),
_("OBJECT_ADMDOMAIN"),
_("OBJECT_SEGMENT"),
_("CONTAINER_ADDRESS"),
_("IFACE_NAME"),
_("IFACE_DESCRIPTION"),
_("IFACE_SPEED"),
_("MAX_LOAD_IN, Mbps"),
_("MAX_LOAD_IN_TIME"),
_("MAX_LOAD_OUT, Mbps"),
_("MAX_LOAD_OUT_TIME"),
_("AVG_LOAD_IN, Mbps"),
_("AVG_LOAD_OUT, Mbps"),
_("TOTAL_IN, Mbyte"),
_("TOTAL_OUT, Mbyte"),
_("UPLINK_IFACE_NAME"),
_("UPLINK_IFACE_DESCRIPTION"),
_("UPLINK_IFACE_SPEED"),
_("UPLINK_MAX_LOAD_IN, Mbps"),
_("UPLINK_MAX_TIME_IN"),
_("UPLINK_MAX_LOAD_OUT, Mbps"),
_("UPLINK_MAX_TIME_OUT"),
_("UPLINK_AVG_LOAD_IN, Mbps"),
_("UPLINK_AVG_LOAD_OUT, Mbps"),
_("UPLINK_TOTAL_IN, Mbyte"),
_("UPLINK_TOTAL_OUT, Mbyte"),
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
columns_order = columns.split(",")
columns_filter = set(columns_order)
r = [translate_row(header_row, cmap)]
# Date Time Block
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(days=1)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
diff = to_date - from_date
# Load managed objects
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if selector:
mos = mos.filter(
ManagedObjectSelector.objects.get(id=int(selector)).Q)
if administrative_domain:
mos = mos.filter(
administrative_domain__in=AdministrativeDomain.get_nested_ids(
int(administrative_domain)))
if object_profile:
mos = mos.filter(object_profile=object_profile)
if interface_profile:
interface_profile = InterfaceProfile.objects.filter(
id=interface_profile).first()
mo_attrs = namedtuple("MOATTRs",
[c for c in cols if c.startswith("object")])
containers_address = {}
if "object_container" in columns_filter:
containers_address = ReportContainerData(
set(mos.values_list("id", flat=True)))
containers_address = dict(list(containers_address.extract()))
moss = {}
for row in mos.values_list("bi_id", "name", "address", "platform",
"administrative_domain__name", "segment",
"id"):
moss[row[0]] = mo_attrs(*[
row[6],
row[1],
row[2],
smart_text(Platform.get_by_id(row[3]) if row[3] else ""),
row[4],
smart_text(NetworkSegment.get_by_id(row[5])) if row[5] else "",
containers_address.
get(row[6], "") if containers_address and row[6] else "",
])
report_metric = ReportInterfaceMetrics(tuple(sorted(moss)),
from_date,
to_date,
columns=None)
report_metric.SELECT_QUERY_MAP = {
(0, "managed_object", "id"): "managed_object",
(1, "path", "iface_name"): "arrayStringConcat(path)",
(
2,
"",
"iface_description",
):
"dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))",
(
3,
"",
"profile",
):
"dictGetString('interfaceattributes', 'profile', (managed_object, arrayStringConcat(path)))",
(
4,
"speed",
"iface_speed",
):
"dictGetUInt64('interfaceattributes', 'in_speed', (managed_object, arrayStringConcat(path)))",
(5, "load_in_max", "load_in_max"): "divide(max(load_in),1048576)",
(6, "load_out_max", "load_out_max"):
"divide(max(load_out),1048576)",
(7, "max_load_in_time", "max_load_in_time"): "argMax(ts,load_in)",
(8, "max_load_out_time", "max_load_out_time"):
"argMax(ts,load_out)",
(9, "avg_load_in", "avg_load_in"): "divide(avg(load_in),1048576)",
(10, "avg_load_out", "avg_load_out"):
"divide(avg(load_out),1048576)",
}
ifaces_metrics = defaultdict(dict)
for row in report_metric.do_query():
avg_in = str_to_float(row[9])
avg_out = str_to_float(row[10])
total_in = avg_in * diff.total_seconds() / 8
total_out = avg_out * diff.total_seconds() / 8
ifaces_metrics[row[0]][row[1]] = {
"description": row[2],
"profile": row[3],
"bandwidth": row[4],
"max_load_in": str_to_float(row[5]),
"max_load_out": str_to_float(row[6]),
"max_load_in_time": row[7],
"max_load_out_time": row[8],
"avg_load_in": avg_in,
"avg_load_out": avg_out,
"total_in": float("{0:.1f}".format(total_in)),
"total_out": float("{0:.1f}".format(total_out)),
}
# find uplinks
links = {}
if cmap[-1] > 17:
mos_id = list(mos.values_list("id", flat=True))
uplinks = {obj: [] for obj in mos_id}
for d in ObjectData._get_collection().find(
{"_id": {
"$in": mos_id
}}, {
"_id": 1,
"uplinks": 1
}):
uplinks[d["_id"]] = d.get("uplinks", [])
rld = load(mos_id)
for mo in uplinks:
for uplink in uplinks[mo]:
if rld[mo]:
if mo in links:
links[mo] += [rld[mo][uplink]]
else:
links[mo] = [rld[mo][uplink]]
for mo_bi in ifaces_metrics:
mo_id = moss[int(mo_bi)]
mo_ids = getattr(mo_id, "object_id")
for i in ifaces_metrics[mo_bi]:
if not exclude_zero:
if (ifaces_metrics[mo_bi][i]["max_load_in"] == 0
and ifaces_metrics[mo_bi][i]["max_load_out"] == 0):
continue
if description:
if description not in ifaces_metrics[mo_bi][i][
"description"]:
continue
if interface_profile:
if interface_profile.name not in ifaces_metrics[mo_bi][i][
"profile"]:
continue
row2 = [
mo_ids,
getattr(mo_id, "object_name"),
getattr(mo_id, "object_address"),
getattr(mo_id, "object_platform"),
getattr(mo_id, "object_adm_domain"),
getattr(mo_id, "object_segment"),
getattr(mo_id, "object_container"),
i,
ifaces_metrics[mo_bi][i]["description"],
ifaces_metrics[mo_bi][i]["bandwidth"],
ifaces_metrics[mo_bi][i]["max_load_in"],
ifaces_metrics[mo_bi][i]["max_load_in_time"],
ifaces_metrics[mo_bi][i]["max_load_out"],
ifaces_metrics[mo_bi][i]["max_load_out_time"],
ifaces_metrics[mo_bi][i]["avg_load_in"],
ifaces_metrics[mo_bi][i]["avg_load_out"],
ifaces_metrics[mo_bi][i]["total_in"],
ifaces_metrics[mo_bi][i]["total_out"],
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
"",
]
ss = True
if mo_ids in links:
for ifname_uplink in links[mo_ids]:
if ifname_uplink in ifaces_metrics[mo_bi]:
row2[18] = ifname_uplink
row2[19] = ifaces_metrics[mo_bi][ifname_uplink][
"description"]
row2[20] = ifaces_metrics[mo_bi][ifname_uplink][
"bandwidth"]
row2[21] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_in"]
row2[22] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_in_time"]
row2[23] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_out"]
row2[24] = ifaces_metrics[mo_bi][ifname_uplink][
"max_load_out_time"]
row2[25] = ifaces_metrics[mo_bi][ifname_uplink][
"avg_load_in"]
row2[26] = ifaces_metrics[mo_bi][ifname_uplink][
"avg_load_out"]
row2[27] = ifaces_metrics[mo_bi][ifname_uplink][
"total_in"]
row2[28] = ifaces_metrics[mo_bi][ifname_uplink][
"total_out"]
r += [translate_row(row2, cmap)]
ss = False
if ss:
r += [translate_row(row2, cmap)]
filename = "metrics_detail_report_%s" % datetime.datetime.now(
).strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response,
dialect="excel",
delimiter=",",
quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr("%s.csv" % filename, f.read())
zf.filename = "%s.csv.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Metrics")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def api_report(
self,
request,
from_date,
to_date,
o_format,
min_duration=0,
max_duration=0,
min_objects=0,
min_subscribers=0,
segment=None,
administrative_domain=None,
selector=None,
ex_selector=None,
columns=None,
source="both",
alarm_class=None,
subscribers=None,
enable_autowidth=False,
):
def row(row, container_path, segment_path):
def qe(v):
if v is None:
return ""
if isinstance(v, unicode):
return v.encode("utf-8")
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
r = [qe(x) for x in row]
if len(container_path) < self.CONTAINER_PATH_DEPTH:
container_path += [""] * (self.CONTAINER_PATH_DEPTH -
len(container_path))
else:
container_path = container_path[:self.CONTAINER_PATH_DEPTH]
if len(segment_path) < self.SEGMENT_PATH_DEPTH:
segment_path += [""] * (self.SEGMENT_PATH_DEPTH -
len(segment_path))
else:
segment_path = segment_path[:self.SEGMENT_PATH_DEPTH]
return r + container_path + segment_path
def translate_row(row, cmap):
return [row[i] for i in cmap]
cols = ([
"id",
"root_id",
"from_ts",
"to_ts",
"duration_sec",
"object_name",
"object_address",
"object_hostname",
"object_profile",
"object_admdomain",
"object_platform",
"object_version",
"alarm_class",
"alarm_subject",
"maintenance",
"objects",
"subscribers",
"tt",
"escalation_ts",
"location",
"container_address",
] + ["container_%d" % i for i in range(self.CONTAINER_PATH_DEPTH)] +
["segment_%d" % i for i in range(self.SEGMENT_PATH_DEPTH)])
header_row = (
[
"ID",
_("ROOT_ID"),
_("FROM_TS"),
_("TO_TS"),
_("DURATION_SEC"),
_("OBJECT_NAME"),
_("OBJECT_ADDRESS"),
_("OBJECT_HOSTNAME"),
_("OBJECT_PROFILE"),
_("OBJECT_ADMDOMAIN"),
_("OBJECT_PLATFORM"),
_("OBJECT_VERSION"),
_("ALARM_CLASS"),
_("ALARM_SUBJECT"),
_("MAINTENANCE"),
_("OBJECTS"),
_("SUBSCRIBERS"),
_("TT"),
_("ESCALATION_TS"),
_("LOCATION"),
_("CONTAINER_ADDRESS"),
] + ["CONTAINER_%d" % i
for i in range(self.CONTAINER_PATH_DEPTH)] +
["SEGMENT_%d" % i for i in range(self.SEGMENT_PATH_DEPTH)])
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
subscribers_profile = self.default_subscribers_profile
if subscribers:
subscribers_profile = set(
SubscriberProfile.objects.filter(
id__in=subscribers.split(",")).scalar("id"))
r = [translate_row(header_row, cmap)]
fd = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
match = {
"timestamp": {
"$gte": datetime.datetime.strptime(from_date, "%d.%m.%Y"),
"$lte": fd
}
}
match_duration = {"duration": {"$gte": min_duration}}
if max_duration:
match_duration = {
"duration": {
"$gte": min_duration,
"$lte": max_duration
}
}
mos = ManagedObject.objects.filter(is_managed=True)
if segment:
try:
match["segment_path"] = bson.ObjectId(segment)
except bson.errors.InvalidId:
pass
ads = []
if administrative_domain:
if administrative_domain.isdigit():
administrative_domain = [int(administrative_domain)]
ads = AdministrativeDomain.get_nested_ids(
administrative_domain[0])
if not request.user.is_superuser:
user_ads = UserAccess.get_domains(request.user)
if administrative_domain and ads:
if administrative_domain[0] not in user_ads:
ads = list(set(ads) & set(user_ads))
else:
ads = administrative_domain
else:
ads = user_ads
if ads:
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if ex_selector:
ex_selector = ManagedObjectSelector.get_by_id(int(ex_selector))
mos = mos.exclude(ex_selector.Q)
# Working if Administrative domain set
if ads:
try:
match["adm_path"] = {"$in": ads}
# @todo More 2 level hierarhy
except bson.errors.InvalidId:
pass
mos_id = list(mos.order_by("id").values_list("id", flat=True))
mo_hostname = {}
maintenance = []
if mos_id and (selector or ex_selector):
match["managed_object"] = {"$in": mos_id}
if "maintenance" in columns.split(","):
maintenance = Maintenance.currently_affected()
if "object_hostname" in columns.split(","):
mo_hostname = ReportObjectsHostname1(sync_ids=mos_id)
mo_hostname = mo_hostname.get_dictionary()
moss = ReportAlarmObjects(mos_id).get_all()
# container_lookup = ReportContainer(mos_id)
container_lookup = None
subject = "alarm_subject" in columns
loc = AlarmApplication([])
if source in ["archive", "both"]:
# Archived Alarms
for a in (ArchivedAlarm._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED
).aggregate([
{
"$match": match
},
{
"$addFields": {
"duration": {
"$divide": [
{
"$subtract":
["$clear_timestamp", "$timestamp"]
},
1000,
]
}
}
},
{
"$match": match_duration
},
# {"$sort": {"timestamp": 1}}
])):
if int(a["managed_object"]) not in moss:
continue
dt = a["clear_timestamp"] - a["timestamp"]
duration = int(dt.total_seconds())
total_objects = sum(ss["summary"] for ss in a["total_objects"])
if min_objects and total_objects < min_objects:
continue
total_subscribers = sum(
ss["summary"] for ss in a["total_subscribers"]
if subscribers_profile
and ss["profile"] in subscribers_profile)
if min_subscribers and total_subscribers < min_subscribers:
continue
if "segment_" in columns.split(
",") or "container_" in columns.split(","):
path = ObjectPath.get_path(a["managed_object"])
if path:
segment_path = [
NetworkSegment.get_by_id(s).name
for s in path.segment_path
if NetworkSegment.get_by_id(s)
]
container_path = [
Object.get_by_id(s).name
for s in path.container_path if Object.get_by_id(s)
]
else:
segment_path = []
container_path = []
else:
segment_path = []
container_path = []
r += [
translate_row(
row(
[
str(a["_id"]),
str(a["root"]) if a.get("root") else "",
a["timestamp"],
a["clear_timestamp"],
str(duration),
moss[a["managed_object"]][0],
moss[a["managed_object"]][1],
mo_hostname.get(a["managed_object"], ""),
Profile.get_by_id(
moss[a["managed_object"]][3]).name
if moss[a["managed_object"]][5] else "",
moss[a["managed_object"]][6],
Platform.get_by_id(
moss[a["managed_object"]][9])
if moss[a["managed_object"]][9] else "",
Firmware.get_by_id(
moss[a["managed_object"]][10])
if moss[a["managed_object"]][10] else "",
AlarmClass.get_by_id(a["alarm_class"]).name,
ArchivedAlarm.objects.get(
id=a["_id"]).subject if subject else "",
"",
total_objects,
total_subscribers,
a.get("escalation_tt"),
a.get("escalation_ts"),
", ".join(l for l in (
loc.location(moss[a["managed_object"]][5]
) if moss[a["managed_object"]]
[5] is not None else "") if l),
container_lookup[a["managed_object"]].get(
"text", "") if container_lookup else "",
],
container_path,
segment_path,
),
cmap,
)
]
# Active Alarms
if source in ["active", "both"]:
for a in (ActiveAlarm._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED).
aggregate([
{
"$match": match
},
{
"$addFields": {
"duration": {
"$divide": [{
"$subtract": [fd, "$timestamp"]
}, 1000]
}
}
},
{
"$match": match_duration
},
# {"$sort": {"timestamp": 1}}
])):
dt = fd - a["timestamp"]
duration = int(dt.total_seconds())
total_objects = sum(ss["summary"] for ss in a["total_objects"])
if min_objects and total_objects < min_objects:
continue
total_subscribers = sum(
ss["summary"] for ss in a["total_subscribers"]
if subscribers_profile
and ss["profile"] in subscribers_profile)
if min_subscribers and total_subscribers < min_subscribers:
continue
if "segment_" in columns.split(
",") or "container_" in columns.split(","):
path = ObjectPath.get_path(a["managed_object"])
if path:
segment_path = [
NetworkSegment.get_by_id(s).name
for s in path.segment_path
if NetworkSegment.get_by_id(s)
]
container_path = [
Object.get_by_id(s).name
for s in path.container_path if Object.get_by_id(s)
]
else:
segment_path = []
container_path = []
else:
segment_path = []
container_path = []
r += [
translate_row(
row(
[
str(a["_id"]),
str(a["root"]) if a.get("root") else "",
a["timestamp"],
# a["clear_timestamp"],
"",
str(duration),
moss[a["managed_object"]][0],
moss[a["managed_object"]][1],
mo_hostname.get(a["managed_object"], ""),
Profile.get_by_id(moss[a["managed_object"]][3])
if moss[a["managed_object"]][5] else "",
moss[a["managed_object"]][6],
Platform.get_by_id(
moss[a["managed_object"]][9])
if moss[a["managed_object"]][9] else "",
Firmware.get_by_id(
moss[a["managed_object"]][10])
if moss[a["managed_object"]][10] else "",
AlarmClass.get_by_id(a["alarm_class"]).name,
ActiveAlarm.objects.get(
id=a["_id"]).subject if subject else None,
"Yes" if a["managed_object"] in maintenance
else "No",
total_objects,
total_subscribers,
a.get("escalation_tt"),
a.get("escalation_ts"),
", ".join(l for l in (
loc.location(moss[a["managed_object"]][5]
) if moss[a["managed_object"]]
[5] is not None else "") if l),
container_lookup[a["managed_object"]].get(
"text", "") if container_lookup else "",
],
container_path,
segment_path,
),
cmap,
)
]
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="alarms.csv"'
writer = csv.writer(response)
writer.writerows(r)
return response
elif o_format == "xlsx":
response = StringIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="alarms.xlsx"'
response.close()
return response
def get_ad(name):
ad = AdministrativeDomain(name=name, parent=get_root_ad())
ad.save()
return ad
def get_root_ad():
ad = AdministrativeDomain(name="Discovery Tests")
ad.save()
return ad
def api_report(
self,
request,
o_format,
is_managed=None,
administrative_domain=None,
selector=None,
pool=None,
segment=None,
avail_status=False,
columns=None,
ids=None,
enable_autowidth=False,
):
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, str):
return smart_text(v)
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
type_columns = ["Up/10G", "Up/1G", "Up/100M", "Down/-", "-"]
cols = [
"object1_admin_domain",
# "id",
"object1_name",
"object1_address",
"object1_platform",
"object1_segment",
"object1_tags",
"object1_iface",
"object1_descr",
"object1_speed",
"object2_admin_domain",
"object2_name",
"object2_address",
"object2_platform",
"object2_segment",
"object2_tags",
"object2_iface",
"object2_descr",
"object2_speed",
"link_proto",
"last_seen",
]
header_row = [
"OBJECT1_ADMIN_DOMAIN",
"OBJECT1_NAME",
"OBJECT1_ADDRESS",
"OBJECT1_PLATFORM",
"OBJECT1_SEGMENT",
"OBJECT1_TAGS",
"OBJECT1_IFACE",
"OBJECT1_DESCR",
"OBJECT1_SPEED",
"OBJECT2_ADMIN_DOMAIN",
"OBJECT2_NAME",
"OBJECT2_ADDRESS",
"OBJECT2_PLATFORM",
"OBJECT2_SEGMENT",
"OBJECT2_TAGS",
"OBJECT2_IFACE",
"OBJECT2_DESCR",
"OBJECT2_SPEED",
"LINK_PROTO",
"LAST_SEEN",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
if "interface_type_count" in columns.split(","):
r[-1].extend(type_columns)
# self.logger.info(r)
# self.logger.info("---------------------------------")
# print("-----------%s------------%s" % (administrative_domain, columns))
p = Pool.get_by_name(pool or "default")
mos = ManagedObject.objects.filter()
if request.user.is_superuser and not administrative_domain and not selector and not segment:
mos = ManagedObject.objects.filter(pool=p)
if ids:
mos = ManagedObject.objects.filter(id__in=[ids])
if is_managed is not None:
mos = ManagedObject.objects.filter(is_managed=is_managed)
if pool:
mos = mos.filter(pool=p)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if administrative_domain:
ads = AdministrativeDomain.get_nested_ids(
int(administrative_domain))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if segment:
segment = NetworkSegment.objects.filter(id=segment).first()
if segment:
mos = mos.filter(segment__in=segment.get_nested_ids())
mos_id = list(mos.values_list("id", flat=True))
rld = ReportLinksDetail(mos_id)
mo_resolv = {
mo[0]: mo[1:]
for mo in ManagedObject.objects.filter().values_list(
"id",
"administrative_domain__name",
"name",
"address",
"segment",
"platform",
"labels",
)
}
for link in rld.out:
if len(rld.out[link]) != 2:
# Multilink or bad link
continue
s1, s2 = rld.out[link]
seg1, seg2 = None, None
if "object1_segment" in columns.split(
",") or "object2_segment" in columns.split(","):
seg1, seg2 = mo_resolv[s1["mo"][0]][3], mo_resolv[s2["mo"]
[0]][3]
plat1, plat2 = None, None
if "object1_platform" in columns.split(
",") or "object2_platform" in columns.split(","):
plat1, plat2 = mo_resolv[s1["mo"][0]][4], mo_resolv[s2["mo"]
[0]][4]
r += [
translate_row(
row([
mo_resolv[s1["mo"][0]][0],
mo_resolv[s1["mo"][0]][1],
mo_resolv[s1["mo"][0]][2],
"" if not plat1 else Platform.get_by_id(plat1),
"" if not seg1 else NetworkSegment.get_by_id(seg1),
";".join(mo_resolv[s1["mo"][0]][5] or []),
s1["iface_n"][0],
s1.get("iface_descr")[0]
if s1.get("iface_descr") else "",
s1.get("iface_speed")[0]
if s1.get("iface_speed") else 0,
mo_resolv[s2["mo"][0]][0],
mo_resolv[s2["mo"][0]][1],
mo_resolv[s2["mo"][0]][2],
"" if not plat2 else Platform.get_by_id(plat2),
"" if not seg2 else NetworkSegment.get_by_id(seg2),
";".join(mo_resolv[s2["mo"][0]][5] or []),
s2["iface_n"][0],
s2.get("iface_descr")[0]
if s2.get("iface_descr") else "",
s2.get("iface_speed")[0]
if s2.get("iface_speed") else 0,
s2.get("dis_method", ""),
s2.get("last_seen", ""),
]),
cmap,
)
]
filename = "links_detail_report_%s" % datetime.datetime.now().strftime(
"%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response,
dialect="excel",
delimiter=",",
quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f,
dialect="excel",
delimiter=";",
quotechar='"')
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr("%s.csv" % filename, f.read())
zf.filename = "%s.csv.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/zip")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Objects")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
# response = HttpResponse(
# content_type="application/x-ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def api_report(
self,
request,
reporttype=None,
from_date=None,
to_date=None,
object_profile=None,
filter_default=None,
exclude_zero=None,
interface_profile=None,
selector=None,
administrative_domain=None,
columns=None,
o_format=None,
enable_autowidth=False,
**kwargs
):
def translate_row(row, cmap):
return [row[i] for i in cmap]
map_table = {
"load_interfaces": "/Interface\s\|\sLoad\s\|\s[In|Out]/",
"load_cpu": "/[CPU|Memory]\s\|\sUsage/",
"errors": "/Interface\s\|\s[Errors|Discards]\s\|\s[In|Out]/",
"ping": "/Ping\s\|\sRTT/",
}
cols = [
"id",
"object_name",
"object_address",
"object_platform",
"object_adm_domain",
"object_segment",
# "object_hostname",
# "object_status",
# "profile_name",
# "object_profile",
# "object_vendor",
"iface_name",
"iface_description",
"iface_speed",
"load_in",
"load_in_p",
"load_out",
"load_out_p",
"errors_in",
"errors_out",
"slot",
"cpu_usage",
"memory_usage",
"ping_rtt",
"ping_attempts",
"interface_flap",
"interface_load_url",
]
header_row = [
"ID",
"OBJECT_NAME",
"OBJECT_ADDRESS",
"OBJECT_PLATFORM",
"OBJECT_ADM_DOMAIN",
"OBJECT_SEGMENT",
"IFACE_NAME",
"IFACE_DESCRIPTION",
"IFACE_SPEED",
"LOAD_IN",
"LOAD_IN_P",
"LOAD_OUT",
"LOAD_OUT_P",
"ERRORS_IN",
"ERRORS_OUT",
"CPU_USAGE",
"MEMORY_USAGE",
"PING_RTT",
"PING_ATTEMPTS",
"INTERFACE_FLAP",
"INTERFACE_LOAD_URL",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
columns_order = columns.split(",")
columns_filter = set(columns_order)
r = [translate_row(header_row, cmap)]
object_columns = [c for c in columns_order if c.startswith("object")]
# Date Time Block
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(days=1)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
# interval = (to_date - from_date).days
ts_from_date = time.mktime(from_date.timetuple())
ts_to_date = time.mktime(to_date.timetuple())
# Load managed objects
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
if selector:
mos = mos.filter(ManagedObjectSelector.objects.get(id=int(selector)).Q)
if administrative_domain:
mos = mos.filter(
administrative_domain__in=AdministrativeDomain.get_nested_ids(
int(administrative_domain)
)
)
if object_profile:
mos = mos.filter(object_profile=object_profile)
# iface_dict = {}
d_url = {
"path": "/ui/grafana/dashboard/script/report.js",
"rname": map_table[reporttype],
"from": str(int(ts_from_date * 1000)),
"to": str(int(ts_to_date * 1000)),
# o.name.replace("#", "%23")
"biid": "",
"oname": "",
"iname": "",
}
report_map = {
"load_interfaces": {
"url": "%(path)s?title=interface&biid=%(biid)s"
"&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s",
"q_group": ["interface"],
"q_select": {
(0, "managed_object", "id"): "managed_object",
(1, "path", "iface_name"): "arrayStringConcat(path)",
},
},
"errors": {
"url": """%(path)s?title=errors&biid=%(biid)s&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s""",
"q_group": ["interface"],
},
"load_cpu": {
"url": """%(path)s?title=cpu&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
"q_select": {
(0, "managed_object", "id"): "managed_object",
(1, "path", "slot"): "arrayStringConcat(path)",
},
},
"ping": {
"url": """%(path)s?title=ping&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
"q_select": {(0, "managed_object", "id"): "managed_object"},
},
}
query_map = {
# "iface_description": ('', 'iface_description', "''"),
"iface_description": (
"",
"iface_description",
"dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))",
),
"iface_speed": (
"speed",
"iface_speed",
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed))",
),
"load_in": ("load_in", "l_in", "round(quantile(0.90)(load_in), 0)"),
"load_in_p": (
"load_in",
"l_in_p",
"replaceOne(toString(round(quantile(0.90)(load_in) / "
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
),
"load_out": ("load_out", "l_out", "round(quantile(0.90)(load_out), 0)"),
"load_out_p": (
"load_out",
"l_out_p",
"replaceOne(toString(round(quantile(0.90)(load_out) / "
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
),
"errors_in": ("errors_in", "err_in", "quantile(0.90)(errors_in)"),
"errors_out": ("errors_out", "err_out", "quantile(0.90)(errors_out)"),
"cpu_usage": ("usage", "cpu_usage", "quantile(0.90)(usage)"),
"ping_rtt": ("rtt", "ping_rtt", "round(quantile(0.90)(rtt) / 1000, 2)"),
"ping_attempts": ("attempts", "ping_attempts", "avg(attempts)"),
}
query_fields = []
for c in report_map[reporttype]["q_select"]:
query_fields += [c[2]]
field_shift = len(query_fields) # deny replacing field
for c in columns.split(","):
if c not in query_map:
continue
field, alias, func = query_map[c]
report_map[reporttype]["q_select"][
(columns_order.index(c) + field_shift, field, alias)
] = func
query_fields += [c]
metrics_attrs = namedtuple("METRICSATTRs", query_fields)
mo_attrs = namedtuple("MOATTRs", [c for c in cols if c.startswith("object")])
moss = {}
for row in mos.values_list(
"bi_id", "name", "address", "platform", "administrative_domain__name", "segment"
):
moss[row[0]] = mo_attrs(
*[
row[1],
row[2],
str(Platform.get_by_id(row[3]) if row[3] else ""),
row[4],
str(NetworkSegment.get_by_id(row[5])) if row[5] else "",
]
)
url = report_map[reporttype].get("url", "")
report_metric = self.metric_source[reporttype](
tuple(sorted(moss)), from_date, to_date, columns=None
)
report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"]
if exclude_zero and reporttype == "load_interfaces":
report_metric.CUSTOM_FILTER["having"] += ["max(load_in) != 0 AND max(load_out) != 0"]
if interface_profile:
interface_profile = InterfaceProfile.objects.filter(id=interface_profile).first()
report_metric.CUSTOM_FILTER["having"] += [
"dictGetString('interfaceattributes', 'profile', "
"(managed_object, arrayStringConcat(path))) = '%s'" % interface_profile.name
]
# OBJECT_PLATFORM, ADMIN_DOMAIN, SEGMENT, OBJECT_HOSTNAME
for row in report_metric.do_query():
mm = metrics_attrs(*row)
mo = moss[int(mm.id)]
res = []
for y in columns_order:
if y in object_columns:
res += [getattr(mo, y)]
else:
res += [getattr(mm, y)]
if "interface_load_url" in columns_filter:
d_url["biid"] = mm.id
d_url["oname"] = mo[2].replace("#", "%23")
# res += [url % d_url, interval]
res.insert(columns_order.index("interface_load_url"), url % d_url)
r += [res]
filename = "metrics_detail_report_%s" % datetime.datetime.now().strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response, dialect="excel", delimiter=",", quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "xlsx":
response = StringIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (
r[0][cn] not in max_column_data_length
or len(str(c)) > max_column_data_length[r[0][cn]]
):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(), content_type="application/vnd.ms-excel")
response["Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def get_data(self,
request,
repo="config",
days=1,
adm_domain=None,
managed_object=None,
**kwargs):
baseline = datetime.datetime.now() - datetime.timedelta(days=days)
coll = get_db()["noc.gridvcs.%s.files" % repo].with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED)
pipeline = [
{
"$match": {
"ts": {
"$gte": baseline
}
}
},
{
"$group": {
"_id": "$object",
"last_ts": {
"$max": "$ts"
}
}
},
{
"$sort": {
"_id": 1
}
},
]
if repo == "config":
objects = ManagedObject.objects.filter()
if not request.user.is_superuser:
objects = objects.filter(administrative_domain__in=UserAccess.
get_domains(request.user))
if adm_domain:
adm_domain = AdministrativeDomain.get_nested_ids(adm_domain)
objects = objects.filter(administrative_domain__in=adm_domain)
if managed_object:
mo_q = ManagedObject.get_search_Q(managed_object)
if not mo_q:
objects = objects.filter(name__contains=managed_object)
else:
objects = objects.filter(mo_q)
pipeline = [{
"$match": {
"object": {
"$in": list(objects.values_list("id", flat=True))
}
}
}] + pipeline
# Perform query
data = list(coll.aggregate(pipeline))
# Resolve names
result = []
if data:
seen_ids = list(set(d["_id"] for d in data))
n_map = {}
if repo == "config":
n_map = {
x[0]: x[1:]
for x in ManagedObject.objects.filter(
id__in=list(seen_ids)).values_list(
"id", "name", "address", "profile")
}
elif repo == "dns":
n_map = {
x[0]: x[1:]
for x in DNSZone.objects.filter(
id__in=list(seen_ids)).values_list(
"id", "name", "address", "profile")
}
for d in data:
name, profile = n_map.get(d["_id"], ("-", None))
result += [(d["_id"], name,
Profile.get_by_id(profile) if profile else "-",
d["last_ts"])]
return self.from_dataset(
title="%s: %s in %d days" % (self.title, repo, days),
columns=[
"ID",
"Name",
"Address",
"Profile",
TableColumn(_("Last Changed"), format="datetime"),
],
data=result,
enumerate=True,
)
def api_report(self, request, o_format, is_managed=None,
administrative_domain=None, selector=None, pool=None,
segment=None, avail_status=False, columns=None, ids=None):
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, unicode):
return v.encode("utf-8")
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
type_columns = ["Up/10G", "Up/1G", "Up/100M", "Down/-", "-"]
cols = [
"admin_domain",
# "id",
"object1_name",
"object1_address",
"object1_iface",
"object2_name",
"object2_address",
"object2_iface",
"link_proto",
"last_seen"
]
header_row = [
"ADMIN_DOMAIN",
"OBJECT1_NAME",
"OBJECT1_ADDRESS",
"OBJECT1_IFACE",
"OBJECT2_NAME",
"OBJECT2_ADDRESS",
"OBJECT2_IFACE",
"LINK_PROTO",
"LAST_SEEN"
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
if "interface_type_count" in columns.split(","):
r[-1].extend(type_columns)
# self.logger.info(r)
# self.logger.info("---------------------------------")
# print("-----------%s------------%s" % (administrative_domain, columns))
p = Pool.get_by_name(pool or "default")
mos = ManagedObject.objects.filter()
if request.user.is_superuser and not administrative_domain and not selector and not segment:
mos = ManagedObject.objects.filter(pool=p)
if ids:
mos = ManagedObject.objects.filter(id__in=[ids])
if is_managed is not None:
mos = ManagedObject.objects.filter(is_managed=is_managed)
if pool:
mos = mos.filter(pool=p)
if not request.user.is_superuser:
mos = mos.filter(administrative_domain__in=UserAccess.get_domains(request.user))
if administrative_domain:
ads = AdministrativeDomain.get_nested_ids(int(administrative_domain))
mos = mos.filter(administrative_domain__in=ads)
if selector:
selector = ManagedObjectSelector.get_by_id(int(selector))
mos = mos.filter(selector.Q)
if segment:
segment = NetworkSegment.objects.filter(id=segment).first()
if segment:
mos = mos.filter(segment__in=segment.get_nested_ids())
mos_id = list(mos.values_list("id", flat=True))
rld = ReportLinksDetail(mos_id)
mo_resolv = dict((mo[0], mo[1:]) for mo in ManagedObject.objects.filter().values_list(
"id", "administrative_domain__name", "name", "address"))
for link in rld.out:
if len(rld.out[link]) != 2:
# Multilink or bad link
continue
s1, s2 = rld.out[link]
r += [translate_row(row([
mo_resolv[s1["mo"][0]][0],
mo_resolv[s1["mo"][0]][1],
mo_resolv[s1["mo"][0]][2],
s1["iface_n"][0],
mo_resolv[s2["mo"][0]][1],
mo_resolv[s2["mo"][0]][2],
s2["iface_n"][0],
s1.get("dis_method", ""),
s1.get("last_seen", "")
]), cmap)]
filename = "links_detail_report_%s" % datetime.datetime.now().strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = "attachment; filename=\"%s.csv\"" % filename
writer = csv.writer(response, dialect='excel', delimiter=';')
writer.writerows(r)
return response
elif o_format == "xlsx":
with tempfile.NamedTemporaryFile(mode="wb") as f:
wb = xlsxwriter.Workbook(f.name)
ws = wb.add_worksheet("Objects")
for rn, x in enumerate(r):
for cn, c in enumerate(x):
ws.write(rn, cn, c)
ws.autofilter(0, 0, rn, cn)
wb.close()
response = HttpResponse(
content_type="application/x-ms-excel")
response[
"Content-Disposition"] = "attachment; filename=\"%s.xlsx\"" % filename
with open(f.name) as ff:
response.write(ff.read())
return response