def fix():
max_value = NetworkSegment._get_collection().estimated_document_count()
for ns in progressbar.progressbar(iter_ids_batch(), max_value=max_value):
try:
st = SegmentTopology(ns)
ObjectData.update_uplinks(st.iter_uplinks())
except Exception as e:
print("[%s] %s" % (ns.name, e))
def get_segment_objects(segment):
# Get objects belonging to segment
so = set(
ManagedObject.objects.filter(segment=segment).values_list(
"id", flat=True))
# Get objects from underlying segments
for ns in NetworkSegment._get_collection().find({"parent": segment},
{"_id": 1}):
so |= get_segment_objects(ns["_id"])
return so
def iter_ids_batch():
match = {}
while True:
cursor = (NetworkSegment._get_collection().find(
match, {
"_id": 1
}, no_cursor_timeout=True).sort("_id").limit(BATCH_SIZE))
d = [d["_id"] for d in cursor]
if not d:
break
for link in NetworkSegment.objects.filter(id__in=d).timeout(False):
yield link
# if match and match["_id"]["$gt"] == d[-1]:
# break
match = {"_id": {"$gt": d[-1]}}
def extract(self):
ns_id = dict(
NetworkSegment.objects.filter(
read_preference=ReadPreference.SECONDARY_PREFERRED).scalar(
"id", "bi_id"))
ns = NetworkSegment._get_collection().with_options(
read_preference=ReadPreference.SECONDARY_PREFERRED)
for sub in ns.find({}, {
"_id": 1,
"bi_id": 1,
"name": 1,
"parent": 1
}).sort("parent"):
yield (sub["bi_id"], sub["_id"], sub["name"],
ns_id.get(sub["parent"], "") if sub.get("parent") else "")
def api_report(
self,
request,
o_format,
is_managed=None,
administrative_domain=None,
selector=None,
pool=None,
segment=None,
avail_status=False,
columns=None,
ids=None,
detail_stat=None,
enable_autowidth=False,
):
def row(row):
def qe(v):
if v is None:
return ""
if isinstance(v, str):
return smart_text(v)
elif isinstance(v, datetime.datetime):
return v.strftime("%Y-%m-%d %H:%M:%S")
elif not isinstance(v, str):
return str(v)
else:
return v
return [qe(x) for x in row]
def translate_row(row, cmap):
return [row[i] for i in cmap]
type_columns = ["Up/10G", "Up/1G", "Up/100M", "Up/10M", "Down/-", "-"]
cols = [
"id",
"object_name",
"object_address",
"object_hostname",
"object_status",
"profile_name",
"object_profile",
"object_vendor",
"object_platform",
"object_attr_hwversion",
"object_version",
"object_attr_bootprom",
"object_serial",
"object_attr_patch",
"auth_profile",
"avail",
"admin_domain",
"container",
"segment",
"phys_interface_count",
"link_count",
"last_config_ts"
# "discovery_problem"
# "object_tags"
# "sorted_tags"
# "object_caps"
# "interface_type_count"
]
header_row = [
"ID",
"OBJECT_NAME",
"OBJECT_ADDRESS",
"OBJECT_HOSTNAME",
"OBJECT_STATUS",
"PROFILE_NAME",
"OBJECT_PROFILE",
"OBJECT_VENDOR",
"OBJECT_PLATFORM",
"OBJECT_HWVERSION",
"OBJECT_VERSION",
"OBJECT_BOOTPROM",
"OBJECT_SERIAL",
"OBJECT_ATTR_PATCH",
"AUTH_PROFILE",
"AVAIL",
"ADMIN_DOMAIN",
"CONTAINER",
"SEGMENT",
"PHYS_INTERFACE_COUNT",
"LINK_COUNT",
"LAST_CONFIG_TS",
]
# "DISCOVERY_PROBLEM"
# "ADM_PATH
# "DISCOVERY_PROBLEM"
# "OBJECT_TAGS"
# "SORTED_TAGS"
# "OBJECT_CAPS"
# "INTERFACE_TYPE_COUNT"
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
r = [translate_row(header_row, cmap)]
mos = self.get_report_object(
request.user, is_managed, administrative_domain, selector, pool, segment, ids
)
columns_filter = set(columns.split(","))
mos_id = tuple(mos.order_by("id").values_list("id", flat=True))
mos_filter = None
if detail_stat:
ref = ReportModelFilter()
ids = list(ref.proccessed(detail_stat).values())
mos_filter = set(mos_id).intersection(ids[0])
mos_id = sorted(mos_filter)
avail = {}
if "avail" in columns_filter:
avail = ObjectStatus.get_statuses(mos_id)
link_count = iter(ReportObjectLinkCount(mos_id))
iface_count = iter(ReportObjectIfacesTypeStat(mos_id))
if "container" in columns_filter:
container_lookup = iter(ReportContainerData(mos_id))
else:
container_lookup = None
if "object_serial" in columns_filter:
container_serials = iter(ReportContainer(mos_id))
else:
container_serials = None
if "interface_type_count" in columns_filter:
iss = iter(ReportObjectIfacesStatusStat(mos_id))
else:
iss = None
if (
"object_attr_patch" in columns_filter
or "object_serial" in columns_filter
or "object_attr_hwversion" in columns_filter
or "object_attr_bootprom" in columns_filter
):
roa = iter(ReportObjectAttributes(mos_id))
else:
roa = None
hn = iter(ReportObjectsHostname1(mos_id))
rc = iter(ReportObjectConfig(mos_id))
segment_lookup = {}
# ccc = iter(ReportObjectCaps(mos_id))
if "segment" in columns_filter:
segment_lookup = {
str(n["_id"]): n["name"]
for n in NetworkSegment._get_collection().find({}, {"name": 1})
}
if "adm_path" in columns_filter:
ad_path = ReportAdPath()
r[-1].extend([_("ADM_PATH1"), _("ADM_PATH1"), _("ADM_PATH1")])
if "interface_type_count" in columns_filter:
r[-1].extend(type_columns)
if "object_caps" in columns_filter:
object_caps = ReportObjectCaps(mos_id)
caps_columns = list(object_caps.ATTRS.values())
ccc = iter(object_caps)
r[-1].extend(caps_columns)
if "object_labels" in columns_filter:
r[-1].extend([_("OBJECT_LABELS")])
if "sorted_labels" in columns_filter:
labels = set()
for s in (
ManagedObject.objects.filter()
.exclude(labels=None)
.values_list("labels", flat=True)
.distinct()
):
labels.update(set(s))
labels_o = sorted([t for t in labels if "{" not in t])
r[-1].extend(labels_o)
if "discovery_problem" in columns.split(","):
discovery_result = ReportDiscoveryResult(mos_id)
discovery_result.safe_output = True
discovery_result.unknown_value = ([""] * len(discovery_result.ATTRS),)
dp_columns = discovery_result.ATTRS
dp = iter(discovery_result)
r[-1].extend(dp_columns)
icount = 0
l_count = 10000
self.logger.debug("[%s|reportobjectdetail] Start main Loop", request.user)
for (
mo_id,
name,
address,
is_managed,
sa_profile,
o_profile,
auth_profile,
ad,
m_segment,
vendor,
platform,
version,
labels,
) in (
mos.values_list(
"id",
"name",
"address",
"is_managed",
"profile",
"object_profile__name",
"auth_profile__name",
"administrative_domain__name",
"segment",
"vendor",
"platform",
"version",
"labels",
)
.order_by("id")
.iterator()
):
if (mos_filter and mo_id not in mos_filter) or not mos_id:
continue
if container_serials:
mo_serials = next(container_serials)
else:
mo_serials = [{}]
if container_lookup:
mo_continer = next(container_lookup)
else:
mo_continer = ("",)
if roa:
serial, hw_ver, boot_prom, patch = next(roa)[0] # noqa
else:
serial, hw_ver, boot_prom, patch = "", "", "", "" # noqa
r.append(
translate_row(
row(
[
mo_id,
name,
address,
next(hn)[0],
"managed" if is_managed else "unmanaged",
Profile.get_by_id(sa_profile),
o_profile,
Vendor.get_by_id(vendor) if vendor else "",
Platform.get_by_id(platform) if platform else "",
hw_ver,
Firmware.get_by_id(version) if version else "",
boot_prom,
# Serial
mo_serials[0].get("serial", "") or serial,
patch or "",
auth_profile,
{True: _("Yes"), False: _("No"), None: _("Unknown")}[
avail.get(mo_id, None)
],
ad,
mo_continer[0],
# NetworkSegment.get_by_id(m_segment) if m_segment else "",
segment_lookup.get(m_segment, "") if segment_lookup else "",
next(iface_count)[0],
next(link_count)[0],
next(rc)[0],
]
),
cmap,
)
)
if "adm_path" in columns_filter:
r[-1].extend([ad] + list(ad_path[ad]))
if "interface_type_count" in columns_filter:
r[-1].extend(next(iss)[0])
if "object_caps" in columns_filter:
r[-1].extend(next(ccc)[0])
if "object_labels" in columns_filter:
r[-1].append(",".join(labels if labels else []))
if "sorted_labels" in columns_filter:
out_labels = [""] * len(labels_o)
try:
if labels:
for m in labels:
out_labels[labels_o.index(m)] = m
except ValueError:
logger.warning("Bad value for tag: %s", m)
r[-1].extend(out_labels)
if "discovery_problem" in columns_filter:
r[-1].extend(next(dp)[0])
if not icount:
self.logger.debug("[%s|reportobjectdetail] First chunk", request.user)
if icount // l_count:
self.logger.debug(
"[%s|reportobjectdetail] Proccessed chunk number %d", request.user, icount
)
l_count += 10000
icount += 1
# r.append(x)
self.logger.debug("[%s|reportobjectdetail] End mail loop", request.user)
filename = "mo_detail_report_%s" % datetime.datetime.now().strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response["Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response, dialect="excel", delimiter=";", quotechar='"')
writer.writerows(r)
return response
elif o_format == "csv_zip":
response = BytesIO()
f = TextIOWrapper(TemporaryFile(mode="w+b"), encoding="utf-8")
writer = csv.writer(f, dialect="excel", delimiter=";", quotechar='"')
writer.writerows(r)
f.seek(0)
with ZipFile(response, "w", compression=ZIP_DEFLATED) as zf:
zf.writestr("%s.csv" % filename, f.read())
zf.filename = "%s.csv.zip" % filename
# response = HttpResponse(content_type="text/csv")
response.seek(0)
response = HttpResponse(response.getvalue(), content_type="application/zip")
response["Content-Disposition"] = 'attachment; filename="%s.csv.zip"' % filename
return response
elif o_format == "xlsx":
response = BytesIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Objects")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (
r[0][cn] not in max_column_data_length
or len(str(c)) > max_column_data_length[r[0][cn]]
):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
# for
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(), content_type="application/vnd.ms-excel")
# response = HttpResponse(
# content_type="application/x-ms-excel")
response["Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
def fix():
total = NetworkSegment._get_collection().estimated_document_count()
for ns in progressbar.progressbar(NetworkSegment.objects.timeout(False),
max_value=total):
st = SegmentTopology(ns)
ObjectData.update_uplinks(st.iter_uplinks())