def get_managedobjectprofile():
mop = ManagedObjectProfile(
name="disovery tests",
enable_box_discovery=True,
enable_box_discovery_version=True,
enable_box_discovery_caps=True,
enable_box_discovery_id=True,
enable_box_discovery_interface=True,
cli_session_policy="D", # Must be disabled, overrides BeefCaller
)
mop.save()
return mop
def get_object_profile_metrics(cls, p_id: int) -> Dict[str, MetricConfig]:
r = {}
opr = ManagedObjectProfile.get_by_id(id=p_id)
if not opr:
return r
for m in opr.metrics:
mt_id = m.get("metric_type")
if not mt_id:
continue
mt = MetricType.get_by_id(mt_id)
if not mt:
continue
if m.get("threshold_profile"):
threshold_profile = ThresholdProfile.get_by_id(
m.get("threshold_profile"))
else:
threshold_profile = None
r[mt.name] = MetricConfig(
mt,
m.get("enable_box", True),
m.get("enable_periodic", True),
m.get("is_stored", True),
threshold_profile,
)
return r
def get_interface_metrics(managed_objects):
from noc.sa.models.managedobject import ManagedObject
# mo = self.object
meric_map = {
"load_in": "Interface | Load | In",
"load_out": "Interface | Load | Out",
"errors_in": "Interface | Errors | In",
"errors_out": "Interface | Errors | Out",
}
if not isinstance(managed_objects, Iterable):
managed_objects = [managed_objects]
bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
query_interval = (
ManagedObjectProfile.get_max_metrics_interval(
set(mo.object_profile.id for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))
)
* 1.5
)
from_date = datetime.datetime.now() - datetime.timedelta(seconds=max(query_interval, 3600))
from_date = from_date.replace(microsecond=0)
SQL = """SELECT managed_object, path[4] as iface, argMax(ts, ts), argMax(load_in, ts), argMax(load_out, ts),
argMax(errors_in, ts), argMax(errors_out, ts)
FROM interface
WHERE
date >= toDate('%s')
AND ts >= toDateTime('%s')
AND managed_object IN (%s)
GROUP BY managed_object, iface
""" % (
from_date.date().isoformat(),
from_date.isoformat(sep=" "),
", ".join(bi_map),
)
ch = ch_connection()
mtable = [] # mo_id, mac, iface, ts
metric_map = defaultdict(dict)
last_ts = {} # mo -> ts
try:
for mo_bi_id, iface, ts, load_in, load_out, errors_in, errors_out in ch.execute(post=SQL):
mo = bi_map.get(mo_bi_id)
if mo:
mtable += [[mo, iface, ts, load_in, load_out]]
metric_map[mo][iface] = {
meric_map["load_in"]: int(load_in),
meric_map["load_out"]: int(load_out),
meric_map["errors_in"]: int(errors_in),
meric_map["errors_out"]: int(errors_out),
}
last_ts[mo] = max(ts, last_ts.get(mo, ts))
except ClickhouseError:
pass
return metric_map, last_ts
def get_objects(self, seg: NetworkSegment) -> List[ManagedObject]:
if not self._objects:
profile = ManagedObjectProfile(name="mock", level=self.LEVEL)
self._objects = [
patch_model(
ManagedObject(name="%d" % (i + 1), segment=self.target, object_profile=profile)
)
for i in range(self.N)
]
objects = [mo for mo in self._objects if mo.segment == seg]
self.set_power(seg, sum(mo.object_profile.level for mo in objects))
return objects
def get_object_profile_metrics(cls, p_id):
r = {}
opr = ManagedObjectProfile.get_by_id(id=p_id)
if not opr:
return r
for m in opr.metrics:
mt_id = m.get("metric_type")
if not mt_id:
continue
mt = MetricType.get_by_id(mt_id)
if not mt:
continue
le = m.get("low_error")
lw = m.get("low_warn")
he = m.get("high_error")
hw = m.get("high_warn")
lew = AlarmSeverity.severity_for_weight(int(m.get("low_error_weight", 10)))
lww = AlarmSeverity.severity_for_weight(int(m.get("low_warn_weight", 1)))
hew = AlarmSeverity.severity_for_weight(int(m.get("high_error_weight", 1)))
hww = AlarmSeverity.severity_for_weight(int(m.get("high_warn_weight", 10)))
threshold_profile = None
if m.get("threshold_profile"):
threshold_profile = ThresholdProfile.get_by_id(m.get("threshold_profile"))
r[mt.name] = MetricConfig(
mt,
m.get("enable_box", True),
m.get("enable_periodic", True),
m.get("is_stored", True),
m.get("window_type", "m"),
int(m.get("window", 1)),
m.get("window_function", "last"),
m.get("window_config"),
m.get("window_related", False),
int(le) if le is not None else None,
int(lw) if lw is not None else None,
int(hw) if hw is not None else None,
int(he) if he is not None else None,
lew, lww, hww, hew,
threshold_profile,
le is not None or lw is not None or he is not None or hw is not None
)
return r
def get_weight(cls, summary):
"""
Convert result of *get_object_summary* to alarm weight
"""
w = 0
subscribers = summary.get("subscriber", {})
for s in subscribers:
sp = SubscriberProfile.get_by_id(s)
if sp and sp.weight:
w += sp.weight * subscribers[s]
services = summary.get("service", {})
for s in services:
sp = ServiceProfile.get_by_id(s)
if sp and sp.weight:
w += sp.weight * services[s]
objects = summary.get("object", {})
for s in objects:
sp = ManagedObjectProfile.get_by_id(s)
if sp and sp.weight:
w += sp.weight * objects[s]
return w
def get_objects_metrics(managed_objects):
from noc.sa.models.managedobject import ManagedObject
"""
:param managed_objects:
:return:
"""
if not isinstance(managed_objects, Iterable):
managed_objects = [managed_objects]
# Object Metrics
bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
query_interval = ManagedObjectProfile.get_max_metrics_interval(
set(mo.object_profile.id for mo in ManagedObject.objects.filter(
bi_id__in=bi_map.keys()))) * 2
from_date = datetime.datetime.now() - datetime.timedelta(
seconds=max(query_interval, 3600))
from_date = from_date.replace(microsecond=0)
# @todo Left Join
object_profiles = set(
mo.object_profile.id
for mo in ManagedObject.objects.filter(bi_id__in=bi_map.keys()))
msd = {ms.id: ms.table_name for ms in MetricScope.objects.filter()}
mts = {
str(mt.id): (msd[mt.scope.id], mt.field_name, mt.name)
for mt in MetricType.objects.all()
}
mmm = set()
op_fields_map = defaultdict(list)
for op in ManagedObjectProfile.objects.filter(id__in=object_profiles):
if not op.metrics:
continue
for mt in op.metrics:
mmm.add(mts[mt["metric_type"]])
op_fields_map[op.id] += [mts[mt["metric_type"]][1]]
ch = ch_connection()
mtable = [] # mo_id, mac, iface, ts
metric_map = {}
last_ts = {} # mo -> ts
for table, fields in itertools.groupby(sorted(mmm, key=lambda x: x[0]),
key=lambda x: x[0]):
fields = list(fields)
SQL = """SELECT managed_object, argMax(ts, ts), arrayStringConcat(path) as path, %s
FROM %s
WHERE
date >= toDate('%s')
AND ts >= toDateTime('%s')
AND managed_object IN (%s)
GROUP BY managed_object, path
""" % (", ".join(
["argMax(%s, ts) as %s" % (f[1], f[1])
for f in fields]), table, from_date.date().isoformat(),
from_date.isoformat(sep=" "), ", ".join(bi_map))
try:
for result in ch.execute(post=SQL):
mo_bi_id, ts, path = result[:3]
mo = bi_map.get(mo_bi_id)
i = 0
for r in result[3:]:
f_name = fields[i][2]
mtable += [[mo, ts, path, r]]
if mo not in metric_map:
metric_map[mo] = defaultdict(dict)
metric_map[mo][path][f_name] = r
last_ts[mo] = max(ts, last_ts.get(mo, ts))
i += 1
except ClickhouseError:
pass
return metric_map, last_ts
def get_objects_metrics(
managed_objects: Union[Iterable, int]
) -> Tuple[Dict["ManagedObject", Dict[str, Dict[str, int]]], Dict[
"ManagedObject", datetime.datetime]]:
"""
:param managed_objects:
:return: Dictionary ManagedObject -> Path -> MetricName -> value
"""
if not isinstance(managed_objects, Iterable):
managed_objects = [managed_objects]
# Object Metrics
bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
query_interval = (ManagedObjectProfile.get_max_metrics_interval(
set(mo.object_profile.id
for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))) *
2)
from_date = datetime.datetime.now() - datetime.timedelta(
seconds=max(query_interval, 3600))
from_date = from_date.replace(microsecond=0)
# @todo Left Join
object_profiles = set(
mo.object_profile.id
for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))
msd: Dict[str, str] = {} # Map ScopeID -> TableName
labels_table = set()
for ms in MetricScope.objects.filter():
msd[ms.id] = ms.table_name
if ms.labels:
labels_table.add(ms.table_name)
mts: Dict[str, Tuple[str, str, str]] = {
str(mt.id): (msd[mt.scope.id], mt.field_name, mt.name)
for mt in MetricType.objects.all()
} # Map Metric Type ID -> table_name, column_name, MetricType Name
mmm = set()
op_fields_map: DefaultDict[str, List[str]] = defaultdict(list)
for op in ManagedObjectProfile.objects.filter(id__in=object_profiles):
if not op.metrics:
continue
for mt in op.metrics:
mmm.add(mts[mt["metric_type"]])
op_fields_map[op.id] += [mts[mt["metric_type"]][1]]
ch = ch_connection()
mtable = [] # mo_id, mac, iface, ts
metric_map = {}
last_ts: Dict["ManagedObject", datetime.datetime] = {} # mo -> ts
for table, fields in itertools.groupby(sorted(mmm, key=lambda x: x[0]),
key=lambda x: x[0]):
fields = list(fields)
SQL = """SELECT managed_object, argMax(ts, ts), %%s %s
FROM %s
WHERE
date >= toDate('%s')
AND ts >= toDateTime('%s')
AND managed_object IN (%s)
GROUP BY managed_object %%s
""" % (
", ".join(["argMax(%s, ts) as %s" % (f[1], f[1]) for f in fields]),
table,
from_date.date().isoformat(),
from_date.isoformat(sep=" "),
", ".join(bi_map),
)
if table in labels_table:
# SQL = SQL % ("arrayStringConcat(labels, '|') as ll,", ", labels")
SQL = SQL % (
"arrayStringConcat(arrayMap(x -> splitByString('::', x)[-1], labels), '|') as labels,",
", labels",
)
else:
SQL = SQL % ("", "")
try:
for result in ch.execute(post=SQL):
if table in labels_table:
mo_bi_id, ts, labels = result[:3]
result = result[3:]
else:
mo_bi_id, ts = result[:2]
labels, result = "", result[2:]
mo = bi_map.get(mo_bi_id)
i = 0
for r in result:
f_name = fields[i][2]
mtable += [[mo, ts, labels, r]]
if mo not in metric_map:
metric_map[mo] = defaultdict(dict)
metric_map[mo][labels][f_name] = r
last_ts[mo] = max(ts, last_ts.get(mo, ts))
i += 1
except ClickhouseError as e:
print(e)
return metric_map, last_ts
def get_interface_metrics(
managed_objects: Union[Iterable, int],
meric_map: Optional[Dict[str, Any]] = None
) -> Tuple[Dict["ManagedObject", Dict[str, Dict[str, Union[float, int]]]],
Dict["ManagedObject", datetime.datetime], ]:
"""
:param managed_objects: ManagedObject list or bi_id list
:param meric_map: For customization getting metrics
:return: Dictionary ManagedObject -> Path -> MetricName -> value
"""
# mo = self.object
if not meric_map:
meric_map = {
"table_name": "interface",
"map": {
"load_in": "Interface | Load | In",
"load_out": "Interface | Load | Out",
"errors_in": "Interface | Errors | In",
"errors_out": "Interface | Errors | Out",
},
}
if not isinstance(managed_objects, Iterable):
managed_objects = [managed_objects]
bi_map: Dict[str, "ManagedObject"] = {
str(getattr(mo, "bi_id", mo)): mo
for mo in managed_objects
}
query_interval: float = (ManagedObjectProfile.get_max_metrics_interval(
set(mo.object_profile.id
for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))) *
1.5)
from_date = datetime.datetime.now() - datetime.timedelta(
seconds=max(query_interval, 3600))
from_date = from_date.replace(microsecond=0)
SQL = """SELECT managed_object, argMax(ts, ts), splitByString('::', arrayFirst(x -> startsWith(x, 'noc::interface::'), labels))[-1] as iface, labels, %s
FROM %s
WHERE
date >= toDate('%s')
AND ts >= toDateTime('%s')
AND managed_object IN (%s)
AND NOT arrayExists(x -> startsWith(x, 'noc::unit::'), labels)
GROUP BY managed_object, labels
""" % (
", ".join(
["argMax(%s, ts) as %s" % (f, f)
for f in meric_map["map"].keys()]),
meric_map["table_name"],
from_date.date().isoformat(),
from_date.isoformat(sep=" "),
", ".join(bi_map),
)
ch = ch_connection()
metric_map: DefaultDict["ManagedObject",
Dict[str, Dict[str,
Union[int,
float]]]] = defaultdict(dict)
last_ts: Dict["ManagedObject", datetime.datetime] = {} # mo -> ts
metric_fields = list(meric_map["map"].keys())
try:
for result in ch.execute(post=SQL):
mo_bi_id, ts, iface, labels = result[:4]
labels = ast.literal_eval(labels)
res = dict(zip(metric_fields, result[4:]))
mo = bi_map.get(mo_bi_id)
if len(labels) == 1 and metric_map[mo].get(iface):
# If only interface metric
continue
metric_map[mo][iface] = defaultdict(dict)
for field, value in res.items():
metric_map[mo][iface][meric_map["map"].get(field)] = (
float(value) if is_float(value) else int(value))
last_ts[mo] = max(ts, last_ts.get(mo, ts))
except ClickhouseError:
pass
return metric_map, last_ts