Ejemplo n.º 1
0
def get_managedobjectprofile():
    mop = ManagedObjectProfile(
        name="disovery tests",
        enable_box_discovery=True,
        enable_box_discovery_version=True,
        enable_box_discovery_caps=True,
        enable_box_discovery_id=True,
        enable_box_discovery_interface=True,
        cli_session_policy="D",  # Must be disabled, overrides BeefCaller
    )
    mop.save()
    return mop
Ejemplo n.º 2
0
 def get_object_profile_metrics(cls, p_id: int) -> Dict[str, MetricConfig]:
     r = {}
     opr = ManagedObjectProfile.get_by_id(id=p_id)
     if not opr:
         return r
     for m in opr.metrics:
         mt_id = m.get("metric_type")
         if not mt_id:
             continue
         mt = MetricType.get_by_id(mt_id)
         if not mt:
             continue
         if m.get("threshold_profile"):
             threshold_profile = ThresholdProfile.get_by_id(
                 m.get("threshold_profile"))
         else:
             threshold_profile = None
         r[mt.name] = MetricConfig(
             mt,
             m.get("enable_box", True),
             m.get("enable_periodic", True),
             m.get("is_stored", True),
             threshold_profile,
         )
     return r
Ejemplo n.º 3
0
def get_interface_metrics(managed_objects):
    from noc.sa.models.managedobject import ManagedObject

    # mo = self.object
    meric_map = {
        "load_in": "Interface | Load | In",
        "load_out": "Interface | Load | Out",
        "errors_in": "Interface | Errors | In",
        "errors_out": "Interface | Errors | Out",
    }
    if not isinstance(managed_objects, Iterable):
        managed_objects = [managed_objects]
    bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
    query_interval = (
        ManagedObjectProfile.get_max_metrics_interval(
            set(mo.object_profile.id for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))
        )
        * 1.5
    )
    from_date = datetime.datetime.now() - datetime.timedelta(seconds=max(query_interval, 3600))
    from_date = from_date.replace(microsecond=0)
    SQL = """SELECT managed_object, path[4] as iface, argMax(ts, ts), argMax(load_in, ts), argMax(load_out, ts),
    argMax(errors_in, ts), argMax(errors_out, ts)
            FROM interface
            WHERE
              date >= toDate('%s')
              AND ts >= toDateTime('%s')
              AND managed_object IN (%s)
            GROUP BY managed_object, iface
            """ % (
        from_date.date().isoformat(),
        from_date.isoformat(sep=" "),
        ", ".join(bi_map),
    )
    ch = ch_connection()
    mtable = []  # mo_id, mac, iface, ts
    metric_map = defaultdict(dict)
    last_ts = {}  # mo -> ts
    try:
        for mo_bi_id, iface, ts, load_in, load_out, errors_in, errors_out in ch.execute(post=SQL):
            mo = bi_map.get(mo_bi_id)
            if mo:
                mtable += [[mo, iface, ts, load_in, load_out]]
                metric_map[mo][iface] = {
                    meric_map["load_in"]: int(load_in),
                    meric_map["load_out"]: int(load_out),
                    meric_map["errors_in"]: int(errors_in),
                    meric_map["errors_out"]: int(errors_out),
                }
                last_ts[mo] = max(ts, last_ts.get(mo, ts))
    except ClickhouseError:
        pass
    return metric_map, last_ts
Ejemplo n.º 4
0
    def get_objects(self, seg: NetworkSegment) -> List[ManagedObject]:
        if not self._objects:
            profile = ManagedObjectProfile(name="mock", level=self.LEVEL)
            self._objects = [
                patch_model(
                    ManagedObject(name="%d" % (i + 1), segment=self.target, object_profile=profile)
                )
                for i in range(self.N)
            ]

        objects = [mo for mo in self._objects if mo.segment == seg]
        self.set_power(seg, sum(mo.object_profile.level for mo in objects))
        return objects
Ejemplo n.º 5
0
 def get_object_profile_metrics(cls, p_id):
     r = {}
     opr = ManagedObjectProfile.get_by_id(id=p_id)
     if not opr:
         return r
     for m in opr.metrics:
         mt_id = m.get("metric_type")
         if not mt_id:
             continue
         mt = MetricType.get_by_id(mt_id)
         if not mt:
             continue
         le = m.get("low_error")
         lw = m.get("low_warn")
         he = m.get("high_error")
         hw = m.get("high_warn")
         lew = AlarmSeverity.severity_for_weight(int(m.get("low_error_weight", 10)))
         lww = AlarmSeverity.severity_for_weight(int(m.get("low_warn_weight", 1)))
         hew = AlarmSeverity.severity_for_weight(int(m.get("high_error_weight", 1)))
         hww = AlarmSeverity.severity_for_weight(int(m.get("high_warn_weight", 10)))
         threshold_profile = None
         if m.get("threshold_profile"):
             threshold_profile = ThresholdProfile.get_by_id(m.get("threshold_profile"))
         r[mt.name] = MetricConfig(
             mt,
             m.get("enable_box", True),
             m.get("enable_periodic", True),
             m.get("is_stored", True),
             m.get("window_type", "m"),
             int(m.get("window", 1)),
             m.get("window_function", "last"),
             m.get("window_config"),
             m.get("window_related", False),
             int(le) if le is not None else None,
             int(lw) if lw is not None else None,
             int(hw) if hw is not None else None,
             int(he) if he is not None else None,
             lew, lww, hww, hew,
             threshold_profile,
             le is not None or lw is not None or he is not None or hw is not None
         )
     return r
Ejemplo n.º 6
0
 def get_weight(cls, summary):
     """
     Convert result of *get_object_summary* to alarm weight
     """
     w = 0
     subscribers = summary.get("subscriber", {})
     for s in subscribers:
         sp = SubscriberProfile.get_by_id(s)
         if sp and sp.weight:
             w += sp.weight * subscribers[s]
     services = summary.get("service", {})
     for s in services:
         sp = ServiceProfile.get_by_id(s)
         if sp and sp.weight:
             w += sp.weight * services[s]
     objects = summary.get("object", {})
     for s in objects:
         sp = ManagedObjectProfile.get_by_id(s)
         if sp and sp.weight:
             w += sp.weight * objects[s]
     return w
Ejemplo n.º 7
0
def get_objects_metrics(managed_objects):
    from noc.sa.models.managedobject import ManagedObject
    """

    :param managed_objects:
    :return:
    """
    if not isinstance(managed_objects, Iterable):
        managed_objects = [managed_objects]

    # Object Metrics
    bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
    query_interval = ManagedObjectProfile.get_max_metrics_interval(
        set(mo.object_profile.id for mo in ManagedObject.objects.filter(
            bi_id__in=bi_map.keys()))) * 2
    from_date = datetime.datetime.now() - datetime.timedelta(
        seconds=max(query_interval, 3600))
    from_date = from_date.replace(microsecond=0)

    # @todo Left Join
    object_profiles = set(
        mo.object_profile.id
        for mo in ManagedObject.objects.filter(bi_id__in=bi_map.keys()))
    msd = {ms.id: ms.table_name for ms in MetricScope.objects.filter()}
    mts = {
        str(mt.id): (msd[mt.scope.id], mt.field_name, mt.name)
        for mt in MetricType.objects.all()
    }
    mmm = set()
    op_fields_map = defaultdict(list)
    for op in ManagedObjectProfile.objects.filter(id__in=object_profiles):
        if not op.metrics:
            continue
        for mt in op.metrics:
            mmm.add(mts[mt["metric_type"]])
            op_fields_map[op.id] += [mts[mt["metric_type"]][1]]

    ch = ch_connection()
    mtable = []  # mo_id, mac, iface, ts
    metric_map = {}
    last_ts = {}  # mo -> ts

    for table, fields in itertools.groupby(sorted(mmm, key=lambda x: x[0]),
                                           key=lambda x: x[0]):
        fields = list(fields)
        SQL = """SELECT managed_object, argMax(ts, ts), arrayStringConcat(path) as path, %s
              FROM %s
              WHERE
                date >= toDate('%s')
                AND ts >= toDateTime('%s')
                AND managed_object IN (%s)
              GROUP BY managed_object, path
              """ % (", ".join(
            ["argMax(%s, ts) as %s" % (f[1], f[1])
             for f in fields]), table, from_date.date().isoformat(),
                     from_date.isoformat(sep=" "), ", ".join(bi_map))
        try:
            for result in ch.execute(post=SQL):
                mo_bi_id, ts, path = result[:3]
                mo = bi_map.get(mo_bi_id)
                i = 0
                for r in result[3:]:
                    f_name = fields[i][2]
                    mtable += [[mo, ts, path, r]]
                    if mo not in metric_map:
                        metric_map[mo] = defaultdict(dict)
                    metric_map[mo][path][f_name] = r
                    last_ts[mo] = max(ts, last_ts.get(mo, ts))
                    i += 1
        except ClickhouseError:
            pass
    return metric_map, last_ts
Ejemplo n.º 8
0
def get_objects_metrics(
    managed_objects: Union[Iterable, int]
) -> Tuple[Dict["ManagedObject", Dict[str, Dict[str, int]]], Dict[
        "ManagedObject", datetime.datetime]]:
    """

    :param managed_objects:
    :return: Dictionary ManagedObject -> Path -> MetricName -> value
    """
    if not isinstance(managed_objects, Iterable):
        managed_objects = [managed_objects]

    # Object Metrics
    bi_map = {str(getattr(mo, "bi_id", mo)): mo for mo in managed_objects}
    query_interval = (ManagedObjectProfile.get_max_metrics_interval(
        set(mo.object_profile.id
            for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))) *
                      2)
    from_date = datetime.datetime.now() - datetime.timedelta(
        seconds=max(query_interval, 3600))
    from_date = from_date.replace(microsecond=0)

    # @todo Left Join
    object_profiles = set(
        mo.object_profile.id
        for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))
    msd: Dict[str, str] = {}  # Map ScopeID -> TableName
    labels_table = set()
    for ms in MetricScope.objects.filter():
        msd[ms.id] = ms.table_name
        if ms.labels:
            labels_table.add(ms.table_name)
    mts: Dict[str, Tuple[str, str, str]] = {
        str(mt.id): (msd[mt.scope.id], mt.field_name, mt.name)
        for mt in MetricType.objects.all()
    }  # Map Metric Type ID -> table_name, column_name, MetricType Name
    mmm = set()
    op_fields_map: DefaultDict[str, List[str]] = defaultdict(list)
    for op in ManagedObjectProfile.objects.filter(id__in=object_profiles):
        if not op.metrics:
            continue
        for mt in op.metrics:
            mmm.add(mts[mt["metric_type"]])
            op_fields_map[op.id] += [mts[mt["metric_type"]][1]]

    ch = ch_connection()
    mtable = []  # mo_id, mac, iface, ts
    metric_map = {}
    last_ts: Dict["ManagedObject", datetime.datetime] = {}  # mo -> ts

    for table, fields in itertools.groupby(sorted(mmm, key=lambda x: x[0]),
                                           key=lambda x: x[0]):
        fields = list(fields)
        SQL = """SELECT managed_object, argMax(ts, ts), %%s %s
              FROM %s
              WHERE
                date >= toDate('%s')
                AND ts >= toDateTime('%s')
                AND managed_object IN (%s)
              GROUP BY managed_object %%s
              """ % (
            ", ".join(["argMax(%s, ts) as %s" % (f[1], f[1]) for f in fields]),
            table,
            from_date.date().isoformat(),
            from_date.isoformat(sep=" "),
            ", ".join(bi_map),
        )
        if table in labels_table:
            # SQL = SQL % ("arrayStringConcat(labels, '|') as ll,", ", labels")
            SQL = SQL % (
                "arrayStringConcat(arrayMap(x -> splitByString('::', x)[-1], labels), '|') as labels,",
                ", labels",
            )
        else:
            SQL = SQL % ("", "")
        try:
            for result in ch.execute(post=SQL):
                if table in labels_table:
                    mo_bi_id, ts, labels = result[:3]
                    result = result[3:]
                else:
                    mo_bi_id, ts = result[:2]
                    labels, result = "", result[2:]
                mo = bi_map.get(mo_bi_id)
                i = 0
                for r in result:
                    f_name = fields[i][2]
                    mtable += [[mo, ts, labels, r]]
                    if mo not in metric_map:
                        metric_map[mo] = defaultdict(dict)
                    metric_map[mo][labels][f_name] = r
                    last_ts[mo] = max(ts, last_ts.get(mo, ts))
                    i += 1
        except ClickhouseError as e:
            print(e)
    return metric_map, last_ts
Ejemplo n.º 9
0
def get_interface_metrics(
    managed_objects: Union[Iterable, int],
    meric_map: Optional[Dict[str, Any]] = None
) -> Tuple[Dict["ManagedObject", Dict[str, Dict[str, Union[float, int]]]],
           Dict["ManagedObject", datetime.datetime], ]:
    """

    :param managed_objects: ManagedObject list or bi_id list
    :param meric_map: For customization getting metrics
    :return: Dictionary ManagedObject -> Path -> MetricName -> value
    """

    # mo = self.object
    if not meric_map:
        meric_map = {
            "table_name": "interface",
            "map": {
                "load_in": "Interface | Load | In",
                "load_out": "Interface | Load | Out",
                "errors_in": "Interface | Errors | In",
                "errors_out": "Interface | Errors | Out",
            },
        }
    if not isinstance(managed_objects, Iterable):
        managed_objects = [managed_objects]
    bi_map: Dict[str, "ManagedObject"] = {
        str(getattr(mo, "bi_id", mo)): mo
        for mo in managed_objects
    }
    query_interval: float = (ManagedObjectProfile.get_max_metrics_interval(
        set(mo.object_profile.id
            for mo in ManagedObject.objects.filter(bi_id__in=list(bi_map)))) *
                             1.5)
    from_date = datetime.datetime.now() - datetime.timedelta(
        seconds=max(query_interval, 3600))
    from_date = from_date.replace(microsecond=0)
    SQL = """SELECT managed_object, argMax(ts, ts),  splitByString('::', arrayFirst(x -> startsWith(x, 'noc::interface::'), labels))[-1] as iface, labels, %s
            FROM %s
            WHERE
              date >= toDate('%s')
              AND ts >= toDateTime('%s')
              AND managed_object IN (%s)
              AND NOT arrayExists(x -> startsWith(x, 'noc::unit::'), labels)
            GROUP BY managed_object, labels
            """ % (
        ", ".join(
            ["argMax(%s, ts) as %s" % (f, f)
             for f in meric_map["map"].keys()]),
        meric_map["table_name"],
        from_date.date().isoformat(),
        from_date.isoformat(sep=" "),
        ", ".join(bi_map),
    )
    ch = ch_connection()
    metric_map: DefaultDict["ManagedObject",
                            Dict[str, Dict[str,
                                           Union[int,
                                                 float]]]] = defaultdict(dict)
    last_ts: Dict["ManagedObject", datetime.datetime] = {}  # mo -> ts
    metric_fields = list(meric_map["map"].keys())
    try:
        for result in ch.execute(post=SQL):
            mo_bi_id, ts, iface, labels = result[:4]
            labels = ast.literal_eval(labels)
            res = dict(zip(metric_fields, result[4:]))
            mo = bi_map.get(mo_bi_id)
            if len(labels) == 1 and metric_map[mo].get(iface):
                # If only interface metric
                continue
            metric_map[mo][iface] = defaultdict(dict)
            for field, value in res.items():
                metric_map[mo][iface][meric_map["map"].get(field)] = (
                    float(value) if is_float(value) else int(value))
                last_ts[mo] = max(ts, last_ts.get(mo, ts))
    except ClickhouseError:
        pass
    return metric_map, last_ts