예제 #1
0
    def get_annotated_graph(self,
                            graph,
                            ts_from,
                            ts_to,
                            utilization=True,
                            saturation=True):
        internal_graph = graph.copy()
        i = 0
        threads = []
        cpu_count = multiprocessing.cpu_count()
        no_node_thread = len(internal_graph.nodes()) / (cpu_count)
        node_pool = []
        node_pools = []
        for node in internal_graph.nodes(data=True):
            if i < no_node_thread:
                node_pool.append(node)
                i = i + 1
            else:
                thread1 = ParallelTelemetryAnnotation(
                    i, "Thread-{}".format(InfoGraphNode.get_name(node)), i,
                    node_pool, internal_graph, self.telemetry, ts_to, ts_from)
                threads.append(thread1)
                node_pools.append(node_pool)
                i = 1
                node_pool = [node]
        if len(node_pool) != 0:
            node_pools.append(node_pool)
            thread1 = ParallelTelemetryAnnotation(
                i, "Thread-{}".format(InfoGraphNode.get_name(node)), i,
                node_pool, internal_graph, self.telemetry, ts_to, ts_from)
            threads.append(thread1)

        [t.start() for t in threads]
        [t.join() for t in threads]

        for node in internal_graph.nodes(data=True):
            if InfoGraphNode.get_type(node) == InfoGraphNodeType.PHYSICAL_PU:
                self.utils.annotate_machine_pu_util(internal_graph, node)
            elif InfoGraphNode.node_is_disk(node):
                self.utils.annotate_machine_disk_util(internal_graph, node)
            elif InfoGraphNode.node_is_nic(node):
                self.utils.annotate_machine_network_util(internal_graph, node)
        return internal_graph
예제 #2
0
    def get_annotated_graph(self,
                            graph,
                            ts_from,
                            ts_to,
                            utilization=False,
                            saturation=False):
        """
        Collect data from cimmaron tsdb in relation to the specified graph and
         time windows and store an annotated subgraph in specified directory

        :param graph: (NetworkX Graph) Graph to be annotated with data
        :param ts_from: (str) Epoch time representation of start time
        :param ts_to: (str) Epoch time representation of stop time
        :param utilization: (bool) if True the method calculates also
                                    utilization for each node, if available
        :return: NetworkX Graph annotated with telemetry data
        """
        TelemetryAnnotation._get_annotated_graph_input_validation(
            graph, ts_from, ts_to)
        internal_graph = graph.copy()
        self.internal_graph = internal_graph
        for node in internal_graph.nodes(data=True):
            if isinstance(self.telemetry, SnapAnnotation):
                queries = list()
                try:
                    queries = self.telemetry.get_queries(
                        internal_graph, node, ts_from, ts_to)
                    # queries = self.telemetry.get_queries(graph, node, ts_from, ts_to)
                except Exception as e:
                    LOG.error("Exception: {}".format(e))
                    LOG.error(e)
                    import traceback
                    traceback.print_exc()
                if len(queries) != 0:
                    InfoGraphNode.set_queries(node, queries)

                    telemetry_data = self.telemetry.get_data(node)
                    InfoGraphNode.set_telemetry_data(node, telemetry_data)
                    if utilization and not telemetry_data.empty:
                        SnapUtils.utilization(internal_graph, node,
                                              self.telemetry)
                        # if only procfs is available, results needs to be
                        # propagated at machine level
                        if InfoGraphNode.get_type(
                                node) == InfoGraphNodeType.PHYSICAL_PU:
                            SnapUtils.annotate_machine_pu_util(
                                internal_graph, node)
                        if InfoGraphNode.node_is_disk(node):
                            SnapUtils.annotate_machine_disk_util(
                                internal_graph, node)
                        if InfoGraphNode.node_is_nic(node):
                            SnapUtils.annotate_machine_network_util(
                                internal_graph, node)
                    if saturation:
                        SnapUtils.saturation(internal_graph, node,
                                             self.telemetry)
            elif isinstance(self.telemetry, PrometheusAnnotation):
                queries = list()
                try:
                    queries = self.telemetry.get_queries(
                        internal_graph, node, ts_from, ts_to)
                    # queries = self.telemetry.get_queries(graph, node, ts_from, ts_to)
                except Exception as e:
                    LOG.error("Exception: {}".format(e))
                    LOG.error(e)
                    import traceback
                    traceback.print_exc()
                if len(queries) != 0:
                    InfoGraphNode.set_queries(node, queries)

                    telemetry_data = self.telemetry.get_data(node)
                    InfoGraphNode.set_telemetry_data(node, telemetry_data)
                    # if utilization and not telemetry_data.empty:
                    #PrometheusUtils.utilization(internal_graph, node, self.telemetry)
                    # if only procfs is available, results needs to be
                    # propagated at machine level
                    #if InfoGraphNode.get_type(node) == InfoGraphNodeType.PHYSICAL_PU:
                    #    PrometheusUtils.annotate_machine_pu_util(internal_graph, node)
                    #if InfoGraphNode.node_is_disk(node):
                    #    PrometheusUtils.annotate_machine_disk_util(internal_graph, node)
                    #if InfoGraphNode.node_is_nic(node):
                    #    PrometheusUtils.annotate_machine_network_util(internal_graph, node)
                    #if saturation:
                    #PrometheusUtils.saturation(internal_graph, node, self.telemetry)
            else:
                telemetry_data = self.telemetry.get_data(node)
                InfoGraphNode.set_telemetry_data(node, telemetry_data)
                if utilization and not telemetry_data.empty:
                    SnapUtils.utilization(internal_graph, node, self.telemetry)
                    # if only procfs is available, results needs to be
                    # propagated at machine level
                    if InfoGraphNode.get_type(
                            node) == InfoGraphNodeType.PHYSICAL_PU:
                        source = InfoGraphNode.get_machine_name_of_pu(node)
                        machine = InfoGraphNode.get_node(
                            internal_graph, source)
                        machine_util = InfoGraphNode.get_compute_utilization(
                            machine)
                        if '/intel/use/compute/utilization' not in machine_util.columns:
                            sum_util = None
                            pu_util = InfoGraphNode.get_compute_utilization(
                                node
                            )['intel/procfs/cpu/utilization_percentage']
                            pu_util = pu_util.fillna(0)
                            if 'intel/procfs/cpu/utilization_percentage' in machine_util.columns:

                                machine_util = machine_util[
                                    'intel/procfs/cpu/utilization_percentage']
                                machine_util = machine_util.fillna(0)
                                sum_util = machine_util.add(pu_util,
                                                            fill_value=0)
                            else:
                                sum_util = pu_util
                            if isinstance(sum_util, pandas.Series):
                                # sum_util.index.name = None
                                sum_util = pandas.DataFrame(
                                    sum_util,
                                    columns=[
                                        'intel/procfs/cpu/utilization_percentage'
                                    ])
                            InfoGraphNode.set_compute_utilization(
                                machine, sum_util)
                        else:
                            LOG.debug('Found use for node {}'.format(
                                InfoGraphNode.get_name(node)))
                if saturation:
                    self._saturation(internal_graph, node, self.telemetry)
        return internal_graph