def get_annotated_graph(self,
graph,
ts_from,
ts_to,
utilization=True,
saturation=True):
internal_graph = graph.copy()
i = 0
threads = []
cpu_count = multiprocessing.cpu_count()
no_node_thread = len(internal_graph.nodes()) / (cpu_count)
node_pool = []
node_pools = []
for node in internal_graph.nodes(data=True):
if i < no_node_thread:
node_pool.append(node)
i = i + 1
else:
thread1 = ParallelTelemetryAnnotation(
i, "Thread-{}".format(InfoGraphNode.get_name(node)), i,
node_pool, internal_graph, self.telemetry, ts_to, ts_from)
threads.append(thread1)
node_pools.append(node_pool)
i = 1
node_pool = [node]
if len(node_pool) != 0:
node_pools.append(node_pool)
thread1 = ParallelTelemetryAnnotation(
i, "Thread-{}".format(InfoGraphNode.get_name(node)), i,
node_pool, internal_graph, self.telemetry, ts_to, ts_from)
threads.append(thread1)
[t.start() for t in threads]
[t.join() for t in threads]
for node in internal_graph.nodes(data=True):
if InfoGraphNode.get_type(node) == InfoGraphNodeType.PHYSICAL_PU:
self.utils.annotate_machine_pu_util(internal_graph, node)
elif InfoGraphNode.node_is_disk(node):
self.utils.annotate_machine_disk_util(internal_graph, node)
elif InfoGraphNode.node_is_nic(node):
self.utils.annotate_machine_network_util(internal_graph, node)
return internal_graph
def get_annotated_graph(self,
graph,
ts_from,
ts_to,
utilization=False,
saturation=False):
"""
Collect data from cimmaron tsdb in relation to the specified graph and
time windows and store an annotated subgraph in specified directory
:param graph: (NetworkX Graph) Graph to be annotated with data
:param ts_from: (str) Epoch time representation of start time
:param ts_to: (str) Epoch time representation of stop time
:param utilization: (bool) if True the method calculates also
utilization for each node, if available
:return: NetworkX Graph annotated with telemetry data
"""
TelemetryAnnotation._get_annotated_graph_input_validation(
graph, ts_from, ts_to)
internal_graph = graph.copy()
self.internal_graph = internal_graph
for node in internal_graph.nodes(data=True):
if isinstance(self.telemetry, SnapAnnotation):
queries = list()
try:
queries = self.telemetry.get_queries(
internal_graph, node, ts_from, ts_to)
# queries = self.telemetry.get_queries(graph, node, ts_from, ts_to)
except Exception as e:
LOG.error("Exception: {}".format(e))
LOG.error(e)
import traceback
traceback.print_exc()
if len(queries) != 0:
InfoGraphNode.set_queries(node, queries)
telemetry_data = self.telemetry.get_data(node)
InfoGraphNode.set_telemetry_data(node, telemetry_data)
if utilization and not telemetry_data.empty:
SnapUtils.utilization(internal_graph, node,
self.telemetry)
# if only procfs is available, results needs to be
# propagated at machine level
if InfoGraphNode.get_type(
node) == InfoGraphNodeType.PHYSICAL_PU:
SnapUtils.annotate_machine_pu_util(
internal_graph, node)
if InfoGraphNode.node_is_disk(node):
SnapUtils.annotate_machine_disk_util(
internal_graph, node)
if InfoGraphNode.node_is_nic(node):
SnapUtils.annotate_machine_network_util(
internal_graph, node)
if saturation:
SnapUtils.saturation(internal_graph, node,
self.telemetry)
elif isinstance(self.telemetry, PrometheusAnnotation):
queries = list()
try:
queries = self.telemetry.get_queries(
internal_graph, node, ts_from, ts_to)
# queries = self.telemetry.get_queries(graph, node, ts_from, ts_to)
except Exception as e:
LOG.error("Exception: {}".format(e))
LOG.error(e)
import traceback
traceback.print_exc()
if len(queries) != 0:
InfoGraphNode.set_queries(node, queries)
telemetry_data = self.telemetry.get_data(node)
InfoGraphNode.set_telemetry_data(node, telemetry_data)
# if utilization and not telemetry_data.empty:
#PrometheusUtils.utilization(internal_graph, node, self.telemetry)
# if only procfs is available, results needs to be
# propagated at machine level
#if InfoGraphNode.get_type(node) == InfoGraphNodeType.PHYSICAL_PU:
# PrometheusUtils.annotate_machine_pu_util(internal_graph, node)
#if InfoGraphNode.node_is_disk(node):
# PrometheusUtils.annotate_machine_disk_util(internal_graph, node)
#if InfoGraphNode.node_is_nic(node):
# PrometheusUtils.annotate_machine_network_util(internal_graph, node)
#if saturation:
#PrometheusUtils.saturation(internal_graph, node, self.telemetry)
else:
telemetry_data = self.telemetry.get_data(node)
InfoGraphNode.set_telemetry_data(node, telemetry_data)
if utilization and not telemetry_data.empty:
SnapUtils.utilization(internal_graph, node, self.telemetry)
# if only procfs is available, results needs to be
# propagated at machine level
if InfoGraphNode.get_type(
node) == InfoGraphNodeType.PHYSICAL_PU:
source = InfoGraphNode.get_machine_name_of_pu(node)
machine = InfoGraphNode.get_node(
internal_graph, source)
machine_util = InfoGraphNode.get_compute_utilization(
machine)
if '/intel/use/compute/utilization' not in machine_util.columns:
sum_util = None
pu_util = InfoGraphNode.get_compute_utilization(
node
)['intel/procfs/cpu/utilization_percentage']
pu_util = pu_util.fillna(0)
if 'intel/procfs/cpu/utilization_percentage' in machine_util.columns:
machine_util = machine_util[
'intel/procfs/cpu/utilization_percentage']
machine_util = machine_util.fillna(0)
sum_util = machine_util.add(pu_util,
fill_value=0)
else:
sum_util = pu_util
if isinstance(sum_util, pandas.Series):
# sum_util.index.name = None
sum_util = pandas.DataFrame(
sum_util,
columns=[
'intel/procfs/cpu/utilization_percentage'
])
InfoGraphNode.set_compute_utilization(
machine, sum_util)
else:
LOG.debug('Found use for node {}'.format(
InfoGraphNode.get_name(node)))
if saturation:
self._saturation(internal_graph, node, self.telemetry)
return internal_graph
