def _tag_value(self, tag_key, node, metric):
# TODO: fully qualify this with metric name, if metric is this and tag
tag_value = None
if tag_key == "source":
tag_value = self._source(node)
elif tag_key in set(["device_id", "disk", "device_name"]):
tag_value = self._disk(node)
elif tag_key in set(["cpu_id", "cpuID", "core_id"]):
tag_value = self._pu(node, metric)
elif tag_key in set([
"nic_id", "interface", "network_interface", "interface_name",
"hardware_addr"
]):
tag_value = self._nic(node, tag_key)
elif tag_key == "nova_uuid":
tag_value = self._nova_uuid(node)
elif tag_key == "stack_name":
tag_value = self._stack(node)
elif tag_key == "dev_id":
if "intel/use/network" in metric:
tag_value = self._nic(node)
elif "intel/use/disk" in metric:
tag_value = self._disk(node)
elif tag_key == "docker_id":
tag_value = InfoGraphNode.get_docker_id(node)
return tag_value
def _source_metrics(self, node):
"""
Retrieves metrics associated with a source/host. The source is
identified by the node and then all metrics types are collected for
that source. If the node is physical then the metric types are
retrieved using just the machine name as the source, if the node is
virtual then the source (the vm hostname) and the stack name are
required.
"""
metric_types = []
node_layer = InfoGraphNode.get_layer(node)
node_type = InfoGraphNode.get_type(node)
if node_layer == GRAPH_LAYER.PHYSICAL \
or node_type == NODE_TYPE.INSTANCE_DISK:
try:
source = self._source(node)
identifier = source
query_tags = {"source": source}
metric_types = self._cached_metrics(identifier, query_tags)
except Exception as ex:
LOG.error('Malformed graph: {}'.format(
InfoGraphNode.get_name(node)))
LOG.error(ex)
elif node_layer == GRAPH_LAYER.VIRTUAL:
source = self._source(node)
stack = self._stack(node)
#LOG.info("SOURCE: {}".format(source))
#LOG.info("STACK: {}".format(stack))
if stack is not None:
identifier = "{}-{}".format(source, stack)
# query_tags = {"source": source, "stack": stack}
query_tags = {"stack_name": stack}
metric_types = self._cached_metrics(identifier, query_tags)
elif node_type == NODE_TYPE.DOCKER_CONTAINER:
source = self._source(node)
docker_id = InfoGraphNode.get_docker_id(node)
if docker_id is not None and source is not None:
identifier = "{}-{}".format(source, docker_id)
query_tags = {"docker_id": docker_id, "source": source}
metric_types = self._cached_metrics(identifier, query_tags)
return metric_types
def run(self, workload, optimal_node_type='machine'):
"""
Ranks machines by CPU utilization.
:param workload: Contains workload related info and results.
:return: heuristic results
"""
workload_config = workload.get_configuration()
graph = workload.get_latest_graph()
if not graph:
raise KeyError('No graph to be processed.')
scores = LandscapeScore.utilization_scores(graph)
scores_sat = LandscapeScore.saturation_scores(graph)
heuristic_results = pd.DataFrame(columns=[
'node_name',
'type',
'ipaddress',
'compute utilization',
'compute saturation',
'memory utilization',
'memory saturation',
'network utilization',
'network saturation',
'disk utilization',
'disk saturation',
])
heuristic_results_nt = heuristic_results.copy()
device_id_col_name = None
project = None
if workload_config.get('project'):
project = workload_config['project']
device_id_col_name = workload_config['project'] + '_device_id'
heuristic_results[device_id_col_name] = None
telemetry_filter = workload_config.get('telemetry_filter')
for node in graph.nodes(data=True):
node_name = InfoGraphNode.get_name(node)
node_type = InfoGraphNode.get_type(node)
list_node_name = node_name
if node_type == optimal_node_type:
if InfoGraphNode.node_is_vm(node):
vm_name = InfoGraphNode.get_properties(node).get('vm_name')
if vm_name:
list_node_name = vm_name
data = {
'node_name':
list_node_name,
'type':
node_type,
'ipaddress':
InfoGraphNode.get_attributes(node).get('ipaddress'),
'compute utilization':
scores[node_name]['compute'],
'compute saturation':
scores_sat[node_name]['compute'],
'memory utilization':
scores[node_name]['memory'],
'memory saturation':
scores_sat[node_name]['memory'],
'network utilization':
scores[node_name]['network'],
'network saturation':
scores_sat[node_name]['network'],
'disk utilization':
scores[node_name]['disk'],
'disk saturation':
scores_sat[node_name]['disk']
}
if device_id_col_name:
dev_id = InfoGraphNode.get_properties(node).get(
device_id_col_name)
if project == 'mf2c':
dev_id = dev_id.replace('_', '-')
data[device_id_col_name] = dev_id
if InfoGraphNode.get_properties(node).get(
"telemetry_data") is not None:
heuristic_results = heuristic_results.append(
data, ignore_index=True)
elif not telemetry_filter:
heuristic_results_nt = heuristic_results.append(
data, ignore_index=True)
if not workload.get_workload_name().startswith('optimal_'):
if InfoGraphNode.get_type(
node
) == "docker_container" and optimal_node_type == 'machine':
node_name = InfoGraphNode.get_docker_id(node)
heuristic_results = heuristic_results.append(
{
'node_name': node_name,
'type': node_type,
'ipaddress': None,
'compute utilization':
scores[node_name]['compute'],
'compute saturation': None,
'memory utilization': scores[node_name]['memory'],
'memory saturation': None,
'network utilization':
scores[node_name]['network'],
'network saturation': None,
'disk utilization': scores[node_name]['disk'],
'disk saturation': None
},
ignore_index=True)
sort_fields = ['compute utilization']
sort_order = workload_config.get('sort_order')
if sort_order:
sort_fields = []
for val in sort_order:
if val == 'cpu':
sort_fields.append('compute utilization')
if val == 'memory':
sort_fields.append('memory utilization')
if val == 'network':
sort_fields.append('network utilization')
if val == 'disk':
sort_fields.append('disk utilization')
heuristic_results_nt = heuristic_results_nt.replace([0], [None])
heuristic_results = heuristic_results.sort_values(by=sort_fields,
ascending=True)
heuristic_results = heuristic_results.append(heuristic_results_nt,
ignore_index=True)
workload.append_metadata(self.__filter_name__, heuristic_results)
LOG.info('AVG: {}'.format(heuristic_results))
return heuristic_results
def utilization_scores(graph):
"""
Returns a dictionary with the scores of
all the nodes of the graph.
:param graph: InfoGraph
:return: dict[node_name] = score
"""
res = dict()
for node in graph.nodes(data=True):
node_name = InfoGraphNode.get_name(node)
res[node_name] = dict()
util = InfoGraphNode.get_utilization(node)
import analytics_engine.common as common
LOG = common.LOG
res[node_name]['compute'] = 0
res[node_name]['disk'] = 0
res[node_name]['network'] = 0
res[node_name]['memory'] = 0
if (isinstance(util, pandas.DataFrame) and
util.empty) or \
(not isinstance(util, pandas.DataFrame) and
util==None):
continue
# intel/use/
if 'intel/use/compute/utilization' in util:
res[node_name]['compute'] = (
util.get('intel/use/compute/utilization').mean()) / 100.0
elif 'intel/procfs/cpu/utilization_percentage' in util:
res[node_name]['compute'] = (util.get(
'intel/procfs/cpu/utilization_percentage').mean()) / 100.0
if 'intel/use/memory/utilization' in util:
res[node_name]['memory'] = (
util.get('intel/use/memory/utilization').mean()) / 100.0
elif 'intel/procfs/memory/utilization_percentage' in util:
res[node_name]['memory'] = (
util.get('intel/procfs/memory/utilization_percentage'
).mean()) / 100.0
if 'intel/use/disk/utilization' in util:
res[node_name]['disk'] = (
util.get('intel/use/disk/utilization').mean()) / 100.0
elif 'intel/procfs/disk/utilization_percentage' in util:
res[node_name]['disk'] = (util.get(
'intel/procfs/disk/utilization_percentage').mean()) / 100.0
if 'intel/use/network/utilization' in util:
res[node_name]['network'] = (
util.get('intel/use/network/utilization').mean()) / 100.0
elif 'intel/psutil/net/utilization_percentage' in util:
res[node_name]['network'] = (util.get(
'intel/psutil/net/utilization_percentage').mean()) / 100.0
# special handling of cpu, disk & network utilization if node is a machine
if InfoGraphNode.node_is_machine(node):
# mean from all cpu columns
cpu_util = InfoGraphNode.get_compute_utilization(node)
cpu_util['total'] = [
sum(row) / len(row) for index, row in cpu_util.iterrows()
]
res[node_name]['compute'] = cpu_util['total'].mean() / 100
# mean from all disk columns
disk_util = InfoGraphNode.get_disk_utilization(node)
if disk_util.empty:
res[node_name]['disk'] = 0.0
else:
disk_util['total'] = [
sum(row) / len(row)
for index, row in disk_util.iterrows()
]
res[node_name]['disk'] = disk_util['total'].mean() / 100
# mean from all nic columns
net_util = InfoGraphNode.get_network_utilization(node)
if net_util.empty:
res[node_name]['network'] = 0.0
else:
net_util['total'] = [
sum(row) / len(row)
for index, row in net_util.iterrows()
]
res[node_name]['network'] = net_util['total'].mean() / 100
# custom metric
if InfoGraphNode.get_type(
node) == InfoGraphNodeType.DOCKER_CONTAINER:
node_name = InfoGraphNode.get_docker_id(node)
res[node_name] = {}
if 'intel/docker/stats/cgroups/cpu_stats/cpu_usage/percentage' in util.columns:
res[node_name]['compute'] = util[
'intel/docker/stats/cgroups/cpu_stats/cpu_usage/percentage'].mean(
) / 100
else:
res[node_name]['compute'] = 0
if 'intel/docker/stats/cgroups/memory_stats/usage/percentage' in util.columns:
res[node_name]['memory'] = util[
'intel/docker/stats/cgroups/memory_stats/usage/percentage'].mean(
) / 100
else:
res[node_name]['memory'] = 0
if 'intel/docker/stats/network/utilization_percentage' in util.columns:
res[node_name]['network'] = util[
'intel/docker/stats/network/utilization_percentage'].mean(
) / 100
else:
res[node_name]['network'] = 0
if 'intel/docker/stats/cgroups/blkio_stats/io_time_recursive/percentage' in util.columns:
res[node_name]['disk'] = util[
'intel/docker/stats/cgroups/blkio_stats/io_time_recursive/percentage'].mean(
) / 100
else:
res[node_name]['disk'] = 0
return res
