def _container_in_pod(gs, container, pod):
"""Returns True when 'container' is a part of 'pod'.
Args:
gs: global state.
container: a wrapped container object.
pod: a wrapped pod object.
Raises:
CollectorError: if the 'container' or the 'pod' are missing essential
attributes.
Returns:
True iff container 'container' is a part of 'pod'.
"""
assert isinstance(gs, global_state.GlobalState)
assert utilities.is_wrapped_object(container, 'Container')
assert utilities.is_wrapped_object(pod, 'Pod')
parent_pod_id = utilities.get_parent_pod_id(container)
if not utilities.valid_string(parent_pod_id):
msg = 'could not find parent pod ID in container %s' % container['id']
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
return parent_pod_id == pod['id']
def annotate_container(project_id, container, parent_pod):
"""Annotate the given container with Heapster GCM metric information.
Args:
project_id: the project ID
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Raises:
AssertionError: if the input arguments are invalid or if
'parent_pod' is not the parent of 'container'
"""
assert utilities.valid_string(project_id)
assert utilities.is_wrapped_object(container, "Container")
assert utilities.is_wrapped_object(parent_pod, "Pod")
parent_name = utilities.get_attribute(container, ["properties", "Config", "Hostname"])
assert utilities.valid_string(parent_name)
pod_name = utilities.get_attribute(parent_pod, ["properties", "metadata", "name"])
assert utilities.valid_string(pod_name)
# The 'parent_name' value is truncated to the first 64 characters.
# Thus it must be the prefix of the full pod name.
assert pod_name.startswith(parent_name)
m = _make_gcm_metrics(project_id, _get_container_labels(container, parent_pod))
if m is None:
return
if container.get("annotations") is None:
container["annotations"] = {}
container["annotations"]["metrics"] = m
def annotate_container(project_id, container, parent_pod):
"""Annotate the given container with Heapster GCM metric information.
Args:
project_id: the project ID
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Raises:
AssertionError: if the input arguments are invalid or if
'parent_pod' is not the parent of 'container'
"""
assert utilities.valid_string(project_id)
assert utilities.is_wrapped_object(container, 'Container')
assert utilities.is_wrapped_object(parent_pod, 'Pod')
parent_name = utilities.get_attribute(container,
['properties', 'Config', 'Hostname'])
assert utilities.valid_string(parent_name)
pod_name = utilities.get_attribute(parent_pod,
['properties', 'metadata', 'name'])
assert utilities.valid_string(pod_name)
# The 'parent_name' value is truncated to the first 64 characters.
# Thus it must be the prefix of the full pod name.
assert pod_name.startswith(parent_name)
m = _make_gcm_metrics(project_id,
_get_container_labels(container, parent_pod))
if m is None:
return
if container.get('annotations') is None:
container['annotations'] = {}
container['annotations']['metrics'] = m
def _get_container_labels(container, parent_pod):
"""Returns key/value pairs identifying all metrics of this container.
Args:
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Returns:
A dictionary of key/value pairs.
If any error was detected, returns None.
"""
if not utilities.is_wrapped_object(container, "Container"):
return None
if not utilities.is_wrapped_object(parent_pod, "Pod"):
return None
pod_id = utilities.get_attribute(parent_pod, ["properties", "metadata", "uid"])
if not utilities.valid_string(pod_id):
return None
hostname = utilities.get_attribute(parent_pod, ["properties", "spec", "host"])
if not utilities.valid_string(hostname):
return None
short_container_name = utilities.get_short_container_name(container, parent_pod)
if not utilities.valid_string(short_container_name):
return None
return {"pod_id": pod_id, "hostname": hostname, "container_name": short_container_name}
def _container_in_pod(gs, container, pod):
"""Returns True when 'container' is a part of 'pod'.
Args:
gs: global state.
container: a wrapped container object.
pod: a wrapped pod object.
Raises:
CollectorError: if the 'container' or the 'pod' are missing essential
attributes.
Returns:
True iff container 'container' is a part of 'pod'.
"""
assert isinstance(gs, global_state.GlobalState)
assert utilities.is_wrapped_object(container, 'Container')
assert utilities.is_wrapped_object(pod, 'Pod')
parent_pod_id = utilities.get_parent_pod_id(container)
if not utilities.valid_string(parent_pod_id):
msg = 'could not find parent pod ID in container %s' % container['id']
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
return parent_pod_id == pod['id']
def _do_compute_other_nodes(gs, cluster_guid, nodes_list, oldest_timestamp, g):
"""Adds nodes not in the node list but running pods to the graph.
This handles the case when there are pods running on the master node,
in which case we add a dummy node representing the master to the graph.
The nodes list does not include the master.
Args:
gs: the global state.
cluster_guid: the cluster's ID.
nodes_list: a list of wrapped Node objects.
oldest_timestamp: the timestamp of the oldest Node object.
g: the context graph under construction.
"""
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(cluster_guid)
assert isinstance(nodes_list, list)
assert utilities.valid_string(oldest_timestamp)
assert isinstance(g, ContextGraph)
# Compute the set of known Node names.
known_node_ids = set()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
known_node_ids.add(node['id'])
# Compute the set of Nodes referenced by pods but not in the known set.
# The set of unknown node names may be empty.
missing_node_ids = set()
for pod in kubernetes.get_pods(gs):
assert utilities.is_wrapped_object(pod, 'Pod')
# pod.properties.spec.nodeName may be missing if the pod is waiting.
parent_node_id = utilities.get_attribute(
pod, ['properties', 'spec', 'nodeName'])
if not utilities.valid_string(parent_node_id):
continue
if parent_node_id in known_node_ids:
continue
# Found a pod that does not belong to any of the known nodes.
missing_node_ids.add(parent_node_id)
# Process the missing nodes.
for node_id in missing_node_ids:
# Create a dummy node object just as a placeholder for metric
# annotations.
node = utilities.wrap_object({}, 'Node', node_id, time.time())
metrics.annotate_node(node)
node_guid = 'Node:' + node_id
g.add_resource(node_guid, node['annotations'], 'Node', oldest_timestamp, {})
g.add_relation(cluster_guid, node_guid, 'contains') # Cluster contains Node
def test_is_wrapped_object(self):
"""Tests is_wrapped_object()."""
self.assertTrue(utilities.is_wrapped_object(CONTAINER, 'Container'))
self.assertTrue(utilities.is_wrapped_object(CONTAINER))
self.assertFalse(utilities.is_wrapped_object(CONTAINER, 'Pod'))
self.assertTrue(utilities.is_wrapped_object(PARENT_POD, 'Pod'))
self.assertTrue(utilities.is_wrapped_object(PARENT_POD))
self.assertFalse(utilities.is_wrapped_object(PARENT_POD, 'Container'))
self.assertFalse(utilities.is_wrapped_object({}))
self.assertFalse(utilities.is_wrapped_object({}, 'Pod'))
self.assertFalse(utilities.is_wrapped_object(None))
self.assertFalse(utilities.is_wrapped_object('hello, world'))
def test_is_wrapped_object(self):
"""Tests is_wrapped_object()."""
self.assertTrue(utilities.is_wrapped_object(CONTAINER, 'Container'))
self.assertTrue(utilities.is_wrapped_object(CONTAINER))
self.assertFalse(utilities.is_wrapped_object(CONTAINER, 'Pod'))
self.assertTrue(utilities.is_wrapped_object(PARENT_POD, 'Pod'))
self.assertTrue(utilities.is_wrapped_object(PARENT_POD))
self.assertFalse(utilities.is_wrapped_object(PARENT_POD, 'Container'))
self.assertFalse(utilities.is_wrapped_object({}))
self.assertFalse(utilities.is_wrapped_object({}, 'Pod'))
self.assertFalse(utilities.is_wrapped_object(None))
self.assertFalse(utilities.is_wrapped_object('hello, world'))
def _do_compute_pod(gs, cluster_guid, node_guid, pod, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(cluster_guid)
assert utilities.valid_string(node_guid)
assert utilities.is_wrapped_object(pod, 'Pod')
assert isinstance(g, ContextGraph)
pod_id = pod['id']
pod_guid = 'Pod:' + pod_id
g.add_resource(pod_guid, pod['annotations'], 'Pod', pod['timestamp'],
pod['properties'])
# pod.properties.spec.nodeName may be missing if the pod is waiting
# (not running yet).
docker_host = utilities.get_attribute(
pod, ['properties', 'spec', 'nodeName'])
if utilities.valid_string(docker_host):
# Pod is running.
if node_guid == ('Node:' + docker_host):
g.add_relation(node_guid, pod_guid, 'runs') # Node runs Pod
else:
msg = ('Docker host (pod.properties.spec.nodeName)=%s '
'not matching node ID=%s' % (docker_host, node_guid))
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
else:
# Pod is not running.
g.add_relation(cluster_guid, pod_guid, 'contains') # Cluster contains Pod
def _do_compute_service(gs, cluster_guid, service, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(service, 'Service')
assert isinstance(g, ContextGraph)
service_id = service['id']
service_guid = 'Service:' + service_id
g.add_resource(service_guid, service['annotations'], 'Service',
service['timestamp'], service['properties'])
# Cluster contains Service.
g.add_relation(cluster_guid, service_guid, 'contains')
# Pods load balanced by this service (use the service['spec', 'selector']
# key/value pairs to find matching Pods)
selector = utilities.get_attribute(
service, ['properties', 'spec', 'selector'])
if selector:
if not isinstance(selector, types.DictType):
msg = 'Service id=%s has an invalid "selector" value' % service_id
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
for pod in kubernetes.get_selected_pods(gs, selector):
pod_guid = 'Pod:' + pod['id']
# Service loadBalances Pod
g.add_relation(service_guid, pod_guid, 'loadBalances')
else:
gs.logger_error('Service id=%s has no "selector" attribute', service_id)
def _do_compute_rcontroller(gs, cluster_guid, rcontroller, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(rcontroller, 'ReplicationController')
assert isinstance(g, ContextGraph)
rcontroller_id = rcontroller['id']
rcontroller_guid = 'ReplicationController:' + rcontroller_id
g.add_resource(rcontroller_guid, rcontroller['annotations'],
'ReplicationController',
rcontroller['timestamp'], rcontroller['properties'])
# Cluster contains Rcontroller
g.add_relation(cluster_guid, rcontroller_guid, 'contains')
# Pods that are monitored by this replication controller.
# Use the rcontroller['spec']['selector'] key/value pairs to find matching
# pods.
selector = utilities.get_attribute(
rcontroller, ['properties', 'spec', 'selector'])
if selector:
if not isinstance(selector, types.DictType):
msg = ('Rcontroller id=%s has an invalid "replicaSelector" value' %
rcontroller_id)
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
for pod in kubernetes.get_selected_pods(gs, selector):
pod_guid = 'Pod:' + pod['id']
# Rcontroller monitors Pod
g.add_relation(rcontroller_guid, pod_guid, 'monitors')
else:
gs.logger_error('Rcontroller id=%s has no "spec.selector" attribute',
rcontroller_id)
def get_nodes_with_metrics(gs):
"""Gets the list of all nodes in the current cluster with their metrics.
Args:
gs: global state.
Returns:
list of wrapped node objects.
Each element in the list is the result of
utilities.wrap_object(node, 'Node', ...)
Raises:
CollectorError in case of failure to fetch data from Kubernetes.
Other exceptions may be raised due to exectution errors.
"""
nodes_list = get_nodes(gs)
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
project_id = utilities.node_id_to_project_id(node['id'])
# The project_id may be '_unknown_'. This is not a big
# deal, since the aggregator knows the project ID.
metrics.annotate_node(project_id, node)
return nodes_list
def get_containers_from_pod(pod):
"""Extracts synthesized container resources from a pod.
Only containers for which status is available are included. (The pod may
still be pending.)
"""
assert utilities.is_wrapped_object(pod, "Pod")
specs = utilities.get_attribute(pod, ["properties", "spec", "containers"])
statuses = utilities.get_attribute(pod, ["properties", "status", "containerStatuses"])
timestamp = pod["timestamp"]
spec_dict = {}
for spec in specs or []:
spec = spec.copy()
spec_dict[spec.pop("name")] = spec
containers = []
for status in statuses or []:
status = status.copy()
name = status.pop("name")
unique_id = status.get("containerID", name)
obj = {"metadata": {"name": name}, "spec": spec_dict.get(name, {}), "status": status}
container = utilities.wrap_object(obj, "Container", unique_id, timestamp, label=name)
containers.append(container)
return containers
def _do_compute_container(parent_guid, container, g):
assert utilities.valid_string(parent_guid)
assert utilities.is_wrapped_object(container, 'Container')
assert isinstance(g, ContextGraph)
container_id = container['id']
container_guid = 'Container:' + container_id
# TODO(vasbala): container_id is too verbose?
g.add_resource(container_guid, container['annotations'],
'Container', container['timestamp'],
container['properties'])
# The parent Pod contains Container.
g.add_relation(parent_guid, container_guid, 'contains')
image = kubernetes.get_image_from_container(container)
image_guid = 'Image:' + image['id']
# Add the image to the graph only if we have not added it before.
#
# Different containers might reference the same image using different
# names. Unfortunately, only the first name encountered is recorded.
# TODO(rimey): Record the other names as well, and choose the primary
# name deterministically.
g.add_resource(image_guid, image['annotations'], 'Image',
image['timestamp'], image['properties'])
# Container createdFrom Image
g.add_relation(container_guid, image_guid, 'createdFrom')
def _do_compute_node(gs, input_queue, cluster_guid, node, g):
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(node, 'Node')
assert isinstance(g, ContextGraph)
node_id = node['id']
node_guid = 'Node:' + node_id
g.add_resource(node_guid, node['annotations'], 'Node', node['timestamp'],
node['properties'])
g.add_relation(cluster_guid, node_guid, 'contains') # Cluster contains Node
# Pods in a Node
# Do not compute the pods by worker threads in test mode because the order
# of the output will be different than the golden files due to the effects
# of queuing the work.
for pod in kubernetes.get_pods(gs, node_id):
if gs.get_testing():
_do_compute_pod(gs, input_queue, node_guid, pod, g)
else:
input_queue.put((
gs.get_random_priority(),
_do_compute_pod,
{'gs': gs, 'input_queue': input_queue, 'node_guid': node_guid,
'pod': pod, 'g': g}))
def _do_compute_service(gs, cluster_guid, service, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(service, 'Service')
assert isinstance(g, ContextGraph)
service_id = service['id']
service_guid = 'Service:' + service_id
g.add_resource(service_guid, service['annotations'], 'Service',
service['timestamp'], service['properties'])
# Cluster contains Service.
g.add_relation(cluster_guid, service_guid, 'contains')
# Pods load balanced by this service (use the service['spec', 'selector']
# key/value pairs to find matching Pods)
selector = utilities.get_attribute(service,
['properties', 'spec', 'selector'])
if selector:
if not isinstance(selector, types.DictType):
msg = 'Service id=%s has an invalid "selector" value' % service_id
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
for pod in kubernetes.get_selected_pods(gs, selector):
pod_guid = 'Pod:' + pod['id']
# Service loadBalances Pod
g.add_relation(service_guid, pod_guid, 'loadBalances')
else:
gs.logger_error('Service id=%s has no "selector" attribute',
service_id)
def _do_compute_node(gs, input_queue, cluster_guid, node, g):
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(node, 'Node')
assert isinstance(g, ContextGraph)
node_id = node['id']
node_guid = 'Node:' + node_id
g.add_resource(node_guid, node['annotations'], 'Node', node['timestamp'],
node['properties'])
g.add_relation(cluster_guid, node_guid,
'contains') # Cluster contains Node
# Pods in a Node
# Do not compute the pods by worker threads in test mode because the order
# of the output will be different than the golden files due to the effects
# of queuing the work.
for pod in kubernetes.get_pods(gs, node_id):
if gs.get_testing():
_do_compute_pod(gs, input_queue, node_guid, pod, g)
else:
input_queue.put((gs.get_random_priority(), _do_compute_pod, {
'gs': gs,
'input_queue': input_queue,
'node_guid': node_guid,
'pod': pod,
'g': g
}))
def get_nodes_with_metrics(gs):
"""Gets the list of all nodes in the current cluster with their metrics.
Args:
gs: global state.
Returns:
list of wrapped node objects.
Each element in the list is the result of
utilities.wrap_object(node, 'Node', ...)
Raises:
CollectorError in case of failure to fetch data from Kubernetes.
Other exceptions may be raised due to exectution errors.
"""
nodes_list = get_nodes(gs)
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
project_id = utilities.node_id_to_project_id(node['id'])
# The project_id may be '_unknown_'. This is not a big
# deal, since the aggregator knows the project ID.
metrics.annotate_node(project_id, node)
return nodes_list
def _do_compute_rcontroller(gs, cluster_guid, rcontroller, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(rcontroller, 'ReplicationController')
assert isinstance(g, ContextGraph)
rcontroller_id = rcontroller['id']
rcontroller_guid = 'ReplicationController:' + rcontroller_id
g.add_resource(rcontroller_guid, rcontroller['annotations'],
'ReplicationController', rcontroller['timestamp'],
rcontroller['properties'])
# Cluster contains Rcontroller
g.add_relation(cluster_guid, rcontroller_guid, 'contains')
# Pods that are monitored by this replication controller.
# Use the rcontroller['spec']['selector'] key/value pairs to find matching
# pods.
selector = utilities.get_attribute(rcontroller,
['properties', 'spec', 'selector'])
if selector:
if not isinstance(selector, types.DictType):
msg = ('Rcontroller id=%s has an invalid "replicaSelector" value' %
rcontroller_id)
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
for pod in kubernetes.get_selected_pods(gs, selector):
pod_guid = 'Pod:' + pod['id']
# Rcontroller monitors Pod
g.add_relation(rcontroller_guid, pod_guid, 'monitors')
else:
gs.logger_error('Rcontroller id=%s has no "spec.selector" attribute',
rcontroller_id)
def get_image_from_container(container):
"""Extracts a synthesized image resource from a container."""
assert utilities.is_wrapped_object(container, "Container")
timestamp = container["timestamp"]
image_name = container["properties"]["status"]["image"]
image_id = container["properties"]["status"]["imageID"]
obj = {"metadata": {"name": image_name}}
return utilities.wrap_object(obj, "Image", image_id, timestamp, label=image_name)
def _do_compute_node(gs, input_queue, cluster_guid, node, g):
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(node, 'Node')
assert isinstance(g, ContextGraph)
node_id = node['id']
node_guid = 'Node:' + node_id
g.add_resource(node_guid, node['annotations'], 'Node', node['timestamp'],
node['properties'])
g.add_relation(cluster_guid, node_guid, 'contains') # Cluster contains Node
# Pods in a Node
pod_ids = set()
docker_hosts = set()
# Process pods sequentially because calls to _do_compute_pod() do not call
# lower-level services or wait.
for pod in kubernetes.get_pods(gs, node_id):
_do_compute_pod(gs, cluster_guid, node_guid, pod, g)
pod_ids.add(pod['id'])
# pod.properties.spec.nodeName may be missing if the pod is waiting.
docker_host = utilities.get_attribute(
pod, ['properties', 'spec', 'nodeName'])
if utilities.valid_string(docker_host):
docker_hosts.add(docker_host)
# 'docker_hosts' should contain a single Docker host, because all of
# the pods run in the same Node. However, if it is not the case, we
# cannot fix the situation, so we just log an error message and continue.
if len(docker_hosts) != 1:
gs.logger_error(
'corrupt pod data in node=%s: '
'"docker_hosts" is empty or contains more than one entry: %s',
node_guid, str(docker_hosts))
# Process containers concurrently.
for docker_host in docker_hosts:
for container in docker.get_containers_with_metrics(gs, docker_host):
parent_pod_id = utilities.get_parent_pod_id(container)
if utilities.valid_string(parent_pod_id) and (parent_pod_id in pod_ids):
# This container is contained in a pod.
parent_guid = 'Pod:' + parent_pod_id
else:
# This container is not contained in a pod.
parent_guid = node_guid
# Do not compute the containers by worker threads in test mode
# because the order of the output will be different than the golden
# files due to the effects of queuing the work.
if gs.get_testing():
_do_compute_container(gs, docker_host, parent_guid, container, g)
else:
input_queue.put((
gs.get_random_priority(),
_do_compute_container,
{'gs': gs, 'docker_host': docker_host, 'parent_guid': parent_guid,
'container': container, 'g': g}))
def annotate_container(container, parent_pod):
"""Annotate the given container with Heapster GCM metric information.
Args:
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Raises:
AssertionError: if the input arguments are invalid
"""
assert utilities.is_wrapped_object(container, 'Container')
assert utilities.is_wrapped_object(parent_pod, 'Pod')
m = _make_gcm_metrics(_get_container_labels(container, parent_pod))
if m is not None:
if 'annotations' not in container:
container['annotations'] = {}
container['annotations']['metrics'] = m
def annotate_container(container, parent_pod):
"""Annotate the given container with Heapster GCM metric information.
Args:
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Raises:
AssertionError: if the input arguments are invalid
"""
assert utilities.is_wrapped_object(container, 'Container')
assert utilities.is_wrapped_object(parent_pod, 'Pod')
m = _make_gcm_metrics(_get_container_labels(container, parent_pod))
if m is not None:
if 'annotations' not in container:
container['annotations'] = {}
container['annotations']['metrics'] = m
def _do_compute_node(cluster_guid, node, g):
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(node, 'Node')
assert isinstance(g, ContextGraph)
node_id = node['id']
node_guid = 'Node:' + node_id
g.add_resource(node_guid, node['annotations'], 'Node', node['timestamp'],
node['properties'])
g.add_relation(cluster_guid, node_guid, 'contains') # Cluster contains Node
def get_image_from_container(container):
"""Extracts a synthesized image resource from a container."""
assert utilities.is_wrapped_object(container, 'Container')
timestamp = container['timestamp']
image_name = container['properties']['status']['image']
image_id = container['properties']['status']['imageID']
obj = {'metadata': {'name': image_name}}
return utilities.wrap_object(obj,
'Image',
image_id,
timestamp,
label=image_name)
def count_resources(self, output, type_name):
assert isinstance(output, dict)
assert isinstance(type_name, types.StringTypes)
if not isinstance(output.get('resources'), list):
return 0
n = 0
for r in output.get('resources'):
assert utilities.is_wrapped_object(r)
if r.get('type') == type_name:
n += 1
return n
def count_resources(self, output, type_name):
assert isinstance(output, types.DictType)
assert isinstance(type_name, types.StringTypes)
if not isinstance(output.get('resources'), types.ListType):
return 0
n = 0
for r in output.get('resources'):
assert utilities.is_wrapped_object(r)
if r.get('type') == type_name:
n += 1
return n
def _get_container_labels(container, parent_pod):
"""Returns key/value pairs identifying all metrics of this container.
Args:
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Returns:
A dictionary of key/value pairs.
If any error was detected, returns None.
"""
if not utilities.is_wrapped_object(container, 'Container'):
return None
if not utilities.is_wrapped_object(parent_pod, 'Pod'):
return None
pod_id = utilities.get_attribute(parent_pod,
['properties', 'metadata', 'uid'])
if not utilities.valid_string(pod_id):
return None
hostname = utilities.get_attribute(parent_pod,
['properties', 'spec', 'host'])
if not utilities.valid_string(hostname):
return None
short_container_name = utilities.get_short_container_name(
container, parent_pod)
if not utilities.valid_string(short_container_name):
return None
return {
'pod_id': pod_id,
'hostname': hostname,
'container_name': short_container_name
}
def _get_container_labels(container, parent_pod):
"""Returns key/value pairs identifying all metrics of this container.
Args:
container: the container object to annotate.
parent_pod: the parent pod of 'container'.
Returns:
A dictionary of key/value pairs.
If any error was detected, returns None.
"""
if not utilities.is_wrapped_object(container, 'Container'):
return None
if not utilities.is_wrapped_object(parent_pod, 'Pod'):
return None
pod_id = utilities.get_attribute(
parent_pod, ['properties', 'metadata', 'uid'])
if not utilities.valid_string(pod_id):
return None
hostname = utilities.get_attribute(
parent_pod, ['properties', 'spec', 'host'])
if not utilities.valid_string(hostname):
return None
short_container_name = utilities.get_short_container_name(
container, parent_pod)
if not utilities.valid_string(short_container_name):
return None
return {
'pod_id': pod_id,
'hostname': hostname,
'container_name': short_container_name
}
def annotate_container_error(container, message):
"""Annotate the given container with an error message in lieu of metrics.
Args:
container: the container object to annotate.
message: a message explaining why this container was not annotated with
metrics.
Raises:
AssertionError: if the input arguments are invalid.
"""
assert utilities.is_wrapped_object(container, "Container")
assert utilities.valid_string(message)
if container.get("annotations") is None:
container["annotations"] = {}
container["annotations"]["metrics"] = {"gcmError": message}
def annotate_node(node):
"""Annotate the given node with Heapster GCM metric information.
Args:
node: the node object to annotate.
Raises:
AssertionError: if the input argument is invalid.
"""
assert utilities.is_wrapped_object(node, 'Node')
m = _make_gcm_metrics(_get_node_labels(node))
if m is not None:
if 'annotations' not in node:
node['annotations'] = {}
node['annotations']['metrics'] = m
def annotate_node(node):
"""Annotate the given node with Heapster GCM metric information.
Args:
node: the node object to annotate.
Raises:
AssertionError: if the input argument is invalid.
"""
assert utilities.is_wrapped_object(node, 'Node')
m = _make_gcm_metrics(_get_node_labels(node))
if m is not None:
if 'annotations' not in node:
node['annotations'] = {}
node['annotations']['metrics'] = m
def _do_compute_container(gs, docker_host, pod_guid, container, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(docker_host)
assert utilities.valid_string(pod_guid)
assert utilities.is_wrapped_object(container, 'Container')
assert isinstance(g, ContextGraph)
container_id = container['id']
container_guid = 'Container:' + container_id
# TODO(vasbala): container_id is too verbose?
g.add_resource(container_guid, container['annotations'],
'Container', container['timestamp'],
container['properties'])
# Pod contains Container
g.add_relation(pod_guid, container_guid, 'contains')
# Processes in a Container
for process in docker.get_processes(gs, docker_host, container_id):
process_id = process['id']
process_guid = 'Process:' + process_id
g.add_resource(process_guid, process['annotations'],
'Process', process['timestamp'], process['properties'])
# Container contains Process
g.add_relation(container_guid, process_guid, 'contains')
# Image from which this Container was created
image_id = utilities.get_attribute(
container, ['properties', 'Config', 'Image'])
if not utilities.valid_string(image_id):
# Image ID not found
return
image = docker.get_image(gs, docker_host, image_id)
if image is None:
# image not found
return
image_guid = 'Image:' + image['id']
# Add the image to the graph only if we have not added it before.
g.add_resource(image_guid, image['annotations'], 'Image',
image['timestamp'], image['properties'])
# Container createdFrom Image
g.add_relation(container_guid, image_guid, 'createdFrom')
def _get_node_labels(node):
"""Returns key/value pairs identifying all metrics of this node.
Args:
node: the node object to annotate.
Returns:
A dictionary of key/value pairs.
If any error was detected, returns None.
"""
if not utilities.is_wrapped_object(node, "Node"):
return None
hostname = utilities.get_attribute(node, ["properties", "metadata", "name"])
if not utilities.valid_string(hostname):
return None
return {"pod_id": "", "hostname": hostname, "container_name": "/"}
def annotate_node(project_id, node):
"""Annotate the given node with Heapster GCM metric information.
Args:
project_id: the project ID
node: the node object to annotate.
Raises:
AssertionError: if the input argument is invalid.
"""
assert utilities.valid_string(project_id)
assert utilities.is_wrapped_object(node, "Node")
m = _make_gcm_metrics(project_id, _get_node_labels(node))
if m is None:
return
if node.get("annotations") is None:
node["annotations"] = {}
node["annotations"]["metrics"] = m
def annotate_node(project_id, node):
"""Annotate the given node with Heapster GCM metric information.
Args:
project_id: the project ID
node: the node object to annotate.
Raises:
AssertionError: if the input argument is invalid.
"""
assert utilities.valid_string(project_id)
assert utilities.is_wrapped_object(node, 'Node')
m = _make_gcm_metrics(project_id, _get_node_labels(node))
if m is None:
return
if node.get('annotations') is None:
node['annotations'] = {}
node['annotations']['metrics'] = m
def _get_node_labels(node):
"""Returns key/value pairs identifying all metrics of this node.
Args:
node: the node object to annotate.
Returns:
A dictionary of key/value pairs.
If any error was detected, returns None.
"""
if not utilities.is_wrapped_object(node, 'Node'):
return None
hostname = utilities.get_attribute(node,
['properties', 'metadata', 'name'])
if not utilities.valid_string(hostname):
return None
return {'pod_id': '', 'hostname': hostname, 'container_name': '/'}
def verify_resources(self, result):
assert isinstance(result, types.DictType)
self.assertEqual(1, self.count_resources(result, 'Cluster'))
self.assertEqual(3, self.count_resources(result, 'Node'))
self.assertEqual(6, self.count_resources(result, 'Service'))
# TODO(eran): the pods count does not include the pods running in the
# master. Fix the count once we include pods that run in the master node.
self.assertEqual(10, self.count_resources(result, 'Pod'))
self.assertEqual(4, self.count_resources(result, 'Container'))
self.assertEqual(7, self.count_resources(result, 'Process'))
self.assertEqual(2, self.count_resources(result, 'Image'))
self.assertEqual(3, self.count_resources(result, 'ReplicationController'))
# Verify that all resources are valid wrapped objects.
assert isinstance(result.get('resources'), types.ListType)
for r in result.get('resources'):
# The type of resource must be defined.
assert utilities.valid_string(r.get('type'))
assert utilities.is_wrapped_object(r, r.get('type'))
def _do_compute_pod(gs, input_queue, node_guid, pod, g):
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert utilities.valid_string(node_guid)
assert utilities.is_wrapped_object(pod, 'Pod')
assert isinstance(g, ContextGraph)
pod_id = pod['id']
pod_guid = 'Pod:' + pod_id
docker_host = utilities.get_attribute(
pod, ['properties', 'spec', 'host'])
if not utilities.valid_string(docker_host):
msg = ('Docker host (pod.properties.spec.host) '
'not found in pod ID %s' % pod_id)
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
g.add_resource(pod_guid, pod['annotations'], 'Pod', pod['timestamp'],
pod['properties'])
g.add_relation(node_guid, pod_guid, 'runs') # Node runs Pod
def verify_resources(self, result, start_time, end_time):
assert isinstance(result, dict)
assert utilities.valid_string(start_time)
assert utilities.valid_string(end_time)
self.assertEqual(1, self.count_resources(result, 'Cluster'))
self.assertEqual(5, self.count_resources(result, 'Node'))
self.assertEqual(6, self.count_resources(result, 'Service'))
# TODO(eran): the pods count does not include the pods running in the
# master. Fix the count once we include pods that run in the master node.
self.assertEqual(14, self.count_resources(result, 'Pod'))
self.assertEqual(16, self.count_resources(result, 'Container'))
self.assertEqual(10, self.count_resources(result, 'Image'))
self.assertEqual(3, self.count_resources(result, 'ReplicationController'))
# Verify that all resources are valid wrapped objects.
assert isinstance(result.get('resources'), list)
for r in result['resources']:
# all resources must be valid.
assert utilities.is_wrapped_object(r)
assert start_time <= r['timestamp'] <= end_time
def _get_node_labels(node):
"""Returns key/value pairs identifying all metrics of this node.
Args:
node: the node object to annotate.
Returns:
A dictionary of key/value pairs.
If any error was detected, returns None.
"""
if not utilities.is_wrapped_object(node, 'Node'):
return None
hostname = utilities.get_attribute(node, ['properties', 'metadata', 'name'])
if not utilities.valid_string(hostname):
return None
return {
'pod_id': '',
'hostname': hostname,
'container_name': '/'
}
def _do_compute_container(gs, docker_host, pod_guid, container, g):
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(docker_host)
assert utilities.valid_string(pod_guid)
assert utilities.is_wrapped_object(container, 'Container')
assert isinstance(g, ContextGraph)
container_id = container['id']
container_guid = 'Container:' + container_id
# TODO(vasbala): container_id is too verbose?
g.add_resource(container_guid, container['annotations'], 'Container',
container['timestamp'], container['properties'])
# Pod contains Container
g.add_relation(pod_guid, container_guid, 'contains')
# Processes in a Container
for process in docker.get_processes(gs, docker_host, container_id):
process_id = process['id']
process_guid = 'Process:' + process_id
g.add_resource(process_guid, process['annotations'], 'Process',
process['timestamp'], process['properties'])
# Container contains Process
g.add_relation(container_guid, process_guid, 'contains')
image = docker.get_image(gs, docker_host, container)
if image is None:
# image not found
return
image_guid = 'Image:' + image['id']
# Add the image to the graph only if we have not added it before.
g.add_resource(image_guid, image['annotations'], 'Image',
image['timestamp'], image['properties'])
# Container createdFrom Image
g.add_relation(container_guid, image_guid, 'createdFrom')
def _do_compute_pod(gs, input_queue, node_guid, pod, g):
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert utilities.valid_string(node_guid)
assert utilities.is_wrapped_object(pod, 'Pod')
assert isinstance(g, ContextGraph)
pod_id = pod['id']
pod_guid = 'Pod:' + pod_id
docker_host = utilities.get_attribute(pod, ['properties', 'spec', 'host'])
if not utilities.valid_string(docker_host):
msg = ('Docker host (pod.properties.spec.host) '
'not found in pod ID %s' % pod_id)
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
g.add_resource(pod_guid, pod['annotations'], 'Pod', pod['timestamp'],
pod['properties'])
g.add_relation(node_guid, pod_guid, 'runs') # Node runs Pod
# Containers in a Pod
for container in docker.get_containers_with_metrics(gs, docker_host):
if not _container_in_pod(gs, container, pod):
continue
# Do not compute the containers by worker threads in test mode because the
# order of the output will be different than the golden files due to the
# effects of queuing the work.
if gs.get_testing():
_do_compute_container(gs, docker_host, pod_guid, container, g)
else:
input_queue.put((gs.get_random_priority(), _do_compute_container, {
'gs': gs,
'docker_host': docker_host,
'pod_guid': pod_guid,
'container': container,
'g': g
}))
def get_minions():
"""Computes the response of the '/minions_status' endpoint.
Returns:
A dictionary from node names to the status of their minion collectors
or an error message.
"""
gs = app.context_graph_global_state
minions_status = {}
try:
for node in kubernetes.get_nodes(gs):
assert utilities.is_wrapped_object(node, 'Node')
docker_host = node['id']
minions_status[docker_host] = docker.get_minion_status(gs, docker_host)
except collector_error.CollectorError as e:
return flask.jsonify(utilities.make_error(str(e)))
except:
msg = 'get_minions_status() failed with exception %s' % sys.exc_info()[0]
app.logger.exception(msg)
return flask.jsonify(utilities.make_error(msg))
return flask.jsonify(utilities.make_response(minions_status, 'minionsStatus'))
def get_one_pod(gs, node_id, pod_id):
"""Gets the pod with the given pod_id in the given node_id.
Args:
gs: global state.
node_id: the parent node of requested pod.
pod_id: the ID of the requested pod.
Returns:
If the pod was found, returns the wrapped pod object, which is the result
of utilities.wrap_object(pod, 'Pod', ...).
If the pod was not found, returns None.
Raises:
CollectorError in case of failure to fetch data from Kubernetes.
Other exceptions may be raised due to exectution errors.
"""
for pod in get_pods(gs, node_id):
assert utilities.is_wrapped_object(pod, 'Pod')
if pod['id'] == pod_id:
return pod
return None
def get_one_pod(gs, node_id, pod_id):
"""Gets the pod with the given pod_id in the given node_id.
Args:
gs: global state.
node_id: the parent node of requested pod.
pod_id: the ID of the requested pod.
Returns:
If the pod was found, returns the wrapped pod object, which is the result
of utilities.wrap_object(pod, 'Pod', ...).
If the pod was not found, returns None.
Raises:
CollectorError in case of failure to fetch data from Kubernetes.
Other exceptions may be raised due to exectution errors.
"""
for pod in get_pods(gs, node_id):
assert utilities.is_wrapped_object(pod, 'Pod')
if pod['id'] == pod_id:
return pod
return None
def _do_compute_pod(gs, input_queue, node_guid, pod, g):
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert utilities.valid_string(node_guid)
assert utilities.is_wrapped_object(pod, 'Pod')
assert isinstance(g, ContextGraph)
pod_id = pod['id']
pod_guid = 'Pod:' + pod_id
docker_host = utilities.get_attribute(
pod, ['properties', 'spec', 'host'])
if not utilities.valid_string(docker_host):
msg = ('Docker host (pod.properties.spec.host) '
'not found in pod ID %s' % pod_id)
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
g.add_resource(pod_guid, pod['annotations'], 'Pod', pod['timestamp'],
pod['properties'])
g.add_relation(node_guid, pod_guid, 'runs') # Node runs Pod
# Containers in a Pod
for container in docker.get_containers_with_metrics(gs, docker_host):
if not _container_in_pod(gs, container, pod):
continue
# Do not compute the containers by worker threads in test mode because the
# order of the output will be different than the golden files due to the
# effects of queuing the work.
if gs.get_testing():
_do_compute_container(gs, docker_host, pod_guid, container, g)
else:
input_queue.put((
gs.get_random_priority(),
_do_compute_container,
{'gs': gs, 'docker_host': docker_host, 'pod_guid': pod_guid,
'container': container, 'g': g}))
def get_containers_from_pod(pod):
"""Extracts synthesized container resources from a pod.
Only containers for which status is available are included. (The pod may
still be pending.)
"""
assert utilities.is_wrapped_object(pod, 'Pod')
specs = utilities.get_attribute(pod, ['properties', 'spec', 'containers'])
statuses = utilities.get_attribute(
pod, ['properties', 'status', 'containerStatuses'])
timestamp = pod['timestamp']
spec_dict = {}
for spec in specs or []:
spec = spec.copy()
spec_dict[spec.pop('name')] = spec
containers = []
for status in statuses or []:
status = status.copy()
name = status.pop('name')
unique_id = status.get('containerID', name)
obj = {
'metadata': {
'name': name
},
'spec': spec_dict.get(name, {}),
'status': status,
}
container = utilities.wrap_object(obj,
'Container',
unique_id,
timestamp,
label=name)
containers.append(container)
return containers
def _do_compute_pod(cluster_guid, pod, g):
assert utilities.valid_string(cluster_guid)
assert utilities.is_wrapped_object(pod, 'Pod')
assert isinstance(g, ContextGraph)
pod_id = pod['id']
pod_guid = 'Pod:' + pod_id
g.add_resource(pod_guid, pod['annotations'], 'Pod', pod['timestamp'],
pod['properties'])
# pod.properties.spec.nodeName may be missing if the pod is waiting
# (not running yet).
node_id = utilities.get_attribute(pod, ['properties', 'spec', 'nodeName'])
if utilities.valid_string(node_id):
# Pod is running.
node_guid = 'Node:' + node_id
g.add_relation(node_guid, pod_guid, 'runs') # Node runs Pod
else:
# Pod is not running.
g.add_relation(cluster_guid, pod_guid, 'contains') # Cluster contains Pod
for container in kubernetes.get_containers_from_pod(pod):
metrics.annotate_container(container, pod)
_do_compute_container(pod_guid, container, g)
def get_containers_with_metrics(gs, docker_host):
"""Gets the list of all containers in 'docker_host' with metric annotations.
Args:
gs: global state.
docker_host: the Docker host running the containers.
Returns:
list of wrapped container objects.
Each element in the list is the result of
utilities.wrap_object(container, 'Container', ...)
Raises:
CollectorError: in case of failure to fetch data from Docker.
Other exceptions may be raised due to exectution errors.
"""
# Create a lookup table from pod IDs to pods.
# This lookup table is needed when annotating containers with
# metrics. Also compute the project's name.
containers_list = get_containers(gs, docker_host)
if not containers_list:
return []
pod_id_to_pod = {}
project_id = '_unknown_'
# Populate the pod ID to pod lookup table.
# Compute the project_id from the name of the first pod.
for pod in kubernetes.get_pods(gs, docker_host):
assert utilities.is_wrapped_object(pod, 'Pod')
pod_id_to_pod[pod['id']] = pod
if project_id != '_unknown_':
continue
pod_hostname = utilities.get_attribute(pod,
['properties', 'spec', 'host'])
if utilities.valid_string(pod_hostname):
project_id = utilities.node_id_to_project_id(pod_hostname)
# We know that there are containers in this docker_host.
if not pod_id_to_pod:
# there are no pods in this docker_host.
msg = 'Docker host %s has containers but no pods' % docker_host
gs.logger_exception(msg)
raise collector_error.CollectorError(msg)
# Annotate the containers with their metrics.
for container in containers_list:
assert utilities.is_wrapped_object(container, 'Container')
parent_pod_id = utilities.get_parent_pod_id(container)
if not utilities.valid_string(parent_pod_id):
msg = ('missing or invalid parent pod ID in container %s' %
container['id'])
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
if parent_pod_id not in pod_id_to_pod:
msg = ('could not locate parent pod %s for container %s' %
(parent_pod_id, container['id']))
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Note that the project ID may be '_unknown_'.
# This is not a big deal, because the aggregator knows the project ID.
metrics.annotate_container(project_id, container,
pod_id_to_pod[parent_pod_id])
return containers_list
def get_processes(gs, docker_host, container_id):
"""Gets the list of all processes in the 'docker_host' and 'container_id'.
If the container is not found, returns an empty list of processes.
Args:
gs: global state.
docker_host: the Docker host running the container.
container_id: the container running the processes.
Returns:
list of wrapped process objects.
Each element in the list is the result of
utilities.wrap_object(process, 'Process', ...)
Raises:
CollectorError in case of failure to fetch data from Docker.
Other exceptions may be raised due to exectution errors.
"""
processes_label = '%s/%s' % (docker_host, container_id)
processes, timestamp_secs = gs.get_processes_cache().lookup(
processes_label)
if timestamp_secs is not None:
gs.logger_info(
'get_processes(docker_host=%s, container_id=%s) cache hit',
docker_host, container_id)
return processes
container = get_one_container(gs, docker_host, container_id)
if container is not None:
assert utilities.is_wrapped_object(container, 'Container')
container_short_hex_id = utilities.object_to_hex_id(
container['properties'])
assert utilities.valid_string(container_short_hex_id)
else:
# Parent container not found. Container might have crashed while we were
# looking for it.
return []
# NOTE: there is no trailing /json in this URL - this looks like a bug in the
# Docker API
url = ('http://{docker_host}:{port}/containers/{container_id}/top?'
'ps_args=aux'.format(docker_host=docker_host,
port=gs.get_docker_port(),
container_id=container_id))
# A typical value of 'docker_host' is:
# k8s-guestbook-node-3.c.rising-apricot-840.internal
# Use only the first period-seperated element for the test file name.
# The typical value of 'container_id' is:
# k8s_php-redis.b317029a_guestbook-controller-ls6k1.default.api_f991d53e-b949-11e4-8246-42010af0c3dd_8dcdfec8
# Use just the tail of the container ID after the last '_' sign.
fname = '{host}-processes-{id}'.format(host=docker_host.split('.')[0],
id=container_id.split('_')[-1])
try:
# TODO(vasbala): what should we do in cases where the container is gone
# (and replaced by a different one)?
result = fetch_data(gs, url, fname, expect_missing=True)
except ValueError:
# this container does not exist anymore
return []
except collector_error.CollectorError:
raise
except:
msg = 'fetching %s failed with exception %s' % (url, sys.exc_info()[0])
gs.logger_exception(msg)
raise collector_error.CollectorError(msg)
if not isinstance(utilities.get_attribute(result, ['Titles']),
types.ListType):
invalid_processes(gs, url)
if not isinstance(utilities.get_attribute(result, ['Processes']),
types.ListType):
invalid_processes(gs, url)
pstats = result['Titles']
processes = []
now = time.time()
for pvalues in result['Processes']:
process = {}
if not isinstance(pvalues, types.ListType):
invalid_processes(gs, url)
if len(pstats) != len(pvalues):
invalid_processes(gs, url)
for pstat, pvalue in zip(pstats, pvalues):
process[pstat] = pvalue
# Prefix with container Id to ensure uniqueness across the whole graph.
process_id = '%s/%s' % (container_short_hex_id, process['PID'])
processes.append(
utilities.wrap_object(process,
'Process',
process_id,
now,
label=process['PID']))
ret_value = gs.get_processes_cache().update(processes_label, processes,
now)
gs.logger_info(
'get_processes(docker_host=%s, container_id=%s) returns %d processes',
docker_host, container_id, len(processes))
return ret_value
def get_image(gs, docker_host, container):
"""Gets the information of the given image in the given host.
Args:
gs: global state.
docker_host: Docker host name. Must not be empty.
container: the container which runs the image.
Returns:
If image was found, returns the wrapped image object, which is the result of
utilities.wrap_object(image, 'Image', ...)
If the image was not found, returns None.
Raises:
CollectorError: in case of failure to fetch data from Docker.
ValueError: in case the container does not contain a valid image ID.
Other exceptions may be raised due to exectution errors.
"""
assert utilities.is_wrapped_object(container, 'Container')
# The 'image_id' should be a long hexadecimal string.
image_id = utilities.get_attribute(container, ['properties', 'Image'])
if not utilities.valid_hex_id(image_id):
msg = 'missing or invalid image ID in container ID=%s' % container['id']
gs.logger_error(msg)
raise ValueError(msg)
# The 'image_name' should be a symbolic name (not a hexadecimal string).
image_name = utilities.get_attribute(container,
['properties', 'Config', 'Image'])
if ((not utilities.valid_string(image_name))
or utilities.valid_hex_id(image_name)):
msg = 'missing or invalid image name in container ID=%s' % container[
'id']
gs.logger_error(msg)
raise ValueError(msg)
cache_key = '%s|%s' % (docker_host, image_id)
image, timestamp_secs = gs.get_images_cache().lookup(cache_key)
if timestamp_secs is not None:
gs.logger_info('get_image(docker_host=%s, image_id=%s) cache hit',
docker_host, image_id)
return image
# A typical value of 'docker_host' is:
# k8s-guestbook-node-3.c.rising-apricot-840.internal
# Use only the first period-seperated element for the test file name.
# The typical value of 'image_name' is:
# brendanburns/php-redis
# We convert embedded '/' and ':' characters to '-' to avoid interference with
# the directory structure or file system.
url = 'http://{docker_host}:{port}/images/{image_id}/json'.format(
docker_host=docker_host, port=gs.get_docker_port(), image_id=image_id)
fname = '{host}-image-{id}'.format(host=docker_host.split('.')[0],
id=image_name.replace('/', '-').replace(
':', '-'))
try:
image = fetch_data(gs, url, fname, expect_missing=True)
except ValueError:
# image not found.
msg = 'image not found for image_id: %s' % image_id
gs.logger_info(msg)
return None
except collector_error.CollectorError:
raise
except:
msg = 'fetching %s failed with exception %s' % (url, sys.exc_info()[0])
gs.logger_exception(msg)
raise collector_error.CollectorError(msg)
now = time.time()
# compute the two labels of the image.
# The first is a 12-digit hexadecimal number shown by "docker images".
# The second is the symbolic name of the image.
full_hex_label = image.get('Id')
if not (isinstance(full_hex_label, types.StringTypes) and full_hex_label):
msg = 'Image id=%s has an invalid "Id" attribute value' % image_id
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
short_hex_label = utilities.object_to_hex_id(image)
if short_hex_label is None:
msg = 'Could not compute short hex ID of image %s' % image_id
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
wrapped_image = utilities.wrap_object(image,
'Image',
full_hex_label,
now,
label=short_hex_label,
alt_label=image_name)
ret_value = gs.get_images_cache().update(cache_key, wrapped_image, now)
gs.logger_info('get_image(docker_host=%s, image_id=%s, image_name=%s)',
docker_host, image_id, image_name)
return ret_value
def _do_compute_graph(gs, input_queue, output_queue, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
input_queue: the input queue for the worker threads.
output_queue: output queue containing exceptions data from the worker
threads.
output_format: one of 'graph', 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert isinstance(input_queue, Queue.PriorityQueue)
assert isinstance(output_queue, Queue.Queue)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_version(docker.get_version(gs))
g.set_metadata({'timestamp': utilities.now()})
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(gs, output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# Get the cluster name from the first node.
# The cluster name is an approximation. It is not a big deal if it
# is incorrect, since the aggregator knows the cluster name.
cluster_name = utilities.node_id_to_cluster_name(nodes_list[0]['id'])
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
input_queue.put((gs.get_random_priority(), _do_compute_node, {
'gs': gs,
'input_queue': input_queue,
'cluster_guid': cluster_guid,
'node': node,
'g': g
}))
# Services
for service in kubernetes.get_services(gs):
input_queue.put((gs.get_random_priority(), _do_compute_service, {
'gs': gs,
'cluster_guid': cluster_guid,
'service': service,
'g': g
}))
# ReplicationControllers
rcontrollers_list = kubernetes.get_rcontrollers(gs)
for rcontroller in rcontrollers_list:
input_queue.put((gs.get_random_priority(), _do_compute_rcontroller, {
'gs': gs,
'cluster_guid': cluster_guid,
'rcontroller': rcontroller,
'g': g
}))
# Wait until worker threads finished processing all outstanding requests.
# Once we return from the join(), all output was generated already.
input_queue.join()
# Convert any exception caught by the worker threads to an exception
# raised by the current thread.
if not output_queue.empty():
msg = output_queue.get_nowait() # should not fail.
gs.logger_error(msg)
raise collector_error.CollectorError(msg)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
# Dump the resulting graph
return g.dump(gs, output_format)
def test_cluster(self):
"""Test the '/cluster' endpoint."""
start_time = utilities.now()
end_time = None
for _ in range(2):
# Exercise the collector. Read data from golden files and compute
# a context graph.
# The second iteration should read from the cache.
ret_value = self.app.get('/cluster')
if end_time is None:
end_time = utilities.now()
result = json.loads(ret_value.data)
# The timestamps of the second iteration should be the same as in the
# first iteration, because the data of the 2nd iteration should be
# fetched from the cache, and it did not change.
# Even if fetching the data caused an explicit reading from the files
# in the second iteration, the data did not change, so it should keep
# its original timestamp.
self.verify_resources(result, start_time, end_time)
self.assertEqual(5, self.count_relations(
result, 'contains', 'Cluster', 'Node'))
self.assertEqual(6, self.count_relations(
result, 'contains', 'Cluster', 'Service'))
self.assertEqual(3, self.count_relations(
result, 'contains', 'Cluster', 'ReplicationController'))
self.assertEqual(16, self.count_relations(
result, 'contains', 'Pod', 'Container'))
self.assertEqual(30, self.count_relations(result, 'contains'))
self.assertEqual(16, self.count_relations(result, 'createdFrom'))
self.assertEqual(7, self.count_relations(result, 'loadBalances'))
self.assertEqual(6, self.count_relations(result, 'monitors'))
self.assertEqual(14, self.count_relations(result, 'runs'))
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), list))
for r in result['relations']:
self.assertTrue(isinstance(r, dict))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(start_time <= result['timestamp'] <= end_time)
# Wait a little to ensure that the current time is greater than
# end_time
time.sleep(1)
self.assertTrue(utilities.now() > end_time)
# Change the timestamp of the nodes in the cache.
timestamp_before_update = utilities.now()
gs = collector.app.context_graph_global_state
nodes, timestamp_seconds = gs.get_nodes_cache().lookup('')
self.assertTrue(isinstance(nodes, list))
self.assertTrue(start_time <=
utilities.seconds_to_timestamp(timestamp_seconds) <=
end_time)
# Change the first node to force the timestamp in the cache to change.
# We have to change both the properties of the first node and its
# timestamp, so the cache will store the new value (including the new
# timestamp).
self.assertTrue(len(nodes) >= 1)
self.assertTrue(utilities.is_wrapped_object(nodes[0], 'Node'))
nodes[0]['properties']['newAttribute123'] = 'the quick brown fox jumps over'
nodes[0]['timestamp'] = utilities.now()
gs.get_nodes_cache().update('', nodes)
timestamp_after_update = utilities.now()
_, timestamp_seconds = gs.get_nodes_cache().lookup('')
self.assertTrue(timestamp_before_update <=
utilities.seconds_to_timestamp(timestamp_seconds) <=
timestamp_after_update)
# Build the context graph again.
ret_value = self.app.get('/cluster')
result = json.loads(ret_value.data)
self.verify_resources(result, start_time, timestamp_after_update)
# Verify that all relations contain a timestamp in the range
# [start_time, end_time].
self.assertTrue(isinstance(result.get('relations'), list))
for r in result['relations']:
self.assertTrue(isinstance(r, dict))
timestamp = r.get('timestamp')
self.assertTrue(utilities.valid_string(timestamp))
self.assertTrue(start_time <= timestamp <= end_time)
# The overall timestamp must be in the expected range.
self.assertTrue(utilities.valid_string(result.get('timestamp')))
self.assertTrue(timestamp_before_update <= result['timestamp'] <=
timestamp_after_update)
def _do_compute_graph(gs, output_format):
"""Returns the context graph in the specified format.
Args:
gs: the global state.
output_format: one of 'dot', 'context_graph', or 'resources'.
Returns:
A successful response in the specified format.
Raises:
CollectorError: inconsistent or invalid graph data.
"""
assert isinstance(gs, global_state.GlobalState)
assert utilities.valid_string(output_format)
g = ContextGraph()
g.set_relations_to_timestamps(gs.get_relations_to_timestamps())
# Nodes
nodes_list = kubernetes.get_nodes_with_metrics(gs)
if not nodes_list:
return g.dump(output_format)
# Find the timestamp of the oldest node. This will be the timestamp of
# the cluster.
oldest_timestamp = utilities.now()
for node in nodes_list:
assert utilities.is_wrapped_object(node, 'Node')
# note: we cannot call min(oldest_timestamp, node['timestamp']) here
# because min(string) returnes the smallest character in the string.
if node['timestamp'] < oldest_timestamp:
oldest_timestamp = node['timestamp']
# The cluster name may be available through the Kubernetes API someday.
# TODO(rimey): Determine the cluster name.
cluster_name = '_unknown_'
cluster_guid = 'Cluster:' + cluster_name
g.set_title(cluster_name)
g.add_resource(cluster_guid, {'label': cluster_name}, 'Cluster',
oldest_timestamp, {})
# Nodes
for node in nodes_list:
_do_compute_node(cluster_guid, node, g)
# Pods
for pod in kubernetes.get_pods(gs):
_do_compute_pod(cluster_guid, pod, g)
# Services
for service in kubernetes.get_services(gs):
_do_compute_service(gs, cluster_guid, service, g)
# ReplicationControllers
for rcontroller in kubernetes.get_rcontrollers(gs):
_do_compute_rcontroller(gs, cluster_guid, rcontroller, g)
# Other nodes, not on the list, such as the Kubernetes master.
_do_compute_other_nodes(gs, cluster_guid, nodes_list, oldest_timestamp, g)
# Keep the relations_to_timestamps mapping for next call.
gs.set_relations_to_timestamps(g.get_relations_to_timestamps())
g.set_metadata({'timestamp': g.max_resources_and_relations_timestamp()})
# Dump the resulting graph
return g.dump(output_format)
