class TestKubeutil(unittest.TestCase):
def setUp(self):
self.kubeutil = KubeUtil()
@mock.patch('utils.kubeutil.KubeUtil.retrieve_pods_list',
side_effect=['foo'])
@mock.patch('utils.kubeutil.KubeUtil.extract_kube_labels')
def test_get_kube_labels(self, extract_kube_labels, retrieve_pods_list):
self.kubeutil.get_kube_labels(excluded_keys='bar')
retrieve_pods_list.assert_called_once()
extract_kube_labels.assert_called_once_with('foo', excluded_keys='bar')
def test_extract_kube_labels(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_kube_labels({}, ['foo'])
self.assertEqual(len(res), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 8)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 6)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
def test_extract_meta(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_meta({}, 'foo')
self.assertEqual(len(res), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 6)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 4)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_pods_list(self, retrieve_json):
self.kubeutil.retrieve_pods_list()
retrieve_json.assert_called_once_with(self.kubeutil.pods_list_url)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_metrics(self, retrieve_json):
self.kubeutil.retrieve_metrics()
retrieve_json.assert_called_once_with(self.kubeutil.metrics_url)
def test_filter_pods_list(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.filter_pods_list({}, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.9')
self.assertEqual(len(res.get('items')), 5)
pods = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.5')
self.assertEqual(len(res.get('items')), 1)
pods = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
@mock.patch('utils.kubeutil.requests')
def test_retrieve_json_auth(self, r):
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_once_with(
'url',
verify=False,
timeout=10,
headers={'Authorization': 'Bearer foo_tok'})
self.kubeutil.CA_CRT_PATH = __file__
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_with('url',
verify=__file__,
timeout=10,
headers={'Authorization': 'Bearer foo_tok'})
def test_get_node_info(self):
with mock.patch('utils.kubeutil.KubeUtil._fetch_host_data') as f:
self.kubeutil.get_node_info()
f.assert_called_once()
f.reset_mock()
self.kubeutil._node_ip = 'foo'
self.kubeutil._node_name = 'bar'
ip, name = self.kubeutil.get_node_info()
self.assertEqual(ip, 'foo')
self.assertEqual(name, 'bar')
f.assert_not_called()
def test__fetch_host_data(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
with mock.patch(
'utils.kubeutil.KubeUtil.retrieve_pods_list') as mock_pods:
self.kubeutil.host_name = 'dd-agent-1rxlh'
mock_pods.return_value = json.loads(
Fixtures.read_file("pods_list_1.2.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name,
'kubernetes-massi-minion-k23m')
self.kubeutil.host_name = 'heapster-v11-l8sh1'
mock_pods.return_value = json.loads(
Fixtures.read_file("pods_list_1.1.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name,
'gke-cluster-1-8046fdfa-node-ld35')
def test_get_auth_token(self):
KubeUtil.AUTH_TOKEN_PATH = '/foo/bar'
self.assertIsNone(KubeUtil.get_auth_token())
KubeUtil.AUTH_TOKEN_PATH = Fixtures.file(
'events.json') # any file could do the trick
self.assertIsNotNone(KubeUtil.get_auth_token())
def test_is_k8s(self):
os.unsetenv('KUBERNETES_PORT')
self.assertFalse(Platform.is_k8s())
os.environ['KUBERNETES_PORT'] = '999'
self.assertTrue(Platform.is_k8s())
def test_extract_event_tags(self):
events = json.loads(
Fixtures.read_file("events.json", string_escape=False))['items']
for ev in events:
tags = KubeUtil().extract_event_tags(ev)
# there should be 4 tags except for some events where source.host is missing
self.assertTrue(len(tags) >= 3)
tag_names = [tag.split(':')[0] for tag in tags]
self.assertIn('reason', tag_names)
self.assertIn('namespace', tag_names)
self.assertIn('object_type', tag_names)
if len(tags) == 4:
self.assertIn('node_name', tag_names)
class TestKubeutil(unittest.TestCase):
def setUp(self):
self.kubeutil = KubeUtil()
@mock.patch("utils.kubeutil.KubeUtil.retrieve_pods_list", side_effect=["foo"])
@mock.patch("utils.kubeutil.KubeUtil.extract_kube_labels")
def test_get_kube_labels(self, extract_kube_labels, retrieve_pods_list):
self.kubeutil.get_kube_labels(excluded_keys="bar")
retrieve_pods_list.assert_called_once()
extract_kube_labels.assert_called_once_with("foo", excluded_keys="bar")
def test_extract_kube_labels(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_kube_labels({}, ["foo"])
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ["foo"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 8)
res = self.kubeutil.extract_kube_labels(pods, ["k8s-app"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ["foo"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
res = self.kubeutil.extract_kube_labels(pods, ["k8s-app"])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
def test_extract_meta(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_meta({}, "foo")
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, "foo")
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, "uid")
self.assertEqual(len(res), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, "foo")
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, "uid")
self.assertEqual(len(res), 4)
@mock.patch("utils.kubeutil.retrieve_json")
def test_retrieve_pods_list(self, retrieve_json):
self.kubeutil.retrieve_pods_list()
retrieve_json.assert_called_once_with(self.kubeutil.pods_list_url)
@mock.patch("utils.kubeutil.retrieve_json")
def test_retrieve_metrics(self, retrieve_json):
self.kubeutil.retrieve_metrics()
retrieve_json.assert_called_once_with(self.kubeutil.metrics_url)
def test_filter_pods_list(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.filter_pods_list({}, "foo")
self.assertEqual(len(res.get("items")), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "10.240.0.9")
self.assertEqual(len(res.get("items")), 5)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "foo")
self.assertEqual(len(res.get("items")), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "10.240.0.5")
self.assertEqual(len(res.get("items")), 1)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, "foo")
self.assertEqual(len(res.get("items")), 0)
@mock.patch("utils.kubeutil.requests")
def test_retrieve_json_auth(self, r):
self.kubeutil.retrieve_json_auth("url", "foo_tok")
r.get.assert_called_once_with("url", verify=False, timeout=10, headers={"Authorization": "Bearer foo_tok"})
self.kubeutil.CA_CRT_PATH = __file__
self.kubeutil.retrieve_json_auth("url", "foo_tok")
r.get.assert_called_with("url", verify=__file__, timeout=10, headers={"Authorization": "Bearer foo_tok"})
def test_get_node_info(self):
with mock.patch("utils.kubeutil.KubeUtil._fetch_host_data") as f:
self.kubeutil.get_node_info()
f.assert_called_once()
f.reset_mock()
self.kubeutil._node_ip = "foo"
self.kubeutil._node_name = "bar"
ip, name = self.kubeutil.get_node_info()
self.assertEqual(ip, "foo")
self.assertEqual(name, "bar")
f.assert_not_called()
def test__fetch_host_data(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
with mock.patch("utils.kubeutil.KubeUtil.retrieve_pods_list") as mock_pods:
self.kubeutil.host_name = "dd-agent-1rxlh"
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, "10.240.0.9")
self.assertEqual(self.kubeutil._node_name, "kubernetes-massi-minion-k23m")
self.kubeutil.host_name = "heapster-v11-l8sh1"
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, "10.240.0.9")
self.assertEqual(self.kubeutil._node_name, "gke-cluster-1-8046fdfa-node-ld35")
def test_get_auth_token(self):
KubeUtil.AUTH_TOKEN_PATH = "/foo/bar"
self.assertIsNone(KubeUtil.get_auth_token())
KubeUtil.AUTH_TOKEN_PATH = Fixtures.file("events.json") # any file could do the trick
self.assertIsNotNone(KubeUtil.get_auth_token())
def test_is_k8s(self):
os.unsetenv("KUBERNETES_PORT")
self.assertFalse(is_k8s())
os.environ["KUBERNETES_PORT"] = "999"
self.assertTrue(is_k8s())
def get_hostname(config=None):
"""
Get the canonical host name this agent should identify as. This is
the authoritative source of the host name for the agent.
Tries, in order:
* agent config (datadog.conf, "hostname:")
* 'hostname -f' (on unix)
* socket.gethostname()
"""
hostname = None
# first, try the config
if config is None:
from config import get_config
config = get_config(parse_args=True)
config_hostname = config.get('hostname')
if config_hostname and is_valid_hostname(config_hostname):
return config_hostname
# Try to get GCE instance name
gce_hostname = GCE.get_hostname(config)
if gce_hostname is not None:
if is_valid_hostname(gce_hostname):
return gce_hostname
# Try to get the docker hostname
if Platform.is_containerized():
# First we try from the Docker API
docker_util = DockerUtil()
docker_hostname = docker_util.get_hostname(use_default_gw=False)
if docker_hostname is not None and is_valid_hostname(docker_hostname):
hostname = docker_hostname
elif Platform.is_k8s(): # Let's try from the kubelet
kube_util = KubeUtil()
_, kube_hostname = kube_util.get_node_info()
if kube_hostname is not None and is_valid_hostname(kube_hostname):
hostname = kube_hostname
# then move on to os-specific detection
if hostname is None:
if Platform.is_unix() or Platform.is_solaris():
unix_hostname = _get_hostname_unix()
if unix_hostname and is_valid_hostname(unix_hostname):
hostname = unix_hostname
# if we have an ec2 default hostname, see if there's an instance-id available
if (Platform.is_ecs_instance()) or (hostname is not None and EC2.is_default(hostname)):
instanceid = EC2.get_instance_id(config)
if instanceid:
hostname = instanceid
# fall back on socket.gethostname(), socket.getfqdn() is too unreliable
if hostname is None:
try:
socket_hostname = socket.gethostname()
except socket.error:
socket_hostname = None
if socket_hostname and is_valid_hostname(socket_hostname):
hostname = socket_hostname
if hostname is None:
log.critical('Unable to reliably determine host name. You can define one in datadog.conf or in your hosts file')
raise Exception('Unable to reliably determine host name. You can define one in datadog.conf or in your hosts file')
return hostname
def get_hostname(config=None):
"""
Get the canonical host name this agent should identify as. This is
the authoritative source of the host name for the agent.
Tries, in order:
* agent config (stackstate.conf, "hostname:")
* 'hostname -f' (on unix)
* socket.gethostname()
"""
hostname = None
# first, try the config
if config is None:
from config import get_config
config = get_config(parse_args=True)
config_hostname = config.get('hostname')
if config_hostname and is_valid_hostname(config_hostname):
return config_hostname
# Try to get GCE instance name
gce_hostname = GCE.get_hostname(config)
if gce_hostname is not None:
if is_valid_hostname(gce_hostname):
return gce_hostname
# Try to get the docker hostname
if Platform.is_containerized():
# First we try from the Docker API
docker_util = DockerUtil()
docker_hostname = docker_util.get_hostname(use_default_gw=False)
if docker_hostname is not None and is_valid_hostname(docker_hostname):
hostname = docker_hostname
elif Platform.is_k8s(): # Let's try from the kubelet
kube_util = KubeUtil()
_, kube_hostname = kube_util.get_node_info()
if kube_hostname is not None and is_valid_hostname(kube_hostname):
hostname = kube_hostname
# then move on to os-specific detection
if hostname is None:
if Platform.is_unix() or Platform.is_solaris():
unix_hostname = _get_hostname_unix()
if unix_hostname and is_valid_hostname(unix_hostname):
hostname = unix_hostname
# if we have an ec2 default hostname, see if there's an instance-id available
if (Platform.is_ecs_instance()) or (hostname is not None
and EC2.is_default(hostname)):
instanceid = EC2.get_instance_id(config)
if instanceid:
hostname = instanceid
# fall back on socket.gethostname(), socket.getfqdn() is too unreliable
if hostname is None:
try:
socket_hostname = socket.gethostname()
except socket.error:
socket_hostname = None
if socket_hostname and is_valid_hostname(socket_hostname):
hostname = socket_hostname
if hostname is None:
log.critical(
'Unable to reliably determine host name. You can define one in stackstate.conf or in your hosts file'
)
raise Exception(
'Unable to reliably determine host name. You can define one in stackstate.conf or in your hosts file'
)
return hostname
class TestKubeutil(unittest.TestCase):
def setUp(self):
self.kubeutil = KubeUtil()
@mock.patch('utils.kubeutil.KubeUtil.retrieve_pods_list', side_effect=['foo'])
@mock.patch('utils.kubeutil.KubeUtil.extract_kube_labels')
def test_get_kube_labels(self, extract_kube_labels, retrieve_pods_list):
self.kubeutil.get_kube_labels(excluded_keys='bar')
retrieve_pods_list.assert_called_once()
extract_kube_labels.assert_called_once_with('foo', excluded_keys='bar')
def test_extract_kube_labels(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_kube_labels({}, ['foo'])
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 8)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_kube_labels(pods, ['foo'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
res = self.kubeutil.extract_kube_labels(pods, ['k8s-app'])
labels = set(inn for out in res.values() for inn in out)
self.assertEqual(len(labels), 3)
def test_extract_meta(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.extract_meta({}, 'foo')
self.assertEqual(len(res), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 6)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.extract_meta(pods, 'foo')
self.assertEqual(len(res), 0)
res = self.kubeutil.extract_meta(pods, 'uid')
self.assertEqual(len(res), 4)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_pods_list(self, retrieve_json):
self.kubeutil.retrieve_pods_list()
retrieve_json.assert_called_once_with(self.kubeutil.pods_list_url)
@mock.patch('utils.kubeutil.retrieve_json')
def test_retrieve_metrics(self, retrieve_json):
self.kubeutil.retrieve_metrics()
retrieve_json.assert_called_once_with(self.kubeutil.metrics_url)
def test_filter_pods_list(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
res = self.kubeutil.filter_pods_list({}, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.9')
self.assertEqual(len(res.get('items')), 5)
pods = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, '10.240.0.5')
self.assertEqual(len(res.get('items')), 1)
pods = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
res = self.kubeutil.filter_pods_list(pods, 'foo')
self.assertEqual(len(res.get('items')), 0)
@mock.patch('utils.kubeutil.requests')
def test_retrieve_json_auth(self, r):
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_once_with('url', verify=False, timeout=10, headers={'Authorization': 'Bearer foo_tok'})
self.kubeutil.CA_CRT_PATH = __file__
self.kubeutil.retrieve_json_auth('url', 'foo_tok')
r.get.assert_called_with('url', verify=__file__, timeout=10, headers={'Authorization': 'Bearer foo_tok'})
def test_get_node_info(self):
with mock.patch('utils.kubeutil.KubeUtil._fetch_host_data') as f:
self.kubeutil.get_node_info()
f.assert_called_once()
f.reset_mock()
self.kubeutil._node_ip = 'foo'
self.kubeutil._node_name = 'bar'
ip, name = self.kubeutil.get_node_info()
self.assertEqual(ip, 'foo')
self.assertEqual(name, 'bar')
f.assert_not_called()
def test__fetch_host_data(self):
"""
Test with both 1.1 and 1.2 version payloads
"""
with mock.patch('utils.kubeutil.KubeUtil.retrieve_pods_list') as mock_pods:
self.kubeutil.host_name = 'dd-agent-1rxlh'
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.2.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name, 'kubernetes-massi-minion-k23m')
self.kubeutil.host_name = 'heapster-v11-l8sh1'
mock_pods.return_value = json.loads(Fixtures.read_file("pods_list_1.1.json", string_escape=False))
self.kubeutil._fetch_host_data()
self.assertEqual(self.kubeutil._node_ip, '10.240.0.9')
self.assertEqual(self.kubeutil._node_name, 'gke-cluster-1-8046fdfa-node-ld35')
def test_get_auth_token(self):
KubeUtil.AUTH_TOKEN_PATH = '/foo/bar'
self.assertIsNone(KubeUtil.get_auth_token())
KubeUtil.AUTH_TOKEN_PATH = Fixtures.file('events.json') # any file could do the trick
self.assertIsNotNone(KubeUtil.get_auth_token())
def test_is_k8s(self):
os.unsetenv('KUBERNETES_PORT')
self.assertFalse(Platform.is_k8s())
os.environ['KUBERNETES_PORT'] = '999'
self.assertTrue(Platform.is_k8s())
