def get_auto_conf_images(agentConfig):
"""Walk through the auto_config folder and build a dict of auto-configurable images."""
from config import PathNotFound, get_auto_confd_path
auto_conf_images = {
# image_name: check_name
}
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error("Couldn't find the check auto-configuration folder, no auto configuration will be used.")
return None
# walk through the auto-config dir
for yaml_file in os.listdir(auto_confd_path):
check_name = yaml_file.split('.')[0]
try:
# load the config file
auto_conf = check_yaml(urljoin(auto_confd_path, yaml_file))
except Exception as e:
log.error("Enable to load the auto-config, yaml file.\n%s" % str(e))
auto_conf = {}
# extract the supported image list
images = auto_conf.get('docker_images', [])
for image in images:
auto_conf_images[image] = check_name
return auto_conf_images
def get_check_config(self):
from util import check_yaml
from utils.checkfiles import get_conf_path
init_config, instances = {}, []
try:
conf_path = get_conf_path(CHECK_NAME)
except IOError as ex:
log.debug(ex.message)
return init_config, {}
if conf_path is not None and os.path.exists(conf_path):
try:
check_config = check_yaml(conf_path)
init_config, instances = check_config.get('init_config', {}), check_config['instances']
init_config = {} if init_config is None else init_config
except Exception:
log.exception('Docker check configuration file is invalid. The docker check and '
'other Docker related components will not work.')
init_config, instances = {}, []
if len(instances) > 0:
instance = instances[0]
else:
instance = {}
log.error('No instance was found in the docker check configuration.'
' Docker related collection will not work.')
return init_config, instance
def get_auto_conf(agentConfig, check_name):
"""Return the yaml auto_config dict for a check name (None if it doesn't exist)."""
from config import PathNotFound, get_auto_confd_path
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error(
"Couldn't find the check auto-configuration folder, no auto configuration will be used."
)
return None
auto_conf_path = os.path.join(auto_confd_path, '%s.yaml' % check_name)
if not os.path.exists(auto_conf_path):
log.error(
"Couldn't find any auto configuration file for the %s check." %
check_name)
return None
try:
auto_conf = check_yaml(auto_conf_path)
except Exception as e:
log.error("Enable to load the auto-config, yaml file."
"Auto-config will not work for this check.\n%s" % str(e))
return None
return auto_conf
def __init__(self, instance=None):
self.docker_util = DockerUtil()
if instance is None:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self.host = instance.get("host") or self.docker_util.get_hostname()
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.cadvisor_port = instance.get('port', KubeUtil.DEFAULT_CADVISOR_PORT)
self.kubelet_port = instance.get('kubelet_port', KubeUtil.DEFAULT_KUBELET_PORT)
self.kubelet_api_url = '%s://%s:%d' % (self.method, self.host, self.kubelet_port)
self.cadvisor_url = '%s://%s:%d' % (self.method, self.host, self.cadvisor_port)
self.kubernetes_api_url = 'https://%s/api/v1' % (os.environ.get('KUBERNETES_SERVICE_HOST') or self.DEFAULT_MASTER_NAME)
self.metrics_url = urljoin(self.cadvisor_url, KubeUtil.METRICS_PATH)
self.pods_list_url = urljoin(self.kubelet_api_url, KubeUtil.PODS_LIST_PATH)
self.kube_health_url = urljoin(self.kubelet_api_url, 'healthz')
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = defaultdict(int)
def __init__(self, instance=None):
self.docker_util = DockerUtil()
if instance is None:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self.host = instance.get("host") or self.docker_util.get_hostname()
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.cadvisor_port = instance.get('port', KubeUtil.DEFAULT_CADVISOR_PORT)
self.kubelet_port = instance.get('kubelet_port', KubeUtil.DEFAULT_KUBELET_PORT)
self.kubelet_api_url = '%s://%s:%d' % (self.method, self.host, self.kubelet_port)
self.cadvisor_url = '%s://%s:%d' % (self.method, self.host, self.cadvisor_port)
self.kubernetes_api_url = 'https://%s/api/v1' % (os.environ.get('KUBERNETES_SERVICE_HOST') or self.DEFAULT_MASTER_NAME)
self.metrics_url = urljoin(self.cadvisor_url, KubeUtil.METRICS_PATH)
self.pods_list_url = urljoin(self.kubelet_api_url, KubeUtil.PODS_LIST_PATH)
self.kube_health_url = urljoin(self.kubelet_api_url, 'healthz')
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = defaultdict(int)
def __init__(self, config=None):
try:
if config:
ntp_config = config
else:
ntp_config = check_yaml(
os.path.join(get_confd_path(), 'ntp.yaml'))
settings = ntp_config['instances'][0]
except Exception:
settings = {}
self.host = settings.get(
'host') or "{0}.serverdensity.pool.ntp.org".format(
random.randint(0, 3))
self.version = int(settings.get("version") or NTPUtil.DEFAULT_VERSION)
self.port = settings.get('port') or NTPUtil.DEFAULT_PORT
self.timeout = float(
settings.get('timeout') or NTPUtil.DEFAULT_TIMEOUT)
self.args = {
'host': self.host,
'port': self.port,
'version': self.version,
'timeout': self.timeout,
}
def get_check_config(self):
"""Read the config from docker_daemon.yaml"""
from util import check_yaml
from utils.checkfiles import get_conf_path
init_config, instances = {}, []
try:
conf_path = get_conf_path(CHECK_NAME)
except IOError:
log.debug("Couldn't find docker settings, trying with defaults.")
return init_config, {}
if conf_path is not None and os.path.exists(conf_path):
try:
check_config = check_yaml(conf_path)
init_config, instances = check_config.get(
'init_config', {}), check_config['instances']
init_config = {} if init_config is None else init_config
except Exception:
log.exception(
'Docker check configuration file is invalid. The docker check and '
'other Docker related components will not work.')
init_config, instances = {}, []
if len(instances) > 0:
instance = instances[0]
else:
instance = {}
log.error(
'No instance was found in the docker check configuration.'
' Docker related collection will not work.')
return init_config, instance
def get_auto_conf_images(agentConfig):
"""Walk through the auto_config folder and build a dict of auto-configurable images."""
from config import PathNotFound, get_auto_confd_path
auto_conf_images = {
# image_name: check_name
}
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error(
"Couldn't find the check auto-configuration folder, no auto configuration will be used."
)
return None
# walk through the auto-config dir
for yaml_file in os.listdir(auto_confd_path):
check_name = yaml_file.split('.')[0]
try:
# load the config file
auto_conf = check_yaml(urljoin(auto_confd_path, yaml_file))
except Exception as e:
log.error("Enable to load the auto-config, yaml file.\n%s" %
str(e))
auto_conf = {}
# extract the supported image list
images = auto_conf.get('docker_images', [])
for image in images:
auto_conf_images[image] = check_name
return auto_conf_images
def get_check_config(self):
"""Read the config from docker_daemon.yaml"""
from util import check_yaml
from utils.checkfiles import get_conf_path
init_config, instances = {}, []
try:
conf_path = get_conf_path(CHECK_NAME)
except IOError as ex:
log.debug(ex.message)
return init_config, {}
if conf_path is not None and os.path.exists(conf_path):
try:
check_config = check_yaml(conf_path)
init_config, instances = check_config.get('init_config', {}), check_config['instances']
init_config = {} if init_config is None else init_config
except Exception:
log.exception('Docker check configuration file is invalid. The docker check and '
'other Docker related components will not work.')
init_config, instances = {}, []
if len(instances) > 0:
instance = instances[0]
else:
instance = {}
log.error('No instance was found in the docker check configuration.'
' Docker related collection will not work.')
return init_config, instance
def _load_file_config(config_path, check_name, agentConfig):
if config_path == 'deprecated/nagios':
log.warning("Configuring Nagios in datadog.conf is deprecated "
"and will be removed in a future version. "
"Please use conf.d")
check_config = {
'instances': [
dict((key, value) for (key, value) in agentConfig.iteritems()
if key in NAGIOS_OLD_CONF_KEYS)
]
}
return True, check_config, {}
try:
check_config = check_yaml(config_path)
except Exception as e:
log.exception("Unable to parse yaml config in %s" % config_path)
traceback_message = traceback.format_exc()
return False, None, {
check_name: {
'error': str(e),
'traceback': traceback_message
}
}
return True, check_config, {}
def __init__(self):
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error(
'Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self.host = instance.get("host") or self._get_default_router()
self.cadvisor_port = instance.get('port',
KubeUtil.DEFAULT_CADVISOR_PORT)
self.kubelet_port = instance.get('kubelet_port',
KubeUtil.DEFAULT_KUBELET_PORT)
self.metrics_url = urljoin(
'%s://%s:%d' % (self.method, self.host, self.cadvisor_port),
KubeUtil.METRICS_PATH)
self.pods_list_url = urljoin(
'%s://%s:%d' % (self.method, self.host, self.kubelet_port),
KubeUtil.PODS_LIST_PATH)
self.kube_health_url = '%s://%s:%d/healthz' % (self.method, self.host,
self.kubelet_port)
def __init__(self):
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
try:
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self.host = instance.get("host") or self._get_default_router()
self.master_host = instance.get('master_host', self.host)
self.cadvisor_port = instance.get('port', KubeUtil.DEFAULT_CADVISOR_PORT)
self.kubelet_port = instance.get('kubelet_port', KubeUtil.DEFAULT_KUBELET_PORT)
self.master_port = instance.get('master_port', KubeUtil.DEFAULT_MASTER_PORT)
self.metrics_url = urljoin(
'%s://%s:%d' % (self.method, self.host, self.cadvisor_port), KubeUtil.METRICS_PATH)
self.pods_list_url = urljoin(
'%s://%s:%d' % (self.method, self.host, self.kubelet_port), KubeUtil.PODS_LIST_PATH)
self.master_url_nodes = '%s://%s:%d/api/v1/nodes' % (self.method, self.master_host, self.master_port)
self.kube_health_url = '%s://%s:%d/healthz' % (self.method, self.host, self.kubelet_port)
def __init__(self):
self.docker_util = DockerUtil()
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self.host = instance.get("host") or self.docker_util.get_hostname()
self.cadvisor_port = instance.get('port', KubeUtil.DEFAULT_CADVISOR_PORT)
self.kubelet_port = instance.get('kubelet_port', KubeUtil.DEFAULT_KUBELET_PORT)
self.metrics_url = urljoin(
'%s://%s:%d' % (self.method, self.host, self.cadvisor_port), KubeUtil.METRICS_PATH)
self.pods_list_url = urljoin(
'%s://%s:%d' % (self.method, self.host, self.kubelet_port), KubeUtil.PODS_LIST_PATH)
self.kube_health_url = '%s://%s:%d/healthz' % (self.method, self.host, self.kubelet_port)
def __init__(self, instance=None):
self.docker_util = DockerUtil()
if instance is None:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error(
'Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...'
)
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.tls_settings = self._init_tls_settings(instance)
# apiserver
self.kubernetes_api_url = 'https://%s/api/v1' % (
os.environ.get('KUBERNETES_SERVICE_HOST')
or self.DEFAULT_MASTER_NAME)
# kubelet
try:
self.kubelet_api_url = self._locate_kubelet(instance)
if not self.kubelet_api_url:
raise Exception(
"Couldn't find a method to connect to kubelet.")
except Exception as ex:
log.error(
"Kubernetes check exiting, cannot run without access to kubelet."
)
raise ex
self.kubelet_host = self.kubelet_api_url.split(':')[1].lstrip('/')
self.pods_list_url = urljoin(self.kubelet_api_url,
KubeUtil.PODS_LIST_PATH)
self.kube_health_url = urljoin(self.kubelet_api_url,
KubeUtil.KUBELET_HEALTH_PATH)
# cadvisor
self.cadvisor_port = instance.get('port',
KubeUtil.DEFAULT_CADVISOR_PORT)
self.cadvisor_url = '%s://%s:%d' % (self.method, self.kubelet_host,
self.cadvisor_port)
self.metrics_url = urljoin(self.cadvisor_url, KubeUtil.METRICS_PATH)
self.machine_info_url = urljoin(self.cadvisor_url,
KubeUtil.MACHINE_INFO_PATH)
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = 0
def configcheck():
all_valid = True
for conf_path in glob.glob(os.path.join(get_confd_path(), "*.yaml")):
basename = os.path.basename(conf_path)
try:
check_yaml(conf_path)
except NoInstancesFound as e:
print "%s contains warning:\n %s" % (basename, e)
except Exception as e:
all_valid = False
print "%s contains errors:\n %s" % (basename, e)
else:
print "%s is valid" % basename
if all_valid:
print "All yaml files passed. You can now run the ServerDensity agent."
return 0
else:
print("Fix the invalid yaml files above in order to start the sd-agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
def configcheck():
all_valid = True
for conf_path in glob.glob(os.path.join(get_confd_path(), "*.yaml")):
basename = os.path.basename(conf_path)
try:
check_yaml(conf_path)
except NoInstancesFound as e:
print "%s contains warning:\n %s" % (basename, e)
except Exception as e:
all_valid = False
print "%s contains errors:\n %s" % (basename, e)
else:
print "%s is valid" % basename
if all_valid:
print "All yaml files passed. You can now run the ServerDensity agent."
return 0
else:
print(
"Fix the invalid yaml files above in order to start the sd-agent. "
"A useful external tool for yaml parsing can be found at "
"http://yaml-online-parser.appspot.com/")
return 1
def get_auto_conf_images(full_tpl=False):
"""Walk through the auto_config folder and build a dict of auto-configurable images."""
from config import PathNotFound, get_auto_confd_path
auto_conf_images = {
# image_name: [check_names] or [[check_names], [init_tpls], [instance_tpls]]
}
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error(
"Couldn't find the check auto-configuration folder, no auto configuration will be used."
)
return None
# walk through the auto-config dir
for yaml_file in os.listdir(auto_confd_path):
# Ignore files that do not end in .yaml
extension = yaml_file.split('.')[-1]
if extension != 'yaml':
continue
else:
check_name = yaml_file.split('.')[0]
try:
# load the config file
auto_conf = check_yaml(urljoin(auto_confd_path, yaml_file))
except Exception as e:
log.error("Unable to load the auto-config, yaml file.\n%s" %
str(e))
auto_conf = {}
# extract the supported image list
images = auto_conf.get('ad_identifiers',
auto_conf.get('docker_images', []))
for image in images:
if full_tpl:
init_tpl = auto_conf.get('init_config') or {}
instance_tpl = auto_conf.get('instances', [])
if image not in auto_conf_images:
auto_conf_images[image] = [[check_name], [init_tpl],
[instance_tpl]]
else:
for idx, item in enumerate(
[check_name, init_tpl, instance_tpl]):
auto_conf_images[image][idx].append(item)
else:
if image in auto_conf_images:
auto_conf_images[image].append(check_name)
else:
auto_conf_images[image] = [check_name]
return auto_conf_images
def _load_file_config(config_path, check_name, agentConfig):
try:
check_config = check_yaml(config_path)
except Exception as e:
log.exception("Unable to parse yaml config in %s" % config_path)
traceback_message = traceback.format_exc()
return False, None, {
check_name: {
'error': str(e),
'traceback': traceback_message,
'version': 'unknown'
}
}
return True, check_config, {}
def _load_file_config(config_path, check_name, agentConfig):
if config_path == 'deprecated/nagios':
log.warning("Configuring Nagios in datadog.conf is deprecated "
"and will be removed in a future version. "
"Please use conf.d")
check_config = {'instances': [dict((key, value) for (key, value) in agentConfig.iteritems() if key in NAGIOS_OLD_CONF_KEYS)]}
return True, check_config, {}
try:
check_config = check_yaml(config_path)
except Exception, e:
log.exception("Unable to parse yaml config in %s" % config_path)
traceback_message = traceback.format_exc()
return False, None, {check_name: {'error': str(e), 'traceback': traceback_message}}
def detect_is_k8s():
"""
Logic for DockerUtil to detect whether to enable Kubernetes code paths
It check whether we have a KUBERNETES_PORT environment variable (running
in a pod) or a valid kubernetes.yaml conf file
"""
if 'KUBERNETES_PORT' in os.environ:
return True
else:
try:
k8_config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
k8_check_config = check_yaml(k8_config_file_path)
return len(k8_check_config['instances']) > 0
except Exception as err:
log.debug("Error detecting kubernetes: %s" % str(err))
return False
def detect_is_k8s():
"""
Logic for DockerUtil to detect whether to enable Kubernetes code paths
It check whether we have a KUBERNETES_PORT environment variable (running
in a pod) or a valid kubernetes.yaml conf file
"""
if 'KUBERNETES_PORT' in os.environ:
return True
else:
try:
k8_config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
k8_check_config = check_yaml(k8_config_file_path)
return len(k8_check_config['instances']) > 0
except Exception as err:
log.debug("Error detecting kubernetes: %s" % str(err))
return False
def get_auto_conf_images(full_tpl=False):
"""Walk through the auto_config folder and build a dict of auto-configurable images."""
from config import PathNotFound, get_auto_confd_path
auto_conf_images = {
# image_name: [check_names] or [[check_names], [init_tpls], [instance_tpls]]
}
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error("Couldn't find the check auto-configuration folder, no auto configuration will be used.")
return None
# walk through the auto-config dir
for yaml_file in os.listdir(auto_confd_path):
# Ignore files that do not end in .yaml
extension = yaml_file.split('.')[-1]
if extension != 'yaml':
continue
else:
check_name = yaml_file.split('.')[0]
try:
# load the config file
auto_conf = check_yaml(urljoin(auto_confd_path, yaml_file))
except Exception as e:
log.error("Unable to load the auto-config, yaml file.\n%s" % str(e))
auto_conf = {}
# extract the supported image list
images = auto_conf.get('docker_images', [])
for image in images:
if full_tpl:
init_tpl = auto_conf.get('init_config') or {}
instance_tpl = auto_conf.get('instances', [])
if image not in auto_conf_images:
auto_conf_images[image] = [[check_name], [init_tpl], [instance_tpl]]
else:
for idx, item in enumerate([check_name, init_tpl, instance_tpl]):
auto_conf_images[image][idx].append(item)
else:
if image in auto_conf_images:
auto_conf_images[image].append(check_name)
else:
auto_conf_images[image] = [check_name]
return auto_conf_images
def __init__(self, config=None):
try:
if config:
ntp_config = config
else:
ntp_config = check_yaml(os.path.join(get_confd_path(), 'ntp.yaml'))
settings = ntp_config['instances'][0]
except Exception:
settings = {}
self.host = settings.get('host') or "{0}.datadog.pool.ntp.org".format(random.randint(0, 3))
self.version = int(settings.get("version") or NTPUtil.DEFAULT_VERSION)
self.port = settings.get('port') or NTPUtil.DEFAULT_PORT
self.timeout = float(settings.get('timeout') or NTPUtil.DEFAULT_TIMEOUT)
self.args = {
'host': self.host,
'port': self.port,
'version': self.version,
'timeout': self.timeout,
}
def get_auto_conf(check_name):
"""Return the yaml auto_config dict for a check name (None if it doesn't exist)."""
from config import PathNotFound, get_auto_confd_path
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error("Couldn't find the check auto-configuration folder, no auto configuration will be used.")
return None
auto_conf_path = os.path.join(auto_confd_path, '%s.yaml' % check_name)
if not os.path.exists(auto_conf_path):
log.error("Couldn't find any auto configuration file for the %s check." % check_name)
return None
try:
auto_conf = check_yaml(auto_conf_path)
except Exception as e:
log.error("Unable to load the auto-config, yaml file."
"Auto-config will not work for this check.\n%s" % str(e))
return None
return auto_conf
def get_auto_conf_images(agentConfig):
from config import PathNotFound, get_auto_confd_path
auto_conf_images = {}
try:
auto_confd_path = get_auto_confd_path()
except PathNotFound:
log.error(
"Couldn't find the check auto-configuration folder, no auto configuration will be used."
)
return None
for yaml_file in os.listdir(auto_confd_path):
check_name = yaml_file.split('.')[0]
try:
auto_conf = check_yaml(urljoin(auto_confd_path, yaml_file))
except Exception as e:
log.error("Enable to load the auto-config, yaml file.\n%s" %
str(e))
auto_conf = {}
images = auto_conf.get('docker_images', [])
for image in images:
auto_conf_images[image] = check_name
return auto_conf_images
def __init__(self, instance=None):
self.docker_util = DockerUtil()
if instance is None:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.tls_settings = self._init_tls_settings(instance)
# apiserver
if 'api_server_url' in instance:
self.kubernetes_api_root_url = instance.get('api_server_url')
else:
master_host = os.environ.get('KUBERNETES_SERVICE_HOST') or self.DEFAULT_MASTER_NAME
master_port = os.environ.get('KUBERNETES_SERVICE_PORT') or self.DEFAULT_MASTER_PORT
self.kubernetes_api_root_url = 'https://%s:%s' % (master_host, master_port)
self.kubernetes_api_url = '%s/api/v1' % self.kubernetes_api_root_url
# leader status triggers event collection
self.is_leader = False
self.leader_elector = None
self.leader_lease_duration = instance.get('lease_duration')
# kubelet
try:
self.kubelet_api_url = self._locate_kubelet(instance)
if not self.kubelet_api_url:
raise Exception("Couldn't find a method to connect to kubelet.")
except Exception as ex:
log.error("Kubernetes check exiting, cannot run without access to kubelet.")
raise ex
# Service mapping helper class
self._service_mapper = PodServiceMapper(self)
self.kubelet_host = self.kubelet_api_url.split(':')[1].lstrip('/')
self.pods_list_url = urljoin(self.kubelet_api_url, KubeUtil.PODS_LIST_PATH)
self.kube_health_url = urljoin(self.kubelet_api_url, KubeUtil.KUBELET_HEALTH_PATH)
self.kube_label_prefix = instance.get('label_to_tag_prefix', KubeUtil.DEFAULT_LABEL_PREFIX)
self.kube_node_labels = instance.get('node_labels_to_host_tags', {})
# cadvisor
self.cadvisor_port = instance.get('port', KubeUtil.DEFAULT_CADVISOR_PORT)
self.cadvisor_url = '%s://%s:%d' % (self.method, self.kubelet_host, self.cadvisor_port)
self.metrics_url = urljoin(self.cadvisor_url, KubeUtil.METRICS_PATH)
self.machine_info_url = urljoin(self.cadvisor_url, KubeUtil.MACHINE_INFO_PATH)
try:
self.self_namespace = self.get_self_namespace()
except Exception:
log.warning("Failed to get the agent pod namespace, defaulting to default.")
self.self_namespace = DEFAULT_NAMESPACE
from config import _is_affirmative
self.collect_service_tag = _is_affirmative(instance.get('collect_service_tags', KubeUtil.DEFAULT_COLLECT_SERVICE_TAG))
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = 0
sd_init_config, sd_instances = service_disco_configs[
check_name]
check_config = {
'init_config': sd_init_config,
'instances': sd_instances
}
skip_config_lookup = True
else:
conf_path = default_conf_path
conf_exists = True
if not skip_config_lookup:
if conf_exists:
try:
check_config = check_yaml(conf_path)
if agentConfig.get(TRACE_CONFIG):
configs_and_sources[check_name] = (CONFIG_FROM_FILE,
check_config)
except Exception, e:
log.exception("Unable to parse yaml config in %s" %
conf_path)
traceback_message = traceback.format_exc()
init_failed_checks[check_name] = {
'error': str(e),
'traceback': traceback_message
}
continue
else:
# Compatibility code for the Nagios checks if it's still configured
# in datadog.conf
configs_and_sources[check_name] = (
service_disco_configs[check_name][0],
{'init_config': sd_init_config, 'instances': sd_instances})
else:
sd_init_config, sd_instances = service_disco_configs[check_name]
check_config = {'init_config': sd_init_config, 'instances': sd_instances}
skip_config_lookup = True
else:
conf_path = default_conf_path
conf_exists = True
if not skip_config_lookup:
if conf_exists:
try:
check_config = check_yaml(conf_path)
if agentConfig.get(TRACE_CONFIG):
configs_and_sources[check_name] = (CONFIG_FROM_FILE, check_config)
except Exception, e:
log.exception("Unable to parse yaml config in %s" % conf_path)
traceback_message = traceback.format_exc()
init_failed_checks[check_name] = {'error': str(e), 'traceback': traceback_message}
continue
else:
# Compatibility code for the Nagios checks if it's still configured
# in datadog.conf
# FIXME: 6.x, should be removed
if check_name == 'nagios':
if any([nagios_key in agentConfig for nagios_key in NAGIOS_OLD_CONF_KEYS]):
log.warning("Configuring Nagios in datadog.conf is deprecated "
"and will be removed in a future version. "
def __init__(self, **kwargs):
self.docker_util = DockerUtil()
if 'init_config' in kwargs and 'instance' in kwargs:
init_config = kwargs.get('init_config', {})
instance = kwargs.get('instance', {})
else:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
init_config = check_config['init_config'] or {}
instance = check_config['instances'][0] or {}
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
init_config, instance = {}, {}
except Exception:
log.error(
'Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...'
)
init_config, instance = {}, {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.pod_name = os.environ.get('KUBERNETES_POD_NAME') or self.host_name
self.tls_settings = self._init_tls_settings(instance)
# apiserver
if 'api_server_url' in instance:
self.kubernetes_api_root_url = instance.get('api_server_url')
else:
master_host = os.environ.get(
'KUBERNETES_SERVICE_HOST') or self.DEFAULT_MASTER_NAME
master_port = os.environ.get(
'KUBERNETES_SERVICE_PORT') or self.DEFAULT_MASTER_PORT
self.kubernetes_api_root_url = 'https://%s:%s' % (master_host,
master_port)
self.kubernetes_api_url = '%s/api/v1' % self.kubernetes_api_root_url
# Service mapping helper class
self._service_mapper = PodServiceMapper(self)
from config import _is_affirmative
self.collect_service_tag = _is_affirmative(
instance.get('collect_service_tags',
KubeUtil.DEFAULT_COLLECT_SERVICE_TAG))
# leader status triggers event collection
self.is_leader = False
self.leader_elector = None
self.leader_lease_duration = instance.get('leader_lease_duration')
# kubelet
# If kubelet_api_url is None, init_kubelet didn't succeed yet.
self.init_success = False
self.kubelet_api_url = None
self.init_retry_interval = init_config.get('init_retry_interval',
DEFAULT_RETRY_INTERVAL)
self.last_init_retry = None
self.left_init_retries = init_config.get('init_retries',
DEFAULT_INIT_RETRIES) + 1
self.init_kubelet(instance)
self.kube_label_prefix = instance.get('label_to_tag_prefix',
KubeUtil.DEFAULT_LABEL_PREFIX)
self.kube_node_labels = instance.get('node_labels_to_host_tags', {})
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = 0
def __init__(self, instance=None):
self.docker_util = DockerUtil()
if instance is None:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error(
'Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...'
)
instance = {}
self.timeoutSeconds = instance.get("timeoutSeconds",
KubeUtil.DEFAULT_TIMEOUT_SECONDS)
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self.host = instance.get("host") or self.docker_util.get_hostname()
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.cadvisor_port = instance.get('port',
KubeUtil.DEFAULT_CADVISOR_PORT)
self.kubelet_port = instance.get('kubelet_port',
KubeUtil.DEFAULT_KUBELET_PORT)
self.master_method = instance.get('master_method',
KubeUtil.DEFAULT_MASTER_METHOD)
self.master_name = instance.get('master_name',
KubeUtil.DEFAULT_MASTER_NAME)
self.master_port = instance.get('master_port',
KubeUtil.DEFAULT_MASTER_PORT)
self.use_kube_auth = instance.get('use_kube_auth',
KubeUtil.DEFAULT_USE_KUBE_AUTH)
self.kubelet_api_url = '%s://%s:%d' % (self.method, self.host,
self.kubelet_port)
self.cadvisor_url = '%s://%s:%d' % (self.method, self.host,
self.cadvisor_port)
self.master_host = os.environ.get('KUBERNETES_SERVICE_HOST') or (
'%s:%d' % (self.master_name, self.master_port))
self.kubernetes_api_url = '%s://%s/api/v1/' % (self.master_method,
self.master_host)
self.kubernetes_api_extension_url = '%s://%s/apis/extensions/v1beta1/' % (
self.master_method, self.master_host)
self.metrics_url = urljoin(self.cadvisor_url, KubeUtil.METRICS_PATH)
self.machine_info_url = urljoin(self.cadvisor_url,
KubeUtil.MACHINE_INFO_PATH)
self.nodes_list_url = urljoin(self.kubernetes_api_url,
KubeUtil.NODES_LIST_PATH)
self.services_list_url = urljoin(self.kubernetes_api_url,
KubeUtil.SERVICES_LIST_PATH)
self.endpoints_list_url = urljoin(self.kubernetes_api_url,
KubeUtil.ENDPOINTS_LIST_PATH)
self.pods_list_url = urljoin(self.kubernetes_api_url,
KubeUtil.PODS_LIST_PATH)
self.deployments_list_url = urljoin(self.kubernetes_api_extension_url,
KubeUtil.DEPLOYMENTS_LIST_PATH)
self.kube_health_url = urljoin(self.kubelet_api_url, 'healthz')
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = defaultdict(int)
def __init__(self, **kwargs):
self.docker_util = DockerUtil()
if 'init_config' in kwargs and 'instance' in kwargs:
init_config = kwargs.get('init_config', {})
instance = kwargs.get('instance', {})
else:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
init_config = check_config['init_config'] or {}
instance = check_config['instances'][0] or {}
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
init_config, instance = {}, {}
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
init_config, instance = {}, {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.pod_name = os.environ.get('KUBERNETES_POD_NAME') or self.host_name
self.tls_settings = self._init_tls_settings(instance)
# apiserver
if 'api_server_url' in instance:
self.kubernetes_api_root_url = instance.get('api_server_url')
else:
master_host = os.environ.get('KUBERNETES_SERVICE_HOST') or self.DEFAULT_MASTER_NAME
master_port = os.environ.get('KUBERNETES_SERVICE_PORT') or self.DEFAULT_MASTER_PORT
self.kubernetes_api_root_url = 'https://%s:%s' % (master_host, master_port)
self.kubernetes_api_url = '%s/api/v1' % self.kubernetes_api_root_url
# Service mapping helper class
self._service_mapper = PodServiceMapper(self)
from config import _is_affirmative
self.collect_service_tag = _is_affirmative(instance.get('collect_service_tags', KubeUtil.DEFAULT_COLLECT_SERVICE_TAG))
# leader status triggers event collection
self.is_leader = False
self.leader_elector = None
self.leader_lease_duration = instance.get('leader_lease_duration')
# kubelet
# If kubelet_api_url is None, init_kubelet didn't succeed yet.
self.init_success = False
self.kubelet_api_url = None
self.init_retry_interval = init_config.get('init_retry_interval', DEFAULT_RETRY_INTERVAL)
self.last_init_retry = None
self.left_init_retries = init_config.get('init_retries', DEFAULT_INIT_RETRIES) + 1
self.init_kubelet(instance)
self.kube_label_prefix = instance.get('label_to_tag_prefix', KubeUtil.DEFAULT_LABEL_PREFIX)
self.kube_node_labels = instance.get('node_labels_to_host_tags', {})
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = 0
def __init__(self, instance=None):
self.docker_util = DockerUtil()
if instance is None:
try:
config_file_path = get_conf_path(KUBERNETES_CHECK_NAME)
check_config = check_yaml(config_file_path)
instance = check_config['instances'][0]
# kubernetes.yaml was not found
except IOError as ex:
log.error(ex.message)
instance = {}
except Exception:
log.error('Kubernetes configuration file is invalid. '
'Trying connecting to kubelet with default settings anyway...')
instance = {}
self.method = instance.get('method', KubeUtil.DEFAULT_METHOD)
self._node_ip = self._node_name = None # lazy evaluation
self.host_name = os.environ.get('HOSTNAME')
self.tls_settings = self._init_tls_settings(instance)
# apiserver
if 'api_server_url' in instance:
self.kubernetes_api_root_url = instance.get('api_server_url')
else:
master_host = os.environ.get('KUBERNETES_SERVICE_HOST') or self.DEFAULT_MASTER_NAME
master_port = os.environ.get('KUBERNETES_SERVICE_PORT') or self.DEFAULT_MASTER_PORT
self.kubernetes_api_root_url = 'https://%s:%s' % (master_host, master_port)
self.kubernetes_api_url = '%s/api/v1' % self.kubernetes_api_root_url
# kubelet
try:
self.kubelet_api_url = self._locate_kubelet(instance)
if not self.kubelet_api_url:
raise Exception("Couldn't find a method to connect to kubelet.")
except Exception as ex:
log.error("Kubernetes check exiting, cannot run without access to kubelet.")
raise ex
# Service mapping helper class
self._service_mapper = PodServiceMapper(self)
self.kubelet_host = self.kubelet_api_url.split(':')[1].lstrip('/')
self.pods_list_url = urljoin(self.kubelet_api_url, KubeUtil.PODS_LIST_PATH)
self.kube_health_url = urljoin(self.kubelet_api_url, KubeUtil.KUBELET_HEALTH_PATH)
self.kube_label_prefix = instance.get('label_to_tag_prefix', KubeUtil.DEFAULT_LABEL_PREFIX)
self.kube_node_labels = instance.get('node_labels_to_host_tags', {})
# cadvisor
self.cadvisor_port = instance.get('port', KubeUtil.DEFAULT_CADVISOR_PORT)
self.cadvisor_url = '%s://%s:%d' % (self.method, self.kubelet_host, self.cadvisor_port)
self.metrics_url = urljoin(self.cadvisor_url, KubeUtil.METRICS_PATH)
self.machine_info_url = urljoin(self.cadvisor_url, KubeUtil.MACHINE_INFO_PATH)
from config import _is_affirmative
self.collect_service_tag = _is_affirmative(instance.get('collect_service_tags', KubeUtil.DEFAULT_COLLECT_SERVICE_TAG))
# keep track of the latest k8s event we collected and posted
# default value is 0 but TTL for k8s events is one hour anyways
self.last_event_collection_ts = 0
