def should_run(cls, confd_path):
"""
Return a tuple (jmx_check_configured, java_bin_path)
jmx_check_configured: boolean that shows that either one of the
check in JMX_CHECKS is enabled or there is a configured check
that have the "is_jmx" flag enabled in its init_config
java_bin_path: is the path to the java executable. It was
previously set in the "instance" part of the yaml file of the
jmx check. So we need to parse yaml files to get it.
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
"""
jmx_check_configured = False
java_bin_path = None
for conf in glob.glob(os.path.join(confd_path, '*.yaml')):
if jmx_check_configured and java_bin_path is not None:
return (jmx_check_configured, java_bin_path)
check_name = os.path.basename(conf).split('.')[0]
if os.path.exists(conf):
f = open(conf)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except Exception:
f.close()
log.error("Unable to parse yaml config in %s" % conf)
continue
init_config = check_config.get('init_config', {})
if init_config is None:
init_config = {}
instances = check_config.get('instances', [])
if instances is None:
instances = []
if instances:
if type(instances) != list or len(instances) == 0:
continue
if java_bin_path is None:
if init_config and init_config.get('java_bin_path'):
# We get the java bin path from the yaml file for backward compatibility purposes
java_bin_path = check_config.get('init_config').get('java_bin_path')
for instance in instances:
if instance and instance.get('java_bin_path'):
java_bin_path = instance.get('java_bin_path')
if init_config.get('is_jmx') or check_name in JMX_CHECKS:
jmx_check_configured = True
return (jmx_check_configured, java_bin_path)
def should_run(cls, confd_path):
"""
Return a tuple (jmx_check_configured, java_bin_path)
jmx_check_configured: boolean that shows that either one of the
check in JMX_CHECKS is enabled or there is a configured check
that have the "is_jmx" flag enabled in its init_config
java_bin_path: is the path to the java executable. It was
previously set in the "instance" part of the yaml file of the
jmx check. So we need to parse yaml files to get it.
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
"""
jmx_check_configured = False
java_bin_path = None
for conf in glob.glob(os.path.join(confd_path, '*.yaml')):
if jmx_check_configured and java_bin_path is not None:
return (jmx_check_configured, java_bin_path)
check_name = os.path.basename(conf).split('.')[0]
if os.path.exists(conf):
f = open(conf)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except Exception:
f.close()
log.error("Unable to parse yaml config in %s" % conf)
continue
init_config = check_config.get('init_config', {})
if init_config is None:
init_config = {}
instances = check_config.get('instances', [])
if instances is None:
instances = []
if instances:
if type(instances) != list or len(instances) == 0:
continue
if java_bin_path is None:
if init_config and init_config.get('java_bin_path'):
# We get the java bin path from the yaml file for backward compatibility purposes
java_bin_path = check_config.get('init_config').get('java_bin_path')
for instance in instances:
if instance and instance.get('java_bin_path'):
java_bin_path = instance.get('java_bin_path')
if init_config.get('is_jmx') or check_name in JMX_CHECKS:
jmx_check_configured = True
return (jmx_check_configured, java_bin_path)
def check_yaml(conf_path):
f = open(conf_path)
check_name = os.path.basename(conf_path).split('.')[0]
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert 'init_config' in check_config, "No 'init_config' section found"
assert 'instances' in check_config, "No 'instances' section found"
valid_instances = True
if check_config['instances'] is None or not isinstance(
check_config['instances'], list):
valid_instances = False
else:
for i in check_config['instances']:
if not isinstance(i, dict):
valid_instances = False
break
if not valid_instances:
raise Exception(
'You need to have at least one instance defined in the YAML file for this check'
)
else:
return check_config
finally:
f.close()
def get_jmx_status():
"""This function tries to read the jmxfetch status file which is a yaml file
located in the same directory as the jmxfetch jar file.
Its format is as the following:
###
timestamp: 1377303057441
instances:
cassandra_localhost: {message: null, metric_count: 40}
tomcat: {message: null, metric_count: 57}
instance_name: {message: 'Cannot connect to instance localhost:3033. Is a JMX Server running at this address?', metric_count: 0}
###
"""
check_statuses = []
path = os.path.join(tempfile.gettempdir(), "jmx_status.yaml")
if not os.path.exists(path):
log.debug("There is no jmx_status file at: %s" % path)
return []
try:
jmx_stats = yaml.load(file(path))
status_age = time.time() - jmx_stats.get('timestamp')/1000 # JMX timestamp is saved in milliseconds
jmx_instances = jmx_stats.get('instances', {})
if status_age > 60:
check_statuses.append(CheckStatus("jmx", [InstanceStatus(
0,
STATUS_ERROR,
error="JMXfetch didn't return any metrics during the last minute"
)], 0, 0))
return check_statuses
for instance, info in jmx_instances.iteritems():
message = info.get('message', None)
metric_count = info.get('metric_count', 0)
status = info.get('status')
if status == STATUS_ERROR:
instance_status = InstanceStatus(0, STATUS_ERROR, error=message)
elif status == STATUS_WARNING:
instance_status = InstanceStatus(0, STATUS_WARNING, warnings=[message])
elif status == STATUS_OK:
instance_status = InstanceStatus(0, STATUS_OK)
check_status = CheckStatus(instance, [instance_status], metric_count, 0)
check_statuses.append(check_status)
return check_statuses
except Exception, e:
log.exception("Couldn't load latest jmx status")
return []
def should_run(cls, confd_path, checks_list):
"""
Return a tuple (jmx_checks, invalid_checks, java_bin_path, java_options)
jmx_checks: list of yaml files that are jmx checks
(they have the is_jmx flag enabled or they are in JMX_CHECKS)
and that have at least one instance configured
invalid_checks: dictionary whose keys are check names that are JMX checks but
they have a bad configuration. Values of the dictionary are exceptions generated
when checking the configuration
java_bin_path: is the path to the java executable. It was
previously set in the "instance" part of the yaml file of the
jmx check. So we need to parse yaml files to get it.
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
java_options: is string contains options that will be passed to java_bin_path
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
"""
jmx_checks = []
java_bin_path = None
java_options = None
invalid_checks = {}
for conf in glob.glob(os.path.join(confd_path, '*.yaml')):
java_bin_path_is_set = java_bin_path is not None
java_options_is_set = java_options is not None
filename = os.path.basename(conf)
check_name = filename.split('.')[0]
if os.path.exists(conf):
f = open(conf)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except Exception:
f.close()
log.error("Unable to parse yaml config in %s" % conf)
continue
try:
is_jmx, check_java_bin_path, check_java_options = JMXFetch.is_jmx_check(check_config, check_name, checks_list)
if is_jmx:
jmx_checks.append(filename)
if java_bin_path is None and check_java_bin_path is not None:
java_bin_path = check_java_bin_path
if java_options is None and check_java_options is not None:
java_options = check_java_options
except InvalidJMXConfiguration, e:
log.error("%s check is not a valid jmx configuration: %s" % (check_name, e))
invalid_checks[check_name] = e
def should_run(cls, confd_path, checks_list):
"""
Return a tuple (jmx_checks, invalid_checks, java_bin_path, java_options)
jmx_checks: list of yaml files that are jmx checks
(they have the is_jmx flag enabled or they are in JMX_CHECKS)
and that have at least one instance configured
invalid_checks: dictionary whose keys are check names that are JMX checks but
they have a bad configuration. Values of the dictionary are exceptions generated
when checking the configuration
java_bin_path: is the path to the java executable. It was
previously set in the "instance" part of the yaml file of the
jmx check. So we need to parse yaml files to get it.
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
java_options: is string contains options that will be passed to java_bin_path
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
"""
jmx_checks = []
java_bin_path = None
java_options = None
invalid_checks = {}
for conf in glob.glob(os.path.join(confd_path, '*.yaml')):
filename = os.path.basename(conf)
check_name = filename.split('.')[0]
if os.path.exists(conf):
f = open(conf)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except Exception:
f.close()
log.error("Unable to parse yaml config in %s" % conf)
continue
try:
is_jmx, check_java_bin_path, check_java_options = JMXFetch.is_jmx_check(
check_config, check_name, checks_list)
if is_jmx:
jmx_checks.append(filename)
if java_bin_path is None and check_java_bin_path is not None:
java_bin_path = check_java_bin_path
if java_options is None and check_java_options is not None:
java_options = check_java_options
except InvalidJMXConfiguration, e:
log.error("%s check is not a valid jmx configuration: %s" %
(check_name, e))
invalid_checks[check_name] = e
def from_yaml(cls, path_to_yaml=None, agentConfig=None, yaml_text=None, check_name=None):
"""
A method used for testing your check without running the agent.
"""
from util import yaml, yLoader
if path_to_yaml:
check_name = os.path.basename(path_to_yaml).split('.')[0]
try:
f = open(path_to_yaml)
except IOError:
raise Exception('Unable to open yaml config: %s' % path_to_yaml)
yaml_text = f.read()
f.close()
config = yaml.load(yaml_text, Loader=yLoader)
check = cls(check_name, config.get('init_config') or {}, agentConfig or {})
return check, config.get('instances', [])
def check_yaml(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert 'init_config' in check_config, "No 'init_config' section found"
assert 'instances' in check_config, "No 'instances' section found"
valid_instances = True
if check_config['instances'] is None or not isinstance(check_config['instances'], list):
valid_instances = False
else:
for i in check_config['instances']:
if not isinstance(i, dict):
valid_instances = False
break
if not valid_instances:
raise Exception('You need to have at least one instance defined in the YAML file for this check')
else:
return check_config
finally:
f.close()
def from_yaml(cls,
path_to_yaml=None,
agentConfig=None,
yaml_text=None,
check_name=None):
"""
A method used for testing your check without running the agent.
"""
from util import yaml, yLoader
if path_to_yaml:
check_name = os.path.basename(path_to_yaml).split('.')[0]
try:
f = open(path_to_yaml)
except IOError:
raise Exception('Unable to open yaml config: %s' %
path_to_yaml)
yaml_text = f.read()
f.close()
config = yaml.load(yaml_text, Loader=yLoader)
check = cls(check_name,
config.get('init_config') or {}, agentConfig or {})
return check, config.get('instances', [])
def load_check_directory(agentConfig):
''' Return the checks from checks.d. Only checks that have a configuration
file in conf.d will be returned. '''
from util import yaml, yLoader
from checks import AgentCheck
checks = []
log = logging.getLogger('checks')
osname = getOS()
checks_path = get_checksd_path(osname)
confd_path = get_confd_path(osname)
check_glob = os.path.join(checks_path, '*.py')
# Update the python path before the import
sys.path.append(checks_path)
# For backwards-compatability with old style checks, we have to load every
# checks.d module and check for a corresponding config OR check if the old
# config will "activate" the check.
#
# Once old-style checks aren't supported, we'll just read the configs and
# import the corresponding check module
for check in glob.glob(check_glob):
check_name = os.path.basename(check).split('.')[0]
try:
check_module = __import__(check_name)
except:
log.exception('Unable to import check module %s.py from checks.d' % check_name)
continue
check_class = None
classes = inspect.getmembers(check_module, inspect.isclass)
for name, clsmember in classes:
if clsmember == AgentCheck:
continue
if issubclass(clsmember, AgentCheck):
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
log.error('No check class (inheriting from AgentCheck) found in %s.py' % check_name)
continue
# Check if the config exists OR we match the old-style config
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except:
f.close()
log.exception("Unable to parse yaml config in %s" % conf_path)
continue
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
check_config = check_class.parse_agent_config(agentConfig)
if not check_config:
continue
else:
log.debug('No conf.d/%s.yaml found for checks.d/%s.py' % (check_name, check_name))
continue
# Look for the per-check config, which *must* exist
if not check_config.get('instances'):
log.error("Config %s is missing 'instances'" % conf_path)
continue
# Accept instances as a list, as a single dict, or as non-existant
instances = check_config.get('instances', {})
if type(instances) != type([]):
instances = [instances]
# Init all of the check's classes with
init_config = check_config.get('init_config', {})
# init_config: in the configuration triggers init_config to be defined
# to None.
if init_config is None:
init_config = {}
init_config['instances_number'] = len(instances)
check_class = check_class(check_name, init_config=init_config,
agentConfig=agentConfig)
# Add custom pythonpath(s) if available
if 'pythonpath' in check_config:
pythonpath = check_config['pythonpath']
if not isinstance(pythonpath, list):
pythonpath = [pythonpath]
sys.path.extend(pythonpath)
log.debug('Loaded check.d/%s.py' % check_name)
checks.append({
'name': check_name,
'instances': check_config['instances'],
'class': check_class
})
log.info('checks.d checks: %s' % [c['name'] for c in checks])
return checks
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
log.error('No check class (inheriting from AgentCheck) found in %s.py' % check_name)
continue
# Check if the config exists OR we match the old-style config
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except Exception:
f.close()
log.exception("Unable to parse yaml config in %s" % conf_path)
continue
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
try:
check_config = check_class.parse_agent_config(agentConfig)
except Exception, e:
continue
if not check_config:
continue
d = [
def get_jmx_status():
"""This function tries to read the jmxfetch status file which is a yaml file
located in the same directory as the jmxfetch jar file.
Its format is as the following:
###
timestamp: 1377303057441
instances:
cassandra_localhost: {message: null, metric_count: 40}
tomcat: {message: null, metric_count: 57}
instance_name: {message: 'Cannot connect to instance localhost:3033. Is a JMX Server running at this address?', metric_count: 0}
###
"""
check_statuses = []
path = os.path.join(tempfile.gettempdir(), "jmx_status.yaml")
if not os.path.exists(path):
log.debug("There is no jmx_status file at: %s" % path)
return []
try:
jmx_stats = yaml.load(file(path))
status_age = time.time() - jmx_stats.get(
'timestamp') / 1000 # JMX timestamp is saved in milliseconds
jmx_instances = jmx_stats.get('instances', {})
if status_age > 60:
check_statuses.append(
CheckStatus("jmx", [
InstanceStatus(
0,
STATUS_ERROR,
error=
"JMXfetch didn't return any metrics during the last minute"
)
], 0, 0))
return check_statuses
for instance, info in jmx_instances.iteritems():
message = info.get('message', None)
metric_count = info.get('metric_count', 0)
status = info.get('status')
if status == STATUS_ERROR:
instance_status = InstanceStatus(0,
STATUS_ERROR,
error=message)
elif status == STATUS_WARNING:
instance_status = InstanceStatus(0,
STATUS_WARNING,
warnings=[message])
elif status == STATUS_OK:
instance_status = InstanceStatus(0, STATUS_OK)
check_status = CheckStatus(instance, [instance_status],
metric_count, 0)
check_statuses.append(check_status)
return check_statuses
except Exception, e:
log.exception("Couldn't load latest jmx status")
return []
def load_check_directory(agentConfig):
''' Return the initialized checks from checks.d, and a mapping of checks that failed to
initialize. Only checks that have a configuration
file in conf.d will be returned. '''
from util import yaml, yLoader
from checks import AgentCheck
initialized_checks = {}
init_failed_checks = {}
osname = get_os()
checks_paths = (glob.glob(os.path.join(path, '*.py')) for path
in [agentConfig['additional_checksd'], get_checksd_path(osname)])
confd_path = get_confd_path(osname)
# For backwards-compatability with old style checks, we have to load every
# checks.d module and check for a corresponding config OR check if the old
# config will "activate" the check.
#
# Once old-style checks aren't supported, we'll just read the configs and
# import the corresponding check module
for check in itertools.chain(*checks_paths):
check_name = os.path.basename(check).split('.')[0]
if check_name in initialized_checks or check_name in init_failed_checks:
log.debug('Skipping check %s because it has already been loaded from another location', check)
continue
try:
check_module = imp.load_source('checksd_%s' % check_name, check)
except Exception, e:
traceback_message = traceback.format_exc()
init_failed_checks[check_name] = {'error':e, 'traceback':traceback_message}
log.exception('Unable to import check module %s.py from checks.d' % check_name)
continue
check_class = None
classes = inspect.getmembers(check_module, inspect.isclass)
for _, clsmember in classes:
if clsmember == AgentCheck:
continue
if issubclass(clsmember, AgentCheck):
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
log.error('No check class (inheriting from AgentCheck) found in %s.py' % check_name)
continue
# Check if the config exists OR we match the old-style config
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except:
f.close()
log.exception("Unable to parse yaml config in %s" % conf_path)
continue
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
try:
check_config = check_class.parse_agent_config(agentConfig)
except Exception, e:
continue
if not check_config:
continue
d = [
"Configuring %s in datadog.conf is deprecated." % (check_name),
"Please use conf.d. In a future release, support for the",
"old style of configuration will be dropped.",
]
log.warn(" ".join(d))
def load_check_directory(agentConfig):
''' Return the checks from checks.d. Only checks that have a configuration
file in conf.d will be returned. '''
from util import yaml, yLoader
from checks import AgentCheck
checks = []
log = logging.getLogger('checks')
osname = getOS()
checks_path = get_checksd_path(osname)
confd_path = get_confd_path(osname)
check_glob = os.path.join(checks_path, '*.py')
# Update the python path before the import
sys.path.append(checks_path)
# For backwards-compatability with old style checks, we have to load every
# checks.d module and check for a corresponding config OR check if the old
# config will "activate" the check.
#
# Once old-style checks aren't supported, we'll just read the configs and
# import the corresponding check module
for check in glob.glob(check_glob):
check_name = os.path.basename(check).split('.')[0]
try:
check_module = __import__(check_name)
except:
log.exception('Unable to import check module %s.py from checks.d' %
check_name)
continue
check_class = None
classes = inspect.getmembers(check_module, inspect.isclass)
for name, clsmember in classes:
if clsmember == AgentCheck:
continue
if issubclass(clsmember, AgentCheck):
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
log.error(
'No check class (inheriting from AgentCheck) found in %s.py' %
check_name)
continue
# Check if the config exists OR we match the old-style config
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except:
f.close()
log.exception("Unable to parse yaml config in %s" % conf_path)
continue
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
check_config = check_class.parse_agent_config(agentConfig)
if not check_config:
continue
else:
log.debug('No conf.d/%s.yaml found for checks.d/%s.py' %
(check_name, check_name))
continue
# Look for the per-check config, which *must* exist
if not check_config.get('instances'):
log.error("Config %s is missing 'instances'" % conf_path)
continue
# Accept instances as a list, as a single dict, or as non-existant
instances = check_config.get('instances', {})
if type(instances) != type([]):
instances = [instances]
# Init all of the check's classes with
init_config = check_config.get('init_config', {})
# init_config: in the configuration triggers init_config to be defined
# to None.
if init_config is None:
init_config = {}
init_config['instances_number'] = len(instances)
check_class = check_class(check_name,
init_config=init_config,
agentConfig=agentConfig)
# Add custom pythonpath(s) if available
if 'pythonpath' in check_config:
pythonpath = check_config['pythonpath']
if not isinstance(pythonpath, list):
pythonpath = [pythonpath]
sys.path.extend(pythonpath)
log.debug('Loaded check.d/%s.py' % check_name)
checks.append({
'name': check_name,
'instances': check_config['instances'],
'class': check_class
})
log.info('checks.d checks: %s' % [c['name'] for c in checks])
return checks
def check_yaml_syntax(content):
try:
yaml.load(content, Loader=yLoader)
except Exception, e:
warning_popup("Unable to parse yaml: \n %s" % str(e))
raise
def check_yaml_syntax(content):
try:
yaml.load(content, Loader=yLoader)
except Exception, e:
warning_popup("Unable to parse yaml: \n %s" % str(e))
raise
def get_jmx_status():
"""This function tries to read the 2 jmxfetch status file which are yaml file
located in the temp directory.
There are 2 files:
- One generated by the Agent itself, for jmx checks that can't be initialized because
there are missing stuff.
Its format is as following:
###
invalid_checks:
jmx: !!python/object/apply:jmxfetch.InvalidJMXConfiguration [You need to have at
least one instance defined in the YAML file for this check]
timestamp: 1391040927.136523
###
- One generated by jmxfetch that return information about the collection of metrics
its format is as following:
###
timestamp: 1391037347435
checks:
failed_checks:
jmx:
- {message: Unable to create instance. Please check your yaml file, status: ERROR}
initialized_checks:
tomcat:
- {message: null, status: OK, metric_count: 7, instance_name: jmx-remihakim.fr-3000}
###
"""
check_statuses = []
java_status_path = os.path.join(tempfile.gettempdir(), "jmx_status.yaml")
python_status_path = os.path.join(tempfile.gettempdir(), "jmx_status_python.yaml")
if not os.path.exists(java_status_path) and not os.path.exists(python_status_path):
log.debug("There is no jmx_status file at: %s or at: %s" % (java_status_path, python_status_path))
return []
check_data = defaultdict(lambda: defaultdict(list))
try:
if os.path.exists(java_status_path):
java_jmx_stats = yaml.load(file(java_status_path))
status_age = time.time() - java_jmx_stats.get('timestamp')/1000 # JMX timestamp is saved in milliseconds
jmx_checks = java_jmx_stats.get('checks', {})
if status_age > 60:
check_statuses.append(CheckStatus("jmx", [InstanceStatus(
0,
STATUS_ERROR,
error="JMXfetch didn't return any metrics during the last minute"
)]))
else:
for check_name, instances in jmx_checks.get('failed_checks', {}).iteritems():
for info in instances:
message = info.get('message', None)
metric_count = info.get('metric_count', 0)
status = info.get('status')
instance_name = info.get('instance_name', None)
check_data[check_name]['statuses'].append(get_jmx_instance_status(instance_name, status, message, metric_count))
check_data[check_name]['metric_count'].append(metric_count)
for check_name, instances in jmx_checks.get('initialized_checks', {}).iteritems():
for info in instances:
message = info.get('message', None)
metric_count = info.get('metric_count', 0)
status = info.get('status')
instance_name = info.get('instance_name', None)
check_data[check_name]['statuses'].append(get_jmx_instance_status(instance_name, status, message, metric_count))
check_data[check_name]['metric_count'].append(metric_count)
for check_name, data in check_data.iteritems():
check_status = CheckStatus(check_name, data['statuses'], sum(data['metric_count']))
check_statuses.append(check_status)
if os.path.exists(python_status_path):
python_jmx_stats = yaml.load(file(python_status_path))
jmx_checks = python_jmx_stats.get('invalid_checks', {})
for check_name, excep in jmx_checks.iteritems():
check_statuses.append(CheckStatus(check_name, [], init_failed_error=excep))
return check_statuses
except Exception:
log.exception("Couldn't load latest jmx status")
return []
def load_check_directory(agentConfig):
''' Return the checks from checks.d. Only checks that have a configuration
file in conf.d will be returned. '''
from util import yaml, yLoader
from checks import AgentCheck
checks = []
osname = getOS()
checks_path = get_checksd_path(osname)
confd_path = get_confd_path(osname)
check_glob = os.path.join(checks_path, '*.py')
# Update the python path before the import
sys.path.append(checks_path)
# For backwards-compatability with old style checks, we have to load every
# checks.d module and check for a corresponding config OR check if the old
# config will "activate" the check.
#
# Once old-style checks aren't supported, we'll just read the configs and
# import the corresponding check module
for check in glob.glob(check_glob):
check_name = os.path.basename(check).split('.')[0]
try:
check_module = __import__(check_name)
except:
log.exception('Unable to import check module %s.py from checks.d' % check_name)
continue
check_class = None
classes = inspect.getmembers(check_module, inspect.isclass)
for name, clsmember in classes:
if clsmember == AgentCheck:
continue
if issubclass(clsmember, AgentCheck):
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
log.error('No check class (inheriting from AgentCheck) found in %s.py' % check_name)
continue
# Check if the config exists OR we match the old-style config
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except:
f.close()
log.exception("Unable to parse yaml config in %s" % conf_path)
continue
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
try:
check_config = check_class.parse_agent_config(agentConfig)
except Exception, e:
continue
if not check_config:
continue
d = [
"Configuring %s in datadog.conf is deprecated." % (check_name),
"Please use conf.d. In a future release, support for the",
"old style of configuration will be dropped.",
]
log.warn(" ".join(d))
else:
log.debug('No conf.d/%s.yaml found for checks.d/%s.py' % (check_name, check_name))
continue
# Look for the per-check config, which *must* exist
if not check_config.get('instances'):
log.error("Config %s is missing 'instances'" % conf_path)
continue
# Accept instances as a list, as a single dict, or as non-existant
instances = check_config.get('instances', {})
if type(instances) != type([]):
instances = [instances]
# Init all of the check's classes with
init_config = check_config.get('init_config', {})
# init_config: in the configuration triggers init_config to be defined
# to None.
if init_config is None:
init_config = {}
instances = check_config['instances']
try:
c = check_class(check_name, init_config=init_config,
agentConfig=agentConfig, instances=instances)
except TypeError, e:
# Backwards compatibility for checks which don't support the
# instances argument in the constructor.
c = check_class(check_name, init_config=init_config,
agentConfig=agentConfig)
c.instances = instances
def get_jmx_status():
"""This function tries to read the 2 jmxfetch status file which are yaml file
located in the temp directory.
There are 2 files:
- One generated by the Agent itself, for jmx checks that can't be initialized because
there are missing stuff.
Its format is as following:
###
invalid_checks:
jmx: !!python/object/apply:jmxfetch.InvalidJMXConfiguration [You need to have at
least one instance defined in the YAML file for this check]
timestamp: 1391040927.136523
###
- One generated by jmxfetch that return information about the collection of metrics
its format is as following:
###
timestamp: 1391037347435
checks:
failed_checks:
jmx:
- {message: Unable to create instance. Please check your yaml file, status: ERROR}
initialized_checks:
tomcat:
- {message: null, status: OK, metric_count: 7, instance_name: jmx-remihakim.fr-3000}
###
"""
check_statuses = []
java_status_path = os.path.join(tempfile.gettempdir(), "jmx_status.yaml")
python_status_path = os.path.join(tempfile.gettempdir(), "jmx_status_python.yaml")
if not os.path.exists(java_status_path) and not os.path.exists(python_status_path):
log.debug("There is no jmx_status file at: %s or at: %s" % (java_status_path, python_status_path))
return []
check_data = defaultdict(lambda: defaultdict(list))
try:
if os.path.exists(java_status_path):
java_jmx_stats = yaml.load(file(java_status_path))
status_age = time.time() - java_jmx_stats.get('timestamp')/1000 # JMX timestamp is saved in milliseconds
jmx_checks = java_jmx_stats.get('checks', {})
if status_age > 60:
check_statuses.append(CheckStatus("jmx", [InstanceStatus(
0,
STATUS_ERROR,
error="JMXfetch didn't return any metrics during the last minute"
)], 0, 0))
else:
for check_name, instances in jmx_checks.get('failed_checks', {}).iteritems():
for info in instances:
message = info.get('message', None)
metric_count = info.get('metric_count', 0)
status = info.get('status')
instance_name = info.get('instance_name', None)
check_data[check_name]['statuses'].append(get_jmx_instance_status(instance_name, status, message, metric_count))
check_data[check_name]['metric_count'].append(metric_count)
for check_name, instances in jmx_checks.get('initialized_checks', {}).iteritems():
for info in instances:
message = info.get('message', None)
metric_count = info.get('metric_count', 0)
status = info.get('status')
instance_name = info.get('instance_name', None)
check_data[check_name]['statuses'].append(get_jmx_instance_status(instance_name, status, message, metric_count))
check_data[check_name]['metric_count'].append(metric_count)
for check_name, data in check_data.iteritems():
check_status = CheckStatus(check_name, data['statuses'], sum(data['metric_count']), 0)
check_statuses.append(check_status)
if os.path.exists(python_status_path):
python_jmx_stats = yaml.load(file(python_status_path))
jmx_checks = python_jmx_stats.get('invalid_checks', {})
for check_name, excep in jmx_checks.iteritems():
check_statuses.append(CheckStatus(check_name, [], 0, 0, init_failed_error=excep))
return check_statuses
except Exception, e:
log.exception("Couldn't load latest jmx status")
return []
def load_check_directory(agentConfig):
''' Return the initialized checks from checks.d, and a mapping of checks that failed to
initialize. Only checks that have a configuration
file in conf.d will be returned. '''
from util import yaml, yLoader
from checks import AgentCheck
initialized_checks = {}
init_failed_checks = {}
osname = get_os()
checks_paths = (
glob.glob(os.path.join(path, '*.py')) for path in
[agentConfig['additional_checksd'],
get_checksd_path(osname)])
confd_path = get_confd_path(osname)
# For backwards-compatability with old style checks, we have to load every
# checks.d module and check for a corresponding config OR check if the old
# config will "activate" the check.
#
# Once old-style checks aren't supported, we'll just read the configs and
# import the corresponding check module
for check in itertools.chain(*checks_paths):
check_name = os.path.basename(check).split('.')[0]
if check_name in initialized_checks or check_name in init_failed_checks:
log.debug(
'Skipping check %s because it has already been loaded from another location',
check)
continue
try:
check_module = imp.load_source('checksd_%s' % check_name, check)
except Exception, e:
traceback_message = traceback.format_exc()
init_failed_checks[check_name] = {
'error': e,
'traceback': traceback_message
}
log.exception('Unable to import check module %s.py from checks.d' %
check_name)
continue
check_class = None
classes = inspect.getmembers(check_module, inspect.isclass)
for name, clsmember in classes:
if clsmember == AgentCheck:
continue
if issubclass(clsmember, AgentCheck):
check_class = clsmember
if AgentCheck in clsmember.__bases__:
continue
else:
break
if not check_class:
log.error(
'No check class (inheriting from AgentCheck) found in %s.py' %
check_name)
continue
# Check if the config exists OR we match the old-style config
conf_path = os.path.join(confd_path, '%s.yaml' % check_name)
if os.path.exists(conf_path):
f = open(conf_path)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except:
f.close()
log.exception("Unable to parse yaml config in %s" % conf_path)
continue
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
try:
check_config = check_class.parse_agent_config(agentConfig)
except Exception, e:
continue
if not check_config:
continue
d = [
"Configuring %s in datadog.conf is deprecated." % (check_name),
"Please use conf.d. In a future release, support for the",
"old style of configuration will be dropped.",
]
log.warn(" ".join(d))
def should_run(cls, confd_path):
"""
Return a tuple (jmx_checks, java_bin_path)
jmx_checks: list of yaml files that are jmx checks
(they have the is_jmx flag enabled or they are in JMX_CHECKS)
and that have at least one instance configured
java_bin_path: is the path to the java executable. It was
previously set in the "instance" part of the yaml file of the
jmx check. So we need to parse yaml files to get it.
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
java_options: is string contains options that will be passed to java_bin_path
We assume that this value is alwayws the same for every jmx check
so we can return the first value returned
"""
jmx_checks = []
java_bin_path = None
java_options = None
for conf in glob.glob(os.path.join(confd_path, '*.yaml')):
java_bin_path_is_set = java_bin_path is not None
java_options_is_set = java_options is not None
check_name = os.path.basename(conf).split('.')[0]
if os.path.exists(conf):
f = open(conf)
try:
check_config = yaml.load(f.read(), Loader=yLoader)
assert check_config is not None
f.close()
except Exception:
f.close()
log.error("Unable to parse yaml config in %s" % conf)
continue
init_config = check_config.get('init_config', {})
if init_config is None:
init_config = {}
instances = check_config.get('instances', [])
if instances is None:
instances = []
if instances:
if type(instances) != list or len(instances) == 0:
continue
if java_bin_path is None:
if init_config and init_config.get('java_bin_path'):
# We get the java bin path from the yaml file for backward compatibility purposes
java_bin_path = init_config.get('java_bin_path')
else:
for instance in instances:
if instance and instance.get('java_bin_path'):
java_bin_path = instance.get('java_bin_path')
if java_options is None:
if init_config and init_config.get('java_options'):
java_options = init_config.get('java_options')
else:
for instance in instances:
if instance and instance.get('java_options'):
java_options = instance.get('java_options')
if init_config.get('is_jmx') or check_name in JMX_CHECKS:
jmx_checks.append(os.path.basename(conf))
return (jmx_checks, java_bin_path, java_options)