def options():
parser = import_module(os.getenv('PARSER')).parse_options()
o = parser.parse_args()
logging.config.fileConfig(fileloc(o.log_config))
config = ConfigParser.SafeConfigParser()
config.read(fileloc(o.config))
# parse config in-order
keys = []
for section in config.sections():
for param in config.options(section):
keys.append(param)
setattr(o, param, config.get(section, param))
# cli arguments override configs
for key in provided_arguments(parser):
setattr(o, key, getattr(parser.parse_args(), key))
# environment variables override configs
for key in keys:
setattr(o, key, os.getenv(key.upper(), getattr(o, key)))
return o
def options():
parser = import_module(os.getenv('PARSER')).parse_options()
o = parser.parse_args()
logging.config.fileConfig(fileloc(o.log_config))
config = ConfigParser.SafeConfigParser()
config.read(fileloc(o.config))
# parse config in-order
keys = []
for section in config.sections():
for param in config.options(section):
keys.append(param)
setattr(o, param, config.get(section, param))
# cli arguments override configs
for key in provided_arguments(parser):
setattr(o, key, getattr(parser.parse_args(), key))
# environment variables override configs
for key in keys:
setattr(o, key, os.getenv(key.upper(), getattr(o, key)))
return o
def ConfigMap(section):
config = configparser.ConfigParser()
config.read("/conf/_config.ini")
props = {}
options = config.options(section)
for option in options:
props[option] = config.get(section, option)
return props
def load_config(path):
config = configparser.ConfigParser()
config.read(path)
result = defaultdict(dict)
for s in config.sections():
for o in config.options(s):
result[s][o] = config.get(s, o)
return result
def __init__(self, config, section):
kwargs = dict([(o, config.get(section, o))
for o in config.options(section)])
self.name = section
self.keystone = KeystoneClient(**kwargs)
glance_endpoint = self._get_endpoint('image')
self.glance = GlanceV1Client(glance_endpoint['internalURL'],
token=self.keystone.auth_token)
def __init__(self, config, section):
kwargs = dict([(o, config.get(section, o))
for o in config.options(section)])
self.name = section
self.keystone = KeystoneClient(**kwargs)
glance_endpoint = self._get_endpoint('image')
self.glance = GlanceV1Client(glance_endpoint['internalURL'],
token=self.keystone.auth_token)
def get_config_options(filename):
"Function return dictionary with all options from configuration file"
config = ConfigParser.ConfigParser()
config.read(filename)
conf_options = {}
for section in config.sections():
conf_options[section] = {}
for option in config.options(section):
conf_options[section][option] = config.get(section, option)
return conf_options
def from_config(self, config):
if not config.has_section(self.name):
return {}
res = {}
for opt in config.options(self.name):
value = config.get(self.name, opt)
if not value and opt in self.required:
continue
res[opt] = value
return res
def _adjust_options(self, config):
# config defaults appear in all sections.
# we'll need to filter them out.
defaults = config.defaults().keys()
# see what options are already defined and add missing ones
preset = [(option.section, option.setting) for option in self.items()]
for section in config.sections():
for name in config.options(section):
if ((section, name) not in preset) \
and (name not in defaults):
self.add_option(Option(self, section, name))
def parse_section(config, section):
map = {}
options = config.options(section)
for option in options:
try:
map[option] = config.get(section, option)
if map[option] == -1:
logging.debug("skipping option: %s in section: %s" % option, section)
except:
error_parsing_option = "Error: Parsing option failed on option: %s!" % option
raise Exception(error_parsing_option)
return map
def ConfigSectionMap(section):
dict1 = {}
options = config.options(section)
for option in options:
try:
dict1[option] = config.get(section, option)
if dict1[option] == -1:
print("skip: %s" % option)
except:
print("exception on %s!" % option)
dict1[option] = None
return dict1
def _config_provided(filepath):
config = configparser.ConfigParser()
try:
with open(filepath) as f_prov:
config.read_file(f_prov)
if not config.options('Microsoft Teams'):
raise MissingConnectorConfigKeyException('missing connector key in provided config')
except configparser.NoSectionError:
raise MissingConnectorConfigKeyException('missing required Microsoft Teams / '
'connector key in provided config')
return config
def config_read(cfgFName):
log.debug('Reading config ' + cfgFName)
config = configparser.ConfigParser()
if os.path.exists(cfgFName): config.read(cfgFName, encoding='utf-8')
else: log.debug('Не найден файл конфигурации.')
# в разделе [cols_in] находится список интересующих нас колонок и номера столбцов исходного файла
in_columns_names = config.options('cols_in')
in_columns_j = {}
for vName in in_columns_names:
if ('' != config.get('cols_in', vName)):
in_columns_j[vName] = config.getint('cols_in', vName)
# По разделу [cols_out] формируем перечень выводимых колонок и строку заголовка результирующего CSV файла
out_columns_names = config.options('cols_out')
out_columns = {}
for vName in out_columns_names:
if ('' != config.get('cols_out', vName)):
out_columns[vName] = config.get('cols_out', vName)
return in_columns_j, out_columns
def parse_options(config, section_name, config_class):
options = config.options(section_name)
for option in options:
option = option.upper()
if option in config_class.__dict__ and not option.startswith("__"):
if isinstance(config_class.__dict__[option], bool):
config_class.__dict__[option] = config.getboolean(section_name, option)
elif isinstance(config_class.__dict__[option], float):
config_class.__dict__[option] = config.getfloat(section_name, option)
elif isinstance(config_class.__dict__[option], int):
config_class.__dict__[option] = config.getint(section_name, option)
else:
config_class.__dict__[option] = config.get(section_name, option)
else:
logger = logging.getLogger('InfrastructureManager')
logger.warn("Unknown option in the IM config file. Ignoring it: " + option)
def loadConfig(configFile):
# Charger le fichier de configuration
with open(configFile) as f:
sample_config = f.read()
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.readfp(io.BytesIO(sample_config))
# List all contents
if (False):
print("Parametrage du fichier de configuration :"+configFile)
for section in config.sections():
print("[%s]" % section)
for options in config.options(section):
print("\t%s = %s" % (options,
config.get(section, options)))
return config
def init_ixiacr_logger():
global ixiacr_logger_initialized
if ixiacr_logger_initialized:
return
ixiacr_logger_initialized = True
config = ConfigParser.ConfigParser()
# Filename, LogLevel, modules to override log level
config.read(os.path.join(os.getenv('IXIACR'), 'ixiacrlogger.ini'))
level_str = config.get('ixiacr', 'level')
def get_log_level(level_str):
if level_str[0].isdigit():
log_level = int(level_str)
else:
log_level = logging.getLevelName(level_str)
return log_level
log_level = get_log_level(level_str)
log_format = config.get('ixiacr', 'format', raw=True)
handler = logging.StreamHandler(sys.stdout)
log_formatter = logging.Formatter(log_format)
handler.setFormatter(log_formatter)
root = logging.getLogger()
root.setLevel(log_level)
root.addHandler(handler)
try:
override_level = get_log_level(config.get('ixiacr', 'override_level'))
if override_level != log_level:
overrides = config.options('overrides')
for override in overrides:
root.info('Overriding log level for logger={0} to level={1}'.
format(override, override_level))
olog = logging.getLogger(override)
olog.setLevel(override_level)
except:
pass
def init_logger():
global logger_initialized
if logger_initialized:
return
logger_initialized = True
config = ConfigParser.ConfigParser()
# Filename, LogLevel, modules to override log level
config.read('/home/judo/workspace/github/CRManager/CyberRange/logger.ini')
level_str = config.get('cr', 'level')
def get_log_level(level_str):
if level_str[0].isdigit():
log_level = int(level_str)
else:
log_level = logging.getLevelName(level_str)
return log_level
log_level = get_log_level(level_str)
log_format = config.get('cr', 'format', raw=True)
handler = logging.StreamHandler(sys.stdout)
log_formatter = logging.Formatter(log_format)
handler.setFormatter(log_formatter)
root = logging.getLogger()
root.setLevel(log_level)
root.addHandler(handler)
try:
override_level = get_log_level(config.get('cr', 'override_level'))
if override_level != log_level:
overrides = config.options('overrides')
for override in overrides:
root.info('Overriding log level for logger={0} to level={1}'.
format(override, override_level))
olog = logging.getLogger(override)
olog.setLevel(override_level)
except:
pass
def readConst(section, file):
config = configparser.ConfigParser()
filePath = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'../Utilities/' + file + '.ini')
variable = config.read(filePath)
if variable:
dictionary = {}
try:
options = config.options(section)
except configparser.NoSectionError:
Logger.logr.error("Section " + section + " does not exist in " +
filePath)
else:
for option in options:
dictionary[option] = config.get(section, option)
return dictionary
else:
Logger.logr.info("File " + file +
".ini doesn't exist under provided path:" + filePath)
def parse_options(config, section_name, config_class):
options = config.options(section_name)
for option in options:
option = option.upper()
if option in config_class.__dict__ and not option.startswith("__"):
if isinstance(config_class.__dict__[option], bool):
config_class.__dict__[option] = config.getboolean(
section_name, option)
elif isinstance(config_class.__dict__[option], float):
config_class.__dict__[option] = config.getfloat(
section_name, option)
elif isinstance(config_class.__dict__[option], int):
config_class.__dict__[option] = config.getint(
section_name, option)
else:
config_class.__dict__[option] = config.get(
section_name, option)
else:
logger = logging.getLogger('InfrastructureManager')
logger.warn("Unknown option in the IM config file. Ignoring it: " +
option)
def set_config_data(self, data: dict):
for identifier, update_data in data.items():
# dont rewrite empty...
if not len(update_data):
continue
if identifier == "meta":
hostname = update_data.get("hostname", None)
if hostname:
SysUtil.set_hostname(hostname)
if update_data.get("update", False):
SysUtil.update_from_git()
config = SysUtil.ensure_config(identifier)
sections = set(config.sections()).intersection(set(update_data.keys()))
for section in sections:
update_section = update_data[section]
options = set(config.options(section)).intersection(set(update_section.keys()))
for option in options:
config.set(section, option, str(update_section[option]))
SysUtil.write_config(config, identifier)
def startAgent(agent):
global agents
proc = config.get(agent,'proc')
if agent not in agents:
agents[agent] = {'pid':None,'sp':None,'lasthb':datetime.now()}
agents[agent]['sp'] = subprocess.Popen('python %s &'%proc,shell=True)
agents[agent]['pid'] = agents[agent]['sp'].pid + 1 #shell=True means pid is the shell, so +1 is the agent
agents[agent]['lasthb'] = datetime.now()
agents[agent]['ivy_name'] = config.get(agent,'ivy_name')
if __name__ == "__main__":
config = SafeConfigParser()
config.read('conf/twitterbot.ini')
if 'access_key' not in config.options('Twitter'):
mainlogger.critical('Twitter client needs OAuth dance. Please run it manually, then rerun this script.')
sys.exit(1)
ivy_name = config.get('Manager','ivy_name')
IvyInit(ivy_name,'[%s is ready]'%ivy_name,0,oncxproc,ondieproc)
IvyStart(config.get('General','ivy_bus'))
IvyBindMsg(heartbeat,'^hb_ack')
mainlogger.setLevel(level=getattr(logging,config.get('Manager','log_level')))
timeout = timedelta(seconds=int(config.get('Manager','timeout')))
interested = config.get('Manager','agents').split(',')
for agent in interested:
startAgent(agent)
def get_config(parse_args=True,
cfg_path=None,
init_logging=False,
options=None):
if parse_args:
options, args = get_parsed_args()
elif not options:
args = None
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'debug_mode': False,
'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
'dogstatsd_normalize': 'yes',
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
}
dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=getOS())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
if init_logging:
initialize_logging(config_path, os_name=getOS())
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
#
# Core config
#
if config.has_option('Main', 'use_dd'):
agentConfig['use_dd'] = config.get('Main',
'use_dd').lower() in ("yes",
"true")
else:
agentConfig['use_dd'] = True
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = config.get('Main', 'listen_port')
agentConfig['dd_url'] = "http://localhost:" + str(listen_port)
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Whether also to send to Pup
if config.has_option('Main', 'use_pup'):
agentConfig['use_pup'] = config.get('Main',
'use_pup').lower() in ("yes",
"true")
else:
agentConfig['use_pup'] = True
if agentConfig['use_pup']:
if config.has_option('Main', 'pup_url'):
agentConfig['pup_url'] = config.get('Main', 'pup_url')
else:
agentConfig['pup_url'] = 'http://localhost:17125'
pup_port = 17125
if config.has_option('Main', 'pup_port'):
agentConfig['pup_port'] = int(config.get('Main', 'pup_port'))
# Increases the frequency of statsd metrics when only sending to Pup
if not agentConfig['use_dd'] and agentConfig['use_pup']:
dogstatsd_interval = PUP_STATSD_FREQUENCY
if not agentConfig['use_dd'] and not agentConfig['use_pup']:
sys.stderr.write(
"Please specify at least one endpoint to send metrics to. This can be done in datadog.conf."
)
exit(2)
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# Debug mode
agentConfig['debug_mode'] = config.get(
'Main', 'debug_mode').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = int(
config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'dogstatsd_interval': dogstatsd_interval,
'dogstatsd_normalize': 'yes',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# normalize 'yes'/'no' to boolean
dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(
dogstatsd_defaults['dogstatsd_normalize'])
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
use_ddurl = _is_affirmative(
config.get('Main', 'dogstatsd_use_ddurl'))
if use_ddurl:
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = config.get(
'Main', 'use_mount').lower() in ("yes", "true", "1")
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join(
[log_path,
config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'monitorstatsd_port': 8125,
'monitorstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchmonitor': True,
'additional_checksd': '/etc/monitor-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
if Platform.is_mac():
agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d'
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
# Store developer mode setting in the agentConfig
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(
config.get('Main', 'developer_mode'))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig['developer_mode'] = True
# Get check frequency
if config.has_option("Main", "frequency"):
agentConfig['check_freq'] = config.get("Main", "frequency")
#
# Core config
#
# FIXME unnecessarily complex
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['m_url'] = "http://" + agentConfig[
'bind_host'] + ":" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.m_url:
agentConfig['m_url'] = options.m_url
else:
agentConfig['m_url'] = config.get('Main', 'm_url')
if agentConfig['m_url'].endswith('/'):
agentConfig['m_url'] = agentConfig['m_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'Datamonitor', 'checks.d')
if config.has_option('Main', 'use_monitorstatsd'):
agentConfig['use_monitorstatsd'] = config.get(
'Main', 'use_monitorstatsd').lower() in ("yes", "true")
else:
agentConfig['use_monitorstatsd'] = True
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(
config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(
config.get('Main', 'histogram_percentiles'))
# Disable Watchmonitor (optionally)
if config.has_option('Main', 'watchmonitor'):
if config.get('Main', 'watchmonitor').lower() in ('no', 'false'):
agentConfig['watchmonitor'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# monitorstatsd config
monitorstatsd_defaults = {
'monitorstatsd_port':
8125,
'monitorstatsd_target':
'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in monitorstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
# optionally send monitorstatsd data directly to the agent.
if config.has_option('Main', 'monitorstatsd_use_murl'):
if _is_affirmative(config.get('Main', 'monitorstatsd_use_murl')):
agentConfig['monitorstatsd_target'] = agentConfig['m_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(
config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(
config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datamonitor', 'ddforwarder_log'):
agentConfig['has_datamonitor'] = True
# monitorstream config
if config.has_option("Main", "monitorstream_log"):
# Older version, single log support
log_path = config.get("Main", "monitorstream_log")
if config.has_option("Main", "monitorstream_line_parser"):
agentConfig["monitorstreams"] = ':'.join([
log_path,
config.get("Main", "monitorstream_line_parser")
])
else:
agentConfig["monitorstreams"] = log_path
elif config.has_option("Main", "monitorstreams"):
agentConfig["monitorstreams"] = config.get("Main",
"monitorstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(
config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(
config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(
config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(
config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(
config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.ParsingError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.NoOptionError as e:
sys.stderr.write(
'There are some items missing from your config file, but nothing fatal [%s]'
% e)
# Storing proxy settings in the agentConfig
agentConfig['proxy_settings'] = get_proxy(agentConfig)
if agentConfig.get('ca_certs', None) is None:
agentConfig['ssl_certificate'] = get_ssl_certificate(
get_os(), 'datamonitor-cert.pem')
else:
agentConfig['ssl_certificate'] = agentConfig['ca_certs']
# self-updater relative conf
agentConfig['interval'] = config.get('Main', 'updater_interval')
return agentConfig
def main():
global DBConn, minutesOffset, myMeterId, meterAtable, meterBtable, anyMeterOk
# Determine the complete file paths for the config file and the graph definitions file.
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
configFile = GetConfigFilePath()
# configFileDir = os.path.dirname(configFile)
# Open up the configuration file and extract some parameters.
config.read(configFile)
cfgSection = ProgFile + "/" + os.environ['HOST']
logger.info("INI file cofig section is: %s", cfgSection)
# logger.debug('Config section has options: %s'%set(config.options(cfgSection)))
# logger.debug('Required options are: %s'%RequiredConfigParams)
if not config.has_section(cfgSection):
logger.critical('Config file "%s", has no section "%s".', configFile,
cfgSection)
sys.exit(2)
if len(RequiredConfigParams - set(config.options(cfgSection))) > 0:
logger.critical(
'Config section "%s" does not have all required params: "%s", it has params: "%s".',
cfgSection, RequiredConfigParams, set(config.options(cfgSection)))
logger.debug(
'The missing params are: %s' %
(RequiredConfigParams - set(config.options(cfgSection)), ))
sys.exit(3)
anyMeterOk = config.getboolean(cfgSection,
'use_database_for_any_meter',
fallback=True)
cfg = config[cfgSection]
parser = argparse.ArgumentParser(
description='Display graphs of home parameters.\nDefaults to show all.'
)
parser.add_argument(
"-s",
"--serial_port",
dest="serialPort",
action="store",
default=cfg['meter_serial_port'],
help="The serial port to which the EKM meter is connected.")
parser.add_argument("-i",
"--meterId",
dest="meterId",
action="store",
default=cfg['meter_id'],
help="Numeric Id of EKM meter.")
parser.add_argument(
"-t",
"--start",
dest="startTime",
action="store",
default=None,
help="Date/time at which to start retrieving database data.")
parser.add_argument(
"-o",
"--offsetHours",
dest="hourOffset",
action="store",
default=None,
help=
"Time offset from NOW to retrieve send data to client. Negative values access database, positive values send random digits."
)
parser.add_argument("-a",
"--anyMeter",
dest="anyMeter",
action="store_true",
default=None,
help="Accept any meter Id.")
parser.add_argument("-v",
"--verbosity",
dest="verbosity",
action="count",
help="increase output verbosity",
default=0)
args = parser.parse_args()
Verbosity = args.verbosity
startTime = None
minutesOffset = None
if (args.hourOffset is None) and (args.startTime is None):
minutesOffset = -1440 # default 1 day ago
elif args.startTime is not None:
try:
startTime = datetime.strptime(args.startTime, '%Y-%m-%d %H:%M:%S')
except ValueError as e:
logger.exception(e)
logger.warning(
'Argument for -t (--start) could not be parsed. Using default time offset of 1 day.'
)
if startTime is not None:
logger.debug('Computing minutes offset from now() and startTime.')
minutesOffset = (startTime - datetime.now()) / timedelta(minutes=1)
else:
if (minutesOffset is None) and (args.hourOffset is not None):
minutesOffset = float(args.hourOffset) * 60
else:
minutesOffset = -1440 # default 1 day ago
logger.debug('Using minutesOffset of: %s' % minutesOffset)
if args.anyMeter is not None:
anyMeterOk = args.anyMeter
logger.debug('Any meter is OK? %s' % anyMeterOk)
# setup "serial" port
'''Extract pipe file name (port) from an URL string;
set mode to "client" or "server"'''
try:
parts = urlsplit(args.serialPort)
except URLError:
logger.critical(
'The serial port parameter is not a URL, FakeEKM.py connot proceed.'
)
raise
if parts.scheme != "socket":
raise Exception(
'expected a string in the form "socket://<host>:<port>[?logging={debug|info|warning|error}]": not starting with socket:// (%r)'
% (parts.scheme, ))
# setup database connection
user = cfg['viewer_user']
pwd = cfg['viewer_password']
host = cfg['viewer_host']
port = cfg['viewer_port']
schema = cfg['viewer_schema']
logger.info("user %s" % (user, ))
logger.info("pwd %s" % (pwd, ))
logger.info("host %s" % (host, ))
logger.info("port %s" % (port, ))
logger.info("schema %s" % (schema, ))
# need meterId both as integer and as 12 char string
myMeterIdInt = int(args.meterId)
myMeterId = '%012d' % myMeterIdInt
logger.debug('MeterId as int is %s; as str "%s"' %
(myMeterIdInt, myMeterId))
if myMeterIdInt < 300000000:
logger.critical(
'EKM V3 meters are not supported for database as data source.')
minutesOffset = 1 # => always generate random data
if minutesOffset < 0: # Verify that database tables are available
meterAtable = myMeterId + cfg['meter_table_a_suffix']
meterBtable = myMeterId + cfg['meter_table_b_suffix']
logger.debug('meterAtable is "%s"' % meterAtable)
logger.debug('meterBtable is "%s"' % meterBtable)
connstr = 'mysql+pymysql://{user}:{pwd}@{host}:{port}/{schema}'.format(
user=user, pwd=pwd, host=host, port=port, schema=schema)
logger.debug("database connection string: %s" % (connstr, ))
Eng = create_engine(connstr,
echo=True if Verbosity >= 2 else False,
logging_name=logger.name)
logger.debug(Eng)
try:
with Eng.connect() as DBConn, DBConn.begin():
logger.info("Viewer connection to database established.")
query = "SHOW TABLES LIKE '%s%%'" % myMeterId
logger.debug(
'Checking database for message tables with query: %s' %
query)
result = DBConn.execute(text(query))
requiredTables = [meterAtable, meterBtable]
for r in result:
logger.debug('Checking if %s is a required table.' % r[0])
if r[0] in requiredTables:
requiredTables.remove(r[0])
logger.debug('Table %s is in the database.' % (r[0], ))
if len(requiredTables) == 0:
break
if len(requiredTables) != 0:
logger.critical(
'Not all required tables are in the database; only random messages will be generated.'
)
minutesOffset = 1 # => always generate random data
else:
logger.debug('Found all required tables in database.')
with ThreadedTCPServer((parts.hostname, parts.port),
ThreadedTCPRequestHandler) as server:
ip, port = server.server_address
logger.debug('TCP server started at address "%s:%s"' %
(ip, port))
# Start a thread with the server -- that thread will then start one
# more thread for each request
server_thread = threading.Thread(
target=server.serve_forever)
# Exit the server thread when the main thread terminates
server_thread.daemon = False # was True
server_thread.start()
logger.debug("Server loop running in thread: %s" %
server_thread.name)
response = "keep going"
while response.lower().find("quit") < 0:
response = input(
'Type "quit" to terminate FakeEKM server: ')
logger.info('Server quits.')
server.shutdown()
finally:
pass
logger.info(
' ############## FakeEKM All Done #################')
def get_config(parse_args = True, cfg_path=None, init_logging=False, options=None):
if parse_args:
options, args = get_parsed_args()
elif not options:
args = None
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'debug_mode': False,
'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
'dogstatsd_normalize': 'yes',
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
}
dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=getOS())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
if init_logging:
initialize_logging(config_path, os_name=getOS())
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
#
# Core config
#
if config.has_option('Main', 'use_dd'):
agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true")
else:
agentConfig['use_dd'] = True
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main','listen_port'):
listen_port = config.get('Main','listen_port')
agentConfig['dd_url'] = "http://localhost:" + str(listen_port)
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Whether also to send to Pup
if config.has_option('Main', 'use_pup'):
agentConfig['use_pup'] = config.get('Main', 'use_pup').lower() in ("yes", "true")
else:
agentConfig['use_pup'] = True
if agentConfig['use_pup']:
if config.has_option('Main', 'pup_url'):
agentConfig['pup_url'] = config.get('Main', 'pup_url')
else:
agentConfig['pup_url'] = 'http://localhost:17125'
pup_port = 17125
if config.has_option('Main', 'pup_port'):
agentConfig['pup_port'] = int(config.get('Main', 'pup_port'))
# Increases the frequency of statsd metrics when only sending to Pup
if not agentConfig['use_dd'] and agentConfig['use_pup']:
dogstatsd_interval = PUP_STATSD_FREQUENCY
if not agentConfig['use_dd'] and not agentConfig['use_pup']:
sys.stderr.write("Please specify at least one endpoint to send metrics to. This can be done in datadog.conf.")
exit(2)
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# Debug mode
agentConfig['debug_mode'] = config.get('Main', 'debug_mode').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(config.get('Main', 'check_freq'))
except:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main','graphite_listen_port'):
agentConfig['graphite_listen_port'] = int(config.get('Main','graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port' : 8125,
'dogstatsd_target' : 'http://localhost:17123',
'dogstatsd_interval' : dogstatsd_interval,
'dogstatsd_normalize' : 'yes',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# normalize 'yes'/'no' to boolean
dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(dogstatsd_defaults['dogstatsd_normalize'])
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
use_ddurl = _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl'))
if use_ddurl:
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = config.get('Main', 'use_mount').lower() in ("yes", "true", "1")
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def main():
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
configFile = GetConfigFilePath()
configFileDir = os.path.dirname(configFile)
config.read(configFile)
cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST']
logger.info("INI file cofig section is: %s", cfgSection)
if not config.has_section(cfgSection):
logger.critical('Config file "%s", has no section "%s".', configFile,
cfgSection)
sys.exit(2)
if set(config.options(cfgSection)) < RequiredConfigParams:
logger.critical(
'Config section "%s" does not have all required params: "%s", it has params: "%s".',
cfgSection, RequiredConfigParams, set(config.options(cfgSection)))
sys.exit(3)
cfg = config[cfgSection]
parser = argparse.ArgumentParser(
description=
'Program to read Ambient WS-1002 console archived data and insert it into the database.'
)
parser.add_argument("files",
action="append",
nargs="*",
help="List of CSV files to process.")
parser.add_argument("-b",
"--beginTime",
dest="begin",
action="append",
help="Beginning time(s) to insert data to database.")
parser.add_argument("-e",
"--endTime",
dest="end",
action="append",
help="Ending time(s) to insert data to database.")
parser.add_argument(
"-g",
"--maxGap",
dest="maxGap",
action="store",
help=
"(minutes) Max gap in database data to ignore; gaps bigger that this will be filled in.",
default='20')
parser.add_argument("-d",
"--dryrun",
dest="dryrun",
action="store_true",
help="Don't actually modify database.",
default=False)
parser.add_argument("-v",
"--verbosity",
dest="verbosity",
action="count",
help="increase output verbosity",
default=0)
args = parser.parse_args()
# numDays = float(args.days)
logger.debug('The arguments are: %s' % args)
argfileslist = args.files[0]
logger.debug('There are %s input files on the command line.' %
len(argfileslist))
logger.debug('The input file list is: %s' % (argfileslist))
if len(argfileslist) == 0:
logger.critical('A list of CSV files is required.')
exit(3)
beginTimes = list()
endTimes = list()
if args.begin is not None and args.end is not None:
try:
logger.debug('Intreperting begin times: %s' % args.begin)
for b in args.begin:
logger.debug('Looking at %s' % b)
beginTimes.append(datetime.fromisoformat(b))
except Exception as e:
logger.critical('Unable to parse begin time: "%s"' % args.begin)
logger.exception(e)
exit(3)
try:
logger.debug('Intreperting end times: %s' % args.end)
for b in args.end:
logger.debug('Looking at %s' % b)
endTimes.append(datetime.fromisoformat(b))
except Exception as e:
logger.critical('Unable to parse begin time: "%s"' % args.end)
logger.exception(e)
exit(3)
logger.debug('Begin times: %s, End times: %s' % (beginTimes, endTimes))
if len(beginTimes) != len(endTimes):
logger.critical(
'There must be same number of begin times and end times.')
exit(3)
else:
logger.debug('begin or end time is "None"')
user = cfg['inserter_user']
pwd = cfg['inserter_password']
host = cfg['inserter_host']
port = cfg['inserter_port']
schema = cfg['inserter_schema']
tableName = cfg['weather_table']
logger.info("user %s" % (user, ))
logger.info("pwd %s" % (pwd, ))
logger.info("host %s" % (host, ))
logger.info("port %s" % (port, ))
logger.info("schema %s" % (schema, ))
logger.info("Table %s" % (tableName, ))
'''
SQL commands to view gaps:
create temporary table w1 like weather;
alter table w1 drop primary key;
alter table w1 add column id int auto_increment primary key first;
insert into w1 (dateutc, tempinf, tempf, humidityin, humidity, windspeedmph, windgustmph, maxdailygust, winddir, baromabsin, baromrelin, hourlyrainin, dailyrainin, weeklyrainin, monthlyrainin, yearlyrainin, solarradiation, uv, feelsLike, dewPoint, lastRain, date) select * from weather order by date;
select wB.date as beginTime, wA.date as endTime, round((wA.dateutc - wB.dateutc)/60000) as diff from w1 as wA inner join w1 as wB on wA.id=wB.id+1 where (wA.dateutc - wB.dateutc)/60000 > 20;
drop table w1;
'''
connstr = 'mysql+pymysql://{user}:{pwd}@{host}:{port}/{schema}'.format(
user=user, pwd=pwd, host=host, port=port, schema=schema)
Eng = create_engine(connstr,
echo=True if args.verbosity >= 2 else False,
logging_name=logger.name)
logger.debug(Eng)
with Eng.connect() as conn, conn.begin():
if args.begin is None or args.end is None:
logger.debug('Auto detecting gaps in weather table.')
logger.debug('Create temporary table.')
conn.execute('create temporary table w1 like weather')
logger.debug('Drop primary key on temporary table.')
conn.execute('alter table w1 drop primary key')
logger.debug('Add id column to temporary table.')
conn.execute(
'alter table w1 add column id int auto_increment primary key first'
)
logger.debug('Copy rows from weather to temporary table.')
conn.execute(
'insert into w1 (dateutc, tempinf, tempf, humidityin, humidity, windspeedmph, windgustmph, maxdailygust, winddir, baromabsin, baromrelin, hourlyrainin, dailyrainin, weeklyrainin, monthlyrainin, yearlyrainin, solarradiation, uv, feelsLike, dewPoint, lastRain, date) select * from weather order by date'
)
gapQuery = 'select wB.date as beginTime, wA.date as endTime, round((wA.dateutc - wB.dateutc)/60000) as diff from w1 as wA inner join w1 as wB on wA.id=wB.id+1 where (wA.dateutc - wB.dateutc)/60000 > %s' % args.maxGap
logger.debug('Select time gaps with query: "%s"' % gapQuery)
result = conn.execute(gapQuery)
for row in result:
b = row[0]
e = row[1]
beginTimes.append(b)
endTimes.append(e)
logger.debug(
'Appending %s to beginTimes, %s to endTimes, because the gap was %s'
% (b, e, row[2]))
logger.debug('Auto detected gaps are: begin %s, end %s' %
(beginTimes, endTimes))
logger.debug('Drop temporary table.')
conn.execute('drop table w1')
pass
insertQuery = 'INSERT IGNORE INTO ' + tableName + '(dateutc, date, tempinf, humidityin, tempf, humidity, windspeedmph, windgustmph,\
dewPoint, feelsLike, winddir, baromabsin, baromrelin, hourlyrainin, dailyrainin, weeklyrainin, monthlyrainin, \
yearlyrainin, solarradiation, uv) VALUES \n'
linebegin = ' ('
valuesCount = 0
logger.debug('zipped times: %s' % list(zip(beginTimes, endTimes)))
for beginTime, endTime in list(zip(beginTimes, endTimes)):
logger.debug('Scanning files for entries between "%s" and "%s"' %
(beginTime, endTime))
for theFile in argfileslist:
logger.info('theFile: %s', theFile)
lineCount = 0
with open(theFile, encoding='utf_16_le') as f:
l = f.readline() # discard first line
printFirstTime = True
for l in f:
fields = l.replace('--.-', 'null').replace(
'--', 'null').replace('\n', '').split('\t')
dt = datetime.strptime(fields[1], '%I:%M %p %m/%d/%Y')
if printFirstTime:
logger.debug('First time in file is %s' %
dt.isoformat())
printFirstTime = False
if dt <= beginTime:
continue
if dt >= endTime:
break
lineCount += 1
fields[1] = dt.isoformat()
ol = linebegin + str([
str(round(dt.timestamp() * 1000)), fields[1:9],
fields[9] if fields[9] != 'null' else fields[19],
fields[10:19], fields[21]
]).replace('[', '').replace(']', '').replace(
"'null'", "NULL") + ')\n'
linebegin = ', ('
insertQuery += ol
logger.debug('Last time in file is %s' % dt.isoformat())
logger.debug(
'There were %s lines in the file within the desired times.'
% lineCount)
valuesCount += lineCount
logger.debug('Insert the values.')
insertQuery += ' ON DUPLICATE KEY UPDATE date=value(date)'
logger.debug(
'All together, there were %s values to add to the database.' %
valuesCount)
logger.debug('The insert query is: "%s"' % insertQuery)
if (not args.dryrun) and (valuesCount > 0):
conn.execute(insertQuery)
else:
if valuesCount == 0:
logger.warning('No data found to update database.')
logger.warning("Didn't actually modify database.")
pass
logger.info(' ############## All Done #################')
def parse_config(relation, config, _logger):
from ConfigParser import ConfigParser
from oxsync import OxTaskSync
from ensync import EnClientSync
from tdsync import ToodledoSync
logger = LogAdapter(_logger, {'package': 'config'})
class_map = {
'OxTaskSync': OxTaskSync,
'EnClientSync': EnClientSync,
'ToodledoSync': ToodledoSync
}
relation_section = 'relation' + '_' + relation
if config.has_section(relation_section):
rel = {}
for key in config.options(relation_section):
rel[key] = config.get(relation_section, key)
if rel.get('map'):
rel['map'] = os.path.expanduser(rel.get('map'))
else:
logging.critical('Configuration error: missing map file path for %s' % (relation))
exit(1)
left = config.get(relation_section, 'left')
if left:
engine_section_left = 'engine' + '_' + left
else:
logging.critical('Configuraion error: missing left engine refernce')
exit(1)
left = None
if config.has_section(engine_section_left):
left = {}
for key in config.options(engine_section_left):
left[key] = config.get(engine_section_left, key)
else:
logging.critical('Configuration error: missing section [%s]' % (engine_section_left))
exit(1)
right = config.get(relation_section, 'right')
if right:
engine_section_right = 'engine' + '_' + right
else:
logging.critical('Configuraion error: missing right engine refernce')
exit(1)
right = None
if config.has_section(engine_section_right):
right = {}
for key in config.options(engine_section_right):
right[key] = config.get(engine_section_right, key)
else:
logging.critical('Configuration error: missing section [%s]' % (engine_section_right))
exit(1)
for engine in [left, right]:
secrets = engine['secrets']
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if secret.has_section('secrets'):
engine['secrets'] = {'key': path}
for key in secret.options('secrets'):
engine['secrets'][key] = secret.get('secrets', key)
else:
logger.critical('Configuration error: missing [secrets] in %s' % (path))
exit(1)
else:
if config.has_option('options', 'secrets'):
secrets = config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
section = engine['secrets'][1:-1]
if secret.has_section(section):
engine['secrets'] = {'key': section}
for key in secret.options(section):
engine['secrets'][key] = secret.get(section, key)
else:
logger.critical('Configuration error: missing [%s] in %s' % (section, path))
exit(1)
else:
logger.critical('Configuration error: missing secret file %s' % (path))
exit(1)
else:
logger.critical('Configuration error: missing secrets in [options]')
exit(1)
else:
logging.critical('Configuration error: missing section [%s]' % (relation_section))
exit(1)
for options in [left, right]:
if options.get('class'):
cn = options['class']
if class_map.get(cn):
options['class'] = class_map[cn]
else:
logger.critical('Configuration error: unknown sync engine [%s]' % (cn))
exit(1)
else:
logger.critical('configuration error: missing class tag')
exit(1)
return Options(left), Options(right), Options(rel)
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_normalize': 'yes',
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
'bind_host': get_default_bind_host(),
}
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
#
# Core config
#
# FIXME unnecessarily complex
if config.has_option('Main', 'use_dd'):
agentConfig['use_dd'] = config.get('Main',
'use_dd').lower() in ("yes",
"true")
else:
agentConfig['use_dd'] = True
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://" + agentConfig[
'bind_host'] + ":" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'Datadog', 'checks.d')
if config.has_option('Main', 'use_dogstatsd'):
agentConfig['use_dogstatsd'] = config.get(
'Main', 'use_dogstatsd').lower() in ("yes", "true")
else:
agentConfig['use_dogstatsd'] = True
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
if not agentConfig['use_dd']:
sys.stderr.write(
"Please specify at least one endpoint to send metrics to. This can be done in datadog.conf."
)
exit(2)
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target':
'http://' + agentConfig['bind_host'] + ':17123',
'dogstatsd_normalize': 'yes',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
#Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
# normalize 'yes'/'no' to boolean
dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(
dogstatsd_defaults['dogstatsd_normalize'])
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')):
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(
config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(
config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join(
[log_path,
config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(
config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if config.has_option("Main", "limit_memory_consumption") and \
config.get("Main", "limit_memory_consumption") is not None:
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(
config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(
config.get("Main", "collect_ec2_tags"))
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def parse_input_file(inp_f: str, verbose: bool) -> Dict[str, Dict[str, str]]:
"""Read in input file.
Reads the input file supplied by the user and returns a dict
with the name of run as a key, and the SRR code as value.
First ist searches for a Config section, and sets the option to use ranges
rather than single values for a key, or changes the default prefix
for the codes. If not found, it understands the values as single values
(i.e. not ranges), and uses the default prefix.
At the moment has limited input validation... fingers crossed.
"""
config = configparser.ConfigParser()
config.read(inp_f)
b_ranges = False
# check if we have things defined in Config section, otherwise use defaults
if config.has_option("Config", "ranges"):
if config.get("Config", "ranges") == "True":
logging.debug("Setting it to ranges")
b_ranges = True
if config.has_option("Config", "prefix"):
logging.debug("Setting a non-default prefix")
my_prefix = config.get("Config", "prefix")
else:
my_prefix = PREFIX
if config.has_option("Config", "prefetch_path"):
my_prefetch = config.get("Config", "prefetch_path")
else:
my_prefetch = PREFETCH
if config.has_option("Config", "prefetch_path"):
my_fqdump = config.get("Config", "prefetch_path")
else:
my_fqdump = FQDUMP
if config.has_option("Config", "max_dw_size"):
my_max_dw_size = config.get("Config", "max_dw_size")
else:
my_max_dw_size = MAX_DW_SIZE
my_config = {
"prefetch_exe": my_prefetch,
"fqdump_exe": my_fqdump,
"max_dw_size": my_max_dw_size,
}
if not config.has_section("SRR_code") or len(
config.options("SRR_code")) < 1:
msg = "We could not find SRR_code section, or it was empty.\n\n"
msg += "Add a [SRR_code] section in your ini file, specifying the\n"
msg += "SRR number (without the SRR prefix to the code).\n"
msg += "If your SRR code does not start with SRR, define the prefix\n"
msg += "In the [Config] section, under the key 'prefix'\n"
logging.error(msg)
sys.exit()
sra_dict = {}
if b_ranges:
for item, values in config.items("SRR_code"):
# logging.info(item, values)
ranges = [int(x) for x in values.split(",")]
if len(ranges) != 2 and b_ranges:
msg = "Input range needs to have two integer numbers, "
msg += f"but I found {len(ranges)} for key {item}\n"
msg += f"Make sure to separate the ranges with comas."
logging.error(msg)
sys.exit()
else:
rr = Range(*ranges)
for i, vv in enumerate(range(rr.start, rr.end + 1)):
sra_dict[item + "_" + str(i)] = my_prefix + str(vv)
else:
for item, value in config.items("SRR_code"):
sra_dict[item] = my_prefix + str(value)
my_config["sra_dict"] = sra_dict
if verbose:
logging.info(pformat(my_config))
return my_config
def parse_config(relation, config, _logger):
from ConfigParser import ConfigParser
from oxsync import OxTaskSync
from ensync import EnClientSync
from tdsync import ToodledoSync
logger = LogAdapter(_logger, {'package': 'config'})
class_map = {
'OxTaskSync': OxTaskSync,
'EnClientSync': EnClientSync,
'ToodledoSync': ToodledoSync
}
relation_section = 'relation' + '_' + relation
if config.has_section(relation_section):
rel = {}
for key in config.options(relation_section):
rel[key] = config.get(relation_section, key)
if rel.get('map'):
rel['map'] = os.path.expanduser(rel.get('map'))
else:
logging.critical(
'Configuration error: missing map file path for %s' %
(relation))
exit(1)
left = config.get(relation_section, 'left')
if left:
engine_section_left = 'engine' + '_' + left
else:
logging.critical(
'Configuraion error: missing left engine refernce')
exit(1)
left = None
if config.has_section(engine_section_left):
left = {}
for key in config.options(engine_section_left):
left[key] = config.get(engine_section_left, key)
else:
logging.critical('Configuration error: missing section [%s]' %
(engine_section_left))
exit(1)
right = config.get(relation_section, 'right')
if right:
engine_section_right = 'engine' + '_' + right
else:
logging.critical(
'Configuraion error: missing right engine refernce')
exit(1)
right = None
if config.has_section(engine_section_right):
right = {}
for key in config.options(engine_section_right):
right[key] = config.get(engine_section_right, key)
else:
logging.critical('Configuration error: missing section [%s]' %
(engine_section_right))
exit(1)
for engine in [left, right]:
secrets = engine['secrets']
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if secret.has_section('secrets'):
engine['secrets'] = {'key': path}
for key in secret.options('secrets'):
engine['secrets'][key] = secret.get('secrets', key)
else:
logger.critical(
'Configuration error: missing [secrets] in %s' %
(path))
exit(1)
else:
if config.has_option('options', 'secrets'):
secrets = config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
section = engine['secrets'][1:-1]
if secret.has_section(section):
engine['secrets'] = {'key': section}
for key in secret.options(section):
engine['secrets'][key] = secret.get(
section, key)
else:
logger.critical(
'Configuration error: missing [%s] in %s' %
(section, path))
exit(1)
else:
logger.critical(
'Configuration error: missing secret file %s' %
(path))
exit(1)
else:
logger.critical(
'Configuration error: missing secrets in [options]')
exit(1)
else:
logging.critical('Configuration error: missing section [%s]' %
(relation_section))
exit(1)
for options in [left, right]:
if options.get('class'):
cn = options['class']
if class_map.get(cn):
options['class'] = class_map[cn]
else:
logger.critical(
'Configuration error: unknown sync engine [%s]' % (cn))
exit(1)
else:
logger.critical('configuration error: missing class tag')
exit(1)
return Options(left), Options(right), Options(rel)
def get_config(parse_args = True, cfg_path=None, init_logging=False):
if parse_args:
options, args = get_parsed_args()
else:
options = None
args = None
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'debug_mode': False,
'dogstatsd_interval': 10,
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False,
'version': get_version(),
'watchdog': True,
}
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path)
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
if init_logging:
initialize_logging(config_path)
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
#
# Core config
#
# Where to send the data
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main','listen_port'):
listen_port = config.get('Main', 'listen_port')
agentConfig['dd_url'] = "http://localhost:" + str(listen_port)
elif options is not None and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Debug mode
agentConfig['debug_mode'] = config.get('Main', 'debug_mode').lower() in ("yes", "true")
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(config.get('Main', 'check_freq'))
except:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main','graphite_listen_port'):
agentConfig['graphite_listen_port'] = int(config.get('Main','graphite_listen_port'))
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = config.get('Main', 'use_mount').lower() in ("yes", "true", "1")
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def main():
global Topics
parser = argparse.ArgumentParser(
description='Log MQTT messages to stdOut with timestamp.')
parser.add_argument(
"-t",
"--topic",
dest="topic",
action="append",
help=
"MQTT topic to which to subscribe. May be specified multiple times.")
parser.add_argument("-o",
"--host",
dest="MqttHost",
action="store",
help="MQTT host",
default=None)
parser.add_argument("-p",
"--port",
dest="MqttPort",
action="store",
help="MQTT host port",
type=int,
default=None)
args = parser.parse_args()
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
configFile = GetConfigFilePath()
config.read(configFile)
cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST']
logger.info("INI file cofig section is: %s", cfgSection)
if not config.has_section(cfgSection):
logger.critical('Config file "%s", has no section "%s".', configFile,
cfgSection)
sys.exit(2)
if set(config.options(cfgSection)) < RequiredConfigParams:
logger.critical(
'Config section "%s" does not have all required params: "%s", it has params: "%s".',
cfgSection, RequiredConfigParams, set(config.options(cfgSection)))
sys.exit(3)
logger.info("INI file cofig section is: %s", cfgSection)
cfg = config[cfgSection]
mqtt_host = cfg['mqtt_host']
mqtt_port = cfg['mqtt_port']
if config.has_option(cfgSection, 'mqtt_topics'):
t = cfg['mqtt_topics']
logger.debug('Topics from config is: "%s".', t)
t = t.split()
logger.debug('Split topics from config is: "%s".', t)
Topics = t
logger.debug('Topics from config is: "%s".', Topics)
if (args.topic != None) and (len(args.topic) > 0): Topics = args.topic
if (args.MqttHost != None) and (len(args.MqttHost) > 0):
mqtt_host = args.MqttHost
if (args.MqttPort != None) and (len(args.MqttPort) > 0):
mqtt_port = args.MqttPort
mqtt_port = int(mqtt_port)
if (mqtt_host is None) or (mqtt_port is None) or (len(Topics) == 0):
logger.critical(
'No mqtt_host OR no mqtt_port OR no topics; must quit.')
sys.exit(1)
try:
logger.debug(
'Connecting to MQTT server at "%s", on port "%s", for topics "%s".',
mqtt_host, mqtt_port, Topics)
RecClient.connect(mqtt_host, mqtt_port, 60)
logger.debug('Mqtt connect returned.')
# while not RecClient.connected_flag: #wait in loop
# logger.debug("In wait loop")
# time.sleep(2)
# logger.debug("connection successful")
except Exception as e:
logger.exception(e)
sys.exit(2)
try:
nowTime = time.time()
localtime = time.localtime(nowTime)
timeUs = (nowTime - int(nowTime)) * 1000000
logger.debug("Begin receive loop at: %s.%06d %s" %
(time.strftime("%Y-%m-%d %H:%M:%S", localtime), timeUs,
time.strftime("%Z", localtime)))
RecClient.loop_forever()
finally:
logger.debug('Executing finally clause.')
RecClient.disconnect()
pass
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
if Platform.is_mac():
agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d'
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
# Store developer mode setting in the agentConfig
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(config.get('Main', 'developer_mode'))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig['developer_mode'] = True
#
# Core config
#ap
if not config.has_option('Main', 'api_key'):
log.warning(u"No API key was found. Aborting.")
sys.exit(2)
if not config.has_option('Main', 'dd_url'):
log.warning(u"No dd_url was found. Aborting.")
sys.exit(2)
# Endpoints
dd_urls = map(clean_dd_url, config.get('Main', 'dd_url').split(','))
api_keys = map(lambda el: el.strip(), config.get('Main', 'api_key').split(','))
# For collector and dogstatsd
agentConfig['dd_url'] = dd_urls[0]
agentConfig['api_key'] = api_keys[0]
# Forwarder endpoints logic
# endpoints is:
# {
# 'https://app.datadoghq.com': ['api_key_abc', 'api_key_def'],
# 'https://app.example.com': ['api_key_xyz']
# }
endpoints = {}
dd_urls = remove_empty(dd_urls)
api_keys = remove_empty(api_keys)
if len(dd_urls) == 1:
if len(api_keys) > 0:
endpoints[dd_urls[0]] = api_keys
else:
assert len(dd_urls) == len(api_keys), 'Please provide one api_key for each url'
for i, dd_url in enumerate(dd_urls):
endpoints[dd_url] = endpoints.get(dd_url, []) + [api_keys[i]]
agentConfig['endpoints'] = endpoints
# Forwarder or not forwarder
agentConfig['use_forwarder'] = options is not None and options.use_forwarder
if agentConfig['use_forwarder']:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://{}:{}".format(agentConfig['bind_host'], listen_port)
# FIXME: Legacy dd_url command line switch
elif options is not None and options.dd_url is not None:
agentConfig['dd_url'] = options.dd_url
# Forwarder timeout
agentConfig['forwarder_timeout'] = 20
if config.has_option('Main', 'forwarder_timeout'):
agentConfig['forwarder_timeout'] = int(config.get('Main', 'forwarder_timeout'))
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d')
if config.has_option('Main', 'use_dogstatsd'):
agentConfig['use_dogstatsd'] = config.get('Main', 'use_dogstatsd').lower() in ("yes", "true")
else:
agentConfig['use_dogstatsd'] = True
# Service discovery
if config.has_option('Main', 'service_discovery_backend'):
try:
additional_config = extract_agent_config(config)
agentConfig.update(additional_config)
except:
log.error('Failed to load the agent configuration related to '
'service discovery. It will not be used.')
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(config.get('Main', 'check_freq'))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(config.get('Main', 'histogram_percentiles'))
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port'))
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.ParsingError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.NoOptionError as e:
sys.stderr.write('There are some items missing from your config file, but nothing fatal [%s]' % e)
# Storing proxy settings in the agentConfig
agentConfig['proxy_settings'] = get_proxy(agentConfig)
if agentConfig.get('ca_certs', None) is None:
agentConfig['ssl_certificate'] = get_ssl_certificate(get_os(), 'datadog-cert.pem')
else:
agentConfig['ssl_certificate'] = agentConfig['ca_certs']
return agentConfig
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agent_config = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'monstatsd_interval': DEFAULT_STATSD_FREQUENCY,
'monstatsd_agregator_bucket_size': DEFAULT_STATSD_BUCKET_SIZE,
'monstatsd_normalize': 'yes',
'monstatsd_port': 8125,
'forwarder_url': 'http://localhost:17123',
'hostname': None,
'listen_port': None,
'version': get_version(),
'watchdog': True,
'additional_checksd': DEFAULT_CONFIG_DIR + '/checks_d/',
}
monstatsd_interval = DEFAULT_STATSD_FREQUENCY
monstatsd_agregator_bucket_size = DEFAULT_STATSD_BUCKET_SIZE
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agent_config[option] = config.get('Main', option)
#
# Core config
#
# FIXME unnecessarily complex
# Extra checks_d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agent_config['additional_checksd'] = config.get('Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agent_config['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks_d')
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agent_config['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agent_config['use_web_info_page'] = True
# local traffic only? Default to no
agent_config['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agent_config['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
if config.has_option('Main', 'check_freq'):
try:
agent_config['check_freq'] = int(config.get('Main', 'check_freq'))
except Exception:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agent_config['watchdog'] = False
# monstatsd config
monstatsd_defaults = {
'monstatsd_port': 8125,
'monstatsd_interval': monstatsd_interval,
'monstatsd_agregator_bucket_size': monstatsd_agregator_bucket_size,
'monstatsd_normalize': 'yes',
}
for key, value in monstatsd_defaults.iteritems():
if config.has_option('Main', key):
agent_config[key] = config.get('Main', key)
else:
agent_config[key] = value
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agent_config['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agent_config['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port'))
# normalize 'yes'/'no' to boolean
monstatsd_defaults['monstatsd_normalize'] = _is_affirmative(
monstatsd_defaults['monstatsd_normalize'])
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agent_config['use_mount'] = _is_affirmative(config.get('Main', 'use_mount'))
if config.has_option('Main', 'autorestart'):
agent_config['autorestart'] = _is_affirmative(config.get('Main', 'autorestart'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agent_config['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agent_config['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agent_config["dogstreams"] = ':'.join(
[log_path, config.get("Main", "dogstream_line_parser")])
else:
agent_config["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agent_config["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agent_config["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_section('WMI'):
agent_config['WMI'] = {}
for key, value in config.items('WMI'):
agent_config['WMI'][key] = value
if config.has_option("Main", "limit_memory_consumption") and \
config.get("Main", "limit_memory_consumption") is not None:
agent_config["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agent_config["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agent_config["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agent_config['Api'] = get_mon_api_config(config)
except ConfigParser.NoSectionError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.ParsingError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.NoOptionError as e:
sys.stderr.write(
'There are some items missing from your config file, but nothing fatal [%s]' % e)
# Storing proxy settings in the agent_config
agent_config['proxy_settings'] = get_proxy(agent_config)
return agent_config
def main():
global Topics, TIME_SYNC_UPDATE_MSEC_TOPIC, TIME_SYNC_UPDATE_TOPIC, DontPublish
parser = argparse.ArgumentParser(
description='Respond to Time Sync request messages with current time.')
parser.add_argument(
"-t",
"--topic",
dest="topic",
action="append",
help=
"MQTT topic to which to subscribe. May be specified multiple times.")
parser.add_argument("-o",
"--host",
dest="MqttHost",
action="store",
help="MQTT host",
default=None)
parser.add_argument("-p",
"--port",
dest="MqttPort",
action="store",
help="MQTT host port",
type=int,
default=None)
parser.add_argument("-P",
"--DontPublish",
dest="dontpub",
action="store_true",
help="Do not actually publish time syncs.",
default=False)
args = parser.parse_args()
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
configFile = GetConfigFilePath()
config.read(configFile)
cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST']
logger.info("INI file cofig section is: %s", cfgSection)
if not config.has_section(cfgSection):
logger.critical('Config file "%s", has no section "%s".', configFile,
cfgSection)
sys.exit(2)
if set(config.options(cfgSection)) < RequiredConfigParams:
logger.critical(
'Config section "%s" does not have all required params: "%s", it has params: "%s".',
cfgSection, RequiredConfigParams, set(config.options(cfgSection)))
sys.exit(3)
logger.info("INI file cofig section is: %s", cfgSection)
cfg = config[cfgSection]
mqtt_host = cfg['mqtt_host']
mqtt_port = cfg['mqtt_port']
TIME_SYNC_UPDATE_MSEC_TOPIC = cfg['mqtt_sync_ms_topic']
TIME_SYNC_UPDATE_TOPIC = cfg['mqtt_sync_topic']
if config.has_option(cfgSection, 'mqtt_request_topics'):
t = cfg['mqtt_request_topics']
logger.debug('Topics from config is: "%s".', t)
t = t.split()
logger.debug('Split topics from config is: "%s".', t)
Topics = t
logger.debug('Topics from config is: "%s".', Topics)
if (args.topic != None) and (len(args.topic) > 0): Topics = args.topic
if (args.MqttHost != None) and (len(args.MqttHost) > 0):
mqtt_host = args.MqttHost
if (args.MqttPort != None) and (len(args.MqttPort) > 0):
mqtt_port = args.MqttPort #DontPublish
if (args.dontpub != None): DontPublish = args.dontpub
mqtt_port = int(mqtt_port)
if (mqtt_host is None) or (mqtt_port is None) or (len(Topics) == 0):
logger.critical(
'No mqtt_host OR no mqtt_port OR no topics; must quit.')
sys.exit(1)
try:
logger.debug(
'Connecting to MQTT server at "%s", on port "%s", for topics "%s".',
mqtt_host, mqtt_port, Topics)
RecClient.connect(mqtt_host, mqtt_port, 60)
logger.debug('Mqtt connect returned.')
logger.debug("Loop immediately after connect returns code: %s",
mqtt.error_string(RecClient.loop()))
if (RecClient._state != mqtt.mqtt_cs_connected):
logger.debug("Not immediately connected.")
except Exception as e:
logger.exception(e)
sys.exit(2)
try:
nowTime = time.time()
localtime = time.localtime(nowTime)
timeUs = (nowTime - int(nowTime)) * 1000000
logger.debug("Begin receive loop at: %s.%06d %s" %
(time.strftime("%Y-%m-%d %H:%M:%S", localtime), timeUs,
time.strftime("%Z", localtime)))
RecClient.loop_forever()
finally:
logger.debug('Executing finally clause.')
RecClient.disconnect()
pass
def main():
global Topics, mqtt_msg_table, DBConn, DBCursor, dontWriteDb, RecClient
parser = argparse.ArgumentParser(
description='Log MQTT messages to database.')
parser.add_argument(
"-t",
"--topic",
dest="topic",
action="append",
help=
"MQTT topic to which to subscribe. May be specified multiple times.")
parser.add_argument("-o",
"--host",
dest="MqttHost",
action="store",
help="MQTT host",
default=None)
parser.add_argument("-p",
"--port",
dest="MqttPort",
action="store",
help="MQTT host port",
type=int,
default=None)
parser.add_argument("-O",
"--Host",
dest="DbHost",
action="store",
help="Database host",
default=None)
parser.add_argument("-P",
"--Port",
dest="DbPort",
action="store",
help="Database host port",
type=int,
default=None)
parser.add_argument("-U",
"--User",
dest="DbUser",
action="store",
help="Database user name",
default=None)
parser.add_argument("-D",
"--Password",
dest="DbPass",
action="store",
help="Database user password",
default=None)
parser.add_argument("-S",
"--Schema",
dest="DbSchema",
action="store",
help="Database schema",
default=None)
parser.add_argument("-T",
"--MsgTable",
dest="MsgTable",
action="store",
help="Database table in which to store messages.",
default=None)
parser.add_argument("-W",
"--dontWriteToDB",
dest="noWriteDb",
action="store_true",
help="Don't write to database.")
args = parser.parse_args()
dontWriteDb = args.noWriteDb
config = configparser.ConfigParser(
interpolation=configparser.ExtendedInterpolation())
configFile = GetConfigFilePath()
config.read(configFile)
cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST']
logger.info("INI file cofig section is: %s", cfgSection)
if not config.has_section(cfgSection):
logger.critical('Config file "%s", has no section "%s".', configFile,
cfgSection)
sys.exit(2)
if set(config.options(cfgSection)) < RequiredConfigParams:
logger.critical(
'Config section "%s" does not have all required params: "%s", it has params: "%s".',
cfgSection, RequiredConfigParams, set(config.options(cfgSection)))
sys.exit(3)
cfg = config[cfgSection]
db_user = cfg['inserter_user']
db_pwd = cfg['inserter_password']
db_host = cfg['inserter_host']
db_port = int(cfg['inserter_port'])
myschema = cfg['inserter_schema']
mqtt_host = cfg['mqtt_host']
mqtt_port = cfg['mqtt_port']
mqtt_msg_table = cfg['mqtt_msg_table']
# Create mqtt client and establish call-backs.
RecClient = mqtt.Client()
RecClient.on_connect = on_connect
RecClient.on_message = on_message
RecClient.on_subscribe = on_subscribe
if config.has_option(cfgSection, 'mqtt_topics'):
t = cfg['mqtt_topics']
logger.debug('Topics from config is: "%s".', t)
t = t.split()
logger.debug('Split topics from config is: "%s".', t)
Topics = t
logger.debug('Topics from config is: "%s".', Topics)
if (args.topic is not None) and (len(args.topic) > 0): Topics = args.topic
if (args.MqttHost is not None) and (len(args.MqttHost) > 0):
mqtt_host = args.MqttHost
if (args.MqttPort is not None) and (len(args.MqttPort) > 0):
mqtt_port = args.MqttPort
if (args.MsgTable is not None) and (len(args.MsgTable) > 0):
mqtt_msg_table = args.MsgTable
if (args.DbHost is not None) and (len(args.DbHost) > 0):
db_host = args.DbHost
if (args.DbPass is not None) and (len(args.DbPass) > 0):
db_pwd = args.DbPass
if (args.DbUser is not None) and (len(args.DbUser) > 0):
db_user = args.DbUser
if (args.DbPort is not None) and (len(args.DbPort) > 0):
db_port = args.DbPort
if (args.DbSchema is not None) and (len(args.DbSchema) > 0):
myschema = args.DbSchema
db_port = int(db_port)
mqtt_port = int(mqtt_port)
if len(Topics) <= 0:
logger.critical('No mqtt subscription topics given. Must exit.')
sys.exit(4)
logger.info(
'Database connection args: host: "%s", port: %d, User: "******", Pass: REDACTED, Schema: "%s"',
db_host, db_port, db_user, myschema)
logger.info(
'Mqtt parameters: host: "%s", port: %d, topic(s): "%s", mqtt msg table: "%s".',
mqtt_host, mqtt_port, Topics, mqtt_msg_table)
try:
DBConn = mysql.connector.connect(host=db_host,
port=db_port,
user=db_user,
password=db_pwd,
db=myschema,
charset='utf8mb4')
if DBConn.is_connected():
logger.info('Connected to MySQL database')
logger.info(DBConn)
DBConn.start_transaction()
DBCursor = DBConn.cursor()
try:
RecClient.connect(mqtt_host, mqtt_port, 60)
RecClient.loop_forever()
except Exception as e:
logger.info(
'RecClient.loop_forever() raised exception; exit main(); Wait awhile, then quit -- Launchctl will restart us.'
)
logger.exception(e)
raise
finally:
logger.info(
'RecClient.loop_forever() returned; exit main(); Wait awhile, then quit -- Launchctl will restart us.'
)
RecClient.disconnect()
DBConn.disconnect()
return
else:
logger.error('Failed to connect to database.')
except Error as e:
logger.critical(e)
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
"check_freq": DEFAULT_CHECK_FREQUENCY,
"dogstatsd_port": 8125,
"dogstatsd_target": "http://localhost:17123",
"graphite_listen_port": None,
"hostname": None,
"listen_port": None,
"tags": None,
"use_ec2_instance_id": False, # DEPRECATED
"version": get_version(),
"watchdog": True,
"additional_checksd": "/etc/dd-agent/checks.d/",
"bind_host": get_default_bind_host(),
"statsd_metric_namespace": None,
"utf8_decoding": False,
}
if Platform.is_mac():
agentConfig["additional_checksd"] = "/opt/datadog-agent/etc/checks.d"
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options("Main"):
agentConfig[option] = config.get("Main", option)
# Store developer mode setting in the agentConfig
in_developer_mode = None
if config.has_option("Main", "developer_mode"):
agentConfig["developer_mode"] = _is_affirmative(config.get("Main", "developer_mode"))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig["developer_mode"] = True
#
# Core config
#
# FIXME unnecessarily complex
agentConfig["use_forwarder"] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option("Main", "listen_port"):
listen_port = int(config.get("Main", "listen_port"))
agentConfig["dd_url"] = "http://" + agentConfig["bind_host"] + ":" + str(listen_port)
agentConfig["use_forwarder"] = True
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig["dd_url"] = options.dd_url
else:
agentConfig["dd_url"] = config.get("Main", "dd_url")
if agentConfig["dd_url"].endswith("/"):
agentConfig["dd_url"] = agentConfig["dd_url"][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option("Main", "additional_checksd"):
agentConfig["additional_checksd"] = config.get("Main", "additional_checksd")
elif get_os() == "windows":
# default windows location
common_path = _windows_commondata_path()
agentConfig["additional_checksd"] = os.path.join(common_path, "Datadog", "checks.d")
if config.has_option("Main", "use_dogstatsd"):
agentConfig["use_dogstatsd"] = config.get("Main", "use_dogstatsd").lower() in ("yes", "true")
else:
agentConfig["use_dogstatsd"] = True
# Concerns only Windows
if config.has_option("Main", "use_web_info_page"):
agentConfig["use_web_info_page"] = config.get("Main", "use_web_info_page").lower() in ("yes", "true")
else:
agentConfig["use_web_info_page"] = True
# Which API key to use
agentConfig["api_key"] = config.get("Main", "api_key")
# local traffic only? Default to no
agentConfig["non_local_traffic"] = False
if config.has_option("Main", "non_local_traffic"):
agentConfig["non_local_traffic"] = config.get("Main", "non_local_traffic").lower() in ("yes", "true")
# DEPRECATED
if config.has_option("Main", "use_ec2_instance_id"):
use_ec2_instance_id = config.get("Main", "use_ec2_instance_id")
# translate yes into True, the rest into False
agentConfig["use_ec2_instance_id"] = use_ec2_instance_id.lower() == "yes"
if config.has_option("Main", "check_freq"):
try:
agentConfig["check_freq"] = int(config.get("Main", "check_freq"))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option("Main", "histogram_aggregates"):
agentConfig["histogram_aggregates"] = get_histogram_aggregates(config.get("Main", "histogram_aggregates"))
if config.has_option("Main", "histogram_percentiles"):
agentConfig["histogram_percentiles"] = get_histogram_percentiles(
config.get("Main", "histogram_percentiles")
)
# Disable Watchdog (optionally)
if config.has_option("Main", "watchdog"):
if config.get("Main", "watchdog").lower() in ("no", "false"):
agentConfig["watchdog"] = False
# Optional graphite listener
if config.has_option("Main", "graphite_listen_port"):
agentConfig["graphite_listen_port"] = int(config.get("Main", "graphite_listen_port"))
else:
agentConfig["graphite_listen_port"] = None
# Dogstatsd config
dogstatsd_defaults = {
"dogstatsd_port": 8125,
"dogstatsd_target": "http://" + agentConfig["bind_host"] + ":17123",
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option("Main", key):
agentConfig[key] = config.get("Main", key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig["create_dd_check_tags"] = config.has_option("Main", "create_dd_check_tags") and _is_affirmative(
config.get("Main", "create_dd_check_tags")
)
# Forwarding to external statsd server
if config.has_option("Main", "statsd_forward_host"):
agentConfig["statsd_forward_host"] = config.get("Main", "statsd_forward_host")
if config.has_option("Main", "statsd_forward_port"):
agentConfig["statsd_forward_port"] = int(config.get("Main", "statsd_forward_port"))
# optionally send dogstatsd data directly to the agent.
if config.has_option("Main", "dogstatsd_use_ddurl"):
if _is_affirmative(config.get("Main", "dogstatsd_use_ddurl")):
agentConfig["dogstatsd_target"] = agentConfig["dd_url"]
# Optional config
# FIXME not the prettiest code ever...
if config.has_option("Main", "use_mount"):
agentConfig["use_mount"] = _is_affirmative(config.get("Main", "use_mount"))
if options is not None and options.autorestart:
agentConfig["autorestart"] = True
elif config.has_option("Main", "autorestart"):
agentConfig["autorestart"] = _is_affirmative(config.get("Main", "autorestart"))
if config.has_option("Main", "check_timings"):
agentConfig["check_timings"] = _is_affirmative(config.get("Main", "check_timings"))
if config.has_option("Main", "exclude_process_args"):
agentConfig["exclude_process_args"] = _is_affirmative(config.get("Main", "exclude_process_args"))
try:
filter_device_re = config.get("Main", "device_blacklist_re")
agentConfig["device_blacklist_re"] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option("datadog", "ddforwarder_log"):
agentConfig["has_datadog"] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ":".join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section("WMI"):
agentConfig["WMI"] = {}
for key, value in config.items("WMI"):
agentConfig["WMI"][key] = value
if (
config.has_option("Main", "limit_memory_consumption")
and config.get("Main", "limit_memory_consumption") is not None
):
agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch")
)
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError, e:
sys.stderr.write("Config file not found or incorrectly formatted.\n")
sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
if Platform.is_mac():
agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d'
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
# Store developer mode setting in the agentConfig
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(
config.get('Main', 'developer_mode'))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig['developer_mode'] = True
#
# Core config
#ap
if not config.has_option('Main', 'api_key'):
log.warning(u"No API key was found. Aborting.")
sys.exit(2)
if not config.has_option('Main', 'dd_url'):
log.warning(u"No dd_url was found. Aborting.")
sys.exit(2)
# Endpoints
dd_url = clean_dd_url(config.get('Main', 'dd_url'))
api_key = config.get('Main', 'api_key').strip()
# For collector and dogstatsd
agentConfig['api_key'] = api_key
agentConfig['dd_url'] = dd_url
# multiple endpoints
if config.has_option('Main', 'other_dd_urls'):
other_dd_urls = map(clean_dd_url,
config.get('Main', 'other_dd_urls').split(','))
else:
other_dd_urls = []
if config.has_option('Main', 'other_api_keys'):
other_api_keys = map(
lambda x: x.strip(),
config.get('Main', 'other_api_keys').split(','))
else:
other_api_keys = []
# Forwarder endpoints logic
# endpoints is:
# {
# 'https://app.datadoghq.com': ['api_key_abc', 'api_key_def'],
# 'https://app.example.com': ['api_key_xyz']
# }
endpoints = {dd_url: [api_key]}
if len(other_dd_urls) == 0:
endpoints[dd_url] += other_api_keys
else:
assert len(other_dd_urls) == len(
other_api_keys), 'Please provide one api_key for each url'
for i, other_dd_url in enumerate(other_dd_urls):
endpoints[other_dd_url] = endpoints.get(
other_dd_url, []) + [other_api_keys[i]]
agentConfig['endpoints'] = endpoints
# Forwarder or not forwarder
agentConfig[
'use_forwarder'] = options is not None and options.use_forwarder
if agentConfig['use_forwarder']:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://{}:{}".format(
agentConfig['bind_host'], listen_port)
# FIXME: Legacy dd_url command line switch
elif options is not None and options.dd_url is not None:
agentConfig['dd_url'] = options.dd_url
# Forwarder timeout
agentConfig['forwarder_timeout'] = 20
if config.has_option('Main', 'forwarder_timeout'):
agentConfig['forwarder_timeout'] = int(
config.get('Main', 'forwarder_timeout'))
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'Datadog', 'checks.d')
if config.has_option('Main', 'use_dogstatsd'):
agentConfig['use_dogstatsd'] = config.get(
'Main', 'use_dogstatsd').lower() in ("yes", "true")
else:
agentConfig['use_dogstatsd'] = True
# Service discovery
if config.has_option('Main', 'service_discovery_backend'):
try:
additional_config = extract_agent_config(config)
agentConfig.update(additional_config)
except:
log.error('Failed to load the agent configuration related to '
'service discovery. It will not be used.')
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(
config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(
config.get('Main', 'histogram_percentiles'))
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target':
'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')):
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(
config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(
config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join(
[log_path,
config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(
config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(
config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(
config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(
config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(
config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.ParsingError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.NoOptionError as e:
sys.stderr.write(
'There are some items missing from your config file, but nothing fatal [%s]'
% e)
# Storing proxy settings in the agentConfig
agentConfig['proxy_settings'] = get_proxy(agentConfig)
if agentConfig.get('ca_certs', None) is None:
agentConfig['ssl_certificate'] = get_ssl_certificate(
get_os(), 'datadog-cert.pem')
else:
agentConfig['ssl_certificate'] = agentConfig['ca_certs']
return agentConfig
def get_config(parse_args=True, cfg_path=None, init_logging=False, options=None):
if parse_args:
options, args = get_parsed_args()
elif not options:
args = None
# General config
agentConfig = {
"check_freq": DEFAULT_CHECK_FREQUENCY,
"debug_mode": False,
"dogstatsd_interval": DEFAULT_STATSD_FREQUENCY,
"dogstatsd_normalize": "yes",
"dogstatsd_port": 8125,
"dogstatsd_target": "http://localhost:17123",
"graphite_listen_port": None,
"hostname": None,
"listen_port": None,
"tags": None,
"use_ec2_instance_id": False, # DEPRECATED
"version": get_version(),
"watchdog": True,
}
dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=getOS())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
if init_logging:
initialize_logging(config_path, os_name=getOS())
# bulk import
for option in config.options("Main"):
agentConfig[option] = config.get("Main", option)
#
# Core config
#
# FIXME unnecessarily complex
if config.has_option("Main", "use_dd"):
agentConfig["use_dd"] = config.get("Main", "use_dd").lower() in ("yes", "true")
else:
agentConfig["use_dd"] = True
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option("Main", "listen_port"):
listen_port = int(config.get("Main", "listen_port"))
agentConfig["dd_url"] = "http://localhost:" + str(listen_port)
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig["dd_url"] = options.dd_url
else:
agentConfig["dd_url"] = config.get("Main", "dd_url")
if agentConfig["dd_url"].endswith("/"):
agentConfig["dd_url"] = agentConfig["dd_url"][:-1]
# Whether also to send to Pup
if config.has_option("Main", "use_pup"):
agentConfig["use_pup"] = config.get("Main", "use_pup").lower() in ("yes", "true")
else:
agentConfig["use_pup"] = True
if agentConfig["use_pup"]:
if config.has_option("Main", "pup_url"):
agentConfig["pup_url"] = config.get("Main", "pup_url")
else:
agentConfig["pup_url"] = "http://localhost:17125"
pup_port = 17125
if config.has_option("Main", "pup_port"):
agentConfig["pup_port"] = int(config.get("Main", "pup_port"))
# Increases the frequency of statsd metrics when only sending to Pup
if not agentConfig["use_dd"] and agentConfig["use_pup"]:
dogstatsd_interval = PUP_STATSD_FREQUENCY
if not agentConfig["use_dd"] and not agentConfig["use_pup"]:
sys.stderr.write(
"Please specify at least one endpoint to send metrics to. This can be done in datadog.conf."
)
exit(2)
# Which API key to use
agentConfig["api_key"] = config.get("Main", "api_key")
# Debug mode
agentConfig["debug_mode"] = config.get("Main", "debug_mode").lower() in ("yes", "true")
# local traffic only? Default to no
agentConfig["non_local_traffic"] = False
if config.has_option("Main", "non_local_traffic"):
agentConfig["non_local_traffic"] = config.get("Main", "non_local_traffic").lower() in ("yes", "true")
# DEPRECATED
if config.has_option("Main", "use_ec2_instance_id"):
use_ec2_instance_id = config.get("Main", "use_ec2_instance_id")
# translate yes into True, the rest into False
agentConfig["use_ec2_instance_id"] = use_ec2_instance_id.lower() == "yes"
if config.has_option("Main", "check_freq"):
try:
agentConfig["check_freq"] = int(config.get("Main", "check_freq"))
except:
pass
# Disable Watchdog (optionally)
if config.has_option("Main", "watchdog"):
if config.get("Main", "watchdog").lower() in ("no", "false"):
agentConfig["watchdog"] = False
# Optional graphite listener
if config.has_option("Main", "graphite_listen_port"):
agentConfig["graphite_listen_port"] = int(config.get("Main", "graphite_listen_port"))
else:
agentConfig["graphite_listen_port"] = None
# Dogstatsd config
dogstatsd_defaults = {
"dogstatsd_port": 8125,
"dogstatsd_target": "http://localhost:17123",
"dogstatsd_interval": dogstatsd_interval,
"dogstatsd_normalize": "yes",
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option("Main", key):
agentConfig[key] = config.get("Main", key)
else:
agentConfig[key] = value
# normalize 'yes'/'no' to boolean
dogstatsd_defaults["dogstatsd_normalize"] = _is_affirmative(dogstatsd_defaults["dogstatsd_normalize"])
# optionally send dogstatsd data directly to the agent.
if config.has_option("Main", "dogstatsd_use_ddurl"):
use_ddurl = _is_affirmative(config.get("Main", "dogstatsd_use_ddurl"))
if use_ddurl:
agentConfig["dogstatsd_target"] = agentConfig["dd_url"]
# Optional config
# FIXME not the prettiest code ever...
if config.has_option("Main", "use_mount"):
agentConfig["use_mount"] = config.get("Main", "use_mount").lower() in ("yes", "true", "1")
if config.has_option("datadog", "ddforwarder_log"):
agentConfig["has_datadog"] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ":".join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_section("WMI"):
agentConfig["WMI"] = {}
for key, value in config.items("WMI"):
agentConfig["WMI"][key] = value
except ConfigParser.NoSectionError, e:
sys.stderr.write("Config file not found or incorrectly formatted.\n")
sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
# Store developer mode setting in the agentConfig
in_developer_mode = None
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(config.get('Main', 'developer_mode'))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig['developer_mode'] = True
#
# Core config
#
# FIXME unnecessarily complex
if config.has_option('Main', 'use_dd'):
agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true")
else:
agentConfig['use_dd'] = True
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://" + agentConfig['bind_host'] + ":" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d')
if config.has_option('Main', 'use_dogstatsd'):
agentConfig['use_dogstatsd'] = config.get('Main', 'use_dogstatsd').lower() in ("yes", "true")
else:
agentConfig['use_dogstatsd'] = True
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
if not agentConfig['use_dd']:
sys.stderr.write("Please specify at least one endpoint to send metrics to. This can be done in datadog.conf.")
exit(2)
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(config.get('Main', 'check_freq'))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(config.get('Main', 'histogram_percentiles'))
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port'))
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')):
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(config.get("Main", "utf8_decoding"))
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'monitorstatsd_port': 8125,
'monitorstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False,
'version': get_version(),
'watchmonitor': True,
'additional_checksd': '/etc/monitor-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
if Platform.is_mac():
agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d'
try:
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(
config.get('Main', 'developer_mode'))
if options is not None and options.profile:
agentConfig['developer_mode'] = True
if config.has_option("Main", "frequency"):
agentConfig['check_freq'] = config.get("Main", "frequency")
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['m_url'] = "http://" + agentConfig[
'bind_host'] + ":" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.m_url:
agentConfig['m_url'] = options.m_url
else:
agentConfig['m_url'] = config.get('Main', 'm_url')
if agentConfig['m_url'].endswith('/'):
agentConfig['m_url'] = agentConfig['m_url'][:-1]
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'Datamonitor', 'checks.d')
if config.has_option('Main', 'use_monitorstatsd'):
agentConfig['use_monitorstatsd'] = config.get(
'Main', 'use_monitorstatsd').lower() in ("yes", "true")
else:
agentConfig['use_monitorstatsd'] = True
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
agentConfig['api_key'] = config.get('Main', 'api_key')
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(
config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(
config.get('Main', 'histogram_percentiles'))
if config.has_option('Main', 'watchmonitor'):
if config.get('Main', 'watchmonitor').lower() in ('no', 'false'):
agentConfig['watchmonitor'] = False
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
monitorstatsd_defaults = {
'monitorstatsd_port':
8125,
'monitorstatsd_target':
'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in monitorstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
if config.has_option('Main', 'monitorstatsd_use_murl'):
if _is_affirmative(config.get('Main', 'monitorstatsd_use_murl')):
agentConfig['monitorstatsd_target'] = agentConfig['m_url']
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(
config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(
config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datamonitor', 'ddforwarder_log'):
agentConfig['has_datamonitor'] = True
if config.has_option("Main", "monitorstream_log"):
log_path = config.get("Main", "monitorstream_log")
if config.has_option("Main", "monitorstream_line_parser"):
agentConfig["monitorstreams"] = ':'.join([
log_path,
config.get("Main", "monitorstream_line_parser")
])
else:
agentConfig["monitorstreams"] = log_path
elif config.has_option("Main", "monitorstreams"):
agentConfig["monitorstreams"] = config.get("Main",
"monitorstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(
config.get("Main", "use_curl_http_client"))
else:
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(
config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(
config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(
config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(
config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.ParsingError as e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
except ConfigParser.NoOptionError as e:
sys.stderr.write(
'There are some items missing from your config file, but nothing fatal [%s]'
% e)
agentConfig['proxy_settings'] = get_proxy(agentConfig)
if agentConfig.get('ca_certs', None) is None:
agentConfig['ssl_certificate'] = get_ssl_certificate(
get_os(), 'datamonitor-cert.pem')
else:
agentConfig['ssl_certificate'] = agentConfig['ca_certs']
agentConfig['interval'] = config.get('Main', 'updater_interval')
return agentConfig
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
'dogstatsd_normalize': 'yes',
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
}
dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
#
# Core config
#
# FIXME unnecessarily complex
if config.has_option('Main', 'use_dd'):
agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true")
else:
agentConfig['use_dd'] = True
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://localhost:" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d')
# Whether also to send to Pup
if config.has_option('Main', 'use_pup'):
agentConfig['use_pup'] = config.get('Main', 'use_pup').lower() in ("yes", "true")
else:
agentConfig['use_pup'] = True
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
if agentConfig['use_pup'] or agentConfig['use_web_info_page']:
if config.has_option('Main', 'pup_url'):
agentConfig['pup_url'] = config.get('Main', 'pup_url')
else:
agentConfig['pup_url'] = 'http://localhost:17125'
if config.has_option('Main', 'pup_port'):
agentConfig['pup_port'] = int(config.get('Main', 'pup_port'))
# Increases the frequency of statsd metrics when only sending to Pup
if not agentConfig['use_dd'] and agentConfig['use_pup']:
dogstatsd_interval = PUP_STATSD_FREQUENCY
if not agentConfig['use_dd'] and not agentConfig['use_pup']:
sys.stderr.write("Please specify at least one endpoint to send metrics to. This can be done in datadog.conf.")
exit(2)
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(config.get('Main', 'check_freq'))
except Exception:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'dogstatsd_interval': dogstatsd_interval,
'dogstatsd_normalize': 'yes',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
#Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port'))
# normalize 'yes'/'no' to boolean
dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(dogstatsd_defaults['dogstatsd_normalize'])
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
use_ddurl = _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl'))
if use_ddurl:
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount'))
if config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if config.has_option("Main", "limit_memory_consumption") and \
config.get("Main", "limit_memory_consumption") is not None:
agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags"))
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def _list_ids(self, config, section):
ids = []
options = config.options(section)
for option in options:
ids.append(config.get(section, option))
return ids
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/sd-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
if Platform.is_mac():
agentConfig['additional_checksd'] = '/usr/local/etc/sd-agent/checks.d/'
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
# Store developer mode setting in the agentConfig
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(
config.get('Main', 'developer_mode'))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig['developer_mode'] = True
#
# Core config
#
# FIXME unnecessarily complex
if config.has_option('Main', 'sd_account'):
agentConfig['sd_account'] = config.get('Main', 'sd_account')
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17124
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['sd_url'] = "http://" + agentConfig[
'bind_host'] + ":" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_sd and options.sd_url:
agentConfig['sd_url'] = options.sd_url
elif config.has_option('Main', 'sd_url'):
agentConfig['sd_url'] = config.get('Main', 'sd_url')
else:
# Default agent URL
agentConfig['sd_url'] = "https://" + agentConfig[
'sd_account'] + ".agent.serverdensity.io"
if agentConfig['sd_url'].endswith('/'):
agentConfig['sd_url'] = agentConfig['sd_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'ServerDensity', 'checks.d')
if config.has_option('Main', 'use_dogstatsd'):
agentConfig['use_dogstatsd'] = config.get(
'Main', 'use_dogstatsd').lower() in ("yes", "true")
else:
agentConfig['use_dogstatsd'] = True
# Service discovery
if config.has_option('Main', 'service_discovery_backend'):
try:
additional_config = extract_agent_config(config)
agentConfig.update(additional_config)
except:
log.error('Failed to load the agent configuration related to '
'service discovery. It will not be used.')
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
# Which agent key to use
agentConfig['agent_key'] = config.get('Main', 'agent_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(
config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(
config.get('Main', 'histogram_percentiles'))
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target':
'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')):
agentConfig['dogstatsd_target'] = agentConfig['sd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(
config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(
config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(
config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(
config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(
config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(
config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(
config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def _convert_to_attribute(self, config, obj, name):
for option in config.options(name):
setattr(obj, option, config.get(name, option))
return obj
