def options(): parser = import_module(os.getenv('PARSER')).parse_options() o = parser.parse_args() logging.config.fileConfig(fileloc(o.log_config)) config = ConfigParser.SafeConfigParser() config.read(fileloc(o.config)) # parse config in-order keys = [] for section in config.sections(): for param in config.options(section): keys.append(param) setattr(o, param, config.get(section, param)) # cli arguments override configs for key in provided_arguments(parser): setattr(o, key, getattr(parser.parse_args(), key)) # environment variables override configs for key in keys: setattr(o, key, os.getenv(key.upper(), getattr(o, key))) return o
def ConfigMap(section): config = configparser.ConfigParser() config.read("/conf/_config.ini") props = {} options = config.options(section) for option in options: props[option] = config.get(section, option) return props
def load_config(path): config = configparser.ConfigParser() config.read(path) result = defaultdict(dict) for s in config.sections(): for o in config.options(s): result[s][o] = config.get(s, o) return result
def __init__(self, config, section): kwargs = dict([(o, config.get(section, o)) for o in config.options(section)]) self.name = section self.keystone = KeystoneClient(**kwargs) glance_endpoint = self._get_endpoint('image') self.glance = GlanceV1Client(glance_endpoint['internalURL'], token=self.keystone.auth_token)
def get_config_options(filename): "Function return dictionary with all options from configuration file" config = ConfigParser.ConfigParser() config.read(filename) conf_options = {} for section in config.sections(): conf_options[section] = {} for option in config.options(section): conf_options[section][option] = config.get(section, option) return conf_options
def from_config(self, config): if not config.has_section(self.name): return {} res = {} for opt in config.options(self.name): value = config.get(self.name, opt) if not value and opt in self.required: continue res[opt] = value return res
def _adjust_options(self, config): # config defaults appear in all sections. # we'll need to filter them out. defaults = config.defaults().keys() # see what options are already defined and add missing ones preset = [(option.section, option.setting) for option in self.items()] for section in config.sections(): for name in config.options(section): if ((section, name) not in preset) \ and (name not in defaults): self.add_option(Option(self, section, name))
def parse_section(config, section): map = {} options = config.options(section) for option in options: try: map[option] = config.get(section, option) if map[option] == -1: logging.debug("skipping option: %s in section: %s" % option, section) except: error_parsing_option = "Error: Parsing option failed on option: %s!" % option raise Exception(error_parsing_option) return map
def ConfigSectionMap(section): dict1 = {} options = config.options(section) for option in options: try: dict1[option] = config.get(section, option) if dict1[option] == -1: print("skip: %s" % option) except: print("exception on %s!" % option) dict1[option] = None return dict1
def _config_provided(filepath): config = configparser.ConfigParser() try: with open(filepath) as f_prov: config.read_file(f_prov) if not config.options('Microsoft Teams'): raise MissingConnectorConfigKeyException('missing connector key in provided config') except configparser.NoSectionError: raise MissingConnectorConfigKeyException('missing required Microsoft Teams / ' 'connector key in provided config') return config
def config_read(cfgFName): log.debug('Reading config ' + cfgFName) config = configparser.ConfigParser() if os.path.exists(cfgFName): config.read(cfgFName, encoding='utf-8') else: log.debug('Не найден файл конфигурации.') # в разделе [cols_in] находится список интересующих нас колонок и номера столбцов исходного файла in_columns_names = config.options('cols_in') in_columns_j = {} for vName in in_columns_names: if ('' != config.get('cols_in', vName)): in_columns_j[vName] = config.getint('cols_in', vName) # По разделу [cols_out] формируем перечень выводимых колонок и строку заголовка результирующего CSV файла out_columns_names = config.options('cols_out') out_columns = {} for vName in out_columns_names: if ('' != config.get('cols_out', vName)): out_columns[vName] = config.get('cols_out', vName) return in_columns_j, out_columns
def parse_options(config, section_name, config_class): options = config.options(section_name) for option in options: option = option.upper() if option in config_class.__dict__ and not option.startswith("__"): if isinstance(config_class.__dict__[option], bool): config_class.__dict__[option] = config.getboolean(section_name, option) elif isinstance(config_class.__dict__[option], float): config_class.__dict__[option] = config.getfloat(section_name, option) elif isinstance(config_class.__dict__[option], int): config_class.__dict__[option] = config.getint(section_name, option) else: config_class.__dict__[option] = config.get(section_name, option) else: logger = logging.getLogger('InfrastructureManager') logger.warn("Unknown option in the IM config file. Ignoring it: " + option)
def loadConfig(configFile): # Charger le fichier de configuration with open(configFile) as f: sample_config = f.read() config = ConfigParser.RawConfigParser(allow_no_value=True) config.readfp(io.BytesIO(sample_config)) # List all contents if (False): print("Parametrage du fichier de configuration :"+configFile) for section in config.sections(): print("[%s]" % section) for options in config.options(section): print("\t%s = %s" % (options, config.get(section, options))) return config
def init_ixiacr_logger(): global ixiacr_logger_initialized if ixiacr_logger_initialized: return ixiacr_logger_initialized = True config = ConfigParser.ConfigParser() # Filename, LogLevel, modules to override log level config.read(os.path.join(os.getenv('IXIACR'), 'ixiacrlogger.ini')) level_str = config.get('ixiacr', 'level') def get_log_level(level_str): if level_str[0].isdigit(): log_level = int(level_str) else: log_level = logging.getLevelName(level_str) return log_level log_level = get_log_level(level_str) log_format = config.get('ixiacr', 'format', raw=True) handler = logging.StreamHandler(sys.stdout) log_formatter = logging.Formatter(log_format) handler.setFormatter(log_formatter) root = logging.getLogger() root.setLevel(log_level) root.addHandler(handler) try: override_level = get_log_level(config.get('ixiacr', 'override_level')) if override_level != log_level: overrides = config.options('overrides') for override in overrides: root.info('Overriding log level for logger={0} to level={1}'. format(override, override_level)) olog = logging.getLogger(override) olog.setLevel(override_level) except: pass
def init_logger(): global logger_initialized if logger_initialized: return logger_initialized = True config = ConfigParser.ConfigParser() # Filename, LogLevel, modules to override log level config.read('/home/judo/workspace/github/CRManager/CyberRange/logger.ini') level_str = config.get('cr', 'level') def get_log_level(level_str): if level_str[0].isdigit(): log_level = int(level_str) else: log_level = logging.getLevelName(level_str) return log_level log_level = get_log_level(level_str) log_format = config.get('cr', 'format', raw=True) handler = logging.StreamHandler(sys.stdout) log_formatter = logging.Formatter(log_format) handler.setFormatter(log_formatter) root = logging.getLogger() root.setLevel(log_level) root.addHandler(handler) try: override_level = get_log_level(config.get('cr', 'override_level')) if override_level != log_level: overrides = config.options('overrides') for override in overrides: root.info('Overriding log level for logger={0} to level={1}'. format(override, override_level)) olog = logging.getLogger(override) olog.setLevel(override_level) except: pass
def readConst(section, file): config = configparser.ConfigParser() filePath = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../Utilities/' + file + '.ini') variable = config.read(filePath) if variable: dictionary = {} try: options = config.options(section) except configparser.NoSectionError: Logger.logr.error("Section " + section + " does not exist in " + filePath) else: for option in options: dictionary[option] = config.get(section, option) return dictionary else: Logger.logr.info("File " + file + ".ini doesn't exist under provided path:" + filePath)
def parse_options(config, section_name, config_class): options = config.options(section_name) for option in options: option = option.upper() if option in config_class.__dict__ and not option.startswith("__"): if isinstance(config_class.__dict__[option], bool): config_class.__dict__[option] = config.getboolean( section_name, option) elif isinstance(config_class.__dict__[option], float): config_class.__dict__[option] = config.getfloat( section_name, option) elif isinstance(config_class.__dict__[option], int): config_class.__dict__[option] = config.getint( section_name, option) else: config_class.__dict__[option] = config.get( section_name, option) else: logger = logging.getLogger('InfrastructureManager') logger.warn("Unknown option in the IM config file. Ignoring it: " + option)
def set_config_data(self, data: dict): for identifier, update_data in data.items(): # dont rewrite empty... if not len(update_data): continue if identifier == "meta": hostname = update_data.get("hostname", None) if hostname: SysUtil.set_hostname(hostname) if update_data.get("update", False): SysUtil.update_from_git() config = SysUtil.ensure_config(identifier) sections = set(config.sections()).intersection(set(update_data.keys())) for section in sections: update_section = update_data[section] options = set(config.options(section)).intersection(set(update_section.keys())) for option in options: config.set(section, option, str(update_section[option])) SysUtil.write_config(config, identifier)
def startAgent(agent): global agents proc = config.get(agent,'proc') if agent not in agents: agents[agent] = {'pid':None,'sp':None,'lasthb':datetime.now()} agents[agent]['sp'] = subprocess.Popen('python %s &'%proc,shell=True) agents[agent]['pid'] = agents[agent]['sp'].pid + 1 #shell=True means pid is the shell, so +1 is the agent agents[agent]['lasthb'] = datetime.now() agents[agent]['ivy_name'] = config.get(agent,'ivy_name') if __name__ == "__main__": config = SafeConfigParser() config.read('conf/twitterbot.ini') if 'access_key' not in config.options('Twitter'): mainlogger.critical('Twitter client needs OAuth dance. Please run it manually, then rerun this script.') sys.exit(1) ivy_name = config.get('Manager','ivy_name') IvyInit(ivy_name,'[%s is ready]'%ivy_name,0,oncxproc,ondieproc) IvyStart(config.get('General','ivy_bus')) IvyBindMsg(heartbeat,'^hb_ack') mainlogger.setLevel(level=getattr(logging,config.get('Manager','log_level'))) timeout = timedelta(seconds=int(config.get('Manager','timeout'))) interested = config.get('Manager','agents').split(',') for agent in interested: startAgent(agent)
def get_config(parse_args=True, cfg_path=None, init_logging=False, options=None): if parse_args: options, args = get_parsed_args() elif not options: args = None # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'debug_mode': False, 'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY, 'dogstatsd_normalize': 'yes', 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, } dogstatsd_interval = DEFAULT_STATSD_FREQUENCY # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=getOS()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if init_logging: initialize_logging(config_path, os_name=getOS()) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # # Core config # if config.has_option('Main', 'use_dd'): agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true") else: agentConfig['use_dd'] = True if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = config.get('Main', 'listen_port') agentConfig['dd_url'] = "http://localhost:" + str(listen_port) elif options is not None and not options.disable_dd and options.dd_url: agentConfig['dd_url'] = options.dd_url else: agentConfig['dd_url'] = config.get('Main', 'dd_url') if agentConfig['dd_url'].endswith('/'): agentConfig['dd_url'] = agentConfig['dd_url'][:-1] # Whether also to send to Pup if config.has_option('Main', 'use_pup'): agentConfig['use_pup'] = config.get('Main', 'use_pup').lower() in ("yes", "true") else: agentConfig['use_pup'] = True if agentConfig['use_pup']: if config.has_option('Main', 'pup_url'): agentConfig['pup_url'] = config.get('Main', 'pup_url') else: agentConfig['pup_url'] = 'http://localhost:17125' pup_port = 17125 if config.has_option('Main', 'pup_port'): agentConfig['pup_port'] = int(config.get('Main', 'pup_port')) # Increases the frequency of statsd metrics when only sending to Pup if not agentConfig['use_dd'] and agentConfig['use_pup']: dogstatsd_interval = PUP_STATSD_FREQUENCY if not agentConfig['use_dd'] and not agentConfig['use_pup']: sys.stderr.write( "Please specify at least one endpoint to send metrics to. This can be done in datadog.conf." ) exit(2) # Which API key to use agentConfig['api_key'] = config.get('Main', 'api_key') # Debug mode agentConfig['debug_mode'] = config.get( 'Main', 'debug_mode').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except: pass # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = int( config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'dogstatsd_interval': dogstatsd_interval, 'dogstatsd_normalize': 'yes', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # normalize 'yes'/'no' to boolean dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative( dogstatsd_defaults['dogstatsd_normalize']) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): use_ddurl = _is_affirmative( config.get('Main', 'dogstatsd_use_ddurl')) if use_ddurl: agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = config.get( 'Main', 'use_mount').lower() in ("yes", "true", "1") if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join( [log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'monitorstatsd_port': 8125, 'monitorstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchmonitor': True, 'additional_checksd': '/etc/monitor-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d' # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative( config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # Get check frequency if config.has_option("Main", "frequency"): agentConfig['check_freq'] = config.get("Main", "frequency") # # Core config # # FIXME unnecessarily complex agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['m_url'] = "http://" + agentConfig[ 'bind_host'] + ":" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_dd and options.m_url: agentConfig['m_url'] = options.m_url else: agentConfig['m_url'] = config.get('Main', 'm_url') if agentConfig['m_url'].endswith('/'): agentConfig['m_url'] = agentConfig['m_url'][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'Datamonitor', 'checks.d') if config.has_option('Main', 'use_monitorstatsd'): agentConfig['use_monitorstatsd'] = config.get( 'Main', 'use_monitorstatsd').lower() in ("yes", "true") else: agentConfig['use_monitorstatsd'] = True # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True # Which API key to use agentConfig['api_key'] = config.get('Main', 'api_key') # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates( config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles( config.get('Main', 'histogram_percentiles')) # Disable Watchmonitor (optionally) if config.has_option('Main', 'watchmonitor'): if config.get('Main', 'watchmonitor').lower() in ('no', 'false'): agentConfig['watchmonitor'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # monitorstatsd config monitorstatsd_defaults = { 'monitorstatsd_port': 8125, 'monitorstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in monitorstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) # optionally send monitorstatsd data directly to the agent. if config.has_option('Main', 'monitorstatsd_use_murl'): if _is_affirmative(config.get('Main', 'monitorstatsd_use_murl')): agentConfig['monitorstatsd_target'] = agentConfig['m_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datamonitor', 'ddforwarder_log'): agentConfig['has_datamonitor'] = True # monitorstream config if config.has_option("Main", "monitorstream_log"): # Older version, single log support log_path = config.get("Main", "monitorstream_log") if config.has_option("Main", "monitorstream_line_parser"): agentConfig["monitorstreams"] = ':'.join([ log_path, config.get("Main", "monitorstream_line_parser") ]) else: agentConfig["monitorstreams"] = log_path elif config.has_option("Main", "monitorstreams"): agentConfig["monitorstreams"] = config.get("Main", "monitorstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative( config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative( config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.ParsingError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.NoOptionError as e: sys.stderr.write( 'There are some items missing from your config file, but nothing fatal [%s]' % e) # Storing proxy settings in the agentConfig agentConfig['proxy_settings'] = get_proxy(agentConfig) if agentConfig.get('ca_certs', None) is None: agentConfig['ssl_certificate'] = get_ssl_certificate( get_os(), 'datamonitor-cert.pem') else: agentConfig['ssl_certificate'] = agentConfig['ca_certs'] # self-updater relative conf agentConfig['interval'] = config.get('Main', 'updater_interval') return agentConfig
def main(): global DBConn, minutesOffset, myMeterId, meterAtable, meterBtable, anyMeterOk # Determine the complete file paths for the config file and the graph definitions file. config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) configFile = GetConfigFilePath() # configFileDir = os.path.dirname(configFile) # Open up the configuration file and extract some parameters. config.read(configFile) cfgSection = ProgFile + "/" + os.environ['HOST'] logger.info("INI file cofig section is: %s", cfgSection) # logger.debug('Config section has options: %s'%set(config.options(cfgSection))) # logger.debug('Required options are: %s'%RequiredConfigParams) if not config.has_section(cfgSection): logger.critical('Config file "%s", has no section "%s".', configFile, cfgSection) sys.exit(2) if len(RequiredConfigParams - set(config.options(cfgSection))) > 0: logger.critical( 'Config section "%s" does not have all required params: "%s", it has params: "%s".', cfgSection, RequiredConfigParams, set(config.options(cfgSection))) logger.debug( 'The missing params are: %s' % (RequiredConfigParams - set(config.options(cfgSection)), )) sys.exit(3) anyMeterOk = config.getboolean(cfgSection, 'use_database_for_any_meter', fallback=True) cfg = config[cfgSection] parser = argparse.ArgumentParser( description='Display graphs of home parameters.\nDefaults to show all.' ) parser.add_argument( "-s", "--serial_port", dest="serialPort", action="store", default=cfg['meter_serial_port'], help="The serial port to which the EKM meter is connected.") parser.add_argument("-i", "--meterId", dest="meterId", action="store", default=cfg['meter_id'], help="Numeric Id of EKM meter.") parser.add_argument( "-t", "--start", dest="startTime", action="store", default=None, help="Date/time at which to start retrieving database data.") parser.add_argument( "-o", "--offsetHours", dest="hourOffset", action="store", default=None, help= "Time offset from NOW to retrieve send data to client. Negative values access database, positive values send random digits." ) parser.add_argument("-a", "--anyMeter", dest="anyMeter", action="store_true", default=None, help="Accept any meter Id.") parser.add_argument("-v", "--verbosity", dest="verbosity", action="count", help="increase output verbosity", default=0) args = parser.parse_args() Verbosity = args.verbosity startTime = None minutesOffset = None if (args.hourOffset is None) and (args.startTime is None): minutesOffset = -1440 # default 1 day ago elif args.startTime is not None: try: startTime = datetime.strptime(args.startTime, '%Y-%m-%d %H:%M:%S') except ValueError as e: logger.exception(e) logger.warning( 'Argument for -t (--start) could not be parsed. Using default time offset of 1 day.' ) if startTime is not None: logger.debug('Computing minutes offset from now() and startTime.') minutesOffset = (startTime - datetime.now()) / timedelta(minutes=1) else: if (minutesOffset is None) and (args.hourOffset is not None): minutesOffset = float(args.hourOffset) * 60 else: minutesOffset = -1440 # default 1 day ago logger.debug('Using minutesOffset of: %s' % minutesOffset) if args.anyMeter is not None: anyMeterOk = args.anyMeter logger.debug('Any meter is OK? %s' % anyMeterOk) # setup "serial" port '''Extract pipe file name (port) from an URL string; set mode to "client" or "server"''' try: parts = urlsplit(args.serialPort) except URLError: logger.critical( 'The serial port parameter is not a URL, FakeEKM.py connot proceed.' ) raise if parts.scheme != "socket": raise Exception( 'expected a string in the form "socket://<host>:<port>[?logging={debug|info|warning|error}]": not starting with socket:// (%r)' % (parts.scheme, )) # setup database connection user = cfg['viewer_user'] pwd = cfg['viewer_password'] host = cfg['viewer_host'] port = cfg['viewer_port'] schema = cfg['viewer_schema'] logger.info("user %s" % (user, )) logger.info("pwd %s" % (pwd, )) logger.info("host %s" % (host, )) logger.info("port %s" % (port, )) logger.info("schema %s" % (schema, )) # need meterId both as integer and as 12 char string myMeterIdInt = int(args.meterId) myMeterId = '%012d' % myMeterIdInt logger.debug('MeterId as int is %s; as str "%s"' % (myMeterIdInt, myMeterId)) if myMeterIdInt < 300000000: logger.critical( 'EKM V3 meters are not supported for database as data source.') minutesOffset = 1 # => always generate random data if minutesOffset < 0: # Verify that database tables are available meterAtable = myMeterId + cfg['meter_table_a_suffix'] meterBtable = myMeterId + cfg['meter_table_b_suffix'] logger.debug('meterAtable is "%s"' % meterAtable) logger.debug('meterBtable is "%s"' % meterBtable) connstr = 'mysql+pymysql://{user}:{pwd}@{host}:{port}/{schema}'.format( user=user, pwd=pwd, host=host, port=port, schema=schema) logger.debug("database connection string: %s" % (connstr, )) Eng = create_engine(connstr, echo=True if Verbosity >= 2 else False, logging_name=logger.name) logger.debug(Eng) try: with Eng.connect() as DBConn, DBConn.begin(): logger.info("Viewer connection to database established.") query = "SHOW TABLES LIKE '%s%%'" % myMeterId logger.debug( 'Checking database for message tables with query: %s' % query) result = DBConn.execute(text(query)) requiredTables = [meterAtable, meterBtable] for r in result: logger.debug('Checking if %s is a required table.' % r[0]) if r[0] in requiredTables: requiredTables.remove(r[0]) logger.debug('Table %s is in the database.' % (r[0], )) if len(requiredTables) == 0: break if len(requiredTables) != 0: logger.critical( 'Not all required tables are in the database; only random messages will be generated.' ) minutesOffset = 1 # => always generate random data else: logger.debug('Found all required tables in database.') with ThreadedTCPServer((parts.hostname, parts.port), ThreadedTCPRequestHandler) as server: ip, port = server.server_address logger.debug('TCP server started at address "%s:%s"' % (ip, port)) # Start a thread with the server -- that thread will then start one # more thread for each request server_thread = threading.Thread( target=server.serve_forever) # Exit the server thread when the main thread terminates server_thread.daemon = False # was True server_thread.start() logger.debug("Server loop running in thread: %s" % server_thread.name) response = "keep going" while response.lower().find("quit") < 0: response = input( 'Type "quit" to terminate FakeEKM server: ') logger.info('Server quits.') server.shutdown() finally: pass logger.info( ' ############## FakeEKM All Done #################')
def get_config(parse_args = True, cfg_path=None, init_logging=False, options=None): if parse_args: options, args = get_parsed_args() elif not options: args = None # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'debug_mode': False, 'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY, 'dogstatsd_normalize': 'yes', 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, } dogstatsd_interval = DEFAULT_STATSD_FREQUENCY # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=getOS()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if init_logging: initialize_logging(config_path, os_name=getOS()) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # # Core config # if config.has_option('Main', 'use_dd'): agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true") else: agentConfig['use_dd'] = True if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main','listen_port'): listen_port = config.get('Main','listen_port') agentConfig['dd_url'] = "http://localhost:" + str(listen_port) elif options is not None and not options.disable_dd and options.dd_url: agentConfig['dd_url'] = options.dd_url else: agentConfig['dd_url'] = config.get('Main', 'dd_url') if agentConfig['dd_url'].endswith('/'): agentConfig['dd_url'] = agentConfig['dd_url'][:-1] # Whether also to send to Pup if config.has_option('Main', 'use_pup'): agentConfig['use_pup'] = config.get('Main', 'use_pup').lower() in ("yes", "true") else: agentConfig['use_pup'] = True if agentConfig['use_pup']: if config.has_option('Main', 'pup_url'): agentConfig['pup_url'] = config.get('Main', 'pup_url') else: agentConfig['pup_url'] = 'http://localhost:17125' pup_port = 17125 if config.has_option('Main', 'pup_port'): agentConfig['pup_port'] = int(config.get('Main', 'pup_port')) # Increases the frequency of statsd metrics when only sending to Pup if not agentConfig['use_dd'] and agentConfig['use_pup']: dogstatsd_interval = PUP_STATSD_FREQUENCY if not agentConfig['use_dd'] and not agentConfig['use_pup']: sys.stderr.write("Please specify at least one endpoint to send metrics to. This can be done in datadog.conf.") exit(2) # Which API key to use agentConfig['api_key'] = config.get('Main', 'api_key') # Debug mode agentConfig['debug_mode'] = config.get('Main', 'debug_mode').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int(config.get('Main', 'check_freq')) except: pass # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main','graphite_listen_port'): agentConfig['graphite_listen_port'] = int(config.get('Main','graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port' : 8125, 'dogstatsd_target' : 'http://localhost:17123', 'dogstatsd_interval' : dogstatsd_interval, 'dogstatsd_normalize' : 'yes', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # normalize 'yes'/'no' to boolean dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(dogstatsd_defaults['dogstatsd_normalize']) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): use_ddurl = _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')) if use_ddurl: agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = config.get('Main', 'use_mount').lower() in ("yes", "true", "1") if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def main(): config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) configFile = GetConfigFilePath() configFileDir = os.path.dirname(configFile) config.read(configFile) cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST'] logger.info("INI file cofig section is: %s", cfgSection) if not config.has_section(cfgSection): logger.critical('Config file "%s", has no section "%s".', configFile, cfgSection) sys.exit(2) if set(config.options(cfgSection)) < RequiredConfigParams: logger.critical( 'Config section "%s" does not have all required params: "%s", it has params: "%s".', cfgSection, RequiredConfigParams, set(config.options(cfgSection))) sys.exit(3) cfg = config[cfgSection] parser = argparse.ArgumentParser( description= 'Program to read Ambient WS-1002 console archived data and insert it into the database.' ) parser.add_argument("files", action="append", nargs="*", help="List of CSV files to process.") parser.add_argument("-b", "--beginTime", dest="begin", action="append", help="Beginning time(s) to insert data to database.") parser.add_argument("-e", "--endTime", dest="end", action="append", help="Ending time(s) to insert data to database.") parser.add_argument( "-g", "--maxGap", dest="maxGap", action="store", help= "(minutes) Max gap in database data to ignore; gaps bigger that this will be filled in.", default='20') parser.add_argument("-d", "--dryrun", dest="dryrun", action="store_true", help="Don't actually modify database.", default=False) parser.add_argument("-v", "--verbosity", dest="verbosity", action="count", help="increase output verbosity", default=0) args = parser.parse_args() # numDays = float(args.days) logger.debug('The arguments are: %s' % args) argfileslist = args.files[0] logger.debug('There are %s input files on the command line.' % len(argfileslist)) logger.debug('The input file list is: %s' % (argfileslist)) if len(argfileslist) == 0: logger.critical('A list of CSV files is required.') exit(3) beginTimes = list() endTimes = list() if args.begin is not None and args.end is not None: try: logger.debug('Intreperting begin times: %s' % args.begin) for b in args.begin: logger.debug('Looking at %s' % b) beginTimes.append(datetime.fromisoformat(b)) except Exception as e: logger.critical('Unable to parse begin time: "%s"' % args.begin) logger.exception(e) exit(3) try: logger.debug('Intreperting end times: %s' % args.end) for b in args.end: logger.debug('Looking at %s' % b) endTimes.append(datetime.fromisoformat(b)) except Exception as e: logger.critical('Unable to parse begin time: "%s"' % args.end) logger.exception(e) exit(3) logger.debug('Begin times: %s, End times: %s' % (beginTimes, endTimes)) if len(beginTimes) != len(endTimes): logger.critical( 'There must be same number of begin times and end times.') exit(3) else: logger.debug('begin or end time is "None"') user = cfg['inserter_user'] pwd = cfg['inserter_password'] host = cfg['inserter_host'] port = cfg['inserter_port'] schema = cfg['inserter_schema'] tableName = cfg['weather_table'] logger.info("user %s" % (user, )) logger.info("pwd %s" % (pwd, )) logger.info("host %s" % (host, )) logger.info("port %s" % (port, )) logger.info("schema %s" % (schema, )) logger.info("Table %s" % (tableName, )) ''' SQL commands to view gaps: create temporary table w1 like weather; alter table w1 drop primary key; alter table w1 add column id int auto_increment primary key first; insert into w1 (dateutc, tempinf, tempf, humidityin, humidity, windspeedmph, windgustmph, maxdailygust, winddir, baromabsin, baromrelin, hourlyrainin, dailyrainin, weeklyrainin, monthlyrainin, yearlyrainin, solarradiation, uv, feelsLike, dewPoint, lastRain, date) select * from weather order by date; select wB.date as beginTime, wA.date as endTime, round((wA.dateutc - wB.dateutc)/60000) as diff from w1 as wA inner join w1 as wB on wA.id=wB.id+1 where (wA.dateutc - wB.dateutc)/60000 > 20; drop table w1; ''' connstr = 'mysql+pymysql://{user}:{pwd}@{host}:{port}/{schema}'.format( user=user, pwd=pwd, host=host, port=port, schema=schema) Eng = create_engine(connstr, echo=True if args.verbosity >= 2 else False, logging_name=logger.name) logger.debug(Eng) with Eng.connect() as conn, conn.begin(): if args.begin is None or args.end is None: logger.debug('Auto detecting gaps in weather table.') logger.debug('Create temporary table.') conn.execute('create temporary table w1 like weather') logger.debug('Drop primary key on temporary table.') conn.execute('alter table w1 drop primary key') logger.debug('Add id column to temporary table.') conn.execute( 'alter table w1 add column id int auto_increment primary key first' ) logger.debug('Copy rows from weather to temporary table.') conn.execute( 'insert into w1 (dateutc, tempinf, tempf, humidityin, humidity, windspeedmph, windgustmph, maxdailygust, winddir, baromabsin, baromrelin, hourlyrainin, dailyrainin, weeklyrainin, monthlyrainin, yearlyrainin, solarradiation, uv, feelsLike, dewPoint, lastRain, date) select * from weather order by date' ) gapQuery = 'select wB.date as beginTime, wA.date as endTime, round((wA.dateutc - wB.dateutc)/60000) as diff from w1 as wA inner join w1 as wB on wA.id=wB.id+1 where (wA.dateutc - wB.dateutc)/60000 > %s' % args.maxGap logger.debug('Select time gaps with query: "%s"' % gapQuery) result = conn.execute(gapQuery) for row in result: b = row[0] e = row[1] beginTimes.append(b) endTimes.append(e) logger.debug( 'Appending %s to beginTimes, %s to endTimes, because the gap was %s' % (b, e, row[2])) logger.debug('Auto detected gaps are: begin %s, end %s' % (beginTimes, endTimes)) logger.debug('Drop temporary table.') conn.execute('drop table w1') pass insertQuery = 'INSERT IGNORE INTO ' + tableName + '(dateutc, date, tempinf, humidityin, tempf, humidity, windspeedmph, windgustmph,\ dewPoint, feelsLike, winddir, baromabsin, baromrelin, hourlyrainin, dailyrainin, weeklyrainin, monthlyrainin, \ yearlyrainin, solarradiation, uv) VALUES \n' linebegin = ' (' valuesCount = 0 logger.debug('zipped times: %s' % list(zip(beginTimes, endTimes))) for beginTime, endTime in list(zip(beginTimes, endTimes)): logger.debug('Scanning files for entries between "%s" and "%s"' % (beginTime, endTime)) for theFile in argfileslist: logger.info('theFile: %s', theFile) lineCount = 0 with open(theFile, encoding='utf_16_le') as f: l = f.readline() # discard first line printFirstTime = True for l in f: fields = l.replace('--.-', 'null').replace( '--', 'null').replace('\n', '').split('\t') dt = datetime.strptime(fields[1], '%I:%M %p %m/%d/%Y') if printFirstTime: logger.debug('First time in file is %s' % dt.isoformat()) printFirstTime = False if dt <= beginTime: continue if dt >= endTime: break lineCount += 1 fields[1] = dt.isoformat() ol = linebegin + str([ str(round(dt.timestamp() * 1000)), fields[1:9], fields[9] if fields[9] != 'null' else fields[19], fields[10:19], fields[21] ]).replace('[', '').replace(']', '').replace( "'null'", "NULL") + ')\n' linebegin = ', (' insertQuery += ol logger.debug('Last time in file is %s' % dt.isoformat()) logger.debug( 'There were %s lines in the file within the desired times.' % lineCount) valuesCount += lineCount logger.debug('Insert the values.') insertQuery += ' ON DUPLICATE KEY UPDATE date=value(date)' logger.debug( 'All together, there were %s values to add to the database.' % valuesCount) logger.debug('The insert query is: "%s"' % insertQuery) if (not args.dryrun) and (valuesCount > 0): conn.execute(insertQuery) else: if valuesCount == 0: logger.warning('No data found to update database.') logger.warning("Didn't actually modify database.") pass logger.info(' ############## All Done #################')
def parse_config(relation, config, _logger): from ConfigParser import ConfigParser from oxsync import OxTaskSync from ensync import EnClientSync from tdsync import ToodledoSync logger = LogAdapter(_logger, {'package': 'config'}) class_map = { 'OxTaskSync': OxTaskSync, 'EnClientSync': EnClientSync, 'ToodledoSync': ToodledoSync } relation_section = 'relation' + '_' + relation if config.has_section(relation_section): rel = {} for key in config.options(relation_section): rel[key] = config.get(relation_section, key) if rel.get('map'): rel['map'] = os.path.expanduser(rel.get('map')) else: logging.critical('Configuration error: missing map file path for %s' % (relation)) exit(1) left = config.get(relation_section, 'left') if left: engine_section_left = 'engine' + '_' + left else: logging.critical('Configuraion error: missing left engine refernce') exit(1) left = None if config.has_section(engine_section_left): left = {} for key in config.options(engine_section_left): left[key] = config.get(engine_section_left, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_left)) exit(1) right = config.get(relation_section, 'right') if right: engine_section_right = 'engine' + '_' + right else: logging.critical('Configuraion error: missing right engine refernce') exit(1) right = None if config.has_section(engine_section_right): right = {} for key in config.options(engine_section_right): right[key] = config.get(engine_section_right, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_right)) exit(1) for engine in [left, right]: secrets = engine['secrets'] path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) if secret.has_section('secrets'): engine['secrets'] = {'key': path} for key in secret.options('secrets'): engine['secrets'][key] = secret.get('secrets', key) else: logger.critical('Configuration error: missing [secrets] in %s' % (path)) exit(1) else: if config.has_option('options', 'secrets'): secrets = config.get('options', 'secrets') path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) section = engine['secrets'][1:-1] if secret.has_section(section): engine['secrets'] = {'key': section} for key in secret.options(section): engine['secrets'][key] = secret.get(section, key) else: logger.critical('Configuration error: missing [%s] in %s' % (section, path)) exit(1) else: logger.critical('Configuration error: missing secret file %s' % (path)) exit(1) else: logger.critical('Configuration error: missing secrets in [options]') exit(1) else: logging.critical('Configuration error: missing section [%s]' % (relation_section)) exit(1) for options in [left, right]: if options.get('class'): cn = options['class'] if class_map.get(cn): options['class'] = class_map[cn] else: logger.critical('Configuration error: unknown sync engine [%s]' % (cn)) exit(1) else: logger.critical('configuration error: missing class tag') exit(1) return Options(left), Options(right), Options(rel)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_normalize': 'yes', 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/dd-agent/checks.d/', 'bind_host': get_default_bind_host(), } # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # # Core config # # FIXME unnecessarily complex if config.has_option('Main', 'use_dd'): agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true") else: agentConfig['use_dd'] = True agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['dd_url'] = "http://" + agentConfig[ 'bind_host'] + ":" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_dd and options.dd_url: agentConfig['dd_url'] = options.dd_url else: agentConfig['dd_url'] = config.get('Main', 'dd_url') if agentConfig['dd_url'].endswith('/'): agentConfig['dd_url'] = agentConfig['dd_url'][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'Datadog', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get( 'Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True if not agentConfig['use_dd']: sys.stderr.write( "Please specify at least one endpoint to send metrics to. This can be done in datadog.conf." ) exit(2) # Which API key to use agentConfig['api_key'] = config.get('Main', 'api_key') # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', 'dogstatsd_normalize': 'yes', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value #Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) # normalize 'yes'/'no' to boolean dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative( dogstatsd_defaults['dogstatsd_normalize']) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')): agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join( [log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if config.has_option("Main", "limit_memory_consumption") and \ config.get("Main", "limit_memory_consumption") is not None: agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def parse_input_file(inp_f: str, verbose: bool) -> Dict[str, Dict[str, str]]: """Read in input file. Reads the input file supplied by the user and returns a dict with the name of run as a key, and the SRR code as value. First ist searches for a Config section, and sets the option to use ranges rather than single values for a key, or changes the default prefix for the codes. If not found, it understands the values as single values (i.e. not ranges), and uses the default prefix. At the moment has limited input validation... fingers crossed. """ config = configparser.ConfigParser() config.read(inp_f) b_ranges = False # check if we have things defined in Config section, otherwise use defaults if config.has_option("Config", "ranges"): if config.get("Config", "ranges") == "True": logging.debug("Setting it to ranges") b_ranges = True if config.has_option("Config", "prefix"): logging.debug("Setting a non-default prefix") my_prefix = config.get("Config", "prefix") else: my_prefix = PREFIX if config.has_option("Config", "prefetch_path"): my_prefetch = config.get("Config", "prefetch_path") else: my_prefetch = PREFETCH if config.has_option("Config", "prefetch_path"): my_fqdump = config.get("Config", "prefetch_path") else: my_fqdump = FQDUMP if config.has_option("Config", "max_dw_size"): my_max_dw_size = config.get("Config", "max_dw_size") else: my_max_dw_size = MAX_DW_SIZE my_config = { "prefetch_exe": my_prefetch, "fqdump_exe": my_fqdump, "max_dw_size": my_max_dw_size, } if not config.has_section("SRR_code") or len( config.options("SRR_code")) < 1: msg = "We could not find SRR_code section, or it was empty.\n\n" msg += "Add a [SRR_code] section in your ini file, specifying the\n" msg += "SRR number (without the SRR prefix to the code).\n" msg += "If your SRR code does not start with SRR, define the prefix\n" msg += "In the [Config] section, under the key 'prefix'\n" logging.error(msg) sys.exit() sra_dict = {} if b_ranges: for item, values in config.items("SRR_code"): # logging.info(item, values) ranges = [int(x) for x in values.split(",")] if len(ranges) != 2 and b_ranges: msg = "Input range needs to have two integer numbers, " msg += f"but I found {len(ranges)} for key {item}\n" msg += f"Make sure to separate the ranges with comas." logging.error(msg) sys.exit() else: rr = Range(*ranges) for i, vv in enumerate(range(rr.start, rr.end + 1)): sra_dict[item + "_" + str(i)] = my_prefix + str(vv) else: for item, value in config.items("SRR_code"): sra_dict[item] = my_prefix + str(value) my_config["sra_dict"] = sra_dict if verbose: logging.info(pformat(my_config)) return my_config
def parse_config(relation, config, _logger): from ConfigParser import ConfigParser from oxsync import OxTaskSync from ensync import EnClientSync from tdsync import ToodledoSync logger = LogAdapter(_logger, {'package': 'config'}) class_map = { 'OxTaskSync': OxTaskSync, 'EnClientSync': EnClientSync, 'ToodledoSync': ToodledoSync } relation_section = 'relation' + '_' + relation if config.has_section(relation_section): rel = {} for key in config.options(relation_section): rel[key] = config.get(relation_section, key) if rel.get('map'): rel['map'] = os.path.expanduser(rel.get('map')) else: logging.critical( 'Configuration error: missing map file path for %s' % (relation)) exit(1) left = config.get(relation_section, 'left') if left: engine_section_left = 'engine' + '_' + left else: logging.critical( 'Configuraion error: missing left engine refernce') exit(1) left = None if config.has_section(engine_section_left): left = {} for key in config.options(engine_section_left): left[key] = config.get(engine_section_left, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_left)) exit(1) right = config.get(relation_section, 'right') if right: engine_section_right = 'engine' + '_' + right else: logging.critical( 'Configuraion error: missing right engine refernce') exit(1) right = None if config.has_section(engine_section_right): right = {} for key in config.options(engine_section_right): right[key] = config.get(engine_section_right, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_right)) exit(1) for engine in [left, right]: secrets = engine['secrets'] path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) if secret.has_section('secrets'): engine['secrets'] = {'key': path} for key in secret.options('secrets'): engine['secrets'][key] = secret.get('secrets', key) else: logger.critical( 'Configuration error: missing [secrets] in %s' % (path)) exit(1) else: if config.has_option('options', 'secrets'): secrets = config.get('options', 'secrets') path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) section = engine['secrets'][1:-1] if secret.has_section(section): engine['secrets'] = {'key': section} for key in secret.options(section): engine['secrets'][key] = secret.get( section, key) else: logger.critical( 'Configuration error: missing [%s] in %s' % (section, path)) exit(1) else: logger.critical( 'Configuration error: missing secret file %s' % (path)) exit(1) else: logger.critical( 'Configuration error: missing secrets in [options]') exit(1) else: logging.critical('Configuration error: missing section [%s]' % (relation_section)) exit(1) for options in [left, right]: if options.get('class'): cn = options['class'] if class_map.get(cn): options['class'] = class_map[cn] else: logger.critical( 'Configuration error: unknown sync engine [%s]' % (cn)) exit(1) else: logger.critical('configuration error: missing class tag') exit(1) return Options(left), Options(right), Options(rel)
def get_config(parse_args = True, cfg_path=None, init_logging=False): if parse_args: options, args = get_parsed_args() else: options = None args = None # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'debug_mode': False, 'dogstatsd_interval': 10, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, 'version': get_version(), 'watchdog': True, } # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if init_logging: initialize_logging(config_path) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # # Core config # # Where to send the data if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main','listen_port'): listen_port = config.get('Main', 'listen_port') agentConfig['dd_url'] = "http://localhost:" + str(listen_port) elif options is not None and options.dd_url: agentConfig['dd_url'] = options.dd_url else: agentConfig['dd_url'] = config.get('Main', 'dd_url') if agentConfig['dd_url'].endswith('/'): agentConfig['dd_url'] = agentConfig['dd_url'][:-1] # Debug mode agentConfig['debug_mode'] = config.get('Main', 'debug_mode').lower() in ("yes", "true") if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int(config.get('Main', 'check_freq')) except: pass # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main','graphite_listen_port'): agentConfig['graphite_listen_port'] = int(config.get('Main','graphite_listen_port')) # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = config.get('Main', 'use_mount').lower() in ("yes", "true", "1") if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def main(): global Topics parser = argparse.ArgumentParser( description='Log MQTT messages to stdOut with timestamp.') parser.add_argument( "-t", "--topic", dest="topic", action="append", help= "MQTT topic to which to subscribe. May be specified multiple times.") parser.add_argument("-o", "--host", dest="MqttHost", action="store", help="MQTT host", default=None) parser.add_argument("-p", "--port", dest="MqttPort", action="store", help="MQTT host port", type=int, default=None) args = parser.parse_args() config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) configFile = GetConfigFilePath() config.read(configFile) cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST'] logger.info("INI file cofig section is: %s", cfgSection) if not config.has_section(cfgSection): logger.critical('Config file "%s", has no section "%s".', configFile, cfgSection) sys.exit(2) if set(config.options(cfgSection)) < RequiredConfigParams: logger.critical( 'Config section "%s" does not have all required params: "%s", it has params: "%s".', cfgSection, RequiredConfigParams, set(config.options(cfgSection))) sys.exit(3) logger.info("INI file cofig section is: %s", cfgSection) cfg = config[cfgSection] mqtt_host = cfg['mqtt_host'] mqtt_port = cfg['mqtt_port'] if config.has_option(cfgSection, 'mqtt_topics'): t = cfg['mqtt_topics'] logger.debug('Topics from config is: "%s".', t) t = t.split() logger.debug('Split topics from config is: "%s".', t) Topics = t logger.debug('Topics from config is: "%s".', Topics) if (args.topic != None) and (len(args.topic) > 0): Topics = args.topic if (args.MqttHost != None) and (len(args.MqttHost) > 0): mqtt_host = args.MqttHost if (args.MqttPort != None) and (len(args.MqttPort) > 0): mqtt_port = args.MqttPort mqtt_port = int(mqtt_port) if (mqtt_host is None) or (mqtt_port is None) or (len(Topics) == 0): logger.critical( 'No mqtt_host OR no mqtt_port OR no topics; must quit.') sys.exit(1) try: logger.debug( 'Connecting to MQTT server at "%s", on port "%s", for topics "%s".', mqtt_host, mqtt_port, Topics) RecClient.connect(mqtt_host, mqtt_port, 60) logger.debug('Mqtt connect returned.') # while not RecClient.connected_flag: #wait in loop # logger.debug("In wait loop") # time.sleep(2) # logger.debug("connection successful") except Exception as e: logger.exception(e) sys.exit(2) try: nowTime = time.time() localtime = time.localtime(nowTime) timeUs = (nowTime - int(nowTime)) * 1000000 logger.debug("Begin receive loop at: %s.%06d %s" % (time.strftime("%Y-%m-%d %H:%M:%S", localtime), timeUs, time.strftime("%Z", localtime))) RecClient.loop_forever() finally: logger.debug('Executing finally clause.') RecClient.disconnect() pass
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/dd-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d' # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative(config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # # Core config #ap if not config.has_option('Main', 'api_key'): log.warning(u"No API key was found. Aborting.") sys.exit(2) if not config.has_option('Main', 'dd_url'): log.warning(u"No dd_url was found. Aborting.") sys.exit(2) # Endpoints dd_urls = map(clean_dd_url, config.get('Main', 'dd_url').split(',')) api_keys = map(lambda el: el.strip(), config.get('Main', 'api_key').split(',')) # For collector and dogstatsd agentConfig['dd_url'] = dd_urls[0] agentConfig['api_key'] = api_keys[0] # Forwarder endpoints logic # endpoints is: # { # 'https://app.datadoghq.com': ['api_key_abc', 'api_key_def'], # 'https://app.example.com': ['api_key_xyz'] # } endpoints = {} dd_urls = remove_empty(dd_urls) api_keys = remove_empty(api_keys) if len(dd_urls) == 1: if len(api_keys) > 0: endpoints[dd_urls[0]] = api_keys else: assert len(dd_urls) == len(api_keys), 'Please provide one api_key for each url' for i, dd_url in enumerate(dd_urls): endpoints[dd_url] = endpoints.get(dd_url, []) + [api_keys[i]] agentConfig['endpoints'] = endpoints # Forwarder or not forwarder agentConfig['use_forwarder'] = options is not None and options.use_forwarder if agentConfig['use_forwarder']: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['dd_url'] = "http://{}:{}".format(agentConfig['bind_host'], listen_port) # FIXME: Legacy dd_url command line switch elif options is not None and options.dd_url is not None: agentConfig['dd_url'] = options.dd_url # Forwarder timeout agentConfig['forwarder_timeout'] = 20 if config.has_option('Main', 'forwarder_timeout'): agentConfig['forwarder_timeout'] = int(config.get('Main', 'forwarder_timeout')) # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get('Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Service discovery if config.has_option('Main', 'service_discovery_backend'): try: additional_config = extract_agent_config(config) agentConfig.update(additional_config) except: log.error('Failed to load the agent configuration related to ' 'service discovery. It will not be used.') # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int(config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates(config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles(config.get('Main', 'histogram_percentiles')) # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port')) # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative(config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative(config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative(config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative(config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative(config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative(config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.ParsingError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.NoOptionError as e: sys.stderr.write('There are some items missing from your config file, but nothing fatal [%s]' % e) # Storing proxy settings in the agentConfig agentConfig['proxy_settings'] = get_proxy(agentConfig) if agentConfig.get('ca_certs', None) is None: agentConfig['ssl_certificate'] = get_ssl_certificate(get_os(), 'datadog-cert.pem') else: agentConfig['ssl_certificate'] = agentConfig['ca_certs'] return agentConfig
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agent_config = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'monstatsd_interval': DEFAULT_STATSD_FREQUENCY, 'monstatsd_agregator_bucket_size': DEFAULT_STATSD_BUCKET_SIZE, 'monstatsd_normalize': 'yes', 'monstatsd_port': 8125, 'forwarder_url': 'http://localhost:17123', 'hostname': None, 'listen_port': None, 'version': get_version(), 'watchdog': True, 'additional_checksd': DEFAULT_CONFIG_DIR + '/checks_d/', } monstatsd_interval = DEFAULT_STATSD_FREQUENCY monstatsd_agregator_bucket_size = DEFAULT_STATSD_BUCKET_SIZE # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agent_config[option] = config.get('Main', option) # # Core config # # FIXME unnecessarily complex # Extra checks_d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agent_config['additional_checksd'] = config.get('Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agent_config['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks_d') # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agent_config['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agent_config['use_web_info_page'] = True # local traffic only? Default to no agent_config['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agent_config['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") if config.has_option('Main', 'check_freq'): try: agent_config['check_freq'] = int(config.get('Main', 'check_freq')) except Exception: pass # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agent_config['watchdog'] = False # monstatsd config monstatsd_defaults = { 'monstatsd_port': 8125, 'monstatsd_interval': monstatsd_interval, 'monstatsd_agregator_bucket_size': monstatsd_agregator_bucket_size, 'monstatsd_normalize': 'yes', } for key, value in monstatsd_defaults.iteritems(): if config.has_option('Main', key): agent_config[key] = config.get('Main', key) else: agent_config[key] = value # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agent_config['statsd_forward_host'] = config.get('Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agent_config['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port')) # normalize 'yes'/'no' to boolean monstatsd_defaults['monstatsd_normalize'] = _is_affirmative( monstatsd_defaults['monstatsd_normalize']) # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agent_config['use_mount'] = _is_affirmative(config.get('Main', 'use_mount')) if config.has_option('Main', 'autorestart'): agent_config['autorestart'] = _is_affirmative(config.get('Main', 'autorestart')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agent_config['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datadog', 'ddforwarder_log'): agent_config['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agent_config["dogstreams"] = ':'.join( [log_path, config.get("Main", "dogstream_line_parser")]) else: agent_config["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agent_config["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agent_config["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_section('WMI'): agent_config['WMI'] = {} for key, value in config.items('WMI'): agent_config['WMI'][key] = value if config.has_option("Main", "limit_memory_consumption") and \ config.get("Main", "limit_memory_consumption") is not None: agent_config["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agent_config["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agent_config["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agent_config['Api'] = get_mon_api_config(config) except ConfigParser.NoSectionError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.ParsingError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.NoOptionError as e: sys.stderr.write( 'There are some items missing from your config file, but nothing fatal [%s]' % e) # Storing proxy settings in the agent_config agent_config['proxy_settings'] = get_proxy(agent_config) return agent_config
def main(): global Topics, TIME_SYNC_UPDATE_MSEC_TOPIC, TIME_SYNC_UPDATE_TOPIC, DontPublish parser = argparse.ArgumentParser( description='Respond to Time Sync request messages with current time.') parser.add_argument( "-t", "--topic", dest="topic", action="append", help= "MQTT topic to which to subscribe. May be specified multiple times.") parser.add_argument("-o", "--host", dest="MqttHost", action="store", help="MQTT host", default=None) parser.add_argument("-p", "--port", dest="MqttPort", action="store", help="MQTT host port", type=int, default=None) parser.add_argument("-P", "--DontPublish", dest="dontpub", action="store_true", help="Do not actually publish time syncs.", default=False) args = parser.parse_args() config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) configFile = GetConfigFilePath() config.read(configFile) cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST'] logger.info("INI file cofig section is: %s", cfgSection) if not config.has_section(cfgSection): logger.critical('Config file "%s", has no section "%s".', configFile, cfgSection) sys.exit(2) if set(config.options(cfgSection)) < RequiredConfigParams: logger.critical( 'Config section "%s" does not have all required params: "%s", it has params: "%s".', cfgSection, RequiredConfigParams, set(config.options(cfgSection))) sys.exit(3) logger.info("INI file cofig section is: %s", cfgSection) cfg = config[cfgSection] mqtt_host = cfg['mqtt_host'] mqtt_port = cfg['mqtt_port'] TIME_SYNC_UPDATE_MSEC_TOPIC = cfg['mqtt_sync_ms_topic'] TIME_SYNC_UPDATE_TOPIC = cfg['mqtt_sync_topic'] if config.has_option(cfgSection, 'mqtt_request_topics'): t = cfg['mqtt_request_topics'] logger.debug('Topics from config is: "%s".', t) t = t.split() logger.debug('Split topics from config is: "%s".', t) Topics = t logger.debug('Topics from config is: "%s".', Topics) if (args.topic != None) and (len(args.topic) > 0): Topics = args.topic if (args.MqttHost != None) and (len(args.MqttHost) > 0): mqtt_host = args.MqttHost if (args.MqttPort != None) and (len(args.MqttPort) > 0): mqtt_port = args.MqttPort #DontPublish if (args.dontpub != None): DontPublish = args.dontpub mqtt_port = int(mqtt_port) if (mqtt_host is None) or (mqtt_port is None) or (len(Topics) == 0): logger.critical( 'No mqtt_host OR no mqtt_port OR no topics; must quit.') sys.exit(1) try: logger.debug( 'Connecting to MQTT server at "%s", on port "%s", for topics "%s".', mqtt_host, mqtt_port, Topics) RecClient.connect(mqtt_host, mqtt_port, 60) logger.debug('Mqtt connect returned.') logger.debug("Loop immediately after connect returns code: %s", mqtt.error_string(RecClient.loop())) if (RecClient._state != mqtt.mqtt_cs_connected): logger.debug("Not immediately connected.") except Exception as e: logger.exception(e) sys.exit(2) try: nowTime = time.time() localtime = time.localtime(nowTime) timeUs = (nowTime - int(nowTime)) * 1000000 logger.debug("Begin receive loop at: %s.%06d %s" % (time.strftime("%Y-%m-%d %H:%M:%S", localtime), timeUs, time.strftime("%Z", localtime))) RecClient.loop_forever() finally: logger.debug('Executing finally clause.') RecClient.disconnect() pass
def main(): global Topics, mqtt_msg_table, DBConn, DBCursor, dontWriteDb, RecClient parser = argparse.ArgumentParser( description='Log MQTT messages to database.') parser.add_argument( "-t", "--topic", dest="topic", action="append", help= "MQTT topic to which to subscribe. May be specified multiple times.") parser.add_argument("-o", "--host", dest="MqttHost", action="store", help="MQTT host", default=None) parser.add_argument("-p", "--port", dest="MqttPort", action="store", help="MQTT host port", type=int, default=None) parser.add_argument("-O", "--Host", dest="DbHost", action="store", help="Database host", default=None) parser.add_argument("-P", "--Port", dest="DbPort", action="store", help="Database host port", type=int, default=None) parser.add_argument("-U", "--User", dest="DbUser", action="store", help="Database user name", default=None) parser.add_argument("-D", "--Password", dest="DbPass", action="store", help="Database user password", default=None) parser.add_argument("-S", "--Schema", dest="DbSchema", action="store", help="Database schema", default=None) parser.add_argument("-T", "--MsgTable", dest="MsgTable", action="store", help="Database table in which to store messages.", default=None) parser.add_argument("-W", "--dontWriteToDB", dest="noWriteDb", action="store_true", help="Don't write to database.") args = parser.parse_args() dontWriteDb = args.noWriteDb config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) configFile = GetConfigFilePath() config.read(configFile) cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST'] logger.info("INI file cofig section is: %s", cfgSection) if not config.has_section(cfgSection): logger.critical('Config file "%s", has no section "%s".', configFile, cfgSection) sys.exit(2) if set(config.options(cfgSection)) < RequiredConfigParams: logger.critical( 'Config section "%s" does not have all required params: "%s", it has params: "%s".', cfgSection, RequiredConfigParams, set(config.options(cfgSection))) sys.exit(3) cfg = config[cfgSection] db_user = cfg['inserter_user'] db_pwd = cfg['inserter_password'] db_host = cfg['inserter_host'] db_port = int(cfg['inserter_port']) myschema = cfg['inserter_schema'] mqtt_host = cfg['mqtt_host'] mqtt_port = cfg['mqtt_port'] mqtt_msg_table = cfg['mqtt_msg_table'] # Create mqtt client and establish call-backs. RecClient = mqtt.Client() RecClient.on_connect = on_connect RecClient.on_message = on_message RecClient.on_subscribe = on_subscribe if config.has_option(cfgSection, 'mqtt_topics'): t = cfg['mqtt_topics'] logger.debug('Topics from config is: "%s".', t) t = t.split() logger.debug('Split topics from config is: "%s".', t) Topics = t logger.debug('Topics from config is: "%s".', Topics) if (args.topic is not None) and (len(args.topic) > 0): Topics = args.topic if (args.MqttHost is not None) and (len(args.MqttHost) > 0): mqtt_host = args.MqttHost if (args.MqttPort is not None) and (len(args.MqttPort) > 0): mqtt_port = args.MqttPort if (args.MsgTable is not None) and (len(args.MsgTable) > 0): mqtt_msg_table = args.MsgTable if (args.DbHost is not None) and (len(args.DbHost) > 0): db_host = args.DbHost if (args.DbPass is not None) and (len(args.DbPass) > 0): db_pwd = args.DbPass if (args.DbUser is not None) and (len(args.DbUser) > 0): db_user = args.DbUser if (args.DbPort is not None) and (len(args.DbPort) > 0): db_port = args.DbPort if (args.DbSchema is not None) and (len(args.DbSchema) > 0): myschema = args.DbSchema db_port = int(db_port) mqtt_port = int(mqtt_port) if len(Topics) <= 0: logger.critical('No mqtt subscription topics given. Must exit.') sys.exit(4) logger.info( 'Database connection args: host: "%s", port: %d, User: "******", Pass: REDACTED, Schema: "%s"', db_host, db_port, db_user, myschema) logger.info( 'Mqtt parameters: host: "%s", port: %d, topic(s): "%s", mqtt msg table: "%s".', mqtt_host, mqtt_port, Topics, mqtt_msg_table) try: DBConn = mysql.connector.connect(host=db_host, port=db_port, user=db_user, password=db_pwd, db=myschema, charset='utf8mb4') if DBConn.is_connected(): logger.info('Connected to MySQL database') logger.info(DBConn) DBConn.start_transaction() DBCursor = DBConn.cursor() try: RecClient.connect(mqtt_host, mqtt_port, 60) RecClient.loop_forever() except Exception as e: logger.info( 'RecClient.loop_forever() raised exception; exit main(); Wait awhile, then quit -- Launchctl will restart us.' ) logger.exception(e) raise finally: logger.info( 'RecClient.loop_forever() returned; exit main(); Wait awhile, then quit -- Launchctl will restart us.' ) RecClient.disconnect() DBConn.disconnect() return else: logger.error('Failed to connect to database.') except Error as e: logger.critical(e)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { "check_freq": DEFAULT_CHECK_FREQUENCY, "dogstatsd_port": 8125, "dogstatsd_target": "http://localhost:17123", "graphite_listen_port": None, "hostname": None, "listen_port": None, "tags": None, "use_ec2_instance_id": False, # DEPRECATED "version": get_version(), "watchdog": True, "additional_checksd": "/etc/dd-agent/checks.d/", "bind_host": get_default_bind_host(), "statsd_metric_namespace": None, "utf8_decoding": False, } if Platform.is_mac(): agentConfig["additional_checksd"] = "/opt/datadog-agent/etc/checks.d" # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options("Main"): agentConfig[option] = config.get("Main", option) # Store developer mode setting in the agentConfig in_developer_mode = None if config.has_option("Main", "developer_mode"): agentConfig["developer_mode"] = _is_affirmative(config.get("Main", "developer_mode")) # Allow an override with the --profile option if options is not None and options.profile: agentConfig["developer_mode"] = True # # Core config # # FIXME unnecessarily complex agentConfig["use_forwarder"] = False if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option("Main", "listen_port"): listen_port = int(config.get("Main", "listen_port")) agentConfig["dd_url"] = "http://" + agentConfig["bind_host"] + ":" + str(listen_port) agentConfig["use_forwarder"] = True elif options is not None and not options.disable_dd and options.dd_url: agentConfig["dd_url"] = options.dd_url else: agentConfig["dd_url"] = config.get("Main", "dd_url") if agentConfig["dd_url"].endswith("/"): agentConfig["dd_url"] = agentConfig["dd_url"][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option("Main", "additional_checksd"): agentConfig["additional_checksd"] = config.get("Main", "additional_checksd") elif get_os() == "windows": # default windows location common_path = _windows_commondata_path() agentConfig["additional_checksd"] = os.path.join(common_path, "Datadog", "checks.d") if config.has_option("Main", "use_dogstatsd"): agentConfig["use_dogstatsd"] = config.get("Main", "use_dogstatsd").lower() in ("yes", "true") else: agentConfig["use_dogstatsd"] = True # Concerns only Windows if config.has_option("Main", "use_web_info_page"): agentConfig["use_web_info_page"] = config.get("Main", "use_web_info_page").lower() in ("yes", "true") else: agentConfig["use_web_info_page"] = True # Which API key to use agentConfig["api_key"] = config.get("Main", "api_key") # local traffic only? Default to no agentConfig["non_local_traffic"] = False if config.has_option("Main", "non_local_traffic"): agentConfig["non_local_traffic"] = config.get("Main", "non_local_traffic").lower() in ("yes", "true") # DEPRECATED if config.has_option("Main", "use_ec2_instance_id"): use_ec2_instance_id = config.get("Main", "use_ec2_instance_id") # translate yes into True, the rest into False agentConfig["use_ec2_instance_id"] = use_ec2_instance_id.lower() == "yes" if config.has_option("Main", "check_freq"): try: agentConfig["check_freq"] = int(config.get("Main", "check_freq")) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option("Main", "histogram_aggregates"): agentConfig["histogram_aggregates"] = get_histogram_aggregates(config.get("Main", "histogram_aggregates")) if config.has_option("Main", "histogram_percentiles"): agentConfig["histogram_percentiles"] = get_histogram_percentiles( config.get("Main", "histogram_percentiles") ) # Disable Watchdog (optionally) if config.has_option("Main", "watchdog"): if config.get("Main", "watchdog").lower() in ("no", "false"): agentConfig["watchdog"] = False # Optional graphite listener if config.has_option("Main", "graphite_listen_port"): agentConfig["graphite_listen_port"] = int(config.get("Main", "graphite_listen_port")) else: agentConfig["graphite_listen_port"] = None # Dogstatsd config dogstatsd_defaults = { "dogstatsd_port": 8125, "dogstatsd_target": "http://" + agentConfig["bind_host"] + ":17123", } for key, value in dogstatsd_defaults.iteritems(): if config.has_option("Main", key): agentConfig[key] = config.get("Main", key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig["create_dd_check_tags"] = config.has_option("Main", "create_dd_check_tags") and _is_affirmative( config.get("Main", "create_dd_check_tags") ) # Forwarding to external statsd server if config.has_option("Main", "statsd_forward_host"): agentConfig["statsd_forward_host"] = config.get("Main", "statsd_forward_host") if config.has_option("Main", "statsd_forward_port"): agentConfig["statsd_forward_port"] = int(config.get("Main", "statsd_forward_port")) # optionally send dogstatsd data directly to the agent. if config.has_option("Main", "dogstatsd_use_ddurl"): if _is_affirmative(config.get("Main", "dogstatsd_use_ddurl")): agentConfig["dogstatsd_target"] = agentConfig["dd_url"] # Optional config # FIXME not the prettiest code ever... if config.has_option("Main", "use_mount"): agentConfig["use_mount"] = _is_affirmative(config.get("Main", "use_mount")) if options is not None and options.autorestart: agentConfig["autorestart"] = True elif config.has_option("Main", "autorestart"): agentConfig["autorestart"] = _is_affirmative(config.get("Main", "autorestart")) if config.has_option("Main", "check_timings"): agentConfig["check_timings"] = _is_affirmative(config.get("Main", "check_timings")) if config.has_option("Main", "exclude_process_args"): agentConfig["exclude_process_args"] = _is_affirmative(config.get("Main", "exclude_process_args")) try: filter_device_re = config.get("Main", "device_blacklist_re") agentConfig["device_blacklist_re"] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option("datadog", "ddforwarder_log"): agentConfig["has_datadog"] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ":".join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative(config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section("WMI"): agentConfig["WMI"] = {} for key, value in config.items("WMI"): agentConfig["WMI"][key] = value if ( config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None ): agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch") ) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative(config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative(config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError, e: sys.stderr.write("Config file not found or incorrectly formatted.\n") sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/dd-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d' # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative( config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # # Core config #ap if not config.has_option('Main', 'api_key'): log.warning(u"No API key was found. Aborting.") sys.exit(2) if not config.has_option('Main', 'dd_url'): log.warning(u"No dd_url was found. Aborting.") sys.exit(2) # Endpoints dd_url = clean_dd_url(config.get('Main', 'dd_url')) api_key = config.get('Main', 'api_key').strip() # For collector and dogstatsd agentConfig['api_key'] = api_key agentConfig['dd_url'] = dd_url # multiple endpoints if config.has_option('Main', 'other_dd_urls'): other_dd_urls = map(clean_dd_url, config.get('Main', 'other_dd_urls').split(',')) else: other_dd_urls = [] if config.has_option('Main', 'other_api_keys'): other_api_keys = map( lambda x: x.strip(), config.get('Main', 'other_api_keys').split(',')) else: other_api_keys = [] # Forwarder endpoints logic # endpoints is: # { # 'https://app.datadoghq.com': ['api_key_abc', 'api_key_def'], # 'https://app.example.com': ['api_key_xyz'] # } endpoints = {dd_url: [api_key]} if len(other_dd_urls) == 0: endpoints[dd_url] += other_api_keys else: assert len(other_dd_urls) == len( other_api_keys), 'Please provide one api_key for each url' for i, other_dd_url in enumerate(other_dd_urls): endpoints[other_dd_url] = endpoints.get( other_dd_url, []) + [other_api_keys[i]] agentConfig['endpoints'] = endpoints # Forwarder or not forwarder agentConfig[ 'use_forwarder'] = options is not None and options.use_forwarder if agentConfig['use_forwarder']: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['dd_url'] = "http://{}:{}".format( agentConfig['bind_host'], listen_port) # FIXME: Legacy dd_url command line switch elif options is not None and options.dd_url is not None: agentConfig['dd_url'] = options.dd_url # Forwarder timeout agentConfig['forwarder_timeout'] = 20 if config.has_option('Main', 'forwarder_timeout'): agentConfig['forwarder_timeout'] = int( config.get('Main', 'forwarder_timeout')) # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'Datadog', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get( 'Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Service discovery if config.has_option('Main', 'service_discovery_backend'): try: additional_config = extract_agent_config(config) agentConfig.update(additional_config) except: log.error('Failed to load the agent configuration related to ' 'service discovery. It will not be used.') # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates( config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles( config.get('Main', 'histogram_percentiles')) # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')): agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join( [log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative( config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative( config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.ParsingError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.NoOptionError as e: sys.stderr.write( 'There are some items missing from your config file, but nothing fatal [%s]' % e) # Storing proxy settings in the agentConfig agentConfig['proxy_settings'] = get_proxy(agentConfig) if agentConfig.get('ca_certs', None) is None: agentConfig['ssl_certificate'] = get_ssl_certificate( get_os(), 'datadog-cert.pem') else: agentConfig['ssl_certificate'] = agentConfig['ca_certs'] return agentConfig
def get_config(parse_args=True, cfg_path=None, init_logging=False, options=None): if parse_args: options, args = get_parsed_args() elif not options: args = None # General config agentConfig = { "check_freq": DEFAULT_CHECK_FREQUENCY, "debug_mode": False, "dogstatsd_interval": DEFAULT_STATSD_FREQUENCY, "dogstatsd_normalize": "yes", "dogstatsd_port": 8125, "dogstatsd_target": "http://localhost:17123", "graphite_listen_port": None, "hostname": None, "listen_port": None, "tags": None, "use_ec2_instance_id": False, # DEPRECATED "version": get_version(), "watchdog": True, } dogstatsd_interval = DEFAULT_STATSD_FREQUENCY # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=getOS()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if init_logging: initialize_logging(config_path, os_name=getOS()) # bulk import for option in config.options("Main"): agentConfig[option] = config.get("Main", option) # # Core config # # FIXME unnecessarily complex if config.has_option("Main", "use_dd"): agentConfig["use_dd"] = config.get("Main", "use_dd").lower() in ("yes", "true") else: agentConfig["use_dd"] = True if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option("Main", "listen_port"): listen_port = int(config.get("Main", "listen_port")) agentConfig["dd_url"] = "http://localhost:" + str(listen_port) elif options is not None and not options.disable_dd and options.dd_url: agentConfig["dd_url"] = options.dd_url else: agentConfig["dd_url"] = config.get("Main", "dd_url") if agentConfig["dd_url"].endswith("/"): agentConfig["dd_url"] = agentConfig["dd_url"][:-1] # Whether also to send to Pup if config.has_option("Main", "use_pup"): agentConfig["use_pup"] = config.get("Main", "use_pup").lower() in ("yes", "true") else: agentConfig["use_pup"] = True if agentConfig["use_pup"]: if config.has_option("Main", "pup_url"): agentConfig["pup_url"] = config.get("Main", "pup_url") else: agentConfig["pup_url"] = "http://localhost:17125" pup_port = 17125 if config.has_option("Main", "pup_port"): agentConfig["pup_port"] = int(config.get("Main", "pup_port")) # Increases the frequency of statsd metrics when only sending to Pup if not agentConfig["use_dd"] and agentConfig["use_pup"]: dogstatsd_interval = PUP_STATSD_FREQUENCY if not agentConfig["use_dd"] and not agentConfig["use_pup"]: sys.stderr.write( "Please specify at least one endpoint to send metrics to. This can be done in datadog.conf." ) exit(2) # Which API key to use agentConfig["api_key"] = config.get("Main", "api_key") # Debug mode agentConfig["debug_mode"] = config.get("Main", "debug_mode").lower() in ("yes", "true") # local traffic only? Default to no agentConfig["non_local_traffic"] = False if config.has_option("Main", "non_local_traffic"): agentConfig["non_local_traffic"] = config.get("Main", "non_local_traffic").lower() in ("yes", "true") # DEPRECATED if config.has_option("Main", "use_ec2_instance_id"): use_ec2_instance_id = config.get("Main", "use_ec2_instance_id") # translate yes into True, the rest into False agentConfig["use_ec2_instance_id"] = use_ec2_instance_id.lower() == "yes" if config.has_option("Main", "check_freq"): try: agentConfig["check_freq"] = int(config.get("Main", "check_freq")) except: pass # Disable Watchdog (optionally) if config.has_option("Main", "watchdog"): if config.get("Main", "watchdog").lower() in ("no", "false"): agentConfig["watchdog"] = False # Optional graphite listener if config.has_option("Main", "graphite_listen_port"): agentConfig["graphite_listen_port"] = int(config.get("Main", "graphite_listen_port")) else: agentConfig["graphite_listen_port"] = None # Dogstatsd config dogstatsd_defaults = { "dogstatsd_port": 8125, "dogstatsd_target": "http://localhost:17123", "dogstatsd_interval": dogstatsd_interval, "dogstatsd_normalize": "yes", } for key, value in dogstatsd_defaults.iteritems(): if config.has_option("Main", key): agentConfig[key] = config.get("Main", key) else: agentConfig[key] = value # normalize 'yes'/'no' to boolean dogstatsd_defaults["dogstatsd_normalize"] = _is_affirmative(dogstatsd_defaults["dogstatsd_normalize"]) # optionally send dogstatsd data directly to the agent. if config.has_option("Main", "dogstatsd_use_ddurl"): use_ddurl = _is_affirmative(config.get("Main", "dogstatsd_use_ddurl")) if use_ddurl: agentConfig["dogstatsd_target"] = agentConfig["dd_url"] # Optional config # FIXME not the prettiest code ever... if config.has_option("Main", "use_mount"): agentConfig["use_mount"] = config.get("Main", "use_mount").lower() in ("yes", "true", "1") if config.has_option("datadog", "ddforwarder_log"): agentConfig["has_datadog"] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ":".join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_section("WMI"): agentConfig["WMI"] = {} for key, value in config.items("WMI"): agentConfig["WMI"][key] = value except ConfigParser.NoSectionError, e: sys.stderr.write("Config file not found or incorrectly formatted.\n") sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/dd-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig in_developer_mode = None if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative(config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # # Core config # # FIXME unnecessarily complex if config.has_option('Main', 'use_dd'): agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true") else: agentConfig['use_dd'] = True agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['dd_url'] = "http://" + agentConfig['bind_host'] + ":" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_dd and options.dd_url: agentConfig['dd_url'] = options.dd_url else: agentConfig['dd_url'] = config.get('Main', 'dd_url') if agentConfig['dd_url'].endswith('/'): agentConfig['dd_url'] = agentConfig['dd_url'][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get('Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True if not agentConfig['use_dd']: sys.stderr.write("Please specify at least one endpoint to send metrics to. This can be done in datadog.conf.") exit(2) # Which API key to use agentConfig['api_key'] = config.get('Main', 'api_key') # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int(config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates(config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles(config.get('Main', 'histogram_percentiles')) # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port')) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')): agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative(config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative(config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative(config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative(config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative(config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative(config.get("Main", "utf8_decoding")) except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'monitorstatsd_port': 8125, 'monitorstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, 'version': get_version(), 'watchmonitor': True, 'additional_checksd': '/etc/monitor-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d' try: path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) for option in config.options('Main'): agentConfig[option] = config.get('Main', option) if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative( config.get('Main', 'developer_mode')) if options is not None and options.profile: agentConfig['developer_mode'] = True if config.has_option("Main", "frequency"): agentConfig['check_freq'] = config.get("Main", "frequency") agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['m_url'] = "http://" + agentConfig[ 'bind_host'] + ":" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_dd and options.m_url: agentConfig['m_url'] = options.m_url else: agentConfig['m_url'] = config.get('Main', 'm_url') if agentConfig['m_url'].endswith('/'): agentConfig['m_url'] = agentConfig['m_url'][:-1] if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'Datamonitor', 'checks.d') if config.has_option('Main', 'use_monitorstatsd'): agentConfig['use_monitorstatsd'] = config.get( 'Main', 'use_monitorstatsd').lower() in ("yes", "true") else: agentConfig['use_monitorstatsd'] = True if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True agentConfig['api_key'] = config.get('Main', 'api_key') agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates( config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles( config.get('Main', 'histogram_percentiles')) if config.has_option('Main', 'watchmonitor'): if config.get('Main', 'watchmonitor').lower() in ('no', 'false'): agentConfig['watchmonitor'] = False if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None monitorstatsd_defaults = { 'monitorstatsd_port': 8125, 'monitorstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in monitorstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) if config.has_option('Main', 'monitorstatsd_use_murl'): if _is_affirmative(config.get('Main', 'monitorstatsd_use_murl')): agentConfig['monitorstatsd_target'] = agentConfig['m_url'] if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datamonitor', 'ddforwarder_log'): agentConfig['has_datamonitor'] = True if config.has_option("Main", "monitorstream_log"): log_path = config.get("Main", "monitorstream_log") if config.has_option("Main", "monitorstream_line_parser"): agentConfig["monitorstreams"] = ':'.join([ log_path, config.get("Main", "monitorstream_line_parser") ]) else: agentConfig["monitorstreams"] = log_path elif config.has_option("Main", "monitorstreams"): agentConfig["monitorstreams"] = config.get("Main", "monitorstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative( config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative( config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.ParsingError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.NoOptionError as e: sys.stderr.write( 'There are some items missing from your config file, but nothing fatal [%s]' % e) agentConfig['proxy_settings'] = get_proxy(agentConfig) if agentConfig.get('ca_certs', None) is None: agentConfig['ssl_certificate'] = get_ssl_certificate( get_os(), 'datamonitor-cert.pem') else: agentConfig['ssl_certificate'] = agentConfig['ca_certs'] agentConfig['interval'] = config.get('Main', 'updater_interval') return agentConfig
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY, 'dogstatsd_normalize': 'yes', 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/dd-agent/checks.d/', } dogstatsd_interval = DEFAULT_STATSD_FREQUENCY # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # # Core config # # FIXME unnecessarily complex if config.has_option('Main', 'use_dd'): agentConfig['use_dd'] = config.get('Main', 'use_dd').lower() in ("yes", "true") else: agentConfig['use_dd'] = True agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['dd_url'] = "http://localhost:" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_dd and options.dd_url: agentConfig['dd_url'] = options.dd_url else: agentConfig['dd_url'] = config.get('Main', 'dd_url') if agentConfig['dd_url'].endswith('/'): agentConfig['dd_url'] = agentConfig['dd_url'][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d') # Whether also to send to Pup if config.has_option('Main', 'use_pup'): agentConfig['use_pup'] = config.get('Main', 'use_pup').lower() in ("yes", "true") else: agentConfig['use_pup'] = True # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True if agentConfig['use_pup'] or agentConfig['use_web_info_page']: if config.has_option('Main', 'pup_url'): agentConfig['pup_url'] = config.get('Main', 'pup_url') else: agentConfig['pup_url'] = 'http://localhost:17125' if config.has_option('Main', 'pup_port'): agentConfig['pup_port'] = int(config.get('Main', 'pup_port')) # Increases the frequency of statsd metrics when only sending to Pup if not agentConfig['use_dd'] and agentConfig['use_pup']: dogstatsd_interval = PUP_STATSD_FREQUENCY if not agentConfig['use_dd'] and not agentConfig['use_pup']: sys.stderr.write("Please specify at least one endpoint to send metrics to. This can be done in datadog.conf.") exit(2) # Which API key to use agentConfig['api_key'] = config.get('Main', 'api_key') # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = (use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int(config.get('Main', 'check_freq')) except Exception: pass # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'dogstatsd_interval': dogstatsd_interval, 'dogstatsd_normalize': 'yes', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value #Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port')) # normalize 'yes'/'no' to boolean dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(dogstatsd_defaults['dogstatsd_normalize']) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): use_ddurl = _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')) if use_ddurl: agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount')) if config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg") if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if config.has_option("Main", "limit_memory_consumption") and \ config.get("Main", "limit_memory_consumption") is not None: agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative(config.get("Main", "collect_ec2_tags")) except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def _list_ids(self, config, section): ids = [] options = config.options(section) for option in options: ids.append(config.get(section, option)) return ids
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/sd-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/usr/local/etc/sd-agent/checks.d/' # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative( config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # # Core config # # FIXME unnecessarily complex if config.has_option('Main', 'sd_account'): agentConfig['sd_account'] = config.get('Main', 'sd_account') agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17124 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['sd_url'] = "http://" + agentConfig[ 'bind_host'] + ":" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_sd and options.sd_url: agentConfig['sd_url'] = options.sd_url elif config.has_option('Main', 'sd_url'): agentConfig['sd_url'] = config.get('Main', 'sd_url') else: # Default agent URL agentConfig['sd_url'] = "https://" + agentConfig[ 'sd_account'] + ".agent.serverdensity.io" if agentConfig['sd_url'].endswith('/'): agentConfig['sd_url'] = agentConfig['sd_url'][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'ServerDensity', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get( 'Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Service discovery if config.has_option('Main', 'service_discovery_backend'): try: additional_config = extract_agent_config(config) agentConfig.update(additional_config) except: log.error('Failed to load the agent configuration related to ' 'service discovery. It will not be used.') # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True # Which agent key to use agentConfig['agent_key'] = config.get('Main', 'agent_key') # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates( config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles( config.get('Main', 'histogram_percentiles')) # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')): agentConfig['dogstatsd_target'] = agentConfig['sd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative( config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative( config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def _convert_to_attribute(self, config, obj, name): for option in config.options(name): setattr(obj, option, config.get(name, option)) return obj