def getShares(tsn=''): shares = [(section, Bdict(config.items(section))) for section in config.sections() if not (section.startswith(('_tivo_', 'logger_', 'handler_', 'formatter_')) or section in ('Server', 'loggers', 'handlers', 'formatters'))] tsnsect = '_tivo_' + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, 'shares').split(','): y = x.strip() if config.has_section(y): tsnshares.append((y, Bdict(config.items(y)))) shares = tsnshares shares.sort() if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']: shares.append(('Settings', {'type': 'settings'})) if get_server('tivo_mak') and get_server('togo_path'): shares.append(('ToGo', {'type': 'togo'})) return shares
def getShares(tsn=''): shares = [(section, Bdict(config.items(section))) for section in config.sections() if not (section.startswith(('_tivo_', 'logger_', 'handler_', 'formatter_')) or section in ('Server', 'loggers', 'handlers', 'formatters') ) ] tsnsect = '_tivo_' + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, 'shares').split(','): y = x.strip() if config.has_section(y): tsnshares.append((y, Bdict(config.items(y)))) shares = tsnshares shares.sort() if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']: shares.append(('Settings', {'type': 'settings'})) if get_server('tivo_mak') and get_server('togo_path'): shares.append(('ToGo', {'type': 'togo'})) return shares
def getShares(tsn=''): shares = [(section, Bdict(config.items(section))) for section in config.sections() if not (section.startswith(special_section_prefixes) or section in special_section_names) ] tsnsect = '_tivo_' + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, 'shares').split(','): y = x.strip() if config.has_section(y): tsnshares.append((y, Bdict(config.items(y)))) shares = tsnshares shares.sort() if not config.getboolean('Server', 'nosettings', fallback=False): shares.append(('Settings', {'type': 'settings'})) if get_server('tivo_mak') and get_togo('path'): shares.append(('ToGo', {'type': 'togo'})) return shares
def getShares(tsn=''): shares = [(section, Bdict(config.items(section))) for section in config.sections() if not (section.startswith(('_tivo_', 'logger_', 'handler_', 'formatter_')) or section in ('Server', 'loggers', 'handlers', 'formatters') ) ] tsnsect = '_tivo_' + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, 'shares').split(','): y = x.strip() if config.has_section(y): tsnshares.append((y, Bdict(config.items(y)))) shares = tsnshares shares.sort() if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']: shares.append(('Settings', {'type': 'settings'})) if get_server('tivo_mak') and get_server('togo_path'): shares.append(('ToGo', {'type': 'togo'})) if sys.platform == 'win32': shares.append(('VRD', {'type': 'vrd'})) if getattr(sys, 'frozen', False): shares.append(('Desktop', {'type': 'desktop', 'path': os.path.join(sys._MEIPASS, 'plugins', 'desktop', 'content')})) return shares
def register_schedule(self): channel = config.get('irc', 'channel') for schedule in config.getlist('irc', 'schedules'): sect = ':'.join(('schedule', schedule)) # do not evaluate isenabled() here. # if it does, the disabled action will never be scheduled. if not config.has_section(sect): logging.error('[schedule] [%s] no such schedule', sect) continue if not config.has_option(sect, 'action'): logging.error('[schedule] [%s] no action specified', sect) continue action = ':'.join(('action', config.get(sect, 'action'))) if not config.has_section(action): logging.error('[schedule] [%s] invalid action specified', sect) continue interval = config.getint(sect, 'interval') if interval < 60: logging.error('[schedule] [%s] interval too short', sect) continue self.reactor.register_schedule(interval, self.do_action, action, self.connection, None, {'target': channel}, sect) logging.info('[schedule] [%s] registered', sect)
def create_models(self, config): self.models = collections.OrderedDict() self.main_model = None for section in config.sections(): if section.startswith('table:'): table_schema = config.items(section) table, name = section.split(':', 2) filter_section = 'filter:' + name filters = [] if config.has_section(filter_section): filters = config.items(filter_section) index_section = 'index:' + name index_schema = [] if config.has_section(index_section): index_schema = config.items(index_section) self.init_model(name, table_schema, index_schema, filters) self.sorted_models = sorted( self.models.values(), key=lambda m: len(m._meta.table_options.get('__iter__', []))) if self.client_config['resume'] and not self.main_model: raise ValueError('Main model is required for resume mode') if not self.main_model: logger.warning('Main model is not defined in config')
def getShares(tsn=""): shares = [ (section, dict(config.items(section))) for section in config.sections() if not ( section.startswith("_tivo_") or section.startswith("logger_") or section.startswith("handler_") or section.startswith("formatter_") or section in ("Server", "loggers", "handlers", "formatters") ) ] tsnsect = "_tivo_" + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, "shares"): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, "shares").split(","): y = x.strip() if config.has_section(y): tsnshares.append((y, dict(config.items(y)))) shares = tsnshares shares.sort() if get_server("nosettings", "false").lower() in ["false", "no", "off"]: shares.append(("Settings", {"type": "settings"})) if get_server("tivo_mak") and get_server("togo_path"): shares.append(("ToGo", {"type": "togo"})) return shares
def get_config(service_conf, section=''): config = ConfigParser.ConfigParser() config.read(service_conf) conf_items = dict(config.items('common')) if config.has_section('common') else {} if section and config.has_section(section): conf_items.update(config.items(section)) return conf_items
def get_config(section=''): config = ConfigParser.ConfigParser() service_conf = os.path.join(work_dir,'conf/service.conf') config.read(service_conf) config_items = dict(config.items('common')) if config.has_section('common') else {} if section and config.has_section(section): config_items.update(config.items(section)) return config_items
def init_logging(): if config.has_section("loggers") and config.has_section("handlers") and config.has_section("formatters"): logging.config.fileConfig(config_files) elif getDebug(): logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO)
def init_logging(): if (config.has_section('loggers') and config.has_section('handlers') and config.has_section('formatters')): logging.config.fileConfig(config_files) elif getDebug(): logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO)
def get_logging_config(cfg_path=None): logging_config = { 'log_level': None, 'collector_log_file': '/var/log/datadog/collector.log', 'forwarder_log_file': '/var/log/datadog/forwarder.log', 'dogstatsd_log_file': '/var/log/datadog/dogstatsd.log', 'pup_log_file': '/var/log/datadog/pup.log', 'log_to_syslog': True, 'syslog_host': None, 'syslog_port': None, } config_path = get_config_path(cfg_path, os_name=getOS()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if config.has_section('handlers') or config.has_section('loggers') or config.has_section('formatters'): sys.stderr.write("Python logging config is no longer supported and will be ignored.\nTo configure logging, update the logging portion of 'datadog.conf' to match:\n 'https://github.com/DataDog/dd-agent/blob/master/datadog.conf.example'.\n") for option in logging_config: if config.has_option('Main', option): logging_config[option] = config.get('Main', option) levels = { 'CRITICAL': logging.CRITICAL, 'DEBUG': logging.DEBUG, 'ERROR': logging.ERROR, 'FATAL': logging.FATAL, 'INFO': logging.INFO, 'WARN': logging.WARN, 'WARNING': logging.WARNING, } if config.has_option('Main', 'log_level'): logging_config['log_level'] = levels.get(config.get('Main', 'log_level')) if config.has_option('Main', 'log_to_syslog'): logging_config['log_to_syslog'] = config.get('Main', 'log_to_syslog').strip().lower() in ['yes', 'true', 1] if config.has_option('Main', 'syslog_host'): host = config.get('Main', 'syslog_host').strip() if host: logging_config['syslog_host'] = host else: logging_config['syslog_host'] = None if config.has_option('Main', 'syslog_port'): port = config.get('Main', 'syslog_port').strip() try: logging_config['syslog_port'] = int(port) except: logging_config['syslog_port'] = None return logging_config
def dict2config(jdict, section): config = ConfigParser() if not config.has_section(section): config.add_section(section) for (k, v) in viewitems(jdict): config.set(section, k, str(v)) return config
def reset(): global bin_paths global config global configs_found bin_paths = {} config = ConfigParser.ConfigParser() configs_found = config.read(config_files) if not configs_found: print ('WARNING: pyTivo.conf does not exist.\n' + 'Assuming default values.') configs_found = config_files[-1:] for section in config.sections(): if section.startswith('_tivo_'): tsn = section[6:] if tsn.upper() not in ['SD', 'HD']: if config.has_option(section, 'name'): tivo_names[tsn] = config.get(section, 'name') else: tivo_names[tsn] = tsn if config.has_option(section, 'address'): tivos[tsn] = config.get(section, 'address') for section in ['Server', '_tivo_SD', '_tivo_HD']: if not config.has_section(section): config.add_section(section)
def get_config(config_file=None, server=None): logger.info("Getting config from config file %s" % config_file) if config_file is None: config_file = '/home/minecraft/minecraft/pyredstone.cfg' if not os.path.exists(config_file): raise IOError("Could not open config file") config = ConfigParser.ConfigParser() config.read(config_file) if server is None: try: sections = config.sections() logger.debug(sections) if len(sections) < 1: raise SyntaxError("No sections found in config file") elif len(sections) > 1: logger.warning("More than one server found, no server specified. Using first server.") server = sections[0] except ConfigParser.Error as e: logger.exception("Could not get sections") if not config.has_section(server): raise SyntaxError("Server section '%s' of config file does not exist. Cannot continue." % (server, )) # Now we have a config file and a section. data = {} try: # Take each item in the config file section and dump into a dict for item in config.items(server): data[item[0]] = item[1] logger.info("Config data: %s" % str(data)) except ConfigParser.Error as e: raise SyntaxError("Config file is improperly formated") return data
def reset(): global bin_paths global config global configs_found global tivos_found bin_paths = {} config = ConfigParser.ConfigParser() configs_found = config.read(config_files) if not configs_found: print ('WARNING: pyTivo.conf does not exist.\n' + 'Assuming default values.') configs_found = config_files[-1:] for section in config.sections(): if section.startswith('_tivo_'): tsn = section[6:] if tsn.upper() not in ['SD', 'HD', '4K']: tivos_found = True tivos[tsn] = Bdict(config.items(section)) for section in ['Server', '_tivo_SD', '_tivo_HD', '_tivo_4K']: if not config.has_section(section): config.add_section(section)
def isenabled(sect, data={}): if not config.has_section(sect): return False if not config.getboolean(sect, 'enable'): return False try: per = config.get(sect, 'percentage') # allow '0' if per and int(per) < random.randint(1, 100): return False time_ = config.get(sect, 'time') if time_ and not util.time_in(time_): return False if 'source' in data: pattern = config.get(sect, 'source_pattern') data['source_match'] = re.search(pattern, data['source']) if not data['source_match']: return False if 'message' in data: pattern = config.get(sect, 'pattern') data['match'] = re.search(pattern, data['message']) if not data['match']: return False except: logging.exception('[%s] %s', sect, data) return False return True
def get_config(): """Read the configuration file. Returns the parameters in dictionary format""" section = "hbcal" parameters = { 'input calendar': SingleConfigurationParameter(CALENDAR_TYPES, 'civil'), 'dafbind': SingleConfigurationParameter(DAFBIND_TYPES, 'civil'), 'format': SingleConfigurationParameter(FORMAT_TYPES, 'normal'), 'output calendar': MultiConfigurationParameter(CALENDAR_TYPES, ['civil', 'hebrew'], (("julian", "gregorian", "civil"),)), 'sedrah': BinaryConfigurationParameter(), 'omer': BinaryConfigurationParameter(), 'molad': BinaryConfigurationParameter(), 'israel': BinaryConfigurationParameter(), } config = RawConfigParser() home = path.expanduser("~") filename = path.join(home, '.hbcal.config') if config.read(filename): for key, value in parameters.viewitems(): value.parameter_found(key, section, config) if config.has_section('loggers'): logging.config.fileConfig(filename, disable_existing_loggers=True) return parameters
def set_up_logger_syslog_handler(config): handlers = {} formatters = {} if config.has_section('Logging'): if config.has_option('Logging', 'loghost') and \ config.has_option('Logging', 'logport'): log_host = config.get('Logging', 'loghost') log_port = int(config.get('Logging', 'logport')) log_address = (log_host, log_port) formatters['syslog_formatter'] = { 'format': '%(asctime)s %(name)s: %(levelname)s %(message)s', 'datefmt': '%b %e %H:%M:%S', } socktype = socket.SOCK_DGRAM if config.has_option('Logging', 'logtcp'): if config.getboolean('Logging', 'logtcp'): socktype = socket.SOCK_STREAM else: socktype = socket.SOCK_DGRAM facility = logging.handlers.SysLogHandler.LOG_USER if config.has_option('Logging', 'logfacility'): try: facility = logging.handlers.SysLogHandler.facility_names[ config.get('Logging', 'logfacility')] except KeyError: raise Exception('Invalid "logfacility" value of "%s"' % config.get('Logging', 'logfacility')) handlers['syslog'] = { 'class': 'logging.handlers.SysLogHandler', 'formatter': 'syslog_formatter', 'address': log_address, 'facility': facility, 'socktype': socktype, } return handlers, formatters
def set_up_logger_file_handler(config): handlers = {} formatters = {} if config.has_section('Logging'): # create handler # add to logger # create formatter if config.has_option('Logging', 'logfile'): logfile = config.get('Logging', 'logfile') if config.has_option('Logging', 'loghistory'): loghistory = int(config.get('Logging', 'loghistory')) else: loghistory = 7 formatters['file_formatter'] = { 'format': "%(asctime)s - %(levelname)s - %(message)s", } handlers['file'] = { 'class': 'logging.handlers.TimedRotatingFileHandler', 'formatter': 'file_formatter', 'filename': logfile, 'when': 'd', 'interval': 1, 'backupCount': loghistory, } return handlers, formatters
def reset(): global bin_paths global config global configs_found bin_paths = {} config = ConfigParser.ConfigParser() configs_found = config.read(config_files) if not configs_found: print ("WARNING: pyTivo.conf does not exist.\n" + "Assuming default values.") configs_found = config_files[-1:] for section in config.sections(): if section.startswith("_tivo_"): tsn = section[6:] if tsn.upper() not in ["SD", "HD"]: if config.has_option(section, "name"): tivo_names[tsn] = config.get(section, "name") else: tivo_names[tsn] = tsn if config.has_option(section, "address"): tivos[tsn] = config.get(section, "address") for section in ["Server", "_tivo_SD", "_tivo_HD"]: if not config.has_section(section): config.add_section(section)
def get_mon_api_config(config): mon_api_config = {'is_enabled': False, 'url': '', 'project_name': '', 'username': '', 'password': False, 'use_keystone': True, 'keystone_url': '', 'dimensions': None} if config.has_option("Main", "dimensions"): # parse comma separated dimensions into a dimension list try: dim_list = [dim.split(':') for dim in config.get('Main', 'dimensions').split(',')] mon_api_config['dimensions'] = {key.strip(): value.strip() for key, value in dim_list} except ValueError: mon_api_config['dimensions'] = {} if config.has_section("Api"): options = {"url": config.get, "project_name": config.get, "username": config.get, "password": config.get, "use_keystone": config.getboolean, "keystone_url": config.get} for name, func in options.iteritems(): if config.has_option("Api", name): mon_api_config[name] = func("Api", name) return mon_api_config
def get_config(): config = configparser.ConfigParser() config.read('./conf/diagnose.cnf', encoding='utf-8') section_has = config.has_section('default') if not section_has: sys.exit("Error: The '[default]' not find") processing = config.get("default", "processing") host = config.get("default", "host") user = config.get("default", "user") password = config.get("default", "password") port = config.get("default", "port") database = config.get("default", "database") log_level = config.get("default", "log_level") type = config.get("default", "type") conf_dict = { 'processing': processing, 'user': user, 'host': host, 'password': password, 'port': port, 'database': database, 'log_level': log_level, 'type': type } return conf_dict
def init_cache(self, config): self.cache_model = None if self.no_cache or self.client_config['resume']: return if not config.has_section('cache'): return cache_config = dict(config.items('cache')) cache_table = cache_config.get('table') if not cache_table: return logger.info("Init cache table `%s`", cache_table) blob_type = cache_config.get('blob_type', 'BLOB') max_size = int(cache_config.get('max_size', 65530)) CacheTendersModel.gzip_data.field_type = blob_type self.cache_model = CacheTendersModel self.cache_model._meta.database = self.database self.cache_model._meta.table_name = cache_table self.cache_max_size = max_size self.cache_hit_count = 0 self.cache_miss_count = 0 try: self.cache_model.select().get() cache_table_exists = True except CacheTendersModel.DoesNotExist: cache_table_exists = True except peewee.DatabaseError: cache_table_exists = False self.database.rollback() if self.drop_cache: logger.warning("Drop cache table `%s`", cache_table) self.cache_model.drop_table(safe=True) cache_table_exists = False if not cache_table_exists: logger.info("Create cache table `%s`", cache_table) self.cache_model.create_table(safe=True)
def main(): home_dir = os.path.expanduser('~') program_dir = os.path.abspath(os.path.join(home_dir, '.dbackups')) db_config_file = os.path.join(program_dir, 'databases.ini') if not os.path.isdir(os.path.join(BASE_DIR, '../logs')): os.mkdir(os.path.join(BASE_DIR, '../logs')) if not os.path.isfile(db_config_file): print('Config File not found. {}'.format(db_config_file)) sys.exit(1) #logging_config = resource_filename(__name__, '../config/logging.ini') #logging.config.fileConfig(logging_config) logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-6s line %(lineno)-4s %(message)s') args = parser.parse_args() pprint(args) config = ConfigParser.ConfigParser() config.read(db_config_file) logging.debug(config.sections()) if not config.has_section(args.database): logging.info('DB alias not found in the config file {} -> [{}]'.format(db_config_file, args.database)) sys.exit(1) else: logging.info('Found the DB settings in the config file. Continuing.') db_type = config.get(args.database, 'db_type') db_host = config.get(args.database, 'db_host') db_user = config.get(args.database, 'db_user') db_pass = config.get(args.database, 'db_pass') db_port = config.get(args.database, 'db_port') db_name = config.get(args.database, 'db_name') db_object = get_database_object(db_type, db_host, db_name, db_user, db_pass, db_port) logging.debug('DB object created: {}'.format(db_object)) if args.command == 'backup': logging.info('Chose to backup {}'.format(db_object.db_host)) logging.info('Dump file: [{}]'.format(db_object.dump_file_name)) db_object.dump() logging.info('Dumping DB finished.') if args.upload_url: print('Uploading to the desired URL: {}'.format(args.upload_url)) upload_http_put(db_object.dump_file, args.upload_url) if args.command == 'clone': logging.info('Going to clone_to from one DB to another.') dev_db = get_database_object(db_type, args.dev_host, args.dev_name, args.dev_user, args.dev_pass, args.dev_port) db_object.clone_to(dev_db, args.latest_local) if args.command == 'clean': logging.info('Cleaning the dumps directory to make room for more dumps.') file_list = cleanup_backups.find_files_for_delete(db_object.get_dump_dir(), db_object.db_host, db_object.db_name) cleanup_backups.delete_file_list(file_list)
def dict2config(jdict, section): config = ConfigParser.ConfigParser() if not config.has_section(section): config.add_section(section) for k,v in jdict.iteritems(): config.set(section, k, str(v)) return config
def enabled(self, sect): if not config.has_section(sect): return False if not config.getboolean(sect, 'enable'): return False try: per = config.get(sect, 'percentage') # allow '0' if per and int(per) < random.randint(1, 100): return False time_ = config.get(sect, 'time') if time_ and not util.time_in(time_): return False if 'user_name' in self.data: pattern = config.get(sect, 'user_pattern') self.data['user_match'] = re.search(pattern, self.data['user_name']) if not self.data['user_match']: return False if 'text' in self.data: pattern = config.get(sect, 'pattern') self.data['match'] = re.search(pattern, self.data['text']) if not self.data['match']: return False except: logging.exception('[%s] %s', sect, self.data) return False return True
def write_sensitive_properties(self, pg_id, sensitive_file): config = configparser.RawConfigParser() config.optionxform = str # Preserve case config.read(sensitive_file) # This file shouldn't be checked in processors = self.get_processors_by_pg(pg_id) for processor in processors["processors"]: self.logger.debug("Processor: {}".format( processor["component"]["name"])) if config.has_section(processor["component"]["name"]): self.logger.debug("Found. Setting properties") properties = {} for name, value in config.items( processor["component"]["name"]): properties[name] = value self.logger.debug("{} = {}".format(name, value)) rtn = self.set_processor_properties(processor, properties) self.logger.debug( "set_processor_properties returned {}".format( json.dumps(rtn))) for key, value in processor['component']['config'][ 'properties'].items(): if value is None: continue self.logger.debug("Checking properties. {}/{}".format( key, value)) # If our value is a uuid then it's likely a controller service. Update those properties. if re.search( '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}', value): self.recurse_update_controller(value, config)
def dict2config(jdict, section): config = ConfigParser.ConfigParser() if not config.has_section(section): config.add_section(section) for k, v in jdict.iteritems(): config.set(section, k, str(v)) return config
def reset(): global bin_paths global config global configs_found global tivos_found bin_paths = {} config = ConfigParser.ConfigParser() configs_found = config.read(config_files) if not configs_found: print('WARNING: pyTivo.conf does not exist.\n' + 'Assuming default values.') configs_found = config_files[-1:] for section in config.sections(): if section.startswith('_tivo_'): tsn = section[6:] if tsn.upper() not in ['SD', 'HD', '4K']: tivos_found = True tivos[tsn] = Bdict(config.items(section)) for section in ['Server', '_tivo_SD', '_tivo_HD', '_tivo_4K']: if not config.has_section(section): config.add_section(section)
def action(self, now, name): sect = ':'.join(('schedule', name)) self.state.setdefault(name, { 'name': name, 'sect': sect, 'time': now, }) if not config.has_section(sect): logging.error('[schedule] [%s] invalid name', name) del self.state[name] return if not config.getboolean(sect, 'enable'): del self.state[name] return interval = config.getint(sect, 'interval') if interval < 60: logging.error('[schedule] [%s] interval too short', name) del self.state[name] return actsect = ':'.join(('action', config.get(sect, 'action'))) if not config.has_section(actsect): logging.error('[schedule] [%s] invalid action', name) del self.state[name] return time_ = config.get(name, 'time') if time_ and not util.time_in(time_): return if self.state[name]['time'] + interval > now: return obj = EventHandler({}) if not obj.enabled(actsect): return obj.action(actsect) if obj.body: post(obj.body) self.state[name]['time'] = now
def __init__(self, config_file, tvdb): config = configparser.ConfigParser() config.read(config_file) log_file = config.get('main', 'log.config', fallback=None) if log_file: import yaml with open(log_file) as f: log_config = yaml.load(f) logging.config.dictConfig(log_config) template = config.get( 'directories', 'out.pattern', fallback= '${series}/Season ${season}/${series} - S${season|zpad}E${episode|zpad} - ${episode_name}.${ext}' ) self.template = template self.wtv_in_dir = config.get('directories', 'tv.in') self.tv_pattern = config.get('directories', 'tv.pattern') self.com_in_dir = config.get('directories', 'commercial.in') self.srt_in_dir = config.get('directories', 'srt.in') self.temp_dir = config.get('directories', 'temp.dir') self.out_dir = config.get('directories', 'out.dir') self.delete_source = config.getboolean('directories', 'delete.source.files', fallback=True) self.convert_config = convert_config_from_config_section( config, 'transcode') if config.has_section('ffmpeg'): logger.error('You are using an outdated configuration') raise Exception('You are using an outdated configuration') self.debug = config.getboolean('main', 'debug', fallback=False) self.ccextractor_exe = config.get('ccextractor', 'executable', fallback=ccextractor()) self.ccextractor_run = config.getboolean('ccextractor', 'run.if.missing', fallback=False) self.comskip_exe = config.get('comskip', 'executable', fallback=None) self.comskip_run = config.getboolean('comskip', 'run.if.missing', fallback=comskip()) self.comskip_ini = config.get('comskip', 'comskip.ini', fallback=None) db_file = config.get('main', 'database.file', fallback='db.sqlite') self.wtvdb = WtvDb(db_file) self.tvdb = tvdb if self.convert_config.include_subtitles: logger.warning( 'Include Subtitles is True. This usually does not work with TV captions.' )
def get_tsn(name, tsn=None, raw=False): if tsn and config.has_section('_tivo_' + tsn): try: return config.get('_tivo_' + tsn, name, raw) except NoOptionError: pass section_name = get_section(tsn) if config.has_section(section_name): try: return config.get(section_name, name, raw) except NoOptionError: pass try: return config.get('Server', name, raw) except NoOptionError: pass return None
def session_login(self, login, password): # Initialize the database (kctl). db_init() # Validate parameters. login = validate_login(login) password = validate_password(password) # Load config config = CustomConfigParser() config.read(ini_conf_path) # Check login/pass pair. if not config.has_section(login): raise xmlrpclib.Fault(201, "Invalid login or password.") goodpass = config.get_default(login, "password", "").strip(" ") if goodpass == "": log.error("Invalid configuration 'password' for login '%s'." % ( login ) ) raise xmlrpclib.Fault(201, "Invalid login or password.") if goodpass != password: raise xmlrpclib.Fault(201, "Invalid login or password.") # Create session. session = kweb_session.ksession_get_session(database=sess_database) session.data["kpsapi"] = 1 session.data["start_stamp"] = int(time.time()) # Load security context. security_ctx = config.get_default(login, "security_ctx", "").strip(" ") if security_ctx not in valid_security_contexts: log.error("Invalid configuration 'security_ctx' for login '%s'." % ( login ) ) raise xmlrpclib.Fault(106, "Invalid KPS API configuration.") session.data["security_ctx"] = security_ctx if session.data["security_ctx"] == KAPI_SECURITY_CTX_ORG: str_org_id = config.get_default(login, "org_id", "").strip(" ") if not str_org_id.isdigit(): log.error("Invalid configuration 'org_id' for login '%s'." % ( login ) ) raise xmlrpclib.Fault(106, "Invalid KPS API configuration.") org_id = int(str_org_id) if org_id == 0: org_id = sdb_get_main_org_id() if not org_id: raise xmlrpclib.Fault(241, "No main organization yet.") session.data["org_id"] = org_id # Load keys. session.data["main_key_id"] = main_key_id() try: str_key_id = config.get(login, "key_id").strip(" ") if str_key_id == "": session.data["key_id"] = session.data["main_key_id"] elif str_key_id.isdigit(): session.data["key_id"] = int(str_key_id) else: log.error("Invalid configuration 'key_id' for login '%s'." % ( login ) ) raise xmlrpclib.Fault(106, "Invalid KPS API configuration.") except ConfigParser.NoOptionError, e: session.data["key_id"] = session.data["main_key_id"]
def get_mon_api_config(config): mon_api_config = { 'is_enabled': False, 'url': '', 'project_name': '', 'project_id': '', 'project_domain_name': '', 'project_domain_id': '', 'ca_file': '', 'insecure': '', 'username': '', 'password': '', 'use_keystone': True, 'keystone_url': '', 'dimensions': None, 'max_buffer_size': 1000, 'backlog_send_rate': 5 } if config.has_option("Main", "dimensions"): # parse comma separated dimensions into a dimension list try: dim_list = [ dim.split(':') for dim in config.get('Main', 'dimensions').split(',') ] mon_api_config['dimensions'] = { key.strip(): value.strip() for key, value in dim_list } except ValueError: mon_api_config['dimensions'] = {} if config.has_section("Api"): options = { "url": config.get, "project_name": config.get, "project_id": config.get, "project_domain_name": config.get, "project_domain_id": config.get, "ca_file": config.get, "insecure": config.get, "username": config.get, "password": config.get, "use_keystone": config.getboolean, "keystone_url": config.get, "max_buffer_size": config.getint, "backlog_send_rate": config.getint, "amplifier": config.getint } for name, func in options.iteritems(): if config.has_option("Api", name): mon_api_config[name] = func("Api", name) return mon_api_config
def getConfiguration(cfgfile=None, config_required={'Main': {'key1': 'value1', 'key2': 'value2'}}): ''' read an ini configuration file and return a dictionary of key/value pairs update configuration file if missing any sections accepts: cfgfile - path to configuration file config_required - nested dictionary in the following format: {'Section1': {'key1': 'value1', 'key2': 'value2'}, 'Section 2': {'key1': 'value1'} } ''' if not cfgfile: raise ValueError('no configuration file specified') # required configuraiton options # Section: {'option': 'default value'} logger = logging.getLogger(__name__) logger.debug('getting configuration from file: {}'.format(cfgfile)) cfgpath = os.path.dirname(cfgfile) # config_required = { # 'Main': {'credentials': os.path.join(cfgpath, 'credentials/'), # }, # } config = configuration.get_config(cfgfile) update_config = False logger.debug('checking sections') for section, values in list(config_required.items()): if not config.has_section(section): logger.warning('section: {} not found in {}'.format(section, cfgfile)) logger.debug('adding section {}'.format(section)) config.add_section(section) update_config = True for option, value in list(values.items()): if not config.has_option(section, option): logger.warning('option: {} not found in {}'.format(option, cfgfile)) logger.debug('adding option {}: {}'.format(option, value)) config.set(section, option, value) update_config = True # for section, options in config_required.items(): if update_config: try: logger.debug('updating configuration file at: {}'.format(cfgfile)) configuration.create_config(cfgfile, config) except Exception as e: logger.error(e) return(config)
def from_config(self, config): if not config.has_section(self.name): return {} res = {} for opt in config.options(self.name): value = config.get(self.name, opt) if not value and opt in self.required: continue res[opt] = value return res
def session(self): """ Session property used for sqlalchemy """ if not config.has_section("Database"): return None if not hasattr(self, "_session") or self._session == None: from models.model import engine from sqlalchemy.orm import sessionmaker self._session = sessionmaker(bind=engine)() return self._session
def load_config(defaults, files): config = ConfigParser.ConfigParser() with open(defaults, 'r') as f: config.readfp(f) config.read(files) if 'ACCOUNTING_CLIENT_DIR' in os.environ: # hack to make it possible to find client dir in buildbot if config.has_section('accounting'): config.set('accounting', 'client_dir', os.environ['ACCOUNTING_CLIENT_DIR']) return config
def get_config_value(section, key, default=None): config = get_config_parser() if not config.has_section(section): if default: return default if default is None: return config.get(section=section, option=key) try: return config.get(section=section, option=key) except configparser.NoOptionError: return default except configparser.NoSectionError: return default
def recurse_update_controller(self, controller_id, config): self.logger.debug("Updating controller id {}".format(controller_id)) controller_service = self.get_controller_service(controller_id) if controller_service is None: self.logger.warning( "Could not find controller service with id: {}".format( controller_id)) return self.logger.debug("Updating {}".format( json.dumps(controller_service['component']['name']))) state = controller_service['component']['state'] if state == 'ENABLED': self.logger.debug("Disabling controller") controller_service = self.update_controller_status( controller_service, self.CONTROLLER_DISABLED) self.logger.debug("{}".format(json.dumps(controller_service))) controller_name = controller_service['component']['name'] config_section = controller_name if 'config_section' in controller_service['component']['properties']: config_section = controller_service['component']['properties'][ 'config_section'] self.logger.info("using section: {}".format(config_section)) if config.has_section(config_section): controller_obj = { "component": { "id": controller_service["component"]["id"], "properties": {} }, "revision": { "version": controller_service["revision"]["version"] } } for name, value in config.items(config_section): if not name.startswith("_"): self.logger.debug("Controller Setting {}={}".format( name, value)) controller_obj["component"]["properties"][name] = value self.logger.debug("controller properties {}".format( json.dumps(controller_obj))) controller_service = self.update_controller_service(controller_obj) if controller_service is not None: self.logger.debug("update controller returned: {}".format( json.dumps(controller_service))) else: self.logger.warning( "update controller returned None. Did it fail?") rtn = self.update_controller_status(controller_service, self.CONTROLLER_ENABLED)
def user_update(username): if not config: raise ValueError('Configuration not loaded') if not config.has_section('users'): config.add_section('users') if config.has_option('users', username): print 'Changing password for %s' % username else: print 'Adding new user %s' % username password = getpass.getpass('Password: '******' * Updating OFTG-Ninja config ...' return True else: print ' * Failed to update user' return False
def get169Setting(tsn): if not tsn: return True tsnsect = "_tivo_" + tsn if config.has_section(tsnsect): if config.has_option(tsnsect, "aspect169"): try: return config.getboolean(tsnsect, "aspect169") except ValueError: pass if get169Blacklist(tsn) or get169Letterbox(tsn): return False return True
def __init__(self, libjars_in_hdfs=None, input_format=None): libjars_in_hdfs = libjars_in_hdfs or [] config = configuration.get_config() streaming_jar = config.get('hadoop', 'streaming-jar', '/tmp/hadoop-streaming.jar') if config.has_section('job-conf'): job_confs = dict(config.items('job-conf')) else: job_confs = {} super(MapReduceJobRunner, self).__init__( streaming_jar, input_format=input_format, libjars_in_hdfs=libjars_in_hdfs, jobconfs=job_confs, )
def get169Setting(tsn): if not tsn: return True tsnsect = '_tivo_' + tsn if config.has_section(tsnsect): if config.has_option(tsnsect, 'aspect169'): try: return config.getboolean(tsnsect, 'aspect169') except ValueError: pass if get169Blacklist(tsn) or get169Letterbox(tsn): return False return True
def get_mon_api_config(config): mon_api_config = {'is_enabled': False, 'url': '', 'project_name': '', 'project_id': '', 'project_domain_name': '', 'project_domain_id': '', 'ca_file': '', 'insecure': '', 'username': '', 'password': '', 'use_keystone': True, 'keystone_url': '', 'dimensions': None, 'max_buffer_size': 1000, 'backlog_send_rate': 5} if config.has_option("Main", "dimensions"): # parse comma separated dimensions into a dimension list try: dim_list = [dim.split(':') for dim in config.get('Main', 'dimensions').split(',')] mon_api_config['dimensions'] = {key.strip(): value.strip() for key, value in dim_list} except ValueError: mon_api_config['dimensions'] = {} if config.has_section("Api"): options = {"url": config.get, "project_name": config.get, "project_id": config.get, "project_domain_name": config.get, "project_domain_id": config.get, "ca_file": config.get, "insecure": config.get, "username": config.get, "password": config.get, "use_keystone": config.getboolean, "keystone_url": config.get, "max_buffer_size": config.getint, "backlog_send_rate": config.getint, "amplifier": config.getint} for name, func in options.iteritems(): if config.has_option("Api", name): mon_api_config[name] = func("Api", name) return mon_api_config
def __getitem__(self, key): config = self._config_parser if config and config.has_section('options') and config.has_option('options', key): default = self._config_parser.get('options', key) else: default = self._options.get(key, None) # check command line options (1) if self._opts.get(key): return self._opts[key] # check environment (2) if self._prefix: return os.environ.get(self._prefix + '_' + key.upper(), default) return default
def create_models(self, config): self.models = dict() self.main_model = None for section in config.sections(): if section.startswith('table:'): table_schema = config.items(section) table, name = section.split(':', 2) index_section = 'index:' + name index_schema = [] if config.has_section(index_section): index_schema = config.items(index_section) self.init_model(name, table_schema, index_schema) self.sorted_models = sorted(self.models.values(), key=lambda m: len(m._meta.table_options.get('__iter__', []))) if self.client_config['resume'] and not self.main_model: raise ValueError('Main model is required for resume mode')
def getConfiguration(cfgfile): # required configuraiton options # Section: {'option': 'default value'} logger = logging.getLogger(__name__) logger.debug('getting configuration from file: {}'.format(cfgfile)) cfgpath = os.path.dirname(cfgfile) config_required = { 'Main': { 'credentials': os.path.join(cfgpath, 'credentials/'), }, } config = configuration.get_config(cfgfile) update_config = False logger.debug('checking sections') for section, values in list(config_required.items()): if not config.has_section(section): logger.warning('section: {} not found in {}'.format( section, cfgfile)) logger.debug('adding section {}'.format(section)) config.add_section(section) update_config = True for option, value in list(values.items()): if not config.has_option(section, option): logger.warning('option: {} not found in {}'.format( option, cfgfile)) logger.debug('adding option {}: {}'.format(option, value)) config.set(section, option, value) update_config = True # for section, options in config_required.items(): if update_config: try: logger.debug('updating configuration file at: {}'.format(cfgfile)) configuration.create_config(cfgfile, config) except Exception as e: logger.error(e) return (config)
def __init__(self, config_file): config = configparser.ConfigParser() config.read(config_file) # Directories self.movie_in_dir = config.get('directories', 'movie.dir.in') self.tv_in_dir = config.get('directories', 'tv.dir.in') self.working_dir = config.get('directories', 'working.dir') self.movie_out_dir = config.get('directories', 'movie.dir.out') self.tv_out_dir = config.get('directories', 'tv.dir.out') # Backup self.backup_enabled = config.getboolean('backup', 'enabled', fallback=True) self.rclone_exe = config.get('backup', 'rclone') self.rclone_args = shlex.split( config.get('backup', 'rclone.args', fallback='')) self.split_exe = config.get('backup', 'split') self.backup_path = config.get('backup', 'backup.path') self.max_size = int(config.get('backup', 'max.size')) * (1024**3) self.split_size = config.get('backup', 'split.size') self.movie_convert_config = convert_config_from_config_section( config, 'movie.transcode') self.tv_convert_config = convert_config_from_config_section( config, 'tv.transcode') if config.has_section('transcode'): raise Exception( 'Config file is out dated. Please update to use movie.transcode and tv.transcode' ) # Logging file = config.get('logging', 'config', fallback=None) if file: import yaml with open(file) as f: log_config = yaml.load(f) logging.config.dictConfig(log_config) db_file = config.get('logging', 'db', fallback='processed.shelve') self.db = ProcessedDatabase(db_file)
def _setup(self): """ Sets up the worker task. Setup logging, Pylons globals and so on """ global log config = ConfigParser.ConfigParser() config.read(self.inifile) if not config.has_section('loggers'): # Sorry, the logger has not been set up yet print('Config file does not have [loggers] section') sys.exit(1) logging.config.fileConfig(self.inifile) logger_name = 'debexpo.worker' log = logging.getLogger(logger_name) sys.path.append(os.path.dirname(self.inifile)) conf = appconfig('config:' + self.inifile) pylons.config = load_environment(conf.global_conf, conf.local_conf)
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/sd-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/usr/local/etc/sd-agent/checks.d/' # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative( config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # # Core config # # FIXME unnecessarily complex if config.has_option('Main', 'sd_account'): agentConfig['sd_account'] = config.get('Main', 'sd_account') agentConfig['use_forwarder'] = False if options is not None and options.use_forwarder: listen_port = 17124 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['sd_url'] = "http://" + agentConfig[ 'bind_host'] + ":" + str(listen_port) agentConfig['use_forwarder'] = True elif options is not None and not options.disable_sd and options.sd_url: agentConfig['sd_url'] = options.sd_url elif config.has_option('Main', 'sd_url'): agentConfig['sd_url'] = config.get('Main', 'sd_url') else: # Default agent URL agentConfig['sd_url'] = "https://" + agentConfig[ 'sd_account'] + ".agent.serverdensity.io" if agentConfig['sd_url'].endswith('/'): agentConfig['sd_url'] = agentConfig['sd_url'][:-1] # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'ServerDensity', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get( 'Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Service discovery if config.has_option('Main', 'service_discovery_backend'): try: additional_config = extract_agent_config(config) agentConfig.update(additional_config) except: log.error('Failed to load the agent configuration related to ' 'service discovery. It will not be used.') # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True # Which agent key to use agentConfig['agent_key'] = config.get('Main', 'agent_key') # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates( config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles( config.get('Main', 'histogram_percentiles')) # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')): agentConfig['dogstatsd_target'] = agentConfig['sd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative( config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative( config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError, e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2)
def get_logging_config(cfg_path=None): system_os = get_os() logging_config = { 'log_level': None, 'log_to_event_viewer': False, 'log_to_syslog': False, 'syslog_host': None, 'syslog_port': None, } if system_os == 'windows': logging_config['windows_collector_log_file'] = os.path.join( _windows_commondata_path(), 'ServerDensity', 'logs', 'collector.log') logging_config['windows_forwarder_log_file'] = os.path.join( _windows_commondata_path(), 'ServerDensity', 'logs', 'forwarder.log') logging_config['windows_dogstatsd_log_file'] = os.path.join( _windows_commondata_path(), 'ServerDensity', 'logs', 'dogstatsd.log') logging_config['jmxfetch_log_file'] = os.path.join( _windows_commondata_path(), 'ServerDensity', 'logs', 'jmxfetch.log') else: logging_config[ 'collector_log_file'] = '/var/log/sd-agent/collector.log' logging_config[ 'forwarder_log_file'] = '/var/log/sd-agent/forwarder.log' logging_config[ 'dogstatsd_log_file'] = '/var/log/sd-agent/dogstatsd.log' logging_config['jmxfetch_log_file'] = '/var/log/sd-agent/jmxfetch.log' logging_config['go-metro_log_file'] = '/var/log/sd-agent/go-metro.log' logging_config['log_to_syslog'] = True config_path = get_config_path(cfg_path, os_name=system_os) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if config.has_section('handlers') or config.has_section( 'loggers') or config.has_section('formatters'): if system_os == 'windows': config_example_file = "https://github.com/serverdensity/sd-agent/blob/master/packaging/sd-agent/win32/install_files/config_win32.conf" else: config_example_file = "https://github.com/serverdensity/sd-agent/blob/master/config.cfg.example" sys.stderr.write( """Python logging config is no longer supported and will be ignored. To configure logging, update the logging portion of 'config.cfg' to match: '%s'. """ % config_example_file) for option in logging_config: if config.has_option('Main', option): logging_config[option] = config.get('Main', option) levels = { 'CRITICAL': logging.CRITICAL, 'DEBUG': logging.DEBUG, 'ERROR': logging.ERROR, 'FATAL': logging.FATAL, 'INFO': logging.INFO, 'WARN': logging.WARN, 'WARNING': logging.WARNING, } if config.has_option('Main', 'log_level'): logging_config['log_level'] = levels.get( config.get('Main', 'log_level')) if config.has_option('Main', 'log_to_syslog'): logging_config['log_to_syslog'] = config.get( 'Main', 'log_to_syslog').strip().lower() in ['yes', 'true', 1] if config.has_option('Main', 'log_to_event_viewer'): logging_config['log_to_event_viewer'] = config.get( 'Main', 'log_to_event_viewer').strip().lower() in ['yes', 'true', 1] if config.has_option('Main', 'syslog_host'): host = config.get('Main', 'syslog_host').strip() if host: logging_config['syslog_host'] = host else: logging_config['syslog_host'] = None if config.has_option('Main', 'syslog_port'): port = config.get('Main', 'syslog_port').strip() try: logging_config['syslog_port'] = int(port) except Exception: logging_config['syslog_port'] = None if config.has_option('Main', 'disable_file_logging'): logging_config['disable_file_logging'] = config.get( 'Main', 'disable_file_logging').strip().lower() in ['yes', 'true', 1] else: logging_config['disable_file_logging'] = False return logging_config
def main(): global Topics, TIME_SYNC_UPDATE_MSEC_TOPIC, TIME_SYNC_UPDATE_TOPIC, DontPublish parser = argparse.ArgumentParser( description='Respond to Time Sync request messages with current time.') parser.add_argument( "-t", "--topic", dest="topic", action="append", help= "MQTT topic to which to subscribe. May be specified multiple times.") parser.add_argument("-o", "--host", dest="MqttHost", action="store", help="MQTT host", default=None) parser.add_argument("-p", "--port", dest="MqttPort", action="store", help="MQTT host port", type=int, default=None) parser.add_argument("-P", "--DontPublish", dest="dontpub", action="store_true", help="Do not actually publish time syncs.", default=False) args = parser.parse_args() config = configparser.ConfigParser( interpolation=configparser.ExtendedInterpolation()) configFile = GetConfigFilePath() config.read(configFile) cfgSection = os.path.basename(sys.argv[0]) + "/" + os.environ['HOST'] logger.info("INI file cofig section is: %s", cfgSection) if not config.has_section(cfgSection): logger.critical('Config file "%s", has no section "%s".', configFile, cfgSection) sys.exit(2) if set(config.options(cfgSection)) < RequiredConfigParams: logger.critical( 'Config section "%s" does not have all required params: "%s", it has params: "%s".', cfgSection, RequiredConfigParams, set(config.options(cfgSection))) sys.exit(3) logger.info("INI file cofig section is: %s", cfgSection) cfg = config[cfgSection] mqtt_host = cfg['mqtt_host'] mqtt_port = cfg['mqtt_port'] TIME_SYNC_UPDATE_MSEC_TOPIC = cfg['mqtt_sync_ms_topic'] TIME_SYNC_UPDATE_TOPIC = cfg['mqtt_sync_topic'] if config.has_option(cfgSection, 'mqtt_request_topics'): t = cfg['mqtt_request_topics'] logger.debug('Topics from config is: "%s".', t) t = t.split() logger.debug('Split topics from config is: "%s".', t) Topics = t logger.debug('Topics from config is: "%s".', Topics) if (args.topic != None) and (len(args.topic) > 0): Topics = args.topic if (args.MqttHost != None) and (len(args.MqttHost) > 0): mqtt_host = args.MqttHost if (args.MqttPort != None) and (len(args.MqttPort) > 0): mqtt_port = args.MqttPort #DontPublish if (args.dontpub != None): DontPublish = args.dontpub mqtt_port = int(mqtt_port) if (mqtt_host is None) or (mqtt_port is None) or (len(Topics) == 0): logger.critical( 'No mqtt_host OR no mqtt_port OR no topics; must quit.') sys.exit(1) try: logger.debug( 'Connecting to MQTT server at "%s", on port "%s", for topics "%s".', mqtt_host, mqtt_port, Topics) RecClient.connect(mqtt_host, mqtt_port, 60) logger.debug('Mqtt connect returned.') logger.debug("Loop immediately after connect returns code: %s", mqtt.error_string(RecClient.loop())) if (RecClient._state != mqtt.mqtt_cs_connected): logger.debug("Not immediately connected.") except Exception as e: logger.exception(e) sys.exit(2) try: nowTime = time.time() localtime = time.localtime(nowTime) timeUs = (nowTime - int(nowTime)) * 1000000 logger.debug("Begin receive loop at: %s.%06d %s" % (time.strftime("%Y-%m-%d %H:%M:%S", localtime), timeUs, time.strftime("%Z", localtime))) RecClient.loop_forever() finally: logger.debug('Executing finally clause.') RecClient.disconnect() pass
def get_config(parse_args=True, cfg_path=None, options=None): if parse_args: options, _ = get_parsed_args() # General config agentConfig = { 'check_freq': DEFAULT_CHECK_FREQUENCY, 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://localhost:17123', 'graphite_listen_port': None, 'hostname': None, 'listen_port': None, 'tags': None, 'use_ec2_instance_id': False, # DEPRECATED 'version': get_version(), 'watchdog': True, 'additional_checksd': '/etc/dd-agent/checks.d/', 'bind_host': get_default_bind_host(), 'statsd_metric_namespace': None, 'utf8_decoding': False } if Platform.is_mac(): agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d' # Config handling try: # Find the right config file path = os.path.realpath(__file__) path = os.path.dirname(path) config_path = get_config_path(cfg_path, os_name=get_os()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) # bulk import for option in config.options('Main'): agentConfig[option] = config.get('Main', option) # Store developer mode setting in the agentConfig if config.has_option('Main', 'developer_mode'): agentConfig['developer_mode'] = _is_affirmative( config.get('Main', 'developer_mode')) # Allow an override with the --profile option if options is not None and options.profile: agentConfig['developer_mode'] = True # # Core config #ap if not config.has_option('Main', 'api_key'): log.warning(u"No API key was found. Aborting.") sys.exit(2) if not config.has_option('Main', 'dd_url'): log.warning(u"No dd_url was found. Aborting.") sys.exit(2) # Endpoints dd_url = clean_dd_url(config.get('Main', 'dd_url')) api_key = config.get('Main', 'api_key').strip() # For collector and dogstatsd agentConfig['api_key'] = api_key agentConfig['dd_url'] = dd_url # multiple endpoints if config.has_option('Main', 'other_dd_urls'): other_dd_urls = map(clean_dd_url, config.get('Main', 'other_dd_urls').split(',')) else: other_dd_urls = [] if config.has_option('Main', 'other_api_keys'): other_api_keys = map( lambda x: x.strip(), config.get('Main', 'other_api_keys').split(',')) else: other_api_keys = [] # Forwarder endpoints logic # endpoints is: # { # 'https://app.datadoghq.com': ['api_key_abc', 'api_key_def'], # 'https://app.example.com': ['api_key_xyz'] # } endpoints = {dd_url: [api_key]} if len(other_dd_urls) == 0: endpoints[dd_url] += other_api_keys else: assert len(other_dd_urls) == len( other_api_keys), 'Please provide one api_key for each url' for i, other_dd_url in enumerate(other_dd_urls): endpoints[other_dd_url] = endpoints.get( other_dd_url, []) + [other_api_keys[i]] agentConfig['endpoints'] = endpoints # Forwarder or not forwarder agentConfig[ 'use_forwarder'] = options is not None and options.use_forwarder if agentConfig['use_forwarder']: listen_port = 17123 if config.has_option('Main', 'listen_port'): listen_port = int(config.get('Main', 'listen_port')) agentConfig['dd_url'] = "http://{}:{}".format( agentConfig['bind_host'], listen_port) # FIXME: Legacy dd_url command line switch elif options is not None and options.dd_url is not None: agentConfig['dd_url'] = options.dd_url # Forwarder timeout agentConfig['forwarder_timeout'] = 20 if config.has_option('Main', 'forwarder_timeout'): agentConfig['forwarder_timeout'] = int( config.get('Main', 'forwarder_timeout')) # Extra checks.d path # the linux directory is set by default if config.has_option('Main', 'additional_checksd'): agentConfig['additional_checksd'] = config.get( 'Main', 'additional_checksd') elif get_os() == 'windows': # default windows location common_path = _windows_commondata_path() agentConfig['additional_checksd'] = os.path.join( common_path, 'Datadog', 'checks.d') if config.has_option('Main', 'use_dogstatsd'): agentConfig['use_dogstatsd'] = config.get( 'Main', 'use_dogstatsd').lower() in ("yes", "true") else: agentConfig['use_dogstatsd'] = True # Service discovery if config.has_option('Main', 'service_discovery_backend'): try: additional_config = extract_agent_config(config) agentConfig.update(additional_config) except: log.error('Failed to load the agent configuration related to ' 'service discovery. It will not be used.') # Concerns only Windows if config.has_option('Main', 'use_web_info_page'): agentConfig['use_web_info_page'] = config.get( 'Main', 'use_web_info_page').lower() in ("yes", "true") else: agentConfig['use_web_info_page'] = True # local traffic only? Default to no agentConfig['non_local_traffic'] = False if config.has_option('Main', 'non_local_traffic'): agentConfig['non_local_traffic'] = config.get( 'Main', 'non_local_traffic').lower() in ("yes", "true") # DEPRECATED if config.has_option('Main', 'use_ec2_instance_id'): use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id') # translate yes into True, the rest into False agentConfig['use_ec2_instance_id'] = ( use_ec2_instance_id.lower() == 'yes') if config.has_option('Main', 'check_freq'): try: agentConfig['check_freq'] = int( config.get('Main', 'check_freq')) except Exception: pass # Custom histogram aggregate/percentile metrics if config.has_option('Main', 'histogram_aggregates'): agentConfig['histogram_aggregates'] = get_histogram_aggregates( config.get('Main', 'histogram_aggregates')) if config.has_option('Main', 'histogram_percentiles'): agentConfig['histogram_percentiles'] = get_histogram_percentiles( config.get('Main', 'histogram_percentiles')) # Disable Watchdog (optionally) if config.has_option('Main', 'watchdog'): if config.get('Main', 'watchdog').lower() in ('no', 'false'): agentConfig['watchdog'] = False # Optional graphite listener if config.has_option('Main', 'graphite_listen_port'): agentConfig['graphite_listen_port'] = \ int(config.get('Main', 'graphite_listen_port')) else: agentConfig['graphite_listen_port'] = None # Dogstatsd config dogstatsd_defaults = { 'dogstatsd_port': 8125, 'dogstatsd_target': 'http://' + agentConfig['bind_host'] + ':17123', } for key, value in dogstatsd_defaults.iteritems(): if config.has_option('Main', key): agentConfig[key] = config.get('Main', key) else: agentConfig[key] = value # Create app:xxx tags based on monitored apps agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \ _is_affirmative(config.get('Main', 'create_dd_check_tags')) # Forwarding to external statsd server if config.has_option('Main', 'statsd_forward_host'): agentConfig['statsd_forward_host'] = config.get( 'Main', 'statsd_forward_host') if config.has_option('Main', 'statsd_forward_port'): agentConfig['statsd_forward_port'] = int( config.get('Main', 'statsd_forward_port')) # optionally send dogstatsd data directly to the agent. if config.has_option('Main', 'dogstatsd_use_ddurl'): if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')): agentConfig['dogstatsd_target'] = agentConfig['dd_url'] # Optional config # FIXME not the prettiest code ever... if config.has_option('Main', 'use_mount'): agentConfig['use_mount'] = _is_affirmative( config.get('Main', 'use_mount')) if options is not None and options.autorestart: agentConfig['autorestart'] = True elif config.has_option('Main', 'autorestart'): agentConfig['autorestart'] = _is_affirmative( config.get('Main', 'autorestart')) if config.has_option('Main', 'check_timings'): agentConfig['check_timings'] = _is_affirmative( config.get('Main', 'check_timings')) if config.has_option('Main', 'exclude_process_args'): agentConfig['exclude_process_args'] = _is_affirmative( config.get('Main', 'exclude_process_args')) try: filter_device_re = config.get('Main', 'device_blacklist_re') agentConfig['device_blacklist_re'] = re.compile(filter_device_re) except ConfigParser.NoOptionError: pass if config.has_option('datadog', 'ddforwarder_log'): agentConfig['has_datadog'] = True # Dogstream config if config.has_option("Main", "dogstream_log"): # Older version, single log support log_path = config.get("Main", "dogstream_log") if config.has_option("Main", "dogstream_line_parser"): agentConfig["dogstreams"] = ':'.join( [log_path, config.get("Main", "dogstream_line_parser")]) else: agentConfig["dogstreams"] = log_path elif config.has_option("Main", "dogstreams"): agentConfig["dogstreams"] = config.get("Main", "dogstreams") if config.has_option("Main", "nagios_perf_cfg"): agentConfig["nagios_perf_cfg"] = config.get( "Main", "nagios_perf_cfg") if config.has_option("Main", "use_curl_http_client"): agentConfig["use_curl_http_client"] = _is_affirmative( config.get("Main", "use_curl_http_client")) else: # Default to False as there are some issues with the curl client and ELB agentConfig["use_curl_http_client"] = False if config.has_section('WMI'): agentConfig['WMI'] = {} for key, value in config.items('WMI'): agentConfig['WMI'][key] = value if (config.has_option("Main", "limit_memory_consumption") and config.get("Main", "limit_memory_consumption") is not None): agentConfig["limit_memory_consumption"] = int( config.get("Main", "limit_memory_consumption")) else: agentConfig["limit_memory_consumption"] = None if config.has_option("Main", "skip_ssl_validation"): agentConfig["skip_ssl_validation"] = _is_affirmative( config.get("Main", "skip_ssl_validation")) agentConfig["collect_instance_metadata"] = True if config.has_option("Main", "collect_instance_metadata"): agentConfig["collect_instance_metadata"] = _is_affirmative( config.get("Main", "collect_instance_metadata")) agentConfig["proxy_forbid_method_switch"] = False if config.has_option("Main", "proxy_forbid_method_switch"): agentConfig["proxy_forbid_method_switch"] = _is_affirmative( config.get("Main", "proxy_forbid_method_switch")) agentConfig["collect_ec2_tags"] = False if config.has_option("Main", "collect_ec2_tags"): agentConfig["collect_ec2_tags"] = _is_affirmative( config.get("Main", "collect_ec2_tags")) agentConfig["utf8_decoding"] = False if config.has_option("Main", "utf8_decoding"): agentConfig["utf8_decoding"] = _is_affirmative( config.get("Main", "utf8_decoding")) agentConfig["gce_updated_hostname"] = False if config.has_option("Main", "gce_updated_hostname"): agentConfig["gce_updated_hostname"] = _is_affirmative( config.get("Main", "gce_updated_hostname")) except ConfigParser.NoSectionError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.ParsingError as e: sys.stderr.write('Config file not found or incorrectly formatted.\n') sys.exit(2) except ConfigParser.NoOptionError as e: sys.stderr.write( 'There are some items missing from your config file, but nothing fatal [%s]' % e) # Storing proxy settings in the agentConfig agentConfig['proxy_settings'] = get_proxy(agentConfig) if agentConfig.get('ca_certs', None) is None: agentConfig['ssl_certificate'] = get_ssl_certificate( get_os(), 'datadog-cert.pem') else: agentConfig['ssl_certificate'] = agentConfig['ca_certs'] return agentConfig