def set_up_logger_syslog_handler(config): handlers = {} formatters = {} if config.has_section('Logging'): if config.has_option('Logging', 'loghost') and \ config.has_option('Logging', 'logport'): log_host = config.get('Logging', 'loghost') log_port = int(config.get('Logging', 'logport')) log_address = (log_host, log_port) formatters['syslog_formatter'] = { 'format': '%(asctime)s %(name)s: %(levelname)s %(message)s', 'datefmt': '%b %e %H:%M:%S', } socktype = socket.SOCK_DGRAM if config.has_option('Logging', 'logtcp'): if config.getboolean('Logging', 'logtcp'): socktype = socket.SOCK_STREAM else: socktype = socket.SOCK_DGRAM facility = logging.handlers.SysLogHandler.LOG_USER if config.has_option('Logging', 'logfacility'): try: facility = logging.handlers.SysLogHandler.facility_names[ config.get('Logging', 'logfacility')] except KeyError: raise Exception('Invalid "logfacility" value of "%s"' % config.get('Logging', 'logfacility')) handlers['syslog'] = { 'class': 'logging.handlers.SysLogHandler', 'formatter': 'syslog_formatter', 'address': log_address, 'facility': facility, 'socktype': socktype, } return handlers, formatters
def reset(): global bin_paths global config global configs_found bin_paths = {} config = ConfigParser.ConfigParser() configs_found = config.read(config_files) if not configs_found: print ("WARNING: pyTivo.conf does not exist.\n" + "Assuming default values.") configs_found = config_files[-1:] for section in config.sections(): if section.startswith("_tivo_"): tsn = section[6:] if tsn.upper() not in ["SD", "HD"]: if config.has_option(section, "name"): tivo_names[tsn] = config.get(section, "name") else: tivo_names[tsn] = tsn if config.has_option(section, "address"): tivos[tsn] = config.get(section, "address") for section in ["Server", "_tivo_SD", "_tivo_HD"]: if not config.has_section(section): config.add_section(section)
def perform_prestart_checks(expected_hostname): # Check if current hostname is equal to expected one (got from the server # during bootstrap. global config if expected_hostname is not None: current_hostname = hostname.hostname(config) if current_hostname != expected_hostname: print("Determined hostname does not match expected. Please check agent " "log for details") msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \ "server hostname ({1}). Aborting registration. Please check hostname, " \ "hostname -f and /etc/hosts file to confirm your " \ "hostname is setup correctly".format(current_hostname, expected_hostname) logger.error(msg) sys.exit(1) # Check if there is another instance running if os.path.isfile(ProcessHelper.pidfile) and not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: print("%s already exists, exiting" % ProcessHelper.pidfile) sys.exit(1) # check if ambari prefix exists elif config.has_option('agent', 'prefix') and not os.path.isdir(os.path.abspath(config.get('agent', 'prefix'))): msg = "Ambari prefix dir %s does not exists, can't continue" \ % config.get("agent", "prefix") logger.error(msg) print(msg) sys.exit(1) elif not config.has_option('agent', 'prefix'): msg = "Ambari prefix dir %s not configured, can't continue" logger.error(msg) print(msg) sys.exit(1)
def get_mon_api_config(config): mon_api_config = {'is_enabled': False, 'url': '', 'project_name': '', 'username': '', 'password': False, 'use_keystone': True, 'keystone_url': '', 'dimensions': None} if config.has_option("Main", "dimensions"): # parse comma separated dimensions into a dimension list try: dim_list = [dim.split(':') for dim in config.get('Main', 'dimensions').split(',')] mon_api_config['dimensions'] = {key.strip(): value.strip() for key, value in dim_list} except ValueError: mon_api_config['dimensions'] = {} if config.has_section("Api"): options = {"url": config.get, "project_name": config.get, "username": config.get, "password": config.get, "use_keystone": config.getboolean, "keystone_url": config.get} for name, func in options.iteritems(): if config.has_option("Api", name): mon_api_config[name] = func("Api", name) return mon_api_config
def set_up_logger_file_handler(config): handlers = {} formatters = {} if config.has_section('Logging'): # create handler # add to logger # create formatter if config.has_option('Logging', 'logfile'): logfile = config.get('Logging', 'logfile') if config.has_option('Logging', 'loghistory'): loghistory = int(config.get('Logging', 'loghistory')) else: loghistory = 7 formatters['file_formatter'] = { 'format': "%(asctime)s - %(levelname)s - %(message)s", } handlers['file'] = { 'class': 'logging.handlers.TimedRotatingFileHandler', 'formatter': 'file_formatter', 'filename': logfile, 'when': 'd', 'interval': 1, 'backupCount': loghistory, } return handlers, formatters
def reset(): global bin_paths global config global configs_found bin_paths = {} config = ConfigParser.ConfigParser() configs_found = config.read(config_files) if not configs_found: print ('WARNING: pyTivo.conf does not exist.\n' + 'Assuming default values.') configs_found = config_files[-1:] for section in config.sections(): if section.startswith('_tivo_'): tsn = section[6:] if tsn.upper() not in ['SD', 'HD']: if config.has_option(section, 'name'): tivo_names[tsn] = config.get(section, 'name') else: tivo_names[tsn] = tsn if config.has_option(section, 'address'): tivos[tsn] = config.get(section, 'address') for section in ['Server', '_tivo_SD', '_tivo_HD']: if not config.has_section(section): config.add_section(section)
def connect(config, echo=False): """Connect to a given Clusto datastore. Accepts a config object with (at least) a DSN string e.g. mysql://user:[email protected]/clustodb e.g. sqlite:///somefile.db @param config: the config object """ dsn = config.get('clusto', 'dsn') if dsn.startswith('http'): SESSION.clusto_api = True else: SESSION.configure(bind=create_engine(dsn, echo=echo, poolclass=SingletonThreadPool, pool_recycle=600 )) SESSION.clusto_version = None if config.has_option('clusto', 'versioning'): SESSION.clusto_versioning_enabled = config.getboolean('clusto', 'versioning') else: SESSION.clusto_versioning_enabled = False # Set the log level from config, default is WARNING if config.has_option('clusto', 'loglevel'): rootlog = logging.getLogger() level = logging.getLevelName(config.get('clusto', 'loglevel')) rootlog.setLevel(level) # Setup audit logging to a file if config.has_option('clusto', 'auditlog'): auditlog = logging.getLogger('clusto.audit') auditlog.propagate = False auditlog.setLevel(logging.INFO) logfile = config.get('clusto', 'auditlog') handler = logging.handlers.WatchedFileHandler(logfile) handler.setFormatter(logging.Formatter( '%(asctime)s %(name)s %(levelname)s %(message)s', '%Y-%m-%d %H:%M:%S' )) auditlog.addHandler(handler) try: memcache_servers = config.get('clusto', 'memcached').split(',') # Memcache should only be imported if we're actually using it, yes? import memcache logging.info('Memcache server list: %s' % config.get('clusto', 'memcached')) SESSION.memcache = memcache.Client(memcache_servers, debug=0) except: SESSION.memcache = None
def __init__(self, config, section): self.is_active = False self.name = section if config.has_option(section, 'nickmatch'): self.nickmatch = re.compile(config.get(section, 'nickmatch'), re.I) else: raise ubot.exceptions.ConfigError("No nickmatch specified for %s" % section) if config.has_option(section, 'priority'): self.priority = int(config.get(section, 'priority')) else: raise ubot.exceptions.ConfigError("No priority specified for %s" % section)
def setup_logging(config): if config.has_option('ircbot', 'log_config'): log_config = config.get('ircbot', 'log_config') fp = os.path.expanduser(log_config) if not os.path.exists(fp): raise Exception("Unable to read logging config file at %s" % fp) logging.config.fileConfig(fp) elif config.has_option('ircbot', 'log_file'): log_file = config.get('ircbot', 'log_file') logging.basicConfig(level=logging.DEBUG, filename=log_file) else: logging.basicConfig(level=logging.DEBUG)
def get_logging_config(cfg_path=None): logging_config = { 'log_level': None, 'collector_log_file': '/var/log/datadog/collector.log', 'forwarder_log_file': '/var/log/datadog/forwarder.log', 'dogstatsd_log_file': '/var/log/datadog/dogstatsd.log', 'pup_log_file': '/var/log/datadog/pup.log', 'log_to_syslog': True, 'syslog_host': None, 'syslog_port': None, } config_path = get_config_path(cfg_path, os_name=getOS()) config = ConfigParser.ConfigParser() config.readfp(skip_leading_wsp(open(config_path))) if config.has_section('handlers') or config.has_section('loggers') or config.has_section('formatters'): sys.stderr.write("Python logging config is no longer supported and will be ignored.\nTo configure logging, update the logging portion of 'datadog.conf' to match:\n 'https://github.com/DataDog/dd-agent/blob/master/datadog.conf.example'.\n") for option in logging_config: if config.has_option('Main', option): logging_config[option] = config.get('Main', option) levels = { 'CRITICAL': logging.CRITICAL, 'DEBUG': logging.DEBUG, 'ERROR': logging.ERROR, 'FATAL': logging.FATAL, 'INFO': logging.INFO, 'WARN': logging.WARN, 'WARNING': logging.WARNING, } if config.has_option('Main', 'log_level'): logging_config['log_level'] = levels.get(config.get('Main', 'log_level')) if config.has_option('Main', 'log_to_syslog'): logging_config['log_to_syslog'] = config.get('Main', 'log_to_syslog').strip().lower() in ['yes', 'true', 1] if config.has_option('Main', 'syslog_host'): host = config.get('Main', 'syslog_host').strip() if host: logging_config['syslog_host'] = host else: logging_config['syslog_host'] = None if config.has_option('Main', 'syslog_port'): port = config.get('Main', 'syslog_port').strip() try: logging_config['syslog_port'] = int(port) except: logging_config['syslog_port'] = None return logging_config
def persist_runner(self): config = self.config_parser logger.debug("initializing backend %s" % config.get('core', 'persist_backend')) persist_backend = getattr( persist, '%s_backend' % config.get('core', 'persist_backend')) persist_backend = persist_backend(config) spool = self.spool num_persist_workers = None if config.has_option('core', 'persist_workers'): num_persist_workers = abs(config.getint('core', 'persist_workers')) if not num_persist_workers: num_persist_workers = PERSIST_POOL_WORKERS self.persist_pool = pool = Pool(num_persist_workers) while 1: spool_file = None if self.stop_now: break pool.wait_available() try: spool_file = spool.get(5) except Empty: sleep(1) continue pool.spawn(self.persist_data, spool_file, persist_backend) pool.join()
def run(self): logger.debug("initializing spool") config = self.config_parser self.spool = spool = Spooler(config) num_gather_workers = None if config.has_option('core', 'gather_workers'): num_gather_workers = abs(config.getint('core', 'gather_workers')) if not num_gather_workers: num_gather_workers = GATHER_POOL_WORKERS self._gather_pool = pool = Pool(num_gather_workers) persist_queue = self.persist_queue self.collect_manager = CollectPlugins( base_class=Collect, config=config, init_args=(config, persist_queue, spool, pool), entry_points='arke_plugins', ) self.collect_manager.load(pool=self._gather_pool) try: self.persist_runner() except KeyboardInterrupt: pass self.shutdown()
def setup_logging(config=None): """Turn down dependent library log levels so they aren't noise.""" global CONFIGURED FORMAT = '%(asctime)s %(levelname)-8s [%(name)-15s] %(message)s' DATEFMT = '%Y-%m-%d %H:%M:%S' # set 3rd party library logging levels to sanity points loglevels = { "irc.client": logging.INFO, "gerrit.GerritWatcher": logging.INFO, "paramiko.transport": logging.INFO, "pyelasticsearch": logging.INFO, "requests.packages.urllib3.connectionpool": logging.WARN, "urllib3.connectionpool": logging.WARN } if config is not None and config.has_option('ircbot', 'log_config'): log_config = config.get('ircbot', 'log_config') fp = os.path.expanduser(log_config) if not os.path.exists(fp): raise Exception("Unable to read logging config file at %s" % fp) logging.config.fileConfig(fp) else: logging.basicConfig( level=logging.DEBUG, format=FORMAT, datefmt=DATEFMT ) for module in loglevels: log = logging.getLogger(module) log.setLevel(loglevels[module]) CONFIGURED = True
def register_schedule(self): channel = config.get('irc', 'channel') for schedule in config.getlist('irc', 'schedules'): sect = ':'.join(('schedule', schedule)) # do not evaluate isenabled() here. # if it does, the disabled action will never be scheduled. if not config.has_section(sect): logging.error('[schedule] [%s] no such schedule', sect) continue if not config.has_option(sect, 'action'): logging.error('[schedule] [%s] no action specified', sect) continue action = ':'.join(('action', config.get(sect, 'action'))) if not config.has_section(action): logging.error('[schedule] [%s] invalid action specified', sect) continue interval = config.getint(sect, 'interval') if interval < 60: logging.error('[schedule] [%s] interval too short', sect) continue self.reactor.register_schedule(interval, self.do_action, action, self.connection, None, {'target': channel}, sect) logging.info('[schedule] [%s] registered', sect)
def persist_runner(self): config = self.config_parser logger.debug("initializing backend %s" % config.get('core', 'persist_backend')) persist_backend = getattr(persist, '%s_backend' % config.get('core', 'persist_backend')) persist_backend = persist_backend(config) spool = self.spool num_persist_workers = None if config.has_option('core', 'persist_workers'): num_persist_workers = abs(config.getint('core', 'persist_workers')) if not num_persist_workers: num_persist_workers = PERSIST_POOL_WORKERS self.persist_pool = pool = Pool(num_persist_workers) while 1: spool_file = None if self.stop_now: break pool.wait_available() try: spool_file = spool.get(5) except Empty: sleep(1) continue pool.spawn(self.persist_data, spool_file, persist_backend) pool.join()
def init_cache(self, config): self.cache_model = None if self.no_cache: return if not config.has_option('cache', 'table'): return cache_table = config.get('cache', 'table') if not cache_table: return logger.info("Init cache table `%s`", cache_table) self.cache_model = CacheTendersModel self.cache_model._meta.database = self.database self.cache_model._meta.db_table = cache_table self.cache_max_size = 0xfff0 try: self.cache_model.select().count() cache_table_exists = True except: cache_table_exists = False self.database.rollback() if self.drop_cache and cache_table_exists: logger.warning("Drop cache table `%s`", cache_table) self.cache_model.drop_table() cache_table_exists = False if not cache_table_exists: logger.info("Create cache table `%s`", cache_table) self.cache_model.create_table()
def get_bin(fname): global bin_paths logger = logging.getLogger('pyTivo.config') if fname in bin_paths: return bin_paths[fname] if config.has_option('Server', fname): fpath = config.get('Server', fname) if os.path.exists(fpath) and os.path.isfile(fpath): bin_paths[fname] = fpath return fpath else: logger.error('Bad %s path: %s' % (fname, fpath)) if sys.platform == 'win32': fext = '.exe' else: fext = '' for path in ([os.path.join(os.path.dirname(__file__), 'bin')] + os.getenv('PATH').split(os.pathsep)): fpath = os.path.join(path, fname + fext) if os.path.exists(fpath) and os.path.isfile(fpath): bin_paths[fname] = fpath return fpath logger.warn('%s not found' % fname) return None
def getShares(tsn=''): shares = [(section, Bdict(config.items(section))) for section in config.sections() if not (section.startswith(special_section_prefixes) or section in special_section_names) ] tsnsect = '_tivo_' + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, 'shares').split(','): y = x.strip() if config.has_section(y): tsnshares.append((y, Bdict(config.items(y)))) shares = tsnshares shares.sort() if not config.getboolean('Server', 'nosettings', fallback=False): shares.append(('Settings', {'type': 'settings'})) if get_server('tivo_mak') and get_togo('path'): shares.append(('ToGo', {'type': 'togo'})) return shares
def getShares(tsn=''): shares = [(section, Bdict(config.items(section))) for section in config.sections() if not (section.startswith(('_tivo_', 'logger_', 'handler_', 'formatter_')) or section in ('Server', 'loggers', 'handlers', 'formatters') ) ] tsnsect = '_tivo_' + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, 'shares').split(','): y = x.strip() if config.has_section(y): tsnshares.append((y, Bdict(config.items(y)))) shares = tsnshares shares.sort() if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']: shares.append(('Settings', {'type': 'settings'})) if get_server('tivo_mak') and get_server('togo_path'): shares.append(('ToGo', {'type': 'togo'})) return shares
def run(self): logger.debug("initializing spool") config = self.config_parser self.spool = spool = Spooler(config) num_gather_workers = None if config.has_option('core', 'gather_workers'): num_gather_workers = abs(config.getint('core', 'gather_workers')) if not num_gather_workers: num_gather_workers = GATHER_POOL_WORKERS self._gather_pool = pool = Pool(num_gather_workers) persist_queue = self.persist_queue self.collect_manager = CollectPlugins(base_class=Collect, config=config, init_args=(config, persist_queue, spool, pool), entry_points='arke_plugins', ) self.collect_manager.load(pool=self._gather_pool) try: self.persist_runner() except KeyboardInterrupt: pass self.shutdown()
def extract_agent_config(config): # get merged into the real agentConfig agentConfig = {} backend = config.get('Main', 'service_discovery_backend') agentConfig['service_discovery'] = True conf_backend = None if config.has_option('Main', 'sd_config_backend'): conf_backend = config.get('Main', 'sd_config_backend') if backend not in SD_BACKENDS: log.error( "The backend {0} is not supported. Service discovery won't be enabled." .format(backend)) agentConfig['service_discovery'] = False if conf_backend is None: log.warning( 'No configuration backend provided for service discovery. Only auto config templates will be used.' ) elif conf_backend not in SD_CONFIG_BACKENDS: log.error( "The config backend {0} is not supported. Only auto config templates will be used." .format(conf_backend)) conf_backend = None agentConfig['sd_config_backend'] = conf_backend additional_config = extract_sd_config(config) agentConfig.update(additional_config) return agentConfig
def parse_channel_config_file(file_path): config = ConfigParser() with codecs.open(file_path, 'r', encoding='utf-8') as f: config.readfp(f) channel_list = [] # validate and parse the config file if not config.sections(): print >> sys.stderr, "no section in config file!" sys.exit(1) for section in config.sections(): channel_info = {} OPTIONS = (u'channel_name', u'channel_description', u'rss_url') for option in OPTIONS: if not config.has_option(section, option): print >> sys.stderr, "no option [%s] in section [%s]!" % ( option, section) sys.exit(1) channel_info[option] = config.get(section, option) channel_list.append(channel_info) return channel_list
def main(loop: Optional[asyncio.AbstractEventLoop] = None) -> int: parser = argparse.ArgumentParser( description="PyPI PEP 381 mirroring client.") parser.add_argument("--version", action="version", version=f"%(prog)s {bandersnatch.__version__}") parser.add_argument( "-c", "--config", default="/etc/bandersnatch.conf", help="use configuration file (default: %(default)s)", ) parser.add_argument( "--debug", action="store_true", default=False, help="Turn on extra logging (DEBUG level)", ) subparsers = parser.add_subparsers() _delete_parser(subparsers) _mirror_parser(subparsers) _verify_parser(subparsers) if len(sys.argv) < 2: parser.print_help() parser.exit() args = parser.parse_args() bandersnatch.log.setup_logging(args) # Prepare default config file if needed. config_path = Path(args.config) if not config_path.exists(): logger.warning( f"Config file '{args.config}' missing, creating default config.") logger.warning( "Please review the config file, then run 'bandersnatch' again.") default_config_path = Path(__file__).parent / "default.conf" try: shutil.copy(default_config_path, args.config) except IOError as e: logger.error(f"Could not create config file: {e}") return 1 config = bandersnatch.configuration.BandersnatchConfig( config_file=args.config).config if config.has_option("mirror", "log-config"): logging.config.fileConfig(str(Path(config.get("mirror", "log-config")))) # TODO: Go to asyncio.run() when >= 3.7 loop = loop or asyncio.get_event_loop() try: return loop.run_until_complete(async_main(args, config)) finally: loop.close()
def init_cache(self, config): self.cache_model = None if self.no_cache or self.client_config['resume']: return if not config.has_option('cache', 'table'): return cache_table = config.get('cache', 'table') if not cache_table: return logger.info("Init cache table `%s`", cache_table) self.cache_model = CacheTendersModel self.cache_model._meta.database = self.database self.cache_model._meta.db_table = cache_table self.cache_max_size = 0xfff0 self.cache_hit_count = 0 self.cache_miss_count = 0 try: self.cache_model.select().get() cache_table_exists = True except CacheTendersModel.DoesNotExist: cache_table_exists = True except peewee.DatabaseError: cache_table_exists = False self.database.rollback() if self.drop_cache and cache_table_exists: logger.warning("Drop cache table `%s`", cache_table) self.cache_model.drop_table() cache_table_exists = False if not cache_table_exists: logger.info("Create cache table `%s`", cache_table) self.cache_model.create_table()
def get_bin(fname): global bin_paths logger = logging.getLogger("pyTivo.config") if fname in bin_paths: return bin_paths[fname] if config.has_option("Server", fname): fpath = config.get("Server", fname) if os.path.exists(fpath) and os.path.isfile(fpath): bin_paths[fname] = fpath return fpath else: logger.error("Bad %s path: %s" % (fname, fpath)) if sys.platform == "win32": fext = ".exe" else: fext = "" for path in [os.path.join(os.path.dirname(__file__), "bin")] + os.getenv("PATH").split(os.pathsep): fpath = os.path.join(path, fname + fext) if os.path.exists(fpath) and os.path.isfile(fpath): bin_paths[fname] = fpath return fpath logger.warn("%s not found" % fname) return None
def main(loop: Optional[asyncio.AbstractEventLoop] = None) -> int: parser = _make_parser() if len(sys.argv) < 2: parser.print_help() parser.exit() args = parser.parse_args() bandersnatch.log.setup_logging(args) # Prepare default config file if needed. config_path = Path(args.config) if not config_path.exists(): logger.warning( f"Config file '{args.config}' missing, creating default config.") logger.warning( "Please review the config file, then run 'bandersnatch' again.") default_config_path = Path(__file__).parent / "default.conf" try: shutil.copy(default_config_path, args.config) except OSError as e: logger.error(f"Could not create config file: {e}") return 1 config = bandersnatch.configuration.BandersnatchConfig( config_file=args.config).config if config.has_option("mirror", "log-config"): logging.config.fileConfig(str(Path(config.get("mirror", "log-config")))) if loop: loop.set_debug(args.debug) return asyncio.run(async_main(args, config))
def extract_agent_config(config): # get merged into the real agentConfig agentConfig = {} backend = config.get('Main', 'service_discovery_backend') agentConfig['service_discovery'] = True conf_backend = None if config.has_option('Main', 'sd_config_backend'): conf_backend = config.get('Main', 'sd_config_backend') if backend not in SD_BACKENDS: log.error("The backend {0} is not supported. " "Service discovery won't be enabled.".format(backend)) agentConfig['service_discovery'] = False if conf_backend is None: log.warning('No configuration backend provided for service discovery. ' 'Only auto config templates will be used.') elif conf_backend not in SD_CONFIG_BACKENDS: log.error("The config backend {0} is not supported. " "Only auto config templates will be used.".format(conf_backend)) conf_backend = None agentConfig['sd_config_backend'] = conf_backend additional_config = extract_sd_config(config) agentConfig.update(additional_config) return agentConfig
def getShares(tsn=""): shares = [ (section, dict(config.items(section))) for section in config.sections() if not ( section.startswith("_tivo_") or section.startswith("logger_") or section.startswith("handler_") or section.startswith("formatter_") or section in ("Server", "loggers", "handlers", "formatters") ) ] tsnsect = "_tivo_" + tsn if config.has_section(tsnsect) and config.has_option(tsnsect, "shares"): # clean up leading and trailing spaces & make sure ref is valid tsnshares = [] for x in config.get(tsnsect, "shares").split(","): y = x.strip() if config.has_section(y): tsnshares.append((y, dict(config.items(y)))) shares = tsnshares shares.sort() if get_server("nosettings", "false").lower() in ["false", "no", "off"]: shares.append(("Settings", {"type": "settings"})) if get_server("tivo_mak") and get_server("togo_path"): shares.append(("ToGo", {"type": "togo"})) return shares
def alt(section, name, default, local_val=None): """Provide mechanism to alternate between config, local override and default.""" if local_val is not None: return local_val if config.has_option(section, name): return config.get(section, name) return default
def get_togo(name, default=None): if config.has_option('togo', name): return config.get('togo', name) # many togo options used to be in the server section with # the name prefixed w/ 'togo_', so check for those values # before returning the default return get_server('togo_{}'.format(name), default)
def get_mon_api_config(config): mon_api_config = {'is_enabled': False, 'url': '', 'project_name': '', 'project_id': '', 'project_domain_name': '', 'project_domain_id': '', 'ca_file': '', 'insecure': '', 'username': '', 'password': '', 'use_keystone': True, 'keystone_url': '', 'dimensions': None, 'max_buffer_size': 1000, 'backlog_send_rate': 5} if config.has_option("Main", "dimensions"): # parse comma separated dimensions into a dimension list try: dim_list = [dim.split(':') for dim in config.get('Main', 'dimensions').split(',')] mon_api_config['dimensions'] = {key.strip(): value.strip() for key, value in dim_list} except ValueError: mon_api_config['dimensions'] = {} if config.has_section("Api"): options = {"url": config.get, "project_name": config.get, "project_id": config.get, "project_domain_name": config.get, "project_domain_id": config.get, "ca_file": config.get, "insecure": config.get, "username": config.get, "password": config.get, "use_keystone": config.getboolean, "keystone_url": config.get, "max_buffer_size": config.getint, "backlog_send_rate": config.getint, "amplifier": config.getint} for name, func in options.iteritems(): if config.has_option("Api", name): mon_api_config[name] = func("Api", name) return mon_api_config
def setup_logging(config): if config.has_option('ircbot', 'log_config'): log_config = config.get('ircbot', 'log_config') fp = os.path.expanduser(log_config) if not os.path.exists(fp): raise Exception("Unable to read logging config file at %s" % fp) logging.config.fileConfig(fp) else: logging.basicConfig(level=logging.DEBUG)
def extract_sd_config(config): """Extract configuration about service discovery for the agent""" sd_config = {} if config.has_option('Main', 'sd_config_backend'): sd_config['sd_config_backend'] = config.get('Main', 'sd_config_backend') else: sd_config['sd_config_backend'] = None if config.has_option('Main', 'sd_template_dir'): sd_config['sd_template_dir'] = config.get('Main', 'sd_template_dir') else: sd_config['sd_template_dir'] = SD_TEMPLATE_DIR if config.has_option('Main', 'sd_backend_host'): sd_config['sd_backend_host'] = config.get('Main', 'sd_backend_host') if config.has_option('Main', 'sd_backend_port'): sd_config['sd_backend_port'] = config.get('Main', 'sd_backend_port') if config.has_option('Main', 'sd_jmx_enable'): sd_config['sd_jmx_enable'] = config.get('Main', 'sd_jmx_enable') return sd_config
def create_logdirs(configfile): config = SafeConfigParser() config.read(configfile) if config.has_option('handlers', 'keys'): handlers = config.get('handlers', 'keys').split(',') for handler in handlers: section = 'handler_' + handler if config.has_option(section, 'class') and config.get(section, 'class') in ('FileHandler', 'handlers.RotatingFileHandler', 'handlers.TimedRotatingFileHandler'): if config.has_option(section, 'args'): try: args = eval(config.get(section, 'args')) except Exception: continue if isinstance(args, tuple) and len(args) > 0: dir = dirname(args[0]) if not exists(dir): makedirs(dir)
def getConfiguration(cfgfile=None, config_required={'Main': {'key1': 'value1', 'key2': 'value2'}}): ''' read an ini configuration file and return a dictionary of key/value pairs update configuration file if missing any sections accepts: cfgfile - path to configuration file config_required - nested dictionary in the following format: {'Section1': {'key1': 'value1', 'key2': 'value2'}, 'Section 2': {'key1': 'value1'} } ''' if not cfgfile: raise ValueError('no configuration file specified') # required configuraiton options # Section: {'option': 'default value'} logger = logging.getLogger(__name__) logger.debug('getting configuration from file: {}'.format(cfgfile)) cfgpath = os.path.dirname(cfgfile) # config_required = { # 'Main': {'credentials': os.path.join(cfgpath, 'credentials/'), # }, # } config = configuration.get_config(cfgfile) update_config = False logger.debug('checking sections') for section, values in list(config_required.items()): if not config.has_section(section): logger.warning('section: {} not found in {}'.format(section, cfgfile)) logger.debug('adding section {}'.format(section)) config.add_section(section) update_config = True for option, value in list(values.items()): if not config.has_option(section, option): logger.warning('option: {} not found in {}'.format(option, cfgfile)) logger.debug('adding option {}: {}'.format(option, value)) config.set(section, option, value) update_config = True # for section, options in config_required.items(): if update_config: try: logger.debug('updating configuration file at: {}'.format(cfgfile)) configuration.create_config(cfgfile, config) except Exception as e: logger.error(e) return(config)
def getIsExternal(tsn): tsnsect = '_tivo_' + tsn if tsnsect in config.sections(): if config.has_option(tsnsect, 'external'): try: return config.getboolean(tsnsect, 'external') except ValueError: pass return False
def getIsExternal(tsn): tsnsect = "_tivo_" + tsn if tsnsect in config.sections(): if config.has_option(tsnsect, "external"): try: return config.getboolean(tsnsect, "external") except ValueError: pass return False
def add_syslog_handler(logger): syslog_enabled = config.has_option("logging","syslog_enabled") and (int(config.get("logging","syslog_enabled")) == 1) #add syslog handler if we are on linux and syslog is enabled in ambari config if syslog_enabled and IS_LINUX: logger.info("Adding syslog handler to ambari agent logger") syslog_handler = SysLogHandler(address="/dev/log", facility=SysLogHandler.LOG_LOCAL1) syslog_handler.setFormatter(SYSLOG_FORMATTER) logger.addHandler(syslog_handler)
def create_logdirs(configfile): config = SafeConfigParser() config.read(configfile) if config.has_option('handlers', 'keys'): handlers = config.get('handlers', 'keys').split(',') for handler in handlers: section = 'handler_' + handler if config.has_option(section, 'class') and config.get( section, 'class') in ('FileHandler', 'handlers.RotatingFileHandler', 'handlers.TimedRotatingFileHandler'): if config.has_option(section, 'args'): try: args = eval(config.get(section, 'args')) except Exception: continue if isinstance(args, tuple) and len(args) > 0: dir = dirname(args[0]) if not exists(dir): makedirs(dir)
def generate_certificate(dir_name, file_name, key_length, key_type="rsa"): """Create self-signed cert and key files Args: dir_name: name of the directory to store the files file_name: name of the files that will be created. It will append .crt to certificate file and .key to key file key_length: key length in bits key_type: crypto type: RSA or DSA; defaults to RSA Returns: Nothing Exceptions: Raise exceptions on error """ config = globals()['config'] private_key = OpenSSL.crypto.PKey() if key_type == "rsa": private_key.generate_key(OpenSSL.crypto.TYPE_RSA, key_length) elif key_type == "dsa": private_key.generate_key(OpenSSL.crypto.TYPE_DSA, key_length) else: message = "Invalid key_type" logging.error(message) raise OneViewRedfishError(message) if not config.has_option("ssl-cert-defaults", "commonName"): config["ssl-cert-defaults"]["commonName"] = get_ip() cert = OpenSSL.crypto.X509() cert_subject = cert.get_subject() cert_defaults = dict(config.items("ssl-cert-defaults")) for key, value in cert_defaults.items(): setattr(cert_subject, key, value) cert.set_serial_number(1) cert.gmtime_adj_notBefore(0) cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) cert.set_issuer(cert.get_subject()) cert.set_pubkey(private_key) cert.sign(private_key, "sha1") # Save Files with open(os.path.join(dir_name, file_name + ".crt"), "wt") as f: f.write( OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_PEM, cert).decode("UTF-8")) with open(os.path.join(dir_name, file_name + ".key"), "wt") as f: f.write( OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM, private_key).decode("UTF-8"))
def read_config(self): if self.config: return config = ConfigParser.RawConfigParser() config.read(self.config_file) for s in config.sections(): if s == 'global': if config.has_option(s, 'poll_interval'): self.poll_interval = int(config.get(s, 'poll_interval')) if config.has_option(s, 'newrelic_license_key'): self.license_key = config.get(s, 'newrelic_license_key') continue if not config.has_option(s, 'name') or not config.has_option( s, 'url'): continue ns = NginxApiCollector(s, config.get(s, 'name'), config.get(s, 'url'), self.poll_interval) if config.has_option(s, 'http_user') and config.has_option( s, 'http_pass'): ns.basic_auth = base64.b64encode( config.get(s, 'http_user') + b':' + config.get(s, 'http_pass')) self.sources.append(ns) self.config = config
def extract_sd_config(config): """Extract configuration about service discovery for the agent""" sd_config = {} if config.has_option('Main', 'sd_config_backend'): sd_config['sd_config_backend'] = config.get('Main', 'sd_config_backend') else: sd_config['sd_config_backend'] = None if config.has_option('Main', 'sd_template_dir'): sd_config['sd_template_dir'] = config.get( 'Main', 'sd_template_dir') else: sd_config['sd_template_dir'] = SD_TEMPLATE_DIR if config.has_option('Main', 'sd_backend_host'): sd_config['sd_backend_host'] = config.get( 'Main', 'sd_backend_host') if config.has_option('Main', 'sd_backend_port'): sd_config['sd_backend_port'] = config.get( 'Main', 'sd_backend_port') if config.has_option('Main', 'sd_jmx_enable'): sd_config['sd_jmx_enable'] = config.get( 'Main', 'sd_jmx_enable') return sd_config
def perform_prestart_checks(expected_hostname): # Check if current hostname is equal to expected one (got from the server # during bootstrap. global config if expected_hostname is not None: current_hostname = hostname.hostname(config) if current_hostname != expected_hostname: print( "Determined hostname does not match expected. Please check agent " "log for details") msg = "Ambari agent machine hostname ({0}) does not match expected ambari " \ "server hostname ({1}). Aborting registration. Please check hostname, " \ "hostname -f and /etc/hosts file to confirm your " \ "hostname is setup correctly".format(current_hostname, expected_hostname) logger.error(msg) sys.exit(1) # Check if there is another instance running if os.path.isfile( ProcessHelper.pidfile ) and not OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: print("%s already exists, exiting" % ProcessHelper.pidfile) sys.exit(1) # check if ambari prefix exists elif config.has_option('agent', 'prefix') and not os.path.isdir( os.path.abspath(config.get('agent', 'prefix'))): msg = "Ambari prefix dir %s does not exists, can't continue" \ % config.get("agent", "prefix") logger.error(msg) print(msg) sys.exit(1) elif not config.has_option('agent', 'prefix'): msg = "Ambari prefix dir %s not configured, can't continue" logger.error(msg) print(msg) sys.exit(1) check_sudo()
def stop(self): """ Stops all running services and itself """ for key in self.services: service = self.services[key] if not config.has_option("Sockets", "keyinput") or service.socketName != config.get("Sockets", "keyinput"): try: service.service.running = False except: self.logger.debug(traceback.format_exc()) self.logger.info("Terminating socket: " + service.socketName) service.socket.send_json({"head" : "system_message" , "data" : "stop"}) self.running = False
def get_zc(): opt = get_server('zeroconf', 'auto').lower() if opt == 'auto': for section in config.sections(): if section.startswith('_tivo_'): if config.has_option(section, 'shares'): logger = logging.getLogger('pyTivo.config') logger.info('Shares security in use -- zeroconf disabled') return False elif opt in ['false', 'no', 'off']: return False return True
def read_settings(self): ''' Reads the settings from the consul.ini file ''' config = ConfigParser.SafeConfigParser() config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul.ini') config_options = ['host', 'token', 'datacenter', 'servers_suffix', 'tags', 'kv_metadata', 'kv_groups', 'availability', 'unavailable_suffix', 'available_suffix', 'url', 'domain'] for option in config_options: value = None if config.has_option('consul', option): value = config.get('consul', option) setattr(self, option, value)
def parse_config_for_client_initiated(args): """ Parse connector configuration and generate a result in dictionary format which includes integration names and the required info for pulling out data """ connectors = {} try: config = SpecialConfigParser() if not os.path.isfile(args.ini): # The config file does not yet exist, so generate it. # generate_ini_file(args) raise ConfigError("Error: unable to open ini file: %r" % args.ini) config.read(args.ini) for section in config.sections(): if section == 'converters': for name, filter_str in config.items('converters'): DynamicConverter(name, filter_str) elif section == 'oomnitza' or config.has_option( section, 'enable') and config.getboolean( section, 'enable'): if not connectors and section != 'oomnitza': raise ConfigError( "Error: [oomnitza] must be the first section in the ini file." ) cfg = init_connector_from_configuration( section, config.items(section), args) connectors[section] = cfg else: LOG.debug("Skipping connector '%s' as it is not enabled.", section) pass except IOError: raise ConfigError("Could not open config file.") if len(connectors) <= 1: raise ConfigError("No connectors have been enabled.") if args.show_mappings: for name, connector in connectors.items(): if name == 'oomnitza': continue print(connector["__connector__"].section, "Mappings") pprint.pprint(connector["__connector__"].field_mappings) exit(0) return connectors
def user_update(username): if not config: raise ValueError('Configuration not loaded') if not config.has_section('users'): config.add_section('users') if config.has_option('users', username): print 'Changing password for %s' % username else: print 'Adding new user %s' % username password = getpass.getpass('Password: '******' * Updating OFTG-Ninja config ...' return True else: print ' * Failed to update user' return False
def save_attachment(config, address, attachment): sandbox = config.get('DEFAULT', 'sandbox') code = config.get(address, 'code') seq = 1 if config.has_option(address, 'sequence'): seq = config.getint(address, 'sequence') + 1 filename = os.path.join(sandbox, '{}_{}.xlsx'.format(code, seq)) with open(filename, 'wb') as f: f.write(attachment) f.close() return seq, filename
def get169Setting(tsn): if not tsn: return True tsnsect = '_tivo_' + tsn if config.has_section(tsnsect): if config.has_option(tsnsect, 'aspect169'): try: return config.getboolean(tsnsect, 'aspect169') except ValueError: pass if get169Blacklist(tsn) or get169Letterbox(tsn): return False return True
def _main(config): setup_logging(config) fp = config.get('ircbot', 'channel_config') if fp: fp = os.path.expanduser(fp) if not os.path.exists(fp): raise Exception("Unable to read layout config file at %s" % fp) else: raise Exception("Channel Config must be specified in config file.") try: channel_config = ChannelConfig(yaml.load(open(fp))) except Exception: log = logging.getLogger('gerritbot') log.exception("Syntax error in chanel config file") raise bot = GerritBot(channel_config.channels, config.get('ircbot', 'nick'), config.get('ircbot', 'pass'), config.get('ircbot', 'server'), config.getint('ircbot', 'port'), config.getboolean('ircbot', 'force_ssl'), config.get('ircbot', 'server_password')) if config.has_option('ircbot', 'use_mqtt'): use_mqtt = config.getboolean('ircbot', 'use_mqtt') else: use_mqtt = False if use_mqtt: g = GerritMQTT(bot, channel_config, config.get('mqtt', 'host'), config.get('mqtt', 'base_topic'), config.getint('mqtt', 'port'), config.getboolean('mqtt', 'websocket')) else: g = Gerrit(bot, channel_config, config.get('gerrit', 'host'), config.get('gerrit', 'user'), config.getint('gerrit', 'port'), config.get('gerrit', 'key')) g.start() bot.start()
def read_settings(self): ''' Reads the settings from the consul_io.ini file (or consul.ini for backwards compatibility)''' config = configparser.SafeConfigParser() if os.path.isfile(os.path.dirname(os.path.realpath(__file__)) + '/consul_io.ini'): config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul_io.ini') else: config.read(os.path.dirname(os.path.realpath(__file__)) + '/consul.ini') config_options = ['host', 'token', 'datacenter', 'servers_suffix', 'tags', 'kv_metadata', 'kv_groups', 'availability', 'unavailable_suffix', 'available_suffix', 'url', 'domain', 'suffixes', 'bulk_load'] for option in config_options: value = None if config.has_option('consul', option): value = config.get('consul', option).lower() setattr(self, option, value)
def main(): if len(sys.argv) != 2: print("Usage: %s CONFIGFILE" % sys.argv[0]) sys.exit(1) config = ConfigParser.ConfigParser() config.read(sys.argv[1]) pid_path = "" if config.has_option('ircbot', 'pid'): pid_path = config.get('ircbot', 'pid') else: pid_path = "/var/run/gerritbot/gerritbot.pid" pid = pid_file_module.TimeoutPIDLockFile(pid_path, 10) with daemon.DaemonContext(pidfile=pid): _main(config)
def getConfiguration(cfgfile): # required configuraiton options # Section: {'option': 'default value'} logger = logging.getLogger(__name__) logger.debug('getting configuration from file: {}'.format(cfgfile)) cfgpath = os.path.dirname(cfgfile) config_required = { 'Main': { 'credentials': os.path.join(cfgpath, 'credentials/'), }, } config = configuration.get_config(cfgfile) update_config = False logger.debug('checking sections') for section, values in list(config_required.items()): if not config.has_section(section): logger.warning('section: {} not found in {}'.format( section, cfgfile)) logger.debug('adding section {}'.format(section)) config.add_section(section) update_config = True for option, value in list(values.items()): if not config.has_option(section, option): logger.warning('option: {} not found in {}'.format( option, cfgfile)) logger.debug('adding option {}: {}'.format(option, value)) config.set(section, option, value) update_config = True # for section, options in config_required.items(): if update_config: try: logger.debug('updating configuration file at: {}'.format(cfgfile)) configuration.create_config(cfgfile, config) except Exception as e: logger.error(e) return (config)
def determineGear(ex): ''' determines gear based on the type of workout and if workout is a Run it looks at the category to determine shoes. Returns the determined Gear ''' gear = '' try: if ex.type.lower() == 'running': if config.has_option( 'gear', 'shoe_' + ex.category.replace(' ', '_').lower()): gear = config['gear']['shoe_' + ex.category.replace(' ', '_').lower()] else: gear = config['gear']['shoe_default'] else: gear = config['gear']['default_' + ex.type] except: gear = '' return gear
def _controllers(self, only_ip=False): from configuration import config controllers = [] if config.has_option('General', 'ip_control_dns_name'): control_domain = config.get('General', 'ip_control_dns_name') self_ip = netaddr.IPNetwork(self.bind_ip) for answer in dns.resolver.query(control_domain, 'A'): ip = netaddr.IPNetwork(answer.to_text()) if ip == self_ip: # Ignore itself continue ip = ip.ip if not only_ip: try: controllers.append( jsonrpclib.Server("http://{}:{}/".format( ip, self.bind_port))) except: logging.warning("Could not connect to controller %s.", ip) else: controllers.append(ip) return controllers