def parse_channel_config_file(file_path):
config = ConfigParser()
with codecs.open(file_path, 'r', encoding='utf-8') as f:
config.readfp(f)
channel_list = []
# validate and parse the config file
if not config.sections():
print >> sys.stderr, "no section in config file!"
sys.exit(1)
for section in config.sections():
channel_info = {}
OPTIONS = (u'channel_name', u'channel_description', u'rss_url')
for option in OPTIONS:
if not config.has_option(section, option):
print >> sys.stderr, "no option [%s] in section [%s]!" % (
option, section)
sys.exit(1)
channel_info[option] = config.get(section, option)
channel_list.append(channel_info)
return channel_list
def main(prog_args, logger, config):
status = True
config.sections()
logger.info("You are running %s, %s.",
platform.dist()[0],
platform.dist()[1])
template_config_call = config.get("General", "myvar")
if not template_check(prog_args, logger, config):
status = False
if not status:
exit(1)
def __parseCfgOptions(self):
# Is there a configuration available?
configDir = self.get('core.config')
configFiles = self.__getCfgFiles(os.path.join(configDir, "config.d"))
configFiles.insert(0, os.path.join(configDir, "config"))
config = configparser.RawConfigParser()
filesRead = config.read(configFiles)
# Bail out if there's no configuration file
if not filesRead:
raise ConfigNoFile(
"No usable configuration file (%s/config) found!" % configDir)
# Walk thru core configuration values and push them into the registry
for section in config.sections():
if not section in self.__registry:
self.__registry[section] = {}
self.__registry[section].update(config.items(section))
# Initialize the logging module on the fly
try:
tmp = StringIO()
config.write(tmp)
tmp2 = StringIO(tmp.getvalue())
logging.config.fileConfig(tmp2)
except configparser.NoSectionError:
logging.basicConfig(
level=logging.ERROR,
format='%(asctime)s (%(levelname)s): %(message)s')
def options():
parser = import_module(os.getenv('PARSER')).parse_options()
o = parser.parse_args()
logging.config.fileConfig(fileloc(o.log_config))
config = ConfigParser.SafeConfigParser()
config.read(fileloc(o.config))
# parse config in-order
keys = []
for section in config.sections():
for param in config.options(section):
keys.append(param)
setattr(o, param, config.get(section, param))
# cli arguments override configs
for key in provided_arguments(parser):
setattr(o, key, getattr(parser.parse_args(), key))
# environment variables override configs
for key in keys:
setattr(o, key, os.getenv(key.upper(), getattr(o, key)))
return o
def create_models(self, config):
self.models = collections.OrderedDict()
self.main_model = None
for section in config.sections():
if section.startswith('table:'):
table_schema = config.items(section)
table, name = section.split(':', 2)
filter_section = 'filter:' + name
filters = []
if config.has_section(filter_section):
filters = config.items(filter_section)
index_section = 'index:' + name
index_schema = []
if config.has_section(index_section):
index_schema = config.items(index_section)
self.init_model(name, table_schema, index_schema, filters)
self.sorted_models = sorted(
self.models.values(),
key=lambda m: len(m._meta.table_options.get('__iter__', [])))
if self.client_config['resume'] and not self.main_model:
raise ValueError('Main model is required for resume mode')
if not self.main_model:
logger.warning('Main model is not defined in config')
def reset():
global bin_paths
global config
global configs_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print ('WARNING: pyTivo.conf does not exist.\n' +
'Assuming default values.')
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith('_tivo_'):
tsn = section[6:]
if tsn.upper() not in ['SD', 'HD']:
if config.has_option(section, 'name'):
tivo_names[tsn] = config.get(section, 'name')
else:
tivo_names[tsn] = tsn
if config.has_option(section, 'address'):
tivos[tsn] = config.get(section, 'address')
for section in ['Server', '_tivo_SD', '_tivo_HD']:
if not config.has_section(section):
config.add_section(section)
def set_up_checks(config):
check_section = [s for s in config.sections() if s.startswith('check.')]
for section in check_section:
name = section[len('check.'):]
# legacy method to determine the check name from the section header
class_name = name
# if there is an explicit class, use that one with higher priority
if 'class' in config[section]:
class_name = config[section]['class']
enabled = config.getboolean(section, 'enabled', fallback=False)
if not enabled:
_logger.debug('Skipping disabled check %s', name)
continue
_logger.info('Configuring check %s with class %s', name, class_name)
try:
klass = globals()[class_name]
except KeyError:
_logger.error('Cannot create check named %s: Class does not exist',
class_name)
sys.exit(2)
check = klass.create(name, config[section])
if not isinstance(check, Check):
_logger.exception('Check %s is not a correct Check instance',
check)
sys.exit(2)
_logger.debug('Created check instance %s', check)
_checks.append(check)
if not _checks:
_logger.error('No checks enabled')
sys.exit(2)
def main(options, config):
monitored_pages = list()
page_monitors = list()
if options.web_server:
launch_web_server(monitored_pages)
for section in config.sections():
page = MonitoredPage(section,
config.get(section, 'url'),
float(config.get(section, 'maximum_response_time')),
config.get(section, 'should_contain'))
monitored_pages.append(page)
page_monitors.append(monitor_page(page, float(options.frequency)))
try:
while True:
for page in monitored_pages:
try:
printable_status = str(page)
if page.get_status():
logger.info(printable_status)
else:
logger.warning(printable_status)
except NoResponse:
logger.error(printable_status)
time.sleep(options.frequency)
except KeyboardInterrupt:
logger.info('Ctrl-C caught, exiting.')
map(lambda x: x.cancel(), page_monitors)
map(lambda x: x.join(), page_monitors)
return 0
def reset():
global bin_paths
global config
global configs_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print ('WARNING: pyTivo.conf does not exist.\n' +
'Assuming default values.')
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith('_tivo_'):
tsn = section[6:]
if tsn.upper() not in ['SD', 'HD']:
if config.has_option(section, 'name'):
tivo_names[tsn] = config.get(section, 'name')
else:
tivo_names[tsn] = tsn
if config.has_option(section, 'address'):
tivos[tsn] = config.get(section, 'address')
for section in ['Server', '_tivo_SD', '_tivo_HD']:
if not config.has_section(section):
config.add_section(section)
def get_config(config_file=None, server=None):
logger.info("Getting config from config file %s" % config_file)
if config_file is None:
config_file = '/home/minecraft/minecraft/pyredstone.cfg'
if not os.path.exists(config_file):
raise IOError("Could not open config file")
config = ConfigParser.ConfigParser()
config.read(config_file)
if server is None:
try:
sections = config.sections()
logger.debug(sections)
if len(sections) < 1:
raise SyntaxError("No sections found in config file")
elif len(sections) > 1:
logger.warning("More than one server found, no server specified. Using first server.")
server = sections[0]
except ConfigParser.Error as e:
logger.exception("Could not get sections")
if not config.has_section(server):
raise SyntaxError("Server section '%s' of config file does not exist. Cannot continue." % (server, ))
# Now we have a config file and a section.
data = {}
try:
# Take each item in the config file section and dump into a dict
for item in config.items(server):
data[item[0]] = item[1]
logger.info("Config data: %s" % str(data))
except ConfigParser.Error as e:
raise SyntaxError("Config file is improperly formated")
return data
def getShares(tsn=""):
shares = [
(section, dict(config.items(section)))
for section in config.sections()
if not (
section.startswith("_tivo_")
or section.startswith("logger_")
or section.startswith("handler_")
or section.startswith("formatter_")
or section in ("Server", "loggers", "handlers", "formatters")
)
]
tsnsect = "_tivo_" + tsn
if config.has_section(tsnsect) and config.has_option(tsnsect, "shares"):
# clean up leading and trailing spaces & make sure ref is valid
tsnshares = []
for x in config.get(tsnsect, "shares").split(","):
y = x.strip()
if config.has_section(y):
tsnshares.append((y, dict(config.items(y))))
shares = tsnshares
shares.sort()
if get_server("nosettings", "false").lower() in ["false", "no", "off"]:
shares.append(("Settings", {"type": "settings"}))
if get_server("tivo_mak") and get_server("togo_path"):
shares.append(("ToGo", {"type": "togo"}))
return shares
def reset():
global bin_paths
global config
global configs_found
global tivos_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print('WARNING: pyTivo.conf does not exist.\n' +
'Assuming default values.')
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith('_tivo_'):
tsn = section[6:]
if tsn.upper() not in ['SD', 'HD', '4K']:
tivos_found = True
tivos[tsn] = Bdict(config.items(section))
for section in ['Server', '_tivo_SD', '_tivo_HD', '_tivo_4K']:
if not config.has_section(section):
config.add_section(section)
def reset():
global bin_paths
global config
global configs_found
global tivos_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print ('WARNING: pyTivo.conf does not exist.\n' +
'Assuming default values.')
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith('_tivo_'):
tsn = section[6:]
if tsn.upper() not in ['SD', 'HD', '4K']:
tivos_found = True
tivos[tsn] = Bdict(config.items(section))
for section in ['Server', '_tivo_SD', '_tivo_HD', '_tivo_4K']:
if not config.has_section(section):
config.add_section(section)
def reset():
global bin_paths
global config
global configs_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print ("WARNING: pyTivo.conf does not exist.\n" + "Assuming default values.")
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith("_tivo_"):
tsn = section[6:]
if tsn.upper() not in ["SD", "HD"]:
if config.has_option(section, "name"):
tivo_names[tsn] = config.get(section, "name")
else:
tivo_names[tsn] = tsn
if config.has_option(section, "address"):
tivos[tsn] = config.get(section, "address")
for section in ["Server", "_tivo_SD", "_tivo_HD"]:
if not config.has_section(section):
config.add_section(section)
def options():
parser = import_module(os.getenv('PARSER')).parse_options()
o = parser.parse_args()
logging.config.fileConfig(fileloc(o.log_config))
config = ConfigParser.SafeConfigParser()
config.read(fileloc(o.config))
# parse config in-order
keys = []
for section in config.sections():
for param in config.options(section):
keys.append(param)
setattr(o, param, config.get(section, param))
# cli arguments override configs
for key in provided_arguments(parser):
setattr(o, key, getattr(parser.parse_args(), key))
# environment variables override configs
for key in keys:
setattr(o, key, os.getenv(key.upper(), getattr(o, key)))
return o
def __parseCfgOptions(self):
# Is there a configuration available?
configDir = self.get('core.config')
configFiles = self.__getCfgFiles(os.path.join(configDir, "config.d"))
configFiles.insert(0, os.path.join(configDir, "config"))
config = ConfigParser.RawConfigParser()
filesRead = config.read(configFiles)
# Bail out if there's no configuration file
if not filesRead:
raise ConfigNoFile("No usable configuration file (%s/config) found!" % configDir)
# Walk thru core configuration values and push them into the registry
for section in config.sections():
if not section in self.__registry:
self.__registry[section] = {}
self.__registry[section].update(config.items(section))
# Initialize the logging module on the fly
try:
tmp = StringIO()
config.write(tmp)
tmp2 = StringIO(tmp.getvalue())
logging.config.fileConfig(tmp2)
except ConfigParser.NoSectionError:
logging.basicConfig(level=logging.ERROR, format='%(asctime)s (%(levelname)s): %(message)s')
def read_config(self):
if self.config:
return
config = ConfigParser.RawConfigParser()
config.read(self.config_file)
for s in config.sections():
if s == 'global':
if config.has_option(s, 'poll_interval'):
self.poll_interval = int(config.get(s, 'poll_interval'))
if config.has_option(s, 'newrelic_license_key'):
self.license_key = config.get(s, 'newrelic_license_key')
continue
if not config.has_option(s, 'name') or not config.has_option(
s, 'url'):
continue
ns = NginxApiCollector(s, config.get(s, 'name'),
config.get(s, 'url'), self.poll_interval)
if config.has_option(s, 'http_user') and config.has_option(
s, 'http_pass'):
ns.basic_auth = base64.b64encode(
config.get(s, 'http_user') + b':' +
config.get(s, 'http_pass'))
self.sources.append(ns)
self.config = config
def main():
home_dir = os.path.expanduser('~')
program_dir = os.path.abspath(os.path.join(home_dir, '.dbackups'))
db_config_file = os.path.join(program_dir, 'databases.ini')
if not os.path.isdir(os.path.join(BASE_DIR, '../logs')):
os.mkdir(os.path.join(BASE_DIR, '../logs'))
if not os.path.isfile(db_config_file):
print('Config File not found. {}'.format(db_config_file))
sys.exit(1)
#logging_config = resource_filename(__name__, '../config/logging.ini')
#logging.config.fileConfig(logging_config)
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)-6s line %(lineno)-4s %(message)s')
args = parser.parse_args()
pprint(args)
config = ConfigParser.ConfigParser()
config.read(db_config_file)
logging.debug(config.sections())
if not config.has_section(args.database):
logging.info('DB alias not found in the config file {} -> [{}]'.format(db_config_file, args.database))
sys.exit(1)
else:
logging.info('Found the DB settings in the config file. Continuing.')
db_type = config.get(args.database, 'db_type')
db_host = config.get(args.database, 'db_host')
db_user = config.get(args.database, 'db_user')
db_pass = config.get(args.database, 'db_pass')
db_port = config.get(args.database, 'db_port')
db_name = config.get(args.database, 'db_name')
db_object = get_database_object(db_type, db_host, db_name, db_user, db_pass, db_port)
logging.debug('DB object created: {}'.format(db_object))
if args.command == 'backup':
logging.info('Chose to backup {}'.format(db_object.db_host))
logging.info('Dump file: [{}]'.format(db_object.dump_file_name))
db_object.dump()
logging.info('Dumping DB finished.')
if args.upload_url:
print('Uploading to the desired URL: {}'.format(args.upload_url))
upload_http_put(db_object.dump_file, args.upload_url)
if args.command == 'clone':
logging.info('Going to clone_to from one DB to another.')
dev_db = get_database_object(db_type, args.dev_host, args.dev_name, args.dev_user, args.dev_pass,
args.dev_port)
db_object.clone_to(dev_db, args.latest_local)
if args.command == 'clean':
logging.info('Cleaning the dumps directory to make room for more dumps.')
file_list = cleanup_backups.find_files_for_delete(db_object.get_dump_dir(), db_object.db_host,
db_object.db_name)
cleanup_backups.delete_file_list(file_list)
def create_models(self, config):
self.models = dict()
for section in config.sections():
if section.startswith('table:'):
table_schema = config.items(section)
table, name = section.split(':', 2)
self.init_model(name, table_schema)
def getShares(tsn=''):
shares = [(section, Bdict(config.items(section)))
for section in config.sections()
if not (section.startswith(('_tivo_', 'logger_', 'handler_',
'formatter_')) or section in
('Server', 'loggers', 'handlers', 'formatters'))]
tsnsect = '_tivo_' + tsn
if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'):
# clean up leading and trailing spaces & make sure ref is valid
tsnshares = []
for x in config.get(tsnsect, 'shares').split(','):
y = x.strip()
if config.has_section(y):
tsnshares.append((y, Bdict(config.items(y))))
shares = tsnshares
shares.sort()
if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']:
shares.append(('Settings', {'type': 'settings'}))
if get_server('tivo_mak') and get_server('togo_path'):
shares.append(('ToGo', {'type': 'togo'}))
return shares
def ini_to_dict(ini_str):
import re
try:
from configparser import SafeConfigParser
except ImportError:
from ConfigParser import SafeConfigParser
finally:
config, conf_dict = SafeConfigParser(), {}
config.optionxform = str
try:
config.read_string(ini_str)
except AttributeError:
from io import StringIO
config.readfp(StringIO(ini_str))
for sect in config.sections():
if 'default' != sect:
conf_dict[sect] = tmp_dict = {}
else:
tmp_dict = conf_dict
for key, val in config.items(sect):
match_1 = re.search('([^#]*)', val)
tmp_dict[key] = match_1.groups()[0].strip().strip("'")
return conf_dict
def load_config(self, configpath, pkg=False):
"""
Use ConfigParser module to load config sections
:param pkg: file is inside the package (configpath is the relative path
inside the package)
:param configpath:
:return:
"""
if not pkg and not os.path.exists(configpath):
LOG.error("Configuration file not found (%s)" % configpath)
from errno import ENOENT
raise OSError(ENOENT)
config = SafeConfigParser(allow_no_value=True)
if pkg:
with pkgr.resource_stream(__name__, configpath) as conf:
config.readfp(conf)
else:
config.read(configpath)
LOG.debug(config)
for section in config.sections():
if hasattr(self, section):
tmp = format_dict(dict(config.items(section)))
getattr(self, section).config.update(tmp)
LOG.debug("%s config updated" % section)
LOG.debug(
"%s.%s : %s" %
(self.__class__.__name__, section, getattr(self, section)))
else:
LOG.warning("Unknow config section %s" % section)
def parse_cfg_with_sections(stream):
"""Return as dict of dict of ...
"""
#Experimental:
"""
ConfigParser sections become sub-sub sections when separated by dots.
[foo.bar]
baz = 42
is equivalent to JSON
{"foo": {"bar": {"baz": 42}}}
"""
content = stream.read()
result = dict()
try:
jdict = json.loads(NativeIO(content).read())
return jdict
except ValueError:
pass #logger.exception('Could not parse stream as JSON.')
try:
config = ConfigParser() #strict=False?
config.optionxform = str
config.readfp(NativeIO(content))
sections = config.sections()
for sec in sections:
result[sec] = dict(config.items(sec))
return result
except:
raise
def set_up_checks(config):
check_section = [s for s in config.sections() if s.startswith('check.')]
for section in check_section:
name = section[len('check.'):]
# legacy method to determine the check name from the section header
class_name = name
# if there is an explicit class, use that one with higher priority
if 'class' in config[section]:
class_name = config[section]['class']
enabled = config.getboolean(section, 'enabled', fallback=False)
if not enabled:
_logger.debug('Skipping disabled check %s', name)
continue
_logger.info('Configuring check %s with class %s', name, class_name)
try:
klass = globals()[class_name]
except KeyError:
_logger.error('Cannot create check named %s: Class does not exist',
class_name)
sys.exit(2)
check = klass.create(name, config[section])
if not isinstance(check, Check):
_logger.exception('Check %s is not a correct Check instance',
check)
sys.exit(2)
_logger.debug('Created check instance %s', check)
_checks.append(check)
if not _checks:
_logger.error('No checks enabled')
sys.exit(2)
def getShares(tsn=''):
shares = [(section, Bdict(config.items(section)))
for section in config.sections()
if not (section.startswith(('_tivo_', 'logger_', 'handler_',
'formatter_'))
or section in ('Server', 'loggers', 'handlers',
'formatters')
)
]
tsnsect = '_tivo_' + tsn
if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'):
# clean up leading and trailing spaces & make sure ref is valid
tsnshares = []
for x in config.get(tsnsect, 'shares').split(','):
y = x.strip()
if config.has_section(y):
tsnshares.append((y, Bdict(config.items(y))))
shares = tsnshares
shares.sort()
if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']:
shares.append(('Settings', {'type': 'settings'}))
if get_server('tivo_mak') and get_server('togo_path'):
shares.append(('ToGo', {'type': 'togo'}))
return shares
def loadConfig(self):
logger.debug("Loading configuration")
config = ConfigParser.ConfigParser()
config.read(os.path.join(os.getcwd(), '..', 'config', 'config.ini'))
logger.debug("Config Sections: " + str(config.sections()))
return config
def parse_cfg_with_sections(stream):
"""Return as dict of dict of ...
"""
#Experimental:
"""
ConfigParser sections become sub-sub sections when separated by dots.
[foo.bar]
baz = 42
is equivalent to JSON
{"foo": {"bar": {"baz": 42}}}
"""
content = stream.read()
result = dict()
try:
jdict = json.loads(NativeIO(content).read())
return jdict
except ValueError:
pass #logger.exception('Could not parse stream as JSON.')
try:
config = ConfigParser() #strict=False?
config.optionxform = str
config.readfp(NativeIO(content))
sections = config.sections()
for sec in sections:
result[sec] = dict(config.items(sec))
return result
except:
raise
def getShares(tsn=''):
shares = [(section, Bdict(config.items(section)))
for section in config.sections()
if not (section.startswith(special_section_prefixes)
or section in special_section_names)
]
tsnsect = '_tivo_' + tsn
if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'):
# clean up leading and trailing spaces & make sure ref is valid
tsnshares = []
for x in config.get(tsnsect, 'shares').split(','):
y = x.strip()
if config.has_section(y):
tsnshares.append((y, Bdict(config.items(y))))
shares = tsnshares
shares.sort()
if not config.getboolean('Server', 'nosettings', fallback=False):
shares.append(('Settings', {'type': 'settings'}))
if get_server('tivo_mak') and get_togo('path'):
shares.append(('ToGo', {'type': 'togo'}))
return shares
def _loadConfig(self):
def writeDefaultConfig(config):
config.add_section('STORAGE')
storageConfig = config['STORAGE']
storageConfig['data_path'] = 'data/storage/'
config.add_section('SCHEDULER')
config.add_section('TASK_MANAGER')
config.add_section('INTERFACE')
config.add_section('CONTRACTOR')
config.add_section('ENTITY_MANAGER')
interfaceConfig = config['INTERFACE']
interfaceConfig['xmlrpc_server_addr'] = 'localhost'
interfaceConfig['xmlrpc_server_port'] = '8000'
config.add_section('LOGGING')
loggingConfig = config['LOGGING']
loggingConfig['file'] = CONFIG_PATH + '/logging.conf'
if not os.path.exists(CONFIG_PATH):
os.makedirs(CONFIG_PATH)
with open(CONFIG_FILE,'w') as configFile:
config.write(configFile)
self._config = configparser.ConfigParser()
config = self._config
config.read(CONFIG_FILE)
self._corelogger.info('load configuration')
if len(config.sections()) < 1:
self._corelogger.info('write default configuration to file')
writeDefaultConfig(config)
logging.config.fileConfig(config['LOGGING']['file'])
def getShares(tsn=''):
shares = [(section, Bdict(config.items(section)))
for section in config.sections()
if not (section.startswith(('_tivo_', 'logger_', 'handler_',
'formatter_'))
or section in ('Server', 'loggers', 'handlers',
'formatters')
)
]
tsnsect = '_tivo_' + tsn
if config.has_section(tsnsect) and config.has_option(tsnsect, 'shares'):
# clean up leading and trailing spaces & make sure ref is valid
tsnshares = []
for x in config.get(tsnsect, 'shares').split(','):
y = x.strip()
if config.has_section(y):
tsnshares.append((y, Bdict(config.items(y))))
shares = tsnshares
shares.sort()
if get_server('nosettings', 'false').lower() in ['false', 'no', 'off']:
shares.append(('Settings', {'type': 'settings'}))
if get_server('tivo_mak') and get_server('togo_path'):
shares.append(('ToGo', {'type': 'togo'}))
if sys.platform == 'win32':
shares.append(('VRD', {'type': 'vrd'}))
if getattr(sys, 'frozen', False):
shares.append(('Desktop', {'type': 'desktop', 'path': os.path.join(sys._MEIPASS, 'plugins', 'desktop', 'content')}))
return shares
def create_models(self, config):
self.models = dict()
for section in config.sections():
if section.startswith('table:'):
table_schema = config.items(section)
table, name = section.split(':', 2)
self.init_model(name, table_schema)
def load_config(path):
config = configparser.ConfigParser()
config.read(path)
result = defaultdict(dict)
for s in config.sections():
for o in config.options(s):
result[s][o] = config.get(s, o)
return result
def get_settings(config):
settings = dict()
for section in config.sections():
value = dict()
for setting in config[section]:
value.update({setting: config.get(section, setting)})
settings.update({section: value})
return settings
def read_yml_or_cfg(file: Union[str, Path], prefer_cfg: bool = False,
cfg_ext=CFG_EXT) -> Any:
"""
Read the file and return its content as a python object.
Parameters
----------
file : file-like
The file to read from.
is_cfg : bool, optional
Indicate that the file should be in 'config' format.
Assumed only for ext '.ini', '.cfg', '.conf' and '.config'.
The default is None.
Returns
-------
Any
Most likely a dict or a ConfigParser supporting get().
But maybe just a list or a single object.
"""
err_str = ("utilities.read_yml_or_cfg encountered '{}' while "
"reading config from '{}' and prefer_cfg={}")
file = Path(file)
if not file.exists():
raise FileNotFoundError(file.absolute())
if not file.is_file():
raise IsADirectoryError(file.resolve())
# continue with file
prefer_cfg = prefer_cfg or file.suffix.lower() in cfg_ext
if prefer_cfg:
try:
config = ConfigParser()
config.read(file)
if config.sections():
return config
print(err_str.format("empty config", file.resolve(), prefer_cfg))
except (ParsingError, CfgError) as exc:
print(err_str.format(exc, file.resolve(), prefer_cfg))
# try yaml file next
try:
result = yaml.safe_load(file.open())
if result is not None:
return result
except yaml.error.MarkedYAMLError as exc:
print(err_str.format(exc, file.resolve(), prefer_cfg))
if not prefer_cfg:
try:
config = ConfigParser()
config.read(file.open())
return config
except (ParsingError, CfgError) as exc:
print(err_str.format(exc, file.resolve(), prefer_cfg))
return dict()
def getIsExternal(tsn):
tsnsect = '_tivo_' + tsn
if tsnsect in config.sections():
if config.has_option(tsnsect, 'external'):
try:
return config.getboolean(tsnsect, 'external')
except ValueError:
pass
return False
def get_config_options(filename):
"Function return dictionary with all options from configuration file"
config = ConfigParser.ConfigParser()
config.read(filename)
conf_options = {}
for section in config.sections():
conf_options[section] = {}
for option in config.options(section):
conf_options[section][option] = config.get(section, option)
return conf_options
def read_users(filename):
config = ConfigParser()
config.read(filename)
for i in config.sections():
USERS.update(
dict([(j, {
'password': k,
'group': i
}) for j, k in config.items(i)]))
def getIsExternal(tsn):
tsnsect = "_tivo_" + tsn
if tsnsect in config.sections():
if config.has_option(tsnsect, "external"):
try:
return config.getboolean(tsnsect, "external")
except ValueError:
pass
return False
def _adjust_options(self, config):
# config defaults appear in all sections.
# we'll need to filter them out.
defaults = config.defaults().keys()
# see what options are already defined and add missing ones
preset = [(option.section, option.setting) for option in self.items()]
for section in config.sections():
for name in config.options(section):
if ((section, name) not in preset) \
and (name not in defaults):
self.add_option(Option(self, section, name))
def init_ordered_configparser(self):
confs = collections.OrderedDict()
for root, dirs, files in os.walk("../config"):
for name in files:
if name.endswith('conf'):
parts = name.split('_')
confs[int(parts[0])] = os.path.join(root, name)
config = configparser.ConfigParser()
config.read(confs.values())
logger.debug(config.sections())
return config
def __init__(self, config):
self.config = config
topics = config.sections()
self.listener = ListenerContainer(topics=topics)
self._loop = True
if GSHHS_DATA_ROOT:
self._cw = ContourWriter(GSHHS_DATA_ROOT)
else:
self._cw = None
self._force_gc = False
def initialize(config_file: str = None,
dev: bool = False,
local: bool = False,
cache: bool = True) -> 'Skill':
""" Initialize the app
* configure logging
* load the intents
* setup the routes
* return default app
:param config_file: read configuration from file
:param dev: initialize development mode
:param local: activate local proxy for services
:param cache: use local caches (False to deactivate)
:return:
"""
config.read_conf(config_file)
from .caching import decorators
decorators.CACHES_ACTIVE = cache
from . import requests
requests.USE_LOCAL_SERVICES = local
max_request_size = config.getint('skill', 'max_request_size', fallback=0)
if max_request_size:
BaseRequest.MEMFILE_MAX = max_request_size
from . import K8sChecks
with K8sChecks('init'):
configure_logging()
skill = app()
if not skill.get_intents():
raise RuntimeError(
"No intent handlers loaded. Check the log messages for import errors..."
)
if not l10n.translations:
l10n.translations = l10n.load_translations()
from . import routes # Add standard routes
from . import swagger # Add swagger route
set_dev_mode() if dev else setup_services()
# Copy configuration to Skill instance
skill.config.load_dict({
section: dict(config.items(section))
for section in config.sections()
})
return skill
def get_quotas(config):
quotas = {}
for section in config.sections():
if not section.startswith('quota'):
continue
try:
quotas[section] = build_quota(config, section)
except:
e = ''.join(traceback.format_exception(*sys.exc_info()))
logging.error('Could not build quota for {0}:\n{1}'.format(
section, e))
return quotas
def as_dict(config):
"""
Converts a ConfigParser object into a dictionary.
The resulting dictionary has sections as keys which point to a dict of the
sections options as key => value pairs.
"""
the_dict = {}
for section in config.sections():
the_dict[section] = {}
for key, val in config.items(section):
the_dict[section][key] = val
return the_dict
def get_zc():
opt = get_server('zeroconf', 'auto').lower()
if opt == 'auto':
for section in config.sections():
if section.startswith('_tivo_'):
if config.has_option(section, 'shares'):
logger = logging.getLogger('pyTivo.config')
logger.info('Shares security in use -- zeroconf disabled')
return False
elif opt in ['false', 'no', 'off']:
return False
return True
def as_dict(config):
"""
https://stackoverflow.com/questions/1773793/convert-configparser-items-to-dictionary/23944270#23944270
Converts a ConfigParser object into a dictionary.
The resulting dictionary has sections as keys which point to a dict of the
sections options as key => value pairs.
"""
_dict = {}
for section in config.sections():
_dict[section] = {}
for key, val in config.items(section):
_dict[section][key] = val
return _dict
def __parseCfgOptions(self):
# Is there a configuration available?
configDir = self.get('core.config')
configFiles = self.__getCfgFiles(os.path.join(configDir, "config.d"))
configFiles.insert(0, os.path.join(configDir, "config"))
config = configparser.RawConfigParser()
filesRead = config.read(configFiles)
# Bail out if there's no configuration file
if not filesRead:
raise ConfigNoFile(
"No usable configuration file (%s/config) found!" % configDir)
# Walk thru core configuration values and push them into the registry
for section in config.sections():
if not section in self.__registry:
self.__registry[section] = {}
self.__registry[section].update(config.items(section))
if not hasattr(sys, "_called_from_test") or getattr(
sys, "_called_from_test") is False:
# do not use this in tests
self.__user_config = configparser.RawConfigParser()
# read the settings changed via gui client
if os.path.exists(
os.path.join(self.get('core.user-config'), "user-config")):
filesRead = self.__user_config.read(
os.path.join(self.get('core.user-config'), "user-config"))
if not filesRead:
raise ConfigNoFile(
"No usable GUI configuration file (%s/user-config) found!"
% configDir)
# Initialize the logging module on the fly
try:
if self.__user_config is not None:
for section in self.__user_config.sections():
for option, value in self.__user_config.items(section):
config.set(section, option, value)
tmp = StringIO()
config.write(tmp)
tmp2 = StringIO(tmp.getvalue())
logging.config.fileConfig(tmp2)
except configparser.NoSectionError:
logging.basicConfig(
level=logging.ERROR,
format='%(asctime)s (%(levelname)s): %(message)s')
def get_zc():
opt = get_server('zeroconf', 'auto').lower()
if opt == 'auto':
for section in config.sections():
if section.startswith('_tivo_'):
if config.has_option(section, 'shares'):
logger = logging.getLogger('pyTivo.config')
logger.info('Shares security in use -- zeroconf disabled')
return False
elif opt in ['false', 'no', 'off']:
return False
return True
def get_zc():
opt = get_server("zeroconf", "auto").lower()
if opt == "auto":
for section in config.sections():
if section.startswith("_tivo_"):
if config.has_option(section, "shares"):
logger = logging.getLogger("pyTivo.config")
logger.info("Shares security in use -- zeroconf disabled")
return False
elif opt in ["false", "no", "off"]:
return False
return True
def loadConfig(configFile):
# Charger le fichier de configuration
with open(configFile) as f:
sample_config = f.read()
config = ConfigParser.RawConfigParser(allow_no_value=True)
config.readfp(io.BytesIO(sample_config))
# List all contents
if (False):
print("Parametrage du fichier de configuration :"+configFile)
for section in config.sections():
print("[%s]" % section)
for options in config.options(section):
print("\t%s = %s" % (options,
config.get(section, options)))
return config
def parse_cfg(cfg_file):
with open(cfg_file) as cfg:
test_parameters = dict()
data_args = dict()
descriptors = []
flann_args = dict()
config = ConfigParser.RawConfigParser()
config.readfp(cfg)
sections = config.sections()
for section in sections:
if section == 'Test':
for name, value in config.items(section):
if name == 'batch_size':
if value == 'None':
test_parameters[name] = None
else:
test_parameters[name] = int(value)
else:
test_parameters[name] = value
elif section == 'Dataset':
for name, value in config.items(section):
if name == 'classes':
data_args[name] = value.split(',')
elif name == 'trainsize' or name == 'testsize' or \
name == 'no_classes':
data_args[name] = int(value)
else:
data_args[name] = value
elif section == 'Flann':
for name, value in config.items(section):
if name in ['k','checks','trees']:
flann_args[name] = int(value)
else:
flann_args[name] = value
else:
d = [section, dict()]
for name, value in config.items(section):
if name=='alpha':
d[1][name] = float(value)
elif name=='verbose':
d[1][name] = value=='True'
else:
d[1][name] = value
descriptors.append(d)
return test_parameters, data_args, descriptors, flann_args
def create_config():
options = parse_arguments()
# 1) read the default config at "~/.ethereum"
config = konfig.read_config()
# 2) read config from file
if getattr(options, 'config_file'):
config.read(getattr(options, 'config_file'))
# 3) apply cmd line options to config
for section in config.sections():
for a,v in config.items(section):
if getattr(options, a, None) is not None:
config.set(section, a, getattr(options,a))
return config
def create_models(self, config):
self.models = dict()
self.main_model = None
for section in config.sections():
if section.startswith('table:'):
table_schema = config.items(section)
table, name = section.split(':', 2)
index_section = 'index:' + name
index_schema = []
if config.has_section(index_section):
index_schema = config.items(index_section)
self.init_model(name, table_schema, index_schema)
self.sorted_models = sorted(self.models.values(),
key=lambda m: len(m._meta.table_options.get('__iter__', [])))
if self.client_config['resume'] and not self.main_model:
raise ValueError('Main model is required for resume mode')
def __parseCfgOptions(self):
# Is there a configuration available?
configDir = self.get('core.config')
configFiles = self.__getCfgFiles(os.path.join(configDir, "config.d"))
configFiles.insert(0, os.path.join(configDir, "config"))
config = configparser.RawConfigParser()
filesRead = config.read(configFiles)
# Bail out if there's no configuration file
if not filesRead:
raise ConfigNoFile("No usable configuration file (%s/config) found!" % configDir)
# Walk thru core configuration values and push them into the registry
for section in config.sections():
if not section in self.__registry:
self.__registry[section] = {}
self.__registry[section].update(config.items(section))
if not hasattr(sys, "_called_from_test") or getattr(sys, "_called_from_test") is False:
# do not use this in tests
self.__user_config = configparser.RawConfigParser()
# read the settings changed via gui client
if os.path.exists(os.path.join(self.get('core.user-config'), "user-config")):
filesRead = self.__user_config.read(os.path.join(self.get('core.user-config'), "user-config"))
if not filesRead:
raise ConfigNoFile("No usable GUI configuration file (%s/user-config) found!" % configDir)
# Initialize the logging module on the fly
try:
if self.__user_config is not None:
for section in self.__user_config.sections():
for option, value in self.__user_config.items(section):
config.set(section, option, value)
tmp = StringIO()
config.write(tmp)
tmp2 = StringIO(tmp.getvalue())
logging.config.fileConfig(tmp2)
except configparser.NoSectionError:
logging.basicConfig(level=logging.ERROR, format='%(asctime)s (%(levelname)s): %(message)s')
def create_config():
options = parse_arguments()
# 1) read the default config at "~/.ethereum"
config = konfig.read_config()
# 2) read config from file
cfg_fn = getattr(options, 'config_file')
if cfg_fn:
if not os.path.exists(cfg_fn):
konfig.read_config(cfg_fn) # creates default
config.read(cfg_fn)
# 3) apply cmd line options to config
for section in config.sections():
for a,v in config.items(section):
if getattr(options, a, None) is not None:
config.set(section, a, getattr(options,a))
# set datadir
data_dir.set(config.get('misc', 'data_dir'))
return config
def _initConfig(self):
def writeDefaultConfig(config):
config.add_section('STORAGE')
storageConfig = config['STORAGE']
storageConfig['data_path'] = 'data/storage/'
config.add_section('TASK_MANAGER')
config.add_section('SERVER')
config.add_section('SCHEDULER')
config.add_section('LOGGING')
loggingConfig = config['LOGGING']
loggingConfig['file'] = CONFIG_PATH + '/logging.conf'
if not os.path.exists(CONFIG_PATH):
os.makedirs(CONFIG_PATH)
with open(CONFIG_FILE,'w') as configFile:
config.write(configFile)
self._config = configparser.ConfigParser()
config = self._config
config.read(CONFIG_FILE)
if len(config.sections()) < 1:
writeDefaultConfig(config)
logging.config.fileConfig(config['LOGGING']['file'])
def set_config_data(self, data: dict):
for identifier, update_data in data.items():
# dont rewrite empty...
if not len(update_data):
continue
if identifier == "meta":
hostname = update_data.get("hostname", None)
if hostname:
SysUtil.set_hostname(hostname)
if update_data.get("update", False):
SysUtil.update_from_git()
config = SysUtil.ensure_config(identifier)
sections = set(config.sections()).intersection(set(update_data.keys()))
for section in sections:
update_section = update_data[section]
options = set(config.options(section)).intersection(set(update_section.keys()))
for option in options:
config.set(section, option, str(update_section[option]))
SysUtil.write_config(config, identifier)
def __parseCfgOptions(self):
# Is there a configuration available?
configFile = self.get('core.config')
configFiles = self.__getCfgFiles(configFile + ".d")
configFiles.insert(0, configFile)
config = ConfigParser.RawConfigParser()
filesRead = config.read(configFiles)
# Bail out if there's no configuration file
if not filesRead:
raise ConfigNoFile("No usable configuration file (%s) found!" % configFile)
# Walk thru core configuration values and push them into the registry
for section in config.sections():
if not section in self.__registry:
self.__registry[section] = {}
self.__registry[section].update(config.items(section))
# Initialize the logging module on the fly
tmp = StringIO.StringIO()
config.write(tmp)
tmp2 = StringIO.StringIO(tmp.getvalue())
logging.config.fileConfig(tmp2)
def reset():
global bin_paths
global config
global configs_found
global tivos_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print ("WARNING: pyTivo.conf does not exist.\n" + "Assuming default values.")
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith("_tivo_"):
tsn = section[6:]
if tsn.upper() not in ["SD", "HD", "4K"]:
tivos_found = True
tivos[tsn] = Bdict(config.items(section))
for section in ["Server", "_tivo_SD", "_tivo_HD", "_tivo_4K"]:
if not config.has_section(section):
config.add_section(section)