def __init__(self,
logger,
path,
args_files,
args_data,
req_files=None,
req_keys=None):
self.log = logger
self.conf_path = path
req_files = req_files or []
req_keys = req_keys or []
self.conf_keys = req_keys
self.required_conf_files = [
os.path.join(self.conf_path, cf) for cf in req_files
]
self.conf_files = []
if args_files is not None:
for cf in args_files:
if not os.path.isabs(cf):
cf = os.path.join(os.getcwd(), cf)
self.conf_files += [cf]
self.conf_data = args_data or []
self.conf_builder = ConfigBuilder()
def __init__(self, logger, path, args_files, args_data, req_files=None, req_keys=None): self.log = logger self.conf_path = path req_files = req_files or [] req_keys = req_keys or [] self.conf_keys = req_keys self.required_conf_files = [os.path.join(self.conf_path, cf) for cf in req_files] self.conf_files = [] if args_files is not None: for cf in args_files: if not os.path.isabs(cf): cf = os.path.join(os.getcwd(), cf) self.conf_files += [cf] self.conf_data = args_data or [] self.conf_builder = ConfigBuilder()
def __init__(self, initial_conf_files = None, initial_conf = None, required = [], args_usage = "", add_options = None, expand_vars = False):
DataElement.__init__(self)
from optparse import OptionParser
parser = OptionParser(usage = "usage: %prog [options] " + args_usage, version = VERSION)
parser.add_option("-L", "--log-level", dest="log_level",
default=None, choices=["debug", "info", "warn", "error", "critical", "notset"],
help="Which log level: debug, info, warn, error, critical, notset")
parser.add_option("-P", "--project", action="append", dest="projects", default=[], metavar="PATH",
help="Include this project. Multiple projects can be specified.")
parser.add_option("-C", "--conf", action="append", dest="conf_files", default=[], metavar="FILE",
help="Load configuration from a file. Multiple files can be specified")
parser.add_option("-D", action="append", dest="data", default=[], metavar="PARAM=VALUE",
help="External data value. example -D param1=value")
if add_options is not None:
add_options(parser)
(self.options, self.args) = parser.parse_args()
self.builder = ConfigBuilder()
if initial_conf is not None:
if isinstance(initial_conf, dict):
initial_conf = Data.create(initial_conf)
self.builder.add_merge(initial_conf)
"""
if self.options.log_level is not None:
self.builder.add_value("wok.log.level", self.options.log_level)
self.builder.add_value("wok.platform.log.level", self.options.log_level)
self.builder.add_value("wok.platform.jobs.log.level", self.options.log_level)
"""
# Load configuration
conf_files = []
if initial_conf_files is not None:
conf_files.extend(initial_conf_files)
conf_files.extend(self.options.conf_files)
if len(conf_files) > 0:
files = []
for conf_file in conf_files:
self.builder.add_file(conf_file)
files.append(os.path.abspath(conf_file))
self.builder.add_value("__files", Data.create(files))
for data in self.options.data:
d = data.split("=")
if len(d) != 2:
raise Exception("Data argument wrong: " + data)
self.builder.add_value(d[0], d[1])
# Projects
self.builder.add_value("wok.projects", self.options.projects, merge=True)
# Build configuration
self.builder.merge_into(self)
if len(required) > 0:
self.check_required(required)
if expand_vars:
self.expand_vars()
class OptionsConfig(DataElement):
"""
Command line options parser and configuration loader.
It parses the arguments, loads configuration files (with -c option)
and appends new configuration parameters (with -D option)
Constructor:
initial_conf: Base configuration
required: A list of required configuration keys
args_usage: The command line arguments usage help string
add_options: A function that will be called to populate the OptionParser with new options
Example:
def more_options(parser):
parser.add_option("-o", "--output", dest="output")
c = OptionsConfig(add_options = more_options)
print c.options.output
expand_vars: whether to expand references like ${key1} or not
Attributes:
options: The option variables got from the OptionParser
args: The arguments got from the OptionParser
builder: The configuration builder
"""
def __init__(self, initial_conf_files = None, initial_conf = None, required = [], args_usage = "", add_options = None, expand_vars = False):
DataElement.__init__(self)
from optparse import OptionParser
parser = OptionParser(usage = "usage: %prog [options] " + args_usage, version = VERSION)
parser.add_option("-L", "--log-level", dest="log_level",
default=None, choices=["debug", "info", "warn", "error", "critical", "notset"],
help="Which log level: debug, info, warn, error, critical, notset")
parser.add_option("-P", "--project", action="append", dest="projects", default=[], metavar="PATH",
help="Include this project. Multiple projects can be specified.")
parser.add_option("-C", "--conf", action="append", dest="conf_files", default=[], metavar="FILE",
help="Load configuration from a file. Multiple files can be specified")
parser.add_option("-D", action="append", dest="data", default=[], metavar="PARAM=VALUE",
help="External data value. example -D param1=value")
if add_options is not None:
add_options(parser)
(self.options, self.args) = parser.parse_args()
self.builder = ConfigBuilder()
if initial_conf is not None:
if isinstance(initial_conf, dict):
initial_conf = Data.create(initial_conf)
self.builder.add_merge(initial_conf)
"""
if self.options.log_level is not None:
self.builder.add_value("wok.log.level", self.options.log_level)
self.builder.add_value("wok.platform.log.level", self.options.log_level)
self.builder.add_value("wok.platform.jobs.log.level", self.options.log_level)
"""
# Load configuration
conf_files = []
if initial_conf_files is not None:
conf_files.extend(initial_conf_files)
conf_files.extend(self.options.conf_files)
if len(conf_files) > 0:
files = []
for conf_file in conf_files:
self.builder.add_file(conf_file)
files.append(os.path.abspath(conf_file))
self.builder.add_value("__files", Data.create(files))
for data in self.options.data:
d = data.split("=")
if len(d) != 2:
raise Exception("Data argument wrong: " + data)
self.builder.add_value(d[0], d[1])
# Projects
self.builder.add_value("wok.projects", self.options.projects, merge=True)
# Build configuration
self.builder.merge_into(self)
if len(required) > 0:
self.check_required(required)
if expand_vars:
self.expand_vars()
def check_required(self, required):
for name in required:
if not name in self:
raise Exception("Missing required configuration: %s" % name)
class ConfigProcessor(object):
def __init__(self, logger, path, args_files, args_data, req_files=None, req_keys=None):
self.log = logger
self.conf_path = path
req_files = req_files or []
req_keys = req_keys or []
self.conf_keys = req_keys
self.required_conf_files = [os.path.join(self.conf_path, cf) for cf in req_files]
self.conf_files = []
if args_files is not None:
for cf in args_files:
if not os.path.isabs(cf):
cf = os.path.join(os.getcwd(), cf)
self.conf_files += [cf]
self.conf_data = args_data or []
self.conf_builder = ConfigBuilder()
def load_files(self):
conf_files = self.required_conf_files
user_conf_file = os.path.join(self.conf_path, "user.conf")
if os.path.exists(user_conf_file):
conf_files += [user_conf_file]
conf_files += self.conf_files
# Check that configuration files exist
missing_conf_files = [cf for cf in conf_files if not os.path.exists(cf)]
if len(missing_conf_files) > 0:
self.log.error("Configuration files not found:\n{}".format(
"\n".join(" {}".format(cf) for cf in missing_conf_files)))
exit(-1)
for cf in conf_files:
self.conf_builder.add_file(cf)
return self.conf_builder
def parse_data(self):
for data in self.conf_data:
try:
pos = data.index("=")
key = data[0:pos]
value = data[pos+1:]
try:
v = json.loads(value)
except:
v = value
self.conf_builder.add_value(key, v)
except:
raise Exception("Wrong configuration data: KEY=VALUE expected but found '{}'".format(data))
return self.conf_builder
def load(self):
self.load_files()
self.parse_data()
def validated_conf(self, expand_vars=True):
self.conf = self.conf_builder.get_conf()
if expand_vars:
self.conf.expand_vars()
#self.log.debug(repr(self.conf))
# Validate that required keys exist
mk = self.conf.missing_keys(self.conf_keys)
if len(mk) > 0:
sb = ["The following configuration parameters were not found:\n",
"\n".join("* " + k for k in mk),
"\nin any of the following configuration files:\n",
"\n".join("* " + k for k in conf_files)]
self.log.error("".join(sb))
exit(-1)
return self.conf
def log_debug(self):
if len(self.conf_files) > 0:
self.log.debug("User defined configuration files:\n{}".format(
"\n".join(" {}".format(cf) for cf in self.conf_files)))
self.log.debug("Effective configuration: " + str(self.conf))
class ConfigProcessor(object):
def __init__(self,
logger,
path,
args_files,
args_data,
req_files=None,
req_keys=None):
self.log = logger
self.conf_path = path
req_files = req_files or []
req_keys = req_keys or []
self.conf_keys = req_keys
self.required_conf_files = [
os.path.join(self.conf_path, cf) for cf in req_files
]
self.conf_files = []
if args_files is not None:
for cf in args_files:
if not os.path.isabs(cf):
cf = os.path.join(os.getcwd(), cf)
self.conf_files += [cf]
self.conf_data = args_data or []
self.conf_builder = ConfigBuilder()
def load_files(self):
conf_files = self.required_conf_files
user_conf_file = os.path.join(self.conf_path, "user.conf")
if os.path.exists(user_conf_file):
conf_files += [user_conf_file]
conf_files += self.conf_files
# Check that configuration files exist
missing_conf_files = [
cf for cf in conf_files if not os.path.exists(cf)
]
if len(missing_conf_files) > 0:
self.log.error("Configuration files not found:\n{}".format(
"\n".join(" {}".format(cf) for cf in missing_conf_files)))
exit(-1)
for cf in conf_files:
self.conf_builder.add_file(cf)
return self.conf_builder
def parse_data(self):
for data in self.conf_data:
try:
pos = data.index("=")
key = data[0:pos]
value = data[pos + 1:]
try:
v = json.loads(value)
except:
v = value
self.conf_builder.add_value(key, v)
except:
raise Exception(
"Wrong configuration data: KEY=VALUE expected but found '{}'"
.format(data))
return self.conf_builder
def load(self):
self.load_files()
self.parse_data()
def validated_conf(self, expand_vars=True):
self.conf = self.conf_builder.get_conf()
if expand_vars:
self.conf.expand_vars()
#self.log.debug(repr(self.conf))
# Validate that required keys exist
mk = self.conf.missing_keys(self.conf_keys)
if len(mk) > 0:
sb = [
"The following configuration parameters were not found:\n",
"\n".join("* " + k for k in mk),
"\nin any of the following configuration files:\n",
"\n".join("* " + k for k in conf_files)
]
self.log.error("".join(sb))
exit(-1)
return self.conf
def log_debug(self):
if len(self.conf_files) > 0:
self.log.debug("User defined configuration files:\n{}".format(
"\n".join(" {}".format(cf) for cf in self.conf_files)))
self.log.debug("Effective configuration: " + str(self.conf))
