class ConfigProcessor(object): def __init__(self, logger, path, args_files, args_data, req_files=None, req_keys=None): self.log = logger self.conf_path = path req_files = req_files or [] req_keys = req_keys or [] self.conf_keys = req_keys self.required_conf_files = [os.path.join(self.conf_path, cf) for cf in req_files] self.conf_files = [] if args_files is not None: for cf in args_files: if not os.path.isabs(cf): cf = os.path.join(os.getcwd(), cf) self.conf_files += [cf] self.conf_data = args_data or [] self.conf_builder = ConfigBuilder() def load_files(self): conf_files = self.required_conf_files user_conf_file = os.path.join(self.conf_path, "user.conf") if os.path.exists(user_conf_file): conf_files += [user_conf_file] conf_files += self.conf_files # Check that configuration files exist missing_conf_files = [cf for cf in conf_files if not os.path.exists(cf)] if len(missing_conf_files) > 0: self.log.error("Configuration files not found:\n{}".format( "\n".join(" {}".format(cf) for cf in missing_conf_files))) exit(-1) for cf in conf_files: self.conf_builder.add_file(cf) return self.conf_builder def parse_data(self): for data in self.conf_data: try: pos = data.index("=") key = data[0:pos] value = data[pos+1:] try: v = json.loads(value) except: v = value self.conf_builder.add_value(key, v) except: raise Exception("Wrong configuration data: KEY=VALUE expected but found '{}'".format(data)) return self.conf_builder def load(self): self.load_files() self.parse_data() def validated_conf(self, expand_vars=True): self.conf = self.conf_builder.get_conf() if expand_vars: self.conf.expand_vars() #self.log.debug(repr(self.conf)) # Validate that required keys exist mk = self.conf.missing_keys(self.conf_keys) if len(mk) > 0: sb = ["The following configuration parameters were not found:\n", "\n".join("* " + k for k in mk), "\nin any of the following configuration files:\n", "\n".join("* " + k for k in conf_files)] self.log.error("".join(sb)) exit(-1) return self.conf def log_debug(self): if len(self.conf_files) > 0: self.log.debug("User defined configuration files:\n{}".format( "\n".join(" {}".format(cf) for cf in self.conf_files))) self.log.debug("Effective configuration: " + str(self.conf))
class OptionsConfig(DataElement): """ Command line options parser and configuration loader. It parses the arguments, loads configuration files (with -c option) and appends new configuration parameters (with -D option) Constructor: initial_conf: Base configuration required: A list of required configuration keys args_usage: The command line arguments usage help string add_options: A function that will be called to populate the OptionParser with new options Example: def more_options(parser): parser.add_option("-o", "--output", dest="output") c = OptionsConfig(add_options = more_options) print c.options.output expand_vars: whether to expand references like ${key1} or not Attributes: options: The option variables got from the OptionParser args: The arguments got from the OptionParser builder: The configuration builder """ def __init__(self, initial_conf_files = None, initial_conf = None, required = [], args_usage = "", add_options = None, expand_vars = False): DataElement.__init__(self) from optparse import OptionParser parser = OptionParser(usage = "usage: %prog [options] " + args_usage, version = VERSION) parser.add_option("-L", "--log-level", dest="log_level", default=None, choices=["debug", "info", "warn", "error", "critical", "notset"], help="Which log level: debug, info, warn, error, critical, notset") parser.add_option("-P", "--project", action="append", dest="projects", default=[], metavar="PATH", help="Include this project. Multiple projects can be specified.") parser.add_option("-C", "--conf", action="append", dest="conf_files", default=[], metavar="FILE", help="Load configuration from a file. Multiple files can be specified") parser.add_option("-D", action="append", dest="data", default=[], metavar="PARAM=VALUE", help="External data value. example -D param1=value") if add_options is not None: add_options(parser) (self.options, self.args) = parser.parse_args() self.builder = ConfigBuilder() if initial_conf is not None: if isinstance(initial_conf, dict): initial_conf = Data.create(initial_conf) self.builder.add_merge(initial_conf) """ if self.options.log_level is not None: self.builder.add_value("wok.log.level", self.options.log_level) self.builder.add_value("wok.platform.log.level", self.options.log_level) self.builder.add_value("wok.platform.jobs.log.level", self.options.log_level) """ # Load configuration conf_files = [] if initial_conf_files is not None: conf_files.extend(initial_conf_files) conf_files.extend(self.options.conf_files) if len(conf_files) > 0: files = [] for conf_file in conf_files: self.builder.add_file(conf_file) files.append(os.path.abspath(conf_file)) self.builder.add_value("__files", Data.create(files)) for data in self.options.data: d = data.split("=") if len(d) != 2: raise Exception("Data argument wrong: " + data) self.builder.add_value(d[0], d[1]) # Projects self.builder.add_value("wok.projects", self.options.projects, merge=True) # Build configuration self.builder.merge_into(self) if len(required) > 0: self.check_required(required) if expand_vars: self.expand_vars() def check_required(self, required): for name in required: if not name in self: raise Exception("Missing required configuration: %s" % name)
class ConfigProcessor(object): def __init__(self, logger, path, args_files, args_data, req_files=None, req_keys=None): self.log = logger self.conf_path = path req_files = req_files or [] req_keys = req_keys or [] self.conf_keys = req_keys self.required_conf_files = [ os.path.join(self.conf_path, cf) for cf in req_files ] self.conf_files = [] if args_files is not None: for cf in args_files: if not os.path.isabs(cf): cf = os.path.join(os.getcwd(), cf) self.conf_files += [cf] self.conf_data = args_data or [] self.conf_builder = ConfigBuilder() def load_files(self): conf_files = self.required_conf_files user_conf_file = os.path.join(self.conf_path, "user.conf") if os.path.exists(user_conf_file): conf_files += [user_conf_file] conf_files += self.conf_files # Check that configuration files exist missing_conf_files = [ cf for cf in conf_files if not os.path.exists(cf) ] if len(missing_conf_files) > 0: self.log.error("Configuration files not found:\n{}".format( "\n".join(" {}".format(cf) for cf in missing_conf_files))) exit(-1) for cf in conf_files: self.conf_builder.add_file(cf) return self.conf_builder def parse_data(self): for data in self.conf_data: try: pos = data.index("=") key = data[0:pos] value = data[pos + 1:] try: v = json.loads(value) except: v = value self.conf_builder.add_value(key, v) except: raise Exception( "Wrong configuration data: KEY=VALUE expected but found '{}'" .format(data)) return self.conf_builder def load(self): self.load_files() self.parse_data() def validated_conf(self, expand_vars=True): self.conf = self.conf_builder.get_conf() if expand_vars: self.conf.expand_vars() #self.log.debug(repr(self.conf)) # Validate that required keys exist mk = self.conf.missing_keys(self.conf_keys) if len(mk) > 0: sb = [ "The following configuration parameters were not found:\n", "\n".join("* " + k for k in mk), "\nin any of the following configuration files:\n", "\n".join("* " + k for k in conf_files) ] self.log.error("".join(sb)) exit(-1) return self.conf def log_debug(self): if len(self.conf_files) > 0: self.log.debug("User defined configuration files:\n{}".format( "\n".join(" {}".format(cf) for cf in self.conf_files))) self.log.debug("Effective configuration: " + str(self.conf))