class Config(ConfigParser):
"""
Custom law configuration parser with a few additions on top of the standard python
``ConfigParser``. Most notably, this class adds config *inheritance* via :py:meth:`update` and
:py:meth:`include`, as well as a mechanism to synchronize with the luigi configuration parser.
When *config_file* is set, it is loaded during setup. When empty, and *skip_fallbacks* is
*False*, the default config file locations defined in :py:attr:`_config_files` are checked. By
default, the default configuration :py:attr:`_default_config` is loaded, which can be prevented
by setting *skip_defaults* to *True*.
.. py:classattribute:: _instance
type: Config
Global instance of this class.
.. py:classattribute:: _default_config
type: dict
Default configuration.
.. py:classattribute:: _config_files
type: list
List of configuration files that are checked during setup (unless *skip_fallbacks* is
*True*). When a file exists, the check is stopped. Therefore, the order is important here.
"""
_instance = None
_default_config = {
"core": {
"index_file":
os.getenv("LAW_INDEX_FILE", law_home_path("index")),
"software_dir":
law_home_path("software"),
"inherit_configs":
"",
"extend_configs":
"",
"sync_luigi_config":
check_bool_flag(os.getenv("LAW_SYNC_LUIGI_CONFIG", "yes")),
},
"logging": {
"law": os.getenv("LAW_LOG_LEVEL", "WARNING"),
},
"target": {
"tmp_dir": os.getenv("LAW_TARGET_TMP_DIR", tempfile.gettempdir()),
"tmp_dir_permission": 0o0770,
"gfal2_log_level": "WARNING",
# contrib
"default_dropbox_fs": "dropbox_fs",
"default_wlcg_fs": "wlcg_fs",
},
"job": {
"job_file_dir": os.getenv("LAW_JOB_FILE_DIR",
tempfile.gettempdir()),
"job_file_dir_mkdtemp": True,
"job_file_dir_cleanup": True,
# contrib
# the three options above can be also be set per workflow type (currently htcondor,
# lsf, glite, arc) by prefixing the option, e.g. "htcondor_job_file_dir"
},
"modules": {},
"bash_env": {},
"docker": {
"forward_dir": "/law_forward",
"python_dir": "py",
"bin_dir": "bin",
"stagein_dir": "stagein",
"stageout_dir": "stageout",
},
"docker_env": {},
"docker_volumes": {},
"singularity": {
"forward_dir": "/law_forward",
"python_dir": "py",
"bin_dir": "bin",
"stagein_dir": "stagein",
"stageout_dir": "stageout",
},
"singularity_env": {},
"singularity_volumes": {},
"notifications": {
"mail_recipient": "",
"mail_sender": "",
"mail_smtp_host": "127.0.0.1",
"mail_smtp_port": 25,
# contrib
"slack_token": "",
"slack_channel": "",
"slack_mention_user": "",
"telegram_token": "",
"telegram_chat": "",
"telegram_mention_user": "",
},
}
_config_files = [
"$LAW_CONFIG_FILE", "law.cfg",
law_home_path("config"), "etc/law/config"
]
@classmethod
def instance(cls, *args, **kwargs):
"""
Creates an instance of this class with all *args* and *kwargs*, saves it in
:py:attr:`_instance`, and returns it. When :py:attr:`_instance` was already set before, no
new instance is created.
"""
if cls._instance is None:
cls._instance = cls(*args, **kwargs)
return cls._instance
def __init__(self,
config_file="",
skip_defaults=False,
skip_fallbacks=False):
ConfigParser.__init__(self, allow_no_value=True)
self.config_file = None
# load defaults
if not skip_defaults:
self.update(self._default_config)
# read from files
files = [config_file]
if not skip_fallbacks:
files += self._config_files
for f in files:
f = os.path.expandvars(os.path.expanduser(f))
f = os.path.normpath(os.path.abspath(f))
if os.path.isfile(f):
self.read(f)
self.config_file = f
logger.debug("config instance created from '{}'".format(f))
break
else:
logger.debug("config instance created without a file")
# inherit from and/or extend by other configs
for option, overwrite_options in [("include_configs", False),
("extend_configs", True)]:
for filename in self.get_default("core", option, "").split(","):
filename = filename.strip()
if filename:
# resolve filename relative to the main config file
if self.config_file:
basedir = os.path.dirname(self.config_file)
filename = os.path.normpath(
os.path.join(basedir, filename))
self.include(filename, overwrite_options=overwrite_options)
# sync with luigi configuration
if self.getboolean("core", "sync_luigi_config"):
self.sync_luigi_config()
def _convert_to_boolean(self, value):
# py2 backport
if six.PY3:
return super(Config, self)._convert_to_boolean(value)
else:
if value.lower() not in self._boolean_states:
raise ValueError("Not a boolean: {}".format(value))
return self._boolean_states[value.lower()]
def _get_type_converter(self, type):
if type in (str, "str"):
return str
if type in (int, "int"):
return int
elif type in (float, "float"):
return float
elif type in (bool, "bool", "boolean"):
return self._convert_to_boolean
else:
raise ValueError(
"unknown 'type' argument ({}), must be 'str', 'int', 'float', or "
"'bool'".format(type))
def optionxform(self, option):
""""""
return option
def get_default(self,
section,
option,
default=None,
type=None,
expandvars=False,
expanduser=False):
"""
Returns the config value defined by *section* and *option*. When either the section or the
option does not exist, the *default* value is returned instead. When *type* is set, it must
be either `"str"`, `"int"`, `"float"`, or `"boolean"`. When *expandvars* is *True*,
environment variables are expanded. When *expanduser* is *True*, user variables are
expanded as well.
"""
if self.has_section(section) and self.has_option(section, option):
value = self.get(section, option)
if isinstance(value, six.string_types):
if expandvars:
value = os.path.expandvars(value)
if expanduser:
value = os.path.expanduser(value)
return value if not type else self._get_type_converter(type)(value)
else:
return default
def get_expanded(self, *args, **kwargs):
"""
Same as :py:meth:`get_default`, but *expandvars* and *expanduser* arguments are set to
*True* by default.
"""
kwargs.setdefault("expandvars", True)
kwargs.setdefault("expanduser", True)
return self.get_default(*args, **kwargs)
def update(self,
data,
overwrite=None,
overwrite_sections=True,
overwrite_options=True):
"""
Updates the currently stored configuration with new *data*, given as a dictionary. When
*overwrite_sections* is *False*, sections in *data* that are already present in the current
config are skipped. When *overwrite_options* is *False*, existing options are not
overwritten. When *overwrite* is not *None*, both *overwrite_sections* and
*overwrite_options* are set to its value.
"""
if overwrite is not None:
overwrite_sections = overwrite
overwrite_options = overwrite
for section, _data in six.iteritems(data):
if not self.has_section(section):
self.add_section(section)
elif not overwrite_sections:
continue
for option, value in six.iteritems(_data):
if overwrite_options or not self.has_option(section, option):
self.set(section, option, str(value))
def include(self, filename, *args, **kwargs):
"""
Updates the current configc with the config found in *filename*. All *args* and *kwargs* are
forwarded to :py:meth:`update`.
"""
p = self.__class__(filename, skip_defaults=True, skip_fallbacks=True)
self.update(p._sections, *args, **kwargs)
def keys(self, section, prefix=None):
"""
Returns all keys of a *section* in a list. When *prefix* is set, only keys starting with
that prefix are returned
"""
return [
key for key, _ in self.items(section)
if (not prefix or key.startswith(prefix))
]
def sync_luigi_config(self, push=True, pull=True, expand=True):
"""
Synchronizes sections starting with ``"luigi_"`` with the luigi configuration parser. First,
when *push* is *True*, options that exist in law but **not** in luigi are stored as defaults
in the luigi config. Then, when *pull* is *True*, all luigi-related options in the law
config are overwritten with those from luigi. This way, options set via luigi defaults
(environment variables, global configuration files, `LUIGI_CONFIG_PATH`) always have
precendence. When *expand* is *True*, environment variables are expanded before pushing them
to the luigi config.
"""
prefix = "luigi_"
lparser = luigi.configuration.LuigiConfigParser.instance()
if push:
for section in self.sections():
if not section.startswith(prefix):
continue
lsection = section[len(prefix):]
if not lparser.has_section(lsection):
lparser.add_section(lsection)
for option in self.options(section):
if not lparser.has_option(lsection, option):
if expand:
value = self.get_expanded(section, option)
else:
value = self.get(section, option)
lparser.set(lsection, option, value)
if pull:
for lsection in lparser.sections():
section = prefix + lsection
if not self.has_section(section):
self.add_section(section)
for option, value in lparser.items(lsection):
self.set(section, option, value)
class Config(ConfigParser):
_instance = None
_default_config = {
"core": {
"db_file": os.getenv("LAW_DB_FILE", law_home_path("db")),
"software_dir": law_home_path("software"),
"inherit_configs": "",
"extend_configs": "",
},
"logging": {
"law": os.getenv("LAW_LOG_LEVEL", "WARNING"),
},
"target": {
"tmp_dir": os.getenv("LAW_TARGET_TMP_DIR", tempfile.gettempdir()),
"tmp_dir_permission": 0o0770,
"gfal2_log_level": "WARNING",
# contrib
"default_dropbox_fs": "dropbox_fs",
"default_wlcg_fs": "wlcg_fs",
},
"job": {
"job_file_dir": tempfile.gettempdir(),
},
"modules": {},
"bash_env": {},
"docker": {
"forward_dir": "/law_forward",
"python_dir": "py",
"bin_dir": "bin",
"stagein_dir": "stagein",
"stageout_dir": "stageout",
},
"docker_env": {},
"docker_volumes": {},
"singularity": {
"forward_dir": "/law_forward",
"python_dir": "py",
"bin_dir": "bin",
"stagein_dir": "stagein",
"stageout_dir": "stageout",
},
"singularity_env": {},
"singularity_volumes": {},
}
_config_files = [
"$LAW_CONFIG_FILE", "law.cfg",
law_home_path("config"), "etc/law/config"
]
@classmethod
def instance(cls, config_file=""):
if cls._instance is None:
cls._instance = cls(config_file=config_file)
return cls._instance
def __init__(self,
config_file="",
skip_defaults=False,
skip_fallbacks=False):
ConfigParser.__init__(self, allow_no_value=True)
self.config_file = None
# load defaults
if not skip_defaults:
self.update(self._default_config)
# read from files
files = [config_file]
if not skip_fallbacks:
files += self._config_files
for f in files:
f = os.path.expandvars(os.path.expanduser(f))
f = os.path.normpath(os.path.abspath(f))
if os.path.isfile(f):
self.read(f)
self.config_file = f
logger.debug("config instance created from '{}'".format(f))
break
else:
logger.debug("config instance created without a file")
# inherit from and/or extend by other configs
for option, overwrite_options in [("include_configs", False),
("extend_configs", True)]:
for filename in self.get_default("core", option, "").split(","):
filename = filename.strip()
if filename:
# resolve filename relative to the main config file
if self.config_file:
basedir = os.path.dirname(self.config_file)
filename = os.path.normpath(
os.path.join(basedir, filename))
self.include(filename, overwrite_options=overwrite_options)
def optionxform(self, option):
return option
def get_default(self, section, option, default=None):
if self.has_section(section) and self.has_option(section, option):
return self.get(section, option)
else:
return default
def get_expanded(self, section, option, default=None):
value = self.get_default(section, option, default=default)
if isinstance(value, six.string_types):
value = os.path.expandvars(os.path.expanduser(value))
return value
def update(self,
data,
overwrite=None,
overwrite_sections=True,
overwrite_options=True):
if overwrite is not None:
overwrite_sections = overwrite
overwrite_options = overwrite
for section, _data in six.iteritems(data):
if not self.has_section(section):
self.add_section(section)
elif not overwrite_sections:
continue
for option, value in six.iteritems(_data):
if overwrite_options or not self.has_option(section, option):
self.set(section, option, str(value))
def include(self, filename, *args, **kwargs):
p = self.__class__(filename, skip_defaults=True, skip_fallbacks=True)
self.update(p._sections, *args, **kwargs)
def keys(self, section):
return [key for key, _ in self.items(section)]