def setup_config(config_file=None, custom_config_file=None, **kwargs):
"""Helper function to setup config at startup"""
if config_file:
config = get_config()
config.add_config_path(config_file)
backend.__global_config__ = update(backend.__global_config__, vars(config)["_sections"])
if custom_config_file:
custom_config = get_custom_config()
custom_config.add_config_path(custom_config_file)
backend.__global_config__ = update(backend.__global_config__, vars(custom_config)["_sections"])
def setup_config(config_file=None, custom_config_file=None, **kwargs):
"""Helper function to setup config at startup"""
if config_file:
config = get_config()
config.add_config_path(config_file)
backend.__global_config__ = update(backend.__global_config__,
vars(config)["_sections"])
if custom_config_file:
custom_config = get_custom_config()
custom_config.add_config_path(custom_config_file)
backend.__global_config__ = update(backend.__global_config__,
vars(custom_config)["_sections"])
def _update_config(self, config, param_values_dict, disable_parent_task_update=False, *args, **kwargs):
"""Update configuration for this task. All task options should
have a default. Order of preference:
1. if command line option encountered, override all config file settings
2. if custom config file setting, override config and default
3. if config file, override default
4. default value
:param config: configuration instance
:param param_values_dict: task parameter dict
:param disable_parent_task_update: disable parent task update for custom configurations (best practice pipeline execution order should stay fixed)
:returns: an updated parameter list for the task.
"""
# Set section to module name and subsection to class name
# unless _config_section and _config_subsection set. The
# latter are needed for classes that live outside their
# namespace, e.g. subclasses in pipelines
_section = self.__module__
try:
_subsection = self.__class__.__name__
except:
_subsection = None
if self._config_section:
_section = self._config_section
if not config:
return kwargs
if not config.has_section(_section):
return kwargs
if not _subsection:
d = {_section:param_values_dict}
else:
d = {_section:{_subsection:param_values_dict}}
backend.__global_config__ = update(backend.__global_config__, d)
for key, value in self.get_params():
new_value = None
if config.has_key(_section, key):
new_value = config.get(_section, key)
if config.has_section(_section, _subsection):
if config.has_key(_section, key, _subsection):
new_value = config.get(_section, key, _subsection)
logger.debug("Reading config file, setting '{0}' to '{1}' for task class '{2}'".format(key, new_value, self.__class__))
if new_value:
if key == "parent_task" and disable_parent_task_update:
logger.debug("disable_parent_task_update set; not updating '{0}' for task class '{1}'".format(key, self.__class__))
else:
kwargs[key] = new_value
logger.debug("Updating config, setting '{0}' to '{1}' for task class '{2}'".format(key, new_value, self.__class__))
else:
logger.debug("Using default value '{0}' for '{1}' for task class '{2}'".format(value.default, key, self.__class__))
pass
return kwargs
def options(self, section, subsection=None):
try:
opts = self._sections[section].copy()
except KeyError:
raise NoSectionError(section)
if subsection:
try:
opts = opts[subsection].copy()
except KeyError:
raise NoSectionError(subsection)
opts = update(opts, self._defaults)
if '__name__' in opts:
del opts['__name__']
return opts.keys()
def options(self, section, subsection=None):
try:
opts = self._sections[section].copy()
except KeyError:
raise NoSectionError(section)
if subsection:
try:
opts = opts[subsection].copy()
except KeyError:
raise NoSectionError(subsection)
opts = update(opts, self._defaults)
if '__name__' in opts:
del opts['__name__']
return opts.keys()
def read(self, file_path):
"""
Read config file.
:param file_path: The file system path to the configuration file.
:returns: boolean
"""
try:
with open(file_path) as fp:
_sections = yaml.load(fp)
if _sections is None:
_sections = {}
self._sections = update(self._sections, _sections)
except IOError:
return False
return True
def read(self, file_paths):
"""
Read config files.
:param file_path: The file system path to the configuration file.
:returns: boolean
"""
for path in file_paths:
try:
with open(path) as fp:
_sections = yaml.load(fp)
if _sections is None:
_sections = {}
self._sections = update(self._sections, _sections)
except IOError:
logging.warn("No such file {}".format(path))
return False
return True
def read(self, file_paths):
"""
Read config files.
:param file_path: The file system path to the configuration file.
:returns: boolean
"""
for path in file_paths:
try:
with open(path) as fp:
_sections = yaml.load(fp)
if _sections is None:
_sections = {}
self._sections = update(self._sections, _sections)
except IOError:
logging.warn("No such file {}".format(path))
return False
return True
def _update_config(self,
config,
param_values_dict,
disable_parent_task_update=False,
*args,
**kwargs):
"""Update configuration for this task. All task options should
have a default. Order of preference:
1. if command line option encountered, override all config file settings
2. if custom config file setting, override config and default
3. if config file, override default
4. default value
:param config: configuration instance
:param param_values_dict: task parameter dict
:param disable_parent_task_update: disable parent task update for custom configurations (best practice pipeline execution order should stay fixed)
:returns: an updated parameter list for the task.
"""
# Set section to module name and subsection to class name
# unless _config_section and _config_subsection set. The
# latter are needed for classes that live outside their
# namespace, e.g. subclasses in pipelines
_section = self.__module__
try:
_subsection = self.__class__.__name__
except:
_subsection = None
if self._config_section:
_section = self._config_section
if not config:
return kwargs
if not config.has_section(_section):
return kwargs
if not _subsection:
d = {_section: param_values_dict}
else:
d = {_section: {_subsection: param_values_dict}}
backend.__global_config__ = update(backend.__global_config__, d)
for key, value in self.get_params():
new_value = None
if config.has_key(_section, key):
new_value = config.get(_section, key)
if config.has_section(_section, _subsection):
if config.has_key(_section, key, _subsection):
new_value = config.get(_section, key, _subsection)
logger.debug(
"Reading config file, setting '{0}' to '{1}' for task class '{2}'"
.format(key, new_value, self.__class__))
if new_value:
if key == "parent_task" and disable_parent_task_update:
logger.debug(
"disable_parent_task_update set; not updating '{0}' for task class '{1}'"
.format(key, self.__class__))
else:
kwargs[key] = new_value
logger.debug(
"Updating config, setting '{0}' to '{1}' for task class '{2}'"
.format(key, new_value, self.__class__))
else:
logger.debug(
"Using default value '{0}' for '{1}' for task class '{2}'".
format(value.default, key, self.__class__))
pass
return kwargs
