def __init__(self, filler=None, config_file_path=None, load_old_config=True, path_base=None):
def _get_name(prefix=''):
if config_file_path:
return ('%s.%s' % (prefix, get_file_name(config_file_path))).strip('.')
elif prefix:
return prefix
return 'unnamed'
try:
config_dn = os.getcwd()
except Exception:
raise ConfigError('The current directory does not exist!')
if config_file_path:
config_dn = os.path.dirname(resolve_path(config_file_path,
search_path_list=[os.getcwd()], exception_type=ConfigError))
if path_base:
config_dn = os.path.dirname(resolve_path(path_base, search_path_list=[os.getcwd()]))
# Init config containers
self._container_cur = ConfigContainer('current')
if filler: # Read in the current configuration ...
filler.fill(self._container_cur)
self._container_cur.resolve() # resolve interpolations
logging.getLogger('config.stored').propagate = False
container_old = ConfigContainer('stored')
container_old.enabled = False
# Create config view and temporary config interface
self._view = SimpleConfigView(_get_name(), container_old, self._container_cur)
self._view.config_vault['path:search'] = UniqueList([os.getcwd(), config_dn])
# Determine work directory using config interface with "global" scope
tmp_config = SimpleConfigInterface(self._view.get_view(set_sections=['global']))
work_dn_base = tmp_config.get_dn('workdir base', config_dn, must_exist=False)
work_dn_default = os.path.join(work_dn_base, _get_name('work'))
work_dn = tmp_config.get_dn('workdir', work_dn_default, must_exist=False)
self._view.config_vault['path:work_dn'] = work_dn # tmp_config still has undefinied
# Set dynamic plugin search path
sys.path.extend(tmp_config.get_dn_list('plugin paths', [os.getcwd()]))
# Determine and load stored config settings
self._config_path_min = os.path.join(work_dn, 'current.conf') # Minimal config file
self._config_path_old = os.path.join(work_dn, 'work.conf') # Config file with saved settings
if load_old_config:
if os.path.exists(self._config_path_old):
GeneralFileConfigFiller([self._config_path_old]).fill(container_old)
old_setting_file = os.path.join(work_dn, 'task.dat')
if os.path.exists(old_setting_file):
ConfigFiller.create_instance('CompatConfigFiller', old_setting_file).fill(container_old)
container_old.enabled = True
container_old.protect()
# Get persistent variables - only possible after container_old was enabled
self._view.set_config_name(tmp_config.get('config id', _get_name(), persistent=True))
def __init__(self, config_fn_list):
from grid_control_settings import Settings
for config_fn in config_fn_list:
exec_wrapper(
SafeFile(resolve_path(config_fn, ['.'])).read_close(),
{'Settings': Settings})
DictConfigFiller.__init__(self, Settings.get_config_dict())
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def resolve_path(self, value, must_exist, error_msg):
# Resolve path
try:
return resolve_path(value, self._config_view.config_vault.get('path:search', []),
must_exist, ConfigError)
except Exception:
raise ConfigError(error_msg)
def _fill_content_deep(self, config_fn, search_path_list, content_configfile):
log = logging.getLogger(('config.%s' % get_file_name(config_fn)).rstrip('.').lower())
log.log(logging.INFO1, 'Reading config file %s', config_fn)
config_fn = resolve_path(config_fn, search_path_list, exception_type=ConfigError)
config_str_list = list(SafeFile(config_fn).iter_close())
# Single pass, non-recursive list retrieval
tmp_content_configfile = {}
self._fill_content_shallow(config_fn, config_str_list,
search_path_list, tmp_content_configfile)
def _get_list_shallow(section, option):
for (opt, value, _) in tmp_content_configfile.get(section, []):
if opt == option:
for entry in parse_list(value, None):
yield entry
search_path_list_new = [os.path.dirname(config_fn)]
# Add entries from include statement recursively
for include_fn in _get_list_shallow('global', 'include'):
self._fill_content_deep(include_fn, search_path_list + search_path_list_new, content_configfile)
# Process all other entries in current file
self._fill_content_shallow(config_fn, config_str_list, search_path_list, content_configfile)
# Override entries in current config file
for override_fn in _get_list_shallow('global', 'include override'):
self._fill_content_deep(override_fn, search_path_list + search_path_list_new, content_configfile)
# Filter special global options
if content_configfile.get('global', []):
def _ignore_includes(opt_v_s_tuple):
return opt_v_s_tuple[0] not in ['include', 'include override']
content_configfile['global'] = lfilter(_ignore_includes, content_configfile['global'])
return search_path_list + search_path_list_new
def __init__(self):
# Collect host / user / installation specific config files
def _resolve_hostname():
import socket
host = socket.gethostname()
return ignore_exception(Exception, host, lambda: socket.gethostbyaddr(host)[0])
try:
hostname = hang_protection(_resolve_hostname, timeout=5)
except TimeoutException:
clear_current_exception()
hostname = None
logging.getLogger('console').warning('System call to resolve hostname is hanging!')
def _get_default_config_fn_iter(): # return possible default config files
if hostname: # host / domain specific
for part_idx in irange(hostname.count('.') + 1, -1, -1):
yield get_path_pkg('../config/%s.conf' % hostname.split('.', part_idx)[-1])
yield '/etc/grid-control.conf' # system specific
yield '~/.grid-control.conf' # user specific
yield get_path_pkg('../config/default.conf') # installation specific
if os.environ.get('GC_CONFIG'):
yield '$GC_CONFIG' # environment specific
config_fn_list = list(_get_default_config_fn_iter())
log = logging.getLogger('config.sources.default')
log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', config_fn_list))
config_fn_iter = imap(lambda fn: resolve_path(fn, must_exist=False), config_fn_list)
FileConfigFiller.__init__(self, lfilter(os.path.exists, config_fn_iter), add_search_path=False)
def _get_sandbox_file_list(self, task, sm_list):
# Prepare all input files
dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list)))
dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep,
lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list)
task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(),
*imap(lambda x: x.get_task_dict(), [task] + sm_list))
task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list),
'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name})
task_config_str_list = DictFormat(escape_strings=True).format(
task_config_dict, format='export %s%s%s\n')
vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(),
self._remote_event_handler.get_mon_env_dict().keys()))
vn_alias_dict.update(task.get_var_alias_map())
vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n')
# Resolve wildcards in task input files
def _get_task_fn_list():
for fpi in task.get_sb_in_fpi_list():
matched = glob.glob(fpi.path_abs)
if matched != []:
for match in matched:
yield match
else:
yield fpi.path_abs
return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [
VirtualFile('_config.sh', sorted(task_config_str_list)),
VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def resolve_path(self, value, must_exist, error_msg):
# Resolve path
try:
return resolve_path(
value, self._config_view.config_vault.get('path:search', []),
must_exist, ConfigError)
except Exception:
raise ConfigError(error_msg)
def _fill_content_deep(self, config_fn, search_path_list,
content_configfile):
log = logging.getLogger(
('config.%s' % get_file_name(config_fn)).rstrip('.').lower())
log.log(logging.INFO1, 'Reading config file %s', config_fn)
config_fn = resolve_path(config_fn,
search_path_list,
exception_type=ConfigError)
config_str_list = list(SafeFile(config_fn).iter_close())
# Single pass, non-recursive list retrieval
tmp_content_configfile = {}
self._fill_content_shallow(config_fn, config_str_list,
search_path_list, tmp_content_configfile)
def _get_list_shallow(section, option):
for (opt, value, _) in tmp_content_configfile.get(section, []):
if opt == option:
for entry in parse_list(value, None):
yield entry
search_path_list_new = [os.path.dirname(config_fn)]
# Add entries from include statement recursively
for include_fn in _get_list_shallow('global', 'include'):
self._fill_content_deep(include_fn,
search_path_list + search_path_list_new,
content_configfile)
# Process all other entries in current file
self._fill_content_shallow(config_fn, config_str_list,
search_path_list, content_configfile)
# Override entries in current config file
for override_fn in _get_list_shallow('global', 'include override'):
self._fill_content_deep(override_fn,
search_path_list + search_path_list_new,
content_configfile)
# Filter special global options
if content_configfile.get('global', []):
def _ignore_includes(opt_v_s_tuple):
return opt_v_s_tuple[0] not in ['include', 'include override']
content_configfile['global'] = lfilter(
_ignore_includes, content_configfile['global'])
return search_path_list + search_path_list_new
def __init__(self):
# Collect host / user / installation specific config files
def _resolve_hostname():
import socket
host = socket.gethostname()
return ignore_exception(Exception, host,
lambda: socket.gethostbyaddr(host)[0])
try:
hostname = hang_protection(_resolve_hostname, timeout=5)
except TimeoutException:
clear_current_exception()
hostname = None
logging.getLogger('console').warning(
'System call to resolve hostname is hanging!')
def _get_default_config_fn_iter(
): # return possible default config files
if hostname: # host / domain specific
for part_idx in irange(hostname.count('.') + 1, -1, -1):
yield get_path_pkg('../config/%s.conf' %
hostname.split('.', part_idx)[-1])
yield '/etc/grid-control.conf' # system specific
yield '~/.grid-control.conf' # user specific
yield get_path_pkg(
'../config/default.conf') # installation specific
if os.environ.get('GC_CONFIG'):
yield '$GC_CONFIG' # environment specific
config_fn_list = list(_get_default_config_fn_iter())
log = logging.getLogger('config.sources.default')
log.log(logging.DEBUG1, 'Possible default config files: %s',
str.join(', ', config_fn_list))
config_fn_iter = imap(lambda fn: resolve_path(fn, must_exist=False),
config_fn_list)
FileConfigFiller.__init__(self,
lfilter(os.path.exists, config_fn_iter),
add_search_path=False)
def __init__(self, config_fn_list):
from grid_control_settings import Settings
for config_fn in config_fn_list:
exec_wrapper(SafeFile(resolve_path(config_fn, ['.'])).read_close(), {'Settings': Settings})
DictConfigFiller.__init__(self, Settings.get_config_dict())