def __init__(self, filler=None, config_file_path=None, load_old_config=True, path_base=None): def _get_name(prefix=''): if config_file_path: return ('%s.%s' % (prefix, get_file_name(config_file_path))).strip('.') elif prefix: return prefix return 'unnamed' try: config_dn = os.getcwd() except Exception: raise ConfigError('The current directory does not exist!') if config_file_path: config_dn = os.path.dirname(resolve_path(config_file_path, search_path_list=[os.getcwd()], exception_type=ConfigError)) if path_base: config_dn = os.path.dirname(resolve_path(path_base, search_path_list=[os.getcwd()])) # Init config containers self._container_cur = ConfigContainer('current') if filler: # Read in the current configuration ... filler.fill(self._container_cur) self._container_cur.resolve() # resolve interpolations logging.getLogger('config.stored').propagate = False container_old = ConfigContainer('stored') container_old.enabled = False # Create config view and temporary config interface self._view = SimpleConfigView(_get_name(), container_old, self._container_cur) self._view.config_vault['path:search'] = UniqueList([os.getcwd(), config_dn]) # Determine work directory using config interface with "global" scope tmp_config = SimpleConfigInterface(self._view.get_view(set_sections=['global'])) work_dn_base = tmp_config.get_dn('workdir base', config_dn, must_exist=False) work_dn_default = os.path.join(work_dn_base, _get_name('work')) work_dn = tmp_config.get_dn('workdir', work_dn_default, must_exist=False) self._view.config_vault['path:work_dn'] = work_dn # tmp_config still has undefinied # Set dynamic plugin search path sys.path.extend(tmp_config.get_dn_list('plugin paths', [os.getcwd()])) # Determine and load stored config settings self._config_path_min = os.path.join(work_dn, 'current.conf') # Minimal config file self._config_path_old = os.path.join(work_dn, 'work.conf') # Config file with saved settings if load_old_config: if os.path.exists(self._config_path_old): GeneralFileConfigFiller([self._config_path_old]).fill(container_old) old_setting_file = os.path.join(work_dn, 'task.dat') if os.path.exists(old_setting_file): ConfigFiller.create_instance('CompatConfigFiller', old_setting_file).fill(container_old) container_old.enabled = True container_old.protect() # Get persistent variables - only possible after container_old was enabled self._view.set_config_name(tmp_config.get('config id', _get_name(), persistent=True))
def __init__(self, config_fn_list): from grid_control_settings import Settings for config_fn in config_fn_list: exec_wrapper( SafeFile(resolve_path(config_fn, ['.'])).read_close(), {'Settings': Settings}) DictConfigFiller.__init__(self, Settings.get_config_dict())
def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def resolve_path(self, value, must_exist, error_msg): # Resolve path try: return resolve_path(value, self._config_view.config_vault.get('path:search', []), must_exist, ConfigError) except Exception: raise ConfigError(error_msg)
def _fill_content_deep(self, config_fn, search_path_list, content_configfile): log = logging.getLogger(('config.%s' % get_file_name(config_fn)).rstrip('.').lower()) log.log(logging.INFO1, 'Reading config file %s', config_fn) config_fn = resolve_path(config_fn, search_path_list, exception_type=ConfigError) config_str_list = list(SafeFile(config_fn).iter_close()) # Single pass, non-recursive list retrieval tmp_content_configfile = {} self._fill_content_shallow(config_fn, config_str_list, search_path_list, tmp_content_configfile) def _get_list_shallow(section, option): for (opt, value, _) in tmp_content_configfile.get(section, []): if opt == option: for entry in parse_list(value, None): yield entry search_path_list_new = [os.path.dirname(config_fn)] # Add entries from include statement recursively for include_fn in _get_list_shallow('global', 'include'): self._fill_content_deep(include_fn, search_path_list + search_path_list_new, content_configfile) # Process all other entries in current file self._fill_content_shallow(config_fn, config_str_list, search_path_list, content_configfile) # Override entries in current config file for override_fn in _get_list_shallow('global', 'include override'): self._fill_content_deep(override_fn, search_path_list + search_path_list_new, content_configfile) # Filter special global options if content_configfile.get('global', []): def _ignore_includes(opt_v_s_tuple): return opt_v_s_tuple[0] not in ['include', 'include override'] content_configfile['global'] = lfilter(_ignore_includes, content_configfile['global']) return search_path_list + search_path_list_new
def __init__(self): # Collect host / user / installation specific config files def _resolve_hostname(): import socket host = socket.gethostname() return ignore_exception(Exception, host, lambda: socket.gethostbyaddr(host)[0]) try: hostname = hang_protection(_resolve_hostname, timeout=5) except TimeoutException: clear_current_exception() hostname = None logging.getLogger('console').warning('System call to resolve hostname is hanging!') def _get_default_config_fn_iter(): # return possible default config files if hostname: # host / domain specific for part_idx in irange(hostname.count('.') + 1, -1, -1): yield get_path_pkg('../config/%s.conf' % hostname.split('.', part_idx)[-1]) yield '/etc/grid-control.conf' # system specific yield '~/.grid-control.conf' # user specific yield get_path_pkg('../config/default.conf') # installation specific if os.environ.get('GC_CONFIG'): yield '$GC_CONFIG' # environment specific config_fn_list = list(_get_default_config_fn_iter()) log = logging.getLogger('config.sources.default') log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', config_fn_list)) config_fn_iter = imap(lambda fn: resolve_path(fn, must_exist=False), config_fn_list) FileConfigFiller.__init__(self, lfilter(os.path.exists, config_fn_iter), add_search_path=False)
def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def resolve_path(self, value, must_exist, error_msg): # Resolve path try: return resolve_path( value, self._config_view.config_vault.get('path:search', []), must_exist, ConfigError) except Exception: raise ConfigError(error_msg)
def _fill_content_deep(self, config_fn, search_path_list, content_configfile): log = logging.getLogger( ('config.%s' % get_file_name(config_fn)).rstrip('.').lower()) log.log(logging.INFO1, 'Reading config file %s', config_fn) config_fn = resolve_path(config_fn, search_path_list, exception_type=ConfigError) config_str_list = list(SafeFile(config_fn).iter_close()) # Single pass, non-recursive list retrieval tmp_content_configfile = {} self._fill_content_shallow(config_fn, config_str_list, search_path_list, tmp_content_configfile) def _get_list_shallow(section, option): for (opt, value, _) in tmp_content_configfile.get(section, []): if opt == option: for entry in parse_list(value, None): yield entry search_path_list_new = [os.path.dirname(config_fn)] # Add entries from include statement recursively for include_fn in _get_list_shallow('global', 'include'): self._fill_content_deep(include_fn, search_path_list + search_path_list_new, content_configfile) # Process all other entries in current file self._fill_content_shallow(config_fn, config_str_list, search_path_list, content_configfile) # Override entries in current config file for override_fn in _get_list_shallow('global', 'include override'): self._fill_content_deep(override_fn, search_path_list + search_path_list_new, content_configfile) # Filter special global options if content_configfile.get('global', []): def _ignore_includes(opt_v_s_tuple): return opt_v_s_tuple[0] not in ['include', 'include override'] content_configfile['global'] = lfilter( _ignore_includes, content_configfile['global']) return search_path_list + search_path_list_new
def __init__(self): # Collect host / user / installation specific config files def _resolve_hostname(): import socket host = socket.gethostname() return ignore_exception(Exception, host, lambda: socket.gethostbyaddr(host)[0]) try: hostname = hang_protection(_resolve_hostname, timeout=5) except TimeoutException: clear_current_exception() hostname = None logging.getLogger('console').warning( 'System call to resolve hostname is hanging!') def _get_default_config_fn_iter( ): # return possible default config files if hostname: # host / domain specific for part_idx in irange(hostname.count('.') + 1, -1, -1): yield get_path_pkg('../config/%s.conf' % hostname.split('.', part_idx)[-1]) yield '/etc/grid-control.conf' # system specific yield '~/.grid-control.conf' # user specific yield get_path_pkg( '../config/default.conf') # installation specific if os.environ.get('GC_CONFIG'): yield '$GC_CONFIG' # environment specific config_fn_list = list(_get_default_config_fn_iter()) log = logging.getLogger('config.sources.default') log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', config_fn_list)) config_fn_iter = imap(lambda fn: resolve_path(fn, must_exist=False), config_fn_list) FileConfigFiller.__init__(self, lfilter(os.path.exists, config_fn_iter), add_search_path=False)
def __init__(self, config_fn_list): from grid_control_settings import Settings for config_fn in config_fn_list: exec_wrapper(SafeFile(resolve_path(config_fn, ['.'])).read_close(), {'Settings': Settings}) DictConfigFiller.__init__(self, Settings.get_config_dict())