def resolvePath(self, value, mustExist, errorMsg): try: return utils.resolvePath( value, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError) except Exception: raise ConfigError(errorMsg)
def _fillContentWithIncludes(self, configFile, searchPaths, configContent): log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.').lower()) log.log(logging.INFO1, 'Reading config file %s', configFile) configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError) configFileLines = SafeFile(configFile).readlines() # Single pass, non-recursive list retrieval tmpConfigContent = {} self._fillContentSingleFile(configFile, configFileLines, searchPaths, tmpConfigContent) def getFlatList(section, option): for (opt, value, src) in tmpConfigContent.get(section, []): try: if opt == option: for entry in parseList(value, None): yield entry except Exception: raise ConfigError('Unable to parse [%s] %s from %s' % (section, option, src)) newSearchPaths = [os.path.dirname(configFile)] # Add entries from include statement recursively for includeFile in getFlatList('global', 'include'): self._fillContentWithIncludes(includeFile, searchPaths + newSearchPaths, configContent) # Process all other entries in current file self._fillContentSingleFile(configFile, configFileLines, searchPaths, configContent) # Override entries in current config file for overrideFile in getFlatList('global', 'include override'): self._fillContentWithIncludes(overrideFile, searchPaths + newSearchPaths, configContent) # Filter special global options if configContent.get('global', []): configContent['global'] = lfilter(lambda opt_v_s: opt_v_s[0] not in ['include', 'include override'], configContent['global']) return searchPaths + newSearchPaths
def queryDBSv3(self, api, **params): if not os.environ.get('X509_USER_PROXY', ''): raise UserError('VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is not set!') proxyPath = utils.resolvePath(os.environ.get('X509_USER_PROXY', ''), mustExist = False) if not os.path.exists(proxyPath): raise UserError('VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is "%s"' % proxyPath) return readJSON(self.url + '/%s' % api, params, cert = proxyPath)
def _parseFile(self, container, configFile, defaults = None, searchPaths = []): try: configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError) log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.')) log.log(logging.INFO1, 'Reading config file %s' % configFile) for line in map(lambda x: x.rstrip() + '=:', open(configFile, 'r').readlines()): if line.startswith('[') or line.lstrip().startswith(';'): continue # skip section and comment lines # Abort if non-indented line with ":" preceeding "=" was found if (line.lstrip() == line) and (line.find(":") < line.find("=")): raise ConfigError('Invalid config line:\n\t%s\nPlease use "key = value" syntax or indent values!' % line) # Parse with python config parser parser = ConfigParser.ConfigParser(defaults) parser.readfp(open(configFile, 'r')) # Parse include files if parser.has_option('global', 'include'): includeFiles = parser.get('global', 'include').split('#')[0].split(';')[0] for includeFile in utils.parseList(includeFiles, None): self._parseFile(container, includeFile, parser.defaults(), searchPaths + [os.path.dirname(configFile)]) # Store config settings for section in parser.sections(): for option in parser.options(section): if (section, option) != ('global', 'include'): value_list = parser.get(section, option).splitlines() # Strip comments value_list = map(lambda l: l.rsplit(';', 1)[0].strip(), value_list) value_list = filter(lambda l: l != '', value_list) container.setEntry(section, option, str.join('\n', value_list), configFile) except: raise RethrowError('Error while reading configuration file "%s"!' % configFile, ConfigError)
def __init__(self): # Collect host / user / installation specific config files def resolve_hostname(): import socket host = socket.gethostname() try: return socket.gethostbyaddr(host)[0] except Exception: return host try: host = hang_protection(resolve_hostname, timeout=5) hostCfg = lmap( lambda c: utils.pathPKG('../config/%s.conf' % host.split( '.', c)[-1]), irange(host.count('.') + 1, -1, -1)) except TimeoutException: sys.stderr.write('System call to resolve hostname is hanging!\n') sys.stderr.flush() hostCfg = [] defaultCfg = [ '/etc/grid-control.conf', '~/.grid-control.conf', utils.pathPKG('../config/default.conf') ] if os.environ.get('GC_CONFIG'): defaultCfg.append('$GC_CONFIG') log = logging.getLogger('config.default') log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', defaultCfg)) fqConfigFiles = lmap(lambda p: utils.resolvePath(p, mustExist=False), hostCfg + defaultCfg) FileConfigFiller.__init__(self, lfilter(os.path.exists, fqConfigFiles), addSearchPath=False)
def __init__(self, filler=None, configFilePath=None): def getName(prefix=''): if configFilePath: return ('%s.%s' % (prefix, utils.getRootName(configFilePath))).strip('.') elif prefix: return prefix return 'unnamed' pathMain = os.getcwd() if configFilePath: pathMain = os.path.dirname( utils.resolvePath(configFilePath, searchPaths=[os.getcwd()], ErrorClass=ConfigError)) # Init config containers self._curContainer = ConfigContainer('current') if filler: # Read in the current configuration ... filler.fill(self._curContainer) logging.getLogger('config.stored').propagate = False oldContainer = ConfigContainer('stored') oldContainer.enabled = False # Create config view and temporary config interface self._view = SimpleConfigView(getName(), oldContainer, self._curContainer) self._view.pathDict['search_paths'] = UniqueList( [os.getcwd(), pathMain]) # Determine work directory using config interface with "global" scope tmpInterface = SimpleConfigInterface( self._view.getView(setSections=['global'])) wdBase = tmpInterface.getPath('workdir base', pathMain, mustExist=False) pathWork = tmpInterface.getPath('workdir', os.path.join(wdBase, getName('work')), mustExist=False) self._view.pathDict[ '<WORKDIR>'] = pathWork # tmpInterface still has undefinied # Set dynamic plugin search path sys.path.extend(tmpInterface.getPaths('plugin paths', [os.getcwd()])) # Determine and load stored config settings self._flatCfgPath = os.path.join(pathWork, 'current.conf') # Minimal config file self._oldCfgPath = os.path.join( pathWork, 'work.conf') # Config file with saved settings if os.path.exists(self._oldCfgPath): GeneralFileConfigFiller([self._oldCfgPath]).fill(oldContainer) CompatConfigFiller(os.path.join(pathWork, 'task.dat')).fill(oldContainer) oldContainer.enabled = True oldContainer.setReadOnly() # Get persistent variables - only possible after oldContainer was enabled self._view.setConfigName( tmpInterface.get('config id', getName(), persistent=True))
def __init__(self): # Collect host / user / installation specific config files host = socket.gethostbyaddr(socket.gethostname())[0] hostCfg = map(lambda c: utils.pathGC('config/%s.conf' % host.split('.', c)[-1]), range(host.count('.') + 1, 0, -1)) defaultCfg = ['/etc/grid-control.conf', '~/.grid-control.conf', utils.pathGC('config/default.conf')] if os.environ.get('GC_CONFIG'): defaultCfg.append('$GC_CONFIG') fqConfigFiles = map(lambda p: utils.resolvePath(p, mustExist = False), hostCfg + defaultCfg) FileConfigFiller.__init__(self, filter(os.path.exists, fqConfigFiles))
def _fillContentFromFile(self, configFile, searchPaths, configContent = {}): log = logging.getLogger(('config.%s' % utils.getRootName(configFile)).rstrip('.')) log.log(logging.INFO1, 'Reading config file %s' % configFile) configFile = utils.resolvePath(configFile, searchPaths, ErrorClass = ConfigError) configFileData = open(configFile, 'r').readlines() # Single pass, non-recursive list retrieval tmpConfigContent = {} self._fillContentFromSingleFile(configFile, configFileData, searchPaths, tmpConfigContent) def getFlatList(section, option): for (opt, value, s) in filter(lambda (opt, v, s): opt == option, tmpConfigContent.get(section, [])): for entry in utils.parseList(value, None): yield entry
def __init__(self, filler = None, configFilePath = None): def getName(prefix = ''): if configFilePath: return ('%s.%s' % (prefix, utils.getRootName(configFilePath))).strip('.') elif prefix: return prefix return 'unnamed' try: pathMain = os.getcwd() except Exception: raise ConfigError('The current directory does not exist!') if configFilePath: pathMain = os.path.dirname(utils.resolvePath(configFilePath, searchPaths = [os.getcwd()], ErrorClass = ConfigError)) # Init config containers self._curContainer = ConfigContainer('current') if filler: # Read in the current configuration ... filler.fill(self._curContainer) self._curContainer.resolve() # resolve interpolations logging.getLogger('config.stored').propagate = False oldContainer = ConfigContainer('stored') oldContainer.enabled = False # Create config view and temporary config interface self._view = SimpleConfigView(getName(), oldContainer, self._curContainer) self._view.pathDict['search_paths'] = UniqueList([os.getcwd(), pathMain]) # Determine work directory using config interface with "global" scope tmp_config = SimpleConfigInterface(self._view.getView(setSections = ['global'])) wdBase = tmp_config.getPath('workdir base', pathMain, mustExist = False) pathWork = tmp_config.getPath('workdir', os.path.join(wdBase, getName('work')), mustExist = False) self._view.pathDict['<WORKDIR>'] = pathWork # tmp_config still has undefinied # Set dynamic plugin search path sys.path.extend(tmp_config.getPaths('plugin paths', [os.getcwd()])) # Determine and load stored config settings self._flatCfgPath = os.path.join(pathWork, 'current.conf') # Minimal config file self._oldCfgPath = os.path.join(pathWork, 'work.conf') # Config file with saved settings if os.path.exists(self._oldCfgPath): GeneralFileConfigFiller([self._oldCfgPath]).fill(oldContainer) CompatConfigFiller(os.path.join(pathWork, 'task.dat')).fill(oldContainer) oldContainer.enabled = True oldContainer.setReadOnly() # Get persistent variables - only possible after oldContainer was enabled self._view.setConfigName(tmp_config.get('config id', getName(), persistent = True))
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set( ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg=pkg), os.listdir(utils.pathPKG())) depFiles = lmap( lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts( imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({ 'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self.wmsName }) taskConfig = sorted( utils.DictFormat(escapeString=True).format( taskEnv, format='export %s%s%s\n')) varMappingDict = dict( izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted( utils.DictFormat(delimeter=' ').format(varMappingDict, format='%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([ monitor.getFiles(), depFiles, getTaskFiles(), [ VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping) ] ])
def __init__(self): # Collect host / user / installation specific config files def resolve_hostname(): import socket host = socket.gethostname() try: return socket.gethostbyaddr(host)[0] except Exception: return host try: host = hang_protection(resolve_hostname, timeout = 5) hostCfg = lmap(lambda c: utils.pathPKG('../config/%s.conf' % host.split('.', c)[-1]), irange(host.count('.') + 1, -1, -1)) except TimeoutException: sys.stderr.write('System call to resolve hostname is hanging!\n') sys.stderr.flush() hostCfg = [] defaultCfg = ['/etc/grid-control.conf', '~/.grid-control.conf', utils.pathPKG('../config/default.conf')] if os.environ.get('GC_CONFIG'): defaultCfg.append('$GC_CONFIG') log = logging.getLogger('config.default') log.log(logging.DEBUG1, 'Possible default config files: %s', str.join(', ', defaultCfg)) fqConfigFiles = lmap(lambda p: utils.resolvePath(p, mustExist = False), hostCfg + defaultCfg) FileConfigFiller.__init__(self, lfilter(os.path.exists, fqConfigFiles), addSearchPath = False)
def _getSandboxFiles(self, task, monitor, smList): # Prepare all input files depList = set(ichain(imap(lambda x: x.getDependencies(), [task] + smList))) depPaths = lmap(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathPKG())) depFiles = lmap(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = utils.mergeDicts(imap(lambda x: x.getTaskConfig(), [monitor, task] + smList)) taskEnv.update({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self._token.getUsername(), 'GC_WMS_NAME': self._name}) taskConfig = sorted(utils.DictFormat(escapeString = True).format(taskEnv, format = 'export %s%s%s\n')) varMappingDict = dict(izip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(task.getVarMapping()) varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n')) # Resolve wildcards in task input files def getTaskFiles(): for f in task.getSBInFiles(): matched = glob.glob(f.pathAbs) if matched != []: for match in matched: yield match else: yield f.pathAbs return lchain([monitor.getFiles(), depFiles, getTaskFiles(), [VirtualFile('_config.sh', taskConfig), VirtualFile('_varmap.dat', varMapping)]])
def _getSandboxFiles(self, module, monitor, smList): # Prepare all input files depList = set(itertools.chain(*map(lambda x: x.getDependencies(), [module] + smList))) depPaths = map(lambda pkg: utils.pathShare('', pkg = pkg), os.listdir(utils.pathGC('packages'))) depFiles = map(lambda dep: utils.resolvePath('env.%s.sh' % dep, depPaths), depList) taskEnv = list(itertools.chain(map(lambda x: x.getTaskConfig(), [monitor, module] + smList))) taskEnv.append({'GC_DEPFILES': str.join(' ', depList), 'GC_USERNAME': self.proxy.getUsername(), 'GC_WMS_NAME': self.wmsName}) taskConfig = sorted(utils.DictFormat(escapeString = True).format(utils.mergeDicts(taskEnv), format = 'export %s%s%s\n')) varMappingDict = dict(zip(monitor.getTaskConfig().keys(), monitor.getTaskConfig().keys())) varMappingDict.update(module.getVarMapping()) varMapping = sorted(utils.DictFormat(delimeter = ' ').format(varMappingDict, format = '%s%s%s\n')) # Resolve wildcards in module input files def getModuleFiles(): for f in module.getSBInFiles(): matched = glob.glob(f) if matched != []: for match in matched: yield match else: yield f return list(itertools.chain(monitor.getFiles(), depFiles, getModuleFiles(), [utils.VirtualFile('_config.sh', taskConfig), utils.VirtualFile('_varmap.dat', varMapping)]))
def resolvePath(self, value, mustExist, errorMsg): try: return utils.resolvePath(value, self._configView.pathDict.get('search_paths', []), mustExist, ConfigError) except Exception: raise ConfigError(errorMsg)
def resolvePath(self, value, mustExist, errorMsg): try: return utils.resolvePath(value, [self._pathBase], mustExist, ConfigError) except: raise RethrowError(errorMsg, ConfigError)